diff --git a/MODULE.bazel b/MODULE.bazel index e9c4c9d3..80256f63 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -19,6 +19,8 @@ bazel_dep(name = "aspect_rules_py", version = "1.8.4") bazel_dep(name = "bazel_skylib", version = "1.9.0") ## Needed by iree +bazel_dep(name = "googletest", version = "1.17.0.bcr.2", dev_dependency = True, repo_name = "com_google_googletest") + bazel_dep(name = "libbacktrace", version = "1.0.0-20250926-7939218") bazel_dep(name = "patchelf", version = "0.18.0.bcr.1") bazel_dep(name = "pcre2", version = "10.45") diff --git a/pjrt/BUILD.bazel b/pjrt/BUILD.bazel index 5370afdc..1747e0c9 100644 --- a/pjrt/BUILD.bazel +++ b/pjrt/BUILD.bazel @@ -1,4 +1,4 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") zig_library( name = "pjrt", @@ -22,3 +22,8 @@ zig_library( "@xla//xla/pjrt/c:pjrt_c_api_stream_extension_hdrs", ], ) + +zig_test( + name = "profiler_collect_data_test", + deps = [":pjrt"], +) diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 3715361f..04c870e1 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -24,6 +24,7 @@ pub const meta = struct { pub fn structSize(comptime T: type) usize { // unsafe on purpose, we want this to fail if that ever changes const typedef_name = comptime blk: { + @setEvalBranchQuota(10_000); const needle = ".struct_"; const idx = std.mem.indexOf(u8, @typeName(T), needle).?; break :blk @typeName(T)[idx + needle.len ..]; @@ -943,6 +944,8 @@ pub const Executable = opaque { /// much: How argument is reused for output. alias_size_in_bytes: u64, temp_size_in_bytes: u64, + total_allocation_bytes: u64, + indefinite_allocations: u64, /// memory: Host usage stats. host_generated_code_size_in_bytes: u64, @@ -962,6 +965,8 @@ pub const Executable = opaque { .output_size_in_bytes = @intCast(ret.output_size_in_bytes), .alias_size_in_bytes = @intCast(ret.alias_size_in_bytes), .temp_size_in_bytes = @intCast(ret.temp_size_in_bytes), + .total_allocation_bytes = @intCast(ret.total_allocation_bytes), + .indefinite_allocations = @intCast(ret.indefinite_allocations), .host_generated_code_size_in_bytes = @intCast(ret.host_generated_code_size_in_bytes), .host_argument_size_in_bytes = @intCast(ret.host_argument_size_in_bytes), .host_output_size_in_bytes = @intCast(ret.host_output_size_in_bytes), @@ -1745,7 +1750,7 @@ pub const Ffi = extern struct { .type_id = 0, // let the plugin assign a unique type ID .type_info = @ptrCast(@constCast(type_info)), }; - const result = self.inner.type_register.?(&ret); + const result = self.inner.type_register.?(@ptrCast(&ret)); if (result) |pjrt_c_error| { const pjrt_error: *Error = @ptrCast(pjrt_c_error); return pjrt_error.getCode(api).toApiError(); @@ -1759,7 +1764,7 @@ pub const Ffi = extern struct { .context = @ptrCast(context), .user_data = user_data.toCStruct(), }; - const result = self.inner.user_data_add.?(&ret); + const result = self.inner.user_data_add.?(@ptrCast(&ret)); if (result) |pjrt_c_error| { const pjrt_error: *Error = @ptrCast(pjrt_c_error); log.err("addUserData error: {s}", .{pjrt_error.getMessage(api)}); @@ -1832,11 +1837,8 @@ pub const Profiler = struct { } if (args.buffer != null) { - var data = args.buffer[0..args.buffer_size_in_bytes]; - if (data.len > 0 and data[data.len - 1] == 0) { - data = data[0 .. data.len - 1]; - } - return allocator.dupe(u8, data); + // CollectData returns binary serialized bytes, not a C string. + return allocator.dupe(u8, args.buffer[0..args.buffer_size_in_bytes]); } if (args.buffer_size_in_bytes == 0) return &[_]u8{}; @@ -1850,15 +1852,52 @@ pub const Profiler = struct { return pjrt_err.getCode(self.pjrt_api).toApiError(); } - // Trim trailing null byte logic - var data = buffer[0..args.buffer_size_in_bytes]; - if (data.len > 0 and data[data.len - 1] == 0) { - data = data[0 .. data.len - 1]; - } - - const pb = try allocator.dupe(u8, data); + const pb = try allocator.dupe(u8, buffer[0..args.buffer_size_in_bytes]); allocator.free(buffer); return pb; } }; + +test "Profiler collectData preserves binary protobuf terminal zero in both API paths" { + const Fixture = struct { + // A valid XSpace wire message: planes[0] has an explicit int64 id = 0. + const protobuf = [_]u8{ 0x0a, 0x02, 0x08, 0x00 }; + + fn direct(args: [*c]c.PLUGIN_Profiler_CollectData_Args) callconv(.c) ?*c.PLUGIN_Profiler_Error { + args.*.buffer = @constCast(&protobuf); + args.*.buffer_size_in_bytes = protobuf.len; + return null; + } + + fn twoPass(args: [*c]c.PLUGIN_Profiler_CollectData_Args) callconv(.c) ?*c.PLUGIN_Profiler_Error { + if (args.*.buffer) |buffer| @memcpy(buffer[0..protobuf.len], &protobuf); + args.*.buffer_size_in_bytes = protobuf.len; + return null; + } + }; + inline for (.{ Fixture.direct, Fixture.twoPass }) |collect| { + var api = std.mem.zeroes(c.PLUGIN_Profiler_Api); + api.collect_data = &collect; + var instance: Profiler = .{ .pjrt_api = undefined, .api = &api, .inner = @ptrFromInt(1) }; + const data = try instance.collectData(std.testing.allocator); + defer std.testing.allocator.free(data); + try std.testing.expectEqualSlices(u8, &Fixture.protobuf, data); + } +} + +test "Profiler collectData accepts empty serialized output" { + const Fixture = struct { + fn collect(args: [*c]c.PLUGIN_Profiler_CollectData_Args) callconv(.c) ?*c.PLUGIN_Profiler_Error { + args.*.buffer = null; + args.*.buffer_size_in_bytes = 0; + return null; + } + }; + var api = std.mem.zeroes(c.PLUGIN_Profiler_Api); + api.collect_data = &Fixture.collect; + var instance: Profiler = .{ .pjrt_api = undefined, .api = &api, .inner = @ptrFromInt(1) }; + const data = try instance.collectData(std.testing.allocator); + defer std.testing.allocator.free(data); + try std.testing.expectEqual(@as(usize, 0), data.len); +} diff --git a/platforms/cuda/BUILD.bazel b/platforms/cuda/BUILD.bazel index 7f7aed55..b59640fa 100644 --- a/platforms/cuda/BUILD.bazel +++ b/platforms/cuda/BUILD.bazel @@ -1,6 +1,16 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library", "zig_shared_library") +filegroup( + name = "rdma_license_texts", + srcs = [ + "rdma/GPL-2", + "rdma/LGPL-2.1", + "rdma/README.md", + ], + visibility = ["//visibility:public"], +) + zig_binary( name = "compat_probe", main = "compat_probe.zig", diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index 8482e52a..d5b4a374 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -202,6 +202,42 @@ NVSHMEM_PACKAGES = { } _UBUNTU_PACKAGES = { + "ibverbs-providers": [ + packages.filegroup( + name = "rdma", + srcs = [ + "usr/lib/aarch64-linux-gnu/libmlx5.so.1", + "usr/share/doc/ibverbs-providers/copyright", + ], + ), + ], + "libibverbs1": [ + packages.filegroup( + name = "rdma", + srcs = [ + "usr/lib/aarch64-linux-gnu/libibverbs.so.1", + "usr/share/doc/libibverbs1/copyright", + ], + ), + ], + "libnl-3-200": [ + packages.filegroup( + name = "rdma", + srcs = [ + "usr/lib/aarch64-linux-gnu/libnl-3.so.200", + "usr/share/doc/libnl-3-200/copyright", + ], + ), + ], + "libnl-route-3-200": [ + packages.filegroup( + name = "rdma", + srcs = [ + "usr/lib/aarch64-linux-gnu/libnl-route-3.so.200", + "usr/share/doc/libnl-route-3-200/copyright", + ], + ), + ], "llvm-libunwind1": [ """filegroup( name = "libunwind", @@ -228,8 +264,8 @@ _PLUGINS = { "url": "https://mirror.zml.ai/plugins/202611101651.363.1.d46c7471c419/zml-cuda-linux-amd64.tar.zst", }, "arm64": { - "sha256": "caeefd717960fa2acec6304cd8296299e262abdba669ecf4d487e3af76d73a7b", - "url": "https://mirror.zml.ai/plugins/202611101651.363.1.d46c7471c419/zml-cuda-linux-arm64.tar.zst", + "sha256": "842746eaec0fd2403ce2b5567d51dbba400fc43b46f37175c8d6d25a7c65b814", + "url": "file:///home/benjamin/deepseek41-cp-master-revalidate-20261111/xla/cuda-d79645f2-nccl23004-linux-arm64.tar.zst", }, } @@ -239,8 +275,8 @@ _NCCL_ASSETS = { "sha256": "534dbf3058cadb625f08ab0d17f1dffad3b961a2bfa360d66633fcf21be53f57", }, "arm64": { - "url": "https://pypi.nvidia.com/nvidia-nccl-cu13/nvidia_nccl_cu13-2.30.4-py3-none-manylinux_2_18_aarch64.whl", - "sha256": "e99308a3a89fba78918d50886e81072a6c8b0b4199feb02c3903e63713a6525a", + "url": "file:///home/benjamin/deepseek41-cp-evidence-20261107/workspaces/tmp-benjamin/llmd-cp-kernel-tuning-20261027/nccl-single-host-backport-review/v1/gnu9-build/frozen/nccl-2.30.4-host-cache-arm64.zip", + "sha256": "38cc66b24f83adfe9e2ae8d96642ef3f0b8674bf6fae048e0fbda4e86fe02c41", }, } @@ -262,6 +298,7 @@ def _read_redist_json(mctx, url, sha256): def _cuda_impl(mctx): loaded_packages = packages.read(mctx, [ "@zml//platforms/cuda:packages.lock.json", + "@zml//platforms/cuda:rdma.packages.lock.json", ]) CUDA_REDIST = _read_redist_json( diff --git a/platforms/cuda/libzml_cuda.BUILD.bazel b/platforms/cuda/libzml_cuda.BUILD.bazel index e79ab900..1d4d9664 100644 --- a/platforms/cuda/libzml_cuda.BUILD.bazel +++ b/platforms/cuda/libzml_cuda.BUILD.bazel @@ -15,11 +15,31 @@ patchelf( set_rpath = "$ORIGIN", ) +# GIN's optional ARM64 host transport. These are unmodified pinned DSOs; +# LLMD loads them by absolute SONAME path only when selecting that transport. +filegroup( + name = "rdma_runtime", + srcs = select({ + "@llvm//platforms/config:linux_aarch64": [ + "@ibverbs-providers_linux_arm64//:rdma", + "@libibverbs1_linux_arm64//:rdma", + "@libnl-3-200_linux_arm64//:rdma", + "@libnl-route-3-200_linux_arm64//:rdma", + "@zml//platforms/cuda:rdma_license_texts", + ], + "//conditions:default": [], + }), + visibility = ["//visibility:public"], +) + copy_to_directory( name = "sandbox", - srcs = [ + # Some plugin archives include the separately loaded Engram device module. + # Keep those files beside the plugin; older archives need no such assets. + srcs = glob(["lib/engram_gin.*"]) + [ "lib/cutlass_ir.so", ":libzml_cuda_so", + ":rdma_runtime", "@zml//platforms/cuda:compat_probe", "@zml//platforms/cuda:zmlxcuda", ] + select({ @@ -75,6 +95,8 @@ copy_to_directory( "lib/aarch64-linux-gnu": "lib", "usr/lib/x86_64-linux-gnu": "lib", "usr/lib/aarch64-linux-gnu": "lib", + "usr/share/doc": "share/licenses/rdma", + "platforms/cuda/rdma": "share/licenses/rdma", "platforms/cuda/compat_probe": "bin/compat_probe", "platforms/cuda/libzmlxcuda": "lib/libzmlxcuda", } | { diff --git a/platforms/cuda/rdma.packages.lock.json b/platforms/cuda/rdma.packages.lock.json new file mode 100644 index 00000000..3894ac2c --- /dev/null +++ b/platforms/cuda/rdma.packages.lock.json @@ -0,0 +1,41 @@ +{ + "scope": "Only the four unmodified ARM64 GIN transport payloads and their package copyright files. System glibc is not bundled; the pinned Ubuntu libraries require glibc 2.38 or later.", + "packages": [ + { + "name": "libnl-3-200", + "version": "3.7.0-0.3build1.1", + "arch": "arm64", + "urls": [ + "https://ports.ubuntu.com/ubuntu-ports/pool/main/libn/libnl3/libnl-3-200_3.7.0-0.3build1.1_arm64.deb" + ], + "sha256": "be2e96607e4c6e745b9ef1ec11d8192f5603721a7a5a9989524ea3d7a7f6db16" + }, + { + "name": "libnl-route-3-200", + "version": "3.7.0-0.3build1.1", + "arch": "arm64", + "urls": [ + "https://ports.ubuntu.com/ubuntu-ports/pool/main/libn/libnl3/libnl-route-3-200_3.7.0-0.3build1.1_arm64.deb" + ], + "sha256": "cce89a1737de041f03f47f71a7b177c3c22b4bc8085a39d84158ce56a8d07032" + }, + { + "name": "libibverbs1", + "version": "50.0-2ubuntu0.2", + "arch": "arm64", + "urls": [ + "https://ports.ubuntu.com/ubuntu-ports/pool/main/r/rdma-core/libibverbs1_50.0-2ubuntu0.2_arm64.deb" + ], + "sha256": "20b72b2c6b882ee4c155658be8a0a01d6edc2ee580eaeb330d9bb3ad643f386d" + }, + { + "name": "ibverbs-providers", + "version": "50.0-2ubuntu0.2", + "arch": "arm64", + "urls": [ + "https://ports.ubuntu.com/ubuntu-ports/pool/main/r/rdma-core/ibverbs-providers_50.0-2ubuntu0.2_arm64.deb" + ], + "sha256": "39e91a145faaa332aad6837783d6b600b7e1b136fa805fb7fc824a87d21897ab" + } + ] +} diff --git a/platforms/cuda/rdma/GPL-2 b/platforms/cuda/rdma/GPL-2 new file mode 100644 index 00000000..d159169d --- /dev/null +++ b/platforms/cuda/rdma/GPL-2 @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/platforms/cuda/rdma/LGPL-2.1 b/platforms/cuda/rdma/LGPL-2.1 new file mode 100644 index 00000000..4362b491 --- /dev/null +++ b/platforms/cuda/rdma/LGPL-2.1 @@ -0,0 +1,502 @@ + GNU LESSER GENERAL PUBLIC LICENSE + Version 2.1, February 1999 + + Copyright (C) 1991, 1999 Free Software Foundation, Inc. + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + +[This is the first released version of the Lesser GPL. It also counts + as the successor of the GNU Library Public License, version 2, hence + the version number 2.1.] + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +Licenses are intended to guarantee your freedom to share and change +free software--to make sure the software is free for all its users. + + This license, the Lesser General Public License, applies to some +specially designated software packages--typically libraries--of the +Free Software Foundation and other authors who decide to use it. You +can use it too, but we suggest you first think carefully about whether +this license or the ordinary General Public License is the better +strategy to use in any particular case, based on the explanations below. + + When we speak of free software, we are referring to freedom of use, +not price. Our General Public Licenses are designed to make sure that +you have the freedom to distribute copies of free software (and charge +for this service if you wish); that you receive source code or can get +it if you want it; that you can change the software and use pieces of +it in new free programs; and that you are informed that you can do +these things. + + To protect your rights, we need to make restrictions that forbid +distributors to deny you these rights or to ask you to surrender these +rights. These restrictions translate to certain responsibilities for +you if you distribute copies of the library or if you modify it. + + For example, if you distribute copies of the library, whether gratis +or for a fee, you must give the recipients all the rights that we gave +you. You must make sure that they, too, receive or can get the source +code. If you link other code with the library, you must provide +complete object files to the recipients, so that they can relink them +with the library after making changes to the library and recompiling +it. And you must show them these terms so they know their rights. + + We protect your rights with a two-step method: (1) we copyright the +library, and (2) we offer you this license, which gives you legal +permission to copy, distribute and/or modify the library. + + To protect each distributor, we want to make it very clear that +there is no warranty for the free library. Also, if the library is +modified by someone else and passed on, the recipients should know +that what they have is not the original version, so that the original +author's reputation will not be affected by problems that might be +introduced by others. + + Finally, software patents pose a constant threat to the existence of +any free program. We wish to make sure that a company cannot +effectively restrict the users of a free program by obtaining a +restrictive license from a patent holder. Therefore, we insist that +any patent license obtained for a version of the library must be +consistent with the full freedom of use specified in this license. + + Most GNU software, including some libraries, is covered by the +ordinary GNU General Public License. This license, the GNU Lesser +General Public License, applies to certain designated libraries, and +is quite different from the ordinary General Public License. We use +this license for certain libraries in order to permit linking those +libraries into non-free programs. + + When a program is linked with a library, whether statically or using +a shared library, the combination of the two is legally speaking a +combined work, a derivative of the original library. The ordinary +General Public License therefore permits such linking only if the +entire combination fits its criteria of freedom. The Lesser General +Public License permits more lax criteria for linking other code with +the library. + + We call this license the "Lesser" General Public License because it +does Less to protect the user's freedom than the ordinary General +Public License. It also provides other free software developers Less +of an advantage over competing non-free programs. These disadvantages +are the reason we use the ordinary General Public License for many +libraries. However, the Lesser license provides advantages in certain +special circumstances. + + For example, on rare occasions, there may be a special need to +encourage the widest possible use of a certain library, so that it becomes +a de-facto standard. To achieve this, non-free programs must be +allowed to use the library. A more frequent case is that a free +library does the same job as widely used non-free libraries. In this +case, there is little to gain by limiting the free library to free +software only, so we use the Lesser General Public License. + + In other cases, permission to use a particular library in non-free +programs enables a greater number of people to use a large body of +free software. For example, permission to use the GNU C Library in +non-free programs enables many more people to use the whole GNU +operating system, as well as its variant, the GNU/Linux operating +system. + + Although the Lesser General Public License is Less protective of the +users' freedom, it does ensure that the user of a program that is +linked with the Library has the freedom and the wherewithal to run +that program using a modified version of the Library. + + The precise terms and conditions for copying, distribution and +modification follow. Pay close attention to the difference between a +"work based on the library" and a "work that uses the library". The +former contains code derived from the library, whereas the latter must +be combined with the library in order to run. + + GNU LESSER GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License Agreement applies to any software library or other +program which contains a notice placed by the copyright holder or +other authorized party saying it may be distributed under the terms of +this Lesser General Public License (also called "this License"). +Each licensee is addressed as "you". + + A "library" means a collection of software functions and/or data +prepared so as to be conveniently linked with application programs +(which use some of those functions and data) to form executables. + + The "Library", below, refers to any such software library or work +which has been distributed under these terms. A "work based on the +Library" means either the Library or any derivative work under +copyright law: that is to say, a work containing the Library or a +portion of it, either verbatim or with modifications and/or translated +straightforwardly into another language. (Hereinafter, translation is +included without limitation in the term "modification".) + + "Source code" for a work means the preferred form of the work for +making modifications to it. For a library, complete source code means +all the source code for all modules it contains, plus any associated +interface definition files, plus the scripts used to control compilation +and installation of the library. + + Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running a program using the Library is not restricted, and output from +such a program is covered only if its contents constitute a work based +on the Library (independent of the use of the Library in a tool for +writing it). Whether that is true depends on what the Library does +and what the program that uses the Library does. + + 1. You may copy and distribute verbatim copies of the Library's +complete source code as you receive it, in any medium, provided that +you conspicuously and appropriately publish on each copy an +appropriate copyright notice and disclaimer of warranty; keep intact +all the notices that refer to this License and to the absence of any +warranty; and distribute a copy of this License along with the +Library. + + You may charge a fee for the physical act of transferring a copy, +and you may at your option offer warranty protection in exchange for a +fee. + + 2. You may modify your copy or copies of the Library or any portion +of it, thus forming a work based on the Library, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) The modified work must itself be a software library. + + b) You must cause the files modified to carry prominent notices + stating that you changed the files and the date of any change. + + c) You must cause the whole of the work to be licensed at no + charge to all third parties under the terms of this License. + + d) If a facility in the modified Library refers to a function or a + table of data to be supplied by an application program that uses + the facility, other than as an argument passed when the facility + is invoked, then you must make a good faith effort to ensure that, + in the event an application does not supply such function or + table, the facility still operates, and performs whatever part of + its purpose remains meaningful. + + (For example, a function in a library to compute square roots has + a purpose that is entirely well-defined independent of the + application. Therefore, Subsection 2d requires that any + application-supplied function or table used by this function must + be optional: if the application does not supply it, the square + root function must still compute square roots.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Library, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Library, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote +it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Library. + +In addition, mere aggregation of another work not based on the Library +with the Library (or with a work based on the Library) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may opt to apply the terms of the ordinary GNU General Public +License instead of this License to a given copy of the Library. To do +this, you must alter all the notices that refer to this License, so +that they refer to the ordinary GNU General Public License, version 2, +instead of to this License. (If a newer version than version 2 of the +ordinary GNU General Public License has appeared, then you can specify +that version instead if you wish.) Do not make any other change in +these notices. + + Once this change is made in a given copy, it is irreversible for +that copy, so the ordinary GNU General Public License applies to all +subsequent copies and derivative works made from that copy. + + This option is useful when you wish to copy part of the code of +the Library into a program that is not a library. + + 4. You may copy and distribute the Library (or a portion or +derivative of it, under Section 2) in object code or executable form +under the terms of Sections 1 and 2 above provided that you accompany +it with the complete corresponding machine-readable source code, which +must be distributed under the terms of Sections 1 and 2 above on a +medium customarily used for software interchange. + + If distribution of object code is made by offering access to copy +from a designated place, then offering equivalent access to copy the +source code from the same place satisfies the requirement to +distribute the source code, even though third parties are not +compelled to copy the source along with the object code. + + 5. A program that contains no derivative of any portion of the +Library, but is designed to work with the Library by being compiled or +linked with it, is called a "work that uses the Library". Such a +work, in isolation, is not a derivative work of the Library, and +therefore falls outside the scope of this License. + + However, linking a "work that uses the Library" with the Library +creates an executable that is a derivative of the Library (because it +contains portions of the Library), rather than a "work that uses the +library". The executable is therefore covered by this License. +Section 6 states terms for distribution of such executables. + + When a "work that uses the Library" uses material from a header file +that is part of the Library, the object code for the work may be a +derivative work of the Library even though the source code is not. +Whether this is true is especially significant if the work can be +linked without the Library, or if the work is itself a library. The +threshold for this to be true is not precisely defined by law. + + If such an object file uses only numerical parameters, data +structure layouts and accessors, and small macros and small inline +functions (ten lines or less in length), then the use of the object +file is unrestricted, regardless of whether it is legally a derivative +work. (Executables containing this object code plus portions of the +Library will still fall under Section 6.) + + Otherwise, if the work is a derivative of the Library, you may +distribute the object code for the work under the terms of Section 6. +Any executables containing that work also fall under Section 6, +whether or not they are linked directly with the Library itself. + + 6. As an exception to the Sections above, you may also combine or +link a "work that uses the Library" with the Library to produce a +work containing portions of the Library, and distribute that work +under terms of your choice, provided that the terms permit +modification of the work for the customer's own use and reverse +engineering for debugging such modifications. + + You must give prominent notice with each copy of the work that the +Library is used in it and that the Library and its use are covered by +this License. You must supply a copy of this License. If the work +during execution displays copyright notices, you must include the +copyright notice for the Library among them, as well as a reference +directing the user to the copy of this License. Also, you must do one +of these things: + + a) Accompany the work with the complete corresponding + machine-readable source code for the Library including whatever + changes were used in the work (which must be distributed under + Sections 1 and 2 above); and, if the work is an executable linked + with the Library, with the complete machine-readable "work that + uses the Library", as object code and/or source code, so that the + user can modify the Library and then relink to produce a modified + executable containing the modified Library. (It is understood + that the user who changes the contents of definitions files in the + Library will not necessarily be able to recompile the application + to use the modified definitions.) + + b) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (1) uses at run time a + copy of the library already present on the user's computer system, + rather than copying library functions into the executable, and (2) + will operate properly with a modified version of the library, if + the user installs one, as long as the modified version is + interface-compatible with the version that the work was made with. + + c) Accompany the work with a written offer, valid for at + least three years, to give the same user the materials + specified in Subsection 6a, above, for a charge no more + than the cost of performing this distribution. + + d) If distribution of the work is made by offering access to copy + from a designated place, offer equivalent access to copy the above + specified materials from the same place. + + e) Verify that the user has already received a copy of these + materials or that you have already sent this user a copy. + + For an executable, the required form of the "work that uses the +Library" must include any data and utility programs needed for +reproducing the executable from it. However, as a special exception, +the materials to be distributed need not include anything that is +normally distributed (in either source or binary form) with the major +components (compiler, kernel, and so on) of the operating system on +which the executable runs, unless that component itself accompanies +the executable. + + It may happen that this requirement contradicts the license +restrictions of other proprietary libraries that do not normally +accompany the operating system. Such a contradiction means you cannot +use both them and the Library together in an executable that you +distribute. + + 7. You may place library facilities that are a work based on the +Library side-by-side in a single library together with other library +facilities not covered by this License, and distribute such a combined +library, provided that the separate distribution of the work based on +the Library and of the other library facilities is otherwise +permitted, and provided that you do these two things: + + a) Accompany the combined library with a copy of the same work + based on the Library, uncombined with any other library + facilities. This must be distributed under the terms of the + Sections above. + + b) Give prominent notice with the combined library of the fact + that part of it is a work based on the Library, and explaining + where to find the accompanying uncombined form of the same work. + + 8. You may not copy, modify, sublicense, link with, or distribute +the Library except as expressly provided under this License. Any +attempt otherwise to copy, modify, sublicense, link with, or +distribute the Library is void, and will automatically terminate your +rights under this License. However, parties who have received copies, +or rights, from you under this License will not have their licenses +terminated so long as such parties remain in full compliance. + + 9. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Library or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Library (or any work based on the +Library), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Library or works based on it. + + 10. Each time you redistribute the Library (or any work based on the +Library), the recipient automatically receives a license from the +original licensor to copy, distribute, link with or modify the Library +subject to these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties with +this License. + + 11. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Library at all. For example, if a patent +license would not permit royalty-free redistribution of the Library by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Library. + +If any portion of this section is held invalid or unenforceable under any +particular circumstance, the balance of the section is intended to apply, +and the section as a whole is intended to apply in other circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 12. If the distribution and/or use of the Library is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Library under this License may add +an explicit geographical distribution limitation excluding those countries, +so that distribution is permitted only in or among countries not thus +excluded. In such case, this License incorporates the limitation as if +written in the body of this License. + + 13. The Free Software Foundation may publish revised and/or new +versions of the Lesser General Public License from time to time. +Such new versions will be similar in spirit to the present version, +but may differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Library +specifies a version number of this License which applies to it and +"any later version", you have the option of following the terms and +conditions either of that version or of any later version published by +the Free Software Foundation. If the Library does not specify a +license version number, you may choose any version ever published by +the Free Software Foundation. + + 14. If you wish to incorporate parts of the Library into other free +programs whose distribution conditions are incompatible with these, +write to the author to ask for permission. For software which is +copyrighted by the Free Software Foundation, write to the Free +Software Foundation; we sometimes make exceptions for this. Our +decision will be guided by the two goals of preserving the free status +of all derivatives of our free software and of promoting the sharing +and reuse of software generally. + + NO WARRANTY + + 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO +WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. +EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR +OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY +KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE +LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME +THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN +WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY +AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU +FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR +CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE +LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING +RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A +FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF +SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH +DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Libraries + + If you develop a new library, and you want it to be of the greatest +possible use to the public, we recommend making it free software that +everyone can redistribute and change. You can do so by permitting +redistribution under these terms (or, alternatively, under the terms of the +ordinary General Public License). + + To apply these terms, attach the following notices to the library. It is +safest to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least the +"copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +Also add information on how to contact you by electronic and paper mail. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the library, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the + library `Frob' (a library for tweaking knobs) written by James Random Hacker. + + , 1 April 1990 + Ty Coon, President of Vice + +That's all there is to it! diff --git a/platforms/cuda/rdma/README.md b/platforms/cuda/rdma/README.md new file mode 100644 index 00000000..ab1b01a2 --- /dev/null +++ b/platforms/cuda/rdma/README.md @@ -0,0 +1,22 @@ +# Optional ARM64 GIN host transport + +The CUDA runtime includes four unmodified Ubuntu ARM64 libraries for the +optional GIN host-memory transport. The archives and SHA-256 hashes are pinned +in `../rdma.packages.lock.json`: libnl3 version 3.7.0-0.3build1.1 and rdma-core +version 50.0-2ubuntu0.2. No host package installation is required. + +Only libnl-3.so.200, libnl-route-3.so.200, libibverbs.so.1 and libmlx5.so.1 +are included. Their SONAMEs and bytes match the individually validated +transport closure. Load them in that order using absolute paths with global +symbol visibility. Loading libmlx5 registers its provider directly; this path +does not need a host provider configuration or IBV_DRIVERS. It requires a +compatible system glibc (the pinned packages require glibc 2.38 or later). + +Each package's unmodified copyright file is under its own directory in +`share/licenses/rdma`. GPL-2 and LGPL-2.1 contain the complete standard license +texts referenced by those copyright files; the latter governs libnl3 library +code. The libraries' source locations are recorded by their package notices: +https://github.com/thom311/libnl and https://github.com/linux-rdma/rdma-core. + +This packaging does not preload the transport, initialize CUDA/NCCL, change +provider environment variables, or alter the CUDA plugin/device-module bytes. diff --git a/third_party/iree/regex-skip-failed-start.patch b/third_party/iree/regex-skip-failed-start.patch new file mode 100644 index 00000000..6f4bb62f --- /dev/null +++ b/third_party/iree/regex-skip-failed-start.patch @@ -0,0 +1,260 @@ +diff --git a/runtime/src/iree/tokenizer/regex/exec.c b/runtime/src/iree/tokenizer/regex/exec.c +--- a/runtime/src/iree/tokenizer/regex/exec.c ++++ b/runtime/src/iree/tokenizer/regex/exec.c +@@ -1330,6 +1330,20 @@ + return iree_ok_status(); + } + ++ // Exact ranges take precedence over the byte table and may include ASCII. ++ // Only use the initial state's byte table directly when no such range can ++ // override it. Start-anchored patterns retain the ordinary per-byte exit. ++ bool scan_failed_start_ascii = ++ !requires_start && ++ dfa->header->start_state == dfa->header->unanchored_start_state; ++ for (uint16_t i = 0; i < dfa->num_ranges && scan_failed_start_ascii; ++i) { ++ const iree_tokenizer_regex_dfa_range_transition_t* range = &dfa->ranges[i]; ++ if (range->from_state == dfa->header->start_state && range->start < 0x80) { ++ scan_failed_start_ascii = false; ++ } ++ } ++ const uint16_t* start_transitions = ++ dfa->transitions + (iree_host_size_t)dfa->header->start_state * 256; + iree_host_size_t position = 0; + + // Resolve pending lookahead from previous chunk if we have new data. +@@ -1511,6 +1525,29 @@ + // With start anchor, give up if not at position 0. + if (requires_start && abs_pos > 0) { + return iree_ok_status(); ++ } ++ // This transition was already attempted from the start state. No ++ // match state changed, so retrying the same character cannot succeed. ++ if (state->dfa_state == dfa->header->start_state) { ++ state->match_start = abs_pos; ++ if (scan_failed_start_ascii) { ++ const iree_host_size_t scan_begin = position; ++ while (position < chunk.size) { ++ const uint8_t b = (uint8_t)chunk.data[position]; ++ if (b >= 0x80 || ++ start_transitions[b] != ++ IREE_TOKENIZER_UTIL_REGEX_NO_TRANSITION) { ++ break; ++ } ++ ++position; ++ } ++ // Preserve the same last failed character as scalar execution, ++ // including when the initial failure was a multi-byte codepoint. ++ if (position != scan_begin) { ++ state->match_start = base_offset + position - 1; ++ } ++ } ++ continue; + } + // If we already tried, advance. + if (state->match_start == abs_pos) { +diff --git a/runtime/src/iree/tokenizer/regex/exec_test.cc b/runtime/src/iree/tokenizer/regex/exec_test.cc +--- a/runtime/src/iree/tokenizer/regex/exec_test.cc ++++ b/runtime/src/iree/tokenizer/regex/exec_test.cc +@@ -504,6 +504,40 @@ + &collector)); + + EXPECT_EQ(collector.matches.size(), 0); ++} ++ ++TEST(DfaExec, UnmatchedPrefixAndStreamingPartialMatch) { ++ auto data = BuildAbcDfa(); ++ iree_tokenizer_regex_dfa_t dfa; ++ IREE_ASSERT_OK(iree_tokenizer_regex_dfa_load( ++ iree_make_const_byte_span(data.data(), data.size()), &dfa)); ++ std::string text(4096, 'x'); ++ text += "abxabc"; ++ text.append(512, 'x'); ++ text += "abc"; ++ MatchCollector reference; ++ IREE_ASSERT_OK(iree_tokenizer_regex_exec( ++ &dfa, iree_make_string_view(text.data(), text.size()), ++ MatchCollector::Callback, &reference)); ++ ASSERT_EQ(reference.matches.size(), 2); ++ EXPECT_EQ(reference.matches[0].first, 4099); ++ EXPECT_EQ(reference.matches[0].second, 4102); ++ EXPECT_EQ(reference.matches[1].first, 4614); ++ EXPECT_EQ(reference.matches[1].second, 4617); ++ for (size_t chunk_size : {1, 2, 7, 64, 1024}) { ++ iree_tokenizer_regex_exec_state_t state; ++ iree_tokenizer_regex_exec_initialize(&state, &dfa); ++ MatchCollector actual; ++ for (size_t offset = 0; offset < text.size(); offset += chunk_size) { ++ const size_t length = std::min(chunk_size, text.size() - offset); ++ IREE_ASSERT_OK(iree_tokenizer_regex_exec_feed( ++ &dfa, &state, iree_make_string_view(text.data() + offset, length), ++ offset, nullptr, MatchCollector::Callback, &actual)); ++ } ++ IREE_ASSERT_OK(iree_tokenizer_regex_exec_finalize( ++ &dfa, &state, text.size(), MatchCollector::Callback, &actual)); ++ EXPECT_EQ(actual.matches, reference.matches); ++ } + } + + TEST(DfaExec, EmptyInput) { +@@ -2839,4 +2873,161 @@ + TestWithChunkSizes(&dfa, "a b ", expected, {1, 2, 3, 4, 5, 6, 10}); + } + ++ ++// Exercises the failed-start scan with hand-built exact range transitions. ++// Ranges may cover ASCII even when the byte table has NO_TRANSITION. ++std::vector BuildAbcWithExactRange(uint16_t from_state, ++ uint16_t target_state, ++ uint32_t codepoint) { ++ auto data = BuildAbcDfa(); ++ const size_t offset = data.size(); ++ data.resize(offset + sizeof(uint32_t) + ++ sizeof(iree_tokenizer_regex_dfa_range_transition_t)); ++ auto* header = ++ reinterpret_cast(data.data()); ++ header->flags |= IREE_TOKENIZER_UTIL_REGEX_DFA_FLAG_HAS_RANGES; ++ const uint16_t counts[] = {1, 0}; ++ std::memcpy(data.data() + offset, counts, sizeof(counts)); ++ const iree_tokenizer_regex_dfa_range_transition_t range = { ++ from_state, target_state, codepoint, codepoint}; ++ std::memcpy(data.data() + offset + sizeof(counts), &range, sizeof(range)); ++ return data; ++} ++ ++void CheckFailedStartScanChunks( ++ const iree_tokenizer_regex_dfa_t* dfa, const std::string& text, ++ const std::vector>& expected) { ++ for (bool use_stride : {false, true}) { ++ iree_tokenizer_regex_stride_t* stride = nullptr; ++ if (use_stride) { ++ IREE_ASSERT_OK(iree_tokenizer_regex_stride_allocate( ++ dfa, iree_allocator_system(), &stride)); ++ } ++ for (size_t chunk_size : {1, 2, 7, 64, 1024, 4097}) { ++ iree_tokenizer_regex_exec_state_t state; ++ iree_tokenizer_regex_exec_initialize(&state, dfa); ++ MatchCollector actual; ++ size_t offset = 0; ++ while (offset < text.size()) { ++ size_t end = std::min(offset + chunk_size, text.size()); ++ // exec_feed requires complete UTF-8 codepoints in each chunk. ++ while (end > offset && end < text.size() && ++ (static_cast(text[end]) & 0xC0) == 0x80) { ++ --end; ++ } ++ if (end == offset) { ++ ++end; ++ while (end < text.size() && ++ (static_cast(text[end]) & 0xC0) == 0x80) { ++ ++end; ++ } ++ } ++ IREE_ASSERT_OK(iree_tokenizer_regex_exec_feed( ++ dfa, &state, ++ iree_make_string_view(text.data() + offset, end - offset), offset, ++ stride, MatchCollector::Callback, &actual)); ++ offset = end; ++ // Empty feeds must preserve pending/backtrack state and absolute offsets. ++ IREE_ASSERT_OK(iree_tokenizer_regex_exec_feed( ++ dfa, &state, iree_make_string_view("", 0), offset, stride, ++ MatchCollector::Callback, &actual)); ++ } ++ IREE_ASSERT_OK(iree_tokenizer_regex_exec_finalize( ++ dfa, &state, text.size(), MatchCollector::Callback, &actual)); ++ EXPECT_EQ(actual.matches, expected) ++ << "stride=" << use_stride << " chunk_size=" << chunk_size; ++ } ++ iree_tokenizer_regex_stride_free(stride, iree_allocator_system()); ++ } ++} ++ ++TEST(FailedStartAsciiScan, HonorsExactAsciiRangeAtInitialState) { ++ auto data = BuildAbcWithExactRange(0, 1, 'x'); ++ iree_tokenizer_regex_dfa_t dfa; ++ IREE_ASSERT_OK(iree_tokenizer_regex_dfa_load( ++ iree_make_const_byte_span(data.data(), data.size()), &dfa)); ++ std::string text(2048, 'z'); ++ text += "xbc"; ++ text.append(33, 'z'); ++ text += "abc"; ++ CheckFailedStartScanChunks(&dfa, text, {{2048, 2051}, {2084, 2087}}); ++} ++ ++TEST(FailedStartAsciiScan, StopsBeforeExactNonAsciiRange) { ++ auto data = BuildAbcWithExactRange(0, 1, 0x4E2D); ++ iree_tokenizer_regex_dfa_t dfa; ++ IREE_ASSERT_OK(iree_tokenizer_regex_dfa_load( ++ iree_make_const_byte_span(data.data(), data.size()), &dfa)); ++ std::string text(4096, 'z'); ++ text += "\xE4\xB8\xAD" "bc"; ++ text.append(127, 'z'); ++ text += "abc"; ++ CheckFailedStartScanChunks(&dfa, text, {{4096, 4101}, {4228, 4231}}); ++} ++ ++TEST(FailedStartAsciiScan, PreservesExactRangeFromAnotherState) { ++ auto data = BuildAbcWithExactRange(1, 2, 'x'); ++ iree_tokenizer_regex_dfa_t dfa; ++ IREE_ASSERT_OK(iree_tokenizer_regex_dfa_load( ++ iree_make_const_byte_span(data.data(), data.size()), &dfa)); ++ std::string text(4096, 'z'); ++ text += "axc"; ++ text.append(511, 'z'); ++ text += "abc"; ++ CheckFailedStartScanChunks(&dfa, text, {{4096, 4099}, {4610, 4613}}); ++} ++ ++TEST(FailedStartAsciiScan, PreservesFailedCodepointOffsetAtChunkEnd) { ++ auto data = BuildAbcDfa(); ++ iree_tokenizer_regex_dfa_t dfa; ++ IREE_ASSERT_OK(iree_tokenizer_regex_dfa_load( ++ iree_make_const_byte_span(data.data(), data.size()), &dfa)); ++ std::string text(4096, 'x'); ++ text += "\xE4\xB8\xAD"; ++ iree_tokenizer_regex_exec_state_t state; ++ iree_tokenizer_regex_exec_initialize(&state, &dfa); ++ MatchCollector actual; ++ IREE_ASSERT_OK(iree_tokenizer_regex_exec_feed( ++ &dfa, &state, iree_make_string_view(text.data(), text.size()), 0, nullptr, ++ MatchCollector::Callback, &actual)); ++ EXPECT_EQ(state.match_start, 4096); ++ EXPECT_FALSE(state.in_match); ++ EXPECT_FALSE(state.has_accept); ++ EXPECT_FALSE(state.has_accept_fallback); ++ EXPECT_FALSE(state.pending_lookahead); ++ EXPECT_TRUE(actual.matches.empty()); ++ CheckFailedStartScanChunks(&dfa, text, {}); ++} ++ ++TEST(FailedStartAsciiScan, PreservesLookaheadFallbackAndRewind) { ++ auto data = BuildOnePlusAWithFallbackDfa(); ++ iree_tokenizer_regex_dfa_t dfa; ++ IREE_ASSERT_OK(iree_tokenizer_regex_dfa_load( ++ iree_make_const_byte_span(data.data(), data.size()), &dfa)); ++ std::string text(2048, 'z'); ++ text += "aaXa"; ++ text.append(1024, 'z'); ++ text += "aX"; ++ CheckFailedStartScanChunks( ++ &dfa, text, {{2048, 2049}, {2049, 2050}, {2051, 2052}, {3076, 3077}}); ++} ++ ++TEST(FailedStartAsciiScan, PreservesStartAnchorExit) { ++ auto data = BuildStartAnchoredHelloDfa(); ++ // Set the actual per-state start constraint that exec_feed checks. The ++ // original fixture has different start states but no anchor bitmap. ++ const size_t offset = data.size(); ++ const uint64_t anchors[] = {(1ull << 0) | (1ull << 5), 0}; ++ data.resize(offset + sizeof(anchors)); ++ auto* header = ++ reinterpret_cast(data.data()); ++ header->flags |= IREE_TOKENIZER_UTIL_REGEX_DFA_FLAG_HAS_ANCHORS; ++ std::memcpy(data.data() + offset, anchors, sizeof(anchors)); ++ iree_tokenizer_regex_dfa_t dfa; ++ IREE_ASSERT_OK(iree_tokenizer_regex_dfa_load( ++ iree_make_const_byte_span(data.data(), data.size()), &dfa)); ++ CheckFailedStartScanChunks(&dfa, "Hello" + std::string(4096, 'x'), {{0, 5}}); ++ CheckFailedStartScanChunks(&dfa, std::string(4096, 'x') + "Hello", {}); ++} ++ + } // namespace diff --git a/third_party/iree/repo.bzl b/third_party/iree/repo.bzl index ee89fb6d..cb17cfaa 100644 --- a/third_party/iree/repo.bzl +++ b/third_party/iree/repo.bzl @@ -19,6 +19,7 @@ def repo(): "//third_party/iree:laguna-tokenizer-compat.patch", "//third_party/iree:fix-normalizer-lifetime-and-boundaries.patch", "//third_party/iree:mistral-right-aligned-numbers.patch", + "//third_party/iree:regex-skip-failed-start.patch", ], patch_args = ["-p1"], ) diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index fff3b8a8..48a4a761 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -1,5 +1,5 @@ load("@com_google_protobuf//bazel:upb_c_proto_library.bzl", "upb_c_proto_library") -load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") upb_c_proto_library( @@ -65,6 +65,16 @@ cc_library( }), ) +cc_test( + name = "profiling_traceme_test", + srcs = ["profiling/traceme_test.cc"], + deps = [ + ":profiling_traceme_lib", + "@tsl//tsl/profiler/protobuf:xplane_proto_cc", + "@xla//xla/tsl/profiler/backends/cpu:traceme_recorder", + ], +) + zig_library( name = "logo", srcs = ["logo.zig"], @@ -104,6 +114,7 @@ zig_library( "ops.zig", "ops/manual_computation.zig", "ops/manual_computation2.zig", + "packed_scale_layout.zig", "pjrtx.zig", "platform.zig", "profiling/profiler.zig", @@ -198,3 +209,8 @@ zig_test( name = "test", deps = ["//zml"], ) + +zig_test( + name = "packed_scale_layout_test", + main = "packed_scale_layout.zig", +) diff --git a/zml/floats.zig b/zml/floats.zig index 689c22a8..6d8976c9 100644 --- a/zml/floats.zig +++ b/zml/floats.zig @@ -500,7 +500,11 @@ pub const Float4E2M1 = packed struct(u4) { for (&from_f32_res, 0..) |*r, i| { r.* = .fromF32(Float4E2M1.values[i]); } - try std.testing.expectEqualSlices(u4, &.{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }, @ptrCast(&from_f32_res)); + // An unpacked FP4 value occupies a byte with unspecified padding bits. + // Compare each four-bit value instead of the backing bytes through a cast. + for (from_f32_res, 0..) |value, expected| { + try std.testing.expectEqual(@as(u4, @intCast(expected)), @as(u4, @bitCast(value))); + } } pub const Packed = packed struct(@Int(.unsigned, @bitSizeOf(Float4E2M1) * 2)) { diff --git a/zml/nn.zig b/zml/nn.zig index a2b582ee..1a97a8dc 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -48,7 +48,7 @@ pub const Linear = struct { if (quantization.quantizeInput(q, x, self.tag, zml.Compiler.current().platform)) |input| { return self.forwardQuantized(input, out_dtype); } - return self.forwardScaled(x.convert(.bf16), null, null, out_dtype); + return self.forwardScaled(x.convert(.bf16), null, null, .logical, out_dtype); } const y = x.dot(self.weight.convert(x.dtype()), self.tag).convert(out_dtype); return if (self.bias) |bias| y.add(bias.convert(y.dtype()).broad(y.shape())) else y; @@ -58,15 +58,20 @@ pub const Linear = struct { /// Convert to output_dtype before adding bias. pub fn forwardQuantized(self: Linear, input: quantization.QuantizedInput, output_dtype: DataType) Tensor { stdx.debug.assert(self.quantization != null, "forwardQuantized requires quantized weights", .{}); - return self.forwardScaled(input.values, input.scales, input.global_scale, output_dtype); + return self.forwardScaled(input.values, input.scales, input.global_scale, input.scale_layout, output_dtype); } - fn forwardScaled(self: Linear, lhs: Tensor, lhs_scale: ?Tensor, input_global_scale: ?Tensor, output_dtype: DataType) Tensor { + fn forwardScaled(self: Linear, lhs: Tensor, lhs_scale: ?Tensor, input_global_scale: ?Tensor, scale_layout: quantization.QuantizedInput.ScaleLayout, output_dtype: DataType) Tensor { const q = self.quantization.?; const weight_global_scale: ?Tensor = if (q.global_scale) |s| s.asMultiplier() else null; const weight = if (isPackedFp4(q.scheme, self.weight.dtype())) unpackFp4(self.weight, self.tag, self.tag) else self.weight; const scales = if (q.scheme.isMx() and q.scales.dtype() == .u8) q.scales.bitCast(.f8e8m0) else q.scales; - const acc = scaledDot(lhs, weight, lhs_scale, scales, output_dtype, self.tag, .{ .rhs_scale_swizzled = q.swizzled_scales }); + const acc = scaledDot(lhs, weight, lhs_scale, scales, output_dtype, self.tag, .{ + .lhs_scale_swizzled = scale_layout == .blackwell_128x4, + .rhs_scale_swizzled = q.scaleStorageLayout() == .blackwell_128x4, + .lhs_scale_packed_column = scale_layout == .packed_column_4, + .rhs_scale_packed_column = q.scaleStorageLayout() == .packed_column_4, + }); const y = applyGlobalScale(acc, input_global_scale, weight_global_scale).convert(output_dtype); return if (self.bias) |bias| y.add(bias.convert(y.dtype()).broad(y.shape())) else y; } @@ -202,23 +207,41 @@ pub fn scaledDot( .intArray(mlir_ctx, i64, rhs_batching_axes.constSlice()), }), }); - const attributes = [_]mlir.NamedAttribute{ - .named(mlir_ctx, "dimension_numbers", dnums), - .named(mlir_ctx, "rhs_scale_swizzled", .boolean(mlir_ctx, true)), - }; + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(mlir_ctx, "dimension_numbers", dnums)); + stdx.debug.assert(!(opts.lhs_scale_swizzled and opts.lhs_scale_packed_column) and !(opts.rhs_scale_swizzled and opts.rhs_scale_packed_column), "scaledDot: conflicting scale storage layouts", .{}); + if (opts.lhs_scale_swizzled) { + stdx.debug.assert(lhs_scale != null and lhs.rank() == 2 and lhs_scale_operand.rank() == 2 and lhs_contracting_axes.get(0) == 1, "scaledDot: a swizzled lhs scale needs a [m, k] lhs with scales, got {f}", .{lhs}); + attributes.appendAssumeCapacity(.named(mlir_ctx, "lhs_scale_swizzled", .boolean(mlir_ctx, true))); + } if (opts.rhs_scale_swizzled) { stdx.debug.assert(rhs.rank() == 2 and rhs_scale.rank() == 2 and rhs_contracting_axes.get(0) == 1, "scaledDot: a swizzled rhs scale needs a [n, k] rhs, got {f}", .{rhs}); + attributes.appendAssumeCapacity(.named(mlir_ctx, "rhs_scale_swizzled", .boolean(mlir_ctx, true))); + } + if (opts.lhs_scale_packed_column) { + stdx.debug.assert(lhs_scale != null and lhs.rank() == 2 and lhs_scale_operand.rank() == 2 and lhs_contracting_axes.get(0) == 1, "scaledDot: a packed-column lhs scale needs a [m, k] lhs with scales, got {f}", .{lhs}); + attributes.appendAssumeCapacity(.named(mlir_ctx, "lhs_scale_packed_column", .boolean(mlir_ctx, true))); + } + if (opts.rhs_scale_packed_column) { + stdx.debug.assert(rhs.rank() == 2 and rhs_scale.rank() == 2 and rhs_contracting_axes.get(0) == 1, "scaledDot: a packed-column rhs scale needs a [n, k] rhs, got {f}", .{rhs}); + attributes.appendAssumeCapacity(.named(mlir_ctx, "rhs_scale_packed_column", .boolean(mlir_ctx, true))); } return ops.composite("xla.scaled_dot", &.{ lhs, rhs, lhs_scale_operand, rhs_scale_operand }, &.{res_shape}, scaledDotReference, res_shape, .{ - .composite_attributes = if (opts.rhs_scale_swizzled) &attributes else attributes[0..1], + .composite_attributes = attributes.constSlice(), })[0]; } pub const ScaledDotOpts = struct { + /// The lhs scale uses `quantization.BlackwellScaleLayout`, per device. + lhs_scale_swizzled: bool = false, /// The rhs scale's `[n, k / block]` grid is stored in the 128x4 blocked layout of Blackwell's block-scaled mma, /// each device's slice on its own. // TODO: fine for now, but we should find a better way to express how a scale is laid out. rhs_scale_swizzled: bool = false, + /// The lhs scale uses `quantization.PackedColumnScaleLayout`, per device. + lhs_scale_packed_column: bool = false, + /// The rhs scale uses `quantization.PackedColumnScaleLayout`, per device. + rhs_scale_packed_column: bool = false, }; test "block128 scaled dot layouts" { @@ -309,6 +332,93 @@ test "block128 scaled dot layouts" { } } +test "packed activation scales linear matches independent dequantization" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = zml.testing.env(); + if (platform.target != .cuda) return error.SkipZigTest; + const Local = struct { + const Outputs = struct { actual: Tensor, expected: Tensor }; + + fn dequantize(values: Tensor, scales: Tensor) Tensor { + const expanded = scales.renameTag(.k, .ks) + .broad(values.shape().splitAxis(.k, .{ .ks = -1, .block = 32 }).withDtype(.f32)) + .reshape(values.shape().withDtype(.f32)); + return values.convert(.f32).mul(expanded); + } + + fn forward(x: Tensor, w: Tensor, packed_scales: Tensor, logical: Tensor, weight_scales: Tensor, packed_weight_scales: Tensor, packed_rhs: bool) Outputs { + const input: quantization.QuantizedInput = .{ + .values = x.convert(.f8e4m3fn), + .scales = packed_scales.bitCast(.f8e8m0), + .global_scale = null, + .scale_layout = .blackwell_128x4, + }; + const linear: Linear = .{ + .weight = w.convert(.f8e4m3fn), + .tag = Shape.toTag(.k), + .quantization = .{ + .scheme = .mxfp8, + .scales = if (packed_rhs) packed_weight_scales.bitCast(.f8e8m0) else weight_scales.convert(.f8e8m0), + .swizzled_scales = packed_rhs, + }, + }; + return .{ + .actual = linear.forwardQuantized(input, .bf16), + .expected = dequantize(x, logical).dot(dequantize(w, weight_scales), .k).convert(.bf16), + }; + } + }; + inline for (.{ 1, 31, 32, 127, 128, 129 }) |rows| { + inline for (.{ false, true }) |packed_rhs| { + const x: Tensor = .init(.{ .m = rows, .k = 160 }, .bf16); + const w: Tensor = .init(.{ .n = 64, .k = 160 }, .bf16); + const logical: Tensor = .init(.{ .m = rows, .k = 5 }, .f32); + const packed_scales: Tensor = .init(quantization.BlackwellScaleLayout.paddedShape(logical.shape()).withDtype(.u8), .u8); + const weight_scales: Tensor = .init(.{ .n = 64, .k = 5 }, .f32); + const packed_weight_scales: Tensor = .init(quantization.BlackwellScaleLayout.paddedShape(weight_scales.shape()).withDtype(.u8), .u8); + var exe = try platform.compileFn(allocator, io, Local.forward, .{ x, w, packed_scales, logical, weight_scales, packed_weight_scales, packed_rhs }, .{}); + defer exe.deinit(); + var buffers: [6]zml.Buffer = undefined; + var initialized: usize = 0; + defer for (buffers[0..initialized]) |*buffer| buffer.deinit(); + for ([_]Shape{ x.shape(), w.shape(), packed_scales.shape(), logical.shape(), weight_scales.shape(), packed_weight_scales.shape() }, 0..) |shape, input| { + const slice = try Slice.alloc(allocator, shape); + defer slice.free(allocator); + if (input < 2) { + for (slice.items(zml.floats.BFloat16), 0..) |*value, i| { + value.* = .fromF32(@as(f32, @floatFromInt(@as(i32, @intCast((i * 7 + i / 160) % 9)) - 4)) / 2); + } + } else if (input == 2 or input == 5) { + @memset(slice.items(u8), 0); + for (0..if (input == 2) rows else 64) |row| { + for (0..5) |column| { + slice.items(u8)[quantization.BlackwellScaleLayout.offset(row, column, @intCast(shape.dim(1)))] = @intCast(124 + (row * 3 + column) % 7); + } + } + } else { + for (slice.items(f32), 0..) |*value, i| { + const row = i / 5; + const column = i % 5; + const exponent: i32 = @as(i32, @intCast((row * 3 + column) % 7)) - 3; + value.* = std.math.pow(f32, 2, @floatFromInt(exponent)); + } + } + buffers[input] = try zml.Buffer.fromSlice(io, platform, slice); + initialized += 1; + } + var output: zml.Bufferized(Local.Outputs) = undefined; + var runner = try exe.runner(allocator); + defer runner.deinit(allocator); + runner.run(io, .{ buffers[0], buffers[1], buffers[2], buffers[3], buffers[4], buffers[5] }, .{&output}, .{ .wait = true }); + defer zml.Buffer.deinitAll(Local.Outputs, &output); + var expected = try output.expected.toSliceAlloc(allocator, io); + defer expected.free(allocator); + try zml.testing.expectClose(io, expected, output.actual, .{ .absolute_tolerance = 0.125, .relative_tolerance = 0.02 }); + } + } +} + /// `shape` with every dimension collapsed to 1 fn onesGrid(shape: Shape, dt: DataType) Shape { var res = shape.withDtype(dt); diff --git a/zml/packed_scale_layout.zig b/zml/packed_scale_layout.zig new file mode 100644 index 00000000..d9de8686 --- /dev/null +++ b/zml/packed_scale_layout.zig @@ -0,0 +1,81 @@ +//! DeepGEMM's TMA-aligned, column-major packed UE8M0 scale storage. +//! +//! Four adjacent K scales form a little-endian u32. These words are column +//! major: their row stride is one and their column stride is paddedRows(rows). +//! This describes one device's local plane. Pad it before joining rank slices. +//! Reference: DeepGEMM 1fc9c669, csrc/jit_kernels/impls/smxx_layout.hpp, +//! get_mn_major_tma_aligned_packed_ue8m0_tensor_torch (MIT license). +const std = @import("std"); + +pub const row_alignment = 4; +pub const column_alignment = 4; + +pub fn paddedRows(rows: usize) usize { + return std.mem.alignForward(usize, rows, row_alignment); +} + +pub fn paddedColumns(columns: usize) usize { + return std.mem.alignForward(usize, columns, column_alignment); +} + +pub fn byteSize(rows: usize, columns: usize) usize { + return paddedRows(rows) * paddedColumns(columns); +} + +pub fn offset(row: usize, column: usize, padded_rows: usize, padded_columns: usize) usize { + std.debug.assert(padded_rows % row_alignment == 0 and row < padded_rows); + std.debug.assert(padded_columns % column_alignment == 0 and column < padded_columns); + return (column / column_alignment * padded_rows + row) * column_alignment + column % column_alignment; +} + +test "packed column scales match four-byte column-major words including padding" { + for ([_]usize{ 1, 3, 4, 5, 31, 32, 127, 128, 129, 136, 224, 256, 512, 768, 4096, 5120 }) |rows| { + for ([_]usize{ 1, 3, 4, 5, 17, 192, 256 }) |columns| { + const rp = paddedRows(rows); + const cp = paddedColumns(columns); + const seen = try std.testing.allocator.alloc(bool, byteSize(rows, columns)); + defer std.testing.allocator.free(seen); + @memset(seen, false); + for (0..rp) |row| { + for (0..cp) |column| { + const address = offset(row, column, rp, cp); + try std.testing.expect(address < seen.len and !seen[address]); + seen[address] = true; + // Independent indexing of an int32 matrix with strides + // (1, rp), followed by selecting its little-endian byte. + const word = row + rp * (column >> 2); + try std.testing.expectEqual(word * @sizeOf(u32) + (column & 3), address); + } + } + for (seen) |visited| try std.testing.expect(visited); + } + } +} + +test "packed column scales preserve logical bytes and separately padded rank planes" { + const rows = 5; + const columns = 7; + const rp = paddedRows(rows); + const cp = paddedColumns(columns); + const rank_bytes = comptime byteSize(rows, columns); + var storage: [2 * rank_bytes]u8 = @splat(0); + for (0..2) |rank| { + for (0..rows) |row| { + for (0..columns) |column| { + const exponent: u8 = @intCast(105 + rank * 11 + row + 2 * column); + storage[rank * rank_bytes + offset(row, column, rp, cp)] = exponent; + } + } + } + for (0..2) |rank| { + for (0..rp) |row| { + for (0..cp) |column| { + const actual = storage[rank * rank_bytes + offset(row, column, rp, cp)]; + const expected: u8 = if (row < rows and column < columns) @intCast(105 + rank * 11 + row + 2 * column) else 0; + try std.testing.expectEqual(expected, actual); + } + } + } + // A global row padding calculation would silently lose four rows here. + try std.testing.expect(2 * paddedRows(rows) != paddedRows(2 * rows)); +} diff --git a/zml/profiling/profiler.zig b/zml/profiling/profiler.zig index d46a0c4c..2e3d0ced 100644 --- a/zml/profiling/profiler.zig +++ b/zml/profiling/profiler.zig @@ -230,6 +230,9 @@ pub const Profiler = struct { try inner.stop(); break :blk try inner.collectData(self.arena.allocator()); } else &.{}; + // Keep the exact backend bytes independently of later parsing/merging. + // These .raw artifacts are diagnostics, not normalized profile files. + try self.writeRawData("profiling.backend.raw", protobuf); const merged_protobuf = c.zml_traceme_session_merge(zffi.ZigSlice.from(protobuf)); defer if (merged_protobuf.len != 0) { c.zml_traceme_str_free(merged_protobuf); @@ -239,6 +242,7 @@ pub const Profiler = struct { return null; } + try self.writeRawData("profiling.merged.raw", final_protobuf); const normalized_protobuf = c.zml_traceme_session_postprocess( zffi.ZigSlice.from(final_protobuf), self.start_timestamp_ns orelse self.stop_timestamp_ns.?, @@ -273,6 +277,12 @@ pub const Profiler = struct { return self.profile; } + fn writeRawData(self: *Profiler, name: []const u8, contents: []const u8) !void { + const path = try std.Io.Dir.path.join(self.arena.allocator(), &.{ self.session_dir, name }); + try self.writeFile(path, contents); + log.info("Profiler raw bytes preserved: {s} ({d} bytes)", .{ path, contents.len }); + } + fn writeFile(self: *const Profiler, path: []const u8, contents: []const u8) !void { const file = try std.Io.Dir.createFile(.cwd(), self.io, path, .{}); defer file.close(self.io); diff --git a/zml/profiling/traceme_lib.cc b/zml/profiling/traceme_lib.cc index ca37dee7..9840807d 100644 --- a/zml/profiling/traceme_lib.cc +++ b/zml/profiling/traceme_lib.cc @@ -1,5 +1,6 @@ #include "zml/profiling/traceme.h" +#include #include #include #include @@ -58,6 +59,35 @@ zig_slice CopyToOwnedSlice(const std::string& data) { return {ptr, data.size()}; } +// PJRT's generic contract is binary data with an explicit length. Some +// providers append one NUL sentinel; accept that only if the exact bytes fail +// and removing that sentinel produces a valid XSpace. +bool ParseXSpace(zig_slice bytes, tensorflow::profiler::XSpace* space) { + if (bytes.len > INT_MAX || (bytes.len != 0 && bytes.ptr == nullptr)) { + return false; + } + if (bytes.len == 0) { + return true; + } + const int size = static_cast(bytes.len); + if (space->ParseFromArray(bytes.ptr, size)) { + return true; + } + const auto* data = static_cast(bytes.ptr); + if (data[bytes.len - 1] != 0) { + return false; + } + space->Clear(); + return space->ParseFromArray(bytes.ptr, size - 1); +} + +void DiscardLocalHostTraceSession() { + if (g_local_host_trace_session.active) { + (void)tsl::profiler::TraceMeRecorder::Stop(); + g_local_host_trace_session.active = false; + } +} + #if defined(ZML_RUNTIME_CUDA) struct CudaNvtxRangeBackend { explicit CudaNvtxRangeBackend(const std::string& name) @@ -215,10 +245,9 @@ extern "C" void zml_traceme_session_start(int level, uint64_t filter_mask, extern "C" zig_slice zml_traceme_session_merge(zig_slice xspace) { tensorflow::profiler::XSpace space; - if (xspace.len != 0 && - !space.ParseFromArray(xspace.ptr, static_cast(xspace.len))) { - return CopyToOwnedSlice( - std::string(static_cast(xspace.ptr), xspace.len)); + if (!ParseXSpace(xspace, &space)) { + DiscardLocalHostTraceSession(); + return {}; } if (g_local_host_trace_session.active) { @@ -238,8 +267,8 @@ extern "C" zig_slice zml_traceme_session_postprocess(zig_slice xspace, uint64_t start_time_ns, uint64_t stop_time_ns) { tensorflow::profiler::XSpace space; - if (xspace.len != 0 && - !space.ParseFromArray(xspace.ptr, static_cast(xspace.len))) { + if (!ParseXSpace(xspace, &space)) { + DiscardLocalHostTraceSession(); return {}; } diff --git a/zml/profiling/traceme_test.cc b/zml/profiling/traceme_test.cc new file mode 100644 index 00000000..2d855382 --- /dev/null +++ b/zml/profiling/traceme_test.cc @@ -0,0 +1,87 @@ +#include "zml/profiling/traceme.h" + +#include +#include +#include + +#include "tsl/profiler/protobuf/xplane.pb.h" +#include "xla/tsl/profiler/backends/cpu/traceme_recorder.h" + +namespace { + +int failures = 0; + +void Check(bool condition, const char* name) { + if (!condition) { + std::cerr << "FAIL: " << name << '\n'; + ++failures; + } +} + +zig_slice Slice(const char* bytes, size_t size) { + return {const_cast(bytes), size}; +} + +void CheckAccepted(zig_slice input, const char* name) { + const bool modes[] = {false, true}; + for (bool postprocess : modes) { + zig_slice result = postprocess + ? zml_traceme_session_postprocess(input, 0, 1) + : zml_traceme_session_merge(input); + tensorflow::profiler::XSpace space; + Check(result.len > 0 && result.len <= INT_MAX && + space.ParseFromArray(result.ptr, static_cast(result.len)) && + space.planes_size() > 0 && space.planes(0).id() == 0, + name); + zml_traceme_str_free(result); + } +} + +void CheckRejected(zig_slice input, const char* name) { + zig_slice merged = zml_traceme_session_merge(input); + Check(merged.len == 0, name); + zml_traceme_str_free(merged); + zig_slice normalized = zml_traceme_session_postprocess(input, 0, 1); + Check(normalized.len == 0, name); + zml_traceme_str_free(normalized); +} + +} // namespace + +int main() { + // A real XSpace wire message: planes[0] has an explicit int64 id = 0. + const char terminal_zero[] = {0x0a, 0x02, 0x08, 0x00}; + const char sentinel_zero[] = {0x0a, 0x02, 0x08, 0x00, 0x00}; + const char two_sentinels[] = {0x0a, 0x02, 0x08, 0x00, 0x00, 0x00}; + const char malformed[] = {0x0a, 0x02, 0x08}; + const char malformed_zero[] = {static_cast(0xff), 0x00}; + CheckAccepted(Slice(terminal_zero, sizeof(terminal_zero)), + "valid binary terminal zero survives"); + CheckAccepted(Slice(sentinel_zero, sizeof(sentinel_zero)), + "one provider sentinel is accepted after validated retry"); + CheckRejected(Slice(two_sentinels, sizeof(two_sentinels)), + "two sentinels are rejected"); + CheckRejected(Slice(malformed, sizeof(malformed)), "truncated protobuf rejected"); + CheckRejected(Slice(malformed_zero, sizeof(malformed_zero)), + "invalid bytes remain invalid after sentinel retry"); + CheckRejected(Slice(terminal_zero, static_cast(INT_MAX) + 1), + "oversized input rejected before reading or int conversion"); + CheckRejected({nullptr, 1}, "nonempty null input rejected"); + + const bool modes[] = {false, true}; + for (bool postprocess : modes) { + zml_traceme_session_start(1, 0, false); + zig_slice input = Slice(malformed, sizeof(malformed)); + zig_slice result = postprocess + ? zml_traceme_session_postprocess(input, 0, 1) + : zml_traceme_session_merge(input); + Check(result.len == 0, "malformed active session rejected"); + zml_traceme_str_free(result); + const bool restarted = tsl::profiler::TraceMeRecorder::Start(1); + Check(restarted, "parse failure releases local host TraceMe session"); + if (restarted) (void)tsl::profiler::TraceMeRecorder::Stop(); + } + + if (failures == 0) std::cout << "All XSpace compatibility checks passed.\n"; + return failures == 0 ? 0 : 1; +} diff --git a/zml/quantization.zig b/zml/quantization.zig index db20619a..be53e420 100644 --- a/zml/quantization.zig +++ b/zml/quantization.zig @@ -14,6 +14,16 @@ const Tensor = @import("tensor.zig").Tensor; pub const nvfp4_block_size = 16; pub const mx_block_size = 32; +pub const PackedColumnScaleLayout = @import("packed_scale_layout.zig"); + +pub const ScaleStorageLayout = enum { + logical, + /// Each device's scale plane uses the independently padded 128x4 layout. + blackwell_128x4, + /// Four adjacent scale bytes per column-major u32, independently per device. + packed_column_4, +}; + pub const Quantization = struct { scheme: Scheme, // decided when the checkpoint is read scales: Tensor, @@ -22,6 +32,16 @@ pub const Quantization = struct { /// HACK: `scales` is stored in the 128x4 blocked layout of Blackwell's block-scaled mma. // Something cleaner will need to be done to differentiate different kinds of swizzling swizzled_scales: bool = false, + /// Explicit storage for new producers; null preserves swizzled_scales. + scale_layout: ?ScaleStorageLayout = null, + + pub fn scaleStorageLayout(self: Quantization) ScaleStorageLayout { + if (self.scale_layout) |layout| { + stdx.debug.assert(!self.swizzled_scales or layout == .blackwell_128x4, "conflicting scale storage layouts", .{}); + return layout; + } + return if (self.swizzled_scales) .blackwell_128x4 else .logical; + } /// A per-tensor scale and the polarity its producer wrote it in pub const GlobalScale = struct { @@ -111,16 +131,82 @@ pub const QuantizedInput = struct { values: Tensor, scales: Tensor, global_scale: ?Tensor, + scale_layout: ScaleLayout = .logical, + + pub const ScaleLayout = ScaleStorageLayout; pub fn reuseBuffer(self: QuantizedInput, other: QuantizedInput) QuantizedInput { + stdx.debug.assert(self.scale_layout == other.scale_layout, "cannot reuse quantized scales with different layouts", .{}); return .{ .values = self.values.reuseBuffer(other.values), .scales = self.scales.reuseBuffer(other.scales), .global_scale = if (self.global_scale != null and other.global_scale != null) self.global_scale.?.reuseBuffer(other.global_scale.?) else self.global_scale, + .scale_layout = self.scale_layout, }; } }; +/// The scale-byte layout consumed by Blackwell block-scaled matrix multiplication. +/// `paddedShape` and `offset` describe one device's local two-dimensional grid. +/// Pad local slices before joining them; padding never denotes additional values. +pub const BlackwellScaleLayout = struct { + pub const rows = 128; + pub const columns = 4; + + pub fn paddedShape(logical: Shape) Shape { + stdx.debug.assert(logical.rank() == 2, "Blackwell scales require a matrix, got {f}", .{logical}); + return logical.setDim(0, std.mem.alignForward(i64, logical.dim(0), rows)) + .setDim(1, std.mem.alignForward(i64, logical.dim(1), columns)); + } + + pub fn offset(row: usize, column: usize, padded_columns: usize) usize { + stdx.debug.assert(padded_columns % columns == 0 and column < padded_columns, "invalid Blackwell scale column {}/{}", .{ column, padded_columns }); + return (row / rows * (padded_columns / columns) + column / columns) * (rows * columns) + + row % 32 * 16 + row % rows / 32 * columns + column % columns; + } +}; + +test "scale storage layout preserves legacy flags and explicit packed metadata" { + var q: Quantization = .{ .scheme = .mxfp8, .scales = undefined }; + try std.testing.expectEqual(.logical, q.scaleStorageLayout()); + q.swizzled_scales = true; + try std.testing.expectEqual(.blackwell_128x4, q.scaleStorageLayout()); + q.scale_layout = .blackwell_128x4; + try std.testing.expectEqual(.blackwell_128x4, q.scaleStorageLayout()); + q.swizzled_scales = false; + q.scale_layout = .packed_column_4; + try std.testing.expectEqual(.packed_column_4, q.scaleStorageLayout()); + q.scale_layout = .logical; + try std.testing.expectEqual(.logical, q.scaleStorageLayout()); +} + +test "Blackwell scale layout padding and address bijection" { + inline for (.{ 1, 31, 32, 127, 128, 129, 255, 256 }) |m| { + inline for (.{ 1, 3, 4, 5, 17 }) |k| { + const logical: Shape = .init(.{ .m = m, .k = k }, .f8e8m0); + const padded = BlackwellScaleLayout.paddedShape(logical); + try std.testing.expectEqual(@as(i64, ((m + 127) / 128) * 128), padded.dim(.m)); + try std.testing.expectEqual(@as(i64, ((k + 3) / 4) * 4), padded.dim(.k)); + const num_rows: usize = @intCast(padded.dim(.m)); + const num_cols: usize = @intCast(padded.dim(.k)); + var seen = try std.testing.allocator.alloc(bool, num_rows * num_cols); + defer std.testing.allocator.free(seen); + @memset(seen, false); + for (0..num_rows) |row| { + for (0..num_cols) |col| { + const index = BlackwellScaleLayout.offset(row, col, num_cols); + try std.testing.expect(index < seen.len); + try std.testing.expect(!seen[index]); + seen[index] = true; + const reference = (((row / 128 * (num_cols / 4) + col / 4) * 32 + row % 32) * 4 + row % 128 / 32) * 4 + col % 4; + try std.testing.expectEqual(reference, index); + } + } + for (seen) |visited| try std.testing.expect(visited); + } + } +} + pub fn quantizeInput(quantization: Quantization, input: Tensor, axis: Shape.Tag, platform: *const Platform) ?QuantizedInput { const global_scale: ?Tensor = if (quantization.input_scale) |scale| scale.asMultiplier() else null; diff --git a/zml/tokenizer/BUILD.bazel b/zml/tokenizer/BUILD.bazel index f2cbc132..96dd51cb 100644 --- a/zml/tokenizer/BUILD.bazel +++ b/zml/tokenizer/BUILD.bazel @@ -32,6 +32,13 @@ zig_test( ], ) +zig_binary( + name = "encode_bench", + main = "encode_bench.zig", + tags = ["manual"], + deps = [":tokenizer"], +) + # todo: reenable after fix # zig_srcs( # name = "sources", diff --git a/zml/tokenizer/encode_bench.zig b/zml/tokenizer/encode_bench.zig new file mode 100644 index 00000000..888681b3 --- /dev/null +++ b/zml/tokenizer/encode_bench.zig @@ -0,0 +1,63 @@ +//! CPU encode timing with an independent full-token-ID reference. +const std = @import("std"); + +const tokenizer_lib = @import("zml/tokenizer"); + +pub fn main(init: std.process.Init) !void { + const allocator = init.gpa; + const io = init.io; + var tokenizer_path: ?[]const u8 = null; + var text_path: ?[]const u8 = null; + var expected_path: ?[]const u8 = null; + var iterations: usize = 100; + var warmups: usize = 10; + var args = init.minimal.args.iterate(); + _ = args.next(); + while (args.next()) |arg| { + const eq = std.mem.indexOfScalar(u8, arg, '=') orelse return error.InvalidArgument; + const key = arg[0..eq]; + const value = arg[eq + 1 ..]; + if (std.mem.eql(u8, key, "--tokenizer")) tokenizer_path = value else if (std.mem.eql(u8, key, "--text")) text_path = value else if (std.mem.eql(u8, key, "--expected")) expected_path = value else if (std.mem.eql(u8, key, "--iterations")) iterations = try std.fmt.parseInt(usize, value, 10) else if (std.mem.eql(u8, key, "--warmups")) warmups = try std.fmt.parseInt(usize, value, 10) else return error.InvalidArgument; + } + if (iterations == 0) return error.InvalidArgument; + const text = try std.Io.Dir.cwd().readFileAlloc(io, text_path orelse return error.MissingText, allocator, .unlimited); + defer allocator.free(text); + const expected = try std.Io.Dir.cwd().readFileAlloc(io, expected_path orelse return error.MissingReference, allocator, .unlimited); + defer allocator.free(expected); + if (@mod(expected.len, 4) != 0) return error.InvalidReference; + var tokenizer = try tokenizer_lib.Tokenizer.fromFile(allocator, io, tokenizer_path orelse return error.MissingTokenizer); + defer tokenizer.deinit(); + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + const samples = try allocator.alloc(i96, iterations); + defer allocator.free(samples); + for (0..warmups + iterations) |iteration| { + encoder.reset(); + const start = std.Io.Clock.awake.now(io); + const ids = try encoder.encodeAlloc(allocator, text); + const elapsed = start.durationTo(std.Io.Clock.awake.now(io)).toNanoseconds(); + defer allocator.free(ids); + if (ids.len != expected.len / 4) return error.TokenCountMismatch; + for (ids, 0..) |id, i| if (id != std.mem.readInt(u32, expected[i * 4 ..][0..4], .little)) return error.TokenIdMismatch; + if (iteration >= warmups) samples[iteration - warmups] = elapsed; + } + const ordered = try allocator.dupe(i96, samples); + defer allocator.free(ordered); + std.mem.sort(i96, ordered, {}, std.sort.asc(i96)); + var hash: [32]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(expected, &hash, .{}); + const result = .{ + .bytes = text.len, + .tokens = expected.len / 4, + .warmups = warmups, + .iterations = iterations, + .median_ns = ordered[ordered.len / 2], + .min_ns = ordered[0], + .samples_ns = samples, + .reference_sha256 = std.fmt.bytesToHex(hash, .lower), + .every_iteration_matches_reference = true, + }; + const output = try std.json.Stringify.valueAlloc(allocator, result, .{}); + defer allocator.free(output); + std.debug.print("{s}\n", .{output}); +}