From 8b89334af281ddef0b57e288f0013eef44736fd5 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 12 Jun 2025 21:24:32 +0200 Subject: [PATCH 01/40] add build.zig for mlir module --- build.zig | 65 +++++++++++++++++++++++++++++++++++++++++++++++- mlir/BUILD.bazel | 25 +++++++++++++++++++ mlir/mlir.zig | 2 ++ 3 files changed, 91 insertions(+), 1 deletion(-) mode change 120000 => 100644 build.zig mode change 100644 => 100755 mlir/mlir.zig diff --git a/build.zig b/build.zig deleted file mode 120000 index 0468ff14..00000000 --- a/build.zig +++ /dev/null @@ -1 +0,0 @@ -examples/build.zig \ No newline at end of file diff --git a/build.zig b/build.zig new file mode 100644 index 00000000..7949a2c4 --- /dev/null +++ b/build.zig @@ -0,0 +1,64 @@ +const std = @import("std"); + +pub fn build(b: *std.Build) void { + const target = b.standardTargetOptions(.{}); + const optimize = b.standardOptimizeOption(.{}); + + const mlir_c_deps = moduleFromBazelSrcs( + b, + "//mlir:sources", + "mlir/sources.tar", + "mlir/test_test_lib_c.zig", + .{ .link_libcpp = true }, + ); + addObjectFromBazel(mlir_c_deps, "//mlir:static_c", "mlir/libstatic_c.a"); + + const mlir_mod = b.addModule("mlir", .{ + .root_source_file = b.path("mlir/mlir.zig"), + .target = target, + .optimize = optimize, + .imports = &.{ + .{ .name = "c", .module = mlir_c_deps }, + }, + }); + + const mlir_test = b.addTest(.{ .root_module = mlir_mod }); + + const run_mlir_tests = b.addRunArtifact(mlir_test); + + const test_step = b.step("test", "Run unit tests"); + test_step.dependOn(&run_mlir_tests.step); +} + +/// Take the name of a Bazel `cc_static_library` and add it to the given module. +fn addObjectFromBazel(module: *std.Build.Module, name: []const u8, output: []const u8) void { + const b = module.owner; + // TODO: consider parsing bazel name to generate output name. + const cmd = b.addSystemCommand(&.{ "bazel", "build", "-c", "opt", name }); + const obj = b.path(b.pathJoin(&.{ "bazel-bin", output })); + obj.addStepDependencies(&cmd.step); + module.addObjectFile(obj); +} + +fn moduleFromBazelSrcs( + b: *std.Build, + name: []const u8, + output: []const u8, + root: []const u8, + options: std.Build.Module.CreateOptions, +) *std.Build.Module { + // TODO: consider parsing bazel name to generate output name. + const bazel_cmd = b.addSystemCommand(&.{ "bazel", "build", name }); + const srcs_tar = b.path(b.pathJoin(&.{ "bazel-bin", output })); + srcs_tar.addStepDependencies(&bazel_cmd.step); + + const tar_cmd = b.addSystemCommand(&.{ "tar", "-xf" }); + tar_cmd.addFileArg(srcs_tar); + tar_cmd.addArg("-C"); + const out_dir = tar_cmd.addOutputDirectoryArg("untarred_sources"); + + var opts = options; + if (opts.root_source_file != null) @panic("moduleFromBazelSrcs is already setting the root_source_file option"); + opts.root_source_file = out_dir.path(b, root); + return b.addModule(name, opts); +} diff --git a/mlir/BUILD.bazel b/mlir/BUILD.bazel index 755d6bef..307450d9 100644 --- a/mlir/BUILD.bazel +++ b/mlir/BUILD.bazel @@ -1,4 +1,6 @@ load("@rules_cc//cc:defs.bzl", "cc_library") +load("@aspect_bazel_lib//lib:tar.bzl", "mtree_spec", "tar") + load("@rules_zig//zig:defs.bzl", "zig_library") load("//bazel:zig.bzl", "zig_cc_test") @@ -30,3 +32,26 @@ zig_cc_test( name = "test", deps = [":mlir"], ) + +cc_static_library( + name="mlir_static", + deps = ["c"] +) + +filegroup( + name = "srcs", + srcs = [":test_test_lib"], + output_group = "srcs", +) + +mtree_spec( + name = "mtree", + srcs = [":srcs"], +) + +tar( + name = "sources", + srcs = [":srcs"], + args = [], + mtree = ":mtree", +) diff --git a/mlir/mlir.zig b/mlir/mlir.zig old mode 100644 new mode 100755 index 918c3049..10d26c93 --- a/mlir/mlir.zig +++ b/mlir/mlir.zig @@ -8,6 +8,8 @@ const log = std.log.scoped(.mlir); test { std.testing.refAllDecls(@This()); + + _ = try Context.init(); } const Error = error{ From 86a8b8c36c3eac0b548a44656bd0331c487dacf0 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 12 Jun 2025 21:32:46 +0200 Subject: [PATCH 02/40] add zig_srcs helper --- bazel/zig_srcs.bzl | 23 +++++++++++++++++++++++ mlir/BUILD.bazel | 18 +++--------------- 2 files changed, 26 insertions(+), 15 deletions(-) create mode 100644 bazel/zig_srcs.bzl diff --git a/bazel/zig_srcs.bzl b/bazel/zig_srcs.bzl new file mode 100644 index 00000000..84696e10 --- /dev/null +++ b/bazel/zig_srcs.bzl @@ -0,0 +1,23 @@ +load("@aspect_bazel_lib//lib:tar.bzl", "mtree_spec", "tar") +load("@bazel_tools//tools/build_defs/repo:filegroup.bzl", "filegroup") + +def zig_srcs(name, zig_bin): + """For a given zig_binary, recursivel extract all zig sources into a tarball. + + This also includes the files translated from C headers. + """ + filegroup( + name = "{}_zig_srcs".format(name), + srcs = [zig_bin], + output_group = "srcs", + ) + mtree_spec( + name = "{}_mtree".format(name), + srcs = [":{}_zig_srcs".format(name)], + ) + tar( + name = name, + srcs = ["{}_zig_srcs".format(name)], + args = [], + mtree = "{}_mtree".format(name), + ) diff --git a/mlir/BUILD.bazel b/mlir/BUILD.bazel index 307450d9..5a2a0909 100644 --- a/mlir/BUILD.bazel +++ b/mlir/BUILD.bazel @@ -3,6 +3,7 @@ load("@aspect_bazel_lib//lib:tar.bzl", "mtree_spec", "tar") load("@rules_zig//zig:defs.bzl", "zig_library") load("//bazel:zig.bzl", "zig_cc_test") +load("//bazel:zig_srcs.bzl", "zig_srcs") cc_library( name = "c", @@ -38,20 +39,7 @@ cc_static_library( deps = ["c"] ) -filegroup( - name = "srcs", - srcs = [":test_test_lib"], - output_group = "srcs", -) - -mtree_spec( - name = "mtree", - srcs = [":srcs"], -) - -tar( +zig_srcs( name = "sources", - srcs = [":srcs"], - args = [], - mtree = ":mtree", + zig_bin = ":test_test_lib", ) From a13bdb7281501c7a2cff8fb25c6628b97a6e24ca Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 12 Jun 2025 22:10:49 +0200 Subject: [PATCH 03/40] add pjrt module --- build.zig | 30 +++++++++++++++++++++++++++--- mlir/BUILD.bazel | 18 +++++++++++++++--- pjrt/BUILD.bazel | 27 ++++++++++++++++++++++++++- 3 files changed, 68 insertions(+), 7 deletions(-) diff --git a/build.zig b/build.zig index 7949a2c4..0a95b36f 100644 --- a/build.zig +++ b/build.zig @@ -4,6 +4,9 @@ pub fn build(b: *std.Build) void { const target = b.standardTargetOptions(.{}); const optimize = b.standardOptimizeOption(.{}); + const test_step = b.step("test", "Run unit tests"); + + // MLIR const mlir_c_deps = moduleFromBazelSrcs( b, "//mlir:sources", @@ -23,11 +26,30 @@ pub fn build(b: *std.Build) void { }); const mlir_test = b.addTest(.{ .root_module = mlir_mod }); - const run_mlir_tests = b.addRunArtifact(mlir_test); - - const test_step = b.step("test", "Run unit tests"); test_step.dependOn(&run_mlir_tests.step); + + // PJRT + const pjrt_c_deps = moduleFromBazelSrcs( + b, + "//pjrt:sources", + "pjrt/sources.tar", + "pjrt/test_test_lib_c.zig", + .{ .link_libcpp = true }, + ); + + const pjrt_mod = b.addModule("pjrt", .{ + .root_source_file = b.path("pjrt/pjrt.zig"), + .target = target, + .optimize = optimize, + .imports = &.{ + .{ .name = "c", .module = pjrt_c_deps }, + }, + }); + + const pjrt_test = b.addTest(.{ .root_module = pjrt_mod }); + const run_pjrt_tests = b.addRunArtifact(pjrt_test); + test_step.dependOn(&run_pjrt_tests.step); } /// Take the name of a Bazel `cc_static_library` and add it to the given module. @@ -36,6 +58,7 @@ fn addObjectFromBazel(module: *std.Build.Module, name: []const u8, output: []con // TODO: consider parsing bazel name to generate output name. const cmd = b.addSystemCommand(&.{ "bazel", "build", "-c", "opt", name }); const obj = b.path(b.pathJoin(&.{ "bazel-bin", output })); + // TODO: fix me this isn't working ! The dep is not added. obj.addStepDependencies(&cmd.step); module.addObjectFile(obj); } @@ -50,6 +73,7 @@ fn moduleFromBazelSrcs( // TODO: consider parsing bazel name to generate output name. const bazel_cmd = b.addSystemCommand(&.{ "bazel", "build", name }); const srcs_tar = b.path(b.pathJoin(&.{ "bazel-bin", output })); + // TODO: fix me this isn't working ! The dep is not added. srcs_tar.addStepDependencies(&bazel_cmd.step); const tar_cmd = b.addSystemCommand(&.{ "tar", "-xf" }); diff --git a/mlir/BUILD.bazel b/mlir/BUILD.bazel index 5a2a0909..307450d9 100644 --- a/mlir/BUILD.bazel +++ b/mlir/BUILD.bazel @@ -3,7 +3,6 @@ load("@aspect_bazel_lib//lib:tar.bzl", "mtree_spec", "tar") load("@rules_zig//zig:defs.bzl", "zig_library") load("//bazel:zig.bzl", "zig_cc_test") -load("//bazel:zig_srcs.bzl", "zig_srcs") cc_library( name = "c", @@ -39,7 +38,20 @@ cc_static_library( deps = ["c"] ) -zig_srcs( +filegroup( + name = "srcs", + srcs = [":test_test_lib"], + output_group = "srcs", +) + +mtree_spec( + name = "mtree", + srcs = [":srcs"], +) + +tar( name = "sources", - zig_bin = ":test_test_lib", + srcs = [":srcs"], + args = [], + mtree = ":mtree", ) diff --git a/pjrt/BUILD.bazel b/pjrt/BUILD.bazel index 3ed3483a..e1955cf7 100644 --- a/pjrt/BUILD.bazel +++ b/pjrt/BUILD.bazel @@ -1,6 +1,8 @@ load("@rules_cc//cc:defs.bzl", "cc_library") load("@rules_zig//zig:defs.bzl", "zig_library") -load("@zml//bazel:zig.bzl", "zig_cc_binary") +load("@aspect_bazel_lib//lib:tar.bzl", "mtree_spec", "tar") + +load("@zml//bazel:zig.bzl", "zig_cc_binary", "zig_cc_test") load("//bazel:zig_proto_library.bzl", "zig_proto_library") cc_library( @@ -38,6 +40,29 @@ zig_library( }), ) +zig_cc_test( + name = "test", + deps = [":pjrt"], +) + +filegroup( + name = "srcs", + srcs = [":test_test_lib"], + output_group = "srcs", +) + +mtree_spec( + name = "mtree", + srcs = [":srcs"], +) + +tar( + name = "sources", + srcs = [":srcs"], + args = [], + mtree = ":mtree", +) + zig_proto_library( name = "profiler_options_proto", import_name = "//tsl:profiler_options_proto", From a6f491fac72cd01e2f673dbabfdc3b3ff5338639 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Fri, 13 Jun 2025 09:16:53 +0200 Subject: [PATCH 04/40] fix zig_srcs macro --- bazel/zig_srcs.bzl | 3 +-- mlir/BUILD.bazel | 19 +++---------------- pjrt/BUILD.bazel | 20 ++++---------------- 3 files changed, 8 insertions(+), 34 deletions(-) diff --git a/bazel/zig_srcs.bzl b/bazel/zig_srcs.bzl index 84696e10..b40bc8ac 100644 --- a/bazel/zig_srcs.bzl +++ b/bazel/zig_srcs.bzl @@ -1,12 +1,11 @@ load("@aspect_bazel_lib//lib:tar.bzl", "mtree_spec", "tar") -load("@bazel_tools//tools/build_defs/repo:filegroup.bzl", "filegroup") def zig_srcs(name, zig_bin): """For a given zig_binary, recursivel extract all zig sources into a tarball. This also includes the files translated from C headers. """ - filegroup( + native.filegroup( name = "{}_zig_srcs".format(name), srcs = [zig_bin], output_group = "srcs", diff --git a/mlir/BUILD.bazel b/mlir/BUILD.bazel index 307450d9..63b9ede6 100644 --- a/mlir/BUILD.bazel +++ b/mlir/BUILD.bazel @@ -1,7 +1,7 @@ load("@rules_cc//cc:defs.bzl", "cc_library") -load("@aspect_bazel_lib//lib:tar.bzl", "mtree_spec", "tar") load("@rules_zig//zig:defs.bzl", "zig_library") +load("//bazel:zig_srcs.bzl", "zig_srcs") load("//bazel:zig.bzl", "zig_cc_test") cc_library( @@ -38,20 +38,7 @@ cc_static_library( deps = ["c"] ) -filegroup( - name = "srcs", - srcs = [":test_test_lib"], - output_group = "srcs", -) - -mtree_spec( - name = "mtree", - srcs = [":srcs"], -) - -tar( +zig_srcs( name = "sources", - srcs = [":srcs"], - args = [], - mtree = ":mtree", + zig_bin = ":test_test_lib", ) diff --git a/pjrt/BUILD.bazel b/pjrt/BUILD.bazel index e1955cf7..bfba0dda 100644 --- a/pjrt/BUILD.bazel +++ b/pjrt/BUILD.bazel @@ -3,7 +3,8 @@ load("@rules_zig//zig:defs.bzl", "zig_library") load("@aspect_bazel_lib//lib:tar.bzl", "mtree_spec", "tar") load("@zml//bazel:zig.bzl", "zig_cc_binary", "zig_cc_test") -load("//bazel:zig_proto_library.bzl", "zig_proto_library") +load("@zml//bazel:zig_srcs.bzl", "zig_srcs") +load("@zml//bazel:zig_proto_library.bzl", "zig_proto_library") cc_library( name = "dlfcn", @@ -45,22 +46,9 @@ zig_cc_test( deps = [":pjrt"], ) -filegroup( - name = "srcs", - srcs = [":test_test_lib"], - output_group = "srcs", -) - -mtree_spec( - name = "mtree", - srcs = [":srcs"], -) - -tar( +zig_srcs( name = "sources", - srcs = [":srcs"], - args = [], - mtree = ":mtree", + zig_bin = ":test_test_lib", ) zig_proto_library( From f7622d06ca37bb230a525a05ae34a5301ddb52c0 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Fri, 13 Jun 2025 09:49:25 +0200 Subject: [PATCH 05/40] add stdx --- build.zig | 37 ++++++++++++++++++++++++++++++------- stdx/BUILD.bazel | 14 +++++++++++++- stdx/queue.zig | 4 ++-- stdx/stdx.zig | 5 +++++ 4 files changed, 50 insertions(+), 10 deletions(-) diff --git a/build.zig b/build.zig index 0a95b36f..bfe5e02e 100644 --- a/build.zig +++ b/build.zig @@ -6,9 +6,10 @@ pub fn build(b: *std.Build) void { const test_step = b.step("test", "Run unit tests"); - // MLIR + // mlir const mlir_c_deps = moduleFromBazelSrcs( b, + null, "//mlir:sources", "mlir/sources.tar", "mlir/test_test_lib_c.zig", @@ -29,9 +30,10 @@ pub fn build(b: *std.Build) void { const run_mlir_tests = b.addRunArtifact(mlir_test); test_step.dependOn(&run_mlir_tests.step); - // PJRT + // pjrt const pjrt_c_deps = moduleFromBazelSrcs( b, + null, "//pjrt:sources", "pjrt/sources.tar", "pjrt/test_test_lib_c.zig", @@ -50,6 +52,23 @@ pub fn build(b: *std.Build) void { const pjrt_test = b.addTest(.{ .root_module = pjrt_mod }); const run_pjrt_tests = b.addRunArtifact(pjrt_test); test_step.dependOn(&run_pjrt_tests.step); + + // stdx + const stdx_mod = moduleFromBazelSrcs( + b, + "stdx", + "//stdx:sources", + "stdx/sources.tar", + "stdx/stdx.zig", + .{ + .target = target, + .optimize = optimize, + }, + ); + + const stdx_test = b.addTest(.{ .root_module = stdx_mod }); + const run_stdx_tests = b.addRunArtifact(stdx_test); + test_step.dependOn(&run_stdx_tests.step); } /// Take the name of a Bazel `cc_static_library` and add it to the given module. @@ -65,14 +84,15 @@ fn addObjectFromBazel(module: *std.Build.Module, name: []const u8, output: []con fn moduleFromBazelSrcs( b: *std.Build, - name: []const u8, - output: []const u8, + module_name: ?[]const u8, + sources_target: []const u8, + sources_tar_path: []const u8, root: []const u8, options: std.Build.Module.CreateOptions, ) *std.Build.Module { // TODO: consider parsing bazel name to generate output name. - const bazel_cmd = b.addSystemCommand(&.{ "bazel", "build", name }); - const srcs_tar = b.path(b.pathJoin(&.{ "bazel-bin", output })); + const bazel_cmd = b.addSystemCommand(&.{ "bazel", "build", sources_target }); + const srcs_tar = b.path(b.pathJoin(&.{ "bazel-bin", sources_tar_path })); // TODO: fix me this isn't working ! The dep is not added. srcs_tar.addStepDependencies(&bazel_cmd.step); @@ -84,5 +104,8 @@ fn moduleFromBazelSrcs( var opts = options; if (opts.root_source_file != null) @panic("moduleFromBazelSrcs is already setting the root_source_file option"); opts.root_source_file = out_dir.path(b, root); - return b.addModule(name, opts); + return if (module_name) |name| + b.addModule(name, opts) + else + b.createModule(opts); } diff --git a/stdx/BUILD.bazel b/stdx/BUILD.bazel index f2fb4ea4..59abb65e 100644 --- a/stdx/BUILD.bazel +++ b/stdx/BUILD.bazel @@ -1,4 +1,5 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") +load("@zml//bazel:zig_srcs.bzl", "zig_srcs") zig_library( name = "stdx", @@ -17,3 +18,14 @@ zig_library( main = "stdx.zig", visibility = ["//visibility:public"], ) + +zig_test( + name = "test", + deps = [":stdx"], + testonly = False, +) + +zig_srcs( + name = "sources", + zig_bin = ":test", +) diff --git a/stdx/queue.zig b/stdx/queue.zig index e4722af1..ebbf0589 100644 --- a/stdx/queue.zig +++ b/stdx/queue.zig @@ -91,7 +91,7 @@ test SPSC { try testing.expect(q.empty()); // Elems - var elems: [10]Elem = .{.{}} ** 10; + var elems: [10]Elem = @splat(.{}); // One try testing.expect(q.pop() == null); @@ -207,7 +207,7 @@ test MPSC { q.init(); // Elems - var elems: [10]Elem = .{.{}} ** 10; + var elems: [10]Elem = @splat(.{}); // One try testing.expect(q.pop() == null); diff --git a/stdx/stdx.zig b/stdx/stdx.zig index 226c1225..cda9635d 100644 --- a/stdx/stdx.zig +++ b/stdx/stdx.zig @@ -8,6 +8,11 @@ pub const meta = @import("meta.zig"); pub const queue = @import("queue.zig"); pub const time = @import("time.zig"); +test { + const std = @import("std"); + std.testing.refAllDecls(@This()); +} + pub inline fn stackSlice(comptime max_len: usize, T: type, len: usize) []T { debug.assert(len <= max_len, "stackSlice can only create a slice of up to {} elements, got: {}", .{ max_len, len }); var storage: [max_len]T = undefined; From 2c007f69aad175f5f59b35053ede444cc6b3af3b Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Fri, 13 Jun 2025 10:14:45 +0200 Subject: [PATCH 06/40] fix dependency issues --- async/BUILD.bazel | 15 ++++++++++++++- build.zig | 32 +++++++++++++++++++++++++------- 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/async/BUILD.bazel b/async/BUILD.bazel index 9d6efffe..65e44d0e 100644 --- a/async/BUILD.bazel +++ b/async/BUILD.bazel @@ -1,4 +1,6 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") +load("@zml//bazel:zig_srcs.bzl", "zig_srcs") + zig_library( name = "async", @@ -18,3 +20,14 @@ zig_library( "@libxev//:xev", ], ) + +zig_test( + name = "test", + deps = [":async"], + testonly = False, +) + +zig_srcs( + name = "sources", + zig_bin = ":test", +) diff --git a/build.zig b/build.zig index bfe5e02e..9b68df3a 100644 --- a/build.zig +++ b/build.zig @@ -15,7 +15,7 @@ pub fn build(b: *std.Build) void { "mlir/test_test_lib_c.zig", .{ .link_libcpp = true }, ); - addObjectFromBazel(mlir_c_deps, "//mlir:static_c", "mlir/libstatic_c.a"); + addObjectFromBazel(mlir_c_deps, "//mlir:mlir_static", "mlir/libmlir_static.a"); const mlir_mod = b.addModule("mlir", .{ .root_source_file = b.path("mlir/mlir.zig"), @@ -69,6 +69,23 @@ pub fn build(b: *std.Build) void { const stdx_test = b.addTest(.{ .root_module = stdx_mod }); const run_stdx_tests = b.addRunArtifact(stdx_test); test_step.dependOn(&run_stdx_tests.step); + + // async + const async_mod = moduleFromBazelSrcs( + b, + "async", + "//async:sources", + "async/sources.tar", + "async/async.zig", + .{ + .target = target, + .optimize = optimize, + }, + ); + + const async_test = b.addTest(.{ .root_module = async_mod }); + const run_async_tests = b.addRunArtifact(async_test); + test_step.dependOn(&run_async_tests.step); } /// Take the name of a Bazel `cc_static_library` and add it to the given module. @@ -76,10 +93,12 @@ fn addObjectFromBazel(module: *std.Build.Module, name: []const u8, output: []con const b = module.owner; // TODO: consider parsing bazel name to generate output name. const cmd = b.addSystemCommand(&.{ "bazel", "build", "-c", "opt", name }); - const obj = b.path(b.pathJoin(&.{ "bazel-bin", output })); - // TODO: fix me this isn't working ! The dep is not added. - obj.addStepDependencies(&cmd.step); - module.addObjectFile(obj); + const obj_path = b.pathJoin(&.{ "bazel-bin", output }); + const generated_file = b.allocator.create(std.Build.GeneratedFile) catch @panic("OOM"); + generated_file.* = .{ .step = &cmd.step, .path = obj_path }; + const obj: std.Build.LazyPath = .{ .generated = .{ .file = generated_file } }; + + module.link_objects.append(b.allocator, .{ .static_path = obj }) catch @panic("OOM"); } fn moduleFromBazelSrcs( @@ -93,10 +112,9 @@ fn moduleFromBazelSrcs( // TODO: consider parsing bazel name to generate output name. const bazel_cmd = b.addSystemCommand(&.{ "bazel", "build", sources_target }); const srcs_tar = b.path(b.pathJoin(&.{ "bazel-bin", sources_tar_path })); - // TODO: fix me this isn't working ! The dep is not added. - srcs_tar.addStepDependencies(&bazel_cmd.step); const tar_cmd = b.addSystemCommand(&.{ "tar", "-xf" }); + tar_cmd.step.dependOn(&bazel_cmd.step); tar_cmd.addFileArg(srcs_tar); tar_cmd.addArg("-C"); const out_dir = tar_cmd.addOutputDirectoryArg("untarred_sources"); From 017ec8d7a7438cec17b833a7f25acef205a51b92 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Fri, 13 Jun 2025 10:57:57 +0200 Subject: [PATCH 07/40] add libxev, fix async tests --- async/async.zig | 12 +++++++++--- async/coro.zig | 27 +++------------------------ build.zig | 46 +++++++++++++++++++++++++++++++++++----------- 3 files changed, 47 insertions(+), 38 deletions(-) diff --git a/async/async.zig b/async/async.zig index 5cee9947..93f69927 100644 --- a/async/async.zig +++ b/async/async.zig @@ -1,13 +1,19 @@ const std = @import("std"); + const stdx = @import("stdx"); const xev = @import("xev").Dynamic; +const XevThreadPool = @import("xev").ThreadPool; + +const aio = @import("asyncio.zig"); +const channel_mod = @import("channel.zig"); const coro = @import("coro.zig"); const executor = @import("executor.zig"); -const channel_mod = @import("channel.zig"); -const aio = @import("asyncio.zig"); const stack = @import("stack.zig"); -const XevThreadPool = @import("xev").ThreadPool; +test { + std.testing.refAllDecls(@This()); + std.testing.refAllDecls(coro); +} pub const Condition = struct { inner: executor.Condition, diff --git a/async/coro.zig b/async/coro.zig index cfad9c82..be26b95b 100644 --- a/async/coro.zig +++ b/async/coro.zig @@ -265,29 +265,6 @@ const CoroT = struct { } }; -/// Estimates the remaining stack size in the currently running coroutine -pub noinline fn remainingStackSize() usize { - var dummy: usize = 0; - dummy += 1; - const addr = @intFromPtr(&dummy); - - // Check if the stack was already overflowed - const current = xframe(); - StackOverflow.check(current) catch return 0; - - // Check if the stack is currently overflowed - const bottom = @intFromPtr(current.stack.ptr); - if (addr < bottom) { - return 0; - } - - // Debug check that we're actually in the stack - const top = @intFromPtr(current.stack.ptr + current.stack.len); - std.debug.assert(addr < top); // should never have popped beyond the top - - return addr - bottom; -} - // ============================================================================ /// Thread-local coroutine runtime @@ -450,7 +427,9 @@ fn testSetIdx(val: usize) void { } fn testFn() void { - std.debug.assert(remainingStackSize() > 2048); + // Check if the stack was already overflowed + const current = xframe(); + std.debug.assert(current.stack.remaining().len > 2048); testSetIdx(2); xsuspend(); testSetIdx(4); diff --git a/build.zig b/build.zig index 9b68df3a..6840920c 100644 --- a/build.zig +++ b/build.zig @@ -17,7 +17,7 @@ pub fn build(b: *std.Build) void { ); addObjectFromBazel(mlir_c_deps, "//mlir:mlir_static", "mlir/libmlir_static.a"); - const mlir_mod = b.addModule("mlir", .{ + const mlir = b.addModule("mlir", .{ .root_source_file = b.path("mlir/mlir.zig"), .target = target, .optimize = optimize, @@ -26,7 +26,7 @@ pub fn build(b: *std.Build) void { }, }); - const mlir_test = b.addTest(.{ .root_module = mlir_mod }); + const mlir_test = b.addTest(.{ .root_module = mlir }); const run_mlir_tests = b.addRunArtifact(mlir_test); test_step.dependOn(&run_mlir_tests.step); @@ -40,7 +40,7 @@ pub fn build(b: *std.Build) void { .{ .link_libcpp = true }, ); - const pjrt_mod = b.addModule("pjrt", .{ + const pjrt = b.addModule("pjrt", .{ .root_source_file = b.path("pjrt/pjrt.zig"), .target = target, .optimize = optimize, @@ -49,12 +49,12 @@ pub fn build(b: *std.Build) void { }, }); - const pjrt_test = b.addTest(.{ .root_module = pjrt_mod }); + const pjrt_test = b.addTest(.{ .root_module = pjrt }); const run_pjrt_tests = b.addRunArtifact(pjrt_test); test_step.dependOn(&run_pjrt_tests.step); // stdx - const stdx_mod = moduleFromBazelSrcs( + const stdx = moduleFromBazelSrcs( b, "stdx", "//stdx:sources", @@ -66,10 +66,27 @@ pub fn build(b: *std.Build) void { }, ); - const stdx_test = b.addTest(.{ .root_module = stdx_mod }); + const stdx_test = b.addTest(.{ .root_module = stdx }); const run_stdx_tests = b.addRunArtifact(stdx_test); test_step.dependOn(&run_stdx_tests.step); + // xev + const xev = moduleFromBazelSrcs( + b, + "xev", + "//async:sources", + "async/sources.tar", + "src/main.zig", + .{ + .target = target, + .optimize = optimize, + }, + ); + + const xev_test = b.addTest(.{ .root_module = xev }); + const run_xev_tests = b.addRunArtifact(xev_test); + test_step.dependOn(&run_xev_tests.step); + // async const async_mod = moduleFromBazelSrcs( b, @@ -80,6 +97,10 @@ pub fn build(b: *std.Build) void { .{ .target = target, .optimize = optimize, + .imports = &.{ + .{ .name = "xev", .module = xev }, + .{ .name = "stdx", .module = stdx }, + }, }, ); @@ -92,13 +113,16 @@ pub fn build(b: *std.Build) void { fn addObjectFromBazel(module: *std.Build.Module, name: []const u8, output: []const u8) void { const b = module.owner; // TODO: consider parsing bazel name to generate output name. - const cmd = b.addSystemCommand(&.{ "bazel", "build", "-c", "opt", name }); + const bazel_cmd = b.addSystemCommand(&.{ "bazel", "build", "-c", "opt", name }); const obj_path = b.pathJoin(&.{ "bazel-bin", output }); - const generated_file = b.allocator.create(std.Build.GeneratedFile) catch @panic("OOM"); - generated_file.* = .{ .step = &cmd.step, .path = obj_path }; - const obj: std.Build.LazyPath = .{ .generated = .{ .file = generated_file } }; - module.link_objects.append(b.allocator, .{ .static_path = obj }) catch @panic("OOM"); + // Copy bazel output into zig-cache, cause bazel may remove the file later. + const cp = b.addWriteFiles(); + cp.step.dependOn(&bazel_cmd.step); + const obj = cp.addCopyFile(b.path(obj_path), output); + + // Module depends on the copied object. + module.addObjectFile(obj); } fn moduleFromBazelSrcs( From 6d514b379faccfb6ca3b29d4bdc3b5107f6f97cd Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Fri, 13 Jun 2025 11:26:50 +0200 Subject: [PATCH 08/40] simplify a bit build.zig --- build.zig | 90 ++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 59 insertions(+), 31 deletions(-) diff --git a/build.zig b/build.zig index 6840920c..e86e8b93 100644 --- a/build.zig +++ b/build.zig @@ -1,5 +1,24 @@ const std = @import("std"); +/// !!! This build.zig is work in progress. !!! +/// +/// It shows how to bridge Bazel and build.zig. +/// It requires the user to have `bazel` and `tar` installed on their machine. +/// +/// Bazel is used to: +/// +/// * compile C and C++ deps into .a files. +/// * call zig-translate C +/// * generating Zig files +/// * tarring Zig sources +/// +/// build.zig finishes the work by: +/// * untarring Zig sources +/// * copying .a files into zig-cache +/// * creating "zig modules" visible to other build.zig. +/// +/// `zig build test --summary all` will run tests for several ZML deps, +/// but not yet ZML itself. pub fn build(b: *std.Build) void { const target = b.standardTargetOptions(.{}); const optimize = b.standardOptimizeOption(.{}); @@ -10,9 +29,7 @@ pub fn build(b: *std.Build) void { const mlir_c_deps = moduleFromBazelSrcs( b, null, - "//mlir:sources", - "mlir/sources.tar", - "mlir/test_test_lib_c.zig", + .canonical(b.allocator, "mlir", "test_test_lib_c.zig"), .{ .link_libcpp = true }, ); addObjectFromBazel(mlir_c_deps, "//mlir:mlir_static", "mlir/libmlir_static.a"); @@ -34,9 +51,7 @@ pub fn build(b: *std.Build) void { const pjrt_c_deps = moduleFromBazelSrcs( b, null, - "//pjrt:sources", - "pjrt/sources.tar", - "pjrt/test_test_lib_c.zig", + .canonical(b.allocator, "pjrt", "test_test_lib_c.zig"), .{ .link_libcpp = true }, ); @@ -54,17 +69,11 @@ pub fn build(b: *std.Build) void { test_step.dependOn(&run_pjrt_tests.step); // stdx - const stdx = moduleFromBazelSrcs( - b, - "stdx", - "//stdx:sources", - "stdx/sources.tar", - "stdx/stdx.zig", - .{ - .target = target, - .optimize = optimize, - }, - ); + const stdx = b.addModule("stdx", .{ + .root_source_file = b.path("stdx/stdx.zig"), + .target = target, + .optimize = optimize, + }); const stdx_test = b.addTest(.{ .root_module = stdx }); const run_stdx_tests = b.addRunArtifact(stdx_test); @@ -74,9 +83,13 @@ pub fn build(b: *std.Build) void { const xev = moduleFromBazelSrcs( b, "xev", - "//async:sources", - "async/sources.tar", - "src/main.zig", + .{ + // xev sources can be find inside async sources cause it's a dependency. + .target = "//async:sources", + .tar_path = "async/sources.tar", + .directory = "src", + .root = "main.zig", + }, .{ .target = target, .optimize = optimize, @@ -91,9 +104,7 @@ pub fn build(b: *std.Build) void { const async_mod = moduleFromBazelSrcs( b, "async", - "//async:sources", - "async/sources.tar", - "async/async.zig", + .canonical(b.allocator, "async", "async.zig"), .{ .target = target, .optimize = optimize, @@ -125,27 +136,44 @@ fn addObjectFromBazel(module: *std.Build.Module, name: []const u8, output: []con module.addObjectFile(obj); } +const BazelSrcs = struct { + target: []const u8, + tar_path: []const u8, + directory: []const u8, + root: []const u8, + + pub fn canonical(allocator: std.mem.Allocator, name: []const u8, root: []const u8) BazelSrcs { + return .{ + .target = std.mem.concat(allocator, u8, &.{ "//", name, ":sources" }) catch @panic("OOM"), + .tar_path = std.fs.path.join(allocator, &.{ name, "sources.tar" }) catch @panic("OOM"), + .directory = name, + .root = root, + }; + } +}; + +/// Ask bazel for the full sources of a zig module. +/// This is needed for module that have generated zig sources, +/// like the output of zig translate-c or protobuf generated sources. fn moduleFromBazelSrcs( b: *std.Build, module_name: ?[]const u8, - sources_target: []const u8, - sources_tar_path: []const u8, - root: []const u8, + srcs: BazelSrcs, options: std.Build.Module.CreateOptions, ) *std.Build.Module { - // TODO: consider parsing bazel name to generate output name. - const bazel_cmd = b.addSystemCommand(&.{ "bazel", "build", sources_target }); - const srcs_tar = b.path(b.pathJoin(&.{ "bazel-bin", sources_tar_path })); + const bazel_cmd = b.addSystemCommand(&.{ "bazel", "build", srcs.target }); + const srcs_tar = b.path(b.pathJoin(&.{ "bazel-bin", srcs.tar_path })); const tar_cmd = b.addSystemCommand(&.{ "tar", "-xf" }); tar_cmd.step.dependOn(&bazel_cmd.step); tar_cmd.addFileArg(srcs_tar); tar_cmd.addArg("-C"); - const out_dir = tar_cmd.addOutputDirectoryArg("untarred_sources"); + const out_dir = tar_cmd.addOutputDirectoryArg("sources"); + tar_cmd.addArg(srcs.directory); var opts = options; if (opts.root_source_file != null) @panic("moduleFromBazelSrcs is already setting the root_source_file option"); - opts.root_source_file = out_dir.path(b, root); + opts.root_source_file = out_dir.path(b, b.pathJoin(&.{ srcs.directory, srcs.root })); return if (module_name) |name| b.addModule(name, opts) else From abc078a39df3f548dc69dd1007f7e2c81cd9b7d2 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Fri, 13 Jun 2025 11:37:42 +0200 Subject: [PATCH 09/40] ffi --- build.zig | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/build.zig b/build.zig index e86e8b93..c6500703 100644 --- a/build.zig +++ b/build.zig @@ -118,6 +118,18 @@ pub fn build(b: *std.Build) void { const async_test = b.addTest(.{ .root_module = async_mod }); const run_async_tests = b.addRunArtifact(async_test); test_step.dependOn(&run_async_tests.step); + + // ffi + const ffi = moduleFromBazelSrcs( + b, + "ffi", + .fromZml("ffi", "ffi.zig"), + .{ .target = target, .optimize = optimize }, + ); + + const ffi_test = b.addTest(.{ .root_module = ffi }); + const run_ffi_tests = b.addRunArtifact(ffi_test); + test_step.dependOn(&run_ffi_tests.step); } /// Take the name of a Bazel `cc_static_library` and add it to the given module. @@ -150,6 +162,15 @@ const BazelSrcs = struct { .root = root, }; } + + pub fn fromZml(name: []const u8, root: []const u8) BazelSrcs { + return .{ + .target = "//zml:sources", + .tar_path = "zml/sources.tar", + .directory = name, + .root = root, + }; + } }; /// Ask bazel for the full sources of a zig module. From 3fe81ce4e3fd51c06bb14c93a13ddc485c354ee9 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Fri, 13 Jun 2025 12:02:24 +0200 Subject: [PATCH 10/40] zml/tokenizers --- build.zig | 66 +++++++++++++++++++++++-- zml/tokenizer/hftokenizers/BUILD.bazel | 8 +++ zml/tokenizer/sentencepiece/BUILD.bazel | 12 +++++ zml/tokenizer/tokenizer.zig | 10 +++- 4 files changed, 91 insertions(+), 5 deletions(-) diff --git a/build.zig b/build.zig index c6500703..5738a282 100644 --- a/build.zig +++ b/build.zig @@ -32,7 +32,6 @@ pub fn build(b: *std.Build) void { .canonical(b.allocator, "mlir", "test_test_lib_c.zig"), .{ .link_libcpp = true }, ); - addObjectFromBazel(mlir_c_deps, "//mlir:mlir_static", "mlir/libmlir_static.a"); const mlir = b.addModule("mlir", .{ .root_source_file = b.path("mlir/mlir.zig"), @@ -44,6 +43,7 @@ pub fn build(b: *std.Build) void { }); const mlir_test = b.addTest(.{ .root_module = mlir }); + addObjectFromBazel(mlir_test, "//mlir:mlir_static", "mlir/libmlir_static.a"); const run_mlir_tests = b.addRunArtifact(mlir_test); test_step.dependOn(&run_mlir_tests.step); @@ -130,11 +130,69 @@ pub fn build(b: *std.Build) void { const ffi_test = b.addTest(.{ .root_module = ffi }); const run_ffi_tests = b.addRunArtifact(ffi_test); test_step.dependOn(&run_ffi_tests.step); + + // hftokenizers + const hftokenizers = moduleFromBazelSrcs( + b, + "//zml/tokenizer/hftokenizers", + .fromZml("zml/tokenizer/hftokenizers", "hftokenizers.zig"), + .{ .target = target, .optimize = optimize }, + ); + + // sentencepiece + const sentencepiece_c_deps = moduleFromBazelSrcs( + b, + null, + .canonical(b.allocator, "zml/tokenizer/sentencepiece", "test_test_lib_c.zig"), + .{ .link_libcpp = true }, + ); + const sentencepiece = moduleFromBazelSrcs( + b, + "//zml/tokenizer/sentencepiece", + .fromZml("zml/tokenizer/sentencepiece", "sentencepiece.zig"), + .{ + .target = target, + .optimize = optimize, + .imports = &.{ + .{ .name = "c", .module = sentencepiece_c_deps }, + }, + }, + ); + + // tokenizer + const tokenizer = moduleFromBazelSrcs( + b, + "tokenizer", + .fromZml("zml/tokenizer", "tokenizer.zig"), + .{ + .target = target, + .optimize = optimize, + .imports = &.{ + .{ .name = "async", .module = async_mod }, + .{ .name = "ffi", .module = ffi }, + // Note: even though we import sentencepiece and hftokenizers, + // we don't link the corresponding .a cause there is no tests + // for them directly. + .{ .name = "hftokenizers", .module = hftokenizers }, + .{ .name = "sentencepiece", .module = sentencepiece }, + }, + }, + ); + + const tokenizer_test = b.addTest(.{ .root_module = tokenizer }); + const run_tokenizer_tests = b.addRunArtifact(tokenizer_test); + test_step.dependOn(&run_tokenizer_tests.step); } /// Take the name of a Bazel `cc_static_library` and add it to the given module. -fn addObjectFromBazel(module: *std.Build.Module, name: []const u8, output: []const u8) void { - const b = module.owner; +fn addObjectFromBazel(entry: *std.Build.Step.Compile, name: []const u8, output: []const u8) void { + // TODO: I'm not sure what's the best idea: add the object to the compile step or directly to the module. + const b = switch (@TypeOf(entry)) { + *std.Build.Module => entry.owner, + *std.Build.Step.Compile => entry.step.owner, + else => @compileError("addObjectFromBazel takes a Module or a Compile Step"), + }; + // TODO: consider parsing bazel name to generate output name. const bazel_cmd = b.addSystemCommand(&.{ "bazel", "build", "-c", "opt", name }); const obj_path = b.pathJoin(&.{ "bazel-bin", output }); @@ -145,7 +203,7 @@ fn addObjectFromBazel(module: *std.Build.Module, name: []const u8, output: []con const obj = cp.addCopyFile(b.path(obj_path), output); // Module depends on the copied object. - module.addObjectFile(obj); + entry.addObjectFile(obj); } const BazelSrcs = struct { diff --git a/zml/tokenizer/hftokenizers/BUILD.bazel b/zml/tokenizer/hftokenizers/BUILD.bazel index d2ac5ac4..68188e38 100644 --- a/zml/tokenizer/hftokenizers/BUILD.bazel +++ b/zml/tokenizer/hftokenizers/BUILD.bazel @@ -28,3 +28,11 @@ zig_library( "//ffi:zig", ], ) + +cc_static_library( + name="hftokenizer_static", + deps = [ + ":hftokenizers_rs", + "//ffi:cc", + ] +) diff --git a/zml/tokenizer/sentencepiece/BUILD.bazel b/zml/tokenizer/sentencepiece/BUILD.bazel index 354b289a..567f76be 100644 --- a/zml/tokenizer/sentencepiece/BUILD.bazel +++ b/zml/tokenizer/sentencepiece/BUILD.bazel @@ -1,5 +1,7 @@ load("@rules_zig//zig:defs.bzl", "zig_library") load("//bazel:swig.bzl", "swig_cc_library") +load("//bazel:zig_srcs.bzl", "zig_srcs") +load("//bazel:zig.bzl", "zig_cc_test") swig_cc_library( name = "sentencepiece_swig", @@ -21,3 +23,13 @@ zig_library( "//ffi:zig", ], ) + +zig_cc_test( + name = "test", + deps = [":sentencepiece"], +) + +zig_srcs( + name = "sources", + zig_bin = ":test_test_lib", +) diff --git a/zml/tokenizer/tokenizer.zig b/zml/tokenizer/tokenizer.zig index a28b3a06..443e2fb8 100644 --- a/zml/tokenizer/tokenizer.zig +++ b/zml/tokenizer/tokenizer.zig @@ -1,10 +1,18 @@ const std = @import("std"); + +const asynk = @import("async"); const hftokenizers = @import("hftokenizers"); const sentencepiece = @import("sentencepiece"); -const asynk = @import("async"); const homemade = @import("homemade.zig"); +test { + std.testing.refAllDecls(@This()); + std.testing.refAllDecls(hftokenizers); + std.testing.refAllDecls(sentencepiece); + std.testing.refAllDecls(homemade); +} + const Tokenizers = enum { hftokenizers, sentencepiece, From 3715f6b3ea2ceebceed95a7355c481acf6623434 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Fri, 13 Jun 2025 12:22:47 +0200 Subject: [PATCH 11/40] mlir_dialects --- build.zig | 75 ++++++++++++++++++++++++--------------- mlir/dialects/BUILD.bazel | 14 ++++++++ 2 files changed, 61 insertions(+), 28 deletions(-) diff --git a/build.zig b/build.zig index 5738a282..88d57d32 100644 --- a/build.zig +++ b/build.zig @@ -25,13 +25,25 @@ pub fn build(b: *std.Build) void { const test_step = b.step("test", "Run unit tests"); + // stdx + const stdx = b.addModule("stdx", .{ + .root_source_file = b.path("stdx/stdx.zig"), + .target = target, + .optimize = optimize, + }); + + const stdx_test = b.addTest(.{ .root_module = stdx }); + const run_stdx_tests = b.addRunArtifact(stdx_test); + test_step.dependOn(&run_stdx_tests.step); + // mlir const mlir_c_deps = moduleFromBazelSrcs( b, null, - .canonical(b.allocator, "mlir", "test_test_lib_c.zig"), + .canonical(b.allocator, "mlir/dialects", "test_test_lib_c.zig"), .{ .link_libcpp = true }, ); + const mlir_obj = objectFromBazel(b, "//mlir/dialects:mlir_static", "mlir/dialects/libmlir_static.a"); const mlir = b.addModule("mlir", .{ .root_source_file = b.path("mlir/mlir.zig"), @@ -43,10 +55,37 @@ pub fn build(b: *std.Build) void { }); const mlir_test = b.addTest(.{ .root_module = mlir }); - addObjectFromBazel(mlir_test, "//mlir:mlir_static", "mlir/libmlir_static.a"); + // TODO: I'm not sure what's the best idea: add the object to the compile step or directly to the module. + mlir_test.addObjectFile(mlir_obj); const run_mlir_tests = b.addRunArtifact(mlir_test); test_step.dependOn(&run_mlir_tests.step); + const stablehlo = b.addModule("mlir/dialects/stablehlo", .{ + .root_source_file = b.path("mlir/dialects/stablehlo.zig"), + .target = target, + .optimize = optimize, + .imports = &.{ + .{ .name = "c", .module = mlir_c_deps }, + .{ .name = "mlir", .module = mlir }, + .{ .name = "stdx", .module = stdx }, + }, + }); + + const mlir_dialects = b.addModule("mlir/dialects", .{ + .root_source_file = b.path("mlir/dialects/dialects.zig"), + .target = target, + .optimize = optimize, + .imports = &.{ + .{ .name = "mlir", .module = mlir }, + .{ .name = "mlir/dialects/stablehlo", .module = stablehlo }, + }, + }); + + const mlir_dialects_test = b.addTest(.{ .root_module = mlir_dialects }); + mlir_dialects_test.addObjectFile(mlir_obj); + const run_mlir_dialects_tests = b.addRunArtifact(mlir_dialects_test); + test_step.dependOn(&run_mlir_dialects_tests.step); + // pjrt const pjrt_c_deps = moduleFromBazelSrcs( b, @@ -68,17 +107,6 @@ pub fn build(b: *std.Build) void { const run_pjrt_tests = b.addRunArtifact(pjrt_test); test_step.dependOn(&run_pjrt_tests.step); - // stdx - const stdx = b.addModule("stdx", .{ - .root_source_file = b.path("stdx/stdx.zig"), - .target = target, - .optimize = optimize, - }); - - const stdx_test = b.addTest(.{ .root_module = stdx }); - const run_stdx_tests = b.addRunArtifact(stdx_test); - test_step.dependOn(&run_stdx_tests.step); - // xev const xev = moduleFromBazelSrcs( b, @@ -184,26 +212,17 @@ pub fn build(b: *std.Build) void { test_step.dependOn(&run_tokenizer_tests.step); } -/// Take the name of a Bazel `cc_static_library` and add it to the given module. -fn addObjectFromBazel(entry: *std.Build.Step.Compile, name: []const u8, output: []const u8) void { - // TODO: I'm not sure what's the best idea: add the object to the compile step or directly to the module. - const b = switch (@TypeOf(entry)) { - *std.Build.Module => entry.owner, - *std.Build.Step.Compile => entry.step.owner, - else => @compileError("addObjectFromBazel takes a Module or a Compile Step"), - }; - - // TODO: consider parsing bazel name to generate output name. - const bazel_cmd = b.addSystemCommand(&.{ "bazel", "build", "-c", "opt", name }); +/// Take the name of a Bazel `cc_static_library` and create a object LazyPath from it. +/// The object need to be added to a Compile step with `step.addObjectFile(obj)`. +fn objectFromBazel(b: *std.Build, target: []const u8, output: []const u8) std.Build.LazyPath { + // TODO: consider parsing bazel target to generate output path. + const bazel_cmd = b.addSystemCommand(&.{ "bazel", "build", "-c", "opt", target }); const obj_path = b.pathJoin(&.{ "bazel-bin", output }); // Copy bazel output into zig-cache, cause bazel may remove the file later. const cp = b.addWriteFiles(); cp.step.dependOn(&bazel_cmd.step); - const obj = cp.addCopyFile(b.path(obj_path), output); - - // Module depends on the copied object. - entry.addObjectFile(obj); + return cp.addCopyFile(b.path(obj_path), output); } const BazelSrcs = struct { diff --git a/mlir/dialects/BUILD.bazel b/mlir/dialects/BUILD.bazel index c4c82209..bcb3a1bb 100644 --- a/mlir/dialects/BUILD.bazel +++ b/mlir/dialects/BUILD.bazel @@ -1,5 +1,6 @@ load("@rules_zig//zig:defs.bzl", "zig_library") load("//bazel:zig.bzl", "zig_cc_test") +load("//bazel:zig_srcs.bzl", "zig_srcs") zig_library( name = "dialects", @@ -24,6 +25,19 @@ zig_cc_test( deps = [":dialects"], ) +zig_srcs( + name = "sources", + zig_bin = ":test_test_lib", +) + +cc_static_library( + name="mlir_static", + deps = [ + "//mlir:c", + "@stablehlo//:stablehlo_dialect_capi", + ] +) + zig_library( name = "stablehlo", import_name = "mlir/dialects/stablehlo", From 420b7cc3eeecc15163a9b581b9041ea66688639d Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Fri, 13 Jun 2025 12:41:55 +0200 Subject: [PATCH 12/40] drop yaml support --- zml/BUILD.bazel | 23 ++++------------------- zml/aio.zig | 8 ++++---- 2 files changed, 8 insertions(+), 23 deletions(-) diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index b346b043..56f08908 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -1,8 +1,8 @@ -load("@aspect_bazel_lib//lib:tar.bzl", "mtree_spec", "tar") load("@rules_cc//cc:defs.bzl", "cc_library") -load("@rules_zig//zig:defs.bzl", "zig_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") load("//bazel:zig.bzl", "zig_cc_test") load("//bazel:zig_proto_library.bzl", "zig_proto_library") +load("//bazel:zig_srcs.bzl", "zig_srcs") cc_library( name = "posix", @@ -37,7 +37,6 @@ zig_library( "//zml/tools", "@rules_zig//zig/lib:libc", "@rules_zig//zig/runfiles", - "@zig-yaml//:zig-yaml", ], ) @@ -71,21 +70,7 @@ filegroup( visibility = ["//visibility:public"], ) -filegroup( - name = "srcs", - srcs = [":test_test_lib"], - output_group = "srcs", -) - -mtree_spec( - name = "mtree", - srcs = [":srcs"], -) - -tar( +zig_srcs( name = "sources", - srcs = [":srcs"], - args = [ - ], - mtree = ":mtree", + zig_bin = ":test_test_lib", ) diff --git a/zml/aio.zig b/zml/aio.zig index 4dc205c0..7d81f52a 100644 --- a/zml/aio.zig +++ b/zml/aio.zig @@ -5,11 +5,11 @@ const c = @import("c"); const stdx = @import("stdx"); pub const gguf = @import("aio/gguf.zig"); -pub const nemo = @import("aio/nemo.zig"); +// pub const nemo = @import("aio/nemo.zig"); pub const safetensors = @import("aio/safetensors.zig"); pub const tinyllama = @import("aio/tinyllama.zig"); pub const torch = @import("aio/torch.zig"); -pub const yaml = @import("aio/yaml.zig"); +// pub const yaml = @import("aio/yaml.zig"); const HostBuffer = @import("hostbuffer.zig").HostBuffer; const posix = @import("posix.zig"); const zml = @import("zml.zig"); @@ -18,10 +18,10 @@ pub const log = std.log.scoped(.@"zml/aio"); test { std.testing.refAllDecls(@This()); std.testing.refAllDecls(gguf); - std.testing.refAllDecls(nemo); + // std.testing.refAllDecls(nemo); std.testing.refAllDecls(safetensors); std.testing.refAllDecls(torch); - std.testing.refAllDecls(yaml); + // std.testing.refAllDecls(yaml); } // TODO error set for weight loading From 5d7ebc0644ced7ee02202bb7c82fd0a72bf44503 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Fri, 13 Jun 2025 12:43:34 +0200 Subject: [PATCH 13/40] drop libc dep on zml --- zml/BUILD.bazel | 1 - 1 file changed, 1 deletion(-) diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 56f08908..1e47ff1a 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -35,7 +35,6 @@ zig_library( "//stdx", "//zml/tokenizer", "//zml/tools", - "@rules_zig//zig/lib:libc", "@rules_zig//zig/runfiles", ], ) From 910015f1c62e4422f0ecb55ebb1fcab2096ca1a9 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Fri, 13 Jun 2025 13:10:37 +0200 Subject: [PATCH 14/40] add runtimes (WIP) --- build.zig | 90 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/build.zig b/build.zig index 88d57d32..704cc8c9 100644 --- a/build.zig +++ b/build.zig @@ -51,6 +51,7 @@ pub fn build(b: *std.Build) void { .optimize = optimize, .imports = &.{ .{ .name = "c", .module = mlir_c_deps }, + .{ .name = "stdx", .module = stdx }, }, }); @@ -210,6 +211,95 @@ pub fn build(b: *std.Build) void { const tokenizer_test = b.addTest(.{ .root_module = tokenizer }); const run_tokenizer_tests = b.addRunArtifact(tokenizer_test); test_step.dependOn(&run_tokenizer_tests.step); + + // zml c deps + const zml_c_deps = moduleFromBazelSrcs( + b, + null, + .fromZml("zml", "test_test_lib_c.zig"), + .{ .link_libcpp = true }, + ); + + // runfiles + const runfiles = moduleFromBazelSrcs( + b, + "runfiles", + .fromZml("zig/runfiles", "runfiles.zig"), + .{}, + ); + + // runtimes + const runtimes = moduleFromBazelSrcs( + b, + "runtimes", + .fromZml("runtimes", "runtimes.zig"), + .{ + .imports = &.{ + .{ .name = "c", .module = zml_c_deps }, + .{ .name = "pjrt", .module = pjrt }, + }, + }, + ); + const PLATFORMS = [_][]const u8{ "cpu", "cuda", "rocm", "tpu", "neuron" }; + + for (&PLATFORMS) |platform| { + const runtime_path = b.pathJoin(&.{ "runtimes", platform }); + const zig_file = std.mem.concat(b.allocator, u8, &.{ platform, ".zig" }) catch @panic("OOM"); + const platform_runtime = moduleFromBazelSrcs( + b, + runtime_path, + .fromZml(runtime_path, zig_file), + .{ + .imports = &.{ + .{ .name = "c", .module = zml_c_deps }, + .{ .name = "runfiles", .module = runfiles }, + .{ .name = "pjrt", .module = pjrt }, + .{ .name = "async", .module = async_mod }, + }, + }, + ); + const import_name = std.mem.concat(b.allocator, u8, &.{ "runtimes/", platform }) catch @panic("OOM"); + runtimes.addImport(import_name, platform_runtime); + } + + // zml/tools + const zml_tools = moduleFromBazelSrcs( + b, + "zml/tools", + .fromZml("zml/tools", "tools.zig"), + .{ + .link_libcpp = target.query.os_tag == .macos, + .imports = &.{.{ .name = "c", .module = zml_c_deps }}, + }, + ); + + // zml + const zml = moduleFromBazelSrcs( + b, + "zml", + .fromZml("zml", "zml.zig"), + .{ + .target = target, + .optimize = optimize, + .link_libcpp = true, + .imports = &.{ + .{ .name = "c", .module = zml_c_deps }, + .{ .name = "async", .module = async_mod }, + .{ .name = "mlir", .module = mlir }, + .{ .name = "mlir/dialects", .module = mlir_dialects }, + .{ .name = "pjrt", .module = pjrt }, + .{ .name = "runtimes", .module = runtimes }, + .{ .name = "stdx", .module = stdx }, + .{ .name = "zml/tokenizer", .module = tokenizer }, + .{ .name = "zml/tools", .module = zml_tools }, + .{ .name = "runfiles", .module = runfiles }, + }, + }, + ); + const zml_test = b.addTest(.{ .root_module = zml }); + const run_zml_tests = b.addRunArtifact(zml_test); + const zml_test_step = b.step("test-zml", "Run ZML tests (broken)"); + zml_test_step.dependOn(&run_zml_tests.step); } /// Take the name of a Bazel `cc_static_library` and create a object LazyPath from it. From 2ec2e0dbf0a25a08014f3f1be69390e00b0d74a5 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Fri, 13 Jun 2025 14:05:19 +0200 Subject: [PATCH 15/40] remove unused proto --- zml/BUILD.bazel | 6 ------ 1 file changed, 6 deletions(-) diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 1e47ff1a..f7c78923 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -25,7 +25,6 @@ zig_library( visibility = ["//visibility:public"], deps = [ ":posix", - ":sentencepiece_model_proto", ":xla_proto", "//async", "//mlir", @@ -45,11 +44,6 @@ zig_proto_library( deps = ["@xla//xla/pjrt/proto:compile_options_proto"], ) -zig_proto_library( - name = "sentencepiece_model_proto", - import_name = "//sentencepiece:model_proto", - deps = ["@sentencepiece//:sentencepiece_model_proto"], -) # All ZML Tests From 5097d463dfe242ebaa35e64dcb4a7e187f543b87 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Fri, 13 Jun 2025 14:05:43 +0200 Subject: [PATCH 16/40] xla_compile_proto --- build.zig | 58 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/build.zig b/build.zig index 704cc8c9..31712503 100644 --- a/build.zig +++ b/build.zig @@ -101,6 +101,7 @@ pub fn build(b: *std.Build) void { .optimize = optimize, .imports = &.{ .{ .name = "c", .module = pjrt_c_deps }, + .{ .name = "stdx", .module = stdx }, }, }); @@ -212,6 +213,50 @@ pub fn build(b: *std.Build) void { const run_tokenizer_tests = b.addRunArtifact(tokenizer_test); test_step.dependOn(&run_tokenizer_tests.step); + // proto + const protobuf = moduleFromBazelSrcs(b, "protobuf", .fromZml("src", "protobuf.zig"), .{}); + const xla_compile_proto = xla_proto: { + const empty: *std.Build.Module = empty: { + const write = b.addWriteFiles(); + const root = write.add("root.zig", "//! empty module"); + break :empty b.createModule(.{ .root_source_file = root }); + }; + const opts: std.Build.Module.CreateOptions = .{ .imports = &.{.{ .name = "protobuf", .module = protobuf }} }; + + const duration = moduleFromBazelSrcs(b, null, .fromZml("_virtual_imports/duration_proto/google/protobuf", "duration.pb.zig"), opts); + const any = moduleFromBazelSrcs(b, null, .fromZml("_virtual_imports/any_proto/google/protobuf", "any.pb.zig"), opts); + const wrappers = moduleFromBazelSrcs(b, null, .fromZml("_virtual_imports/wrappers_proto/google/protobuf", "wrappers.pb.zig"), opts); + + const data = moduleFromBazelSrcs(b, null, .fromZml("xla", "xla_data.pb.zig"), opts); + const service_hlo = moduleFromBazelSrcs(b, null, .fromZml("xla/service", "hlo.pb.zig"), opts); + const xla = moduleFromBazelSrcs(b, null, .fromZml("xla", "xla.pb.zig"), opts); + xla.addImport("google_protobuf_any_proto", any); + xla.addImport("xla_xla_data_proto", data); + xla.addImport("xla_service_hlo_proto", service_hlo); + xla.addImport("xla_autotune_results_proto", empty); + + const tsl_dnn = moduleFromBazelSrcs(b, null, .fromZml("xla/tsl/protobuf", "dnn.pb.zig"), opts); + tsl_dnn.addImport("google_protobuf_wrappers_proto", wrappers); + + const autotuning = moduleFromBazelSrcs(b, null, .fromZml("xla", "autotuning.pb.zig"), opts); + autotuning.addImport("google_protobuf_duration_proto", duration); + autotuning.addImport("xla_tsl_protobuf_dnn_proto", tsl_dnn); + const autotune = moduleFromBazelSrcs(b, null, .fromZml("xla", "autotune_results.pb.zig"), opts); + autotune.addImport("xla_autotuning_proto", autotuning); + + const stream_executor_cuda_cuda_compute_capability = moduleFromBazelSrcs(b, null, .fromZml("xla/stream_executor/cuda", "cuda_compute_capability.pb.zig"), opts); + const stream_executor_device_description = moduleFromBazelSrcs(b, null, .fromZml("xla/stream_executor", "device_description.pb.zig"), opts); + stream_executor_device_description.addImport("xla_autotune_results_proto", autotune); + stream_executor_device_description.addImport("xla_stream_executor_cuda_cuda_compute_capability_proto", stream_executor_cuda_cuda_compute_capability); + + const compile = moduleFromBazelSrcs(b, null, .fromZml("xla/pjrt/proto", "compile_options.pb.zig"), opts); + compile.addImport("xla_xla_proto", xla); + compile.addImport("xla_xla_data_proto", data); + compile.addImport("xla_stream_executor_device_description_proto", stream_executor_device_description); + + break :xla_proto compile; + }; + // zml c deps const zml_c_deps = moduleFromBazelSrcs( b, @@ -284,6 +329,7 @@ pub fn build(b: *std.Build) void { .link_libcpp = true, .imports = &.{ .{ .name = "c", .module = zml_c_deps }, + .{ .name = "//xla:xla_proto", .module = xla_compile_proto }, .{ .name = "async", .module = async_mod }, .{ .name = "mlir", .module = mlir }, .{ .name = "mlir/dialects", .module = mlir_dialects }, @@ -340,6 +386,8 @@ const BazelSrcs = struct { } }; +var zml_srcs_tar: ?*std.Build.Step.Run = null; + /// Ask bazel for the full sources of a zig module. /// This is needed for module that have generated zig sources, /// like the output of zig translate-c or protobuf generated sources. @@ -349,7 +397,15 @@ fn moduleFromBazelSrcs( srcs: BazelSrcs, options: std.Build.Module.CreateOptions, ) *std.Build.Module { - const bazel_cmd = b.addSystemCommand(&.{ "bazel", "build", srcs.target }); + const bazel_cmd: *std.Build.Step.Run = cmd: { + if (std.mem.eql(u8, srcs.target, "//zml:sources")) { + if (zml_srcs_tar == null) { + zml_srcs_tar = b.addSystemCommand(&.{ "bazel", "build", srcs.target }); + } + break :cmd zml_srcs_tar.?; + } + break :cmd b.addSystemCommand(&.{ "bazel", "build", srcs.target }); + }; const srcs_tar = b.path(b.pathJoin(&.{ "bazel-bin", srcs.tar_path })); const tar_cmd = b.addSystemCommand(&.{ "tar", "-xf" }); From c9a990ec192810356d7d0b1071057916ea3834bb Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Fri, 13 Jun 2025 14:13:05 +0200 Subject: [PATCH 17/40] fix tracing --- build.zig | 4 ++++ zml/tools/BUILD.bazel | 6 +++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/build.zig b/build.zig index 31712503..9e30984a 100644 --- a/build.zig +++ b/build.zig @@ -317,6 +317,10 @@ pub fn build(b: *std.Build) void { .imports = &.{.{ .name = "c", .module = zml_c_deps }}, }, ); + const macos_tools_obj = objectFromBazel(b, "//zml/tools:macos_static_tools", "zml/tools/libmacos_static_tools.a"); + if (target.result.os.tag == .macos) { + zml_tools.addObjectFile(macos_tools_obj); + } // zml const zml = moduleFromBazelSrcs( diff --git a/zml/tools/BUILD.bazel b/zml/tools/BUILD.bazel index 7ffe3a55..a99ea26e 100644 --- a/zml/tools/BUILD.bazel +++ b/zml/tools/BUILD.bazel @@ -7,7 +7,6 @@ cc_library( target_compatible_with = [ "@platforms//os:macos", ], - visibility = ["//zml/tools:__subpackages__"], ) zig_library( @@ -22,3 +21,8 @@ zig_library( "//conditions:default": [], }), ) + +cc_static_library( + name = "macos_static_tools", + deps = ["macos_c"] +) From e3475fd8af5c6b757110debe6ba8c4706afe83ed Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Fri, 13 Jun 2025 14:56:26 +0200 Subject: [PATCH 18/40] use async test runner --- build.zig | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/build.zig b/build.zig index 9e30984a..0a58ee45 100644 --- a/build.zig +++ b/build.zig @@ -17,13 +17,14 @@ const std = @import("std"); /// * copying .a files into zig-cache /// * creating "zig modules" visible to other build.zig. /// -/// `zig build test --summary all` will run tests for several ZML deps, +/// `zig build test --summary all` will run a lot of tests +/// `zig build test-zml` will stick to zml test suite /// but not yet ZML itself. pub fn build(b: *std.Build) void { const target = b.standardTargetOptions(.{}); const optimize = b.standardOptimizeOption(.{}); - const test_step = b.step("test", "Run unit tests"); + const test_step = b.step("test", "Run all tests across ZML and deps"); // stdx const stdx = b.addModule("stdx", .{ @@ -216,6 +217,7 @@ pub fn build(b: *std.Build) void { // proto const protobuf = moduleFromBazelSrcs(b, "protobuf", .fromZml("src", "protobuf.zig"), .{}); const xla_compile_proto = xla_proto: { + // this is horrible, look away. const empty: *std.Build.Module = empty: { const write = b.addWriteFiles(); const root = write.add("root.zig", "//! empty module"); @@ -265,13 +267,8 @@ pub fn build(b: *std.Build) void { .{ .link_libcpp = true }, ); - // runfiles - const runfiles = moduleFromBazelSrcs( - b, - "runfiles", - .fromZml("zig/runfiles", "runfiles.zig"), - .{}, - ); + // bazel runfiles + const runfiles = moduleFromBazelSrcs(b, "runfiles", .fromZml("zig/runfiles", "runfiles.zig"), .{}); // runtimes const runtimes = moduleFromBazelSrcs( @@ -346,10 +343,16 @@ pub fn build(b: *std.Build) void { }, }, ); - const zml_test = b.addTest(.{ .root_module = zml }); + + const zml_test = b.addTest(.{ + .root_module = zml, + .test_runner = .{ .mode = .simple, .path = b.path("zml/test_runner.zig") }, + }); const run_zml_tests = b.addRunArtifact(zml_test); - const zml_test_step = b.step("test-zml", "Run ZML tests (broken)"); + // TODO let copy the pjrt dylib somewhere Zig can see them. + const zml_test_step = b.step("test-zml", "Run ZML tests (assumes pjrt.dylib are in the path)"); zml_test_step.dependOn(&run_zml_tests.step); + test_step.dependOn(&run_zml_tests.step); } /// Take the name of a Bazel `cc_static_library` and create a object LazyPath from it. From bb04337523f5579bac81a88f8755cdd606537df3 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Fri, 13 Jun 2025 16:18:49 +0200 Subject: [PATCH 19/40] bump rules_zig --- MODULE.bazel | 2 +- .../rules_zig/20250613.0-567662a/MODULE.bazel | 68 +++++++++++++++++++ .../rules_zig/20250613.0-567662a/source.json | 5 ++ third_party/modules/rules_zig/metadata.json | 3 +- 4 files changed, 76 insertions(+), 2 deletions(-) create mode 100644 third_party/modules/rules_zig/20250613.0-567662a/MODULE.bazel create mode 100644 third_party/modules/rules_zig/20250613.0-567662a/source.json diff --git a/MODULE.bazel b/MODULE.bazel index 62113917..12c95ae4 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -17,7 +17,7 @@ bazel_dep(name = "rules_proto", version = "7.1.0") bazel_dep(name = "rules_python", version = "0.40.0") bazel_dep(name = "rules_rust", version = "0.60.0") bazel_dep(name = "rules_uv", version = "0.65.0") -bazel_dep(name = "rules_zig", version = "20250530.0-5084f1f") +bazel_dep(name = "rules_zig", version = "20250613.0-567662a") bazel_dep(name = "sentencepiece", version = "20240618.0-d7ace0a") bazel_dep(name = "toolchains_llvm_bootstrapped", version = "0.2.3") bazel_dep(name = "toolchains_protoc", version = "0.4.1") diff --git a/third_party/modules/rules_zig/20250613.0-567662a/MODULE.bazel b/third_party/modules/rules_zig/20250613.0-567662a/MODULE.bazel new file mode 100644 index 00000000..4189634c --- /dev/null +++ b/third_party/modules/rules_zig/20250613.0-567662a/MODULE.bazel @@ -0,0 +1,68 @@ +module( + name = "rules_zig", + version = "20250613.0-567662a", + compatibility_level = 1, +) + +bazel_dep(name = "aspect_bazel_lib", version = "2.8.1") +bazel_dep(name = "bazel_skylib", version = "1.7.1") +bazel_dep(name = "platforms", version = "0.0.10") + +zig = use_extension("//zig:extensions.bzl", "zig") +zig.index(file = "//zig/private:versions.json") +use_repo(zig, "zig_toolchains") + +register_toolchains("@rules_zig//zig/target:all") + +register_toolchains("@zig_toolchains//:all") + +zig_dev = use_extension( + "//zig:extensions.bzl", + "zig", + dev_dependency = True, +) +zig_dev.toolchain(zig_version = "0.13.0") +zig_dev.toolchain(zig_version = "0.12.1") +zig_dev.toolchain(zig_version = "0.12.0") +zig_dev.toolchain(zig_version = "0.11.0") + +bazel_dep(name = "rules_cc", version = "0.0.9") +bazel_dep(name = "stardoc", version = "0.7.0", dev_dependency = True, repo_name = "io_bazel_stardoc") +bazel_dep(name = "gazelle", version = "0.38.0", dev_dependency = True, repo_name = "bazel_gazelle") +bazel_dep(name = "bazel_skylib_gazelle_plugin", version = "1.7.1", dev_dependency = True) +bazel_dep( + name = "buildifier_prebuilt", + version = "7.3.1", + dev_dependency = True, +) +bazel_dep(name = "rules_multirun", version = "0.9.0", dev_dependency = True) +bazel_dep(name = "rules_python", version = "0.35.0", dev_dependency = True) +bazel_dep( + name = "rules_bazel_integration_test", + version = "0.25.0", + dev_dependency = True, +) + +bazel_binaries = use_extension( + "@rules_bazel_integration_test//:extensions.bzl", + "bazel_binaries", + dev_dependency = True, +) + +# NOTE: Keep in sync with WORKSPACE. +bazel_binaries.download(version_file = "//:.bazelversion") +bazel_binaries.download(version = "7.0.0") +use_repo( + bazel_binaries, + "bazel_binaries", + "bazel_binaries_bazelisk", + "build_bazel_bazel_.bazelversion", + "build_bazel_bazel_7_0_0", +) + +# TODO[AH] Should be an implicit transitive dependency through rules_bazel_integration_test. +# However, if we do not include it explicitly, then the runfiles resolution for +# cgrindel_bazel_starlib/shlib/lib/message.sh fails in +# rules_bazel_integration_test/tools/update_deleted_packages.sh when invoked +# through the rules_multirun target //util:update. +bazel_dep(name = "cgrindel_bazel_starlib", version = "0.21.0", dev_dependency = True) diff --git a/third_party/modules/rules_zig/20250613.0-567662a/source.json b/third_party/modules/rules_zig/20250613.0-567662a/source.json new file mode 100644 index 00000000..4dbdafee --- /dev/null +++ b/third_party/modules/rules_zig/20250613.0-567662a/source.json @@ -0,0 +1,5 @@ +{ + "strip_prefix": "rules_zig-567662a3ce5e87894950d56c69d8de5ad6e0b5f0", + "url": "https://github.com/zml/rules_zig/archive/567662a3ce5e87894950d56c69d8de5ad6e0b5f0.tar.gz", + "integrity": "sha256-PjFpDJXO0BGx2CAZ54Ppv6d4TwTDOWS+mVangKQFvZc=" +} diff --git a/third_party/modules/rules_zig/metadata.json b/third_party/modules/rules_zig/metadata.json index 51e22e7d..90794cf9 100644 --- a/third_party/modules/rules_zig/metadata.json +++ b/third_party/modules/rules_zig/metadata.json @@ -16,7 +16,8 @@ "20240912.0-41bfe84", "20240913.0-1957d05", "20250314.0-b9739c6", - "20250519.0-233b207" + "20250519.0-233b207", + "20250613.0-567662a" ], "yanked_versions": {} } From 9240d5ea7cd6bcf8f4a3dd74a24f4e2d98d640f8 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Fri, 13 Jun 2025 16:24:20 +0200 Subject: [PATCH 20/40] update disclaimer --- build.zig | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/build.zig b/build.zig index 0a58ee45..e7863f2a 100644 --- a/build.zig +++ b/build.zig @@ -1,6 +1,6 @@ const std = @import("std"); -/// !!! This build.zig is work in progress. !!! +/// !!! This build.zig is experimental !!! /// /// It shows how to bridge Bazel and build.zig. /// It requires the user to have `bazel` and `tar` installed on their machine. @@ -18,8 +18,14 @@ const std = @import("std"); /// * creating "zig modules" visible to other build.zig. /// /// `zig build test --summary all` will run a lot of tests -/// `zig build test-zml` will stick to zml test suite -/// but not yet ZML itself. +/// `zig build test-zml` will stick to ZML test suite +/// +/// Caveats: +/// +/// * requires a local bazel installed and tar utility +/// * requires to have the CPU pjrt plugin visible in the path when running the executables +/// you can do so eg by find -L ./bazel-out -name libpjrt_cpu.dylib then cp ./bazel-out/.../libpjt_cpu.dylib . +/// * only tested with CPU plugins pub fn build(b: *std.Build) void { const target = b.standardTargetOptions(.{}); const optimize = b.standardOptimizeOption(.{}); From c0a2b3baac536d3f461cb6e48974864d7383c477 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Sun, 15 Jun 2025 15:34:17 +0200 Subject: [PATCH 21/40] start working on cuda support --- build.zig | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/build.zig b/build.zig index e7863f2a..380d7aad 100644 --- a/build.zig +++ b/build.zig @@ -130,6 +130,7 @@ pub fn build(b: *std.Build) void { .{ .target = target, .optimize = optimize, + .link_libc = target.result.os.tag == .linux, }, ); @@ -293,6 +294,9 @@ pub fn build(b: *std.Build) void { for (&PLATFORMS) |platform| { const runtime_path = b.pathJoin(&.{ "runtimes", platform }); const zig_file = std.mem.concat(b.allocator, u8, &.{ platform, ".zig" }) catch @panic("OOM"); + const builtin_name = std.fmt.allocPrint(b.allocator, "bazel_builtin_A_S_Sruntimes_S{0s}_C{0s}.zig", .{platform}) catch @panic("OOM"); + const bazel_builtin = moduleFromBazelSrcs(b, null, .fromZml(runtime_path, builtin_name), .{}); + const platform_runtime = moduleFromBazelSrcs( b, runtime_path, @@ -301,8 +305,10 @@ pub fn build(b: *std.Build) void { .imports = &.{ .{ .name = "c", .module = zml_c_deps }, .{ .name = "runfiles", .module = runfiles }, + .{ .name = "stdx", .module = stdx }, .{ .name = "pjrt", .module = pjrt }, .{ .name = "async", .module = async_mod }, + .{ .name = "bazel_builtin", .module = bazel_builtin }, }, }, ); @@ -316,7 +322,7 @@ pub fn build(b: *std.Build) void { "zml/tools", .fromZml("zml/tools", "tools.zig"), .{ - .link_libcpp = target.query.os_tag == .macos, + .link_libcpp = target.result.os.tag == .macos, .imports = &.{.{ .name = "c", .module = zml_c_deps }}, }, ); @@ -413,7 +419,13 @@ fn moduleFromBazelSrcs( const bazel_cmd: *std.Build.Step.Run = cmd: { if (std.mem.eql(u8, srcs.target, "//zml:sources")) { if (zml_srcs_tar == null) { - zml_srcs_tar = b.addSystemCommand(&.{ "bazel", "build", srcs.target }); + zml_srcs_tar = b.addSystemCommand(&.{ + "bazel", + "build", + // "--@zml//runtimes:cuda=true", + "--@zml//runtimes:cpu=true", + srcs.target, + }); } break :cmd zml_srcs_tar.?; } From 1658a1bd787a4c11f48252094640c04d8edfea80 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Sun, 15 Jun 2025 15:35:18 +0200 Subject: [PATCH 22/40] REVERTME: disable cuda tracer --- zml/tools/tracer.zig | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/zml/tools/tracer.zig b/zml/tools/tracer.zig index 05769563..c1000cb8 100644 --- a/zml/tools/tracer.zig +++ b/zml/tools/tracer.zig @@ -1,9 +1,11 @@ const builtin = @import("builtin"); + const c = @import("c"); pub const Tracer = switch (builtin.os.tag) { .macos => MacOsTracer, - .linux => if (@hasDecl(c, "ZML_RUNTIME_CUDA")) CudaTracer else FakeTracer, + // .linux => if (@hasDecl(c, "ZML_RUNTIME_CUDA")) CudaTracer else FakeTracer, + .linux => FakeTracer, else => FakeTracer, }; From 01f3b4bc0948c5dc7385414157573dde126809be Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Mon, 16 Jun 2025 11:40:45 +0200 Subject: [PATCH 23/40] download plugin from pjrt-artifacts --- build.zig | 35 +++++++++++++++++++++++++++++++++-- build.zig.zon | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 2 deletions(-) create mode 100644 build.zig.zon diff --git a/build.zig b/build.zig index 380d7aad..16f7fb92 100644 --- a/build.zig +++ b/build.zig @@ -116,6 +116,20 @@ pub fn build(b: *std.Build) void { const run_pjrt_tests = b.addRunArtifact(pjrt_test); test_step.dependOn(&run_pjrt_tests.step); + const pjrt_plugin_so = plugin: { + switch (target.result.os.tag) { + .macos => { + const dep = b.lazyDependency("pjrt_cpu_darwin_arm64", .{}).?; + break :plugin b.addInstallLibFile(dep.path("libpjrt_cpu.dylib"), "libpjrt_cpu.dylib"); + }, + .linux => { + const dep = b.lazyDependency("pjrt_cuda_linux_amd64", .{}).?; + break :plugin b.addInstallLibFile(dep.path("libpjrt_cuda.so"), "libpjrt_cuda.so"); + }, + else => |os| std.debug.panic("Target not supported: {s}", .{@tagName(os)}), + } + }; + // xev const xev = moduleFromBazelSrcs( b, @@ -360,8 +374,11 @@ pub fn build(b: *std.Build) void { .root_module = zml, .test_runner = .{ .mode = .simple, .path = b.path("zml/test_runner.zig") }, }); + // Note: I failed to make rpath work, so I'm using the "dlpreload" trick. + // zml_test.addRPath(pjrt_plugin_so.source); const run_zml_tests = b.addRunArtifact(zml_test); - // TODO let copy the pjrt dylib somewhere Zig can see them. + dlPreload(target, run_zml_tests, pjrt_plugin_so); + const zml_test_step = b.step("test-zml", "Run ZML tests (assumes pjrt.dylib are in the path)"); zml_test_step.dependOn(&run_zml_tests.step); test_step.dependOn(&run_zml_tests.step); @@ -422,7 +439,8 @@ fn moduleFromBazelSrcs( zml_srcs_tar = b.addSystemCommand(&.{ "bazel", "build", - // "--@zml//runtimes:cuda=true", + // TODO parse this from build.zig flags. + "--@zml//runtimes:cuda=true", "--@zml//runtimes:cpu=true", srcs.target, }); @@ -448,3 +466,16 @@ fn moduleFromBazelSrcs( else b.createModule(opts); } + +fn dlPreload(target: std.Build.ResolvedTarget, run_step: *std.Build.Step.Run, lib: *std.Build.Step.InstallFile) void { + const env_key = switch (target.result.os.tag) { + .macos => "DYLD_INSERT_LIBRARIES", + .linux => "LD_PRELOAD", + else => |os| std.debug.panic("Target not supported: {s}", .{@tagName(os)}), + }; + + run_step.step.dependOn(&lib.step); + const b = run_step.step.owner; + const lib_path = b.getInstallPath(lib.dir, lib.dest_rel_path); + run_step.setEnvironmentVariable(env_key, lib_path); +} diff --git a/build.zig.zon b/build.zig.zon new file mode 100644 index 00000000..4d4b986f --- /dev/null +++ b/build.zig.zon @@ -0,0 +1,49 @@ +.{ + .name = .zml, + .version = "0.0.0", + + .fingerprint = 0x32700c0d288fdb37, // Changing this has security and trust implications. + + .minimum_zig_version = "0.14.0", + + // This field is optional. + // Each dependency must either provide a `url` and `hash`, or a `path`. + // `zig build --fetch` can be used to fetch all dependencies of a package, recursively. + // Once all dependencies are fetched, `zig build` no longer requires + // internet connectivity. + .dependencies = .{ + .pjrt_cpu_darwin_arm64 = .{ + .url = "https://github.com/zml/pjrt-artifacts/releases/download/v10.0.0/pjrt-cpu_darwin-arm64.tar.gz", + .hash = "N-V-__8AADBK6gU12pCj6Yo3Q5zYgUefD3uH5kShV2pHGyOS", + .lazy = true, + }, + .pjrt_cuda_linux_amd64 = .{ + .url = "https://github.com/zml/pjrt-artifacts/releases/download/v10.0.0/pjrt-cuda_linux-amd64.tar.gz", + .hash = "N-V-__8AAJASFhgOL6MRfJwNEy7Y6vRUmu5OPlIE0hYXKVTc", + .lazy = true, + }, + }, + + // Specifies the set of files and directories that are included in this package. + // Only files and directories listed here are included in the `hash` that + // is computed for this package. Only files listed here will remain on disk + // when using the zig package manager. As a rule of thumb, one should list + // files required for compilation plus any license(s). + // Paths are relative to the build root. Use the empty string (`""`) to refer to + // the build root itself. + // A directory listed here means that all files within, recursively, are included. + .paths = .{ + "async", + "bazel", + "ffi", + "mlir", + "pjrt", + "platforms", + "runtimes", + "stdx", + "third_party", + "zml", + "build.zig", + "build.zig.zon", + }, +} From b012146d7326aaa518b7715fcfec12ebe391fe1b Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Mon, 16 Jun 2025 11:40:57 +0200 Subject: [PATCH 24/40] warn in case of failed dlopen --- pjrt/pjrt.zig | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 294243ac..2b7f48c4 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -87,7 +87,10 @@ pub const Api = struct { }, }; }, - else => try std.DynLib.open(library), + else => std.DynLib.open(library) catch |err| { + log.warn("Failed to load {s}", .{library}); + return err; + }, }; const DynGetPjrtApi = lib.lookup(*const fn () callconv(.C) *const Api, "GetPjrtApi") orelse { std.debug.panic("Unable to find GetPjrtApi symbol in library: {s}", .{library}); From 5f06901fb8841466eea6da99c8464257d95ed114 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Mon, 16 Jun 2025 15:06:49 +0200 Subject: [PATCH 25/40] switch to RPath --- build.zig | 43 +++++++++++++++++++++++-------------------- build.zig.zon | 5 +++++ 2 files changed, 28 insertions(+), 20 deletions(-) diff --git a/build.zig b/build.zig index 16f7fb92..fc0be5cf 100644 --- a/build.zig +++ b/build.zig @@ -116,15 +116,25 @@ pub fn build(b: *std.Build) void { const run_pjrt_tests = b.addRunArtifact(pjrt_test); test_step.dependOn(&run_pjrt_tests.step); - const pjrt_plugin_so = plugin: { + const pjrt_dynamic_deps: []const *std.Build.Step.InstallFile = deps: { switch (target.result.os.tag) { .macos => { - const dep = b.lazyDependency("pjrt_cpu_darwin_arm64", .{}).?; - break :plugin b.addInstallLibFile(dep.path("libpjrt_cpu.dylib"), "libpjrt_cpu.dylib"); + const dep = b.lazyDependency("pjrt_cpu_darwin_arm64", .{}) orelse break :deps &.{}; + break :deps &.{b.addInstallLibFile(dep.path("libpjrt_cpu.dylib"), "libpjrt_cpu.dylib")}; }, .linux => { - const dep = b.lazyDependency("pjrt_cuda_linux_amd64", .{}).?; - break :plugin b.addInstallLibFile(dep.path("libpjrt_cuda.so"), "libpjrt_cuda.so"); + const dep = b.lazyDependency("pjrt_cpu_linux_amd64", .{}) orelse break :deps &.{}; + break :deps &.{b.addInstallLibFile(dep.path("libpjrt_cpu.so"), "libpjrt_cpu.so")}; + }, + .cuda => { + // TODO: this is not enough for libpjrt_cuda, there are a lot more deps needed. + const zmlxcuda = objectFromBazel(b, "@libpjrt_cuda//:zmlxcuda_so", "external/+cuda_packages+libpjrt_cuda/libzmlxcuda.so.0"); + const plugin = objectFromBazel(b, "//runtimes/cuda:libpjrt_cuda", "runtimes/cuda/libpjrt_cuda.so"); + + break :deps &.{ + b.addInstallLibFile(zmlxcuda, "libzmlxcuda.so.0"), + b.addInstallLibFile(plugin, "libpjrt_cuda.so"), + }; }, else => |os| std.debug.panic("Target not supported: {s}", .{@tagName(os)}), } @@ -303,7 +313,6 @@ pub fn build(b: *std.Build) void { }, }, ); - const PLATFORMS = [_][]const u8{ "cpu", "cuda", "rocm", "tpu", "neuron" }; for (&PLATFORMS) |platform| { const runtime_path = b.pathJoin(&.{ "runtimes", platform }); @@ -374,10 +383,11 @@ pub fn build(b: *std.Build) void { .root_module = zml, .test_runner = .{ .mode = .simple, .path = b.path("zml/test_runner.zig") }, }); - // Note: I failed to make rpath work, so I'm using the "dlpreload" trick. - // zml_test.addRPath(pjrt_plugin_so.source); + // This is where we will put all the .so needed. + zml_test.addRPath(.{ .cwd_relative = b.lib_dir }); + const run_zml_tests = b.addRunArtifact(zml_test); - dlPreload(target, run_zml_tests, pjrt_plugin_so); + addRuntimeDeps(run_zml_tests, pjrt_dynamic_deps); const zml_test_step = b.step("test-zml", "Run ZML tests (assumes pjrt.dylib are in the path)"); zml_test_step.dependOn(&run_zml_tests.step); @@ -467,15 +477,8 @@ fn moduleFromBazelSrcs( b.createModule(opts); } -fn dlPreload(target: std.Build.ResolvedTarget, run_step: *std.Build.Step.Run, lib: *std.Build.Step.InstallFile) void { - const env_key = switch (target.result.os.tag) { - .macos => "DYLD_INSERT_LIBRARIES", - .linux => "LD_PRELOAD", - else => |os| std.debug.panic("Target not supported: {s}", .{@tagName(os)}), - }; - - run_step.step.dependOn(&lib.step); - const b = run_step.step.owner; - const lib_path = b.getInstallPath(lib.dir, lib.dest_rel_path); - run_step.setEnvironmentVariable(env_key, lib_path); +fn addRuntimeDeps(run_step: *std.Build.Step.Run, libs: []const *std.Build.Step.InstallFile) void { + for (libs) |lib| { + run_step.step.dependOn(&lib.step); + } } diff --git a/build.zig.zon b/build.zig.zon index 4d4b986f..caf36784 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -17,6 +17,11 @@ .hash = "N-V-__8AADBK6gU12pCj6Yo3Q5zYgUefD3uH5kShV2pHGyOS", .lazy = true, }, + .pjrt_cpu_linux_amd64 = .{ + .url = "https://github.com/zml/pjrt-artifacts/releases/download/v10.0.0/pjrt-cpu_linux-amd64.tar.gz", + .hash = "N-V-__8AAAjaOQyQKeieiaG2csxnfXzIkaPNN4aX7p247PLE", + .lazy = true, + }, .pjrt_cuda_linux_amd64 = .{ .url = "https://github.com/zml/pjrt-artifacts/releases/download/v10.0.0/pjrt-cuda_linux-amd64.tar.gz", .hash = "N-V-__8AAJASFhgOL6MRfJwNEy7Y6vRUmu5OPlIE0hYXKVTc", From 8791e531e537b13ea45830cdb2a408da2c9a8019 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Mon, 16 Jun 2025 15:07:11 +0200 Subject: [PATCH 26/40] cuda: allow no runfiles --- runtimes/cuda/cuda.zig | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/runtimes/cuda/cuda.zig b/runtimes/cuda/cuda.zig index bc749693..46eb69dd 100644 --- a/runtimes/cuda/cuda.zig +++ b/runtimes/cuda/cuda.zig @@ -36,7 +36,8 @@ fn setupXlaGpuCudaDirFlag() !void { defer arena.deinit(); var r_ = try runfiles.Runfiles.create(.{ .allocator = arena.allocator() }) orelse { - stdx.debug.panic("Unable to find CUDA directory", .{}); + log.warn("Unable to find CUDA directory. Using system defaults.", .{}); + return; }; const source_repo = bazel_builtin.current_repository; From 1f4286d10a8deacc98a1bcbac8d7dd3d9c8d04b9 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Mon, 16 Jun 2025 16:20:33 +0200 Subject: [PATCH 27/40] add flags to select runtime --- build.zig | 294 +++++++++++++++++++++++++------------------------- build.zig.zon | 5 - 2 files changed, 148 insertions(+), 151 deletions(-) diff --git a/build.zig b/build.zig index fc0be5cf..638ded00 100644 --- a/build.zig +++ b/build.zig @@ -29,6 +29,7 @@ const std = @import("std"); pub fn build(b: *std.Build) void { const target = b.standardTargetOptions(.{}); const optimize = b.standardOptimizeOption(.{}); + const platforms = Platforms.parse(b); const test_step = b.step("test", "Run all tests across ZML and deps"); @@ -44,10 +45,11 @@ pub fn build(b: *std.Build) void { test_step.dependOn(&run_stdx_tests.step); // mlir - const mlir_c_deps = moduleFromBazelSrcs( - b, - null, - .canonical(b.allocator, "mlir/dialects", "test_test_lib_c.zig"), + const mlir_srcs = Tarball.sources(b, "mlir/dialects"); + const mlir_c_deps = mlir_srcs.extractModule( + "mlir/dialects", + "mlir/dialects", + "test_test_lib_c.zig", .{ .link_libcpp = true }, ); const mlir_obj = objectFromBazel(b, "//mlir/dialects:mlir_static", "mlir/dialects/libmlir_static.a"); @@ -95,10 +97,11 @@ pub fn build(b: *std.Build) void { test_step.dependOn(&run_mlir_dialects_tests.step); // pjrt - const pjrt_c_deps = moduleFromBazelSrcs( - b, - null, - .canonical(b.allocator, "pjrt", "test_test_lib_c.zig"), + const pjrt_srcs = Tarball.sources(b, "pjrt"); + const pjrt_c_deps = pjrt_srcs.extractModule( + "pjrt", + "pjrt", + "test_test_lib_c.zig", .{ .link_libcpp = true }, ); @@ -116,41 +119,43 @@ pub fn build(b: *std.Build) void { const run_pjrt_tests = b.addRunArtifact(pjrt_test); test_step.dependOn(&run_pjrt_tests.step); - const pjrt_dynamic_deps: []const *std.Build.Step.InstallFile = deps: { + var _d: std.BoundedArray(*std.Build.Step.InstallFile, 8) = .{}; + deps: { switch (target.result.os.tag) { .macos => { - const dep = b.lazyDependency("pjrt_cpu_darwin_arm64", .{}) orelse break :deps &.{}; - break :deps &.{b.addInstallLibFile(dep.path("libpjrt_cpu.dylib"), "libpjrt_cpu.dylib")}; + if (platforms.cpu) { + const dep = b.lazyDependency("pjrt_cpu_darwin_arm64", .{}) orelse break :deps; + const pjrt_cpu = b.addInstallLibFile(dep.path("libpjrt_cpu.dylib"), "libpjrt_cpu.dylib"); + _d.appendAssumeCapacity(pjrt_cpu); + } }, .linux => { - const dep = b.lazyDependency("pjrt_cpu_linux_amd64", .{}) orelse break :deps &.{}; - break :deps &.{b.addInstallLibFile(dep.path("libpjrt_cpu.so"), "libpjrt_cpu.so")}; - }, - .cuda => { - // TODO: this is not enough for libpjrt_cuda, there are a lot more deps needed. - const zmlxcuda = objectFromBazel(b, "@libpjrt_cuda//:zmlxcuda_so", "external/+cuda_packages+libpjrt_cuda/libzmlxcuda.so.0"); - const plugin = objectFromBazel(b, "//runtimes/cuda:libpjrt_cuda", "runtimes/cuda/libpjrt_cuda.so"); + if (platforms.cpu) { + const dep = b.lazyDependency("pjrt_cpu_linux_amd64", .{}) orelse break :deps; + const pjrt_cpu = b.addInstallLibFile(dep.path("libpjrt_cpu.so"), "libpjrt_cpu.so"); + _d.appendAssumeCapacity(pjrt_cpu); + } - break :deps &.{ - b.addInstallLibFile(zmlxcuda, "libzmlxcuda.so.0"), - b.addInstallLibFile(plugin, "libpjrt_cuda.so"), - }; + if (platforms.cuda) { + // TODO: this is not enough for libpjrt_cuda, there are a lot more deps needed. + const zmlxcuda = objectFromBazel(b, "@libpjrt_cuda//:zmlxcuda_so", "external/+cuda_packages+libpjrt_cuda/libzmlxcuda.so.0"); + const plugin = objectFromBazel(b, "//runtimes/cuda:libpjrt_cuda", "runtimes/cuda/libpjrt_cuda.so"); + + _d.appendAssumeCapacity(b.addInstallLibFile(zmlxcuda, "libzmlxcuda.so.0")); + _d.appendAssumeCapacity(b.addInstallLibFile(plugin, "libpjrt_cuda.so")); + } }, else => |os| std.debug.panic("Target not supported: {s}", .{@tagName(os)}), } - }; + } + const pjrt_dynamic_deps = _d.constSlice(); // xev - const xev = moduleFromBazelSrcs( - b, + const async_srcs = Tarball.sources(b, "async"); + const xev = async_srcs.extractModule( "xev", - .{ - // xev sources can be find inside async sources cause it's a dependency. - .target = "//async:sources", - .tar_path = "async/sources.tar", - .directory = "src", - .root = "main.zig", - }, + "src", + "main.zig", .{ .target = target, .optimize = optimize, @@ -163,10 +168,10 @@ pub fn build(b: *std.Build) void { test_step.dependOn(&run_xev_tests.step); // async - const async_mod = moduleFromBazelSrcs( - b, + const async_mod = async_srcs.extractModule( "async", - .canonical(b.allocator, "async", "async.zig"), + "async", + "async.zig", .{ .target = target, .optimize = optimize, @@ -181,11 +186,12 @@ pub fn build(b: *std.Build) void { const run_async_tests = b.addRunArtifact(async_test); test_step.dependOn(&run_async_tests.step); + const zml_srcs = Tarball.zml(b, platforms); // ffi - const ffi = moduleFromBazelSrcs( - b, + const ffi = zml_srcs.extractModule( "ffi", - .fromZml("ffi", "ffi.zig"), + "ffi", + "ffi.zig", .{ .target = target, .optimize = optimize }, ); @@ -194,24 +200,25 @@ pub fn build(b: *std.Build) void { test_step.dependOn(&run_ffi_tests.step); // hftokenizers - const hftokenizers = moduleFromBazelSrcs( - b, + const hftokenizers = zml_srcs.extractModule( "//zml/tokenizer/hftokenizers", - .fromZml("zml/tokenizer/hftokenizers", "hftokenizers.zig"), + "zml/tokenizer/hftokenizers", + "hftokenizers.zig", .{ .target = target, .optimize = optimize }, ); // sentencepiece - const sentencepiece_c_deps = moduleFromBazelSrcs( - b, + const sentencepiece_c_deps_srcs = Tarball.sources(b, "zml/tokenizer/sentencepiece"); + const sentencepiece_c_deps = sentencepiece_c_deps_srcs.extractModule( null, - .canonical(b.allocator, "zml/tokenizer/sentencepiece", "test_test_lib_c.zig"), + "zml/tokenizer/sentencepiece", + "test_test_lib_c.zig", .{ .link_libcpp = true }, ); - const sentencepiece = moduleFromBazelSrcs( - b, + const sentencepiece = zml_srcs.extractModule( "//zml/tokenizer/sentencepiece", - .fromZml("zml/tokenizer/sentencepiece", "sentencepiece.zig"), + "zml/tokenizer/sentencepiece", + "sentencepiece.zig", .{ .target = target, .optimize = optimize, @@ -222,10 +229,10 @@ pub fn build(b: *std.Build) void { ); // tokenizer - const tokenizer = moduleFromBazelSrcs( - b, + const tokenizer = zml_srcs.extractModule( "tokenizer", - .fromZml("zml/tokenizer", "tokenizer.zig"), + "zml/tokenizer", + "tokenizer.zig", .{ .target = target, .optimize = optimize, @@ -246,9 +253,9 @@ pub fn build(b: *std.Build) void { test_step.dependOn(&run_tokenizer_tests.step); // proto - const protobuf = moduleFromBazelSrcs(b, "protobuf", .fromZml("src", "protobuf.zig"), .{}); + const protobuf = zml_srcs.extractModule("protobuf", "src", "protobuf.zig", .{}); const xla_compile_proto = xla_proto: { - // this is horrible, look away. + // Ideally we should ask bazel to generate this graph of dependencies between the different proto files. const empty: *std.Build.Module = empty: { const write = b.addWriteFiles(); const root = write.add("root.zig", "//! empty module"); @@ -256,33 +263,33 @@ pub fn build(b: *std.Build) void { }; const opts: std.Build.Module.CreateOptions = .{ .imports = &.{.{ .name = "protobuf", .module = protobuf }} }; - const duration = moduleFromBazelSrcs(b, null, .fromZml("_virtual_imports/duration_proto/google/protobuf", "duration.pb.zig"), opts); - const any = moduleFromBazelSrcs(b, null, .fromZml("_virtual_imports/any_proto/google/protobuf", "any.pb.zig"), opts); - const wrappers = moduleFromBazelSrcs(b, null, .fromZml("_virtual_imports/wrappers_proto/google/protobuf", "wrappers.pb.zig"), opts); + const duration = zml_srcs.extractModule(null, "_virtual_imports/duration_proto/google/protobuf", "duration.pb.zig", opts); + const any = zml_srcs.extractModule(null, "_virtual_imports/any_proto/google/protobuf", "any.pb.zig", opts); + const wrappers = zml_srcs.extractModule(null, "_virtual_imports/wrappers_proto/google/protobuf", "wrappers.pb.zig", opts); - const data = moduleFromBazelSrcs(b, null, .fromZml("xla", "xla_data.pb.zig"), opts); - const service_hlo = moduleFromBazelSrcs(b, null, .fromZml("xla/service", "hlo.pb.zig"), opts); - const xla = moduleFromBazelSrcs(b, null, .fromZml("xla", "xla.pb.zig"), opts); + const data = zml_srcs.extractModule(null, "xla", "xla_data.pb.zig", opts); + const service_hlo = zml_srcs.extractModule(null, "xla/service", "hlo.pb.zig", opts); + const xla = zml_srcs.extractModule(null, "xla", "xla.pb.zig", opts); xla.addImport("google_protobuf_any_proto", any); xla.addImport("xla_xla_data_proto", data); xla.addImport("xla_service_hlo_proto", service_hlo); xla.addImport("xla_autotune_results_proto", empty); - const tsl_dnn = moduleFromBazelSrcs(b, null, .fromZml("xla/tsl/protobuf", "dnn.pb.zig"), opts); + const tsl_dnn = zml_srcs.extractModule(null, "xla/tsl/protobuf", "dnn.pb.zig", opts); tsl_dnn.addImport("google_protobuf_wrappers_proto", wrappers); - const autotuning = moduleFromBazelSrcs(b, null, .fromZml("xla", "autotuning.pb.zig"), opts); + const autotuning = zml_srcs.extractModule(null, "xla", "autotuning.pb.zig", opts); autotuning.addImport("google_protobuf_duration_proto", duration); autotuning.addImport("xla_tsl_protobuf_dnn_proto", tsl_dnn); - const autotune = moduleFromBazelSrcs(b, null, .fromZml("xla", "autotune_results.pb.zig"), opts); + const autotune = zml_srcs.extractModule(null, "xla", "autotune_results.pb.zig", opts); autotune.addImport("xla_autotuning_proto", autotuning); - const stream_executor_cuda_cuda_compute_capability = moduleFromBazelSrcs(b, null, .fromZml("xla/stream_executor/cuda", "cuda_compute_capability.pb.zig"), opts); - const stream_executor_device_description = moduleFromBazelSrcs(b, null, .fromZml("xla/stream_executor", "device_description.pb.zig"), opts); + const stream_executor_cuda_cuda_compute_capability = zml_srcs.extractModule(null, "xla/stream_executor/cuda", "cuda_compute_capability.pb.zig", opts); + const stream_executor_device_description = zml_srcs.extractModule(null, "xla/stream_executor", "device_description.pb.zig", opts); stream_executor_device_description.addImport("xla_autotune_results_proto", autotune); stream_executor_device_description.addImport("xla_stream_executor_cuda_cuda_compute_capability_proto", stream_executor_cuda_cuda_compute_capability); - const compile = moduleFromBazelSrcs(b, null, .fromZml("xla/pjrt/proto", "compile_options.pb.zig"), opts); + const compile = zml_srcs.extractModule(null, "xla/pjrt/proto", "compile_options.pb.zig", opts); compile.addImport("xla_xla_proto", xla); compile.addImport("xla_xla_data_proto", data); compile.addImport("xla_stream_executor_device_description_proto", stream_executor_device_description); @@ -291,21 +298,21 @@ pub fn build(b: *std.Build) void { }; // zml c deps - const zml_c_deps = moduleFromBazelSrcs( - b, + const zml_c_deps = zml_srcs.extractModule( null, - .fromZml("zml", "test_test_lib_c.zig"), + "zml", + "test_test_lib_c.zig", .{ .link_libcpp = true }, ); // bazel runfiles - const runfiles = moduleFromBazelSrcs(b, "runfiles", .fromZml("zig/runfiles", "runfiles.zig"), .{}); + const runfiles = zml_srcs.extractModule("runfiles", "zig/runfiles", "runfiles.zig", .{}); // runtimes - const runtimes = moduleFromBazelSrcs( - b, + const runtimes = zml_srcs.extractModule( "runtimes", - .fromZml("runtimes", "runtimes.zig"), + "runtimes", + "runtimes.zig", .{ .imports = &.{ .{ .name = "c", .module = zml_c_deps }, @@ -314,16 +321,17 @@ pub fn build(b: *std.Build) void { }, ); - for (&PLATFORMS) |platform| { + inline for (@typeInfo(Platforms).@"struct".fields) |field| { + const platform = field.name; const runtime_path = b.pathJoin(&.{ "runtimes", platform }); const zig_file = std.mem.concat(b.allocator, u8, &.{ platform, ".zig" }) catch @panic("OOM"); const builtin_name = std.fmt.allocPrint(b.allocator, "bazel_builtin_A_S_Sruntimes_S{0s}_C{0s}.zig", .{platform}) catch @panic("OOM"); - const bazel_builtin = moduleFromBazelSrcs(b, null, .fromZml(runtime_path, builtin_name), .{}); + const bazel_builtin = zml_srcs.extractModule(null, runtime_path, builtin_name, .{}); - const platform_runtime = moduleFromBazelSrcs( - b, + const platform_runtime = zml_srcs.extractModule( runtime_path, - .fromZml(runtime_path, zig_file), + runtime_path, + zig_file, .{ .imports = &.{ .{ .name = "c", .module = zml_c_deps }, @@ -340,10 +348,10 @@ pub fn build(b: *std.Build) void { } // zml/tools - const zml_tools = moduleFromBazelSrcs( - b, + const zml_tools = zml_srcs.extractModule( "zml/tools", - .fromZml("zml/tools", "tools.zig"), + "zml/tools", + "tools.zig", .{ .link_libcpp = target.result.os.tag == .macos, .imports = &.{.{ .name = "c", .module = zml_c_deps }}, @@ -355,10 +363,13 @@ pub fn build(b: *std.Build) void { } // zml - const zml = moduleFromBazelSrcs( - b, + const zml_srcs_tar = b.addSystemCommand(&.{ "bazel", "build", "//zml:sources" }); + if (platforms.cpu) zml_srcs_tar.addArg("--@zml//runtimes:cpu=true"); + if (platforms.cuda) zml_srcs_tar.addArg("--@zml//runtimes:cuda=true"); + const zml = zml_srcs.extractModule( "zml", - .fromZml("zml", "zml.zig"), + "zml", + "zml.zig", .{ .target = target, .optimize = optimize, @@ -407,78 +418,69 @@ fn objectFromBazel(b: *std.Build, target: []const u8, output: []const u8) std.Bu return cp.addCopyFile(b.path(obj_path), output); } -const BazelSrcs = struct { - target: []const u8, - tar_path: []const u8, - directory: []const u8, - root: []const u8, +const Tarball = struct { + create_cmd: *std.Build.Step.Run, + path: std.Build.LazyPath, - pub fn canonical(allocator: std.mem.Allocator, name: []const u8, root: []const u8) BazelSrcs { - return .{ - .target = std.mem.concat(allocator, u8, &.{ "//", name, ":sources" }) catch @panic("OOM"), - .tar_path = std.fs.path.join(allocator, &.{ name, "sources.tar" }) catch @panic("OOM"), - .directory = name, - .root = root, - }; + /// Ask bazel for the full sources of a zig module. + /// This is needed for module that have generated zig sources, + /// like the output of zig translate-c or protobuf generated sources. + pub fn sources(b: *std.Build, name: []const u8) Tarball { + const allocator = b.allocator; + const target = std.mem.concat(allocator, u8, &.{ "//", name, ":sources" }) catch @panic("OOM"); + const bazel_cmd: *std.Build.Step.Run = b.addSystemCommand(&.{ "bazel", "build", target }); + + const srcs_tar = b.path(b.pathJoin(&.{ "bazel-bin", name, "sources.tar" })); + return .{ .create_cmd = bazel_cmd, .path = srcs_tar }; } - pub fn fromZml(name: []const u8, root: []const u8) BazelSrcs { - return .{ - .target = "//zml:sources", - .tar_path = "zml/sources.tar", - .directory = name, - .root = root, - }; + /// Zml is a bit different from other targets cause the code of the "runtimes/**.zig" + /// will change based on the platforms selected. + pub fn zml(b: *std.Build, platforms: Platforms) Tarball { + const bazel_cmd: *std.Build.Step.Run = b.addSystemCommand(&.{ "bazel", "build", "//zml:sources" }); + bazel_cmd.addArg(if (platforms.cpu) "--@zml//runtimes:cpu=true" else "--@zml//runtimes:cpu=false"); + bazel_cmd.addArg(if (platforms.cuda) "--@zml//runtimes:cuda=true" else "--@zml//runtimes:cuda=false"); + + const srcs_tar = b.path(b.pathJoin(&.{ "bazel-bin", "zml", "sources.tar" })); + return .{ .create_cmd = bazel_cmd, .path = srcs_tar }; + } + + pub fn extractModule(self: Tarball, module_name: ?[]const u8, directory: []const u8, root: []const u8, options: std.Build.Module.CreateOptions) *std.Build.Module { + const b = self.create_cmd.step.owner; + const tar_cmd = b.addSystemCommand(&.{ "tar", "-xf" }); + tar_cmd.step.dependOn(&self.create_cmd.step); + tar_cmd.addFileArg(self.path); + tar_cmd.addArg("-C"); + const out_dir = tar_cmd.addOutputDirectoryArg("sources"); + tar_cmd.addArg(directory); + + var opts = options; + if (opts.root_source_file != null) @panic("moduleFromBazelSrcs is already setting the root_source_file option"); + opts.root_source_file = out_dir.path(b, b.pathJoin(&.{ directory, root })); + return if (module_name) |name| + b.addModule(name, opts) + else + b.createModule(opts); } }; -var zml_srcs_tar: ?*std.Build.Step.Run = null; - -/// Ask bazel for the full sources of a zig module. -/// This is needed for module that have generated zig sources, -/// like the output of zig translate-c or protobuf generated sources. -fn moduleFromBazelSrcs( - b: *std.Build, - module_name: ?[]const u8, - srcs: BazelSrcs, - options: std.Build.Module.CreateOptions, -) *std.Build.Module { - const bazel_cmd: *std.Build.Step.Run = cmd: { - if (std.mem.eql(u8, srcs.target, "//zml:sources")) { - if (zml_srcs_tar == null) { - zml_srcs_tar = b.addSystemCommand(&.{ - "bazel", - "build", - // TODO parse this from build.zig flags. - "--@zml//runtimes:cuda=true", - "--@zml//runtimes:cpu=true", - srcs.target, - }); - } - break :cmd zml_srcs_tar.?; - } - break :cmd b.addSystemCommand(&.{ "bazel", "build", srcs.target }); - }; - const srcs_tar = b.path(b.pathJoin(&.{ "bazel-bin", srcs.tar_path })); - - const tar_cmd = b.addSystemCommand(&.{ "tar", "-xf" }); - tar_cmd.step.dependOn(&bazel_cmd.step); - tar_cmd.addFileArg(srcs_tar); - tar_cmd.addArg("-C"); - const out_dir = tar_cmd.addOutputDirectoryArg("sources"); - tar_cmd.addArg(srcs.directory); - - var opts = options; - if (opts.root_source_file != null) @panic("moduleFromBazelSrcs is already setting the root_source_file option"); - opts.root_source_file = out_dir.path(b, b.pathJoin(&.{ srcs.directory, srcs.root })); - return if (module_name) |name| - b.addModule(name, opts) - else - b.createModule(opts); -} - fn addRuntimeDeps(run_step: *std.Build.Step.Run, libs: []const *std.Build.Step.InstallFile) void { for (libs) |lib| { run_step.step.dependOn(&lib.step); } } + +const Platforms = struct { + cpu: bool, + cuda: bool, + rocm: bool = false, + tpu: bool = false, + neuron: bool = false, + + pub fn parse(b: *std.Build) Platforms { + return .{ + .cpu = b.option(bool, "runtimes:cpu", "Enable cpu runtime (default: true)") orelse true, + .cuda = b.option(bool, "runtimes:cuda", "Enable cuda runtime") orelse false, + }; + } +}; diff --git a/build.zig.zon b/build.zig.zon index caf36784..f4456fff 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -22,11 +22,6 @@ .hash = "N-V-__8AAAjaOQyQKeieiaG2csxnfXzIkaPNN4aX7p247PLE", .lazy = true, }, - .pjrt_cuda_linux_amd64 = .{ - .url = "https://github.com/zml/pjrt-artifacts/releases/download/v10.0.0/pjrt-cuda_linux-amd64.tar.gz", - .hash = "N-V-__8AAJASFhgOL6MRfJwNEy7Y6vRUmu5OPlIE0hYXKVTc", - .lazy = true, - }, }, // Specifies the set of files and directories that are included in this package. From be4fe2e16fe77cfd4ab2b68710fe3aa8dd4378a3 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Mon, 16 Jun 2025 16:35:34 +0200 Subject: [PATCH 28/40] update docs --- build.zig | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/build.zig b/build.zig index 638ded00..4944499a 100644 --- a/build.zig +++ b/build.zig @@ -19,13 +19,25 @@ const std = @import("std"); /// /// `zig build test --summary all` will run a lot of tests /// `zig build test-zml` will stick to ZML test suite +/// `zig build -Druntimes:cpu=true -Druntimes:cuda=false test` will enable/disable runtimes /// /// Caveats: /// /// * requires a local bazel installed and tar utility -/// * requires to have the CPU pjrt plugin visible in the path when running the executables -/// you can do so eg by find -L ./bazel-out -name libpjrt_cpu.dylib then cp ./bazel-out/.../libpjt_cpu.dylib . -/// * only tested with CPU plugins +/// * only work for the CPU plugins +/// * pjrt_cuda.so fails to load +/// * requires a build of zig with the following patch: +/// https://github.com/ziglang/zig/commit/8eb58132eb02c843a54ec7baa6337f54dd024648 +/// this works around https://github.com/ziglang/zig/issues/23152 +/// +/// Potential follow ups: +/// * The main missing is that Bazel takes care of sandboxing +/// ZML executables and the PJRT runtimes and all the dependencies. +/// This part has not been added to build.zig beyond mininal rPath for the the libpjrt_xxx.so, +/// but for cuda you have more depedencies that would need to be sandboxed. +/// +/// * The module graph of the various protobuffers is alson a bit big, +/// and if it changes it will be tedious to update. Ideally bazel should generate this graph of deps for us. pub fn build(b: *std.Build) void { const target = b.standardTargetOptions(.{}); const optimize = b.standardOptimizeOption(.{}); @@ -455,7 +467,7 @@ const Tarball = struct { tar_cmd.addArg(directory); var opts = options; - if (opts.root_source_file != null) @panic("moduleFromBazelSrcs is already setting the root_source_file option"); + if (opts.root_source_file != null) @panic("extractModule is already setting the root_source_file option"); opts.root_source_file = out_dir.path(b, b.pathJoin(&.{ directory, root })); return if (module_name) |name| b.addModule(name, opts) From f6dfda5227a22828cb889d3a8faebefd463cbd05 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Tue, 17 Jun 2025 10:01:14 +0200 Subject: [PATCH 29/40] use std.c.dlopen instead of including it ourselves --- pjrt/BUILD.bazel | 13 +------------ pjrt/dlfcn.h | 1 - pjrt/pjrt.zig | 16 ++++++---------- 3 files changed, 7 insertions(+), 23 deletions(-) delete mode 100644 pjrt/dlfcn.h diff --git a/pjrt/BUILD.bazel b/pjrt/BUILD.bazel index bfba0dda..5e219fa0 100644 --- a/pjrt/BUILD.bazel +++ b/pjrt/BUILD.bazel @@ -6,14 +6,6 @@ load("@zml//bazel:zig.bzl", "zig_cc_binary", "zig_cc_test") load("@zml//bazel:zig_srcs.bzl", "zig_srcs") load("@zml//bazel:zig_proto_library.bzl", "zig_proto_library") -cc_library( - name = "dlfcn", - hdrs = ["dlfcn.h"], - target_compatible_with = [ - "@platforms//os:linux", - ], -) - zig_library( name = "pjrt", srcs = [ @@ -35,10 +27,7 @@ zig_library( "@xla//xla/pjrt/c:pjrt_c_api_hdrs", "@xla//xla/pjrt/c:pjrt_c_api_profiler_extension_hdrs", "@xla//xla/pjrt/c:pjrt_c_api_triton_extension_hdrs", - ] + select({ - "@platforms//os:linux": [":dlfcn"], - "//conditions:default": [], - }), + ], ) zig_cc_test( diff --git a/pjrt/dlfcn.h b/pjrt/dlfcn.h deleted file mode 100644 index 9e446a2b..00000000 --- a/pjrt/dlfcn.h +++ /dev/null @@ -1 +0,0 @@ -#include diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 2b7f48c4..8826098c 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -74,21 +74,17 @@ pub const Api = struct { inner: c.PJRT_Api, - pub fn loadFrom(library: []const u8) !*const Api { + pub fn loadFrom(library: [:0]const u8) !*const Api { var lib: std.DynLib = switch (builtin.os.tag) { .linux => blk: { - const library_c = try std.posix.toPosixPath(library); - break :blk .{ - .inner = .{ - .handle = c.dlopen(&library_c, c.RTLD_LAZY | c.RTLD_LOCAL | c.RTLD_NODELETE) orelse { - log.err("Unable to dlopen plugin: {s}", .{library}); - return error.FileNotFound; - }, - }, + const handle = std.c.dlopen(library, .{ .LAZY = true, .GLOBAL = false, .NODELETE = true }) orelse { + log.err("Unable to dlopen plugin: {s}", .{library}); + return error.FileNotFound; }; + break :blk .{ .inner = .{ .handle = handle } }; }, else => std.DynLib.open(library) catch |err| { - log.warn("Failed to load {s}", .{library}); + log.err("Unable to dlopen plugin: {s}", .{library}); return err; }, }; From fcea3bd7fe6bb1af57a008862dc7a0ae989fd517 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Tue, 17 Jun 2025 10:36:25 +0200 Subject: [PATCH 30/40] rewrite zig_srcs to avoid exposing broken intermediary targets --- bazel/zig_srcs.bzl | 22 +++++++++++++++++----- build.zig | 6 +++--- pjrt/BUILD.bazel | 7 +------ zml/tokenizer/sentencepiece/BUILD.bazel | 7 +------ 4 files changed, 22 insertions(+), 20 deletions(-) diff --git a/bazel/zig_srcs.bzl b/bazel/zig_srcs.bzl index b40bc8ac..2aa79568 100644 --- a/bazel/zig_srcs.bzl +++ b/bazel/zig_srcs.bzl @@ -1,22 +1,34 @@ load("@aspect_bazel_lib//lib:tar.bzl", "mtree_spec", "tar") +load("@rules_zig//zig:defs.bzl", "zig_binary", "BINARY_KIND") -def zig_srcs(name, zig_bin): - """For a given zig_binary, recursivel extract all zig sources into a tarball. +def zig_srcs(name, zig_bin="", zig_lib=""): + """For a given zig_library, recursively extract all zig sources into a tarball. This also includes the files translated from C headers. + It's also possible to pass zig_lib instead of zig_bin in which case, + The rule takes care of creating an intermediary binary from the lib. """ + if zig_bin == "": + zig_bin = "{}_bin".format(name) + zig_binary( + name = zig_bin, + kind = BINARY_KIND.bc, + tags = ["manual", "@rules_zig//zig/lib:libc"], + deps = [zig_lib], + ) + native.filegroup( - name = "{}_zig_srcs".format(name), + name = "{}_files".format(name), srcs = [zig_bin], output_group = "srcs", ) mtree_spec( name = "{}_mtree".format(name), - srcs = [":{}_zig_srcs".format(name)], + srcs = [":{}_files".format(name)], ) tar( name = name, - srcs = ["{}_zig_srcs".format(name)], + srcs = ["{}_files".format(name)], args = [], mtree = "{}_mtree".format(name), ) diff --git a/build.zig b/build.zig index 4944499a..9e018ed3 100644 --- a/build.zig +++ b/build.zig @@ -111,9 +111,9 @@ pub fn build(b: *std.Build) void { // pjrt const pjrt_srcs = Tarball.sources(b, "pjrt"); const pjrt_c_deps = pjrt_srcs.extractModule( + null, "pjrt", - "pjrt", - "test_test_lib_c.zig", + "sources_bin_c.zig", .{ .link_libcpp = true }, ); @@ -224,7 +224,7 @@ pub fn build(b: *std.Build) void { const sentencepiece_c_deps = sentencepiece_c_deps_srcs.extractModule( null, "zml/tokenizer/sentencepiece", - "test_test_lib_c.zig", + "sources_bin_c.zig", .{ .link_libcpp = true }, ); const sentencepiece = zml_srcs.extractModule( diff --git a/pjrt/BUILD.bazel b/pjrt/BUILD.bazel index 5e219fa0..dffc049e 100644 --- a/pjrt/BUILD.bazel +++ b/pjrt/BUILD.bazel @@ -30,14 +30,9 @@ zig_library( ], ) -zig_cc_test( - name = "test", - deps = [":pjrt"], -) - zig_srcs( name = "sources", - zig_bin = ":test_test_lib", + zig_lib = ":pjrt", ) zig_proto_library( diff --git a/zml/tokenizer/sentencepiece/BUILD.bazel b/zml/tokenizer/sentencepiece/BUILD.bazel index 567f76be..729e6078 100644 --- a/zml/tokenizer/sentencepiece/BUILD.bazel +++ b/zml/tokenizer/sentencepiece/BUILD.bazel @@ -24,12 +24,7 @@ zig_library( ], ) -zig_cc_test( - name = "test", - deps = [":sentencepiece"], -) - zig_srcs( name = "sources", - zig_bin = ":test_test_lib", + zig_lib = ":sentencepiece", ) From 622a3aed7a03e2f44ef9a2bd1161b4141fda7f63 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Tue, 17 Jun 2025 10:51:04 +0200 Subject: [PATCH 31/40] remove unused import --- pjrt/BUILD.bazel | 5 +---- zml/tokenizer/sentencepiece/BUILD.bazel | 1 - 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/pjrt/BUILD.bazel b/pjrt/BUILD.bazel index dffc049e..682b569c 100644 --- a/pjrt/BUILD.bazel +++ b/pjrt/BUILD.bazel @@ -1,8 +1,5 @@ -load("@rules_cc//cc:defs.bzl", "cc_library") load("@rules_zig//zig:defs.bzl", "zig_library") -load("@aspect_bazel_lib//lib:tar.bzl", "mtree_spec", "tar") - -load("@zml//bazel:zig.bzl", "zig_cc_binary", "zig_cc_test") +load("@zml//bazel:zig.bzl", "zig_cc_binary") load("@zml//bazel:zig_srcs.bzl", "zig_srcs") load("@zml//bazel:zig_proto_library.bzl", "zig_proto_library") diff --git a/zml/tokenizer/sentencepiece/BUILD.bazel b/zml/tokenizer/sentencepiece/BUILD.bazel index 729e6078..457d811b 100644 --- a/zml/tokenizer/sentencepiece/BUILD.bazel +++ b/zml/tokenizer/sentencepiece/BUILD.bazel @@ -1,7 +1,6 @@ load("@rules_zig//zig:defs.bzl", "zig_library") load("//bazel:swig.bzl", "swig_cc_library") load("//bazel:zig_srcs.bzl", "zig_srcs") -load("//bazel:zig.bzl", "zig_cc_test") swig_cc_library( name = "sentencepiece_swig", From c12e4a57b06c488057c64366da92cd160286adfd Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Tue, 17 Jun 2025 17:22:21 +0200 Subject: [PATCH 32/40] publish bazel files --- build.zig | 3 --- build.zig.zon | 12 ++++++++++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/build.zig b/build.zig index 9e018ed3..8ab9c7c5 100644 --- a/build.zig +++ b/build.zig @@ -375,9 +375,6 @@ pub fn build(b: *std.Build) void { } // zml - const zml_srcs_tar = b.addSystemCommand(&.{ "bazel", "build", "//zml:sources" }); - if (platforms.cpu) zml_srcs_tar.addArg("--@zml//runtimes:cpu=true"); - if (platforms.cuda) zml_srcs_tar.addArg("--@zml//runtimes:cuda=true"); const zml = zml_srcs.extractModule( "zml", "zml", diff --git a/build.zig.zon b/build.zig.zon index f4456fff..4bad5174 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -45,5 +45,17 @@ "zml", "build.zig", "build.zig.zon", + // bazel files + "BUILD.bazel", + "MODULE.bazel", + "MODULE.bazel.lock", + // symlinks + ".bazelrc", + ".bazelrc-zml", + "examples/.bazelrc", + ".bazelignore", + "examples/.bazelignore", + ".bazelversion", + "examples/.bazelversion", }, } From e6ec751e6e478c64113169c28aaed278d6ce13e9 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Fri, 20 Jun 2025 11:13:27 +0200 Subject: [PATCH 33/40] add "zml_sandbox" fixup! add "zml_sandbox" --- bazel/zig_srcs.bzl | 4 ++++ build.zig | 50 +++++++++++++++++++++------------------------- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/bazel/zig_srcs.bzl b/bazel/zig_srcs.bzl index 2aa79568..4fcb5698 100644 --- a/bazel/zig_srcs.bzl +++ b/bazel/zig_srcs.bzl @@ -8,6 +8,10 @@ def zig_srcs(name, zig_bin="", zig_lib=""): It's also possible to pass zig_lib instead of zig_bin in which case, The rule takes care of creating an intermediary binary from the lib. """ + # TODO: this forces to build the test target which isn't needed. + # The problem is we need the binary target to get the translate-c output. + # What is missing from rules_zig is a "translate_c" target that we could call directly + # without needing to build the corresponding binary. if zig_bin == "": zig_bin = "{}_bin".format(name) zig_binary( diff --git a/build.zig b/build.zig index 8ab9c7c5..9b41ab26 100644 --- a/build.zig +++ b/build.zig @@ -131,36 +131,33 @@ pub fn build(b: *std.Build) void { const run_pjrt_tests = b.addRunArtifact(pjrt_test); test_step.dependOn(&run_pjrt_tests.step); - var _d: std.BoundedArray(*std.Build.Step.InstallFile, 8) = .{}; - deps: { - switch (target.result.os.tag) { - .macos => { - if (platforms.cpu) { - const dep = b.lazyDependency("pjrt_cpu_darwin_arm64", .{}) orelse break :deps; - const pjrt_cpu = b.addInstallLibFile(dep.path("libpjrt_cpu.dylib"), "libpjrt_cpu.dylib"); - _d.appendAssumeCapacity(pjrt_cpu); + const zml_sandbox = b.addNamedWriteFiles("zml_sandbox"); + switch (target.result.os.tag) { + .macos => { + if (platforms.cpu) { + if (b.lazyDependency("pjrt_cpu_darwin_arm64", .{})) |dep| { + _ = zml_sandbox.addCopyFile(dep.path("libpjrt_cpu.dylib"), "libpjrt_cpu.dylib"); } - }, - .linux => { - if (platforms.cpu) { - const dep = b.lazyDependency("pjrt_cpu_linux_amd64", .{}) orelse break :deps; - const pjrt_cpu = b.addInstallLibFile(dep.path("libpjrt_cpu.so"), "libpjrt_cpu.so"); - _d.appendAssumeCapacity(pjrt_cpu); + } + }, + .linux => { + if (platforms.cpu) { + if (b.lazyDependency("pjrt_cpu_linux_amd64", .{})) |dep| { + _ = zml_sandbox.addCopyFile(dep.path("libpjrt_cpu.so"), "libpjrt_cpu.so"); } + } - if (platforms.cuda) { - // TODO: this is not enough for libpjrt_cuda, there are a lot more deps needed. - const zmlxcuda = objectFromBazel(b, "@libpjrt_cuda//:zmlxcuda_so", "external/+cuda_packages+libpjrt_cuda/libzmlxcuda.so.0"); - const plugin = objectFromBazel(b, "//runtimes/cuda:libpjrt_cuda", "runtimes/cuda/libpjrt_cuda.so"); + if (platforms.cuda) { + // TODO: this is not enough for libpjrt_cuda, there are a lot more deps needed. + const zmlxcuda = objectFromBazel(b, "@libpjrt_cuda//:zmlxcuda_so", "external/+cuda_packages+libpjrt_cuda/libzmlxcuda.so.0"); + _ = zml_sandbox.addCopyFile(zmlxcuda, "libzmlxcuda.so.0"); - _d.appendAssumeCapacity(b.addInstallLibFile(zmlxcuda, "libzmlxcuda.so.0")); - _d.appendAssumeCapacity(b.addInstallLibFile(plugin, "libpjrt_cuda.so")); - } - }, - else => |os| std.debug.panic("Target not supported: {s}", .{@tagName(os)}), - } + const plugin = objectFromBazel(b, "//runtimes/cuda:libpjrt_cuda", "runtimes/cuda/libpjrt_cuda.so"); + _ = zml_sandbox.addCopyFile(plugin, "libpjrt_cuda.so"); + } + }, + else => |os| std.debug.panic("Target not supported: {s}", .{@tagName(os)}), } - const pjrt_dynamic_deps = _d.constSlice(); // xev const async_srcs = Tarball.sources(b, "async"); @@ -404,10 +401,9 @@ pub fn build(b: *std.Build) void { .test_runner = .{ .mode = .simple, .path = b.path("zml/test_runner.zig") }, }); // This is where we will put all the .so needed. - zml_test.addRPath(.{ .cwd_relative = b.lib_dir }); + zml_test.addRPath(zml_sandbox.getDirectory()); const run_zml_tests = b.addRunArtifact(zml_test); - addRuntimeDeps(run_zml_tests, pjrt_dynamic_deps); const zml_test_step = b.step("test-zml", "Run ZML tests (assumes pjrt.dylib are in the path)"); zml_test_step.dependOn(&run_zml_tests.step); From dfd92ddf8aabcf413693eeb0ef9738d355be2584 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Fri, 20 Jun 2025 11:16:10 +0200 Subject: [PATCH 34/40] examples/mnist/build.zig --- examples/mnist/build.zig | 45 ++++++++++++++++++++++++++++++++++++ examples/mnist/build.zig.zon | 25 ++++++++++++++++++++ 2 files changed, 70 insertions(+) create mode 100644 examples/mnist/build.zig create mode 100644 examples/mnist/build.zig.zon diff --git a/examples/mnist/build.zig b/examples/mnist/build.zig new file mode 100644 index 00000000..f53c8c08 --- /dev/null +++ b/examples/mnist/build.zig @@ -0,0 +1,45 @@ +const std = @import("std"); + +pub fn build(b: *std.Build) void { + const target = b.standardTargetOptions(.{}); + const optimize = b.standardOptimizeOption(.{}); + + // Note: setting -Dcuda on CLI fails cause zml_sandbox doesn't yet contains all CUDA deps. + const cuda = b.option(bool, "cuda", "Enable cuda (broken)") orelse false; + const zml_pkg = b.dependency("zml", .{ + .@"runtimes:cuda" = cuda, + .@"runtimes:cpu" = !cuda, + }); + const zml_sandbox = zml_pkg.namedWriteFiles("zml_sandbox"); + + const mnist = b.createModule(.{ + .root_source_file = b.path("mnist.zig"), + .target = target, + .optimize = optimize, + .imports = &.{ + .{ .name = "async", .module = zml_pkg.module("async") }, + .{ .name = "zml", .module = zml_pkg.module("zml") }, + }, + }); + + const exe = b.addExecutable(.{ .name = "mnist", .root_module = mnist }); + exe.addRPath(zml_sandbox.getDirectory()); + b.installArtifact(exe); + + const run_cmd = b.addRunArtifact(exe); + { + const ggml = b.dependency("ggml", .{}); + const mnist_model = ggml.path("examples/mnist/models/mnist/mnist_model.state_dict"); + const mnist_data = ggml.path("examples/mnist/models/mnist/t10k-images.idx3-ubyte"); + + run_cmd.addFileArg(mnist_model); + run_cmd.addFileArg(mnist_data); + run_cmd.step.dependOn(b.getInstallStep()); + if (b.args) |args| { + run_cmd.addArgs(args); + } + } + + const run_step = b.step("run", "Run the app"); + run_step.dependOn(&run_cmd.step); +} diff --git a/examples/mnist/build.zig.zon b/examples/mnist/build.zig.zon new file mode 100644 index 00000000..a2dd4cc5 --- /dev/null +++ b/examples/mnist/build.zig.zon @@ -0,0 +1,25 @@ +.{ + .name = .mnist, + .version = "0.0.0", + + .fingerprint = 0x6805ab6e16b3f141, // Changing this has security and trust implications. + + .minimum_zig_version = "0.14.0", + .dependencies = .{ + .zml = .{ + .url = "git+https://github.com/zml/zml#e6ec751e6e478c64113169c28aaed278d6ce13e9", + .hash = "zml-0.0.0-N9uPKHO8IgABeyC0QM0tqVA4wyL4lVr8vwCvGEQgrDqU", + }, + // Get MNIST model and data from GGML repo. + .ggml = .{ + .url = "git+https://github.com/ggerganov/ggml#18703ad600cc68dbdb04d57434c876989a841d12", + .hash = "N-V-__8AAGUBBAESVQ00NaQbKKLu9KDR54aQyFBxUmYvJLF4", + }, + }, + + .paths = .{ + "build.zig", + "build.zig.zon", + "mnist.zig", + }, +} From aa8348d9602a19c5703a4cc3e1f1e4ce7ace8ef8 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Mon, 17 Mar 2025 18:55:48 +0100 Subject: [PATCH 35/40] asyncio: remove usingnamespace --- async/asyncio.zig | 58 ++++++++++++++++++++++++++--------------------- 1 file changed, 32 insertions(+), 26 deletions(-) diff --git a/async/asyncio.zig b/async/asyncio.zig index 1f41f90a..7fe05903 100644 --- a/async/asyncio.zig +++ b/async/asyncio.zig @@ -115,10 +115,10 @@ pub const TCP = struct { exec: *Executor, tcp: xev.TCP, - pub usingnamespace Closeable(Self, xev.TCP); - pub usingnamespace Pollable(Self, xev.TCP); - pub usingnamespace Readable(Self, xev.TCP); - pub usingnamespace Writeable(Self, xev.TCP); + pub const close = Closeable(Self, xev.TCP); + pub const poll = Pollable(Self, xev.TCP); + pub const read = Readable(Self, xev.TCP); + pub const write = Writeable(Self, xev.TCP); pub fn init(exec: *Executor, tcp: xev.TCP) Self { return .{ .exec = exec, .tcp = tcp }; @@ -214,11 +214,20 @@ pub const TCP = struct { } }; -fn Closeable(comptime T: type, comptime StreamT: type) type { +pub const AsyncError = error{ + AccessDenied, + EventNotFound, + SystemResources, + ProcessNotFound, + Overflow, + NestedRunsNotAllowed, +}; + +fn Closeable(comptime T: type, comptime StreamT: type) fn (T) anyerror!void { return struct { const Self = T; const CloseResult = xev.CloseError!void; - pub fn close(self: Self) !void { + pub fn close(self: Self) anyerror!void { const ResultT = CloseResult; const Data = struct { result: ResultT = undefined, @@ -251,14 +260,13 @@ fn Closeable(comptime T: type, comptime StreamT: type) type { return data.result; } - }; + }.close; } -fn Pollable(comptime T: type, comptime StreamT: type) type { +fn Pollable(comptime T: type, comptime StreamT: type) fn (T) anyerror!void { return struct { - const Self = T; const PollResult = xev.PollError!void; - pub fn poll(self: Self) !void { + pub fn poll(self: T) anyerror!void { const ResultT = PollResult; const Data = struct { result: ResultT = undefined, @@ -291,14 +299,14 @@ fn Pollable(comptime T: type, comptime StreamT: type) type { return data.result; } - }; + }.poll; } -fn Readable(comptime T: type, comptime StreamT: type) type { +fn Readable(comptime T: type, comptime StreamT: type) fn (T, xev.ReadBuffer) anyerror!usize { return struct { const Self = T; const ReadResult = xev.ReadError!usize; - pub fn read(self: Self, buf: xev.ReadBuffer) !usize { + pub fn read(self: Self, buf: xev.ReadBuffer) anyerror!usize { const ResultT = ReadResult; const Data = struct { result: ResultT = undefined, @@ -333,10 +341,10 @@ fn Readable(comptime T: type, comptime StreamT: type) type { return data.result; } - }; + }.read; } -fn Writeable(comptime T: type, comptime StreamT: type) type { +fn Writeable(comptime T: type, comptime StreamT: type) fn (T, xev.WriteBuffer) anyerror!usize { return struct { const Self = T; const WriteResult = xev.WriteError!usize; @@ -374,7 +382,7 @@ fn Writeable(comptime T: type, comptime StreamT: type) type { try waitForCompletion(self.exec, &c); return data.result; } - }; + }.write; } pub const File = struct { @@ -383,10 +391,10 @@ pub const File = struct { exec: *Executor, file: xev.File, - pub usingnamespace Closeable(Self, xev.File); - pub usingnamespace Pollable(Self, xev.File); - pub usingnamespace Readable(Self, xev.File); - pub usingnamespace Writeable(Self, xev.File); + pub const close = Closeable(Self, xev.File); + pub const poll = Pollable(Self, xev.File); + pub const read = Readable(Self, xev.File); + pub const write = Writeable(Self, xev.File); pub fn init(exec: *Executor, file: xev.File) Self { return .{ .exec = exec, .file = file }; @@ -527,10 +535,8 @@ pub const UDP = struct { exec: *Executor, udp: xev.UDP, - pub usingnamespace Closeable(Self, xev.TCP); - pub usingnamespace Pollable(Self, xev.TCP); - pub usingnamespace Readable(Self, xev.TCP); - pub usingnamespace Writeable(Self, xev.TCP); + pub const close = Closeable(Self, xev.UDP); + pub const poll = Pollable(Self, xev.UDP); pub fn init(exec: *Executor, udp: xev.UDP) Self { return .{ .exec = exec, .udp = udp }; @@ -541,7 +547,7 @@ pub const UDP = struct { } const ReadResult = xev.ReadError!usize; - pub fn read(self: Self, buf: xev.ReadBuffer) !usize { + pub fn read(self: Self, buf: xev.ReadBuffer) anyerror!usize { const ResultT = ReadResult; const Data = struct { result: ResultT = undefined, @@ -582,7 +588,7 @@ pub const UDP = struct { } const WriteResult = xev.WriteError!usize; - pub fn write(self: Self, addr: std.net.Address, buf: xev.WriteBuffer) !usize { + pub fn write(self: Self, addr: std.net.Address, buf: xev.WriteBuffer) anyerror!usize { const ResultT = WriteResult; const Data = struct { result: ResultT = undefined, From 1602667a3c6d86e0f77a6795df0278b645e6f85d Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Tue, 15 Jul 2025 17:59:50 +0200 Subject: [PATCH 36/40] move protobuf virtual imports --- build.zig | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/build.zig b/build.zig index 9b41ab26..fc695752 100644 --- a/build.zig +++ b/build.zig @@ -264,7 +264,8 @@ pub fn build(b: *std.Build) void { // proto const protobuf = zml_srcs.extractModule("protobuf", "src", "protobuf.zig", .{}); const xla_compile_proto = xla_proto: { - // Ideally we should ask bazel to generate this graph of dependencies between the different proto files. + // Ideally we should ask bazel to generate this graph of dependencies between the different proto files, + // Or change altogether how protobuf are generated and only create one zig module for all protobuf stuff. const empty: *std.Build.Module = empty: { const write = b.addWriteFiles(); const root = write.add("root.zig", "//! empty module"); @@ -272,9 +273,9 @@ pub fn build(b: *std.Build) void { }; const opts: std.Build.Module.CreateOptions = .{ .imports = &.{.{ .name = "protobuf", .module = protobuf }} }; - const duration = zml_srcs.extractModule(null, "_virtual_imports/duration_proto/google/protobuf", "duration.pb.zig", opts); - const any = zml_srcs.extractModule(null, "_virtual_imports/any_proto/google/protobuf", "any.pb.zig", opts); - const wrappers = zml_srcs.extractModule(null, "_virtual_imports/wrappers_proto/google/protobuf", "wrappers.pb.zig", opts); + const duration = zml_srcs.extractModule(null, googleProtobuf("duration"), "duration.pb.zig", opts); + const any = zml_srcs.extractModule(null, googleProtobuf("any"), "any.pb.zig", opts); + const wrappers = zml_srcs.extractModule(null, googleProtobuf("wrappers"), "wrappers.pb.zig", opts); const data = zml_srcs.extractModule(null, "xla", "xla_data.pb.zig", opts); const service_hlo = zml_srcs.extractModule(null, "xla/service", "hlo.pb.zig", opts); @@ -475,6 +476,10 @@ fn addRuntimeDeps(run_step: *std.Build.Step.Run, libs: []const *std.Build.Step.I } } +fn googleProtobuf(comptime name: []const u8) []const u8 { + return "src/google/protobuf/_virtual_imports/" ++ name ++ "_proto/google/protobuf"; +} + const Platforms = struct { cpu: bool, cuda: bool, From df95a3242036fb1d8b84cde4489a6ffc7db147b7 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Wed, 16 Jul 2025 11:32:53 +0200 Subject: [PATCH 37/40] remove usingnamespace from libxev and zig-protobuf relevant PRs: libxev: https://github.com/mitchellh/libxev/commit/7f803181b158a10fec8619f793e3b4df515566cb zig-protobuf: https://github.com/zml/zig-protobuf/pull/1/files Signed-off-by: Guillaume Wenzek --- MODULE.bazel | 4 +-- .../libxev/20250716.0-7f80318/MODULE.bazel | 7 ++++ .../20250716.0-7f80318/overlay/BUILD.bazel | 13 ++++++++ .../20250716.0-7f80318/overlay/MODULE.bazel | 7 ++++ .../libxev/20250716.0-7f80318/source.json | 10 ++++++ third_party/modules/libxev/metadata.json | 1 + .../20250716.0-97f1e31/MODULE.bazel | 8 +++++ .../20250716.0-97f1e31/overlay/BUILD.bazel | 32 +++++++++++++++++++ .../20250716.0-97f1e31/overlay/MODULE.bazel | 8 +++++ .../20250716.0-97f1e31/source.json | 9 ++++++ .../modules/zig-protobuf/metadata.json | 5 +-- zml/module.zig | 20 ++++++------ 12 files changed, 109 insertions(+), 15 deletions(-) create mode 100644 third_party/modules/libxev/20250716.0-7f80318/MODULE.bazel create mode 100644 third_party/modules/libxev/20250716.0-7f80318/overlay/BUILD.bazel create mode 100644 third_party/modules/libxev/20250716.0-7f80318/overlay/MODULE.bazel create mode 100644 third_party/modules/libxev/20250716.0-7f80318/source.json create mode 100644 third_party/modules/zig-protobuf/20250716.0-97f1e31/MODULE.bazel create mode 100644 third_party/modules/zig-protobuf/20250716.0-97f1e31/overlay/BUILD.bazel create mode 100644 third_party/modules/zig-protobuf/20250716.0-97f1e31/overlay/MODULE.bazel create mode 100644 third_party/modules/zig-protobuf/20250716.0-97f1e31/source.json diff --git a/MODULE.bazel b/MODULE.bazel index 45fb05de..17a68ac6 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -6,7 +6,7 @@ bazel_dep(name = "abseil-cpp", version = "20240722.0.bcr.2") bazel_dep(name = "aspect_bazel_lib", version = "2.14.0") bazel_dep(name = "aspect_rules_py", version = "1.3.2") bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "libxev", version = "20250313.0-5773f46") +bazel_dep(name = "libxev", version = "20250716.0-7f80318") bazel_dep(name = "patchelf", version = "0.18.0") bazel_dep(name = "pcre2", version = "10.43") bazel_dep(name = "platforms", version = "0.0.11") @@ -23,7 +23,7 @@ bazel_dep(name = "toolchains_llvm_bootstrapped", version = "0.2.3") bazel_dep(name = "toolchains_protoc", version = "0.4.1") bazel_dep(name = "with_cfg.bzl", version = "0.9.1") bazel_dep(name = "xla", version = "20250710.0-22ea002") -bazel_dep(name = "zig-protobuf", version = "20250318.0-930153e") +bazel_dep(name = "zig-protobuf", version = "20250716.0-97f1e31") bazel_dep(name = "zig-yaml", version = "20240903.0-83d5fdf") bazel_dep(name = "buildifier_prebuilt", version = "8.0.3", dev_dependency = True) diff --git a/third_party/modules/libxev/20250716.0-7f80318/MODULE.bazel b/third_party/modules/libxev/20250716.0-7f80318/MODULE.bazel new file mode 100644 index 00000000..62de098a --- /dev/null +++ b/third_party/modules/libxev/20250716.0-7f80318/MODULE.bazel @@ -0,0 +1,7 @@ +module( + name = "libxev", + version = "20250716.0-7f80318", + compatibility_level = 1, +) + +bazel_dep(name = "rules_zig", version = "20250613.0-567662a") diff --git a/third_party/modules/libxev/20250716.0-7f80318/overlay/BUILD.bazel b/third_party/modules/libxev/20250716.0-7f80318/overlay/BUILD.bazel new file mode 100644 index 00000000..872ef7fb --- /dev/null +++ b/third_party/modules/libxev/20250716.0-7f80318/overlay/BUILD.bazel @@ -0,0 +1,13 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "xev", + srcs = glob([ + "src/*.zig", + "src/backend/*.zig", + "src/linux/*.zig", + "src/watcher/*.zig", + ]), + main = "src/main.zig", + visibility = ["//visibility:public"], +) diff --git a/third_party/modules/libxev/20250716.0-7f80318/overlay/MODULE.bazel b/third_party/modules/libxev/20250716.0-7f80318/overlay/MODULE.bazel new file mode 100644 index 00000000..62de098a --- /dev/null +++ b/third_party/modules/libxev/20250716.0-7f80318/overlay/MODULE.bazel @@ -0,0 +1,7 @@ +module( + name = "libxev", + version = "20250716.0-7f80318", + compatibility_level = 1, +) + +bazel_dep(name = "rules_zig", version = "20250613.0-567662a") diff --git a/third_party/modules/libxev/20250716.0-7f80318/source.json b/third_party/modules/libxev/20250716.0-7f80318/source.json new file mode 100644 index 00000000..f578c73c --- /dev/null +++ b/third_party/modules/libxev/20250716.0-7f80318/source.json @@ -0,0 +1,10 @@ +{ + "strip_prefix": "libxev-7f803181b158a10fec8619f793e3b4df515566cb", + "url": "https://github.com/mitchellh/libxev/archive/7f803181b158a10fec8619f793e3b4df515566cb.tar.gz", + "integrity": "sha256-KaozYKEhhT/6sInef7/8O/60LDBJN+8QmdLuNY1Gkmc", + "overlay": { + "MODULE.bazel": "", + "BUILD.bazel": "" + }, + "patch_strip": 1 +} diff --git a/third_party/modules/libxev/metadata.json b/third_party/modules/libxev/metadata.json index 59deb2c9..c897c980 100644 --- a/third_party/modules/libxev/metadata.json +++ b/third_party/modules/libxev/metadata.json @@ -20,6 +20,7 @@ "20250124.0-31eed4e", "20250222.0-07bcffa", "20250313.0-5773f46", + "20250716.0-7f80318", ], "yanked_versions": {} } diff --git a/third_party/modules/zig-protobuf/20250716.0-97f1e31/MODULE.bazel b/third_party/modules/zig-protobuf/20250716.0-97f1e31/MODULE.bazel new file mode 100644 index 00000000..b71798cc --- /dev/null +++ b/third_party/modules/zig-protobuf/20250716.0-97f1e31/MODULE.bazel @@ -0,0 +1,8 @@ +module( + name = "zig-protobuf", + version = "20250716.0-97f1e31", + compatibility_level = 1, +) + +bazel_dep(name = "rules_zig", version = "20250613.0-567662a") +bazel_dep(name = "rules_proto", version = "6.0.0-rc1") diff --git a/third_party/modules/zig-protobuf/20250716.0-97f1e31/overlay/BUILD.bazel b/third_party/modules/zig-protobuf/20250716.0-97f1e31/overlay/BUILD.bazel new file mode 100644 index 00000000..832c7591 --- /dev/null +++ b/third_party/modules/zig-protobuf/20250716.0-97f1e31/overlay/BUILD.bazel @@ -0,0 +1,32 @@ +load("@rules_proto//proto:defs.bzl", "proto_lang_toolchain") +load("@rules_zig//zig:defs.bzl", "BINARY_KIND", "zig_binary", "zig_library") + +zig_library( + name = "protobuf", + import_name = "protobuf", + main = "src/protobuf.zig", + visibility = ["//visibility:public"], +) + +zig_binary( + name = "generator", + srcs = [ + "bootstrapped-generator/FullName.zig", + "bootstrapped-generator/google/protobuf/compiler/plugin.pb.zig", + "bootstrapped-generator/google/protobuf/descriptor.pb.zig", + ], + kind = BINARY_KIND.exe, + main = "bootstrapped-generator/main.zig", + visibility = ["//visibility:public"], + deps = [":protobuf"], +) + +proto_lang_toolchain( + name = "zig_toolchain", + command_line = "--zig_out=$(OUT)", + output_files = "multiple", + plugin = ":generator", + plugin_format_flag = "--plugin=protoc-gen-zig=%s", + runtime = ":protobuf", + visibility = ["//visibility:public"], +) diff --git a/third_party/modules/zig-protobuf/20250716.0-97f1e31/overlay/MODULE.bazel b/third_party/modules/zig-protobuf/20250716.0-97f1e31/overlay/MODULE.bazel new file mode 100644 index 00000000..b71798cc --- /dev/null +++ b/third_party/modules/zig-protobuf/20250716.0-97f1e31/overlay/MODULE.bazel @@ -0,0 +1,8 @@ +module( + name = "zig-protobuf", + version = "20250716.0-97f1e31", + compatibility_level = 1, +) + +bazel_dep(name = "rules_zig", version = "20250613.0-567662a") +bazel_dep(name = "rules_proto", version = "6.0.0-rc1") diff --git a/third_party/modules/zig-protobuf/20250716.0-97f1e31/source.json b/third_party/modules/zig-protobuf/20250716.0-97f1e31/source.json new file mode 100644 index 00000000..4975cdb8 --- /dev/null +++ b/third_party/modules/zig-protobuf/20250716.0-97f1e31/source.json @@ -0,0 +1,9 @@ +{ + "strip_prefix": "zig-protobuf-97f1e314d9bbf600fbaadee5bb08c8856e0006de", + "url": "https://github.com/zml/zig-protobuf/archive/97f1e314d9bbf600fbaadee5bb08c8856e0006de.tar.gz", + "integrity": "sha256-iAqsLIKhl15JInJvwMY8NGVll6eUBXzdLW6Xv8wANGY=", + "overlay": { + "MODULE.bazel": "", + "BUILD.bazel": "" + } +} diff --git a/third_party/modules/zig-protobuf/metadata.json b/third_party/modules/zig-protobuf/metadata.json index 87b4f2ba..4a038360 100644 --- a/third_party/modules/zig-protobuf/metadata.json +++ b/third_party/modules/zig-protobuf/metadata.json @@ -8,12 +8,13 @@ } ], "repository": [ - "gwenzek/zig-protobuf" + "zml/zig-protobuf" ], "versions": [ "20240722.0-c644d11", "20250213.0-5304067", - "20250318.0-930153e" + "20250318.0-930153e", + "20250716.0-97f1e31", ], "yanked_versions": {} } diff --git a/zml/module.zig b/zml/module.zig index c6824b4a..b8d4f21f 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -841,8 +841,6 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, platform: Platform, m const sharding = platform.sharding(); - // NOTE(Corendos): Hack needed because Protobuf struct are not public. - const DeviceAssignmentProto = @TypeOf(xla_pb.CompileOptionsProto.init().executable_build_options.?.device_assignment.?); var options: xla_pb.CompileOptionsProto = .{ .executable_build_options = .{ .device_ordinal = -1, @@ -852,18 +850,18 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, platform: Platform, m .device_assignment = .{ .replica_count = sharding.num_replicas, .computation_count = sharding.num_partitions, - .computation_devices = blk: { - var computation_devices = try std.ArrayListUnmanaged(DeviceAssignmentProto.ComputationDevice).initCapacity(arena, sharding.num_partitions); - for (0..sharding.num_partitions) |i| { - var replica_device_ids = std.ArrayListUnmanaged(i64).initCapacity(arena, 1) catch unreachable; - replica_device_ids.appendAssumeCapacity(@intCast(i)); - computation_devices.appendAssumeCapacity(.{ .replica_device_ids = replica_device_ids }); - } - break :blk computation_devices; - }, + // Filled below. + .computation_devices = .{}, }, }, }; + const computation_devices = &options.executable_build_options.?.device_assignment.?.computation_devices; + try computation_devices.ensureTotalCapacity(arena, sharding.num_partitions); + const replica_device_ids = try arena.alloc(i64, sharding.num_partitions); + for (0..sharding.num_partitions) |i| { + replica_device_ids[i] = @intCast(i); + computation_devices.appendAssumeCapacity(.{ .replica_device_ids = .fromOwnedSlice(replica_device_ids[i .. i + 1]) }); + } // Let the arena deinit, zig-protobuf deinit is very slow. try options.env_option_overrides.ensureUnusedCapacity(arena, 16); From 60f31700b6187ef6416a591d0cc69250fd4b21a7 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 17 Jul 2025 10:21:19 +0200 Subject: [PATCH 38/40] -Dzml_deps_tar allow to pass pregenerated tarball to avoid calling `bazel //zml:sources` This help when upgrading Zig versions and ZML code is not fully ported yet, but we want to use build.zig --- build.zig | 89 ++++++++++++++++++++++++++++--------------------------- 1 file changed, 45 insertions(+), 44 deletions(-) diff --git a/build.zig b/build.zig index fc695752..7777cb05 100644 --- a/build.zig +++ b/build.zig @@ -177,11 +177,10 @@ pub fn build(b: *std.Build) void { test_step.dependOn(&run_xev_tests.step); // async - const async_mod = async_srcs.extractModule( + const async_mod = b.addModule( "async", - "async", - "async.zig", .{ + .root_source_file = b.path("async/async.zig"), .target = target, .optimize = optimize, .imports = &.{ @@ -195,9 +194,9 @@ pub fn build(b: *std.Build) void { const run_async_tests = b.addRunArtifact(async_test); test_step.dependOn(&run_async_tests.step); - const zml_srcs = Tarball.zml(b, platforms); + const zml_deps = Tarball.zmlDepsTarball(b, platforms); // ffi - const ffi = zml_srcs.extractModule( + const ffi = zml_deps.extractModule( "ffi", "ffi", "ffi.zig", @@ -209,7 +208,7 @@ pub fn build(b: *std.Build) void { test_step.dependOn(&run_ffi_tests.step); // hftokenizers - const hftokenizers = zml_srcs.extractModule( + const hftokenizers = zml_deps.extractModule( "//zml/tokenizer/hftokenizers", "zml/tokenizer/hftokenizers", "hftokenizers.zig", @@ -224,7 +223,7 @@ pub fn build(b: *std.Build) void { "sources_bin_c.zig", .{ .link_libcpp = true }, ); - const sentencepiece = zml_srcs.extractModule( + const sentencepiece = zml_deps.extractModule( "//zml/tokenizer/sentencepiece", "zml/tokenizer/sentencepiece", "sentencepiece.zig", @@ -238,15 +237,13 @@ pub fn build(b: *std.Build) void { ); // tokenizer - const tokenizer = zml_srcs.extractModule( + const tokenizer = b.addModule( "tokenizer", - "zml/tokenizer", - "tokenizer.zig", .{ + .root_source_file = b.path("zml/tokenizer/tokenizer.zig"), .target = target, .optimize = optimize, .imports = &.{ - .{ .name = "async", .module = async_mod }, .{ .name = "ffi", .module = ffi }, // Note: even though we import sentencepiece and hftokenizers, // we don't link the corresponding .a cause there is no tests @@ -262,7 +259,7 @@ pub fn build(b: *std.Build) void { test_step.dependOn(&run_tokenizer_tests.step); // proto - const protobuf = zml_srcs.extractModule("protobuf", "src", "protobuf.zig", .{}); + const protobuf = zml_deps.extractModule("protobuf", "src", "protobuf.zig", .{}); const xla_compile_proto = xla_proto: { // Ideally we should ask bazel to generate this graph of dependencies between the different proto files, // Or change altogether how protobuf are generated and only create one zig module for all protobuf stuff. @@ -273,33 +270,33 @@ pub fn build(b: *std.Build) void { }; const opts: std.Build.Module.CreateOptions = .{ .imports = &.{.{ .name = "protobuf", .module = protobuf }} }; - const duration = zml_srcs.extractModule(null, googleProtobuf("duration"), "duration.pb.zig", opts); - const any = zml_srcs.extractModule(null, googleProtobuf("any"), "any.pb.zig", opts); - const wrappers = zml_srcs.extractModule(null, googleProtobuf("wrappers"), "wrappers.pb.zig", opts); + const duration = zml_deps.extractModule(null, googleProtobuf("duration"), "duration.pb.zig", opts); + const any = zml_deps.extractModule(null, googleProtobuf("any"), "any.pb.zig", opts); + const wrappers = zml_deps.extractModule(null, googleProtobuf("wrappers"), "wrappers.pb.zig", opts); - const data = zml_srcs.extractModule(null, "xla", "xla_data.pb.zig", opts); - const service_hlo = zml_srcs.extractModule(null, "xla/service", "hlo.pb.zig", opts); - const xla = zml_srcs.extractModule(null, "xla", "xla.pb.zig", opts); + const data = zml_deps.extractModule(null, "xla", "xla_data.pb.zig", opts); + const service_hlo = zml_deps.extractModule(null, "xla/service", "hlo.pb.zig", opts); + const xla = zml_deps.extractModule(null, "xla", "xla.pb.zig", opts); xla.addImport("google_protobuf_any_proto", any); xla.addImport("xla_xla_data_proto", data); xla.addImport("xla_service_hlo_proto", service_hlo); xla.addImport("xla_autotune_results_proto", empty); - const tsl_dnn = zml_srcs.extractModule(null, "xla/tsl/protobuf", "dnn.pb.zig", opts); + const tsl_dnn = zml_deps.extractModule(null, "xla/tsl/protobuf", "dnn.pb.zig", opts); tsl_dnn.addImport("google_protobuf_wrappers_proto", wrappers); - const autotuning = zml_srcs.extractModule(null, "xla", "autotuning.pb.zig", opts); + const autotuning = zml_deps.extractModule(null, "xla", "autotuning.pb.zig", opts); autotuning.addImport("google_protobuf_duration_proto", duration); autotuning.addImport("xla_tsl_protobuf_dnn_proto", tsl_dnn); - const autotune = zml_srcs.extractModule(null, "xla", "autotune_results.pb.zig", opts); + const autotune = zml_deps.extractModule(null, "xla", "autotune_results.pb.zig", opts); autotune.addImport("xla_autotuning_proto", autotuning); - const stream_executor_cuda_cuda_compute_capability = zml_srcs.extractModule(null, "xla/stream_executor/cuda", "cuda_compute_capability.pb.zig", opts); - const stream_executor_device_description = zml_srcs.extractModule(null, "xla/stream_executor", "device_description.pb.zig", opts); + const stream_executor_cuda_cuda_compute_capability = zml_deps.extractModule(null, "xla/stream_executor/cuda", "cuda_compute_capability.pb.zig", opts); + const stream_executor_device_description = zml_deps.extractModule(null, "xla/stream_executor", "device_description.pb.zig", opts); stream_executor_device_description.addImport("xla_autotune_results_proto", autotune); stream_executor_device_description.addImport("xla_stream_executor_cuda_cuda_compute_capability_proto", stream_executor_cuda_cuda_compute_capability); - const compile = zml_srcs.extractModule(null, "xla/pjrt/proto", "compile_options.pb.zig", opts); + const compile = zml_deps.extractModule(null, "xla/pjrt/proto", "compile_options.pb.zig", opts); compile.addImport("xla_xla_proto", xla); compile.addImport("xla_xla_data_proto", data); compile.addImport("xla_stream_executor_device_description_proto", stream_executor_device_description); @@ -308,7 +305,7 @@ pub fn build(b: *std.Build) void { }; // zml c deps - const zml_c_deps = zml_srcs.extractModule( + const zml_c_deps = zml_deps.extractModule( null, "zml", "test_test_lib_c.zig", @@ -316,10 +313,11 @@ pub fn build(b: *std.Build) void { ); // bazel runfiles - const runfiles = zml_srcs.extractModule("runfiles", "zig/runfiles", "runfiles.zig", .{}); + // TODO: should we also generate a "runfiles" folder to provide the expected runtime information ? + const runfiles = zml_deps.extractModule("runfiles", "zig/runfiles", "runfiles.zig", .{}); // runtimes - const runtimes = zml_srcs.extractModule( + const runtimes = zml_deps.extractModule( "runtimes", "runtimes", "runtimes.zig", @@ -336,13 +334,12 @@ pub fn build(b: *std.Build) void { const runtime_path = b.pathJoin(&.{ "runtimes", platform }); const zig_file = std.mem.concat(b.allocator, u8, &.{ platform, ".zig" }) catch @panic("OOM"); const builtin_name = std.fmt.allocPrint(b.allocator, "bazel_builtin_A_S_Sruntimes_S{0s}_C{0s}.zig", .{platform}) catch @panic("OOM"); - const bazel_builtin = zml_srcs.extractModule(null, runtime_path, builtin_name, .{}); + const bazel_builtin = zml_deps.extractModule(null, runtime_path, builtin_name, .{}); - const platform_runtime = zml_srcs.extractModule( + const platform_runtime = b.addModule( runtime_path, - runtime_path, - zig_file, .{ + .root_source_file = b.path(b.pathJoin(&.{ runtime_path, zig_file })), .imports = &.{ .{ .name = "c", .module = zml_c_deps }, .{ .name = "runfiles", .module = runfiles }, @@ -358,11 +355,9 @@ pub fn build(b: *std.Build) void { } // zml/tools - const zml_tools = zml_srcs.extractModule( - "zml/tools", - "zml/tools", - "tools.zig", + const zml_tools = b.createModule( .{ + .root_source_file = b.path("zml/tools/tools.zig"), .link_libcpp = target.result.os.tag == .macos, .imports = &.{.{ .name = "c", .module = zml_c_deps }}, }, @@ -373,11 +368,10 @@ pub fn build(b: *std.Build) void { } // zml - const zml = zml_srcs.extractModule( + const zml = b.addModule( "zml", - "zml", - "zml.zig", .{ + .root_source_file = b.path("zml/zml.zig"), .target = target, .optimize = optimize, .link_libcpp = true, @@ -442,13 +436,20 @@ const Tarball = struct { /// Zml is a bit different from other targets cause the code of the "runtimes/**.zig" /// will change based on the platforms selected. - pub fn zml(b: *std.Build, platforms: Platforms) Tarball { - const bazel_cmd: *std.Build.Step.Run = b.addSystemCommand(&.{ "bazel", "build", "//zml:sources" }); - bazel_cmd.addArg(if (platforms.cpu) "--@zml//runtimes:cpu=true" else "--@zml//runtimes:cpu=false"); - bazel_cmd.addArg(if (platforms.cuda) "--@zml//runtimes:cuda=true" else "--@zml//runtimes:cuda=false"); + pub fn zmlDepsTarball(b: *std.Build, platforms: Platforms) Tarball { + const zml_deps_tar = b.option([]const u8, "zml_deps_tar", "Path to Bazel generated ZML deps (default generate from bazel)"); - const srcs_tar = b.path(b.pathJoin(&.{ "bazel-bin", "zml", "sources.tar" })); - return .{ .create_cmd = bazel_cmd, .path = srcs_tar }; + return if (zml_deps_tar) |tarball| .{ + .create_cmd = b.addSystemCommand(&.{ "echo", "Using zml deps tarball", tarball }), + .path = b.path(tarball), + } else { + const bazel_cmd: *std.Build.Step.Run = b.addSystemCommand(&.{ "bazel", "build", "//zml:sources" }); + bazel_cmd.addArg(if (platforms.cpu) "--@zml//runtimes:cpu=true" else "--@zml//runtimes:cpu=false"); + bazel_cmd.addArg(if (platforms.cuda) "--@zml//runtimes:cuda=true" else "--@zml//runtimes:cuda=false"); + + const srcs_tar = b.path(b.pathJoin(&.{ "bazel-bin", "zml", "sources.tar" })); + return .{ .create_cmd = bazel_cmd, .path = srcs_tar }; + }; } pub fn extractModule(self: Tarball, module_name: ?[]const u8, directory: []const u8, root: []const u8, options: std.Build.Module.CreateOptions) *std.Build.Module { From 7457e26bd59ad1e99c1faecdeecb1edc73361b0b Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 17 Jul 2025 11:43:02 +0200 Subject: [PATCH 39/40] zig-protobuf: remove meta.FieldType --- MODULE.bazel | 2 +- .../modules/zig-protobuf/20250716.0-7627e25/MODULE.bazel | 1 + .../overlay/BUILD.bazel | 0 .../overlay}/MODULE.bazel | 2 +- .../modules/zig-protobuf/20250716.0-7627e25/source.json | 9 +++++++++ .../zig-protobuf/20250716.0-97f1e31/overlay/MODULE.bazel | 8 -------- .../modules/zig-protobuf/20250716.0-97f1e31/source.json | 9 --------- third_party/modules/zig-protobuf/metadata.json | 2 +- 8 files changed, 13 insertions(+), 20 deletions(-) create mode 120000 third_party/modules/zig-protobuf/20250716.0-7627e25/MODULE.bazel rename third_party/modules/zig-protobuf/{20250716.0-97f1e31 => 20250716.0-7627e25}/overlay/BUILD.bazel (100%) rename third_party/modules/zig-protobuf/{20250716.0-97f1e31 => 20250716.0-7627e25/overlay}/MODULE.bazel (83%) create mode 100644 third_party/modules/zig-protobuf/20250716.0-7627e25/source.json delete mode 100644 third_party/modules/zig-protobuf/20250716.0-97f1e31/overlay/MODULE.bazel delete mode 100644 third_party/modules/zig-protobuf/20250716.0-97f1e31/source.json diff --git a/MODULE.bazel b/MODULE.bazel index 17a68ac6..bf7e434e 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -23,7 +23,7 @@ bazel_dep(name = "toolchains_llvm_bootstrapped", version = "0.2.3") bazel_dep(name = "toolchains_protoc", version = "0.4.1") bazel_dep(name = "with_cfg.bzl", version = "0.9.1") bazel_dep(name = "xla", version = "20250710.0-22ea002") -bazel_dep(name = "zig-protobuf", version = "20250716.0-97f1e31") +bazel_dep(name = "zig-protobuf", version = "20250716.0-7627e25") bazel_dep(name = "zig-yaml", version = "20240903.0-83d5fdf") bazel_dep(name = "buildifier_prebuilt", version = "8.0.3", dev_dependency = True) diff --git a/third_party/modules/zig-protobuf/20250716.0-7627e25/MODULE.bazel b/third_party/modules/zig-protobuf/20250716.0-7627e25/MODULE.bazel new file mode 120000 index 00000000..d2cd2542 --- /dev/null +++ b/third_party/modules/zig-protobuf/20250716.0-7627e25/MODULE.bazel @@ -0,0 +1 @@ +overlay/MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/zig-protobuf/20250716.0-97f1e31/overlay/BUILD.bazel b/third_party/modules/zig-protobuf/20250716.0-7627e25/overlay/BUILD.bazel similarity index 100% rename from third_party/modules/zig-protobuf/20250716.0-97f1e31/overlay/BUILD.bazel rename to third_party/modules/zig-protobuf/20250716.0-7627e25/overlay/BUILD.bazel diff --git a/third_party/modules/zig-protobuf/20250716.0-97f1e31/MODULE.bazel b/third_party/modules/zig-protobuf/20250716.0-7627e25/overlay/MODULE.bazel similarity index 83% rename from third_party/modules/zig-protobuf/20250716.0-97f1e31/MODULE.bazel rename to third_party/modules/zig-protobuf/20250716.0-7627e25/overlay/MODULE.bazel index b71798cc..24102b40 100644 --- a/third_party/modules/zig-protobuf/20250716.0-97f1e31/MODULE.bazel +++ b/third_party/modules/zig-protobuf/20250716.0-7627e25/overlay/MODULE.bazel @@ -1,6 +1,6 @@ module( name = "zig-protobuf", - version = "20250716.0-97f1e31", + version = "20250716.0-7627e25", compatibility_level = 1, ) diff --git a/third_party/modules/zig-protobuf/20250716.0-7627e25/source.json b/third_party/modules/zig-protobuf/20250716.0-7627e25/source.json new file mode 100644 index 00000000..37ecb718 --- /dev/null +++ b/third_party/modules/zig-protobuf/20250716.0-7627e25/source.json @@ -0,0 +1,9 @@ +{ + "strip_prefix": "zig-protobuf-7627e25201c60d38d730eb7a0f3fd492b99c47c0", + "url": "https://github.com/zml/zig-protobuf/archive/7627e25201c60d38d730eb7a0f3fd492b99c47c0.tar.gz", + "integrity": "sha256-eantkVVBROb77HdUxaKY1iMNu1wXpxJ44K2+QzGns2E=", + "overlay": { + "MODULE.bazel": "", + "BUILD.bazel": "" + } +} diff --git a/third_party/modules/zig-protobuf/20250716.0-97f1e31/overlay/MODULE.bazel b/third_party/modules/zig-protobuf/20250716.0-97f1e31/overlay/MODULE.bazel deleted file mode 100644 index b71798cc..00000000 --- a/third_party/modules/zig-protobuf/20250716.0-97f1e31/overlay/MODULE.bazel +++ /dev/null @@ -1,8 +0,0 @@ -module( - name = "zig-protobuf", - version = "20250716.0-97f1e31", - compatibility_level = 1, -) - -bazel_dep(name = "rules_zig", version = "20250613.0-567662a") -bazel_dep(name = "rules_proto", version = "6.0.0-rc1") diff --git a/third_party/modules/zig-protobuf/20250716.0-97f1e31/source.json b/third_party/modules/zig-protobuf/20250716.0-97f1e31/source.json deleted file mode 100644 index 4975cdb8..00000000 --- a/third_party/modules/zig-protobuf/20250716.0-97f1e31/source.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "strip_prefix": "zig-protobuf-97f1e314d9bbf600fbaadee5bb08c8856e0006de", - "url": "https://github.com/zml/zig-protobuf/archive/97f1e314d9bbf600fbaadee5bb08c8856e0006de.tar.gz", - "integrity": "sha256-iAqsLIKhl15JInJvwMY8NGVll6eUBXzdLW6Xv8wANGY=", - "overlay": { - "MODULE.bazel": "", - "BUILD.bazel": "" - } -} diff --git a/third_party/modules/zig-protobuf/metadata.json b/third_party/modules/zig-protobuf/metadata.json index 4a038360..efb77d34 100644 --- a/third_party/modules/zig-protobuf/metadata.json +++ b/third_party/modules/zig-protobuf/metadata.json @@ -14,7 +14,7 @@ "20240722.0-c644d11", "20250213.0-5304067", "20250318.0-930153e", - "20250716.0-97f1e31", + "20250716.0-784e5d6", ], "yanked_versions": {} } From a652a8f1e90eaa81ec3975a890eaa6263d8df44f Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 17 Jul 2025 11:45:04 +0200 Subject: [PATCH 40/40] strip always on ci --- .github/workflows/commit.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/commit.yaml b/.github/workflows/commit.yaml index 790ee3ca..d6af07af 100644 --- a/.github/workflows/commit.yaml +++ b/.github/workflows/commit.yaml @@ -45,6 +45,7 @@ jobs: common --test_output=streamed common --color=yes common --show_timestamps + common --strip=always - name: ZML / bazel mod explain run: bazel mod explain