diff --git a/bin/zml-smi/tui/BUILD.bazel b/bin/zml-smi/tui/BUILD.bazel index 1a0b7f5d..29f4824c 100644 --- a/bin/zml-smi/tui/BUILD.bazel +++ b/bin/zml-smi/tui/BUILD.bazel @@ -2,6 +2,8 @@ load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") zig_library( name = "tui", + import_name = "zml-smi/tui", + main = "tui.zig", srcs = [ "data.zig", "image_cache.zig", @@ -38,14 +40,12 @@ zig_library( "assets/nvidia.png", "assets/tpu.png", ], - import_name = "zml-smi/tui", - main = "tui.zig", visibility = ["//visibility:public"], deps = [ "//bazel/stamp:zig", "//bin/zml-smi/info", "//bin/zml-smi/utils:double_buffer", - "//zml/logo", + "//zml:logo", "@libvaxis//:vaxis", ] + select({ "@platforms//os:macos": ["//bin/zml-smi/platforms/macos"], diff --git a/platforms/neuron/BUILD.bazel b/platforms/neuron/BUILD.bazel index b1f0fd30..ad537191 100644 --- a/platforms/neuron/BUILD.bazel +++ b/platforms/neuron/BUILD.bazel @@ -134,10 +134,7 @@ zig_library( import_name = "platforms/neuron/nki_kernel", main = "nki_kernel.zig", tags = ["manual"], - visibility = [ - "//zml:__pkg__", - "//zml/core:__pkg__", - ], + visibility = ["//zml:__pkg__"], deps = [ ":neuron", "//bazel", diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 6680fd65..f52ecdd5 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -1,10 +1,92 @@ +load("@com_google_protobuf//bazel:upb_c_proto_library.bzl", "upb_c_proto_library") +load("@rules_cc//cc:defs.bzl", "cc_library") load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") +upb_c_proto_library( + name = "xla_data_upb", + tags = ["manual"], + deps = ["@xla//xla:xla_data_proto"], +) + +upb_c_proto_library( + name = "xla_compile_options_upb", + tags = ["manual"], + deps = ["@xla//xla/pjrt/proto:compile_options_proto"], +) + +upb_c_proto_library( + name = "profiler_options", + tags = ["manual"], + deps = ["@tsl//tsl/profiler/protobuf:profiler_options_proto"], +) + +cc_library( + name = "posix", + hdrs = ["posix.h"], + tags = ["manual"], +) + +cc_library( + name = "profiling_traceme_lib", + srcs = ["profiling/traceme_lib.cc"], + hdrs = ["profiling/traceme.h"], + defines = select({ + "//platforms:cuda.enabled": ["ZML_RUNTIME_CUDA"], + "//conditions:default": [], + }) + select({ + "//platforms:rocm.enabled": ["ZML_RUNTIME_ROCM"], + "//conditions:default": [], + }), + linkopts = select({ + "//platforms:rocm.enabled": ["-ldl"], + "//conditions:default": [], + }), + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//ffi:cc", + "@tsl//tsl/profiler/lib:traceme", + "@tsl//tsl/profiler/protobuf:xplane_proto_cc", + "@xla//xla/tsl/profiler/backends/cpu:host_tracer_utils", + "@xla//xla/tsl/profiler/backends/cpu:traceme_recorder", + "@xla//xla/tsl/profiler/convert:post_process_single_host_xplane", + "@xla//xla/tsl/profiler/utils:time_utils", + "@xla//xla/tsl/profiler/utils:xplane_utils", + ] + select({ + "//platforms:cuda.enabled": [ + "//platforms/cuda:cuda_nvtx_headers", + ], + "//conditions:default": [], + }) + select({ + "//platforms:rocm.enabled": [ + "@amdrocm-profiler-base//:roctx_headers", + ], + "//conditions:default": [], + }), +) + +zig_library( + name = "logo", + srcs = ["logo.zig"], + import_name = "zml/logo", + main = "logo.zig", + visibility = ["//visibility:public"], +) + zig_library( name = "zml", srcs = [ + "Compiler.zig", + "Sharding.zig", "attention/Options.zig", "attention/attention.zig", + "buffer.zig", + "buffer/HostAccessible.zig", + "constants.zig", + "dtype.zig", + "exe.zig", + "floats.zig", + "io.zig", "kernels/flash_attention/flash_attention.zig", "kernels/flash_attention/tests.zig", "kernels/moe/api.zig", @@ -13,7 +95,25 @@ zig_library( "kernels/moe/weights.zig", "kernels/paged_attention/options.zig", "kernels/paged_attention/paged_attention.zig", + "logo.zig", + "mem.zig", + "meta.zig", + "mlirx.zig", + "nn.zig", + "ops.zig", + "ops/manual_computation.zig", + "pjrtx.zig", + "platform.zig", + "profiling/profiler.zig", + "profiling/tracer.zig", + "quantization.zig", + "safetensors.zig", + "shape.zig", + "slice.zig", + "tensor.zig", + "testing.zig", "zml.zig", + "//zml/kernel", "//zml/kernels/flash_attention/cute", "//zml/kernels/flash_attention/metal", "//zml/kernels/flash_attention/nki", @@ -46,29 +146,43 @@ zig_library( "-freference-trace=20", ], deps = [ + ":posix", + ":profiler_options", + ":profiling_traceme_lib", + ":xla_compile_options_upb", + ":xla_data_upb", "//bazel", + "//dsl/cuda_tile:builder", + "//dsl/cute:builder", + "//dsl/fly:builder", + "//dsl/metal:builder", "//dsl/mosaic_tpu:builder", "//dsl/tcl:builder", + "//dsl/triton:builder", + "//ffi:zig", "//mlir", + "//mlir/dialects/func", "//mlir/dialects/llvm", + "//mlir/dialects/mosaic_tpu", "//mlir/dialects/nvvm", + "//mlir/dialects/shardy", + "//mlir/dialects/stablehlo", "//mlir/dialects/ttir", "//mlir/dialects/vector", "//pjrt", "//platforms", "//platforms/tpu:ragged_paged", "//stdx", - "//zml/constants", - "//zml/core", - "//zml/logo", - "//zml/meta", - "//zml/nn", - "//zml/profiling", - "//zml/quantization", + "//tools/xspace_to_perfetto:xspace_to_perfetto_lib", + "//upb", + "//vfs", "//zml/tokenizer", + "@rules_zig//zig/runfiles", + "@zio", ] + select({ "//platforms:neuron.enabled": [ "//platforms/neuron:nki_kernel", + "//platforms/neuron:topology", ], "//conditions:default": [], }) + select({ @@ -81,5 +195,5 @@ zig_library( zig_test( name = "test", - deps = [":zml"], + deps = ["//zml"], ) diff --git a/zml/core/Compiler.zig b/zml/Compiler.zig similarity index 99% rename from zml/core/Compiler.zig rename to zml/Compiler.zig index 1bf994cc..93eafa8d 100644 --- a/zml/core/Compiler.zig +++ b/zml/Compiler.zig @@ -2,13 +2,11 @@ const std = @import("std"); const c = @import("c"); const func = @import("mlir/dialects/func"); -const meta = @import("zml/meta"); const mlir = @import("mlir"); const pjrt = @import("pjrt"); const shardy = @import("mlir/dialects/shardy"); const stablehlo = @import("mlir/dialects/stablehlo"); const stdx = @import("stdx"); -const tracer = @import("zml/profiling").tracer; const upb = @import("upb"); const zio = @import("zio"); @@ -16,10 +14,12 @@ const Buffer = @import("buffer.zig").Buffer; const DataType = @import("dtype.zig").DataType; const Exe = @import("exe.zig").Exe; const Memory = @import("platform.zig").Memory; +const meta = @import("meta.zig"); const mlirx = @import("mlirx.zig"); const ops = @import("ops.zig"); const pjrtx = @import("pjrtx.zig"); const Platform = @import("platform.zig").Platform; +const tracer = @import("profiling/tracer.zig"); const Shape = @import("shape.zig").Shape; const Sharding = @import("Sharding.zig"); const Tensor = @import("tensor.zig").Tensor; diff --git a/zml/core/Sharding.zig b/zml/Sharding.zig similarity index 99% rename from zml/core/Sharding.zig rename to zml/Sharding.zig index 1f05b4df..040216bc 100644 --- a/zml/core/Sharding.zig +++ b/zml/Sharding.zig @@ -3249,7 +3249,7 @@ test "GSPMD attributes preserve folded logical axis order" { test "GSPMD executes a reduction across a folded logical axis" { if (comptime !Target.cpu.isEnabled()) return error.SkipZigTest; - const zml = @import("core.zig"); + const zml = @import("zml.zig"); const io = std.testing.io; const allocator = std.testing.allocator; diff --git a/zml/core/buffer.zig b/zml/buffer.zig similarity index 99% rename from zml/core/buffer.zig rename to zml/buffer.zig index 81112de9..8714d80f 100644 --- a/zml/core/buffer.zig +++ b/zml/buffer.zig @@ -1,14 +1,14 @@ const std = @import("std"); -const constants = @import("zml/constants"); -const meta = @import("zml/meta"); const pjrt = @import("pjrt"); const platforms = @import("platforms"); const stdx = @import("stdx"); +const constants = @import("constants.zig"); const DataType = @import("dtype.zig").DataType; const mem = @import("mem.zig"); const Memory = @import("platform.zig").Memory; +const meta = @import("meta.zig"); const pjrtx = @import("pjrtx.zig"); const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; @@ -376,7 +376,7 @@ pub const Buffer = struct { }; test "device round-trip" { - const zml = @import("core.zig"); + const zml = @import("zml.zig"); const io = std.testing.io; const allocator = std.testing.allocator; const platform = zml.testing.env(); diff --git a/zml/core/buffer/HostAccessible.zig b/zml/buffer/HostAccessible.zig similarity index 99% rename from zml/core/buffer/HostAccessible.zig rename to zml/buffer/HostAccessible.zig index db17ae7b..b3e82a9c 100644 --- a/zml/core/buffer/HostAccessible.zig +++ b/zml/buffer/HostAccessible.zig @@ -1878,7 +1878,7 @@ fn expectDevicesReadShards(platform: *const Platform, shape_: Shape) !void { } test "HostAccessible executions read shared inputs on every device" { - const zml = @import("../core.zig"); + const zml = @import("../zml.zig"); const allocator = std.testing.allocator; const io = std.testing.io; const platform = testing.env(); @@ -2107,7 +2107,7 @@ test "HostAccessible.View copies across layouts, shards and replicas" { } test "HostAccessible await after executable donation permits the next host update" { - const zml = @import("../core.zig"); + const zml = @import("../zml.zig"); const allocator = std.testing.allocator; const io = std.testing.io; const platform = testing.env(); @@ -2159,7 +2159,7 @@ test "HostAccessible await after executable donation permits the next host updat } test "HostAccessible await follows an output that doesn't reuse the pinned allocation" { - const zml = @import("../core.zig"); + const zml = @import("../zml.zig"); const allocator = std.testing.allocator; const io = std.testing.io; const platform = testing.env(); diff --git a/zml/constants/constants.zig b/zml/constants.zig similarity index 100% rename from zml/constants/constants.zig rename to zml/constants.zig diff --git a/zml/constants/BUILD.bazel b/zml/constants/BUILD.bazel deleted file mode 100644 index b55a6e34..00000000 --- a/zml/constants/BUILD.bazel +++ /dev/null @@ -1,13 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") - -zig_library( - name = "constants", - import_name = "zml/constants", - main = "constants.zig", - visibility = ["//visibility:public"], -) - -zig_test( - name = "test", - deps = [":constants"], -) diff --git a/zml/core/BUILD.bazel b/zml/core/BUILD.bazel deleted file mode 100644 index 18452d0b..00000000 --- a/zml/core/BUILD.bazel +++ /dev/null @@ -1,101 +0,0 @@ -load("@com_google_protobuf//bazel:upb_c_proto_library.bzl", "upb_c_proto_library") -load("@rules_cc//cc:defs.bzl", "cc_library") -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") - -upb_c_proto_library( - name = "xla_data_upb", - tags = ["manual"], - deps = ["@xla//xla:xla_data_proto"], -) - -upb_c_proto_library( - name = "xla_compile_options_upb", - tags = ["manual"], - deps = ["@xla//xla/pjrt/proto:compile_options_proto"], -) - -cc_library( - name = "posix", - hdrs = ["posix.h"], - tags = ["manual"], -) - -zig_library( - name = "core", - srcs = [ - "Compiler.zig", - "Sharding.zig", - "buffer.zig", - "buffer/HostAccessible.zig", - "dtype.zig", - "exe.zig", - "flashinfer_cutlass/contract.zig", - "flashinfer_cutlass/runtime.zig", - "floats.zig", - "io.zig", - "mem.zig", - "mlirx.zig", - "ops.zig", - "ops/manual_computation.zig", - "pjrtx.zig", - "platform.zig", - "safetensors.zig", - "shape.zig", - "slice.zig", - "tensor.zig", - "testing.zig", - "//zml/core/kernel", - ], - import_name = "zml/core", - main = "core.zig", - tags = ["manual"], - visibility = ["//visibility:public"], - zigopts = [ - "-lc", - "-freference-trace=20", - ], - deps = [ - ":posix", - ":xla_compile_options_upb", - ":xla_data_upb", - "//bazel", - "//dsl/cuda_tile:builder", - "//dsl/cute:builder", - "//dsl/fly:builder", - "//dsl/metal:builder", - "//dsl/mosaic_tpu:builder", - "//dsl/tcl:builder", - "//dsl/triton:builder", - "//ffi:zig", - "//mlir", - "//mlir/dialects/func", - "//mlir/dialects/mosaic_tpu", - "//mlir/dialects/shardy", - "//mlir/dialects/stablehlo", - "//pjrt", - "//platforms", - "//stdx", - "//upb", - "//vfs", - "//zml/constants", - "//zml/meta", - "//zml/profiling", - "@zio", - ] + select({ - "//platforms:neuron.enabled": [ - "//platforms/neuron:nki_kernel", - "//platforms/neuron:topology", - ], - "//conditions:default": [], - }) + select({ - "//platforms:cuda.enabled": [ - "//platforms/cuda/flashinfer_cutlass_moe", - ], - "//conditions:default": [], - }), -) - -zig_test( - name = "test", - deps = [":core"], -) diff --git a/zml/core/core.zig b/zml/core/core.zig deleted file mode 100644 index 4ec2bea4..00000000 --- a/zml/core/core.zig +++ /dev/null @@ -1,74 +0,0 @@ -const std = @import("std"); - -const bazel = @import("bazel"); - -pub const Buffer = @import("buffer.zig").Buffer; -pub const Compiler = @import("Compiler.zig"); -pub const dtype = @import("dtype.zig"); -pub const Data = dtype.Data; -pub const DataType = dtype.DataType; -pub const exe = @import("exe.zig"); -pub const Exe = exe.Exe; -pub const FnExe = exe.FnExe; -pub const floats = @import("floats.zig"); -pub const io = @import("io.zig"); -pub const kernel = @import("kernel/kernel.zig"); -pub const mem = @import("mem.zig"); -pub const Bufferized = mem.Bufferized; -pub const mlir = @import("mlirx.zig"); -pub const ops = @import("ops.zig"); -pub const pjrtx = @import("pjrtx.zig"); -pub const platform = @import("platform.zig"); -pub const Memory = platform.Memory; -pub const Platform = platform.Platform; -pub const Target = platform.Target; -pub const safetensors = @import("safetensors.zig"); -pub const shape = @import("shape.zig"); -pub const Shape = shape.Shape; -pub const Sharding = @import("Sharding.zig"); -pub const Mesh = Sharding.Mesh; -pub const slice = @import("slice.zig"); -pub const Slice = slice.Slice; -pub const tensor = @import("tensor.zig"); -pub const Tensor = tensor.Tensor; -pub const testing = @import("testing.zig"); - -// Platform owns registration and runner lifetime until these are separated from core. -pub const flashinfer_cutlass = struct { - pub const contract = @import("flashinfer_cutlass/contract.zig"); - pub const runtime = @import("flashinfer_cutlass/runtime.zig"); -}; - -/// Deprecated, use `zml.Compiler` instead -pub const module = struct { - pub const CompilationContext = Compiler; - pub const CompileError = Compiler.Error; - pub const CompilationOptions = Compiler.Options; - - /// Deprecated, use `platform.compileFn` or `zml.Compiler.compile` - pub fn compile( - allocator: std.mem.Allocator, - io_: std.Io, - comptime func: anytype, - args: std.meta.ArgsTuple(@TypeOf(func)), - platform_: *const Platform, - opts: Compiler.Options, - ) Compiler.Error!Exe { - return Compiler.compile(allocator, io_, platform_, func, args, opts); - } -}; - -test "core" { - std.testing.refAllDecls(@This()); -} - -pub const KiB = 1024; -pub const MiB = 1024 * KiB; -pub const GiB = 1024 * MiB; -pub const TiB = 1024 * GiB; - -comptime { - // Forces zig to compile `@zml//bazel` module even when users don't depend on it even indirectly. - // Otherwise exported functions declared in `bazel` aren't compiled. - std.testing.refAllDecls(bazel); -} diff --git a/zml/core/kernel/BUILD.bazel b/zml/core/kernel/BUILD.bazel deleted file mode 100644 index 31638d07..00000000 --- a/zml/core/kernel/BUILD.bazel +++ /dev/null @@ -1,15 +0,0 @@ -package(default_visibility = ["//zml/core:__pkg__"]) - -filegroup( - name = "kernel", - srcs = [ - "kernel.zig", - "//zml/core/kernel/cuda_tile", - "//zml/core/kernel/cute", - "//zml/core/kernel/fly", - "//zml/core/kernel/metal", - "//zml/core/kernel/mosaic_tpu", - "//zml/core/kernel/tcl", - "//zml/core/kernel/triton", - ], -) diff --git a/zml/core/dtype.zig b/zml/dtype.zig similarity index 100% rename from zml/core/dtype.zig rename to zml/dtype.zig diff --git a/zml/core/exe.zig b/zml/exe.zig similarity index 99% rename from zml/core/exe.zig rename to zml/exe.zig index e370c708..32c0ac48 100644 --- a/zml/core/exe.zig +++ b/zml/exe.zig @@ -1,15 +1,15 @@ const std = @import("std"); const builtin = @import("builtin"); -const meta = @import("zml/meta"); const pjrt = @import("pjrt"); const stdx = @import("stdx"); -const tracer = @import("zml/profiling").tracer; const Buffer = @import("buffer.zig").Buffer; const Compiler = @import("Compiler.zig"); const mem = @import("mem.zig"); +const meta = @import("meta.zig"); const Platform = @import("platform.zig").Platform; +const tracer = @import("profiling/tracer.zig"); const Shape = @import("shape.zig").Shape; const Sharding = @import("Sharding.zig"); const Tensor = @import("tensor.zig").Tensor; diff --git a/zml/core/floats.zig b/zml/floats.zig similarity index 100% rename from zml/core/floats.zig rename to zml/floats.zig diff --git a/zml/core/io.zig b/zml/io.zig similarity index 99% rename from zml/core/io.zig rename to zml/io.zig index 93829541..ab8d8e6e 100644 --- a/zml/core/io.zig +++ b/zml/io.zig @@ -1,19 +1,19 @@ const std = @import("std"); -const meta = @import("zml/meta"); const pjrt = @import("pjrt"); const stdx = @import("stdx"); -const tracer = @import("zml/profiling").tracer; pub const VFS = @import("vfs"); const Buffer = @import("buffer.zig").Buffer; -const Bufferized = @import("core.zig").Bufferized; +const Bufferized = @import("zml.zig").Bufferized; const CreateOptions = @import("platform.zig").CreateOptions; const Exe = @import("exe.zig").Exe; const mem = @import("mem.zig"); const Memory = @import("platform.zig").Memory; +const meta = @import("meta.zig"); const pjrtx = @import("pjrtx.zig"); const Platform = @import("platform.zig").Platform; +const tracer = @import("profiling/tracer.zig"); const safetensors = @import("safetensors.zig"); const Shape = @import("shape.zig").Shape; const Sharding = @import("Sharding.zig"); diff --git a/zml/kernel/BUILD.bazel b/zml/kernel/BUILD.bazel new file mode 100644 index 00000000..61ab44f8 --- /dev/null +++ b/zml/kernel/BUILD.bazel @@ -0,0 +1,15 @@ +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "kernel", + srcs = [ + "kernel.zig", + "//zml/kernel/cuda_tile", + "//zml/kernel/cute", + "//zml/kernel/fly", + "//zml/kernel/metal", + "//zml/kernel/mosaic_tpu", + "//zml/kernel/tcl", + "//zml/kernel/triton", + ], +) diff --git a/zml/core/kernel/cuda_tile/BUILD.bazel b/zml/kernel/cuda_tile/BUILD.bazel similarity index 63% rename from zml/core/kernel/cuda_tile/BUILD.bazel rename to zml/kernel/cuda_tile/BUILD.bazel index a3e7dc19..89b7977e 100644 --- a/zml/core/kernel/cuda_tile/BUILD.bazel +++ b/zml/kernel/cuda_tile/BUILD.bazel @@ -1,4 +1,4 @@ -package(default_visibility = ["//zml/core/kernel:__pkg__"]) +package(default_visibility = ["//zml/kernel:__pkg__"]) filegroup( name = "cuda_tile", diff --git a/zml/core/kernel/cuda_tile/cuda_tile.zig b/zml/kernel/cuda_tile/cuda_tile.zig similarity index 100% rename from zml/core/kernel/cuda_tile/cuda_tile.zig rename to zml/kernel/cuda_tile/cuda_tile.zig diff --git a/zml/core/kernel/cuda_tile/test.zig b/zml/kernel/cuda_tile/test.zig similarity index 100% rename from zml/core/kernel/cuda_tile/test.zig rename to zml/kernel/cuda_tile/test.zig diff --git a/zml/core/kernel/cute/BUILD.bazel b/zml/kernel/cute/BUILD.bazel similarity index 60% rename from zml/core/kernel/cute/BUILD.bazel rename to zml/kernel/cute/BUILD.bazel index 73c8c128..7a613072 100644 --- a/zml/core/kernel/cute/BUILD.bazel +++ b/zml/kernel/cute/BUILD.bazel @@ -1,4 +1,4 @@ -package(default_visibility = ["//zml/core/kernel:__pkg__"]) +package(default_visibility = ["//zml/kernel:__pkg__"]) filegroup( name = "cute", diff --git a/zml/core/kernel/cute/cute.zig b/zml/kernel/cute/cute.zig similarity index 100% rename from zml/core/kernel/cute/cute.zig rename to zml/kernel/cute/cute.zig diff --git a/zml/core/kernel/cute/test.zig b/zml/kernel/cute/test.zig similarity index 100% rename from zml/core/kernel/cute/test.zig rename to zml/kernel/cute/test.zig diff --git a/zml/core/kernel/fly/BUILD.bazel b/zml/kernel/fly/BUILD.bazel similarity index 60% rename from zml/core/kernel/fly/BUILD.bazel rename to zml/kernel/fly/BUILD.bazel index 11fea497..57203521 100644 --- a/zml/core/kernel/fly/BUILD.bazel +++ b/zml/kernel/fly/BUILD.bazel @@ -1,4 +1,4 @@ -package(default_visibility = ["//zml/core/kernel:__pkg__"]) +package(default_visibility = ["//zml/kernel:__pkg__"]) filegroup( name = "fly", diff --git a/zml/core/kernel/fly/fly.zig b/zml/kernel/fly/fly.zig similarity index 100% rename from zml/core/kernel/fly/fly.zig rename to zml/kernel/fly/fly.zig diff --git a/zml/core/kernel/fly/test.zig b/zml/kernel/fly/test.zig similarity index 99% rename from zml/core/kernel/fly/test.zig rename to zml/kernel/fly/test.zig index 6b77ad3e..ae788d96 100644 --- a/zml/core/kernel/fly/test.zig +++ b/zml/kernel/fly/test.zig @@ -3,7 +3,7 @@ const std = @import("std"); const fly_builder = @import("dsl/fly/builder"); const fly = @import("fly.zig"); -const zml = @import("../../core.zig"); +const zml = @import("../../zml.zig"); /// C = A + B over (M, N) f32, 128-bit vectorized and predicated on the /// ragged edge. diff --git a/zml/core/kernel/kernel.zig b/zml/kernel/kernel.zig similarity index 100% rename from zml/core/kernel/kernel.zig rename to zml/kernel/kernel.zig diff --git a/zml/core/kernel/metal/BUILD.bazel b/zml/kernel/metal/BUILD.bazel similarity index 61% rename from zml/core/kernel/metal/BUILD.bazel rename to zml/kernel/metal/BUILD.bazel index d0a2d380..4007beb4 100644 --- a/zml/core/kernel/metal/BUILD.bazel +++ b/zml/kernel/metal/BUILD.bazel @@ -1,4 +1,4 @@ -package(default_visibility = ["//zml/core/kernel:__pkg__"]) +package(default_visibility = ["//zml/kernel:__pkg__"]) filegroup( name = "metal", diff --git a/zml/core/kernel/metal/metal.zig b/zml/kernel/metal/metal.zig similarity index 100% rename from zml/core/kernel/metal/metal.zig rename to zml/kernel/metal/metal.zig diff --git a/zml/core/kernel/metal/test.zig b/zml/kernel/metal/test.zig similarity index 98% rename from zml/core/kernel/metal/test.zig rename to zml/kernel/metal/test.zig index a184869c..0a342af0 100644 --- a/zml/core/kernel/metal/test.zig +++ b/zml/kernel/metal/test.zig @@ -1,7 +1,7 @@ const std = @import("std"); const metal = @import("metal.zig"); -const zml = @import("../../core.zig"); +const zml = @import("../../zml.zig"); /// `out[i] = a[i] * scale + b[i]` over `n` floats. const MetalAxpy = struct { diff --git a/zml/core/kernel/mosaic_tpu/BUILD.bazel b/zml/kernel/mosaic_tpu/BUILD.bazel similarity index 53% rename from zml/core/kernel/mosaic_tpu/BUILD.bazel rename to zml/kernel/mosaic_tpu/BUILD.bazel index ae2d346d..ba3b6b66 100644 --- a/zml/core/kernel/mosaic_tpu/BUILD.bazel +++ b/zml/kernel/mosaic_tpu/BUILD.bazel @@ -1,4 +1,4 @@ -package(default_visibility = ["//zml/core/kernel:__pkg__"]) +package(default_visibility = ["//zml/kernel:__pkg__"]) filegroup( name = "mosaic_tpu", diff --git a/zml/core/kernel/mosaic_tpu/mosaic_tpu.zig b/zml/kernel/mosaic_tpu/mosaic_tpu.zig similarity index 100% rename from zml/core/kernel/mosaic_tpu/mosaic_tpu.zig rename to zml/kernel/mosaic_tpu/mosaic_tpu.zig diff --git a/zml/core/kernel/tcl/BUILD.bazel b/zml/kernel/tcl/BUILD.bazel similarity index 60% rename from zml/core/kernel/tcl/BUILD.bazel rename to zml/kernel/tcl/BUILD.bazel index 6a1229c1..bc7a0aa2 100644 --- a/zml/core/kernel/tcl/BUILD.bazel +++ b/zml/kernel/tcl/BUILD.bazel @@ -1,4 +1,4 @@ -package(default_visibility = ["//zml/core/kernel:__pkg__"]) +package(default_visibility = ["//zml/kernel:__pkg__"]) filegroup( name = "tcl", diff --git a/zml/core/kernel/tcl/tcl.zig b/zml/kernel/tcl/tcl.zig similarity index 100% rename from zml/core/kernel/tcl/tcl.zig rename to zml/kernel/tcl/tcl.zig diff --git a/zml/core/kernel/tcl/test.zig b/zml/kernel/tcl/test.zig similarity index 99% rename from zml/core/kernel/tcl/test.zig rename to zml/kernel/tcl/test.zig index 20804b8b..b9ae1875 100644 --- a/zml/core/kernel/tcl/test.zig +++ b/zml/kernel/tcl/test.zig @@ -1,6 +1,6 @@ const std = @import("std"); -const zml = @import("../../core.zig"); +const zml = @import("../../zml.zig"); const tcl = @import("tcl.zig"); /// `furiosa.tcl.examples.softmax` over f32. diff --git a/zml/core/kernel/triton/BUILD.bazel b/zml/kernel/triton/BUILD.bazel similarity index 50% rename from zml/core/kernel/triton/BUILD.bazel rename to zml/kernel/triton/BUILD.bazel index 0eface24..1e992352 100644 --- a/zml/core/kernel/triton/BUILD.bazel +++ b/zml/kernel/triton/BUILD.bazel @@ -1,4 +1,4 @@ -package(default_visibility = ["//zml/core/kernel:__pkg__"]) +package(default_visibility = ["//zml/kernel:__pkg__"]) filegroup( name = "triton", diff --git a/zml/core/kernel/triton/triton.zig b/zml/kernel/triton/triton.zig similarity index 100% rename from zml/core/kernel/triton/triton.zig rename to zml/kernel/triton/triton.zig diff --git a/zml/kernels/flash_attention/metal/flash_attn.zig b/zml/kernels/flash_attention/metal/flash_attn.zig index 4d584dab..de4b68da 100644 --- a/zml/kernels/flash_attention/metal/flash_attn.zig +++ b/zml/kernels/flash_attention/metal/flash_attn.zig @@ -4,7 +4,7 @@ const std = @import("std"); -const metal = @import("zml/core").kernel.metal; +const metal = @import("../../../kernel/kernel.zig").metal; const ArgSpec = metal.ArgSpec; const Builder = metal.Builder; diff --git a/zml/kernels/moe/api.zig b/zml/kernels/moe/api.zig index b11ac1d5..b66593c1 100644 --- a/zml/kernels/moe/api.zig +++ b/zml/kernels/moe/api.zig @@ -2,7 +2,7 @@ const std = @import("std"); const stdx = @import("stdx"); -const Tensor = @import("zml/core").Tensor; +const Tensor = @import("../../tensor.zig").Tensor; const log = std.log.scoped(.@"zml/moe"); diff --git a/zml/kernels/moe/flashinfer_cutlass/BUILD.bazel b/zml/kernels/moe/flashinfer_cutlass/BUILD.bazel index ada90e9f..c9c5d9a6 100644 --- a/zml/kernels/moe/flashinfer_cutlass/BUILD.bazel +++ b/zml/kernels/moe/flashinfer_cutlass/BUILD.bazel @@ -2,5 +2,9 @@ package(default_visibility = ["//zml:__pkg__"]) filegroup( name = "flashinfer_cutlass", - srcs = ["flashinfer_cutlass.zig"], + srcs = [ + "contract.zig", + "flashinfer_cutlass.zig", + "runtime.zig", + ], ) diff --git a/zml/core/flashinfer_cutlass/contract.zig b/zml/kernels/moe/flashinfer_cutlass/contract.zig similarity index 100% rename from zml/core/flashinfer_cutlass/contract.zig rename to zml/kernels/moe/flashinfer_cutlass/contract.zig diff --git a/zml/kernels/moe/flashinfer_cutlass/flashinfer_cutlass.zig b/zml/kernels/moe/flashinfer_cutlass/flashinfer_cutlass.zig index 715198f9..92317c7c 100644 --- a/zml/kernels/moe/flashinfer_cutlass/flashinfer_cutlass.zig +++ b/zml/kernels/moe/flashinfer_cutlass/flashinfer_cutlass.zig @@ -9,9 +9,9 @@ const GlobalOptions = @import("../api.zig").Options; const Activation = @import("../api.zig").Activation; const LocalRouting = @import("../api.zig").LocalRouting; const unpackedWeight = @import("../weights.zig").unpackedWeight; -const contract = @import("zml/core").flashinfer_cutlass.contract; +const contract = @import("contract.zig"); const Attributes = contract.Attributes; -const runtime = @import("zml/core").flashinfer_cutlass.runtime; +const runtime = @import("runtime.zig"); const log = std.log.scoped(.@"zml/moe/flashinfer_cutlass"); diff --git a/zml/core/flashinfer_cutlass/runtime.zig b/zml/kernels/moe/flashinfer_cutlass/runtime.zig similarity index 99% rename from zml/core/flashinfer_cutlass/runtime.zig rename to zml/kernels/moe/flashinfer_cutlass/runtime.zig index fc72f1be..028ef892 100644 --- a/zml/core/flashinfer_cutlass/runtime.zig +++ b/zml/kernels/moe/flashinfer_cutlass/runtime.zig @@ -3,7 +3,7 @@ const std = @import("std"); const fi_cutlass_moe = @import("platforms/cuda/flashinfer_cutlass_moe"); const pjrt = @import("pjrt"); -const DataType = @import("../dtype.zig").DataType; +const DataType = @import("../../../dtype.zig").DataType; const contract = @import("contract.zig"); const log = std.log.scoped(.@"zml/kernels/moe/flashinfer_cutlass/runtime"); diff --git a/zml/kernels/moe/weights.zig b/zml/kernels/moe/weights.zig index fd3c9e82..0e288856 100644 --- a/zml/kernels/moe/weights.zig +++ b/zml/kernels/moe/weights.zig @@ -1,5 +1,5 @@ -const nn = @import("zml/nn"); -const Tensor = @import("zml/core").Tensor; +const nn = @import("../../nn.zig"); +const Tensor = @import("../../tensor.zig").Tensor; pub fn unpackedWeight(linear: nn.Linear) Tensor { const quantization = linear.quantization orelse return linear.weight; diff --git a/zml/kernels/paged_attention/metal/paged_attn.zig b/zml/kernels/paged_attention/metal/paged_attn.zig index 10a088e5..04f4c1ac 100644 --- a/zml/kernels/paged_attention/metal/paged_attn.zig +++ b/zml/kernels/paged_attention/metal/paged_attn.zig @@ -3,7 +3,7 @@ const std = @import("std"); -const metal = @import("zml/core").kernel.metal; +const metal = @import("../../../kernel/kernel.zig").metal; const ArgSpec = metal.ArgSpec; const Builder = metal.Builder; diff --git a/zml/kernels/paged_attention/options.zig b/zml/kernels/paged_attention/options.zig index 159da9b2..35ad4049 100644 --- a/zml/kernels/paged_attention/options.zig +++ b/zml/kernels/paged_attention/options.zig @@ -1,4 +1,4 @@ -const Tensor = @import("zml/core").Tensor; +const Tensor = @import("../../tensor.zig").Tensor; pub const AttentionMask = union(enum) { standard: struct { diff --git a/zml/logo/logo.zig b/zml/logo.zig similarity index 100% rename from zml/logo/logo.zig rename to zml/logo.zig diff --git a/zml/logo/BUILD.bazel b/zml/logo/BUILD.bazel deleted file mode 100644 index 1a7af734..00000000 --- a/zml/logo/BUILD.bazel +++ /dev/null @@ -1,14 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") - -zig_library( - name = "logo", - srcs = ["logo.zig"], - import_name = "zml/logo", - main = "logo.zig", - visibility = ["//visibility:public"], -) - -zig_test( - name = "test", - deps = [":logo"], -) diff --git a/zml/core/mem.zig b/zml/mem.zig similarity index 99% rename from zml/core/mem.zig rename to zml/mem.zig index 3007044c..a64ae968 100644 --- a/zml/core/mem.zig +++ b/zml/mem.zig @@ -3,12 +3,12 @@ const Allocator = std.mem.Allocator; const Alignment = std.mem.Alignment; const assert = std.debug.assert; -const meta = @import("zml/meta"); const pjrt = @import("pjrt"); const Buffer = @import("buffer.zig").Buffer; const Device = @import("platform.zig").Device; const Memory = @import("platform.zig").Memory; +const meta = @import("meta.zig"); const Platform = @import("platform.zig").Platform; const Tensor = @import("tensor.zig").Tensor; diff --git a/zml/meta/meta.zig b/zml/meta.zig similarity index 100% rename from zml/meta/meta.zig rename to zml/meta.zig diff --git a/zml/meta/BUILD.bazel b/zml/meta/BUILD.bazel deleted file mode 100644 index a795993a..00000000 --- a/zml/meta/BUILD.bazel +++ /dev/null @@ -1,14 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") - -zig_library( - name = "meta", - import_name = "zml/meta", - main = "meta.zig", - visibility = ["//visibility:public"], - deps = ["//stdx"], -) - -zig_test( - name = "test", - deps = [":meta"], -) diff --git a/zml/core/mlirx.zig b/zml/mlirx.zig similarity index 100% rename from zml/core/mlirx.zig rename to zml/mlirx.zig diff --git a/zml/nn/nn.zig b/zml/nn.zig similarity index 99% rename from zml/nn/nn.zig rename to zml/nn.zig index 94347552..eceb316b 100644 --- a/zml/nn/nn.zig +++ b/zml/nn.zig @@ -1,22 +1,21 @@ //! Common layer definition and functions for Neural Networks (NN) const std = @import("std"); -const constants = @import("zml/constants"); -const meta = @import("zml/meta"); const mlir = @import("mlir"); const stablehlo = @import("mlir/dialects/stablehlo"); const stdx = @import("stdx"); -const zml = @import("zml/core"); -const quantization = @import("zml/quantization"); - -const DataType = zml.DataType; -const mlirx = zml.mlir; -const ops = zml.ops; +const constants = @import("constants.zig"); +const DataType = @import("dtype.zig").DataType; +const meta = @import("meta.zig"); +const mlirx = @import("mlirx.zig"); +const ops = @import("ops.zig"); +const quantization = @import("quantization.zig"); pub const Quantization = quantization.Quantization; -const Shape = zml.Shape; -const Slice = zml.Slice; -const Tensor = zml.Tensor; +const Shape = @import("shape.zig").Shape; +const Slice = @import("slice.zig").Slice; +const Tensor = @import("tensor.zig").Tensor; +const zml = @import("zml.zig"); const log = std.log.scoped(.@"zml/nn"); @@ -230,7 +229,7 @@ test "block128 scaled dot layouts" { const platform = zml.testing.env(); // FIXME: Add rocm when we have a PJRT plugin with native fp8 dot support (by the end of Sept 2026) if (platform.target != .cuda and platform.target != .furiosa) return error.SkipZigTest; - const dtype: DataType = if (platform.target == .rocm and zml.platform.rocm.computeCapability(platform) == .gfx942) .f8e4m3fnuz else .f8e4m3fn; + const dtype: DataType = if (platform.target == .rocm and @import("platform.zig").rocm.computeCapability(platform) == .gfx942) .f8e4m3fnuz else .f8e4m3fn; const Local = struct { const Outputs = struct { actual: Tensor, expected: Tensor, linear: Tensor, linear_expected: Tensor }; @@ -632,7 +631,7 @@ pub fn rope(x: Tensor, pos_idx: ?Tensor, opts: RopeOpts) Tensor { const rotary_dim: u32 = opts.scaling.partialRotaryDim(head_dim); stdx.debug.assert(rotary_dim > 0 and @mod(rotary_dim, 2) == 0, "partial rope expects a even head dim (.hd), got {d}", .{rotary_dim}); - const x_real, const x_imag, const x_pass = splitRealImgPass(x, opts.layout, rotary_dim); + const x_real, const x_imag, const x_pass = zml.nn.splitRealImgPass(x, opts.layout, rotary_dim); const inv_freq = invFreq(head_dim, opts).withTags(.{.hd}); // compute sin and cos in f32 before downcasting to x type. @@ -645,7 +644,7 @@ pub fn rope(x: Tensor, pos_idx: ?Tensor, opts: RopeOpts) Tensor { const y_real = x_real.mul(cos).sub(x_imag.mul(sin)); const y_imag = x_real.mul(sin).add(x_imag.mul(cos)); - return mergeRealImgPass(y_real, y_imag, x_pass, opts.layout); + return zml.nn.mergeRealImgPass(y_real, y_imag, x_pass, opts.layout); } pub fn splitRealImg(x: Tensor, layout: RopeOpts.Layout) [2]Tensor { diff --git a/zml/nn/BUILD.bazel b/zml/nn/BUILD.bazel deleted file mode 100644 index 4fa4c325..00000000 --- a/zml/nn/BUILD.bazel +++ /dev/null @@ -1,23 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") - -zig_library( - name = "nn", - import_name = "zml/nn", - main = "nn.zig", - tags = ["manual"], - visibility = ["//visibility:public"], - deps = [ - "//mlir", - "//mlir/dialects/stablehlo", - "//stdx", - "//zml/constants", - "//zml/core", - "//zml/meta", - "//zml/quantization", - ], -) - -zig_test( - name = "test", - deps = [":nn"], -) diff --git a/zml/core/ops.zig b/zml/ops.zig similarity index 99% rename from zml/core/ops.zig rename to zml/ops.zig index 59f5130a..f8fdba7e 100644 --- a/zml/core/ops.zig +++ b/zml/ops.zig @@ -1,9 +1,7 @@ const std = @import("std"); const builtin = @import("builtin"); -const constants = @import("zml/constants"); const func = @import("mlir/dialects/func"); -const meta = @import("zml/meta"); const mlir = @import("mlir"); const pjrt = @import("pjrt"); const platforms = @import("platforms"); @@ -13,8 +11,10 @@ const stdx = @import("stdx"); const Buffer = @import("buffer.zig").Buffer; const Bufferized = @import("mem.zig").Bufferized; const Compiler = @import("Compiler.zig"); +const constants = @import("constants.zig"); const CustomCallBuffer = @import("pjrtx.zig").CustomCallBuffer; const DataType = @import("dtype.zig").DataType; +const meta = @import("meta.zig"); const mlirx = @import("mlirx.zig"); const manual_computation = @import("ops/manual_computation.zig"); pub const manualComputation = manual_computation.manualComputation; @@ -667,7 +667,7 @@ pub fn @"while"( pub const while_ = @"while"; test @"while" { - const zml = @import("core.zig"); + const zml = @import("zml.zig"); const platform = zml.testing.env(); const While = struct { @@ -812,7 +812,7 @@ pub fn @"if"( } test "if" { - const zml = @import("core.zig"); + const zml = @import("zml.zig"); const platform = zml.testing.env(); const allocator = std.testing.allocator; @@ -893,7 +893,7 @@ pub fn if2( } test if2 { - const zml = @import("core.zig"); + const zml = @import("zml.zig"); const platform = zml.testing.env(); const allocator = std.testing.allocator; @@ -1107,7 +1107,7 @@ pub fn fly(inputs: anytype, outputs: anytype, opts: FlyOps) [outputs.len]Tensor } test "fly custom call, both entry ABIs" { - const zml = @import("core.zig"); + const zml = @import("zml.zig"); const platform = zml.testing.env(); if (platform.target != .rocm) return error.SkipZigTest; @@ -1298,7 +1298,7 @@ pub fn neuronNki(inputs: anytype, outputs: anytype, opts: NeuronNkiOps) [outputs } test "triton" { - const zml = @import("core.zig"); + const zml = @import("zml.zig"); const platform = zml.testing.env(); if (platform.target != .cuda and platform.target != .rocm and platform.target != .oneapi) return error.SkipZigTest; @@ -1617,7 +1617,7 @@ pub fn cute(inputs: anytype, outputs: anytype, opts: CuteOps) [outputs.len]Tenso } test "cute" { - const zml = @import("core.zig"); + const zml = @import("zml.zig"); const platform = zml.testing.env(); if (platform.target != .cuda) return error.SkipZigTest; @@ -1682,7 +1682,7 @@ test "cute" { } test "cuda_tile" { - const zml = @import("core.zig"); + const zml = @import("zml.zig"); const platform = zml.testing.env(); if (platform.target != .cuda) return error.SkipZigTest; @@ -1741,7 +1741,7 @@ test "cuda_tile" { } test "cuda_tile grid" { - const zml = @import("core.zig"); + const zml = @import("zml.zig"); const platform = zml.testing.env(); if (platform.target != .cuda) return error.SkipZigTest; @@ -1808,7 +1808,7 @@ test "cuda_tile grid" { } test "cuda_tile zeroed_outputs" { - const zml = @import("core.zig"); + const zml = @import("zml.zig"); const platform = zml.testing.env(); if (platform.target != .cuda) return error.SkipZigTest; @@ -2107,7 +2107,7 @@ fn scatterConfig( } test scatterConfig { - const zml = @import("core.zig"); + const zml = @import("zml.zig"); const platform = zml.testing.env(); var comp: zml.Compiler = .init(std.testing.allocator, std.testing.io, platform, .{}); @@ -2984,7 +2984,7 @@ fn customCallArgsFromPjrtCallFrame(I: type, O: type, A: type, call_frame: *pjrt. } test customCall { - const zml = @import("core.zig"); + const zml = @import("zml.zig"); const platform = zml.testing.env(); const tp = zml.testing.tpMesh(); diff --git a/zml/core/ops/manual_computation.zig b/zml/ops/manual_computation.zig similarity index 99% rename from zml/core/ops/manual_computation.zig rename to zml/ops/manual_computation.zig index 68dfaccf..a0ec0037 100644 --- a/zml/core/ops/manual_computation.zig +++ b/zml/ops/manual_computation.zig @@ -6,7 +6,7 @@ const stablehlo = @import("mlir/dialects/stablehlo"); const stdx = @import("stdx"); const Compiler = @import("../Compiler.zig"); -const meta = @import("zml/meta"); +const meta = @import("../meta.zig"); const mlirx = @import("../mlirx.zig"); const Shape = @import("../shape.zig").Shape; const Sharding = @import("../Sharding.zig"); @@ -376,7 +376,7 @@ const gspmd = struct { }; test manualComputation { - const zml = @import("../core.zig"); + const zml = @import("../zml.zig"); const allocator = std.testing.allocator; const io = std.testing.io; const platform = zml.testing.env(); @@ -436,7 +436,7 @@ test manualComputation { } test "manualComputation composes across logical and physical meshes" { - const zml = @import("../core.zig"); + const zml = @import("../zml.zig"); if (comptime !zml.Target.cpu.isEnabled()) return error.SkipZigTest; const allocator = std.testing.allocator; const io = std.testing.io; @@ -505,7 +505,7 @@ test "manualComputation composes across logical and physical meshes" { } test "partitionSpecs preserves optional tensors and active cache variants" { - const zml = @import("../core.zig"); + const zml = @import("../zml.zig"); const Logical = Sharding.Partitioning.Logical; const Inputs = struct { layers: []const Tensor, diff --git a/zml/core/pjrtx.zig b/zml/pjrtx.zig similarity index 99% rename from zml/core/pjrtx.zig rename to zml/pjrtx.zig index 3185b608..824d2bd1 100644 --- a/zml/core/pjrtx.zig +++ b/zml/pjrtx.zig @@ -1,11 +1,11 @@ const std = @import("std"); -const meta = @import("zml/meta"); const mlir = @import("mlir"); const pjrt = @import("pjrt"); const stablehlo = @import("mlir/dialects/stablehlo"); pub const DataType = @import("dtype.zig").DataType; +const meta = @import("meta.zig"); const Shape = @import("shape.zig").Shape; const Tensor = @import("tensor.zig").Tensor; diff --git a/zml/core/platform.zig b/zml/platform.zig similarity index 99% rename from zml/core/platform.zig rename to zml/platform.zig index 9c9b85a9..54fb90e4 100644 --- a/zml/core/platform.zig +++ b/zml/platform.zig @@ -2,18 +2,18 @@ const std = @import("std"); const builtin = @import("builtin"); const c = @import("c"); -const constants = @import("zml/constants"); const pjrt = @import("pjrt"); -const profiler_ = @import("zml/profiling").profiler; const stdx = @import("stdx"); pub const Target = @import("platforms").Platform; +const constants = @import("constants.zig"); const Exe = @import("exe.zig").Exe; -const flashinfer_cutlass_contract = @import("flashinfer_cutlass/contract.zig"); -const flashinfer_cutlass_runtime = @import("flashinfer_cutlass/runtime.zig"); +const flashinfer_cutlass_contract = @import("kernels/moe/flashinfer_cutlass/contract.zig"); +const flashinfer_cutlass_runtime = @import("kernels/moe/flashinfer_cutlass/runtime.zig"); const pjrtx = @import("pjrtx.zig"); +const profiler_ = @import("profiling/profiler.zig"); const Sharding = @import("Sharding.zig"); -const zml = @import("core.zig"); +const zml = @import("zml.zig"); const log = std.log.scoped(.zml); @@ -1041,7 +1041,7 @@ fn printCallbackInner(call_frame: *pjrt.ffi.CallFrame) !?*pjrt.ffi.Error { .device = device, .layout = .{ .tiled = .{ - .minor_to_major = constants.minorToMajor(shape.rank()), + .minor_to_major = zml.constants.minorToMajor(shape.rank()), .tile_dims = &.{}, .tile_dims_sizes = &.{}, }, diff --git a/zml/core/posix.h b/zml/posix.h similarity index 100% rename from zml/core/posix.h rename to zml/posix.h diff --git a/zml/profiling/BUILD.bazel b/zml/profiling/BUILD.bazel deleted file mode 100644 index 86675b54..00000000 --- a/zml/profiling/BUILD.bazel +++ /dev/null @@ -1,77 +0,0 @@ -load("@com_google_protobuf//bazel:upb_c_proto_library.bzl", "upb_c_proto_library") -load("@rules_cc//cc:defs.bzl", "cc_library") -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") - -upb_c_proto_library( - name = "profiler_options", - tags = ["manual"], - deps = ["@tsl//tsl/profiler/protobuf:profiler_options_proto"], -) - -cc_library( - name = "profiling_traceme_lib", - srcs = ["traceme_lib.cc"], - hdrs = ["traceme.h"], - defines = select({ - "//platforms:cuda.enabled": ["ZML_RUNTIME_CUDA"], - "//conditions:default": [], - }) + select({ - "//platforms:rocm.enabled": ["ZML_RUNTIME_ROCM"], - "//conditions:default": [], - }), - linkopts = select({ - "//platforms:rocm.enabled": ["-ldl"], - "//conditions:default": [], - }), - tags = ["manual"], - visibility = ["//visibility:public"], - deps = [ - "//ffi:cc", - "@tsl//tsl/profiler/lib:traceme", - "@tsl//tsl/profiler/protobuf:xplane_proto_cc", - "@xla//xla/tsl/profiler/backends/cpu:host_tracer_utils", - "@xla//xla/tsl/profiler/backends/cpu:traceme_recorder", - "@xla//xla/tsl/profiler/convert:post_process_single_host_xplane", - "@xla//xla/tsl/profiler/utils:time_utils", - "@xla//xla/tsl/profiler/utils:xplane_utils", - ] + select({ - "//platforms:cuda.enabled": [ - "//platforms/cuda:cuda_nvtx_headers", - ], - "//conditions:default": [], - }) + select({ - "//platforms:rocm.enabled": [ - "@amdrocm-profiler-base//:roctx_headers", - ], - "//conditions:default": [], - }), -) - -zig_library( - name = "profiling", - srcs = [ - "profiler.zig", - "tracer.zig", - ], - import_name = "zml/profiling", - main = "profiling.zig", - tags = ["manual"], - visibility = ["//visibility:public"], - zigopts = ["-lc"], - deps = [ - ":profiler_options", - ":profiling_traceme_lib", - "//bazel", - "//ffi:zig", - "//pjrt", - "//platforms", - "//stdx", - "//tools/xspace_to_perfetto:xspace_to_perfetto_lib", - "//upb", - ], -) - -zig_test( - name = "test", - deps = [":profiling"], -) diff --git a/zml/profiling/profiling.zig b/zml/profiling/profiling.zig deleted file mode 100644 index 41bf94f9..00000000 --- a/zml/profiling/profiling.zig +++ /dev/null @@ -1,12 +0,0 @@ -pub const profiler = @import("profiler.zig"); -pub const tracer = @import("tracer.zig"); - -test { - @import("std").testing.refAllDecls(@This()); -} - -comptime { - // Forces zig to compile `@zml//bazel` module even when users don't depend on it even indirectly. - // Otherwise exported functions declared in `bazel` aren't compiled. - @import("std").testing.refAllDecls(@import("bazel")); -} diff --git a/zml/quantization/quantization.zig b/zml/quantization.zig similarity index 99% rename from zml/quantization/quantization.zig rename to zml/quantization.zig index 70576db5..db20619a 100644 --- a/zml/quantization/quantization.zig +++ b/zml/quantization.zig @@ -3,13 +3,13 @@ const std = @import("std"); const stdx = @import("stdx"); -const zml = @import("zml/core"); +const zml = @import("zml.zig"); -const DataType = zml.DataType; -const platform_mod = zml.platform; +const DataType = @import("dtype.zig").DataType; +const platform_mod = @import("platform.zig"); const Platform = platform_mod.Platform; -const Shape = zml.Shape; -const Tensor = zml.Tensor; +const Shape = @import("shape.zig").Shape; +const Tensor = @import("tensor.zig").Tensor; pub const nvfp4_block_size = 16; pub const mx_block_size = 32; diff --git a/zml/quantization/BUILD.bazel b/zml/quantization/BUILD.bazel deleted file mode 100644 index 8e0bd565..00000000 --- a/zml/quantization/BUILD.bazel +++ /dev/null @@ -1,18 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") - -zig_library( - name = "quantization", - import_name = "zml/quantization", - main = "quantization.zig", - tags = ["manual"], - visibility = ["//visibility:public"], - deps = [ - "//stdx", - "//zml/core", - ], -) - -zig_test( - name = "test", - deps = [":quantization"], -) diff --git a/zml/core/safetensors.zig b/zml/safetensors.zig similarity index 100% rename from zml/core/safetensors.zig rename to zml/safetensors.zig diff --git a/zml/core/shape.zig b/zml/shape.zig similarity index 99% rename from zml/core/shape.zig rename to zml/shape.zig index dddcdf63..386b594e 100644 --- a/zml/core/shape.zig +++ b/zml/shape.zig @@ -2,9 +2,9 @@ const std = @import("std"); const testing = std.testing; const builtin = @import("builtin"); -const constants = @import("zml/constants"); const stdx = @import("stdx"); +const constants = @import("constants.zig"); const DataType = @import("dtype.zig").DataType; const Sharding = @import("Sharding.zig"); diff --git a/zml/core/slice.zig b/zml/slice.zig similarity index 99% rename from zml/core/slice.zig rename to zml/slice.zig index 8b593bd2..9451165b 100644 --- a/zml/core/slice.zig +++ b/zml/slice.zig @@ -1,12 +1,12 @@ const std = @import("std"); -const builtin = @import("builtin"); -const constants = @import("zml/constants"); const stdx = @import("stdx"); +const constants = @import("constants.zig"); const DataType = @import("dtype.zig").DataType; const floats = @import("floats.zig"); const Shape = @import("shape.zig").Shape; +const builtin = @import("builtin"); pub fn isBytes(comptime T: type) bool { const type_info = @typeInfo(T); diff --git a/zml/core/tensor.zig b/zml/tensor.zig similarity index 99% rename from zml/core/tensor.zig rename to zml/tensor.zig index 686f9683..ee084835 100644 --- a/zml/core/tensor.zig +++ b/zml/tensor.zig @@ -1,17 +1,17 @@ const std = @import("std"); const builtin = @import("builtin"); -const constants = @import("zml/constants"); -const meta = @import("zml/meta"); const mlir = @import("mlir"); const shardy = @import("mlir/dialects/shardy"); const stablehlo = @import("mlir/dialects/stablehlo"); const stdx = @import("stdx"); const Compiler = @import("Compiler.zig"); +const constants = @import("constants.zig"); const DataType = @import("dtype.zig").DataType; const mem = @import("mem.zig"); const Memory = @import("platform.zig").Memory; +const meta = @import("meta.zig"); const mlirx = @import("mlirx.zig"); const ops = @import("ops.zig"); const Platform = @import("platform.zig").Platform; @@ -322,7 +322,7 @@ pub const Tensor = struct { } test toMemory { - const zml = @import("core.zig"); + const zml = @import("zml.zig"); const platform = zml.testing.env(); const io = std.testing.io; @@ -479,7 +479,7 @@ pub const Tensor = struct { } test reuseBuffer { - const zml = @import("core.zig"); + const zml = @import("zml.zig"); const platform = zml.testing.env(); const io = std.testing.io; @@ -617,7 +617,7 @@ pub const Tensor = struct { } test fmod { - const zml = @import("core.zig"); + const zml = @import("zml.zig"); const platform = zml.testing.env(); const inputs: [2][6]f32 = .{ .{ -3.0, -2, -1, 1, 2, 3 }, .{ 1, 2, 3, 4, 5, -5 } }; @@ -799,7 +799,7 @@ pub const Tensor = struct { } test "Tensor.fft IRFFT restores RFFT last dimension" { - const zml = @import("core.zig"); + const zml = @import("zml.zig"); const platform = zml.testing.env(); const rfft_input_len: i64 = 10; @@ -909,7 +909,7 @@ pub const Tensor = struct { } test uniform { - const zml = @import("core.zig"); + const zml = @import("zml.zig"); const Stats = struct { const Stats = @This(); @@ -1007,7 +1007,7 @@ pub const Tensor = struct { } test gumbel { - const zml = @import("core.zig"); + const zml = @import("zml.zig"); const Stats = struct { const Stats = @This(); @@ -1382,7 +1382,7 @@ pub const Tensor = struct { test "convert f32 -> f4e2m1" { const floats = @import("floats.zig"); - const zml = @import("core.zig"); + const zml = @import("zml.zig"); const platform = zml.testing.env(); const x_f32 = [_]f32{ 0.0, 0.5, 1, 1.5, 2, 3, 4, 6, -0.0, -0.5, -1, -1.5, -2, -3, -4, -6 }; var x_f4: [x_f32.len / 2]floats.Float4E2M1.Packed = undefined; @@ -1415,7 +1415,7 @@ pub const Tensor = struct { } test "convert f32 -> f8e3m4" { - const zml = @import("core.zig"); + const zml = @import("zml.zig"); const platform = zml.testing.env(); const x = [_]f32{ 1.1 / 4.0, 1.1 / 8.0, 1.1 / 16.0, 1.1 / 32.0, 1.1 / 64.0, 1.1 / 128.0 }; @@ -1446,7 +1446,7 @@ pub const Tensor = struct { } test "convert u2 -> u8" { - const zml = @import("core.zig"); + const zml = @import("zml.zig"); const platform = zml.testing.env(); const x_u2: [2]stdx.Packed(u2) = .{ .pack(0, 1, 2, 3), .pack(3, 2, 1, 0) }; comptime std.debug.assert(@sizeOf(@TypeOf(x_u2)) == 2); @@ -1476,7 +1476,7 @@ pub const Tensor = struct { test "convert f4e2m1 -> f32" { const floats = @import("floats.zig"); - const zml = @import("core.zig"); + const zml = @import("zml.zig"); const platform = zml.testing.env(); const x_f4_packed = [_]floats.Float4E2M1.Packed{ .fromF32(0, 0.5), .fromF32(1, 1.5), .fromF32(2, 3), .fromF32(4, 6) }; @@ -1570,7 +1570,7 @@ pub const Tensor = struct { } test dot { - const zml = @import("core.zig"); + const zml = @import("zml.zig"); const platform = zml.testing.env(); inline for (.{ @@ -1719,7 +1719,7 @@ pub const Tensor = struct { } test leakyReLU { - const zml = @import("core.zig"); + const zml = @import("zml.zig"); const platform = zml.testing.env(); const input: Tensor = .init(.{2}, .f32); @@ -1757,7 +1757,7 @@ pub const Tensor = struct { } test swiglu { - const zml = @import("core.zig"); + const zml = @import("zml.zig"); const platform = zml.testing.env(); const input: Tensor = .init(.{ .d = 4 }, .f32); @@ -2030,7 +2030,7 @@ pub const Tensor = struct { } test cumulativeSum { - const zml = @import("core.zig"); + const zml = @import("zml.zig"); const platform = zml.testing.env(); const Local = struct { @@ -2338,7 +2338,7 @@ pub const Tensor = struct { } test slice { - const zml = @import("core.zig"); + const zml = @import("zml.zig"); const platform = zml.testing.env(); const x: Tensor = .init(.{ 2, 5 }, .f32); @@ -2378,7 +2378,7 @@ pub const Tensor = struct { } test "dynamic slices" { - const zml = @import("core.zig"); + const zml = @import("zml.zig"); const platform = zml.testing.env(); const Local = struct { @@ -2498,7 +2498,7 @@ pub const Tensor = struct { } test repeat1d { - const zml = @import("core.zig"); + const zml = @import("zml.zig"); const platform = zml.testing.env(); const Local = struct { @@ -2561,7 +2561,7 @@ pub const Tensor = struct { } test stutter1d { - const zml = @import("core.zig"); + const zml = @import("zml.zig"); const platform = zml.testing.env(); const Local = struct { @@ -2684,7 +2684,7 @@ pub const Tensor = struct { } test arange { - const zml = @import("core.zig"); + const zml = @import("zml.zig"); const platform = zml.testing.env(); const exe = try zml.module.compile( @@ -2770,7 +2770,7 @@ pub const Tensor = struct { } test scalar { - const zml = @import("core.zig"); + const zml = @import("zml.zig"); const platform = zml.testing.env(); const Local = struct { @@ -3063,7 +3063,7 @@ pub const Tensor = struct { } test gather { - const zml = @import("core.zig"); + const zml = @import("zml.zig"); const platform = zml.testing.env(); const Local = struct { @@ -3217,7 +3217,7 @@ pub const Tensor = struct { } test gatherSlices { - const zml = @import("core.zig"); + const zml = @import("zml.zig"); const platform = zml.testing.env(); const Local = struct { @@ -3426,7 +3426,7 @@ pub const Tensor = struct { } test scatterSlices { - const zml = @import("core.zig"); + const zml = @import("zml.zig"); const platform = zml.testing.env(); const Local = struct { @@ -3648,7 +3648,7 @@ pub const Tensor = struct { } test argMax { - const zml = @import("core.zig"); + const zml = @import("zml.zig"); const platform = zml.testing.env(); const allocator = std.testing.allocator; const ArgMaxTest = struct { @@ -3720,7 +3720,7 @@ pub const Tensor = struct { } test argsort { - const zml = @import("core.zig"); + const zml = @import("zml.zig"); const platform = zml.testing.env(); const Local = struct { @@ -3991,7 +3991,7 @@ pub const Tensor = struct { } test chunkExact { - const zml = @import("core.zig"); + const zml = @import("zml.zig"); const platform = zml.testing.env(); // Only test shapes @@ -4044,7 +4044,7 @@ pub const Tensor = struct { } test chunkAllowTrailing { - const zml = @import("core.zig"); + const zml = @import("zml.zig"); const platform = zml.testing.env(); // Only test shapes @@ -4102,7 +4102,7 @@ pub const Tensor = struct { } test split { - const zml = @import("core.zig"); + const zml = @import("zml.zig"); const platform = zml.testing.env(); const Local = struct { @@ -4236,7 +4236,7 @@ pub const Tensor = struct { } test dynamicUpdateSlice { - const zml = @import("core.zig"); + const zml = @import("zml.zig"); const platform = zml.testing.env(); { @@ -4432,7 +4432,7 @@ pub const Tensor = struct { } test toDiagonal { - const zml = @import("core.zig"); + const zml = @import("zml.zig"); const platform = zml.testing.env(); const Local = struct { @@ -4485,7 +4485,7 @@ pub const Tensor = struct { } test triangular { - const zml = @import("core.zig"); + const zml = @import("zml.zig"); const platform = zml.testing.env(); const Local = struct { @@ -4663,7 +4663,7 @@ pub const Tensor = struct { } test cartesianProduct { - const zml = @import("core.zig"); + const zml = @import("zml.zig"); const platform = zml.testing.env(); const x: Tensor = .init(.{6}, .i32); @@ -4741,7 +4741,7 @@ pub const Tensor = struct { } test cartesianProductStacked { - const zml = @import("core.zig"); + const zml = @import("zml.zig"); const platform = zml.testing.env(); const x: Tensor = .init(.{6}, .i32); const y: Tensor = .init(.{4}, .i32); @@ -4887,7 +4887,7 @@ fn getComparisonType(ctx: *mlir.Context, dtype: DataType) *const stablehlo.Compa } test "Tensor.maxPool1d" { - const zml = @import("core.zig"); + const zml = @import("zml.zig"); const platform = zml.testing.env(); const MaxPool = struct { @@ -4929,7 +4929,7 @@ test "Tensor.maxPool1d" { } test "Tensor.maxPool2d" { - const zml = @import("core.zig"); + const zml = @import("zml.zig"); const platform = zml.testing.env(); const MaxPool = struct { @@ -5038,7 +5038,7 @@ test transposeIsJustAReshape { } test "unused tensor" { - const zml = @import("core.zig"); + const zml = @import("zml.zig"); const platform = zml.testing.env(); const Local = struct { diff --git a/zml/core/testing.zig b/zml/testing.zig similarity index 98% rename from zml/core/testing.zig rename to zml/testing.zig index c4258f56..5e3d6333 100644 --- a/zml/core/testing.zig +++ b/zml/testing.zig @@ -2,12 +2,11 @@ const std = @import("std"); const builtin = @import("builtin"); const stdx = @import("stdx"); -const meta = @import("zml/meta"); const Platform = @import("platform.zig").Platform; const Sharding = @import("Sharding.zig"); const Slice = @import("slice.zig").Slice; -const zml = @import("core.zig"); +const zml = @import("zml.zig"); const log = std.log.scoped(.@"zml/testing"); @@ -367,7 +366,7 @@ pub fn expectEqualShapes(expected: zml.Shape, actual: zml.Shape) error{TestExpec } fn BufferizedWithArgs(comptime T: type) type { - return meta.MapType(zml.Tensor, zml.Buffer).map(T); + return zml.meta.MapType(zml.Tensor, zml.Buffer).map(T); } fn countWithPrefix(store: zml.io.TensorStore.View, prefix: []const u8) usize { @@ -419,7 +418,7 @@ pub fn testLayer( }; const input_name = try std.fmt.allocPrint(arena.allocator(), "{s}.in", .{name}); - const input_count = meta.count(zml.Tensor, &args); + const input_count = zml.meta.count(zml.Tensor, &args); const expected_input_count = countWithPrefix(activation_store, input_name); if (input_count != expected_input_count) { log.warn("Reference models uses {d} inputs, but implementation uses {d}", .{ expected_input_count, input_count }); @@ -427,7 +426,7 @@ pub fn testLayer( const store_input = activation_store.withPrefix(input_name); var ctx = LocalContext{ .activation_store = store_input }; - try meta.visit(struct { + try zml.meta.visit(struct { fn cb(ctx_: *LocalContext, tensor: *zml.Tensor) !void { var buffer: [256]u8 = undefined; const subkey = std.fmt.bufPrint(&buffer, "{d}", .{ctx_.index}) catch unreachable; @@ -455,7 +454,7 @@ pub fn testLayer( defer exe_results.deinit(allocator); var args_buffers = try zml.io.load(ArgsT, &args, allocator, io, platform, activation_store.store, .auto); - defer meta.visit(struct { + defer zml.meta.visit(struct { fn cb(_: void, b: *zml.Buffer) void { b.deinit(); } diff --git a/zml/zml.zig b/zml/zml.zig index 43cb0ecc..dafc3a0f 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -5,62 +5,85 @@ // Namespaces const std = @import("std"); -const core = @import("zml/core"); pub const bazel = @import("bazel"); -pub const constants = @import("zml/constants"); -pub const meta = @import("zml/meta"); +const c = @import("c"); pub const pjrt = @import("pjrt"); +const runfiles = @import("runfiles"); pub const stdx = @import("stdx"); pub const tokenizer = @import("zml/tokenizer"); -pub const tracer = @import("zml/profiling").tracer; pub const attention = @import("attention/attention.zig"); -pub const Buffer = core.Buffer; -pub const Compiler = core.Compiler; -pub const dtype = core.dtype; -pub const Data = core.Data; -pub const DataType = core.DataType; -pub const exe = core.exe; -pub const Exe = core.Exe; -pub const FnExe = core.FnExe; -pub const floats = core.floats; -pub const io = core.io; -pub const kernel = core.kernel; -pub const moe = @import("kernels/moe/moe.zig"); pub const paged_attention = @import("kernels/paged_attention/paged_attention.zig"); -pub const logo = @import("zml/logo"); -pub const mem = core.mem; -pub const Bufferized = core.Bufferized; -pub const mlir = core.mlir; -pub const nn = @import("zml/nn"); -pub const ops = core.ops; -pub const pjrtx = core.pjrtx; -pub const platform = core.platform; -pub const Memory = core.Memory; -pub const Platform = core.Platform; -pub const Target = core.Target; -pub const quantization = @import("zml/quantization"); +pub const Buffer = @import("buffer.zig").Buffer; +pub const Compiler = @import("Compiler.zig"); +pub const constants = @import("constants.zig"); +pub const dtype = @import("dtype.zig"); +pub const Data = dtype.Data; +pub const DataType = dtype.DataType; +pub const exe = @import("exe.zig"); +pub const Exe = exe.Exe; +pub const FnExe = exe.FnExe; +pub const floats = @import("floats.zig"); +pub const io = @import("io.zig"); +pub const kernel = @import("kernel/kernel.zig"); +pub const logo = @import("logo.zig"); +pub const mem = @import("mem.zig"); +pub const Bufferized = mem.Bufferized; +pub const meta = @import("meta.zig"); +pub const mlir = @import("mlirx.zig"); +pub const moe = @import("kernels/moe/moe.zig"); +pub const nn = @import("nn.zig"); +pub const ops = @import("ops.zig"); +pub const pjrtx = @import("pjrtx.zig"); +pub const platform = @import("platform.zig"); +pub const Memory = platform.Memory; +pub const Platform = platform.Platform; +pub const Target = platform.Target; +pub const tracer = @import("profiling/tracer.zig"); +pub const quantization = @import("quantization.zig"); pub const Quantization = quantization.Quantization; -pub const safetensors = core.safetensors; -pub const shape = core.shape; -pub const Shape = core.Shape; -pub const Sharding = core.Sharding; -pub const Mesh = core.Mesh; -pub const slice = core.slice; -pub const Slice = core.Slice; -pub const tensor = core.tensor; -pub const Tensor = core.Tensor; -pub const testing = core.testing; +pub const safetensors = @import("safetensors.zig"); +pub const shape = @import("shape.zig"); +pub const Shape = shape.Shape; +pub const Sharding = @import("Sharding.zig"); +pub const Mesh = Sharding.Mesh; +pub const slice = @import("slice.zig"); +pub const Slice = slice.Slice; +pub const tensor = @import("tensor.zig"); +pub const Tensor = tensor.Tensor; +pub const testing = @import("testing.zig"); /// Deprecated, use `zml.Compiler` instead -pub const module = core.module; +pub const module = struct { + pub const CompilationContext = Compiler; + pub const CompileError = Compiler.Error; + pub const CompilationOptions = Compiler.Options; + + /// Deprecated, use `platform.compileFn` or `zml.Compiler.compile` + pub fn compile( + allocator: std.mem.Allocator, + io_: std.Io, + comptime func: anytype, + args: std.meta.ArgsTuple(@TypeOf(func)), + platform_: *const Platform, + opts: Compiler.Options, + ) Compiler.Error!Exe { + return Compiler.compile(allocator, io_, platform_, func, args, opts); + } +}; test "zml" { std.testing.refAllDecls(@This()); } -pub const KiB = core.KiB; -pub const MiB = core.MiB; -pub const GiB = core.GiB; -pub const TiB = core.TiB; +pub const KiB = 1024; +pub const MiB = 1024 * KiB; +pub const GiB = 1024 * MiB; +pub const TiB = 1024 * GiB; + +comptime { + // Forces zig to compile `@zml//bazel` module even when users don't depend on it even indirectly. + // Otherwise exported functions declared in `bazel` aren't compiled. + std.testing.refAllDecls(bazel); +}