diff --git a/bin/zml-smi/tui/BUILD.bazel b/bin/zml-smi/tui/BUILD.bazel index 29f4824c..1a0b7f5d 100644 --- a/bin/zml-smi/tui/BUILD.bazel +++ b/bin/zml-smi/tui/BUILD.bazel @@ -2,8 +2,6 @@ load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") zig_library( name = "tui", - import_name = "zml-smi/tui", - main = "tui.zig", srcs = [ "data.zig", "image_cache.zig", @@ -40,12 +38,14 @@ zig_library( "assets/nvidia.png", "assets/tpu.png", ], + import_name = "zml-smi/tui", + main = "tui.zig", visibility = ["//visibility:public"], deps = [ "//bazel/stamp:zig", "//bin/zml-smi/info", "//bin/zml-smi/utils:double_buffer", - "//zml:logo", + "//zml/logo", "@libvaxis//:vaxis", ] + select({ "@platforms//os:macos": ["//bin/zml-smi/platforms/macos"], diff --git a/platforms/neuron/BUILD.bazel b/platforms/neuron/BUILD.bazel index ad537191..a4aa29db 100644 --- a/platforms/neuron/BUILD.bazel +++ b/platforms/neuron/BUILD.bazel @@ -134,7 +134,10 @@ zig_library( import_name = "platforms/neuron/nki_kernel", main = "nki_kernel.zig", tags = ["manual"], - visibility = ["//zml:__pkg__"], + visibility = [ + "//zml/core:__pkg__", + "//zml/kernels/flash_attention/nki:__pkg__", + ], deps = [ ":neuron", "//bazel", diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index f52ecdd5..e060f5bf 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -1,143 +1,11 @@ -load("@com_google_protobuf//bazel:upb_c_proto_library.bzl", "upb_c_proto_library") -load("@rules_cc//cc:defs.bzl", "cc_library") load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") -upb_c_proto_library( - name = "xla_data_upb", - tags = ["manual"], - deps = ["@xla//xla:xla_data_proto"], -) - -upb_c_proto_library( - name = "xla_compile_options_upb", - tags = ["manual"], - deps = ["@xla//xla/pjrt/proto:compile_options_proto"], -) - -upb_c_proto_library( - name = "profiler_options", - tags = ["manual"], - deps = ["@tsl//tsl/profiler/protobuf:profiler_options_proto"], -) - -cc_library( - name = "posix", - hdrs = ["posix.h"], - tags = ["manual"], -) - -cc_library( - name = "profiling_traceme_lib", - srcs = ["profiling/traceme_lib.cc"], - hdrs = ["profiling/traceme.h"], - defines = select({ - "//platforms:cuda.enabled": ["ZML_RUNTIME_CUDA"], - "//conditions:default": [], - }) + select({ - "//platforms:rocm.enabled": ["ZML_RUNTIME_ROCM"], - "//conditions:default": [], - }), - linkopts = select({ - "//platforms:rocm.enabled": ["-ldl"], - "//conditions:default": [], - }), - tags = ["manual"], - visibility = ["//visibility:public"], - deps = [ - "//ffi:cc", - "@tsl//tsl/profiler/lib:traceme", - "@tsl//tsl/profiler/protobuf:xplane_proto_cc", - "@xla//xla/tsl/profiler/backends/cpu:host_tracer_utils", - "@xla//xla/tsl/profiler/backends/cpu:traceme_recorder", - "@xla//xla/tsl/profiler/convert:post_process_single_host_xplane", - "@xla//xla/tsl/profiler/utils:time_utils", - "@xla//xla/tsl/profiler/utils:xplane_utils", - ] + select({ - "//platforms:cuda.enabled": [ - "//platforms/cuda:cuda_nvtx_headers", - ], - "//conditions:default": [], - }) + select({ - "//platforms:rocm.enabled": [ - "@amdrocm-profiler-base//:roctx_headers", - ], - "//conditions:default": [], - }), -) - -zig_library( - name = "logo", - srcs = ["logo.zig"], - import_name = "zml/logo", - main = "logo.zig", - visibility = ["//visibility:public"], -) - zig_library( name = "zml", srcs = [ - "Compiler.zig", - "Sharding.zig", - "attention/Options.zig", - "attention/attention.zig", - "buffer.zig", - "buffer/HostAccessible.zig", - "constants.zig", - "dtype.zig", - "exe.zig", - "floats.zig", - "io.zig", - "kernels/flash_attention/flash_attention.zig", - "kernels/flash_attention/tests.zig", - "kernels/moe/api.zig", - "kernels/moe/moe.zig", - "kernels/moe/tests.zig", - "kernels/moe/weights.zig", - "kernels/paged_attention/options.zig", - "kernels/paged_attention/paged_attention.zig", - "logo.zig", - "mem.zig", - "meta.zig", - "mlirx.zig", - "nn.zig", - "ops.zig", - "ops/manual_computation.zig", - "pjrtx.zig", - "platform.zig", - "profiling/profiler.zig", - "profiling/tracer.zig", - "quantization.zig", - "safetensors.zig", - "shape.zig", - "slice.zig", - "tensor.zig", - "testing.zig", "zml.zig", - "//zml/kernel", - "//zml/kernels/flash_attention/cute", - "//zml/kernels/flash_attention/metal", - "//zml/kernels/flash_attention/nki", - "//zml/kernels/moe/cute", - "//zml/kernels/moe/flashinfer_cutlass", - "//zml/kernels/moe/fly", - "//zml/kernels/moe/metal", - "//zml/kernels/moe/mosaic_tpu", - "//zml/kernels/moe/stablehlo", - "//zml/kernels/moe/tcl", - "//zml/kernels/moe/triton", - "//zml/kernels/paged_attention/cute", - "//zml/kernels/paged_attention/metal", - "//zml/kernels/paged_attention/mosaic_tpu", - "//zml/kernels/paged_attention/stablehlo", - "//zml/kernels/paged_attention/tcl", - "//zml/kernels/paged_attention/triton", + "//zml/attention", ], - data = select({ - "//platforms:neuron.enabled": [ - "//zml/kernels/flash_attention/nki:attention_kernel", - ], - "//conditions:default": [], - }), main = "zml.zig", tags = ["manual"], visibility = ["//visibility:public"], @@ -146,54 +14,25 @@ zig_library( "-freference-trace=20", ], deps = [ - ":posix", - ":profiler_options", - ":profiling_traceme_lib", - ":xla_compile_options_upb", - ":xla_data_upb", "//bazel", - "//dsl/cuda_tile:builder", - "//dsl/cute:builder", - "//dsl/fly:builder", - "//dsl/metal:builder", - "//dsl/mosaic_tpu:builder", - "//dsl/tcl:builder", - "//dsl/triton:builder", - "//ffi:zig", - "//mlir", - "//mlir/dialects/func", - "//mlir/dialects/llvm", - "//mlir/dialects/mosaic_tpu", - "//mlir/dialects/nvvm", - "//mlir/dialects/shardy", - "//mlir/dialects/stablehlo", - "//mlir/dialects/ttir", - "//mlir/dialects/vector", "//pjrt", - "//platforms", - "//platforms/tpu:ragged_paged", "//stdx", - "//tools/xspace_to_perfetto:xspace_to_perfetto_lib", - "//upb", - "//vfs", + "//zml/attention:options", + "//zml/constants", + "//zml/core", + "//zml/kernels/flash_attention", + "//zml/kernels/moe", + "//zml/kernels/paged_attention", + "//zml/logo", + "//zml/meta", + "//zml/nn", + "//zml/profiling", + "//zml/quantization", "//zml/tokenizer", - "@rules_zig//zig/runfiles", - "@zio", - ] + select({ - "//platforms:neuron.enabled": [ - "//platforms/neuron:nki_kernel", - "//platforms/neuron:topology", - ], - "//conditions:default": [], - }) + select({ - "//platforms:cuda.enabled": [ - "//platforms/cuda/flashinfer_cutlass_moe", - ], - "//conditions:default": [], - }), + ], ) zig_test( name = "test", - deps = ["//zml"], + deps = [":zml"], ) diff --git a/zml/attention/BUILD.bazel b/zml/attention/BUILD.bazel new file mode 100644 index 00000000..b9fbcc62 --- /dev/null +++ b/zml/attention/BUILD.bazel @@ -0,0 +1,14 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "options", + import_name = "zml/attention/options", + main = "Options.zig", + visibility = ["//visibility:public"], +) + +filegroup( + name = "attention", + srcs = ["attention.zig"], + visibility = ["//zml:__pkg__"], +) diff --git a/zml/attention/attention.zig b/zml/attention/attention.zig index edd9b4ca..c7829966 100644 --- a/zml/attention/attention.zig +++ b/zml/attention/attention.zig @@ -1,7 +1,9 @@ const std = @import("std"); +const flash_attention = @import("zml/kernels/flash_attention"); +pub const Options = @import("zml/attention/options"); + const zml = @import("../zml.zig"); -const flash_attention = @import("../kernels/flash_attention/flash_attention.zig"); test { std.testing.refAllDecls(@This()); @@ -93,8 +95,6 @@ pub const Metadata = union(BackendType) { } }; -pub const Options = @import("Options.zig"); - pub const Partitioning = struct { const Logical = zml.Sharding.Partitioning.Logical; diff --git a/zml/constants/BUILD.bazel b/zml/constants/BUILD.bazel new file mode 100644 index 00000000..b55a6e34 --- /dev/null +++ b/zml/constants/BUILD.bazel @@ -0,0 +1,13 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "constants", + import_name = "zml/constants", + main = "constants.zig", + visibility = ["//visibility:public"], +) + +zig_test( + name = "test", + deps = [":constants"], +) diff --git a/zml/constants.zig b/zml/constants/constants.zig similarity index 100% rename from zml/constants.zig rename to zml/constants/constants.zig diff --git a/zml/core/BUILD.bazel b/zml/core/BUILD.bazel new file mode 100644 index 00000000..18452d0b --- /dev/null +++ b/zml/core/BUILD.bazel @@ -0,0 +1,101 @@ +load("@com_google_protobuf//bazel:upb_c_proto_library.bzl", "upb_c_proto_library") +load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +upb_c_proto_library( + name = "xla_data_upb", + tags = ["manual"], + deps = ["@xla//xla:xla_data_proto"], +) + +upb_c_proto_library( + name = "xla_compile_options_upb", + tags = ["manual"], + deps = ["@xla//xla/pjrt/proto:compile_options_proto"], +) + +cc_library( + name = "posix", + hdrs = ["posix.h"], + tags = ["manual"], +) + +zig_library( + name = "core", + srcs = [ + "Compiler.zig", + "Sharding.zig", + "buffer.zig", + "buffer/HostAccessible.zig", + "dtype.zig", + "exe.zig", + "flashinfer_cutlass/contract.zig", + "flashinfer_cutlass/runtime.zig", + "floats.zig", + "io.zig", + "mem.zig", + "mlirx.zig", + "ops.zig", + "ops/manual_computation.zig", + "pjrtx.zig", + "platform.zig", + "safetensors.zig", + "shape.zig", + "slice.zig", + "tensor.zig", + "testing.zig", + "//zml/core/kernel", + ], + import_name = "zml/core", + main = "core.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + zigopts = [ + "-lc", + "-freference-trace=20", + ], + deps = [ + ":posix", + ":xla_compile_options_upb", + ":xla_data_upb", + "//bazel", + "//dsl/cuda_tile:builder", + "//dsl/cute:builder", + "//dsl/fly:builder", + "//dsl/metal:builder", + "//dsl/mosaic_tpu:builder", + "//dsl/tcl:builder", + "//dsl/triton:builder", + "//ffi:zig", + "//mlir", + "//mlir/dialects/func", + "//mlir/dialects/mosaic_tpu", + "//mlir/dialects/shardy", + "//mlir/dialects/stablehlo", + "//pjrt", + "//platforms", + "//stdx", + "//upb", + "//vfs", + "//zml/constants", + "//zml/meta", + "//zml/profiling", + "@zio", + ] + select({ + "//platforms:neuron.enabled": [ + "//platforms/neuron:nki_kernel", + "//platforms/neuron:topology", + ], + "//conditions:default": [], + }) + select({ + "//platforms:cuda.enabled": [ + "//platforms/cuda/flashinfer_cutlass_moe", + ], + "//conditions:default": [], + }), +) + +zig_test( + name = "test", + deps = [":core"], +) diff --git a/zml/Compiler.zig b/zml/core/Compiler.zig similarity index 99% rename from zml/Compiler.zig rename to zml/core/Compiler.zig index 93eafa8d..1bf994cc 100644 --- a/zml/Compiler.zig +++ b/zml/core/Compiler.zig @@ -2,11 +2,13 @@ const std = @import("std"); const c = @import("c"); const func = @import("mlir/dialects/func"); +const meta = @import("zml/meta"); const mlir = @import("mlir"); const pjrt = @import("pjrt"); const shardy = @import("mlir/dialects/shardy"); const stablehlo = @import("mlir/dialects/stablehlo"); const stdx = @import("stdx"); +const tracer = @import("zml/profiling").tracer; const upb = @import("upb"); const zio = @import("zio"); @@ -14,12 +16,10 @@ const Buffer = @import("buffer.zig").Buffer; const DataType = @import("dtype.zig").DataType; const Exe = @import("exe.zig").Exe; const Memory = @import("platform.zig").Memory; -const meta = @import("meta.zig"); const mlirx = @import("mlirx.zig"); const ops = @import("ops.zig"); const pjrtx = @import("pjrtx.zig"); const Platform = @import("platform.zig").Platform; -const tracer = @import("profiling/tracer.zig"); const Shape = @import("shape.zig").Shape; const Sharding = @import("Sharding.zig"); const Tensor = @import("tensor.zig").Tensor; diff --git a/zml/Sharding.zig b/zml/core/Sharding.zig similarity index 99% rename from zml/Sharding.zig rename to zml/core/Sharding.zig index 040216bc..1f05b4df 100644 --- a/zml/Sharding.zig +++ b/zml/core/Sharding.zig @@ -3249,7 +3249,7 @@ test "GSPMD attributes preserve folded logical axis order" { test "GSPMD executes a reduction across a folded logical axis" { if (comptime !Target.cpu.isEnabled()) return error.SkipZigTest; - const zml = @import("zml.zig"); + const zml = @import("core.zig"); const io = std.testing.io; const allocator = std.testing.allocator; diff --git a/zml/buffer.zig b/zml/core/buffer.zig similarity index 99% rename from zml/buffer.zig rename to zml/core/buffer.zig index 8714d80f..81112de9 100644 --- a/zml/buffer.zig +++ b/zml/core/buffer.zig @@ -1,14 +1,14 @@ const std = @import("std"); +const constants = @import("zml/constants"); +const meta = @import("zml/meta"); const pjrt = @import("pjrt"); const platforms = @import("platforms"); const stdx = @import("stdx"); -const constants = @import("constants.zig"); const DataType = @import("dtype.zig").DataType; const mem = @import("mem.zig"); const Memory = @import("platform.zig").Memory; -const meta = @import("meta.zig"); const pjrtx = @import("pjrtx.zig"); const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; @@ -376,7 +376,7 @@ pub const Buffer = struct { }; test "device round-trip" { - const zml = @import("zml.zig"); + const zml = @import("core.zig"); const io = std.testing.io; const allocator = std.testing.allocator; const platform = zml.testing.env(); diff --git a/zml/buffer/HostAccessible.zig b/zml/core/buffer/HostAccessible.zig similarity index 99% rename from zml/buffer/HostAccessible.zig rename to zml/core/buffer/HostAccessible.zig index b3e82a9c..db17ae7b 100644 --- a/zml/buffer/HostAccessible.zig +++ b/zml/core/buffer/HostAccessible.zig @@ -1878,7 +1878,7 @@ fn expectDevicesReadShards(platform: *const Platform, shape_: Shape) !void { } test "HostAccessible executions read shared inputs on every device" { - const zml = @import("../zml.zig"); + const zml = @import("../core.zig"); const allocator = std.testing.allocator; const io = std.testing.io; const platform = testing.env(); @@ -2107,7 +2107,7 @@ test "HostAccessible.View copies across layouts, shards and replicas" { } test "HostAccessible await after executable donation permits the next host update" { - const zml = @import("../zml.zig"); + const zml = @import("../core.zig"); const allocator = std.testing.allocator; const io = std.testing.io; const platform = testing.env(); @@ -2159,7 +2159,7 @@ test "HostAccessible await after executable donation permits the next host updat } test "HostAccessible await follows an output that doesn't reuse the pinned allocation" { - const zml = @import("../zml.zig"); + const zml = @import("../core.zig"); const allocator = std.testing.allocator; const io = std.testing.io; const platform = testing.env(); diff --git a/zml/core/core.zig b/zml/core/core.zig new file mode 100644 index 00000000..4ec2bea4 --- /dev/null +++ b/zml/core/core.zig @@ -0,0 +1,74 @@ +const std = @import("std"); + +const bazel = @import("bazel"); + +pub const Buffer = @import("buffer.zig").Buffer; +pub const Compiler = @import("Compiler.zig"); +pub const dtype = @import("dtype.zig"); +pub const Data = dtype.Data; +pub const DataType = dtype.DataType; +pub const exe = @import("exe.zig"); +pub const Exe = exe.Exe; +pub const FnExe = exe.FnExe; +pub const floats = @import("floats.zig"); +pub const io = @import("io.zig"); +pub const kernel = @import("kernel/kernel.zig"); +pub const mem = @import("mem.zig"); +pub const Bufferized = mem.Bufferized; +pub const mlir = @import("mlirx.zig"); +pub const ops = @import("ops.zig"); +pub const pjrtx = @import("pjrtx.zig"); +pub const platform = @import("platform.zig"); +pub const Memory = platform.Memory; +pub const Platform = platform.Platform; +pub const Target = platform.Target; +pub const safetensors = @import("safetensors.zig"); +pub const shape = @import("shape.zig"); +pub const Shape = shape.Shape; +pub const Sharding = @import("Sharding.zig"); +pub const Mesh = Sharding.Mesh; +pub const slice = @import("slice.zig"); +pub const Slice = slice.Slice; +pub const tensor = @import("tensor.zig"); +pub const Tensor = tensor.Tensor; +pub const testing = @import("testing.zig"); + +// Platform owns registration and runner lifetime until these are separated from core. +pub const flashinfer_cutlass = struct { + pub const contract = @import("flashinfer_cutlass/contract.zig"); + pub const runtime = @import("flashinfer_cutlass/runtime.zig"); +}; + +/// Deprecated, use `zml.Compiler` instead +pub const module = struct { + pub const CompilationContext = Compiler; + pub const CompileError = Compiler.Error; + pub const CompilationOptions = Compiler.Options; + + /// Deprecated, use `platform.compileFn` or `zml.Compiler.compile` + pub fn compile( + allocator: std.mem.Allocator, + io_: std.Io, + comptime func: anytype, + args: std.meta.ArgsTuple(@TypeOf(func)), + platform_: *const Platform, + opts: Compiler.Options, + ) Compiler.Error!Exe { + return Compiler.compile(allocator, io_, platform_, func, args, opts); + } +}; + +test "core" { + std.testing.refAllDecls(@This()); +} + +pub const KiB = 1024; +pub const MiB = 1024 * KiB; +pub const GiB = 1024 * MiB; +pub const TiB = 1024 * GiB; + +comptime { + // Forces zig to compile `@zml//bazel` module even when users don't depend on it even indirectly. + // Otherwise exported functions declared in `bazel` aren't compiled. + std.testing.refAllDecls(bazel); +} diff --git a/zml/dtype.zig b/zml/core/dtype.zig similarity index 100% rename from zml/dtype.zig rename to zml/core/dtype.zig diff --git a/zml/exe.zig b/zml/core/exe.zig similarity index 99% rename from zml/exe.zig rename to zml/core/exe.zig index 32c0ac48..e370c708 100644 --- a/zml/exe.zig +++ b/zml/core/exe.zig @@ -1,15 +1,15 @@ const std = @import("std"); const builtin = @import("builtin"); +const meta = @import("zml/meta"); const pjrt = @import("pjrt"); const stdx = @import("stdx"); +const tracer = @import("zml/profiling").tracer; const Buffer = @import("buffer.zig").Buffer; const Compiler = @import("Compiler.zig"); const mem = @import("mem.zig"); -const meta = @import("meta.zig"); const Platform = @import("platform.zig").Platform; -const tracer = @import("profiling/tracer.zig"); const Shape = @import("shape.zig").Shape; const Sharding = @import("Sharding.zig"); const Tensor = @import("tensor.zig").Tensor; diff --git a/zml/kernels/moe/flashinfer_cutlass/contract.zig b/zml/core/flashinfer_cutlass/contract.zig similarity index 100% rename from zml/kernels/moe/flashinfer_cutlass/contract.zig rename to zml/core/flashinfer_cutlass/contract.zig diff --git a/zml/kernels/moe/flashinfer_cutlass/runtime.zig b/zml/core/flashinfer_cutlass/runtime.zig similarity index 99% rename from zml/kernels/moe/flashinfer_cutlass/runtime.zig rename to zml/core/flashinfer_cutlass/runtime.zig index 028ef892..fc72f1be 100644 --- a/zml/kernels/moe/flashinfer_cutlass/runtime.zig +++ b/zml/core/flashinfer_cutlass/runtime.zig @@ -3,7 +3,7 @@ const std = @import("std"); const fi_cutlass_moe = @import("platforms/cuda/flashinfer_cutlass_moe"); const pjrt = @import("pjrt"); -const DataType = @import("../../../dtype.zig").DataType; +const DataType = @import("../dtype.zig").DataType; const contract = @import("contract.zig"); const log = std.log.scoped(.@"zml/kernels/moe/flashinfer_cutlass/runtime"); diff --git a/zml/floats.zig b/zml/core/floats.zig similarity index 100% rename from zml/floats.zig rename to zml/core/floats.zig diff --git a/zml/io.zig b/zml/core/io.zig similarity index 99% rename from zml/io.zig rename to zml/core/io.zig index ab8d8e6e..93829541 100644 --- a/zml/io.zig +++ b/zml/core/io.zig @@ -1,19 +1,19 @@ const std = @import("std"); +const meta = @import("zml/meta"); const pjrt = @import("pjrt"); const stdx = @import("stdx"); +const tracer = @import("zml/profiling").tracer; pub const VFS = @import("vfs"); const Buffer = @import("buffer.zig").Buffer; -const Bufferized = @import("zml.zig").Bufferized; +const Bufferized = @import("core.zig").Bufferized; const CreateOptions = @import("platform.zig").CreateOptions; const Exe = @import("exe.zig").Exe; const mem = @import("mem.zig"); const Memory = @import("platform.zig").Memory; -const meta = @import("meta.zig"); const pjrtx = @import("pjrtx.zig"); const Platform = @import("platform.zig").Platform; -const tracer = @import("profiling/tracer.zig"); const safetensors = @import("safetensors.zig"); const Shape = @import("shape.zig").Shape; const Sharding = @import("Sharding.zig"); diff --git a/zml/core/kernel/BUILD.bazel b/zml/core/kernel/BUILD.bazel new file mode 100644 index 00000000..31638d07 --- /dev/null +++ b/zml/core/kernel/BUILD.bazel @@ -0,0 +1,15 @@ +package(default_visibility = ["//zml/core:__pkg__"]) + +filegroup( + name = "kernel", + srcs = [ + "kernel.zig", + "//zml/core/kernel/cuda_tile", + "//zml/core/kernel/cute", + "//zml/core/kernel/fly", + "//zml/core/kernel/metal", + "//zml/core/kernel/mosaic_tpu", + "//zml/core/kernel/tcl", + "//zml/core/kernel/triton", + ], +) diff --git a/zml/kernel/cuda_tile/BUILD.bazel b/zml/core/kernel/cuda_tile/BUILD.bazel similarity index 63% rename from zml/kernel/cuda_tile/BUILD.bazel rename to zml/core/kernel/cuda_tile/BUILD.bazel index 89b7977e..a3e7dc19 100644 --- a/zml/kernel/cuda_tile/BUILD.bazel +++ b/zml/core/kernel/cuda_tile/BUILD.bazel @@ -1,4 +1,4 @@ -package(default_visibility = ["//zml/kernel:__pkg__"]) +package(default_visibility = ["//zml/core/kernel:__pkg__"]) filegroup( name = "cuda_tile", diff --git a/zml/kernel/cuda_tile/cuda_tile.zig b/zml/core/kernel/cuda_tile/cuda_tile.zig similarity index 100% rename from zml/kernel/cuda_tile/cuda_tile.zig rename to zml/core/kernel/cuda_tile/cuda_tile.zig diff --git a/zml/kernel/cuda_tile/test.zig b/zml/core/kernel/cuda_tile/test.zig similarity index 100% rename from zml/kernel/cuda_tile/test.zig rename to zml/core/kernel/cuda_tile/test.zig diff --git a/zml/kernel/cute/BUILD.bazel b/zml/core/kernel/cute/BUILD.bazel similarity index 60% rename from zml/kernel/cute/BUILD.bazel rename to zml/core/kernel/cute/BUILD.bazel index 7a613072..73c8c128 100644 --- a/zml/kernel/cute/BUILD.bazel +++ b/zml/core/kernel/cute/BUILD.bazel @@ -1,4 +1,4 @@ -package(default_visibility = ["//zml/kernel:__pkg__"]) +package(default_visibility = ["//zml/core/kernel:__pkg__"]) filegroup( name = "cute", diff --git a/zml/kernel/cute/cute.zig b/zml/core/kernel/cute/cute.zig similarity index 100% rename from zml/kernel/cute/cute.zig rename to zml/core/kernel/cute/cute.zig diff --git a/zml/kernel/cute/test.zig b/zml/core/kernel/cute/test.zig similarity index 100% rename from zml/kernel/cute/test.zig rename to zml/core/kernel/cute/test.zig diff --git a/zml/kernel/fly/BUILD.bazel b/zml/core/kernel/fly/BUILD.bazel similarity index 60% rename from zml/kernel/fly/BUILD.bazel rename to zml/core/kernel/fly/BUILD.bazel index 57203521..11fea497 100644 --- a/zml/kernel/fly/BUILD.bazel +++ b/zml/core/kernel/fly/BUILD.bazel @@ -1,4 +1,4 @@ -package(default_visibility = ["//zml/kernel:__pkg__"]) +package(default_visibility = ["//zml/core/kernel:__pkg__"]) filegroup( name = "fly", diff --git a/zml/kernel/fly/fly.zig b/zml/core/kernel/fly/fly.zig similarity index 100% rename from zml/kernel/fly/fly.zig rename to zml/core/kernel/fly/fly.zig diff --git a/zml/kernel/fly/test.zig b/zml/core/kernel/fly/test.zig similarity index 99% rename from zml/kernel/fly/test.zig rename to zml/core/kernel/fly/test.zig index ae788d96..6b77ad3e 100644 --- a/zml/kernel/fly/test.zig +++ b/zml/core/kernel/fly/test.zig @@ -3,7 +3,7 @@ const std = @import("std"); const fly_builder = @import("dsl/fly/builder"); const fly = @import("fly.zig"); -const zml = @import("../../zml.zig"); +const zml = @import("../../core.zig"); /// C = A + B over (M, N) f32, 128-bit vectorized and predicated on the /// ragged edge. diff --git a/zml/kernel/kernel.zig b/zml/core/kernel/kernel.zig similarity index 100% rename from zml/kernel/kernel.zig rename to zml/core/kernel/kernel.zig diff --git a/zml/kernel/metal/BUILD.bazel b/zml/core/kernel/metal/BUILD.bazel similarity index 61% rename from zml/kernel/metal/BUILD.bazel rename to zml/core/kernel/metal/BUILD.bazel index 4007beb4..d0a2d380 100644 --- a/zml/kernel/metal/BUILD.bazel +++ b/zml/core/kernel/metal/BUILD.bazel @@ -1,4 +1,4 @@ -package(default_visibility = ["//zml/kernel:__pkg__"]) +package(default_visibility = ["//zml/core/kernel:__pkg__"]) filegroup( name = "metal", diff --git a/zml/kernel/metal/metal.zig b/zml/core/kernel/metal/metal.zig similarity index 100% rename from zml/kernel/metal/metal.zig rename to zml/core/kernel/metal/metal.zig diff --git a/zml/kernel/metal/test.zig b/zml/core/kernel/metal/test.zig similarity index 98% rename from zml/kernel/metal/test.zig rename to zml/core/kernel/metal/test.zig index 0a342af0..a184869c 100644 --- a/zml/kernel/metal/test.zig +++ b/zml/core/kernel/metal/test.zig @@ -1,7 +1,7 @@ const std = @import("std"); const metal = @import("metal.zig"); -const zml = @import("../../zml.zig"); +const zml = @import("../../core.zig"); /// `out[i] = a[i] * scale + b[i]` over `n` floats. const MetalAxpy = struct { diff --git a/zml/kernel/mosaic_tpu/BUILD.bazel b/zml/core/kernel/mosaic_tpu/BUILD.bazel similarity index 53% rename from zml/kernel/mosaic_tpu/BUILD.bazel rename to zml/core/kernel/mosaic_tpu/BUILD.bazel index ba3b6b66..ae2d346d 100644 --- a/zml/kernel/mosaic_tpu/BUILD.bazel +++ b/zml/core/kernel/mosaic_tpu/BUILD.bazel @@ -1,4 +1,4 @@ -package(default_visibility = ["//zml/kernel:__pkg__"]) +package(default_visibility = ["//zml/core/kernel:__pkg__"]) filegroup( name = "mosaic_tpu", diff --git a/zml/kernel/mosaic_tpu/mosaic_tpu.zig b/zml/core/kernel/mosaic_tpu/mosaic_tpu.zig similarity index 100% rename from zml/kernel/mosaic_tpu/mosaic_tpu.zig rename to zml/core/kernel/mosaic_tpu/mosaic_tpu.zig diff --git a/zml/kernel/tcl/BUILD.bazel b/zml/core/kernel/tcl/BUILD.bazel similarity index 60% rename from zml/kernel/tcl/BUILD.bazel rename to zml/core/kernel/tcl/BUILD.bazel index bc7a0aa2..6a1229c1 100644 --- a/zml/kernel/tcl/BUILD.bazel +++ b/zml/core/kernel/tcl/BUILD.bazel @@ -1,4 +1,4 @@ -package(default_visibility = ["//zml/kernel:__pkg__"]) +package(default_visibility = ["//zml/core/kernel:__pkg__"]) filegroup( name = "tcl", diff --git a/zml/kernel/tcl/tcl.zig b/zml/core/kernel/tcl/tcl.zig similarity index 100% rename from zml/kernel/tcl/tcl.zig rename to zml/core/kernel/tcl/tcl.zig diff --git a/zml/kernel/tcl/test.zig b/zml/core/kernel/tcl/test.zig similarity index 99% rename from zml/kernel/tcl/test.zig rename to zml/core/kernel/tcl/test.zig index b9ae1875..20804b8b 100644 --- a/zml/kernel/tcl/test.zig +++ b/zml/core/kernel/tcl/test.zig @@ -1,6 +1,6 @@ const std = @import("std"); -const zml = @import("../../zml.zig"); +const zml = @import("../../core.zig"); const tcl = @import("tcl.zig"); /// `furiosa.tcl.examples.softmax` over f32. diff --git a/zml/kernel/triton/BUILD.bazel b/zml/core/kernel/triton/BUILD.bazel similarity index 50% rename from zml/kernel/triton/BUILD.bazel rename to zml/core/kernel/triton/BUILD.bazel index 1e992352..0eface24 100644 --- a/zml/kernel/triton/BUILD.bazel +++ b/zml/core/kernel/triton/BUILD.bazel @@ -1,4 +1,4 @@ -package(default_visibility = ["//zml/kernel:__pkg__"]) +package(default_visibility = ["//zml/core/kernel:__pkg__"]) filegroup( name = "triton", diff --git a/zml/kernel/triton/triton.zig b/zml/core/kernel/triton/triton.zig similarity index 100% rename from zml/kernel/triton/triton.zig rename to zml/core/kernel/triton/triton.zig diff --git a/zml/mem.zig b/zml/core/mem.zig similarity index 99% rename from zml/mem.zig rename to zml/core/mem.zig index a64ae968..3007044c 100644 --- a/zml/mem.zig +++ b/zml/core/mem.zig @@ -3,12 +3,12 @@ const Allocator = std.mem.Allocator; const Alignment = std.mem.Alignment; const assert = std.debug.assert; +const meta = @import("zml/meta"); const pjrt = @import("pjrt"); const Buffer = @import("buffer.zig").Buffer; const Device = @import("platform.zig").Device; const Memory = @import("platform.zig").Memory; -const meta = @import("meta.zig"); const Platform = @import("platform.zig").Platform; const Tensor = @import("tensor.zig").Tensor; diff --git a/zml/mlirx.zig b/zml/core/mlirx.zig similarity index 100% rename from zml/mlirx.zig rename to zml/core/mlirx.zig diff --git a/zml/ops.zig b/zml/core/ops.zig similarity index 99% rename from zml/ops.zig rename to zml/core/ops.zig index f8fdba7e..59f5130a 100644 --- a/zml/ops.zig +++ b/zml/core/ops.zig @@ -1,7 +1,9 @@ const std = @import("std"); const builtin = @import("builtin"); +const constants = @import("zml/constants"); const func = @import("mlir/dialects/func"); +const meta = @import("zml/meta"); const mlir = @import("mlir"); const pjrt = @import("pjrt"); const platforms = @import("platforms"); @@ -11,10 +13,8 @@ const stdx = @import("stdx"); const Buffer = @import("buffer.zig").Buffer; const Bufferized = @import("mem.zig").Bufferized; const Compiler = @import("Compiler.zig"); -const constants = @import("constants.zig"); const CustomCallBuffer = @import("pjrtx.zig").CustomCallBuffer; const DataType = @import("dtype.zig").DataType; -const meta = @import("meta.zig"); const mlirx = @import("mlirx.zig"); const manual_computation = @import("ops/manual_computation.zig"); pub const manualComputation = manual_computation.manualComputation; @@ -667,7 +667,7 @@ pub fn @"while"( pub const while_ = @"while"; test @"while" { - const zml = @import("zml.zig"); + const zml = @import("core.zig"); const platform = zml.testing.env(); const While = struct { @@ -812,7 +812,7 @@ pub fn @"if"( } test "if" { - const zml = @import("zml.zig"); + const zml = @import("core.zig"); const platform = zml.testing.env(); const allocator = std.testing.allocator; @@ -893,7 +893,7 @@ pub fn if2( } test if2 { - const zml = @import("zml.zig"); + const zml = @import("core.zig"); const platform = zml.testing.env(); const allocator = std.testing.allocator; @@ -1107,7 +1107,7 @@ pub fn fly(inputs: anytype, outputs: anytype, opts: FlyOps) [outputs.len]Tensor } test "fly custom call, both entry ABIs" { - const zml = @import("zml.zig"); + const zml = @import("core.zig"); const platform = zml.testing.env(); if (platform.target != .rocm) return error.SkipZigTest; @@ -1298,7 +1298,7 @@ pub fn neuronNki(inputs: anytype, outputs: anytype, opts: NeuronNkiOps) [outputs } test "triton" { - const zml = @import("zml.zig"); + const zml = @import("core.zig"); const platform = zml.testing.env(); if (platform.target != .cuda and platform.target != .rocm and platform.target != .oneapi) return error.SkipZigTest; @@ -1617,7 +1617,7 @@ pub fn cute(inputs: anytype, outputs: anytype, opts: CuteOps) [outputs.len]Tenso } test "cute" { - const zml = @import("zml.zig"); + const zml = @import("core.zig"); const platform = zml.testing.env(); if (platform.target != .cuda) return error.SkipZigTest; @@ -1682,7 +1682,7 @@ test "cute" { } test "cuda_tile" { - const zml = @import("zml.zig"); + const zml = @import("core.zig"); const platform = zml.testing.env(); if (platform.target != .cuda) return error.SkipZigTest; @@ -1741,7 +1741,7 @@ test "cuda_tile" { } test "cuda_tile grid" { - const zml = @import("zml.zig"); + const zml = @import("core.zig"); const platform = zml.testing.env(); if (platform.target != .cuda) return error.SkipZigTest; @@ -1808,7 +1808,7 @@ test "cuda_tile grid" { } test "cuda_tile zeroed_outputs" { - const zml = @import("zml.zig"); + const zml = @import("core.zig"); const platform = zml.testing.env(); if (platform.target != .cuda) return error.SkipZigTest; @@ -2107,7 +2107,7 @@ fn scatterConfig( } test scatterConfig { - const zml = @import("zml.zig"); + const zml = @import("core.zig"); const platform = zml.testing.env(); var comp: zml.Compiler = .init(std.testing.allocator, std.testing.io, platform, .{}); @@ -2984,7 +2984,7 @@ fn customCallArgsFromPjrtCallFrame(I: type, O: type, A: type, call_frame: *pjrt. } test customCall { - const zml = @import("zml.zig"); + const zml = @import("core.zig"); const platform = zml.testing.env(); const tp = zml.testing.tpMesh(); diff --git a/zml/ops/manual_computation.zig b/zml/core/ops/manual_computation.zig similarity index 99% rename from zml/ops/manual_computation.zig rename to zml/core/ops/manual_computation.zig index a0ec0037..68dfaccf 100644 --- a/zml/ops/manual_computation.zig +++ b/zml/core/ops/manual_computation.zig @@ -6,7 +6,7 @@ const stablehlo = @import("mlir/dialects/stablehlo"); const stdx = @import("stdx"); const Compiler = @import("../Compiler.zig"); -const meta = @import("../meta.zig"); +const meta = @import("zml/meta"); const mlirx = @import("../mlirx.zig"); const Shape = @import("../shape.zig").Shape; const Sharding = @import("../Sharding.zig"); @@ -376,7 +376,7 @@ const gspmd = struct { }; test manualComputation { - const zml = @import("../zml.zig"); + const zml = @import("../core.zig"); const allocator = std.testing.allocator; const io = std.testing.io; const platform = zml.testing.env(); @@ -436,7 +436,7 @@ test manualComputation { } test "manualComputation composes across logical and physical meshes" { - const zml = @import("../zml.zig"); + const zml = @import("../core.zig"); if (comptime !zml.Target.cpu.isEnabled()) return error.SkipZigTest; const allocator = std.testing.allocator; const io = std.testing.io; @@ -505,7 +505,7 @@ test "manualComputation composes across logical and physical meshes" { } test "partitionSpecs preserves optional tensors and active cache variants" { - const zml = @import("../zml.zig"); + const zml = @import("../core.zig"); const Logical = Sharding.Partitioning.Logical; const Inputs = struct { layers: []const Tensor, diff --git a/zml/pjrtx.zig b/zml/core/pjrtx.zig similarity index 99% rename from zml/pjrtx.zig rename to zml/core/pjrtx.zig index 824d2bd1..3185b608 100644 --- a/zml/pjrtx.zig +++ b/zml/core/pjrtx.zig @@ -1,11 +1,11 @@ const std = @import("std"); +const meta = @import("zml/meta"); const mlir = @import("mlir"); const pjrt = @import("pjrt"); const stablehlo = @import("mlir/dialects/stablehlo"); pub const DataType = @import("dtype.zig").DataType; -const meta = @import("meta.zig"); const Shape = @import("shape.zig").Shape; const Tensor = @import("tensor.zig").Tensor; diff --git a/zml/platform.zig b/zml/core/platform.zig similarity index 99% rename from zml/platform.zig rename to zml/core/platform.zig index 54fb90e4..9c9b85a9 100644 --- a/zml/platform.zig +++ b/zml/core/platform.zig @@ -2,18 +2,18 @@ const std = @import("std"); const builtin = @import("builtin"); const c = @import("c"); +const constants = @import("zml/constants"); const pjrt = @import("pjrt"); +const profiler_ = @import("zml/profiling").profiler; const stdx = @import("stdx"); pub const Target = @import("platforms").Platform; -const constants = @import("constants.zig"); const Exe = @import("exe.zig").Exe; -const flashinfer_cutlass_contract = @import("kernels/moe/flashinfer_cutlass/contract.zig"); -const flashinfer_cutlass_runtime = @import("kernels/moe/flashinfer_cutlass/runtime.zig"); +const flashinfer_cutlass_contract = @import("flashinfer_cutlass/contract.zig"); +const flashinfer_cutlass_runtime = @import("flashinfer_cutlass/runtime.zig"); const pjrtx = @import("pjrtx.zig"); -const profiler_ = @import("profiling/profiler.zig"); const Sharding = @import("Sharding.zig"); -const zml = @import("zml.zig"); +const zml = @import("core.zig"); const log = std.log.scoped(.zml); @@ -1041,7 +1041,7 @@ fn printCallbackInner(call_frame: *pjrt.ffi.CallFrame) !?*pjrt.ffi.Error { .device = device, .layout = .{ .tiled = .{ - .minor_to_major = zml.constants.minorToMajor(shape.rank()), + .minor_to_major = constants.minorToMajor(shape.rank()), .tile_dims = &.{}, .tile_dims_sizes = &.{}, }, diff --git a/zml/posix.h b/zml/core/posix.h similarity index 100% rename from zml/posix.h rename to zml/core/posix.h diff --git a/zml/safetensors.zig b/zml/core/safetensors.zig similarity index 100% rename from zml/safetensors.zig rename to zml/core/safetensors.zig diff --git a/zml/shape.zig b/zml/core/shape.zig similarity index 99% rename from zml/shape.zig rename to zml/core/shape.zig index 386b594e..dddcdf63 100644 --- a/zml/shape.zig +++ b/zml/core/shape.zig @@ -2,9 +2,9 @@ const std = @import("std"); const testing = std.testing; const builtin = @import("builtin"); +const constants = @import("zml/constants"); const stdx = @import("stdx"); -const constants = @import("constants.zig"); const DataType = @import("dtype.zig").DataType; const Sharding = @import("Sharding.zig"); diff --git a/zml/slice.zig b/zml/core/slice.zig similarity index 99% rename from zml/slice.zig rename to zml/core/slice.zig index 9451165b..8b593bd2 100644 --- a/zml/slice.zig +++ b/zml/core/slice.zig @@ -1,12 +1,12 @@ const std = @import("std"); +const builtin = @import("builtin"); +const constants = @import("zml/constants"); const stdx = @import("stdx"); -const constants = @import("constants.zig"); const DataType = @import("dtype.zig").DataType; const floats = @import("floats.zig"); const Shape = @import("shape.zig").Shape; -const builtin = @import("builtin"); pub fn isBytes(comptime T: type) bool { const type_info = @typeInfo(T); diff --git a/zml/tensor.zig b/zml/core/tensor.zig similarity index 99% rename from zml/tensor.zig rename to zml/core/tensor.zig index ee084835..686f9683 100644 --- a/zml/tensor.zig +++ b/zml/core/tensor.zig @@ -1,17 +1,17 @@ const std = @import("std"); const builtin = @import("builtin"); +const constants = @import("zml/constants"); +const meta = @import("zml/meta"); const mlir = @import("mlir"); const shardy = @import("mlir/dialects/shardy"); const stablehlo = @import("mlir/dialects/stablehlo"); const stdx = @import("stdx"); const Compiler = @import("Compiler.zig"); -const constants = @import("constants.zig"); const DataType = @import("dtype.zig").DataType; const mem = @import("mem.zig"); const Memory = @import("platform.zig").Memory; -const meta = @import("meta.zig"); const mlirx = @import("mlirx.zig"); const ops = @import("ops.zig"); const Platform = @import("platform.zig").Platform; @@ -322,7 +322,7 @@ pub const Tensor = struct { } test toMemory { - const zml = @import("zml.zig"); + const zml = @import("core.zig"); const platform = zml.testing.env(); const io = std.testing.io; @@ -479,7 +479,7 @@ pub const Tensor = struct { } test reuseBuffer { - const zml = @import("zml.zig"); + const zml = @import("core.zig"); const platform = zml.testing.env(); const io = std.testing.io; @@ -617,7 +617,7 @@ pub const Tensor = struct { } test fmod { - const zml = @import("zml.zig"); + const zml = @import("core.zig"); const platform = zml.testing.env(); const inputs: [2][6]f32 = .{ .{ -3.0, -2, -1, 1, 2, 3 }, .{ 1, 2, 3, 4, 5, -5 } }; @@ -799,7 +799,7 @@ pub const Tensor = struct { } test "Tensor.fft IRFFT restores RFFT last dimension" { - const zml = @import("zml.zig"); + const zml = @import("core.zig"); const platform = zml.testing.env(); const rfft_input_len: i64 = 10; @@ -909,7 +909,7 @@ pub const Tensor = struct { } test uniform { - const zml = @import("zml.zig"); + const zml = @import("core.zig"); const Stats = struct { const Stats = @This(); @@ -1007,7 +1007,7 @@ pub const Tensor = struct { } test gumbel { - const zml = @import("zml.zig"); + const zml = @import("core.zig"); const Stats = struct { const Stats = @This(); @@ -1382,7 +1382,7 @@ pub const Tensor = struct { test "convert f32 -> f4e2m1" { const floats = @import("floats.zig"); - const zml = @import("zml.zig"); + const zml = @import("core.zig"); const platform = zml.testing.env(); const x_f32 = [_]f32{ 0.0, 0.5, 1, 1.5, 2, 3, 4, 6, -0.0, -0.5, -1, -1.5, -2, -3, -4, -6 }; var x_f4: [x_f32.len / 2]floats.Float4E2M1.Packed = undefined; @@ -1415,7 +1415,7 @@ pub const Tensor = struct { } test "convert f32 -> f8e3m4" { - const zml = @import("zml.zig"); + const zml = @import("core.zig"); const platform = zml.testing.env(); const x = [_]f32{ 1.1 / 4.0, 1.1 / 8.0, 1.1 / 16.0, 1.1 / 32.0, 1.1 / 64.0, 1.1 / 128.0 }; @@ -1446,7 +1446,7 @@ pub const Tensor = struct { } test "convert u2 -> u8" { - const zml = @import("zml.zig"); + const zml = @import("core.zig"); const platform = zml.testing.env(); const x_u2: [2]stdx.Packed(u2) = .{ .pack(0, 1, 2, 3), .pack(3, 2, 1, 0) }; comptime std.debug.assert(@sizeOf(@TypeOf(x_u2)) == 2); @@ -1476,7 +1476,7 @@ pub const Tensor = struct { test "convert f4e2m1 -> f32" { const floats = @import("floats.zig"); - const zml = @import("zml.zig"); + const zml = @import("core.zig"); const platform = zml.testing.env(); const x_f4_packed = [_]floats.Float4E2M1.Packed{ .fromF32(0, 0.5), .fromF32(1, 1.5), .fromF32(2, 3), .fromF32(4, 6) }; @@ -1570,7 +1570,7 @@ pub const Tensor = struct { } test dot { - const zml = @import("zml.zig"); + const zml = @import("core.zig"); const platform = zml.testing.env(); inline for (.{ @@ -1719,7 +1719,7 @@ pub const Tensor = struct { } test leakyReLU { - const zml = @import("zml.zig"); + const zml = @import("core.zig"); const platform = zml.testing.env(); const input: Tensor = .init(.{2}, .f32); @@ -1757,7 +1757,7 @@ pub const Tensor = struct { } test swiglu { - const zml = @import("zml.zig"); + const zml = @import("core.zig"); const platform = zml.testing.env(); const input: Tensor = .init(.{ .d = 4 }, .f32); @@ -2030,7 +2030,7 @@ pub const Tensor = struct { } test cumulativeSum { - const zml = @import("zml.zig"); + const zml = @import("core.zig"); const platform = zml.testing.env(); const Local = struct { @@ -2338,7 +2338,7 @@ pub const Tensor = struct { } test slice { - const zml = @import("zml.zig"); + const zml = @import("core.zig"); const platform = zml.testing.env(); const x: Tensor = .init(.{ 2, 5 }, .f32); @@ -2378,7 +2378,7 @@ pub const Tensor = struct { } test "dynamic slices" { - const zml = @import("zml.zig"); + const zml = @import("core.zig"); const platform = zml.testing.env(); const Local = struct { @@ -2498,7 +2498,7 @@ pub const Tensor = struct { } test repeat1d { - const zml = @import("zml.zig"); + const zml = @import("core.zig"); const platform = zml.testing.env(); const Local = struct { @@ -2561,7 +2561,7 @@ pub const Tensor = struct { } test stutter1d { - const zml = @import("zml.zig"); + const zml = @import("core.zig"); const platform = zml.testing.env(); const Local = struct { @@ -2684,7 +2684,7 @@ pub const Tensor = struct { } test arange { - const zml = @import("zml.zig"); + const zml = @import("core.zig"); const platform = zml.testing.env(); const exe = try zml.module.compile( @@ -2770,7 +2770,7 @@ pub const Tensor = struct { } test scalar { - const zml = @import("zml.zig"); + const zml = @import("core.zig"); const platform = zml.testing.env(); const Local = struct { @@ -3063,7 +3063,7 @@ pub const Tensor = struct { } test gather { - const zml = @import("zml.zig"); + const zml = @import("core.zig"); const platform = zml.testing.env(); const Local = struct { @@ -3217,7 +3217,7 @@ pub const Tensor = struct { } test gatherSlices { - const zml = @import("zml.zig"); + const zml = @import("core.zig"); const platform = zml.testing.env(); const Local = struct { @@ -3426,7 +3426,7 @@ pub const Tensor = struct { } test scatterSlices { - const zml = @import("zml.zig"); + const zml = @import("core.zig"); const platform = zml.testing.env(); const Local = struct { @@ -3648,7 +3648,7 @@ pub const Tensor = struct { } test argMax { - const zml = @import("zml.zig"); + const zml = @import("core.zig"); const platform = zml.testing.env(); const allocator = std.testing.allocator; const ArgMaxTest = struct { @@ -3720,7 +3720,7 @@ pub const Tensor = struct { } test argsort { - const zml = @import("zml.zig"); + const zml = @import("core.zig"); const platform = zml.testing.env(); const Local = struct { @@ -3991,7 +3991,7 @@ pub const Tensor = struct { } test chunkExact { - const zml = @import("zml.zig"); + const zml = @import("core.zig"); const platform = zml.testing.env(); // Only test shapes @@ -4044,7 +4044,7 @@ pub const Tensor = struct { } test chunkAllowTrailing { - const zml = @import("zml.zig"); + const zml = @import("core.zig"); const platform = zml.testing.env(); // Only test shapes @@ -4102,7 +4102,7 @@ pub const Tensor = struct { } test split { - const zml = @import("zml.zig"); + const zml = @import("core.zig"); const platform = zml.testing.env(); const Local = struct { @@ -4236,7 +4236,7 @@ pub const Tensor = struct { } test dynamicUpdateSlice { - const zml = @import("zml.zig"); + const zml = @import("core.zig"); const platform = zml.testing.env(); { @@ -4432,7 +4432,7 @@ pub const Tensor = struct { } test toDiagonal { - const zml = @import("zml.zig"); + const zml = @import("core.zig"); const platform = zml.testing.env(); const Local = struct { @@ -4485,7 +4485,7 @@ pub const Tensor = struct { } test triangular { - const zml = @import("zml.zig"); + const zml = @import("core.zig"); const platform = zml.testing.env(); const Local = struct { @@ -4663,7 +4663,7 @@ pub const Tensor = struct { } test cartesianProduct { - const zml = @import("zml.zig"); + const zml = @import("core.zig"); const platform = zml.testing.env(); const x: Tensor = .init(.{6}, .i32); @@ -4741,7 +4741,7 @@ pub const Tensor = struct { } test cartesianProductStacked { - const zml = @import("zml.zig"); + const zml = @import("core.zig"); const platform = zml.testing.env(); const x: Tensor = .init(.{6}, .i32); const y: Tensor = .init(.{4}, .i32); @@ -4887,7 +4887,7 @@ fn getComparisonType(ctx: *mlir.Context, dtype: DataType) *const stablehlo.Compa } test "Tensor.maxPool1d" { - const zml = @import("zml.zig"); + const zml = @import("core.zig"); const platform = zml.testing.env(); const MaxPool = struct { @@ -4929,7 +4929,7 @@ test "Tensor.maxPool1d" { } test "Tensor.maxPool2d" { - const zml = @import("zml.zig"); + const zml = @import("core.zig"); const platform = zml.testing.env(); const MaxPool = struct { @@ -5038,7 +5038,7 @@ test transposeIsJustAReshape { } test "unused tensor" { - const zml = @import("zml.zig"); + const zml = @import("core.zig"); const platform = zml.testing.env(); const Local = struct { diff --git a/zml/testing.zig b/zml/core/testing.zig similarity index 98% rename from zml/testing.zig rename to zml/core/testing.zig index 5e3d6333..c4258f56 100644 --- a/zml/testing.zig +++ b/zml/core/testing.zig @@ -2,11 +2,12 @@ const std = @import("std"); const builtin = @import("builtin"); const stdx = @import("stdx"); +const meta = @import("zml/meta"); const Platform = @import("platform.zig").Platform; const Sharding = @import("Sharding.zig"); const Slice = @import("slice.zig").Slice; -const zml = @import("zml.zig"); +const zml = @import("core.zig"); const log = std.log.scoped(.@"zml/testing"); @@ -366,7 +367,7 @@ pub fn expectEqualShapes(expected: zml.Shape, actual: zml.Shape) error{TestExpec } fn BufferizedWithArgs(comptime T: type) type { - return zml.meta.MapType(zml.Tensor, zml.Buffer).map(T); + return meta.MapType(zml.Tensor, zml.Buffer).map(T); } fn countWithPrefix(store: zml.io.TensorStore.View, prefix: []const u8) usize { @@ -418,7 +419,7 @@ pub fn testLayer( }; const input_name = try std.fmt.allocPrint(arena.allocator(), "{s}.in", .{name}); - const input_count = zml.meta.count(zml.Tensor, &args); + const input_count = meta.count(zml.Tensor, &args); const expected_input_count = countWithPrefix(activation_store, input_name); if (input_count != expected_input_count) { log.warn("Reference models uses {d} inputs, but implementation uses {d}", .{ expected_input_count, input_count }); @@ -426,7 +427,7 @@ pub fn testLayer( const store_input = activation_store.withPrefix(input_name); var ctx = LocalContext{ .activation_store = store_input }; - try zml.meta.visit(struct { + try meta.visit(struct { fn cb(ctx_: *LocalContext, tensor: *zml.Tensor) !void { var buffer: [256]u8 = undefined; const subkey = std.fmt.bufPrint(&buffer, "{d}", .{ctx_.index}) catch unreachable; @@ -454,7 +455,7 @@ pub fn testLayer( defer exe_results.deinit(allocator); var args_buffers = try zml.io.load(ArgsT, &args, allocator, io, platform, activation_store.store, .auto); - defer zml.meta.visit(struct { + defer meta.visit(struct { fn cb(_: void, b: *zml.Buffer) void { b.deinit(); } diff --git a/zml/kernel/BUILD.bazel b/zml/kernel/BUILD.bazel deleted file mode 100644 index 61ab44f8..00000000 --- a/zml/kernel/BUILD.bazel +++ /dev/null @@ -1,15 +0,0 @@ -package(default_visibility = ["//zml:__pkg__"]) - -filegroup( - name = "kernel", - srcs = [ - "kernel.zig", - "//zml/kernel/cuda_tile", - "//zml/kernel/cute", - "//zml/kernel/fly", - "//zml/kernel/metal", - "//zml/kernel/mosaic_tpu", - "//zml/kernel/tcl", - "//zml/kernel/triton", - ], -) diff --git a/zml/kernels/flash_attention/BUILD.bazel b/zml/kernels/flash_attention/BUILD.bazel new file mode 100644 index 00000000..0a52bbb1 --- /dev/null +++ b/zml/kernels/flash_attention/BUILD.bazel @@ -0,0 +1,27 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "flash_attention", + import_name = "zml/kernels/flash_attention", + main = "flash_attention.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//stdx", + "//zml/attention:options", + "//zml/core", + "//zml/kernels/flash_attention/cute", + "//zml/kernels/flash_attention/metal", + "//zml/kernels/flash_attention/nki", + "//zml/nn", + ], +) + +zig_test( + name = "integration_test", + main = "tests.zig", + deps = [ + ":flash_attention", + "//zml", + ], +) diff --git a/zml/kernels/flash_attention/cute/BUILD.bazel b/zml/kernels/flash_attention/cute/BUILD.bazel index b83a642e..b34e4534 100644 --- a/zml/kernels/flash_attention/cute/BUILD.bazel +++ b/zml/kernels/flash_attention/cute/BUILD.bazel @@ -1,9 +1,22 @@ -package(default_visibility = ["//zml:__pkg__"]) +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") -filegroup( +zig_library( name = "cute", - srcs = [ - "attention.zig", - "cute.zig", + srcs = ["attention.zig"], + import_name = "zml/kernels/flash_attention/cute", + main = "cute.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//mlir", + "//mlir/dialects/llvm", + "//mlir/dialects/vector", + "//stdx", + "//zml/core", ], ) + +zig_test( + name = "test", + deps = [":cute"], +) diff --git a/zml/kernels/flash_attention/cute/attention.zig b/zml/kernels/flash_attention/cute/attention.zig index ec0fce16..189edd54 100644 --- a/zml/kernels/flash_attention/cute/attention.zig +++ b/zml/kernels/flash_attention/cute/attention.zig @@ -3,7 +3,7 @@ const mlir = @import("mlir"); const llvm = @import("mlir/dialects/llvm"); const vector = @import("mlir/dialects/vector"); -const zml = @import("../../../zml.zig"); +const zml = @import("zml/core"); const cute = zml.kernel.cute; const B = cute.Builder; const Value = cute.Value; diff --git a/zml/kernels/flash_attention/cute/cute.zig b/zml/kernels/flash_attention/cute/cute.zig index 36e0953b..7bf1cd16 100644 --- a/zml/kernels/flash_attention/cute/cute.zig +++ b/zml/kernels/flash_attention/cute/cute.zig @@ -2,7 +2,7 @@ const std = @import("std"); const stdx = @import("stdx"); -const zml = @import("../../../zml.zig"); +const zml = @import("zml/core"); const kernel = @import("attention.zig"); test { diff --git a/zml/kernels/flash_attention/flash_attention.zig b/zml/kernels/flash_attention/flash_attention.zig index 3fb898db..883d5c40 100644 --- a/zml/kernels/flash_attention/flash_attention.zig +++ b/zml/kernels/flash_attention/flash_attention.zig @@ -1,16 +1,16 @@ const std = @import("std"); const stdx = @import("stdx"); +const nn = @import("zml/nn"); -pub const cute = @import("cute/cute.zig"); -pub const metal = @import("metal/metal.zig"); -pub const nki = @import("nki/nki.zig"); -const zml = @import("../../zml.zig"); -const Options = @import("../../attention/Options.zig"); +pub const cute = @import("zml/kernels/flash_attention/cute"); +pub const metal = @import("zml/kernels/flash_attention/metal"); +pub const nki = @import("zml/kernels/flash_attention/nki"); +const zml = @import("zml/core"); +const Options = @import("zml/attention/options"); test { std.testing.refAllDecls(@This()); - _ = @import("tests.zig"); } pub fn isAvailable(platform: *const zml.Platform) bool { @@ -129,11 +129,11 @@ pub const Metadata = union(Backend) { /// - .k is the number of keys /// - .hd is the head dimension pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters, opts: Options) zml.Tensor { - zml.stdx.debug.assert(zml.Compiler.current().manual_computation_depth > 0, "Attention kernels require an explicit manualComputation boundary", .{}); + stdx.debug.assert(zml.Compiler.current().manual_computation_depth > 0, "Attention kernels require an explicit manualComputation boundary", .{}); return switch (parameters) { .cute => cute.attention(q, k, v, token_index, opts.causal), // The NKI and Metal kernels are causal only. - .nki => if (opts.causal) nki.attention(q, k, v, token_index, parameters.nki) else zml.nn.sdpa(q, k, v, .{}), - .metal => if (opts.causal) metal.attention(q, k, v, token_index, metadata.metal) else zml.nn.sdpa(q, k, v, .{}), + .nki => if (opts.causal) nki.attention(q, k, v, token_index, parameters.nki) else nn.sdpa(q, k, v, .{}), + .metal => if (opts.causal) metal.attention(q, k, v, token_index, metadata.metal) else nn.sdpa(q, k, v, .{}), }; } diff --git a/zml/kernels/flash_attention/metal/BUILD.bazel b/zml/kernels/flash_attention/metal/BUILD.bazel index f82ade45..84ab38fb 100644 --- a/zml/kernels/flash_attention/metal/BUILD.bazel +++ b/zml/kernels/flash_attention/metal/BUILD.bazel @@ -1,9 +1,19 @@ -package(default_visibility = ["//zml:__pkg__"]) +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") -filegroup( +zig_library( name = "metal", - srcs = [ - "flash_attn.zig", - "metal.zig", + srcs = ["flash_attn.zig"], + import_name = "zml/kernels/flash_attention/metal", + main = "metal.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//zml/core", + "//zml/nn", ], ) + +zig_test( + name = "test", + deps = [":metal"], +) diff --git a/zml/kernels/flash_attention/metal/flash_attn.zig b/zml/kernels/flash_attention/metal/flash_attn.zig index de4b68da..4d584dab 100644 --- a/zml/kernels/flash_attention/metal/flash_attn.zig +++ b/zml/kernels/flash_attention/metal/flash_attn.zig @@ -4,7 +4,7 @@ const std = @import("std"); -const metal = @import("../../../kernel/kernel.zig").metal; +const metal = @import("zml/core").kernel.metal; const ArgSpec = metal.ArgSpec; const Builder = metal.Builder; diff --git a/zml/kernels/flash_attention/metal/metal.zig b/zml/kernels/flash_attention/metal/metal.zig index bdfb00c0..ee0b02d3 100644 --- a/zml/kernels/flash_attention/metal/metal.zig +++ b/zml/kernels/flash_attention/metal/metal.zig @@ -1,6 +1,7 @@ const std = @import("std"); -const zml = @import("../../../zml.zig"); +const zml = @import("zml/core"); +const nn = @import("zml/nn"); const flash_attn = @import("flash_attn.zig"); pub const Metadata = struct { @@ -28,9 +29,9 @@ pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.T const head_dim = q.dim(.hd); if (!((dtype == .bf16 or dtype == .f16) and k.dtype() == dtype and v.dtype() == dtype and flash_attn.FaTiled.tileConfig(head_dim) != null)) { const seq_len = k.dim(.k); - const mask = zml.nn.causalAttnMask(.{ .q = seq_len, .k = seq_len }, dtype, null) + const mask = nn.causalAttnMask(.{ .q = seq_len, .k = seq_len }, dtype, null) .gatherSlices(zml.Shape.init(.{ .q = q.dim(.q) }, dtype), token_index.appendAxes(.{.coord}), .{}); - return zml.nn.sdpa(q, k, v, .{ .attn_mask = mask }); + return nn.sdpa(q, k, v, .{ .attn_mask = mask }); } const batched = q.shape().hasTag(.b) != null; diff --git a/zml/kernels/flash_attention/nki/BUILD.bazel b/zml/kernels/flash_attention/nki/BUILD.bazel index f8e4f470..60134177 100644 --- a/zml/kernels/flash_attention/nki/BUILD.bazel +++ b/zml/kernels/flash_attention/nki/BUILD.bazel @@ -1,15 +1,29 @@ load("@rules_python//python:py_binary.bzl", "py_binary") - -package(default_visibility = ["//zml:__pkg__"]) +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") filegroup( name = "attention_kernel", srcs = ["attention.py"], ) -filegroup( +zig_library( name = "nki", - srcs = ["nki.zig"], + data = select({ + "//platforms:neuron.enabled": [":attention_kernel"], + "//conditions:default": [], + }), + import_name = "zml/kernels/flash_attention/nki", + main = "nki.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//platforms", + "//stdx", + "//zml/core", + ] + select({ + "//platforms:neuron.enabled": ["//platforms/neuron:nki_kernel"], + "//conditions:default": [], + }), ) py_binary( diff --git a/zml/kernels/flash_attention/nki/nki.zig b/zml/kernels/flash_attention/nki/nki.zig index f98a545d..429cb8f3 100644 --- a/zml/kernels/flash_attention/nki/nki.zig +++ b/zml/kernels/flash_attention/nki/nki.zig @@ -1,7 +1,7 @@ const platforms = @import("platforms"); -const zml = @import("../../../zml.zig"); -const stdx = zml.stdx; +const zml = @import("zml/core"); +const stdx = @import("stdx"); const KernelSpec = struct { name: []const u8, @@ -24,14 +24,14 @@ pub const Parameters = struct { .trn1, .inf2, .trn1n, .trn2, .trn2n, .trn3 => .{ .name = "nki_attention_trn1_decode", .entrypoint = "decode", - .source_path = "zml/zml/attention/nki/attention.py", + .source_path = "zml/zml/kernels/flash_attention/nki/attention.py", }, }, .prefill_kernel = switch (compiler_target) { .trn1, .inf2, .trn1n, .trn2, .trn2n, .trn3 => .{ .name = "nki_attention_trn1_prefill", .entrypoint = "prefill", - .source_path = "zml/zml/attention/nki/attention.py", + .source_path = "zml/zml/kernels/flash_attention/nki/attention.py", }, }, }; diff --git a/zml/kernels/flash_attention/tests.zig b/zml/kernels/flash_attention/tests.zig index c3c3dca2..b86326fe 100644 --- a/zml/kernels/flash_attention/tests.zig +++ b/zml/kernels/flash_attention/tests.zig @@ -1,7 +1,7 @@ const std = @import("std"); -const zml = @import("../../zml.zig"); -const flash_attention = @import("flash_attention.zig"); +const zml = @import("zml"); +const flash_attention = @import("zml/kernels/flash_attention"); test "attention: q=1,qh=64,kh=8" { try testAttention( diff --git a/zml/kernels/moe/BUILD.bazel b/zml/kernels/moe/BUILD.bazel new file mode 100644 index 00000000..733ca83e --- /dev/null +++ b/zml/kernels/moe/BUILD.bazel @@ -0,0 +1,40 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "moe", + import_name = "zml/kernels/moe", + main = "moe.zig", + visibility = ["//visibility:public"], + deps = [ + "//platforms", + "//stdx", + "//zml/core", + "//zml/kernels/moe/api", + "//zml/kernels/moe/cute", + "//zml/kernels/moe/flashinfer_cutlass", + "//zml/kernels/moe/fly", + "//zml/kernels/moe/metal", + "//zml/kernels/moe/mosaic_tpu", + "//zml/kernels/moe/stablehlo", + "//zml/kernels/moe/tcl", + "//zml/kernels/moe/triton", + "//zml/meta", + "//zml/nn", + "//zml/quantization", + ], +) + +zig_test( + name = "test", + deps = [":moe"], +) + +zig_test( + name = "integration_test", + main = "tests.zig", + deps = [ + "//zml", + "//zml/kernels/moe/api", + "//zml/kernels/moe/stablehlo", + ], +) diff --git a/zml/kernels/moe/api/BUILD.bazel b/zml/kernels/moe/api/BUILD.bazel new file mode 100644 index 00000000..2661f244 --- /dev/null +++ b/zml/kernels/moe/api/BUILD.bazel @@ -0,0 +1,12 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "api", + import_name = "zml/kernels/moe/api", + main = "api.zig", + visibility = ["//visibility:public"], + deps = [ + "//stdx", + "//zml/core", + ], +) diff --git a/zml/kernels/moe/api.zig b/zml/kernels/moe/api/api.zig similarity index 99% rename from zml/kernels/moe/api.zig rename to zml/kernels/moe/api/api.zig index b66593c1..b11ac1d5 100644 --- a/zml/kernels/moe/api.zig +++ b/zml/kernels/moe/api/api.zig @@ -2,7 +2,7 @@ const std = @import("std"); const stdx = @import("stdx"); -const Tensor = @import("../../tensor.zig").Tensor; +const Tensor = @import("zml/core").Tensor; const log = std.log.scoped(.@"zml/moe"); diff --git a/zml/kernels/moe/cute/BUILD.bazel b/zml/kernels/moe/cute/BUILD.bazel index ca87f37a..83744ef7 100644 --- a/zml/kernels/moe/cute/BUILD.bazel +++ b/zml/kernels/moe/cute/BUILD.bazel @@ -1,9 +1,26 @@ -package(default_visibility = ["//zml:__pkg__"]) +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") -filegroup( +zig_library( name = "cute", srcs = [ - "cute.zig", "//zml/kernels/moe/cute/sm10x", ], + import_name = "zml/kernels/moe/cute", + main = "cute.zig", + visibility = ["//visibility:public"], + deps = [ + "//mlir", + "//mlir/dialects/nvvm", + "//stdx", + "//zml/core", + "//zml/kernels/moe/api", + "//zml/kernels/moe/triton", + "//zml/nn", + "//zml/quantization", + ], +) + +zig_test( + name = "test", + deps = [":cute"], ) diff --git a/zml/kernels/moe/cute/cute.zig b/zml/kernels/moe/cute/cute.zig index 8bc03c9c..1cc60f03 100644 --- a/zml/kernels/moe/cute/cute.zig +++ b/zml/kernels/moe/cute/cute.zig @@ -2,10 +2,13 @@ const stdx = @import("stdx"); -const zml = @import("../../../zml.zig"); -const Options = @import("../api.zig").Options; -const RoutingWeightPlacement = @import("../api.zig").RoutingWeightPlacement; -const LocalRouting = @import("../api.zig").LocalRouting; +const nn = @import("zml/nn"); +const Quantization = @import("zml/quantization").Quantization; + +const zml = @import("zml/core"); +const Options = @import("zml/kernels/moe/api").Options; +const RoutingWeightPlacement = @import("zml/kernels/moe/api").RoutingWeightPlacement; +const LocalRouting = @import("zml/kernels/moe/api").LocalRouting; pub const kernels = @import("sm10x/moe.zig"); // `moe.zig`'s `refAllDecls` reaches this file but not its imports, so the @@ -43,8 +46,8 @@ pub fn validateOptions(opts: Options) void { pub fn fusedExperts( input: zml.Tensor, routing: LocalRouting, - gate_up: zml.nn.Linear, - down: zml.nn.Linear, + gate_up: nn.Linear, + down: nn.Linear, options: Options, ) zml.Tensor { validateOptions(options); @@ -72,7 +75,7 @@ pub fn fusedExperts( stdx.debug.assert(gq.swizzled_scales and dq.swizzled_scales, "cute NVFP4 backend expects swizzled block scales", .{}); stdx.debug.assert(gate_up.weight.dtype() == .u8 and down.weight.dtype() == .u8, "cute NVFP4 backend expects packed u8 weights, got {} and {}", .{ gate_up.weight.dtype(), down.weight.dtype() }); const scale = struct { - fn f(global: ?zml.Quantization.GlobalScale) zml.Tensor { + fn f(global: ?Quantization.GlobalScale) zml.Tensor { return (global orelse @panic("cute NVFP4 backend requires global and input scales")).asMultiplier(); } }.f; diff --git a/zml/kernels/moe/cute/sm10x/common.zig b/zml/kernels/moe/cute/sm10x/common.zig index da639c75..21e0164b 100644 --- a/zml/kernels/moe/cute/sm10x/common.zig +++ b/zml/kernels/moe/cute/sm10x/common.zig @@ -4,7 +4,7 @@ const std = @import("std"); const mlir = @import("mlir"); -const zml = @import("../../../../zml.zig"); +const zml = @import("zml/core"); const cute = zml.kernel.cute; const B = cute.Builder; const V = cute.Value; diff --git a/zml/kernels/moe/cute/sm10x/moe.zig b/zml/kernels/moe/cute/sm10x/moe.zig index ff6edec2..9dbc5ecf 100644 --- a/zml/kernels/moe/cute/sm10x/moe.zig +++ b/zml/kernels/moe/cute/sm10x/moe.zig @@ -7,9 +7,9 @@ const std = @import("std"); const stdx = @import("stdx"); -const zml = @import("../../../../zml.zig"); -const RoutingWeightPlacement = @import("../../api.zig").RoutingWeightPlacement; -const triton_kernels = @import("../../triton/moe.zig"); +const zml = @import("zml/core"); +const RoutingWeightPlacement = @import("zml/kernels/moe/api").RoutingWeightPlacement; +const triton_kernels = @import("zml/kernels/moe/triton").kernels; pub const mxfp4 = @import("mxfp4.zig"); pub const mxfp4_persistent = @import("persistent_mxfp4.zig"); pub const nvfp4 = @import("nvfp4.zig"); diff --git a/zml/kernels/moe/cute/sm10x/mxfp4.zig b/zml/kernels/moe/cute/sm10x/mxfp4.zig index 29e68977..09ddaae1 100644 --- a/zml/kernels/moe/cute/sm10x/mxfp4.zig +++ b/zml/kernels/moe/cute/sm10x/mxfp4.zig @@ -2,7 +2,7 @@ //! `persistent_mxfp4.zig`: routed MXFP8 input quantization. const std = @import("std"); -const zml = @import("../../../../zml.zig"); +const zml = @import("zml/core"); const cute = zml.kernel.cute; const B = cute.Builder; const launchBoundary = @import("common.zig").launchBoundary; diff --git a/zml/kernels/moe/cute/sm10x/nvfp4.zig b/zml/kernels/moe/cute/sm10x/nvfp4.zig index 5afee090..2500083b 100644 --- a/zml/kernels/moe/cute/sm10x/nvfp4.zig +++ b/zml/kernels/moe/cute/sm10x/nvfp4.zig @@ -2,7 +2,7 @@ //! `persistent_nvfp4.zig`: routed NVFP4 input quantization. const std = @import("std"); -const zml = @import("../../../../zml.zig"); +const zml = @import("zml/core"); const cute = zml.kernel.cute; const B = cute.Builder; const V = cute.Value; diff --git a/zml/kernels/moe/cute/sm10x/persistent_mxfp4.zig b/zml/kernels/moe/cute/sm10x/persistent_mxfp4.zig index 30062a3a..bdf1c8fa 100644 --- a/zml/kernels/moe/cute/sm10x/persistent_mxfp4.zig +++ b/zml/kernels/moe/cute/sm10x/persistent_mxfp4.zig @@ -14,7 +14,7 @@ const std = @import("std"); const mlir = @import("mlir"); const nvvm = @import("mlir/dialects/nvvm"); -const zml = @import("../../../../zml.zig"); +const zml = @import("zml/core"); const cute = zml.kernel.cute; const B = cute.Builder; const V = cute.Value; diff --git a/zml/kernels/moe/cute/sm10x/persistent_nvfp4.zig b/zml/kernels/moe/cute/sm10x/persistent_nvfp4.zig index 0de45007..f21f0797 100644 --- a/zml/kernels/moe/cute/sm10x/persistent_nvfp4.zig +++ b/zml/kernels/moe/cute/sm10x/persistent_nvfp4.zig @@ -14,7 +14,7 @@ const std = @import("std"); const mlir = @import("mlir"); const nvvm = @import("mlir/dialects/nvvm"); -const zml = @import("../../../../zml.zig"); +const zml = @import("zml/core"); const cute = zml.kernel.cute; const B = cute.Builder; const V = cute.Value; diff --git a/zml/kernels/moe/cute/sm10x/reduce.zig b/zml/kernels/moe/cute/sm10x/reduce.zig index 63244ac4..aded141a 100644 --- a/zml/kernels/moe/cute/sm10x/reduce.zig +++ b/zml/kernels/moe/cute/sm10x/reduce.zig @@ -2,7 +2,7 @@ //! BF16 route rows. const std = @import("std"); -const zml = @import("../../../../zml.zig"); +const zml = @import("zml/core"); const cute = zml.kernel.cute; const B = cute.Builder; const launchBoundary = @import("common.zig").launchBoundary; diff --git a/zml/kernels/moe/flashinfer_cutlass/BUILD.bazel b/zml/kernels/moe/flashinfer_cutlass/BUILD.bazel index c9c5d9a6..4dbd1526 100644 --- a/zml/kernels/moe/flashinfer_cutlass/BUILD.bazel +++ b/zml/kernels/moe/flashinfer_cutlass/BUILD.bazel @@ -1,10 +1,22 @@ -package(default_visibility = ["//zml:__pkg__"]) +load("@rules_zig//zig:defs.bzl", "zig_library") -filegroup( +zig_library( name = "flashinfer_cutlass", - srcs = [ - "contract.zig", - "flashinfer_cutlass.zig", - "runtime.zig", - ], + import_name = "zml/kernels/moe/flashinfer_cutlass", + main = "flashinfer_cutlass.zig", + visibility = ["//visibility:public"], + deps = [ + "//platforms", + "//stdx", + "//zml/core", + "//zml/kernels/moe/api", + "//zml/kernels/moe/weights", + "//zml/nn", + "//zml/quantization", + ] + select({ + "//platforms:cuda.enabled": [ + "//platforms/cuda/flashinfer_cutlass_moe", + ], + "//conditions:default": [], + }), ) diff --git a/zml/kernels/moe/flashinfer_cutlass/flashinfer_cutlass.zig b/zml/kernels/moe/flashinfer_cutlass/flashinfer_cutlass.zig index 92317c7c..f4b6cdf8 100644 --- a/zml/kernels/moe/flashinfer_cutlass/flashinfer_cutlass.zig +++ b/zml/kernels/moe/flashinfer_cutlass/flashinfer_cutlass.zig @@ -4,14 +4,17 @@ const fi_cutlass_moe = @import("platforms/cuda/flashinfer_cutlass_moe"); const platforms = @import("platforms"); const stdx = @import("stdx"); -const zml = @import("../../../zml.zig"); -const GlobalOptions = @import("../api.zig").Options; -const Activation = @import("../api.zig").Activation; -const LocalRouting = @import("../api.zig").LocalRouting; -const unpackedWeight = @import("../weights.zig").unpackedWeight; -const contract = @import("contract.zig"); +const nn = @import("zml/nn"); +const Quantization = @import("zml/quantization").Quantization; + +const zml = @import("zml/core"); +const GlobalOptions = @import("zml/kernels/moe/api").Options; +const Activation = @import("zml/kernels/moe/api").Activation; +const LocalRouting = @import("zml/kernels/moe/api").LocalRouting; +const unpackedWeight = @import("zml/kernels/moe/weights").unpackedWeight; +const contract = @import("zml/core").flashinfer_cutlass.contract; const Attributes = contract.Attributes; -const runtime = @import("runtime.zig"); +const runtime = @import("zml/core").flashinfer_cutlass.runtime; const log = std.log.scoped(.@"zml/moe/flashinfer_cutlass"); @@ -312,8 +315,8 @@ pub fn fusedExpertsNvfp4( pub fn fusedExperts( input: zml.Tensor, routing: LocalRouting, - gate_up: zml.nn.Linear, - down: zml.nn.Linear, + gate_up: nn.Linear, + down: nn.Linear, opts: GlobalOptions, ) zml.Tensor { if (comptime !platforms.isEnabled(.cuda)) { @@ -330,14 +333,14 @@ pub fn fusedExperts( const runner_options: Options = .fromGlobalOptions(opts); - const quant_scheme: ?zml.Quantization.Scheme = if (gate_up.quantization) |q| q.scheme else null; + const quant_scheme: ?Quantization.Scheme = if (gate_up.quantization) |q| q.scheme else null; stdx.debug.assert(quant_scheme == .nvfp4, "FlashInfer CUTLASS MoE only supports NVFP4, got {?}", .{quant_scheme}); const gate_up_weight_unpacked = unpackedWeight(gate_up); const down_weight_unpacked = unpackedWeight(down); // TODO(Corentin): Do error checking on nvfp4 - // Also, maybe pass `zml.nn.Linear` directly + // Also, maybe pass `nn.Linear` directly return fusedExpertsNvfp4( input, gate_up_weight_unpacked, diff --git a/zml/kernels/moe/fly/BUILD.bazel b/zml/kernels/moe/fly/BUILD.bazel index cb3b2ff4..99b4fcd4 100644 --- a/zml/kernels/moe/fly/BUILD.bazel +++ b/zml/kernels/moe/fly/BUILD.bazel @@ -1,9 +1,23 @@ -package(default_visibility = ["//zml:__pkg__"]) +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") -filegroup( +zig_library( name = "fly", srcs = [ - "fly.zig", "moe.zig", ], + import_name = "zml/kernels/moe/fly", + main = "fly.zig", + visibility = ["//visibility:public"], + deps = [ + "//stdx", + "//zml/core", + "//zml/kernels/moe/api", + "//zml/nn", + "//zml/quantization", + ], +) + +zig_test( + name = "test", + deps = [":fly"], ) diff --git a/zml/kernels/moe/fly/fly.zig b/zml/kernels/moe/fly/fly.zig index ba3d3c00..a5054268 100644 --- a/zml/kernels/moe/fly/fly.zig +++ b/zml/kernels/moe/fly/fly.zig @@ -2,16 +2,19 @@ const std = @import("std"); const stdx = @import("stdx"); -const zml = @import("../../../zml.zig"); +const nn = @import("zml/nn"); +const Quantization = @import("zml/quantization").Quantization; + +const zml = @import("zml/core"); const Tensor = zml.Tensor; const Shape = zml.Shape; const DataType = zml.DataType; -const ExpertsLayout = @import("../api.zig").ExpertsLayout; -const ProjectionLayout = @import("../api.zig").ProjectionLayout; -const RoutingWeightPlacement = @import("../api.zig").RoutingWeightPlacement; -const Activation = @import("../api.zig").Activation; -const Options = @import("../api.zig").Options; -const LocalRouting = @import("../api.zig").LocalRouting; +const ExpertsLayout = @import("zml/kernels/moe/api").ExpertsLayout; +const ProjectionLayout = @import("zml/kernels/moe/api").ProjectionLayout; +const RoutingWeightPlacement = @import("zml/kernels/moe/api").RoutingWeightPlacement; +const Activation = @import("zml/kernels/moe/api").Activation; +const Options = @import("zml/kernels/moe/api").Options; +const LocalRouting = @import("zml/kernels/moe/api").LocalRouting; const fly = @import("moe.zig"); const log = std.log.scoped(.@"zml/moe/fused_experts"); @@ -23,8 +26,8 @@ test { pub fn fusedExperts( hidden_states: Tensor, routing_: LocalRouting, - gate_up_: zml.nn.Linear, - down_: zml.nn.Linear, + gate_up_: nn.Linear, + down_: nn.Linear, opts: Options, ) Tensor { const expert_map_ = routing_.expertMap(); @@ -136,7 +139,7 @@ pub fn fusedExperts( } // TODO(Corentin): unused for now -pub fn supports(hidden_states: Tensor, routing: LocalRouting, gate_up_: zml.nn.Linear, down_: zml.nn.Linear, opts: Options) bool { +pub fn supports(hidden_states: Tensor, routing: LocalRouting, gate_up_: nn.Linear, down_: nn.Linear, opts: Options) bool { if (hidden_states.dtype() != .bf16 or gate_up_.quantizationScheme() != .mxfp4 or down_.quantizationScheme() != .mxfp4 or gate_up_.bias != null or down_.bias != null or opts.routing_weight_placement != .before_down) return false; @@ -170,7 +173,7 @@ pub fn supports(hidden_states: Tensor, routing: LocalRouting, gate_up_: zml.nn.L return routes > 0 and routes <= @divTrunc(std.math.maxInt(i32), max_width); } -pub fn prepareInput(x: Tensor, scheme: ?zml.Quantization.Scheme, _: zml.DataType, _: bool) struct { Tensor, ?Tensor } { +pub fn prepareInput(x: Tensor, scheme: ?Quantization.Scheme, _: zml.DataType, _: bool) struct { Tensor, ?Tensor } { std.debug.assert(scheme == .mxfp4); return .{ x.convert(.bf16), null }; } @@ -185,13 +188,13 @@ pub const GemmOptions = struct { routing: Routing, expert_ids: Tensor, launch_config: LaunchConfig, - quant_scheme: ?zml.Quantization.Scheme = null, + quant_scheme: ?Quantization.Scheme = null, top_k: usize, output_shape: Shape, }; pub fn gemm(opts: GemmOptions) zml.Tensor { - zml.stdx.debug.assert(opts.quant_scheme == .mxfp4 and opts.bias == null and opts.routing_weights == null and opts.input_scale == null, "Fly MoE requires MXFP4 weights, BF16 activations and routing weights before the down projection", .{}); + stdx.debug.assert(opts.quant_scheme == .mxfp4 and opts.bias == null and opts.routing_weights == null and opts.input_scale == null, "Fly MoE requires MXFP4 weights, BF16 activations and routing weights before the down projection", .{}); const weight = if (opts.weight.dtype() == .f4e2m1) opts.weight.reshape(opts.weight.shape().setDim(2, @divExact(opts.weight.dim(2), 2)).append(.{ .nibble = 2 })).bitCast(.u8) else diff --git a/zml/kernels/moe/fly/moe.zig b/zml/kernels/moe/fly/moe.zig index 3cb82b96..8c24d56b 100644 --- a/zml/kernels/moe/fly/moe.zig +++ b/zml/kernels/moe/fly/moe.zig @@ -1,5 +1,5 @@ const std = @import("std"); -const zml = @import("../../../zml.zig"); +const zml = @import("zml/core"); const Tensor = zml.Tensor; const Shape = zml.Shape; const fly = zml.kernel.fly; diff --git a/zml/kernels/moe/metal/BUILD.bazel b/zml/kernels/moe/metal/BUILD.bazel index b4240b00..ef08b35b 100644 --- a/zml/kernels/moe/metal/BUILD.bazel +++ b/zml/kernels/moe/metal/BUILD.bazel @@ -1,8 +1,15 @@ -package(default_visibility = ["//zml:__pkg__"]) +load("@rules_zig//zig:defs.bzl", "zig_library") -filegroup( +zig_library( name = "metal", - srcs = [ - "metal.zig", + import_name = "zml/kernels/moe/metal", + main = "metal.zig", + visibility = ["//visibility:public"], + deps = [ + "//stdx", + "//zml/core", + "//zml/kernels/moe/api", + "//zml/kernels/moe/weights", + "//zml/nn", ], ) diff --git a/zml/kernels/moe/metal/metal.zig b/zml/kernels/moe/metal/metal.zig index f835c889..96d93dda 100644 --- a/zml/kernels/moe/metal/metal.zig +++ b/zml/kernels/moe/metal/metal.zig @@ -2,12 +2,14 @@ const std = @import("std"); const stdx = @import("stdx"); -const zml = @import("../../../zml.zig"); +const nn = @import("zml/nn"); + +const zml = @import("zml/core"); const Tensor = zml.Tensor; -const Activation = @import("../api.zig").Activation; -const GlobalOptions = @import("../api.zig").Options; -const LocalRouting = @import("../api.zig").LocalRouting; -const unpackedWeight = @import("../weights.zig").unpackedWeight; +const Activation = @import("zml/kernels/moe/api").Activation; +const GlobalOptions = @import("zml/kernels/moe/api").Options; +const LocalRouting = @import("zml/kernels/moe/api").LocalRouting; +const unpackedWeight = @import("zml/kernels/moe/weights").unpackedWeight; const log = std.log.scoped(.@"zml/moe/metal"); @@ -70,8 +72,8 @@ fn applyDownGlobalScale(output: Tensor, global_scale: ?Tensor, expert_ids: Tenso pub fn fusedExperts( hidden_states: zml.Tensor, routing: LocalRouting, - gate_up_: zml.nn.Linear, - down_: zml.nn.Linear, + gate_up_: nn.Linear, + down_: nn.Linear, opts: GlobalOptions, ) zml.Tensor { validateOptions(opts); diff --git a/zml/kernels/moe/moe.zig b/zml/kernels/moe/moe.zig index eb697f36..1ddaaa5d 100644 --- a/zml/kernels/moe/moe.zig +++ b/zml/kernels/moe/moe.zig @@ -1,30 +1,33 @@ const std = @import("std"); const platforms = @import("platforms"); +const stdx = @import("stdx"); -const zml = @import("../../zml.zig"); -const stdx = zml.stdx; -pub const Activation = @import("api.zig").Activation; -pub const ActivationKind = @import("api.zig").ActivationKind; -pub const cute = @import("cute/cute.zig"); -pub const ExpertsLayout = @import("api.zig").ExpertsLayout; -pub const flashinfer_cutlass = @import("flashinfer_cutlass/flashinfer_cutlass.zig"); -pub const fly = @import("fly/fly.zig"); -pub const LocalRouting = @import("api.zig").LocalRouting; -pub const metal = @import("metal/metal.zig"); -pub const mosaic_tpu = @import("mosaic_tpu/mosaic_tpu.zig"); -pub const Options = @import("api.zig").Options; -pub const ProjectionLayout = @import("api.zig").ProjectionLayout; -pub const RoutingWeightPlacement = @import("api.zig").RoutingWeightPlacement; -pub const stablehlo = @import("stablehlo/stablehlo.zig"); -pub const tcl = @import("tcl/tcl.zig"); -pub const triton = @import("triton/triton.zig"); +const nn = @import("zml/nn"); +const Quantization = @import("zml/quantization").Quantization; +const meta = @import("zml/meta"); + +const zml = @import("zml/core"); +pub const Activation = @import("zml/kernels/moe/api").Activation; +pub const ActivationKind = @import("zml/kernels/moe/api").ActivationKind; +pub const cute = @import("zml/kernels/moe/cute"); +pub const ExpertsLayout = @import("zml/kernels/moe/api").ExpertsLayout; +pub const flashinfer_cutlass = @import("zml/kernels/moe/flashinfer_cutlass"); +pub const fly = @import("zml/kernels/moe/fly"); +pub const LocalRouting = @import("zml/kernels/moe/api").LocalRouting; +pub const metal = @import("zml/kernels/moe/metal"); +pub const mosaic_tpu = @import("zml/kernels/moe/mosaic_tpu"); +pub const Options = @import("zml/kernels/moe/api").Options; +pub const ProjectionLayout = @import("zml/kernels/moe/api").ProjectionLayout; +pub const RoutingWeightPlacement = @import("zml/kernels/moe/api").RoutingWeightPlacement; +pub const stablehlo = @import("zml/kernels/moe/stablehlo"); +pub const tcl = @import("zml/kernels/moe/tcl"); +pub const triton = @import("zml/kernels/moe/triton"); const log = std.log.scoped(.@"zml/moe"); test { std.testing.refAllDecls(@This()); - _ = @import("tests.zig"); } pub const Backend = enum { @@ -37,7 +40,7 @@ pub const Backend = enum { metal, tcl, - pub fn auto(platform: *const zml.Platform, scheme: ?zml.Quantization.Scheme, dtype: zml.DataType) !Backend { + pub fn auto(platform: *const zml.Platform, scheme: ?Quantization.Scheme, dtype: zml.DataType) !Backend { return switch (platform.target) { .cuda => b: { const s = scheme orelse break :b .triton; @@ -109,7 +112,7 @@ pub const Backend = enum { } /// The layout contract the expert weights must have for a given backend with scheme - pub fn expertsLayout(backend: Backend, scheme: ?zml.Quantization.Scheme) !ExpertsLayout { + pub fn expertsLayout(backend: Backend, scheme: ?Quantization.Scheme) !ExpertsLayout { return switch (backend) { .stablehlo => .{ .gate_up = .concatenated, .packing = .plain }, .cute => if (scheme) |s| switch (s) { @@ -184,8 +187,8 @@ pub fn forwardMoe( input: zml.Tensor, topk_ids: zml.Tensor, topk_weights: zml.Tensor, - gate_up: zml.nn.Linear, - down: zml.nn.Linear, + gate_up: nn.Linear, + down: nn.Linear, backend: Backend, mesh: *const zml.Mesh, partitioning: Partitioning, @@ -224,8 +227,8 @@ pub fn forwardMoe( input: zml.Tensor, ids: zml.Tensor, weights: zml.Tensor, - gate_up: zml.nn.Linear, - down: zml.nn.Linear, + gate_up: nn.Linear, + down: nn.Linear, backend: Backend, opts: Options, mesh: *const zml.Mesh, @@ -266,9 +269,9 @@ pub fn forwardMoe( .global_experts = @intCast(gate_up.weight.dim(.expert)), }; const Logical = zml.Sharding.Partitioning.Logical; - var projection_specs: [2]zml.ops.PartitionSpecs(zml.nn.Linear) = undefined; - for ([_]zml.nn.Linear{ gate_up, down }, &projection_specs) |linear, *specs| { - zml.meta.mapAlloc(struct { + var projection_specs: [2]zml.ops.PartitionSpecs(nn.Linear) = undefined; + for ([_]nn.Linear{ gate_up, down }, &projection_specs) |linear, *specs| { + meta.mapAlloc(struct { fn spec(ctx: struct { experts: i64, axis: ?zml.Shape.Tag }, tensor: zml.Tensor) Logical { const axis = ctx.axis orelse return .replicated; if (tensor.count() == 1) return .replicated; diff --git a/zml/kernels/moe/mosaic_tpu/BUILD.bazel b/zml/kernels/moe/mosaic_tpu/BUILD.bazel index ed4dfcd9..fd8ada94 100644 --- a/zml/kernels/moe/mosaic_tpu/BUILD.bazel +++ b/zml/kernels/moe/mosaic_tpu/BUILD.bazel @@ -1,9 +1,18 @@ -package(default_visibility = ["//zml:__pkg__"]) +load("@rules_zig//zig:defs.bzl", "zig_library") -filegroup( +zig_library( name = "mosaic_tpu", srcs = [ "gmm_ep.zig", - "mosaic_tpu.zig", + ], + import_name = "zml/kernels/moe/mosaic_tpu", + main = "mosaic_tpu.zig", + visibility = ["//visibility:public"], + deps = [ + "//dsl/mosaic_tpu:builder", + "//stdx", + "//zml/core", + "//zml/kernels/moe/api", + "//zml/nn", ], ) diff --git a/zml/kernels/moe/mosaic_tpu/gmm_ep.zig b/zml/kernels/moe/mosaic_tpu/gmm_ep.zig index c30d805b..07ef428e 100644 --- a/zml/kernels/moe/mosaic_tpu/gmm_ep.zig +++ b/zml/kernels/moe/mosaic_tpu/gmm_ep.zig @@ -1,6 +1,6 @@ const std = @import("std"); -const zml = @import("../../../zml.zig"); +const zml = @import("zml/core"); const mtt = @import("dsl/mosaic_tpu/builder"); const pipeline = mtt.pipeline; const mtt_kernel = zml.kernel.mosaic_tpu; diff --git a/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig b/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig index bb60594b..e56edbb8 100644 --- a/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig +++ b/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig @@ -3,11 +3,13 @@ const std = @import("std"); const mtt = @import("dsl/mosaic_tpu/builder"); const stdx = @import("stdx"); -const zml = @import("../../../zml.zig"); +const nn = @import("zml/nn"); + +const zml = @import("zml/core"); const Tensor = zml.Tensor; -const Activation = @import("../api.zig").Activation; -const GlobalOptions = @import("../api.zig").Options; -const LocalRouting = @import("../api.zig").LocalRouting; +const Activation = @import("zml/kernels/moe/api").Activation; +const GlobalOptions = @import("zml/kernels/moe/api").Options; +const LocalRouting = @import("zml/kernels/moe/api").LocalRouting; pub const gmm_ep = @import("gmm_ep.zig"); const log = std.log.scoped(.@"zml/moe/mosaic_tpu"); @@ -216,8 +218,8 @@ pub fn callGmmEp( pub fn fusedExperts( hidden_states: zml.Tensor, routing: LocalRouting, - gate_up_: zml.nn.Linear, - down_: zml.nn.Linear, + gate_up_: nn.Linear, + down_: nn.Linear, opts: GlobalOptions, ) zml.Tensor { validateOptions(opts); diff --git a/zml/kernels/moe/stablehlo/BUILD.bazel b/zml/kernels/moe/stablehlo/BUILD.bazel index c31b27f0..2543a904 100644 --- a/zml/kernels/moe/stablehlo/BUILD.bazel +++ b/zml/kernels/moe/stablehlo/BUILD.bazel @@ -1,8 +1,19 @@ -package(default_visibility = ["//zml:__pkg__"]) +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") -filegroup( +zig_library( name = "stablehlo", - srcs = [ - "stablehlo.zig", + import_name = "zml/kernels/moe/stablehlo", + main = "stablehlo.zig", + visibility = ["//visibility:public"], + deps = [ + "//stdx", + "//zml/core", + "//zml/kernels/moe/api", + "//zml/nn", ], ) + +zig_test( + name = "test", + deps = [":stablehlo"], +) diff --git a/zml/kernels/moe/stablehlo/stablehlo.zig b/zml/kernels/moe/stablehlo/stablehlo.zig index ba723234..8b7b3823 100644 --- a/zml/kernels/moe/stablehlo/stablehlo.zig +++ b/zml/kernels/moe/stablehlo/stablehlo.zig @@ -4,13 +4,15 @@ const std = @import("std"); const stdx = @import("stdx"); -const zml = @import("../../../zml.zig"); +const nn = @import("zml/nn"); + +const zml = @import("zml/core"); const Tensor = zml.Tensor; const Shape = zml.Shape; -const GlobalOptions = @import("../api.zig").Options; -const Activation = @import("../api.zig").Activation; -const RoutingWeightPlacement = @import("../api.zig").RoutingWeightPlacement; -const LocalRouting = @import("../api.zig").LocalRouting; +const GlobalOptions = @import("zml/kernels/moe/api").Options; +const Activation = @import("zml/kernels/moe/api").Activation; +const RoutingWeightPlacement = @import("zml/kernels/moe/api").RoutingWeightPlacement; +const LocalRouting = @import("zml/kernels/moe/api").LocalRouting; pub fn activate(x: Tensor, activation: Activation) Tensor { return switch (activation) { @@ -71,11 +73,11 @@ fn expandScales(scales: Tensor, shape: Shape) Tensor { return result.withTags(shape.tags()); } -pub fn dequantize(linear: zml.nn.Linear) Tensor { +pub fn dequantize(linear: nn.Linear) Tensor { var weight = linear.weight; const q = linear.quantization orelse return weight.convert(.f32); stdx.debug.assert(!q.swizzled_scales, "StableHLO reference requires plain block scales", .{}); - if (zml.nn.isPackedFp4(q.scheme, weight.dtype())) { + if (nn.isPackedFp4(q.scheme, weight.dtype())) { // StableHLO bitcast expands the low nibble first along the input axis. weight = weight.bitCast(.f4e2m1).merge(.{ .unpacked = .{ linear.tag, .bitcast } }).renameTag(.unpacked, linear.tag); } @@ -101,7 +103,7 @@ fn roundFp4(x: Tensor) Tensor { // Quantize and dequantize activation groups using tensor math, independently of // Triton's quantizer. The return value is the represented value in FP32. -fn roundedInput(x: Tensor, linear: zml.nn.Linear, opts: Options, dtype: zml.DataType) Tensor { +fn roundedInput(x: Tensor, linear: nn.Linear, opts: Options, dtype: zml.DataType) Tensor { if (opts.input_quantization == .none) return x.convert(dtype).convert(.f32); const q = linear.quantization orelse return x.convert(dtype).convert(.f32); const fixed_mxfp8 = opts.input_quantization == .mxfp8 or opts.input_quantization == .mxfp8_bf16; @@ -138,7 +140,7 @@ fn roundedInput(x: Tensor, linear: zml.nn.Linear, opts: Options, dtype: zml.Data .convert(output_dtype).optimizationBarrier().convert(.f32).mul(broad).reshape(x.shape().withDtype(.f32)); } -fn project(x: Tensor, linear: zml.nn.Linear, ids: Tensor, opts: Options, dtype: zml.DataType) Tensor { +fn project(x: Tensor, linear: nn.Linear, ids: Tensor, opts: Options, dtype: zml.DataType) Tensor { const weight = dequantize(linear).withTags(.{ .expert, .out, .in }); const selected = weight.gather(.{ .expert = ids }, .{}); var result = roundedInput(x, linear, opts, dtype).dot(selected, .in); @@ -163,7 +165,7 @@ pub const Options = struct { }; /// Negative expert IDs denote nonlocal routes and contribute zero, including bias. -pub fn reference(input: Tensor, topk_ids: Tensor, topk_weights: Tensor, gate_up: zml.nn.Linear, down: zml.nn.Linear, opts: Options) Tensor { +pub fn reference(input: Tensor, topk_ids: Tensor, topk_weights: Tensor, gate_up: nn.Linear, down: nn.Linear, opts: Options) Tensor { stdx.debug.assert(topk_ids.dim(.topk) > 0, "reference top-k must be positive", .{}); stdx.debug.assert(topk_ids.dtype() == .i32, "reference expects i32 expert ids", .{}); stdx.debug.assert(gate_up.weight.rank() == 3 and down.weight.rank() == 3, "reference expects rank-three expert weights", .{}); @@ -192,7 +194,7 @@ pub fn reference(input: Tensor, topk_ids: Tensor, topk_weights: Tensor, gate_up: .sum(.topk).squeeze(.topk).convert(input.dtype()); } -pub fn fusedExperts(input: Tensor, routing: LocalRouting, gate_up: zml.nn.Linear, down: zml.nn.Linear, opts: GlobalOptions) Tensor { +pub fn fusedExperts(input: Tensor, routing: LocalRouting, gate_up: nn.Linear, down: nn.Linear, opts: GlobalOptions) Tensor { stdx.debug.assert(routing.ids.dim(.topk) > 0 and routing.ids.dim(.topk) <= routing.global_num_experts, "MoE top-k must fit the global expert count", .{}); return reference(input, routing.localIds(), routing.weights, gate_up, down, .{ .activation = opts.activation, diff --git a/zml/kernels/moe/tcl/BUILD.bazel b/zml/kernels/moe/tcl/BUILD.bazel index 44b56e08..79811e83 100644 --- a/zml/kernels/moe/tcl/BUILD.bazel +++ b/zml/kernels/moe/tcl/BUILD.bazel @@ -1,9 +1,21 @@ -package(default_visibility = ["//zml:__pkg__"]) +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") -filegroup( +zig_library( name = "tcl", - srcs = [ - "moe.zig", - "tcl.zig", + srcs = ["moe.zig"], + import_name = "zml/kernels/moe/tcl", + main = "tcl.zig", + visibility = ["//visibility:public"], + deps = [ + "//dsl/tcl:builder", + "//stdx", + "//zml/core", + "//zml/kernels/moe/api", + "//zml/nn", ], ) + +zig_test( + name = "test", + deps = [":tcl"], +) diff --git a/zml/kernels/moe/tcl/moe.zig b/zml/kernels/moe/tcl/moe.zig index 23fb2274..69c82db1 100644 --- a/zml/kernels/moe/tcl/moe.zig +++ b/zml/kernels/moe/tcl/moe.zig @@ -3,9 +3,9 @@ const std = @import("std"); const builder = @import("dsl/tcl/builder"); -const zml = @import("../../../zml.zig"); +const zml = @import("zml/core"); const tcl = zml.kernel.tcl; -const ActivationKind = @import("../api.zig").ActivationKind; +const ActivationKind = @import("zml/kernels/moe/api").ActivationKind; pub const Format = enum { bf16, fp8_block128 }; diff --git a/zml/kernels/moe/tcl/tcl.zig b/zml/kernels/moe/tcl/tcl.zig index f05727ce..41250dcf 100644 --- a/zml/kernels/moe/tcl/tcl.zig +++ b/zml/kernels/moe/tcl/tcl.zig @@ -3,17 +3,19 @@ const std = @import("std"); const stdx = @import("stdx"); -const zml = @import("../../../zml.zig"); +const nn = @import("zml/nn"); + +const zml = @import("zml/core"); const Tensor = zml.Tensor; const tcl = zml.kernel.tcl; const kernels = @import("moe.zig"); -const api = @import("../api.zig"); +const api = @import("zml/kernels/moe/api"); /// `input` is `{.t, .d}`, routing IDs and weights are `{.t, .topk}`, `gate_up` /// is `{.expert, .dout = 2 * inter (gate then up), .d}`, and `down` is /// `{.expert, .d, .dout = inter}`. Experts are BF16 or block-128 FP8 with /// BF16/F32 scales. Supports plain SwiGLU and BF16 GEGLU with tanh GELU. -pub fn fusedExperts(input: Tensor, routing: api.LocalRouting, gate_up_linear: zml.nn.Linear, down_linear: zml.nn.Linear, opts: api.Options) Tensor { +pub fn fusedExperts(input: Tensor, routing: api.LocalRouting, gate_up_linear: nn.Linear, down_linear: nn.Linear, opts: api.Options) Tensor { const topk_ids = routing.localIds(); stdx.debug.assert(!opts.quantize_input and opts.routing_weight_placement == .after_down, "TCL MoE requires quantize_input=false and routing_weight_placement=after_down", .{}); stdx.debug.assert(gate_up_linear.bias == null and down_linear.bias == null, "TCL MoE takes experts without bias", .{}); @@ -64,7 +66,7 @@ const Format = union(kernels.Format) { fp8_block128: Scales, }; -fn expertFormat(gate_up: zml.nn.Linear, down: zml.nn.Linear) Format { +fn expertFormat(gate_up: nn.Linear, down: nn.Linear) Format { const scheme = gate_up.quantizationScheme(); stdx.debug.assert(scheme == down.quantizationScheme(), "TCL MoE projections must have matching quantization schemes", .{}); return switch (scheme orelse { @@ -80,7 +82,7 @@ fn expertFormat(gate_up: zml.nn.Linear, down: zml.nn.Linear) Format { }; } -fn fp8Scales(linear: zml.nn.Linear) Tensor { +fn fp8Scales(linear: nn.Linear) Tensor { const q = linear.quantization.?; stdx.debug.assert(linear.weight.dtype() == .f8e4m3fn and !q.swizzled_scales and q.global_scale == null and q.input_scale == null, "TCL MoE takes plain E4M3FN block-128 weights", .{}); stdx.debug.assert(q.scales.dtype() == .bf16 or q.scales.dtype() == .f32, "TCL MoE FP8 scales must be BF16 or F32", .{}); @@ -151,7 +153,7 @@ test "tcl fused experts run on furiosa" { try testFusedExperts(.geglu_tanh); } -fn testFusedExperts(activation: zml.moe.Activation) !void { +fn testFusedExperts(activation: api.Activation) !void { const platform = zml.testing.env(); if (platform.target != .furiosa) return error.SkipZigTest; const allocator = std.testing.allocator; @@ -171,8 +173,14 @@ fn testFusedExperts(activation: zml.moe.Activation) !void { const gu: Tensor = .init(.{ .expert = e, .dout = 2 * i, .d = d }, .bf16); const dn: Tensor = .init(.{ .expert = e, .d = d, .dout = i }, .bf16); const Mod = struct { - pub fn forward(x_: Tensor, ids_: Tensor, w_: Tensor, gu_: Tensor, dn_: Tensor, act: zml.moe.Activation) Tensor { - return zml.moe.forwardMoe(x_, ids_, w_, .init(gu_, null, .dout), .init(dn_, null, .dout), .tcl, zml.Compiler.current().platform.replicated_mesh, .expert(.replicated), .{ + pub fn forward(x_: Tensor, ids_: Tensor, w_: Tensor, gu_: Tensor, dn_: Tensor, act: api.Activation) Tensor { + const routing: api.LocalRouting = .{ + .ids = ids_, + .weights = w_, + .global_num_experts = @intCast(gu_.dim(.expert)), + .local_num_experts = @intCast(gu_.dim(.expert)), + }; + return fusedExperts(x_, routing, .init(gu_, null, .dout), .init(dn_, null, .dout), .{ .activation = act, .quantize_input = false, .routing_weight_placement = .after_down, diff --git a/zml/kernels/moe/tests.zig b/zml/kernels/moe/tests.zig index c74406c3..0c3e6aab 100644 --- a/zml/kernels/moe/tests.zig +++ b/zml/kernels/moe/tests.zig @@ -2,14 +2,14 @@ //! supported cases are explicit, and errors in those cases must fail the test. const std = @import("std"); -const zml = @import("../../zml.zig"); +const zml = @import("zml"); const Tensor = zml.Tensor; const Shape = zml.Shape; -const Activation = @import("api.zig").Activation; -const ExpertsLayout = @import("api.zig").ExpertsLayout; -const GlobalOptions = @import("api.zig").Options; -const Placement = @import("api.zig").RoutingWeightPlacement; -const reference = @import("stablehlo/stablehlo.zig"); +const Activation = @import("zml/kernels/moe/api").Activation; +const ExpertsLayout = @import("zml/kernels/moe/api").ExpertsLayout; +const GlobalOptions = @import("zml/kernels/moe/api").Options; +const Placement = @import("zml/kernels/moe/api").RoutingWeightPlacement; +const reference = @import("zml/kernels/moe/stablehlo"); const Case = struct { backend: zml.moe.Backend, diff --git a/zml/kernels/moe/triton/BUILD.bazel b/zml/kernels/moe/triton/BUILD.bazel index 8dda7f88..0dc49f93 100644 --- a/zml/kernels/moe/triton/BUILD.bazel +++ b/zml/kernels/moe/triton/BUILD.bazel @@ -1,9 +1,23 @@ -package(default_visibility = ["//zml:__pkg__"]) +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") -filegroup( +zig_library( name = "triton", srcs = [ "moe.zig", - "triton.zig", + ], + import_name = "zml/kernels/moe/triton", + main = "triton.zig", + visibility = ["//visibility:public"], + deps = [ + "//stdx", + "//zml/core", + "//zml/kernels/moe/api", + "//zml/nn", + "//zml/quantization", ], ) + +zig_test( + name = "test", + deps = [":triton"], +) diff --git a/zml/kernels/moe/triton/moe.zig b/zml/kernels/moe/triton/moe.zig index 6e4006fa..f62ee83c 100644 --- a/zml/kernels/moe/triton/moe.zig +++ b/zml/kernels/moe/triton/moe.zig @@ -7,12 +7,14 @@ const std = @import("std"); -const zml = @import("../../../zml.zig"); +const Quantization = @import("zml/quantization").Quantization; + +const zml = @import("zml/core"); const tri = zml.kernel.triton; const Builder = tri.Builder; const Value = tri.Value; const DType = tri.DType; -const RoutingWeightPlacement = @import("../api.zig").RoutingWeightPlacement; +const RoutingWeightPlacement = @import("zml/kernels/moe/api").RoutingWeightPlacement; const log = std.log.scoped(.@"zml/moe/triton_kernels"); @@ -34,7 +36,7 @@ pub const PerTokenGroupQuantFp8 = struct { output_dtype: DType, scale_dtype: DType, block: usize, - quant_scheme: zml.Quantization.Scheme, + quant_scheme: Quantization.Scheme, }; pub const Kernel = tri.Kernel(Cfg, .{ .name = "per_token_group_quant_fp8", @@ -206,7 +208,7 @@ pub const FusedMoe = struct { top_k: usize, naive_block_assignment: bool, compute_type: DType, - quant_scheme: ?zml.Quantization.Scheme, + quant_scheme: ?Quantization.Scheme, }; pub const Kernel = tri.Kernel(Cfg, .{ .name = "fused_moe_kernel", @@ -573,7 +575,7 @@ pub const FusedMoe = struct { }; test "FP8 activation quantization emits float and E8M0 scales" { - for ([_]zml.Quantization.Scheme{ .mxfp8, .fp8_per_channel, .fp8_per_tensor, .fp8_block128, .fp8_block32 }) |scheme| { + for ([_]Quantization.Scheme{ .mxfp8, .fp8_per_channel, .fp8_per_tensor, .fp8_block128, .fp8_block32 }) |scheme| { for ([_]DType{ .bf16, .f32 }) |input_dtype| { const ir = try PerTokenGroupQuantFp8.Kernel.emit(std.testing.allocator, .{ .input_dtype = input_dtype, @@ -592,7 +594,7 @@ test "FP8 activation quantization emits float and E8M0 scales" { test "FusedMoe emits each FP8 and MXFP4 scaling path" { const allocator = std.testing.allocator; - const schemes = [_]?zml.Quantization.Scheme{ null, .mxfp4, .mxfp8, .fp8_per_tensor, .fp8_per_channel, .fp8_block128, .fp8_block32 }; + const schemes = [_]?Quantization.Scheme{ null, .mxfp4, .mxfp8, .fp8_per_tensor, .fp8_per_channel, .fp8_block128, .fp8_block32 }; for (schemes) |scheme| { for ([_]DType{ .f8e4m3fn, .f8e4m3fnuz }) |fp8_dtype| { if ((scheme == .mxfp4 or scheme == .mxfp8) and fp8_dtype == .f8e4m3fnuz) continue; diff --git a/zml/kernels/moe/triton/triton.zig b/zml/kernels/moe/triton/triton.zig index 8ba799a5..1e649072 100644 --- a/zml/kernels/moe/triton/triton.zig +++ b/zml/kernels/moe/triton/triton.zig @@ -2,18 +2,21 @@ const std = @import("std"); const stdx = @import("stdx"); -const zml = @import("../../../zml.zig"); +const nn = @import("zml/nn"); +const Quantization = @import("zml/quantization").Quantization; + +const zml = @import("zml/core"); const Tensor = zml.Tensor; const Shape = zml.Shape; const DataType = zml.DataType; const toDType = zml.kernel.triton.from; -const ExpertsLayout = @import("../api.zig").ExpertsLayout; -const ProjectionLayout = @import("../api.zig").ProjectionLayout; -const RoutingWeightPlacement = @import("../api.zig").RoutingWeightPlacement; -const Activation = @import("../api.zig").Activation; -const Options = @import("../api.zig").Options; -const LocalRouting = @import("../api.zig").LocalRouting; -const kernels = @import("moe.zig"); +const ExpertsLayout = @import("zml/kernels/moe/api").ExpertsLayout; +const ProjectionLayout = @import("zml/kernels/moe/api").ProjectionLayout; +const RoutingWeightPlacement = @import("zml/kernels/moe/api").RoutingWeightPlacement; +const Activation = @import("zml/kernels/moe/api").Activation; +const Options = @import("zml/kernels/moe/api").Options; +const LocalRouting = @import("zml/kernels/moe/api").LocalRouting; +pub const kernels = @import("moe.zig"); const log = std.log.scoped(.@"zml/moe/triton"); @@ -40,8 +43,8 @@ test { pub fn fusedExperts( input: zml.Tensor, routing: LocalRouting, - gate_up: zml.nn.Linear, - down: zml.nn.Linear, + gate_up: nn.Linear, + down: nn.Linear, options: Options, ) zml.Tensor { // HACK(Corentin): only rely on the scheme of gate_up for now. @@ -60,8 +63,8 @@ pub fn fusedExperts( pub fn fusedExpertsBase( hidden_states: Tensor, routing_: LocalRouting, - gate_up_: zml.nn.Linear, - down_: zml.nn.Linear, + gate_up_: nn.Linear, + down_: nn.Linear, layout: ProjectionLayout, opts: Options, ) Tensor { @@ -182,7 +185,7 @@ pub const GemmOptions = struct { routing: Routing, expert_ids: Tensor, launch_config: LaunchConfig, - quant_scheme: ?zml.Quantization.Scheme = null, + quant_scheme: ?Quantization.Scheme = null, top_k: usize, output_shape: Shape, }; @@ -436,7 +439,7 @@ fn alignBlockSize(topk_ids: Tensor, num_experts: i64, block_size_m: i64) struct return .{ sorted_token_ids, expert_ids, num_tokens_post_padded }; } -pub fn prepareInput(x: Tensor, scheme: ?zml.Quantization.Scheme, output_dtype: zml.DataType, quantize: bool) struct { Tensor, ?Tensor } { +pub fn prepareInput(x: Tensor, scheme: ?Quantization.Scheme, output_dtype: zml.DataType, quantize: bool) struct { Tensor, ?Tensor } { if (quantize) { if (scheme) |scheme_| { if (scheme_ != .mxfp4) { @@ -448,7 +451,7 @@ pub fn prepareInput(x: Tensor, scheme: ?zml.Quantization.Scheme, output_dtype: z return .{ x.convert(.bf16), null }; } -fn quantizeFp8Input(x: Tensor, scheme: zml.Quantization.Scheme, output_dtype: DataType) struct { Tensor, Tensor } { +fn quantizeFp8Input(x: Tensor, scheme: Quantization.Scheme, output_dtype: DataType) struct { Tensor, Tensor } { const group_size: i64, const scale_dtype: DataType, const absmax_epsilon: f32 = switch (scheme) { .mxfp8 => .{ 32, .u8, 1e-10 }, .fp8_per_channel, .fp8_per_tensor => .{ x.dim(1), .f32, 1e-10 }, @@ -548,7 +551,7 @@ test "FP8 routed GEMM with bias matches dequantized weights" { return expanded.withTags(shape.tags()); } - fn forward(x: Tensor, w: Tensor, scales: Tensor, ids: Tensor, weights: Tensor, bias: Tensor, scheme: zml.Quantization.Scheme, mx_native: bool, quantize_input: bool) Outputs { + fn forward(x: Tensor, w: Tensor, scales: Tensor, ids: Tensor, weights: Tensor, bias: Tensor, scheme: Quantization.Scheme, mx_native: bool, quantize_input: bool) Outputs { const weight = w.convert(.f8e4m3fn); var input = x; var input_scale: ?Tensor = null; @@ -592,7 +595,7 @@ test "FP8 routed GEMM with bias matches dequantized weights" { } }; - for ([_]zml.Quantization.Scheme{ .mxfp8, .fp8_per_tensor, .fp8_per_channel, .fp8_block128 }) |scheme| { + for ([_]Quantization.Scheme{ .mxfp8, .fp8_per_tensor, .fp8_per_channel, .fp8_block128 }) |scheme| { for ([_]bool{ false, true }) |quantize_input| { for ([_]i64{ 1, 17 }) |tokens| { const n: i64 = if (scheme == .fp8_block128) 256 else 48; @@ -674,7 +677,7 @@ test "fused experts support BF16 and MXFP4 layouts, bias, and routing weights" { const gate_value = if (fp4) Tensor.scalar(0x11, .u8) else Tensor.scalar(0.5, .bf16); const up_value = if (fp4) Tensor.scalar(0x33, .u8) else Tensor.scalar(1.5, .bf16); const gate_up_values = is_gate.select(gate_value, up_value); - var gate_up: zml.nn.Linear = .{ + var gate_up: nn.Linear = .{ .weight = gate_up_values.broad(Shape.init(.{ .expert = 8, .dout = 256, .d = @as(i64, if (fp4) 64 else 128) }, dtype)), .tag = Shape.toTag(.d), .quantization = if (fp4) .{ @@ -682,7 +685,7 @@ test "fused experts support BF16 and MXFP4 layouts, bias, and routing weights" { .scales = Tensor.scalar(127, .u8).broad(Shape.init(.{ .expert = 8, .dout = 256, .d = 4 }, .u8)), } else null, }; - var down: zml.nn.Linear = .{ + var down: nn.Linear = .{ .weight = (if (fp4) Tensor.scalar(0x22, .u8) else Tensor.scalar(1, .bf16)).broad(Shape.init(.{ .expert = 8, .d = 128, .dout = @as(i64, if (fp4) 64 else 128) }, dtype)), .bias = Tensor.scalar(2, .bf16).broad(Shape.init(.{ .expert = 8, .d = 128 }, .bf16)), .tag = Shape.toTag(.dout), @@ -902,8 +905,8 @@ fn validateInputs( pub fn fusedExpertsMxfp4( input: zml.Tensor, routing: LocalRouting, - gate_up: zml.nn.Linear, - down: zml.nn.Linear, + gate_up: nn.Linear, + down: nn.Linear, options: Options, ) zml.Tensor { validateOptions(options); diff --git a/zml/kernels/moe/weights/BUILD.bazel b/zml/kernels/moe/weights/BUILD.bazel new file mode 100644 index 00000000..0ddf5543 --- /dev/null +++ b/zml/kernels/moe/weights/BUILD.bazel @@ -0,0 +1,12 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "weights", + import_name = "zml/kernels/moe/weights", + main = "weights.zig", + visibility = ["//visibility:public"], + deps = [ + "//zml/core", + "//zml/nn", + ], +) diff --git a/zml/kernels/moe/weights.zig b/zml/kernels/moe/weights/weights.zig similarity index 77% rename from zml/kernels/moe/weights.zig rename to zml/kernels/moe/weights/weights.zig index 0e288856..fd3c9e82 100644 --- a/zml/kernels/moe/weights.zig +++ b/zml/kernels/moe/weights/weights.zig @@ -1,5 +1,5 @@ -const nn = @import("../../nn.zig"); -const Tensor = @import("../../tensor.zig").Tensor; +const nn = @import("zml/nn"); +const Tensor = @import("zml/core").Tensor; pub fn unpackedWeight(linear: nn.Linear) Tensor { const quantization = linear.quantization orelse return linear.weight; diff --git a/zml/kernels/paged_attention/BUILD.bazel b/zml/kernels/paged_attention/BUILD.bazel new file mode 100644 index 00000000..8050c3fa --- /dev/null +++ b/zml/kernels/paged_attention/BUILD.bazel @@ -0,0 +1,25 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "paged_attention", + import_name = "zml/kernels/paged_attention", + main = "paged_attention.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//stdx", + "//zml/core", + "//zml/kernels/paged_attention/cute", + "//zml/kernels/paged_attention/metal", + "//zml/kernels/paged_attention/mosaic_tpu", + "//zml/kernels/paged_attention/options", + "//zml/kernels/paged_attention/stablehlo", + "//zml/kernels/paged_attention/tcl", + "//zml/kernels/paged_attention/triton", + ], +) + +zig_test( + name = "test", + deps = [":paged_attention"], +) diff --git a/zml/kernels/paged_attention/cute/BUILD.bazel b/zml/kernels/paged_attention/cute/BUILD.bazel index 6329ae12..7c16ee7e 100644 --- a/zml/kernels/paged_attention/cute/BUILD.bazel +++ b/zml/kernels/paged_attention/cute/BUILD.bazel @@ -1,15 +1,31 @@ -load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_test") +load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library", "zig_test") package(default_visibility = ["//zml:__pkg__"]) -filegroup( +zig_library( name = "cute", srcs = [ "attention.zig", - "cute.zig", "//zml/kernels/paged_attention/cute/sm10x", "//zml/kernels/paged_attention/cute/sm120", ], + import_name = "zml/kernels/paged_attention/cute", + main = "cute.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//mlir", + "//mlir/dialects/llvm", + "//mlir/dialects/vector", + "//stdx", + "//zml/core", + "//zml/kernels/paged_attention/options", + ], +) + +zig_test( + name = "test", + deps = [":cute"], ) zig_test( diff --git a/zml/kernels/paged_attention/cute/attention.zig b/zml/kernels/paged_attention/cute/attention.zig index 50eb2531..7d88d64d 100644 --- a/zml/kernels/paged_attention/cute/attention.zig +++ b/zml/kernels/paged_attention/cute/attention.zig @@ -1,6 +1,6 @@ const std = @import("std"); -const zml = @import("../../../zml.zig"); +const zml = @import("zml/core"); const cute = zml.kernel.cute; const V = cute.Value; diff --git a/zml/kernels/paged_attention/cute/cute.zig b/zml/kernels/paged_attention/cute/cute.zig index ef78cc49..199b05a9 100644 --- a/zml/kernels/paged_attention/cute/cute.zig +++ b/zml/kernels/paged_attention/cute/cute.zig @@ -1,7 +1,7 @@ const stdx = @import("stdx"); -const zml = @import("../../../zml.zig"); -pub const AttentionOptions = @import("../options.zig").AttentionOptions; +const zml = @import("zml/core"); +pub const AttentionOptions = @import("zml/kernels/paged_attention/options").AttentionOptions; const kernel = @import("attention.zig"); const sm10x = @import("sm10x/sm10x.zig"); const sm120 = @import("sm120/sm120.zig"); diff --git a/zml/kernels/paged_attention/cute/sm10x/attention.zig b/zml/kernels/paged_attention/cute/sm10x/attention.zig index 12d6273c..5fbdee49 100644 --- a/zml/kernels/paged_attention/cute/sm10x/attention.zig +++ b/zml/kernels/paged_attention/cute/sm10x/attention.zig @@ -1,6 +1,6 @@ const std = @import("std"); -const zml = @import("../../../../zml.zig"); +const zml = @import("zml/core"); const cute = zml.kernel.cute; const B = cute.Builder; const V = cute.Value; diff --git a/zml/kernels/paged_attention/cute/sm10x/sm103.zig b/zml/kernels/paged_attention/cute/sm10x/sm103.zig index fe96916d..42750b11 100644 --- a/zml/kernels/paged_attention/cute/sm10x/sm103.zig +++ b/zml/kernels/paged_attention/cute/sm10x/sm103.zig @@ -2,7 +2,7 @@ const std = @import("std"); const mlir = @import("mlir"); const llvm = @import("mlir/dialects/llvm"); -const cute = @import("../../../../zml.zig").kernel.cute; +const cute = @import("zml/core").kernel.cute; pub const Scores = struct { values: [64]cute.Value, diff --git a/zml/kernels/paged_attention/cute/sm10x/sm10x.zig b/zml/kernels/paged_attention/cute/sm10x/sm10x.zig index afc4298e..24d83469 100644 --- a/zml/kernels/paged_attention/cute/sm10x/sm10x.zig +++ b/zml/kernels/paged_attention/cute/sm10x/sm10x.zig @@ -1,7 +1,7 @@ const std = @import("std"); const stdx = @import("stdx"); -const zml = @import("../../../../zml.zig"); +const zml = @import("zml/core"); const cute = @import("../cute.zig"); const kernel = @import("attention.zig"); diff --git a/zml/kernels/paged_attention/cute/sm120/attention.zig b/zml/kernels/paged_attention/cute/sm120/attention.zig index e9675252..8436a897 100644 --- a/zml/kernels/paged_attention/cute/sm120/attention.zig +++ b/zml/kernels/paged_attention/cute/sm120/attention.zig @@ -3,7 +3,7 @@ const mlir = @import("mlir"); const llvm = @import("mlir/dialects/llvm"); const vector = @import("mlir/dialects/vector"); -const zml = @import("../../../../zml.zig"); +const zml = @import("zml/core"); const cute = zml.kernel.cute; const B = cute.Builder; const V = cute.Value; diff --git a/zml/kernels/paged_attention/cute/sm120/sm120.zig b/zml/kernels/paged_attention/cute/sm120/sm120.zig index 6645abf1..787c5cd9 100644 --- a/zml/kernels/paged_attention/cute/sm120/sm120.zig +++ b/zml/kernels/paged_attention/cute/sm120/sm120.zig @@ -1,6 +1,6 @@ const stdx = @import("stdx"); -const zml = @import("../../../../zml.zig"); +const zml = @import("zml/core"); const cute = @import("../cute.zig"); const kernel = @import("attention.zig"); diff --git a/zml/kernels/paged_attention/metal/BUILD.bazel b/zml/kernels/paged_attention/metal/BUILD.bazel index 39d92259..bc91f0dc 100644 --- a/zml/kernels/paged_attention/metal/BUILD.bazel +++ b/zml/kernels/paged_attention/metal/BUILD.bazel @@ -1,9 +1,22 @@ -package(default_visibility = ["//zml:__pkg__"]) +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") -filegroup( +zig_library( name = "metal", srcs = [ - "metal.zig", "paged_attn.zig", ], + import_name = "zml/kernels/paged_attention/metal", + main = "metal.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//stdx", + "//zml/core", + "//zml/kernels/paged_attention/options", + ], +) + +zig_test( + name = "test", + deps = [":metal"], ) diff --git a/zml/kernels/paged_attention/metal/metal.zig b/zml/kernels/paged_attention/metal/metal.zig index 2cec68f5..9e3b6e25 100644 --- a/zml/kernels/paged_attention/metal/metal.zig +++ b/zml/kernels/paged_attention/metal/metal.zig @@ -2,8 +2,8 @@ const std = @import("std"); const stdx = @import("stdx"); -const zml = @import("../../../zml.zig"); -const AttentionOptions = @import("../options.zig").AttentionOptions; +const zml = @import("zml/core"); +const AttentionOptions = @import("zml/kernels/paged_attention/options").AttentionOptions; const paged_attn = @import("paged_attn.zig"); const log = std.log.scoped(.@"zml/attention/metal"); diff --git a/zml/kernels/paged_attention/metal/paged_attn.zig b/zml/kernels/paged_attention/metal/paged_attn.zig index 04f4c1ac..10a088e5 100644 --- a/zml/kernels/paged_attention/metal/paged_attn.zig +++ b/zml/kernels/paged_attention/metal/paged_attn.zig @@ -3,7 +3,7 @@ const std = @import("std"); -const metal = @import("../../../kernel/kernel.zig").metal; +const metal = @import("zml/core").kernel.metal; const ArgSpec = metal.ArgSpec; const Builder = metal.Builder; diff --git a/zml/kernels/paged_attention/mosaic_tpu/BUILD.bazel b/zml/kernels/paged_attention/mosaic_tpu/BUILD.bazel index 6a0e4e87..a6a2143c 100644 --- a/zml/kernels/paged_attention/mosaic_tpu/BUILD.bazel +++ b/zml/kernels/paged_attention/mosaic_tpu/BUILD.bazel @@ -1,9 +1,24 @@ -package(default_visibility = ["//zml:__pkg__"]) +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") -filegroup( +zig_library( name = "mosaic_tpu", srcs = [ - "mosaic_tpu.zig", "ragged_attention.zig", ], + import_name = "zml/kernels/paged_attention/mosaic_tpu", + main = "mosaic_tpu.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//mlir", + "//platforms/tpu:ragged_paged", + "//stdx", + "//zml/core", + "//zml/kernels/paged_attention/options", + ], +) + +zig_test( + name = "test", + deps = [":mosaic_tpu"], ) diff --git a/zml/kernels/paged_attention/mosaic_tpu/mosaic_tpu.zig b/zml/kernels/paged_attention/mosaic_tpu/mosaic_tpu.zig index 84716379..c871cfca 100644 --- a/zml/kernels/paged_attention/mosaic_tpu/mosaic_tpu.zig +++ b/zml/kernels/paged_attention/mosaic_tpu/mosaic_tpu.zig @@ -4,8 +4,8 @@ const mlir = @import("mlir"); const ragged_paged = @import("platforms/tpu/ragged_paged"); const stdx = @import("stdx"); -const zml = @import("../../../zml.zig"); -const AttentionOptions = @import("../options.zig").AttentionOptions; +const zml = @import("zml/core"); +const AttentionOptions = @import("zml/kernels/paged_attention/options").AttentionOptions; const ragged_attention = @import("ragged_attention.zig"); const log = std.log.scoped(.tpu_attention); diff --git a/zml/kernels/paged_attention/mosaic_tpu/ragged_attention.zig b/zml/kernels/paged_attention/mosaic_tpu/ragged_attention.zig index 30536306..ef18328d 100644 --- a/zml/kernels/paged_attention/mosaic_tpu/ragged_attention.zig +++ b/zml/kernels/paged_attention/mosaic_tpu/ragged_attention.zig @@ -1,6 +1,6 @@ const ragged_paged = @import("platforms/tpu/ragged_paged"); -const zml = @import("../../../zml.zig"); +const zml = @import("zml/core"); test { _ = Kernel; diff --git a/zml/kernels/paged_attention/options/BUILD.bazel b/zml/kernels/paged_attention/options/BUILD.bazel new file mode 100644 index 00000000..84ef4ed4 --- /dev/null +++ b/zml/kernels/paged_attention/options/BUILD.bazel @@ -0,0 +1,11 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "options", + import_name = "zml/kernels/paged_attention/options", + main = "options.zig", + visibility = ["//visibility:public"], + deps = [ + "//zml/core", + ], +) diff --git a/zml/kernels/paged_attention/options.zig b/zml/kernels/paged_attention/options/options.zig similarity index 95% rename from zml/kernels/paged_attention/options.zig rename to zml/kernels/paged_attention/options/options.zig index 35ad4049..159da9b2 100644 --- a/zml/kernels/paged_attention/options.zig +++ b/zml/kernels/paged_attention/options/options.zig @@ -1,4 +1,4 @@ -const Tensor = @import("../../tensor.zig").Tensor; +const Tensor = @import("zml/core").Tensor; pub const AttentionMask = union(enum) { standard: struct { diff --git a/zml/kernels/paged_attention/paged_attention.zig b/zml/kernels/paged_attention/paged_attention.zig index 4a97a38e..a3f07605 100644 --- a/zml/kernels/paged_attention/paged_attention.zig +++ b/zml/kernels/paged_attention/paged_attention.zig @@ -2,20 +2,19 @@ const std = @import("std"); const stdx = @import("stdx"); -const zml = @import("../../zml.zig"); -pub const AttentionMask = @import("options.zig").AttentionMask; -pub const AttentionOptions = @import("options.zig").AttentionOptions; -pub const cute = @import("cute/cute.zig"); -const metal = @import("metal/metal.zig"); -const tpu = @import("mosaic_tpu/mosaic_tpu.zig"); -const stablehlo = @import("stablehlo/stablehlo.zig"); -pub const tcl = @import("tcl/tcl.zig"); -const triton = @import("triton/triton.zig"); +const zml = @import("zml/core"); +pub const AttentionMask = @import("zml/kernels/paged_attention/options").AttentionMask; +pub const AttentionOptions = @import("zml/kernels/paged_attention/options").AttentionOptions; +pub const cute = @import("zml/kernels/paged_attention/cute"); +const metal = @import("zml/kernels/paged_attention/metal"); +const tpu = @import("zml/kernels/paged_attention/mosaic_tpu"); +const stablehlo = @import("zml/kernels/paged_attention/stablehlo"); +pub const tcl = @import("zml/kernels/paged_attention/tcl"); +const triton = @import("zml/kernels/paged_attention/triton"); const PagedAttention = @This(); test { - _ = tcl; std.testing.refAllDecls(Backend); std.testing.refAllDecls(Options); std.testing.refAllDecls(Parameters); @@ -63,7 +62,7 @@ pub const Options = union(Backend) { triton: triton.paged.Options, mosaic_tpu: tpu.mosaic_tpu.Options, metal: metal.Options, - stablehlo: triton.paged.Options, + stablehlo: stablehlo.Options, tcl: tcl.paged.Options, const Args = struct { @@ -122,7 +121,7 @@ pub const Parameters = union(Backend) { triton: triton.paged.Parameters, mosaic_tpu: tpu.mosaic_tpu.Parameters, metal: metal.Parameters, - stablehlo: triton.paged.Parameters, + stablehlo: stablehlo.Parameters, tcl: tcl.paged.Parameters, pub fn init(options_: Options) Parameters { diff --git a/zml/kernels/paged_attention/stablehlo/BUILD.bazel b/zml/kernels/paged_attention/stablehlo/BUILD.bazel index f60464f4..3a867a90 100644 --- a/zml/kernels/paged_attention/stablehlo/BUILD.bazel +++ b/zml/kernels/paged_attention/stablehlo/BUILD.bazel @@ -1,6 +1,13 @@ -package(default_visibility = ["//zml:__pkg__"]) +load("@rules_zig//zig:defs.bzl", "zig_library") -filegroup( +zig_library( name = "stablehlo", - srcs = ["stablehlo.zig"], + import_name = "zml/kernels/paged_attention/stablehlo", + main = "stablehlo.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//zml/core", + "//zml/kernels/paged_attention/options", + ], ) diff --git a/zml/kernels/paged_attention/stablehlo/stablehlo.zig b/zml/kernels/paged_attention/stablehlo/stablehlo.zig index 55ee01d5..b4dd5415 100644 --- a/zml/kernels/paged_attention/stablehlo/stablehlo.zig +++ b/zml/kernels/paged_attention/stablehlo/stablehlo.zig @@ -1,9 +1,53 @@ -const zml = @import("../../../zml.zig"); -const AttentionOptions = @import("../options.zig").AttentionOptions; -const triton = @import("../triton/triton.zig"); +const zml = @import("zml/core"); +const AttentionOptions = @import("zml/kernels/paged_attention/options").AttentionOptions; + +pub const Options = struct { + batch_size: usize, + max_num_pages: usize, + max_seqlen_q: usize, + is_prefill: bool, + + pub fn isPrefill(self: Options) bool { + return self.is_prefill; + } + + pub fn maxNumPages(self: Options) usize { + return self.max_num_pages; + } +}; + +pub const Parameters = struct { + block_table: zml.Tensor, + seq_lens: zml.Tensor, + query_start_len: zml.Tensor, + options_: Options, + + pub fn init(options_: Options) Parameters { + return .{ + .block_table = .init(.{ .b = options_.batch_size, .p = options_.max_num_pages }, .i32), + .seq_lens = .init(.{ .b = options_.batch_size }, .i32), + .query_start_len = .init(.{ .b = options_.batch_size + 1 }, .i32), + .options_ = options_, + }; + } + + pub fn allocationSize(self: Parameters) usize { + var allocation_size: usize = 0; + + allocation_size += self.block_table.byteSize(); + allocation_size += self.seq_lens.byteSize(); + allocation_size += self.query_start_len.byteSize(); + + return allocation_size; + } + + pub fn options(self: Parameters) Options { + return self.options_; + } +}; pub fn pagedAttention( - parameters: triton.paged.Parameters, + parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, @@ -21,7 +65,6 @@ pub fn pagedAttention( AttentionLoop, .{ .q = padded_q, .k = k, .v = v, .parameters = parameters, .opts = opts }, .{ - // i32 to match triton conventions .slot_id = .scalar(0, .i32), .seq_id = .scalar(0, .i32), .q_page_idx = .scalar(0, .i32), @@ -40,7 +83,7 @@ const AttentionLoop = struct { q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, - parameters: triton.paged.Parameters, + parameters: Parameters, opts: AttentionOptions, const While = @This(); diff --git a/zml/kernels/paged_attention/tcl/BUILD.bazel b/zml/kernels/paged_attention/tcl/BUILD.bazel index 4057eea3..550f0349 100644 --- a/zml/kernels/paged_attention/tcl/BUILD.bazel +++ b/zml/kernels/paged_attention/tcl/BUILD.bazel @@ -1,6 +1,20 @@ -package(default_visibility = ["//zml:__pkg__"]) +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") -filegroup( +zig_library( name = "tcl", - srcs = ["tcl.zig"], + import_name = "zml/kernels/paged_attention/tcl", + main = "tcl.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//dsl/tcl:builder", + "//stdx", + "//zml/core", + "//zml/kernels/paged_attention/options", + ], +) + +zig_test( + name = "test", + deps = [":tcl"], ) diff --git a/zml/kernels/paged_attention/tcl/tcl.zig b/zml/kernels/paged_attention/tcl/tcl.zig index d04b684e..8de744b8 100644 --- a/zml/kernels/paged_attention/tcl/tcl.zig +++ b/zml/kernels/paged_attention/tcl/tcl.zig @@ -6,11 +6,10 @@ const std = @import("std"); const builder = @import("dsl/tcl/builder"); const stdx = @import("stdx"); -const zml = @import("../../../zml.zig"); +const zml = @import("zml/core"); const Tensor = zml.Tensor; const tcl = zml.kernel.tcl; -const AttentionOptions = @import("../options.zig").AttentionOptions; -const triton = @import("../triton/triton.zig"); +const AttentionOptions = @import("zml/kernels/paged_attention/options").AttentionOptions; /// WORKAROUND (TCC 2026.5.0b0): flash decode with 8192-key chunks fails to /// lower, so decode chunks hold 4096 keys. Repro: repro_flash_decode_8k_chunks.py. @@ -44,8 +43,50 @@ pub const Plan = struct { pub var calibration_plan: ?Plan = null; pub const paged = struct { - pub const Options = triton.paged.Options; - pub const Parameters = triton.paged.Parameters; + pub const Options = struct { + batch_size: usize, + max_num_pages: usize, + max_seqlen_q: usize, + is_prefill: bool, + + pub fn isPrefill(self: Options) bool { + return self.is_prefill; + } + + pub fn maxNumPages(self: Options) usize { + return self.max_num_pages; + } + }; + + pub const Parameters = struct { + block_table: Tensor, + seq_lens: Tensor, + query_start_len: Tensor, + options_: Options, + + pub fn init(options_: Options) Parameters { + return .{ + .block_table = .init(.{ .b = options_.batch_size, .p = options_.max_num_pages }, .i32), + .seq_lens = .init(.{ .b = options_.batch_size }, .i32), + .query_start_len = .init(.{ .b = options_.batch_size + 1 }, .i32), + .options_ = options_, + }; + } + + pub fn allocationSize(self: Parameters) usize { + var allocation_size: usize = 0; + + allocation_size += self.block_table.byteSize(); + allocation_size += self.seq_lens.byteSize(); + allocation_size += self.query_start_len.byteSize(); + + return allocation_size; + } + + pub fn options(self: Parameters) Options { + return self.options_; + } + }; /// `q` is `{.b = tokens, .hkv, .hg, .hd}`, the caches `{.page, .k_chunk, .hkv, .hd}`. /// Sinks are one logit per query head. diff --git a/zml/kernels/paged_attention/triton/BUILD.bazel b/zml/kernels/paged_attention/triton/BUILD.bazel index eff7d096..d921a8d3 100644 --- a/zml/kernels/paged_attention/triton/BUILD.bazel +++ b/zml/kernels/paged_attention/triton/BUILD.bazel @@ -1,11 +1,21 @@ -package(default_visibility = ["//zml:__pkg__"]) +load("@rules_zig//zig:defs.bzl", "zig_library") -filegroup( +zig_library( name = "triton", srcs = [ "sparse_unified_attention.zig", - "triton.zig", "unified_attention.zig", "unified_attention_oneapi.zig", ], + import_name = "zml/kernels/paged_attention/triton", + main = "triton.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//mlir", + "//mlir/dialects/ttir", + "//stdx", + "//zml/core", + "//zml/kernels/paged_attention/options", + ], ) diff --git a/zml/kernels/paged_attention/triton/sparse_unified_attention.zig b/zml/kernels/paged_attention/triton/sparse_unified_attention.zig index 79113f1e..1682a168 100644 --- a/zml/kernels/paged_attention/triton/sparse_unified_attention.zig +++ b/zml/kernels/paged_attention/triton/sparse_unified_attention.zig @@ -1,5 +1,5 @@ const std = @import("std"); -const zml = @import("../../../zml.zig"); +const zml = @import("zml/core"); const tri = zml.kernel.triton; const Builder = tri.Builder; const Value = tri.Value; diff --git a/zml/kernels/paged_attention/triton/triton.zig b/zml/kernels/paged_attention/triton/triton.zig index 2a12e012..e9bd44e6 100644 --- a/zml/kernels/paged_attention/triton/triton.zig +++ b/zml/kernels/paged_attention/triton/triton.zig @@ -2,9 +2,9 @@ const std = @import("std"); const stdx = @import("stdx"); -const zml = @import("../../../zml.zig"); +const zml = @import("zml/core"); const triton = zml.kernel.triton; -const AttentionOptions = @import("../options.zig").AttentionOptions; +const AttentionOptions = @import("zml/kernels/paged_attention/options").AttentionOptions; const kernels = @import("unified_attention.zig"); const kernels_oneapi = @import("unified_attention_oneapi.zig"); const kernels_sparse = @import("sparse_unified_attention.zig"); diff --git a/zml/kernels/paged_attention/triton/unified_attention.zig b/zml/kernels/paged_attention/triton/unified_attention.zig index 7b2b9428..46e5ca76 100644 --- a/zml/kernels/paged_attention/triton/unified_attention.zig +++ b/zml/kernels/paged_attention/triton/unified_attention.zig @@ -3,7 +3,7 @@ const std = @import("std"); const mlir = @import("mlir"); const ttir = @import("mlir/dialects/ttir"); -const zml = @import("../../../zml.zig"); +const zml = @import("zml/core"); const tri = zml.kernel.triton; const Builder = tri.Builder; const Value = tri.Value; diff --git a/zml/kernels/paged_attention/triton/unified_attention_oneapi.zig b/zml/kernels/paged_attention/triton/unified_attention_oneapi.zig index b7c724f9..365108a6 100644 --- a/zml/kernels/paged_attention/triton/unified_attention_oneapi.zig +++ b/zml/kernels/paged_attention/triton/unified_attention_oneapi.zig @@ -3,7 +3,7 @@ const std = @import("std"); const mlir = @import("mlir"); const ttir = @import("mlir/dialects/ttir"); -const zml = @import("../../../zml.zig"); +const zml = @import("zml/core"); const tri = zml.kernel.triton; const Builder = tri.Builder; const Value = tri.Value; diff --git a/zml/logo/BUILD.bazel b/zml/logo/BUILD.bazel new file mode 100644 index 00000000..1a7af734 --- /dev/null +++ b/zml/logo/BUILD.bazel @@ -0,0 +1,14 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "logo", + srcs = ["logo.zig"], + import_name = "zml/logo", + main = "logo.zig", + visibility = ["//visibility:public"], +) + +zig_test( + name = "test", + deps = [":logo"], +) diff --git a/zml/logo.zig b/zml/logo/logo.zig similarity index 100% rename from zml/logo.zig rename to zml/logo/logo.zig diff --git a/zml/meta/BUILD.bazel b/zml/meta/BUILD.bazel new file mode 100644 index 00000000..a795993a --- /dev/null +++ b/zml/meta/BUILD.bazel @@ -0,0 +1,14 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "meta", + import_name = "zml/meta", + main = "meta.zig", + visibility = ["//visibility:public"], + deps = ["//stdx"], +) + +zig_test( + name = "test", + deps = [":meta"], +) diff --git a/zml/meta.zig b/zml/meta/meta.zig similarity index 100% rename from zml/meta.zig rename to zml/meta/meta.zig diff --git a/zml/nn/BUILD.bazel b/zml/nn/BUILD.bazel new file mode 100644 index 00000000..4fa4c325 --- /dev/null +++ b/zml/nn/BUILD.bazel @@ -0,0 +1,23 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "nn", + import_name = "zml/nn", + main = "nn.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//mlir", + "//mlir/dialects/stablehlo", + "//stdx", + "//zml/constants", + "//zml/core", + "//zml/meta", + "//zml/quantization", + ], +) + +zig_test( + name = "test", + deps = [":nn"], +) diff --git a/zml/nn.zig b/zml/nn/nn.zig similarity index 99% rename from zml/nn.zig rename to zml/nn/nn.zig index eceb316b..94347552 100644 --- a/zml/nn.zig +++ b/zml/nn/nn.zig @@ -1,21 +1,22 @@ //! Common layer definition and functions for Neural Networks (NN) const std = @import("std"); +const constants = @import("zml/constants"); +const meta = @import("zml/meta"); const mlir = @import("mlir"); const stablehlo = @import("mlir/dialects/stablehlo"); const stdx = @import("stdx"); -const constants = @import("constants.zig"); -const DataType = @import("dtype.zig").DataType; -const meta = @import("meta.zig"); -const mlirx = @import("mlirx.zig"); -const ops = @import("ops.zig"); -const quantization = @import("quantization.zig"); +const zml = @import("zml/core"); +const quantization = @import("zml/quantization"); + +const DataType = zml.DataType; +const mlirx = zml.mlir; +const ops = zml.ops; pub const Quantization = quantization.Quantization; -const Shape = @import("shape.zig").Shape; -const Slice = @import("slice.zig").Slice; -const Tensor = @import("tensor.zig").Tensor; -const zml = @import("zml.zig"); +const Shape = zml.Shape; +const Slice = zml.Slice; +const Tensor = zml.Tensor; const log = std.log.scoped(.@"zml/nn"); @@ -229,7 +230,7 @@ test "block128 scaled dot layouts" { const platform = zml.testing.env(); // FIXME: Add rocm when we have a PJRT plugin with native fp8 dot support (by the end of Sept 2026) if (platform.target != .cuda and platform.target != .furiosa) return error.SkipZigTest; - const dtype: DataType = if (platform.target == .rocm and @import("platform.zig").rocm.computeCapability(platform) == .gfx942) .f8e4m3fnuz else .f8e4m3fn; + const dtype: DataType = if (platform.target == .rocm and zml.platform.rocm.computeCapability(platform) == .gfx942) .f8e4m3fnuz else .f8e4m3fn; const Local = struct { const Outputs = struct { actual: Tensor, expected: Tensor, linear: Tensor, linear_expected: Tensor }; @@ -631,7 +632,7 @@ pub fn rope(x: Tensor, pos_idx: ?Tensor, opts: RopeOpts) Tensor { const rotary_dim: u32 = opts.scaling.partialRotaryDim(head_dim); stdx.debug.assert(rotary_dim > 0 and @mod(rotary_dim, 2) == 0, "partial rope expects a even head dim (.hd), got {d}", .{rotary_dim}); - const x_real, const x_imag, const x_pass = zml.nn.splitRealImgPass(x, opts.layout, rotary_dim); + const x_real, const x_imag, const x_pass = splitRealImgPass(x, opts.layout, rotary_dim); const inv_freq = invFreq(head_dim, opts).withTags(.{.hd}); // compute sin and cos in f32 before downcasting to x type. @@ -644,7 +645,7 @@ pub fn rope(x: Tensor, pos_idx: ?Tensor, opts: RopeOpts) Tensor { const y_real = x_real.mul(cos).sub(x_imag.mul(sin)); const y_imag = x_real.mul(sin).add(x_imag.mul(cos)); - return zml.nn.mergeRealImgPass(y_real, y_imag, x_pass, opts.layout); + return mergeRealImgPass(y_real, y_imag, x_pass, opts.layout); } pub fn splitRealImg(x: Tensor, layout: RopeOpts.Layout) [2]Tensor { diff --git a/zml/profiling/BUILD.bazel b/zml/profiling/BUILD.bazel new file mode 100644 index 00000000..86675b54 --- /dev/null +++ b/zml/profiling/BUILD.bazel @@ -0,0 +1,77 @@ +load("@com_google_protobuf//bazel:upb_c_proto_library.bzl", "upb_c_proto_library") +load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +upb_c_proto_library( + name = "profiler_options", + tags = ["manual"], + deps = ["@tsl//tsl/profiler/protobuf:profiler_options_proto"], +) + +cc_library( + name = "profiling_traceme_lib", + srcs = ["traceme_lib.cc"], + hdrs = ["traceme.h"], + defines = select({ + "//platforms:cuda.enabled": ["ZML_RUNTIME_CUDA"], + "//conditions:default": [], + }) + select({ + "//platforms:rocm.enabled": ["ZML_RUNTIME_ROCM"], + "//conditions:default": [], + }), + linkopts = select({ + "//platforms:rocm.enabled": ["-ldl"], + "//conditions:default": [], + }), + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//ffi:cc", + "@tsl//tsl/profiler/lib:traceme", + "@tsl//tsl/profiler/protobuf:xplane_proto_cc", + "@xla//xla/tsl/profiler/backends/cpu:host_tracer_utils", + "@xla//xla/tsl/profiler/backends/cpu:traceme_recorder", + "@xla//xla/tsl/profiler/convert:post_process_single_host_xplane", + "@xla//xla/tsl/profiler/utils:time_utils", + "@xla//xla/tsl/profiler/utils:xplane_utils", + ] + select({ + "//platforms:cuda.enabled": [ + "//platforms/cuda:cuda_nvtx_headers", + ], + "//conditions:default": [], + }) + select({ + "//platforms:rocm.enabled": [ + "@amdrocm-profiler-base//:roctx_headers", + ], + "//conditions:default": [], + }), +) + +zig_library( + name = "profiling", + srcs = [ + "profiler.zig", + "tracer.zig", + ], + import_name = "zml/profiling", + main = "profiling.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + zigopts = ["-lc"], + deps = [ + ":profiler_options", + ":profiling_traceme_lib", + "//bazel", + "//ffi:zig", + "//pjrt", + "//platforms", + "//stdx", + "//tools/xspace_to_perfetto:xspace_to_perfetto_lib", + "//upb", + ], +) + +zig_test( + name = "test", + deps = [":profiling"], +) diff --git a/zml/profiling/profiling.zig b/zml/profiling/profiling.zig new file mode 100644 index 00000000..41bf94f9 --- /dev/null +++ b/zml/profiling/profiling.zig @@ -0,0 +1,12 @@ +pub const profiler = @import("profiler.zig"); +pub const tracer = @import("tracer.zig"); + +test { + @import("std").testing.refAllDecls(@This()); +} + +comptime { + // Forces zig to compile `@zml//bazel` module even when users don't depend on it even indirectly. + // Otherwise exported functions declared in `bazel` aren't compiled. + @import("std").testing.refAllDecls(@import("bazel")); +} diff --git a/zml/quantization/BUILD.bazel b/zml/quantization/BUILD.bazel new file mode 100644 index 00000000..8e0bd565 --- /dev/null +++ b/zml/quantization/BUILD.bazel @@ -0,0 +1,18 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "quantization", + import_name = "zml/quantization", + main = "quantization.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//stdx", + "//zml/core", + ], +) + +zig_test( + name = "test", + deps = [":quantization"], +) diff --git a/zml/quantization.zig b/zml/quantization/quantization.zig similarity index 99% rename from zml/quantization.zig rename to zml/quantization/quantization.zig index db20619a..70576db5 100644 --- a/zml/quantization.zig +++ b/zml/quantization/quantization.zig @@ -3,13 +3,13 @@ const std = @import("std"); const stdx = @import("stdx"); -const zml = @import("zml.zig"); +const zml = @import("zml/core"); -const DataType = @import("dtype.zig").DataType; -const platform_mod = @import("platform.zig"); +const DataType = zml.DataType; +const platform_mod = zml.platform; const Platform = platform_mod.Platform; -const Shape = @import("shape.zig").Shape; -const Tensor = @import("tensor.zig").Tensor; +const Shape = zml.Shape; +const Tensor = zml.Tensor; pub const nvfp4_block_size = 16; pub const mx_block_size = 32; diff --git a/zml/zml.zig b/zml/zml.zig index dafc3a0f..ef8887ed 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -5,85 +5,62 @@ // Namespaces const std = @import("std"); +const core = @import("zml/core"); pub const bazel = @import("bazel"); -const c = @import("c"); +pub const constants = @import("zml/constants"); +pub const meta = @import("zml/meta"); pub const pjrt = @import("pjrt"); -const runfiles = @import("runfiles"); pub const stdx = @import("stdx"); pub const tokenizer = @import("zml/tokenizer"); +pub const tracer = @import("zml/profiling").tracer; pub const attention = @import("attention/attention.zig"); -pub const paged_attention = @import("kernels/paged_attention/paged_attention.zig"); -pub const Buffer = @import("buffer.zig").Buffer; -pub const Compiler = @import("Compiler.zig"); -pub const constants = @import("constants.zig"); -pub const dtype = @import("dtype.zig"); -pub const Data = dtype.Data; -pub const DataType = dtype.DataType; -pub const exe = @import("exe.zig"); -pub const Exe = exe.Exe; -pub const FnExe = exe.FnExe; -pub const floats = @import("floats.zig"); -pub const io = @import("io.zig"); -pub const kernel = @import("kernel/kernel.zig"); -pub const logo = @import("logo.zig"); -pub const mem = @import("mem.zig"); -pub const Bufferized = mem.Bufferized; -pub const meta = @import("meta.zig"); -pub const mlir = @import("mlirx.zig"); -pub const moe = @import("kernels/moe/moe.zig"); -pub const nn = @import("nn.zig"); -pub const ops = @import("ops.zig"); -pub const pjrtx = @import("pjrtx.zig"); -pub const platform = @import("platform.zig"); -pub const Memory = platform.Memory; -pub const Platform = platform.Platform; -pub const Target = platform.Target; -pub const tracer = @import("profiling/tracer.zig"); -pub const quantization = @import("quantization.zig"); +pub const Buffer = core.Buffer; +pub const Compiler = core.Compiler; +pub const dtype = core.dtype; +pub const Data = core.Data; +pub const DataType = core.DataType; +pub const exe = core.exe; +pub const Exe = core.Exe; +pub const FnExe = core.FnExe; +pub const floats = core.floats; +pub const io = core.io; +pub const kernel = core.kernel; +pub const moe = @import("zml/kernels/moe"); +pub const paged_attention = @import("zml/kernels/paged_attention"); +pub const logo = @import("zml/logo"); +pub const mem = core.mem; +pub const Bufferized = core.Bufferized; +pub const mlir = core.mlir; +pub const nn = @import("zml/nn"); +pub const ops = core.ops; +pub const pjrtx = core.pjrtx; +pub const platform = core.platform; +pub const Memory = core.Memory; +pub const Platform = core.Platform; +pub const Target = core.Target; +pub const quantization = @import("zml/quantization"); pub const Quantization = quantization.Quantization; -pub const safetensors = @import("safetensors.zig"); -pub const shape = @import("shape.zig"); -pub const Shape = shape.Shape; -pub const Sharding = @import("Sharding.zig"); -pub const Mesh = Sharding.Mesh; -pub const slice = @import("slice.zig"); -pub const Slice = slice.Slice; -pub const tensor = @import("tensor.zig"); -pub const Tensor = tensor.Tensor; -pub const testing = @import("testing.zig"); +pub const safetensors = core.safetensors; +pub const shape = core.shape; +pub const Shape = core.Shape; +pub const Sharding = core.Sharding; +pub const Mesh = core.Mesh; +pub const slice = core.slice; +pub const Slice = core.Slice; +pub const tensor = core.tensor; +pub const Tensor = core.Tensor; +pub const testing = core.testing; /// Deprecated, use `zml.Compiler` instead -pub const module = struct { - pub const CompilationContext = Compiler; - pub const CompileError = Compiler.Error; - pub const CompilationOptions = Compiler.Options; - - /// Deprecated, use `platform.compileFn` or `zml.Compiler.compile` - pub fn compile( - allocator: std.mem.Allocator, - io_: std.Io, - comptime func: anytype, - args: std.meta.ArgsTuple(@TypeOf(func)), - platform_: *const Platform, - opts: Compiler.Options, - ) Compiler.Error!Exe { - return Compiler.compile(allocator, io_, platform_, func, args, opts); - } -}; +pub const module = core.module; test "zml" { std.testing.refAllDecls(@This()); } -pub const KiB = 1024; -pub const MiB = 1024 * KiB; -pub const GiB = 1024 * MiB; -pub const TiB = 1024 * GiB; - -comptime { - // Forces zig to compile `@zml//bazel` module even when users don't depend on it even indirectly. - // Otherwise exported functions declared in `bazel` aren't compiled. - std.testing.refAllDecls(bazel); -} +pub const KiB = core.KiB; +pub const MiB = core.MiB; +pub const GiB = core.GiB; +pub const TiB = core.TiB;