From 3796132f5af42031077d479fd86fed46e433601a Mon Sep 17 00:00:00 2001 From: James Deal Date: Tue, 20 May 2025 22:37:02 -0500 Subject: [PATCH 01/13] Runtimes: (WIP) MLX PjRt Plugin Update plugin implementation to latest ZML --- MODULE.bazel.lock | 6 + runtimes/BUILD.bazel | 4 + runtimes/mlx/BUILD.bazel | 36 + runtimes/mlx/mlx.zig | 18 + runtimes/runtimes.zig | 4 + .../mlx/20250518.0-eebe730/MODULE.bazel | 11 + .../20250518.0-eebe730/overlay/BUILD.bazel | 592 +++ .../20250518.0-eebe730/overlay/MODULE.bazel | 1 + .../mlx/20250518.0-eebe730/overlay/jit.bzl | 23 + .../20250518.0-eebe730/overlay/metal_cpp.bzl | 23 + ...1-Modify-Sort-And-Comparison-Kernels.patch | 1132 ++++ .../mlx/20250518.0-eebe730/source.json | 15 + third_party/modules/mlx/metadata.json | 17 + .../xla/20250317.2-71c67e2/MODULE.bazel | 10 + .../20250317.2-71c67e2/overlay/workspace.bzl | 36 + .../patches/0003-Added-MLX-PjRt-Plugin.patch | 4663 +++++++++++++++++ .../xla/20250317.2-71c67e2/source.json | 29 +- zml/context.zig | 2 +- zml/platform.zig | 1 + 19 files changed, 6608 insertions(+), 15 deletions(-) create mode 100644 runtimes/mlx/BUILD.bazel create mode 100644 runtimes/mlx/mlx.zig create mode 100644 third_party/modules/mlx/20250518.0-eebe730/MODULE.bazel create mode 100644 third_party/modules/mlx/20250518.0-eebe730/overlay/BUILD.bazel create mode 120000 third_party/modules/mlx/20250518.0-eebe730/overlay/MODULE.bazel create mode 100644 third_party/modules/mlx/20250518.0-eebe730/overlay/jit.bzl create mode 100644 third_party/modules/mlx/20250518.0-eebe730/overlay/metal_cpp.bzl create mode 100644 third_party/modules/mlx/20250518.0-eebe730/patches/0001-Modify-Sort-And-Comparison-Kernels.patch create mode 100644 third_party/modules/mlx/20250518.0-eebe730/source.json create mode 100644 third_party/modules/mlx/metadata.json create mode 100644 third_party/modules/xla/20250317.2-71c67e2/patches/0003-Added-MLX-PjRt-Plugin.patch diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 3132fc11..98406392 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -51,6 +51,7 @@ "https://bazel-registry.zml.ai/modules/buildifier_prebuilt/7.3.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/buildifier_prebuilt/8.0.3/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/buildozer/7.1.2/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/fmt/11.2.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/google_benchmark/1.8.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/googletest/1.11.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/googletest/1.14.0.bcr.1/MODULE.bazel": "not found", @@ -83,6 +84,7 @@ "https://bazel-registry.zml.ai/modules/protobuf/3.19.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/pybind11_bazel/2.11.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/pybind11_bazel/2.12.0/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/range-v3/0.12.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/re2/2023-09-01/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/re2/2024-07-02.bcr.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/re2/2024-07-02/MODULE.bazel": "not found", @@ -236,6 +238,8 @@ "https://bcr.bazel.build/modules/buildifier_prebuilt/8.0.3/source.json": "68ee16210562f325ff961aabeb4d46ed10e6638b5e842cd48d8d83dea220dfbd", "https://bcr.bazel.build/modules/buildozer/7.1.2/MODULE.bazel": "2e8dd40ede9c454042645fd8d8d0cd1527966aa5c919de86661e62953cd73d84", "https://bcr.bazel.build/modules/buildozer/7.1.2/source.json": "c9028a501d2db85793a6996205c8de120944f50a0d570438fcae0457a5f9d1f8", + "https://bcr.bazel.build/modules/fmt/11.2.0/MODULE.bazel": "bb2cef47f22163e23aa39ff7812cfe8cec85adb6337493fe38d1fa039133ca5c", + "https://bcr.bazel.build/modules/fmt/11.2.0/source.json": "a2c3846c868dbb75e3793e14c7bb9afbb0e34d3967c9a22b335df71eb2ff1481", "https://bcr.bazel.build/modules/google_benchmark/1.8.2/MODULE.bazel": "a70cf1bba851000ba93b58ae2f6d76490a9feb74192e57ab8e8ff13c34ec50cb", "https://bcr.bazel.build/modules/googletest/1.11.0/MODULE.bazel": "3a83f095183f66345ca86aa13c58b59f9f94a2f81999c093d4eeaa2d262d12f4", "https://bcr.bazel.build/modules/googletest/1.14.0.bcr.1/MODULE.bazel": "22c31a561553727960057361aa33bf20fb2e98584bc4fec007906e27053f80c6", @@ -277,6 +281,8 @@ "https://bcr.bazel.build/modules/pybind11_bazel/2.11.1/MODULE.bazel": "88af1c246226d87e65be78ed49ecd1e6f5e98648558c14ce99176da041dc378e", "https://bcr.bazel.build/modules/pybind11_bazel/2.12.0/MODULE.bazel": "e6f4c20442eaa7c90d7190d8dc539d0ab422f95c65a57cc59562170c58ae3d34", "https://bcr.bazel.build/modules/pybind11_bazel/2.12.0/source.json": "6900fdc8a9e95866b8c0d4ad4aba4d4236317b5c1cd04c502df3f0d33afed680", + "https://bcr.bazel.build/modules/range-v3/0.12.0/MODULE.bazel": "46ee6c1fcb121df6e6345b3898313fef3a173d74c87db2e6196276200daadcba", + "https://bcr.bazel.build/modules/range-v3/0.12.0/source.json": "09f0087034f1b93202d9d43b8b772a709329e45eab9c2ba92ec60525b19c28d2", "https://bcr.bazel.build/modules/re2/2023-09-01/MODULE.bazel": "cb3d511531b16cfc78a225a9e2136007a48cf8a677e4264baeab57fe78a80206", "https://bcr.bazel.build/modules/re2/2024-07-02.bcr.1/MODULE.bazel": "b4963dda9b31080be1905ef085ecd7dd6cd47c05c79b9cdf83ade83ab2ab271a", "https://bcr.bazel.build/modules/re2/2024-07-02.bcr.1/source.json": "2ff292be6ef3340325ce8a045ecc326e92cbfab47c7cbab4bd85d28971b97ac4", diff --git a/runtimes/BUILD.bazel b/runtimes/BUILD.bazel index d0a1bcdb..ab0792ce 100644 --- a/runtimes/BUILD.bazel +++ b/runtimes/BUILD.bazel @@ -6,6 +6,7 @@ load("@rules_zig//zig:defs.bzl", "zig_library") RUNTIMES = { "cpu": True, "cuda": False, + "mlx": False, "rocm": False, "tpu": False, "neuron": False, @@ -48,6 +49,9 @@ filegroup( }) + select({ ":cuda.enabled": ["//runtimes/cuda:layers"], "//conditions:default": [], + }) + select({ + ":mlx.enabled": ["//runtimes/mlx:layers"], + "//conditions:default": [], }) + select({ ":neuron.enabled": ["//runtimes/neuron:layers"], "//conditions:default": [], diff --git a/runtimes/mlx/BUILD.bazel b/runtimes/mlx/BUILD.bazel new file mode 100644 index 00000000..f54f21c4 --- /dev/null +++ b/runtimes/mlx/BUILD.bazel @@ -0,0 +1,36 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +cc_import( + name = "libpjrt_mlx_bindings", + shared_library = "@xla//xla/pjrt/plugin/mlx:stablehlo_mlx_plugin.so", + visibility = ["//visibility:public"], +) + +cc_library( + name = "empty", +) + +cc_library( + name = "libpjrt_mlx", + defines = ["ZML_RUNTIME_MLX"], + deps = [":libpjrt_mlx_bindings"], +) + +zig_library( + name = "mlx", + import_name = "runtimes/mlx", + main = "mlx.zig", + visibility = ["//visibility:public"], + deps = [ + "//pjrt", + ] + select({ + "//runtimes:mlx.enabled": [":libpjrt_mlx"], + "//conditions:default": [":empty"], + }), +) + +filegroup( + name = "layers", + srcs = [], + visibility = ["//visibility:public"], +) diff --git a/runtimes/mlx/mlx.zig b/runtimes/mlx/mlx.zig new file mode 100644 index 00000000..1af0cffa --- /dev/null +++ b/runtimes/mlx/mlx.zig @@ -0,0 +1,18 @@ +const builtin = @import("builtin"); +const asynk = @import("async"); +const pjrt = @import("pjrt"); +const c = @import("c"); + +pub fn isEnabled() bool { + return @hasDecl(c, "ZML_RUNTIME_MLX"); +} + +pub fn load() !*const pjrt.Api { + if (comptime !isEnabled()) { + return error.Unavailable; + } + if (comptime builtin.os.tag != .macos) { + return error.Unavailable; + } + return pjrt.Api.loadFrom("stablehlo_mlx_plugin.so"); +} diff --git a/runtimes/runtimes.zig b/runtimes/runtimes.zig index 7319da76..d9496f0d 100644 --- a/runtimes/runtimes.zig +++ b/runtimes/runtimes.zig @@ -1,6 +1,7 @@ const pjrt = @import("pjrt"); const cpu = @import("runtimes/cpu"); const cuda = @import("runtimes/cuda"); +const mlx = @import("runtimes/mlx"); const rocm = @import("runtimes/rocm"); const tpu = @import("runtimes/tpu"); const neuron = @import("runtimes/neuron"); @@ -8,6 +9,7 @@ const neuron = @import("runtimes/neuron"); pub const Platform = enum { cpu, cuda, + mlx, rocm, tpu, neuron, @@ -17,6 +19,7 @@ pub fn load(tag: Platform) !*const pjrt.Api { return switch (tag) { .cpu => try cpu.load(), .cuda => try cuda.load(), + .mlx => try mlx.load(), .rocm => try rocm.load(), .tpu => try tpu.load(), .neuron => try neuron.load(), @@ -27,6 +30,7 @@ pub fn isEnabled(tag: Platform) bool { return switch (tag) { .cpu => cpu.isEnabled(), .cuda => cuda.isEnabled(), + .mlx => mlx.isEnabled(), .rocm => rocm.isEnabled(), .tpu => tpu.isEnabled(), .neuron => neuron.isEnabled(), diff --git a/third_party/modules/mlx/20250518.0-eebe730/MODULE.bazel b/third_party/modules/mlx/20250518.0-eebe730/MODULE.bazel new file mode 100644 index 00000000..afd89f51 --- /dev/null +++ b/third_party/modules/mlx/20250518.0-eebe730/MODULE.bazel @@ -0,0 +1,11 @@ +module( + name = "mlx", + version = "20250518.0-eebe730", + compatibility_level = 1, +) + +bazel_dep(name = "rules_apple", version = "3.17.0", repo_name = "build_bazel_rules_apple") +bazel_dep(name = "fmt", version = "11.2.0") +bazel_dep(name = "range-v3", version = "0.12.0") +metal_cpp = use_extension("//:metal_cpp.bzl", "metal_cpp") +use_repo(metal_cpp, "metal_cpp") diff --git a/third_party/modules/mlx/20250518.0-eebe730/overlay/BUILD.bazel b/third_party/modules/mlx/20250518.0-eebe730/overlay/BUILD.bazel new file mode 100644 index 00000000..5a664afe --- /dev/null +++ b/third_party/modules/mlx/20250518.0-eebe730/overlay/BUILD.bazel @@ -0,0 +1,592 @@ +load("@build_bazel_rules_apple//apple:resources.bzl", "apple_metal_library") +load("//:jit.bzl", "make_jit_source") + +filegroup( + name = "mlx_h", + srcs = glob(["mlx/*.h"], ["mlx/mlx.h"]), +) + +# Contains `mlx/types/*.h` headers +cc_library( + name = "types", + hdrs = glob(["mlx/types/*.h"]), +) + +cc_library( + name = "pocketfft", + hdrs = [ + "mlx/3rdparty/pocketfft.h", + ], +) + +# mlx/distributed/ring/CMakeLists.txt +cc_library( + name = "ring", + srcs = [ + "mlx/distributed/ring/no_ring.cpp", + + # Other sources (maybe reorganize dependencies?) + ":mlx_h", + "mlx/distributed/distributed.h", + ], + hdrs = [ + "mlx/distributed/ring/ring.h", + ], + deps = [ + ":types" + ], +) + +# mlx/distributed/mpi/CMakeLists.txt +cc_library( + name = "mpi", + srcs = [ + "mlx/distributed/mpi/no_mpi.cpp", + # Other sources (maybe reorganize dependencies?) + "mlx/distributed/distributed.h", + ":mlx_h", + ], + hdrs = [ + "mlx/distributed/mpi/mpi.h", + ], + deps = [ + ":types", + ] +) + +# mlx/distributed/CMakeLists.txt +cc_library( + name = "distributed", + srcs = [ + "mlx/distributed/primitives.cpp", + "mlx/distributed/ops.cpp", + "mlx/distributed/distributed.cpp", + + # Other sources (maybe reorganize dependencies?) + ":mlx_h", + ], + hdrs = [ + "mlx/distributed/distributed_impl.h", + "mlx/distributed/distributed.h", + "mlx/distributed/ops.h", + "mlx/distributed/primitives.h", + ], + deps = [ + ":io", + ":mpi", + ":ring", + ":types", + ] +) + +# mlx/io/CMakeLists.txt +cc_library( + name = "io", + srcs = [ + "mlx/io/load.cpp", + "mlx/io/no_safetensors.cpp", + "mlx/io/no_gguf.cpp", + + # Other sources (maybe reorganize dependencies?) + ":mlx_h", + ], + hdrs = [ + "mlx/io/load.h", + ], + deps = [ + ":types", + ], +) + +# mlx/backend/common/CMakeLists.txt +cc_library( + name = "backend_common", + srcs = [ + "mlx/backend/common/broadcasting.cpp", + "mlx/backend/common/compiled.cpp", + "mlx/backend/common/common.cpp", + "mlx/backend/common/load.cpp", + "mlx/backend/common/reduce.cpp", + "mlx/backend/common/slicing.cpp", + "mlx/backend/common/utils.cpp", + # Other sources (maybe reorganize dependencies?) + "mlx/backend/gpu/eval.h", + ":mlx_h", + ], + hdrs = [ + "mlx/backend/common/binary.h", + "mlx/backend/common/broadcasting.h", + "mlx/backend/common/compiled.h", + "mlx/backend/common/copy.h", + "mlx/backend/common/hadamard.h", + "mlx/backend/common/reduce.h", + "mlx/backend/common/slicing.h", + "mlx/backend/common/ternary.h", + "mlx/backend/common/utils.h", + ], + deps = [ + + ":io", + ":types", + ], +) + +# mlx/backend/cpu/CMakeLists.txt +genrule( + name = "compiled_preamble", + visibility = ["//visibility:public"], + srcs = ["mlx/backend/cpu/compiled_preamble.h"], + outs = ["mlx/backend/cpu/compiled_preamable.cpp"], + cmd = "bash $(location :mlx/backend/cpu/make_compiled_preamble.sh) $@ $(location mlx/backend/cpu/compiled_preamble.h)", + tools = [":mlx/backend/cpu/make_compiled_preamble.sh"], +) + +# TODO (@cryptodeal): finish bazel config for cpu backend +cc_library( + name = "backend_cpu", + linkopts = [ + "-framework", + "Accelerate", + ], + defines = [ + "MLX_USE_ACCELERATE", + "ACCELERATE_NEW_LAPACK" + ], + srcs = [ + "mlx/backend/cpu/arg_reduce.cpp", + "mlx/backend/cpu/binary.cpp", + "mlx/backend/cpu/conv.cpp", + "mlx/backend/cpu/copy.cpp", + "mlx/backend/cpu/eigh.cpp", + "mlx/backend/cpu/fft.cpp", + "mlx/backend/cpu/hadamard.cpp", + "mlx/backend/cpu/matmul.cpp", + "mlx/backend/cpu/gemms/cblas.cpp", + "mlx/backend/cpu/masked_mm.cpp", + "mlx/backend/cpu/primitives.cpp", + "mlx/backend/cpu/quantized.cpp", + "mlx/backend/cpu/reduce.cpp", + "mlx/backend/cpu/scan.cpp", + "mlx/backend/cpu/select.cpp", + "mlx/backend/cpu/softmax.cpp", + "mlx/backend/cpu/sort.cpp", + "mlx/backend/cpu/threefry.cpp", + "mlx/backend/cpu/indexing.cpp", + "mlx/backend/cpu/luf.cpp", + "mlx/backend/cpu/qrf.cpp", + "mlx/backend/cpu/svd.cpp", + "mlx/backend/cpu/inverse.cpp", + "mlx/backend/cpu/cholesky.cpp", + "mlx/backend/cpu/unary.cpp", + # Build with Accelerate + "mlx/backend/cpu/gemms/bnns.cpp", + # Not Building for IOS + "mlx/backend/cpu/compiled.cpp", + "mlx/backend/cpu/jit_compiler.cpp", + # Additional Header Sources + "mlx/backend/cpu/simd/simd.h", + "mlx/backend/cpu/simd/math.h", + "mlx/backend/cpu/simd/type.h", + "mlx/backend/cpu/simd/base_simd.h", + "mlx/backend/cpu/simd/accelerate_simd.h", + "mlx/backend/cpu/simd/accelerate_fp16_simd.h", + # generated `compiled_preamble.cpp` + ":compiled_preamble", + + # Other sources (maybe reorganize dependencies?) + ":mlx_h", + "mlx/io/load.h", + ], + hdrs = [ + "mlx/backend/cpu/arange.h", + "mlx/backend/cpu/binary_ops.h", + "mlx/backend/cpu/binary_two.h", + "mlx/backend/cpu/binary.h", + "mlx/backend/cpu/compiled_preamble.h", + "mlx/backend/cpu/copy.h", + "mlx/backend/cpu/gemm.h", + "mlx/backend/cpu/jit_compiler.h", + "mlx/backend/cpu/lapack.h", + "mlx/backend/cpu/slicing.h", + "mlx/backend/cpu/ternary.h", + "mlx/backend/cpu/threefry.h", + "mlx/backend/cpu/unary_ops.h", + "mlx/backend/cpu/unary.h", + ], + deps = [ + "@fmt", + ":backend_common", + ":pocketfft", + ":types", + ], +) + +# mlx/backend/no_cpu/CMakeLists.txt +cc_library( + name = "backend_no_cpu", + srcs = [ + "mlx/backend/no_cpu/available.cpp", + "mlx/backend/no_cpu/primitives.cpp", + "mlx/backend/cpu/eval.cpp", + "mlx/backend/cpu/encoder.cpp", + "mlx/backend/no_cpu/compiled.cpp", + # Other sources (maybe reorganize dependencies?) + "mlx/backend/cpu/available.h", + "mlx/backend/cpu/encoder.h", + "mlx/backend/cpu/eval.h", + "mlx/backend/gpu/eval.h", + ":mlx_h", + ], + deps = [ + ":distributed", + ":io", + # ":backend_common", + ":types", + ], +) + +# mlx/backend/metal/CMakeLists.txt +make_jit_source( + name = "utils", + deps = [ + "mlx/backend/metal/kernels/jit/bf16.h", + "mlx/backend/metal/kernels/metal_3_0/bf16.h", + "mlx/backend/metal/kernels/metal_3_1/bf16.h", + "mlx/backend/metal/kernels/bf16_math.h", + "mlx/backend/metal/kernels/complex.h", + "mlx/backend/metal/kernels/defines.h", + ], +) + +make_jit_source( + name = "unary_ops", + deps = [ + "mlx/backend/metal/kernels/erf.h", + "mlx/backend/metal/kernels/expm1f.h", + ], +) + +make_jit_source(name = "binary_ops") + +make_jit_source(name = "ternary_ops") + +make_jit_source( + name = "reduce_utils", + deps = [ + "mlx/backend/metal/kernels/atomic.h", + "mlx/backend/metal/kernels/reduction/ops.h" + ], +) + +make_jit_source( + name = "scatter", + deps = [ + "mlx/backend/metal/kernels/indexing.h", + ], +) + +make_jit_source( + name = "gather", + deps = [ + "mlx/backend/metal/kernels/indexing.h", + ], +) + +make_jit_source(name = "gather_axis") + +make_jit_source(name = "scatter_axis") + +make_jit_source(name = "hadamard") + +cc_library( + name = "backend_metal", + linkopts = [ + "-framework Metal", + "-framework Foundation", + "-framework QuartzCore", + ], + # defines = ["MLX_METAL_DEBUG"], + srcs = [ + "mlx/backend/metal/allocator.cpp", + "mlx/backend/metal/binary.cpp", + "mlx/backend/metal/compiled.cpp", + "mlx/backend/metal/conv.cpp", + "mlx/backend/metal/copy.cpp", + "mlx/backend/metal/custom_kernel.cpp", + "mlx/backend/metal/distributed.cpp", + "mlx/backend/metal/device.cpp", + "mlx/backend/metal/event.cpp", + "mlx/backend/metal/eval.cpp", + "mlx/backend/metal/fence.cpp", + "mlx/backend/metal/fft.cpp", + "mlx/backend/metal/hadamard.cpp", + "mlx/backend/metal/indexing.cpp", + "mlx/backend/metal/logsumexp.cpp", + "mlx/backend/metal/matmul.cpp", + "mlx/backend/metal/scaled_dot_product_attention.cpp", + "mlx/backend/metal/metal.cpp", + "mlx/backend/metal/primitives.cpp", + "mlx/backend/metal/quantized.cpp", + "mlx/backend/metal/normalization.cpp", + "mlx/backend/metal/rope.cpp", + "mlx/backend/metal/scan.cpp", + "mlx/backend/metal/slicing.cpp", + "mlx/backend/metal/softmax.cpp", + "mlx/backend/metal/sort.cpp", + "mlx/backend/metal/reduce.cpp", + "mlx/backend/metal/ternary.cpp", + "mlx/backend/metal/unary.cpp", + "mlx/backend/metal/resident.cpp", + "mlx/backend/metal/utils.cpp", + "mlx/backend/metal/nojit_kernels.cpp", + # Additional Header Sources + "mlx/backend/metal/jit/indexing.h", + "mlx/backend/metal/kernels/defines.h", + "mlx/backend/metal/kernels/steel/conv/params.h", + "mlx/backend/metal/kernels/steel/gemm/params.h", + "mlx/backend/metal/kernels/steel/attn/params.h", + "mlx/backend/metal/jit/includes.h", + # Generate METAL_PATH define + ":generate_metal_path", + # Generated sources + ":utils", + ":unary_ops", + ":binary_ops", + ":ternary_ops", + ":reduce_utils", + ":scatter", + ":gather", + ":gather_axis", + ":scatter_axis", + ":hadamard", + + # Other sources (maybe reorganize dependencies?) + ":mlx_h", + ], + hdrs = [ + "mlx/backend/metal/allocator.h", + "mlx/backend/metal/binary.h", + "mlx/backend/metal/device.h", + "mlx/backend/metal/kernels.h", + "mlx/backend/metal/matmul.h", + "mlx/backend/metal/metal.h", + "mlx/backend/metal/reduce.h", + "mlx/backend/metal/resident.h", + "mlx/backend/metal/ternary.h", + "mlx/backend/metal/unary.h", + "mlx/backend/metal/utils.h", + ], + deps = [ + "@metal_cpp//:metal_cpp", + "@fmt", + ":backend_common", + ":backend_gpu", + ":distributed", + ":pocketfft", + ], +) + +genrule( + name = "generate_metal_path", + outs = [ + "mlx/backend/metal/metal_path.h", + ], + cmd = "echo \"#define METAL_PATH \\\"`realpath $(location //:mlx_metallib)`\\\"\" > $@", + tools = [ + "//:mlx_metallib", + ], +) + +# TODO (@cryptodeal): find a better solution than `copts` includes +apple_metal_library( + name = "mlx_metallib", + visibility = ["//visibility:public"], + copts = [ + "-target", + "air64-apple-macosx15.4", + "-Iexternal/mlx+", + "-Iexternal/mlx+/mlx/backend/metal/kernels/metal_3_1", + # "-I.", + # "-Imlx/backend/metal/kernels/metal_3_1", + "-Wall", + "-Wextra", + "-fno-fast-math", + "-Wno-c++17-extensions", + # "-gline-tables-only", + # "-frecord-sources", + ], + srcs = [ + "mlx/backend/metal/kernels/arg_reduce.metal", + "mlx/backend/metal/kernels/conv.metal", + "mlx/backend/metal/kernels/gemv.metal", + "mlx/backend/metal/kernels/layer_norm.metal", + "mlx/backend/metal/kernels/random.metal", + "mlx/backend/metal/kernels/rms_norm.metal", + "mlx/backend/metal/kernels/rope.metal", + "mlx/backend/metal/kernels/scaled_dot_product_attention.metal", + "mlx/backend/metal/kernels/fence.metal", + "mlx/backend/metal/kernels/steel/attn/kernels/steel_attention.metal", + # Disable JIT kernels (for default ops) + "mlx/backend/metal/kernels/arange.metal", + "mlx/backend/metal/kernels/binary.metal", + "mlx/backend/metal/kernels/binary_two.metal", + "mlx/backend/metal/kernels/copy.metal", + "mlx/backend/metal/kernels/fft.metal", + "mlx/backend/metal/kernels/reduce.metal", + "mlx/backend/metal/kernels/quantized.metal", + "mlx/backend/metal/kernels/scan.metal", + "mlx/backend/metal/kernels/softmax.metal", + "mlx/backend/metal/kernels/sort.metal", + "mlx/backend/metal/kernels/ternary.metal", + "mlx/backend/metal/kernels/unary.metal", + "mlx/backend/metal/kernels/steel/conv/kernels/steel_conv.metal", + "mlx/backend/metal/kernels/steel/conv/kernels/steel_conv_general.metal", + "mlx/backend/metal/kernels/steel/gemm/kernels/steel_gemm_fused.metal", + "mlx/backend/metal/kernels/steel/gemm/kernels/steel_gemm_masked.metal", + "mlx/backend/metal/kernels/steel/gemm/kernels/steel_gemm_splitk.metal", + "mlx/backend/metal/kernels/gemv_masked.metal", + ], + hdrs = [ + "mlx/backend/metal/kernels/metal_3_1/bf16.h", + "mlx/backend/metal/kernels/metal_3_0/bf16.h", + "mlx/backend/metal/kernels/bf16_math.h", + "mlx/backend/metal/kernels/complex.h", + "mlx/backend/metal/kernels/defines.h", + "mlx/backend/metal/kernels/erf.h", + "mlx/backend/metal/kernels/expm1f.h", + "mlx/backend/metal/kernels/utils.h", + # `conv` headers + "mlx/backend/metal/kernels/steel/conv/params.h", + # `scaled_dot_product_attention` headers + "mlx/backend/metal/kernels/sdpa_vector.h", + # Steel Headers + "mlx/backend/metal/kernels/steel/defines.h", + "mlx/backend/metal/kernels/steel/utils.h", + "mlx/backend/metal/kernels/steel/conv/conv.h", + "mlx/backend/metal/kernels/steel/conv/loader.h", + "mlx/backend/metal/kernels/steel/conv/loaders/loader_channel_l.h", + "mlx/backend/metal/kernels/steel/conv/loaders/loader_channel_n.h", + "mlx/backend/metal/kernels/steel/conv/loaders/loader_general.h", + "mlx/backend/metal/kernels/steel/conv/kernels/steel_conv.h", + "mlx/backend/metal/kernels/steel/conv/kernels/steel_conv_general.h", + "mlx/backend/metal/kernels/steel/gemm/gemm.h", + "mlx/backend/metal/kernels/steel/gemm/mma.h", + "mlx/backend/metal/kernels/steel/gemm/loader.h", + "mlx/backend/metal/kernels/steel/gemm/params.h", + "mlx/backend/metal/kernels/steel/gemm/transforms.h", + "mlx/backend/metal/kernels/steel/gemm/kernels/steel_gemm_fused.h", + "mlx/backend/metal/kernels/steel/gemm/kernels/steel_gemm_masked.h", + "mlx/backend/metal/kernels/steel/gemm/kernels/steel_gemm_splitk.h", + "mlx/backend/metal/kernels/steel/utils/type_traits.h", + "mlx/backend/metal/kernels/steel/utils/integral_constant.h", + # Steel Attn Headers (not repeating previously listed) + "mlx/backend/metal/kernels/steel/attn/attn.h", + "mlx/backend/metal/kernels/steel/attn/loader.h", + "mlx/backend/metal/kernels/steel/attn/mma.h", + "mlx/backend/metal/kernels/steel/attn/params.h", + "mlx/backend/metal/kernels/steel/attn/transforms.h", + "mlx/backend/metal/kernels/steel/attn/kernels/steel_attention.h", + # `arange` headers + "mlx/backend/metal/kernels/arange.h", + # `binary` headers + "mlx/backend/metal/kernels/binary.h", + "mlx/backend/metal/kernels/binary_ops.h", + # `binary_two` headers + "mlx/backend/metal/kernels/binary_two.h", + # `copy` headers + "mlx/backend/metal/kernels/copy.h", + # `fft` headers + "mlx/backend/metal/kernels/fft.h", + "mlx/backend/metal/kernels/fft/radix.h", + "mlx/backend/metal/kernels/fft/readwrite.h", + # `reduce` headers + "mlx/backend/metal/kernels/reduce.h", + "mlx/backend/metal/kernels/atomic.h", + "mlx/backend/metal/kernels/reduction/ops.h", + "mlx/backend/metal/kernels/reduction/reduce_init.h", + "mlx/backend/metal/kernels/reduction/reduce_all.h", + "mlx/backend/metal/kernels/reduction/reduce_col.h", + "mlx/backend/metal/kernels/reduction/reduce_row.h", + # `quantized` headers + "mlx/backend/metal/kernels/quantized.h", + # `scan` headers + "mlx/backend/metal/kernels/scan.h", + # `softmax` headers + "mlx/backend/metal/kernels/softmax.h", + # `sort` headers + "mlx/backend/metal/kernels/sort.h", + # `ternary` headers + "mlx/backend/metal/kernels/ternary.h", + "mlx/backend/metal/kernels/ternary_ops.h", + # `unary` headers + "mlx/backend/metal/kernels/unary.h", + "mlx/backend/metal/kernels/unary_ops.h", + # `gemv` headers + "mlx/backend/metal/kernels/gemv_masked.h", + ], + out = "mlx.metallib", +) + +cc_library( + name = "backend_gpu", + srcs = [ + "mlx/backend/gpu/copy.cpp", + "mlx/backend/gpu/primitives.cpp", + "mlx/backend/gpu/slicing.cpp", + # Other sources (maybe reorganize dependencies?) + ":mlx_h", + ], + hdrs = [ + "mlx/backend/gpu/available.h", + "mlx/backend/gpu/copy.h", + "mlx/backend/gpu/eval.h", + "mlx/backend/gpu/slicing.h", + ], + deps = [ + ":backend_common", + ], +) + +cc_library( + name = "mlx", + visibility = ["//visibility:public"], + data = [ + ":mlx_metallib", + ], + srcs = [ + "mlx/allocator.cpp", + "mlx/array.cpp", + "mlx/compile.cpp", + "mlx/device.cpp", + "mlx/dtype.cpp", + "mlx/dtype_utils.cpp", + "mlx/export.cpp", + "mlx/einsum.cpp", + "mlx/fast.cpp", + "mlx/fft.cpp", + "mlx/ops.cpp", + "mlx/graph_utils.cpp", + "mlx/primitives.cpp", + "mlx/random.cpp", + "mlx/scheduler.cpp", + "mlx/transforms.cpp", + "mlx/utils.cpp", + "mlx/linalg.cpp", + "mlx/backend/metal/metal.h", + ":mlx_h", + ], + hdrs = [ + "mlx/mlx.h", + ], + deps = [ + ":io", + ":distributed", + ":backend_common", + ":backend_metal", + ":backend_gpu", + ":backend_no_cpu", + # TODO (@cryptodeal): enable cpu backend once changes to `sort`, and + # comparison ops have been made for parity with metal backend changes. + # ":backend_cpu", + ], +) diff --git a/third_party/modules/mlx/20250518.0-eebe730/overlay/MODULE.bazel b/third_party/modules/mlx/20250518.0-eebe730/overlay/MODULE.bazel new file mode 120000 index 00000000..9b599e3a --- /dev/null +++ b/third_party/modules/mlx/20250518.0-eebe730/overlay/MODULE.bazel @@ -0,0 +1 @@ +../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/mlx/20250518.0-eebe730/overlay/jit.bzl b/third_party/modules/mlx/20250518.0-eebe730/overlay/jit.bzl new file mode 100644 index 00000000..b8ba8aa0 --- /dev/null +++ b/third_party/modules/mlx/20250518.0-eebe730/overlay/jit.bzl @@ -0,0 +1,23 @@ +def make_jit_source(name, deps = [], **kwargs): + """ + Bazel equivalent of the CMake make_jit_source function. + Generates a C++ source file from a Metal header file by running a script. + + Args: + name: The Metal header file to process. + deps: Additional dependencies required for processing. + kwargs: Additional arguments to pass to the native.genrule function. + """ + src_name = name.split("/")[-1] # Extract the filename + output_file = "mlx/backend/metal/jit/{}.cpp".format(src_name) + + native.genrule( + name = name, + srcs = ["mlx/backend/metal/kernels/{}.h".format(src_name)] + deps, + outs = [output_file], + cmd = ( + "bash $(location :mlx/backend/metal/make_compiled_preamble.sh) $@ $(location mlx/backend/metal/kernels/{}.h)" + ).format(name), + tools = [":mlx/backend/metal/make_compiled_preamble.sh"], + **kwargs, + ) diff --git a/third_party/modules/mlx/20250518.0-eebe730/overlay/metal_cpp.bzl b/third_party/modules/mlx/20250518.0-eebe730/overlay/metal_cpp.bzl new file mode 100644 index 00000000..64874afe --- /dev/null +++ b/third_party/modules/mlx/20250518.0-eebe730/overlay/metal_cpp.bzl @@ -0,0 +1,23 @@ +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +def _metal_cpp_impl(mctx): + http_archive( + name = "metal_cpp", + build_file_content = """\ +cc_library( + name = "metal_cpp", + hdrs = glob(["metal-cpp/**/*.hpp"]), + strip_include_prefix = "metal-cpp", + visibility = ["//visibility:public"], +) +""", + sha256 = "0433df1e0ab13c2b0becbd78665071e3fa28381e9714a3fce28a497892b8a184", + url = "https://developer.apple.com/metal/cpp/files/metal-cpp_macOS15_iOS18.zip", + ) + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = "all", + root_module_direct_dev_deps = [], + ) +metal_cpp = module_extension( + implementation = _metal_cpp_impl, +) diff --git a/third_party/modules/mlx/20250518.0-eebe730/patches/0001-Modify-Sort-And-Comparison-Kernels.patch b/third_party/modules/mlx/20250518.0-eebe730/patches/0001-Modify-Sort-And-Comparison-Kernels.patch new file mode 100644 index 00000000..dae2293b --- /dev/null +++ b/third_party/modules/mlx/20250518.0-eebe730/patches/0001-Modify-Sort-And-Comparison-Kernels.patch @@ -0,0 +1,1132 @@ +diff --git a/.gitignore b/.gitignore +index 43629548..0b74c60d 100644 +--- a/.gitignore ++++ b/.gitignore +@@ -1,3 +1,8 @@ ++# Bazel symlinks ++bazel-* ++ ++# macOS ++.DS_Store + # Byte-compiled / optimized / DLL files + __pycache__/ + *.py[cod] +diff --git a/mlx/backend/metal/device.cpp b/mlx/backend/metal/device.cpp +index ebc3cc77..7bdca47d 100644 +--- a/mlx/backend/metal/device.cpp ++++ b/mlx/backend/metal/device.cpp +@@ -10,6 +10,7 @@ + + #include "mlx/backend/metal/device.h" + #include "mlx/backend/metal/metal.h" ++#include "mlx/backend/metal/metal_path.h" + #include "mlx/backend/metal/utils.h" + #include "mlx/utils.h" + +diff --git a/mlx/backend/metal/jit_kernels.cpp b/mlx/backend/metal/jit_kernels.cpp +index 5206c9b5..6678e7ac 100644 +--- a/mlx/backend/metal/jit_kernels.cpp ++++ b/mlx/backend/metal/jit_kernels.cpp +@@ -328,7 +328,8 @@ MTL::ComputePipelineState* get_sort_kernel( + const array& in, + const array& out, + int bn, +- int tn) { ++ int tn, ++ const std::string& comparator) { + std::string lib_name = kernel_name.substr(kernel_name.find("_") + 1); + auto lib = d.get_library(lib_name, [&]() { + std::ostringstream kernel_source; +@@ -345,7 +346,8 @@ MTL::ComputePipelineState* get_sort_kernel( + out_type, + bool_string, + bn, +- tn); ++ tn, ++ comparator); + kernel_source << get_template_definition( + "n" + func_string + lib_name, + "block_sort_nc", +@@ -353,7 +355,8 @@ MTL::ComputePipelineState* get_sort_kernel( + out_type, + bool_string, + bn, +- tn); ++ tn, ++ comparator); + } + return kernel_source.str(); + }); +@@ -366,7 +369,8 @@ MTL::ComputePipelineState* get_mb_sort_kernel( + const array& in, + const array& idx, + int bn, +- int tn) { ++ int tn, ++ const std::string& comparator) { + std::string lib_name = kernel_name.substr(kernel_name.find("_") + 1); + auto lib = d.get_library(lib_name, [&]() { + std::ostringstream kernel_source; +@@ -383,7 +387,8 @@ MTL::ComputePipelineState* get_mb_sort_kernel( + get_type_string(idx.dtype()), + "true", + bn, +- tn); ++ tn, ++ comparator); + } + return kernel_source.str(); + }); +diff --git a/mlx/backend/metal/kernels.h b/mlx/backend/metal/kernels.h +index 6d886438..ff2dd14b 100644 +--- a/mlx/backend/metal/kernels.h ++++ b/mlx/backend/metal/kernels.h +@@ -79,7 +79,8 @@ MTL::ComputePipelineState* get_sort_kernel( + const array& in, + const array& out, + int bn, +- int tn); ++ int tn, ++ const std::string& comparator); + + MTL::ComputePipelineState* get_mb_sort_kernel( + metal::Device& d, +@@ -87,7 +88,8 @@ MTL::ComputePipelineState* get_mb_sort_kernel( + const array& in, + const array& idx, + int bn, +- int tn); ++ int tn, ++ const std::string& comparator); + + MTL::ComputePipelineState* get_reduce_init_kernel( + metal::Device& d, +diff --git a/mlx/backend/metal/kernels/binary_ops.h b/mlx/backend/metal/kernels/binary_ops.h +index 4aaf2b4d..2c6e27b1 100644 +--- a/mlx/backend/metal/kernels/binary_ops.h ++++ b/mlx/backend/metal/kernels/binary_ops.h +@@ -37,29 +37,14 @@ struct Divide { + return x / y; + } + }; +- + struct Remainder { + template +- metal::enable_if_t & !metal::is_signed_v, T> +- operator()(T x, T y) { ++ metal::enable_if_t, T> operator()(T x, T y) { + return x % y; + } + template +- metal::enable_if_t & metal::is_signed_v, T> +- operator()(T x, T y) { +- auto r = x % y; +- if (r != 0 && (r < 0 != y < 0)) { +- r += y; +- } +- return r; +- } +- template + metal::enable_if_t, T> operator()(T x, T y) { +- T r = fmod(x, y); +- if (r != 0 && (r < 0 != y < 0)) { +- r += y; +- } +- return r; ++ return fmod(x, y); + } + template <> + complex64_t operator()(complex64_t x, complex64_t y) { +diff --git a/mlx/backend/metal/kernels/complex.h b/mlx/backend/metal/kernels/complex.h +index c88002cb..54fc8d52 100644 +--- a/mlx/backend/metal/kernels/complex.h ++++ b/mlx/backend/metal/kernels/complex.h +@@ -142,11 +142,12 @@ constexpr complex64_t operator/(float a, complex64_t b) { + constexpr complex64_t operator%(complex64_t a, complex64_t b) { + auto real = a.real - (b.real * static_cast(a.real / b.real)); + auto imag = a.imag - (b.imag * static_cast(a.imag / b.imag)); +- if (real != 0 && (real < 0 != b.real < 0)) { +- real += b.real; +- } +- if (imag != 0 && (imag < 0 != b.imag < 0)) { +- imag += b.imag; +- } ++ /* Disable to match `stablehlo` implementation semantics */ ++ // if (real != 0 && (real < 0 != b.real < 0)) { ++ // real += b.real; ++ // } ++ // if (imag != 0 && (imag < 0 != b.imag < 0)) { ++ // imag += b.imag; ++ // } + return {real, imag}; + } +diff --git a/mlx/backend/metal/kernels/sort.h b/mlx/backend/metal/kernels/sort.h +index b067150d..a24e429f 100644 +--- a/mlx/backend/metal/kernels/sort.h ++++ b/mlx/backend/metal/kernels/sort.h +@@ -28,6 +28,15 @@ struct LessThan { + } + }; + ++template ++struct GreaterThan { ++ static constexpr constant T init = Limits::min; ++ ++ METAL_FUNC bool operator()(T a, T b) { ++ return a > b; ++ } ++}; ++ + template < + typename ValT, + typename IdxT, +@@ -216,7 +225,7 @@ template < + bool ARG_SORT, + short BLOCK_THREADS, + short N_PER_THREAD, +- typename CompareOp = LessThan> ++ typename CompareOp> + struct KernelMergeSort { + using ValT = T; + using IdxT = uint; +@@ -278,7 +287,8 @@ template < + typename U, + bool ARG_SORT, + short BLOCK_THREADS, +- short N_PER_THREAD> ++ short N_PER_THREAD, ++ typename CompareOp = LessThan> + [[kernel, max_total_threads_per_threadgroup(BLOCK_THREADS)]] void block_sort( + const device T* inp [[buffer(0)]], + device U* out [[buffer(1)]], +@@ -290,7 +300,7 @@ template < + uint3 tid [[threadgroup_position_in_grid]], + uint3 lid [[thread_position_in_threadgroup]]) { + using sort_kernel = +- KernelMergeSort; ++ KernelMergeSort; + using ValT = typename sort_kernel::ValT; + using IdxT = typename sort_kernel::IdxT; + +@@ -333,7 +343,8 @@ template < + typename U, + bool ARG_SORT, + short BLOCK_THREADS, +- short N_PER_THREAD> ++ short N_PER_THREAD, ++ typename CompareOp> + [[kernel, max_total_threads_per_threadgroup(BLOCK_THREADS)]] void block_sort_nc( + const device T* inp [[buffer(0)]], + device U* out [[buffer(1)]], +@@ -347,7 +358,7 @@ template < + uint3 tid [[threadgroup_position_in_grid]], + uint3 lid [[thread_position_in_threadgroup]]) { + using sort_kernel = +- KernelMergeSort; ++ KernelMergeSort; + using ValT = typename sort_kernel::ValT; + using IdxT = typename sort_kernel::IdxT; + +@@ -394,7 +405,7 @@ template < + bool ARG_SORT, + short BLOCK_THREADS, + short N_PER_THREAD, +- typename CompareOp = LessThan> ++ typename CompareOp> + struct KernelMultiBlockMergeSort { + using block_merge_sort_t = BlockMergeSort< + ValT, +@@ -476,7 +487,8 @@ template < + typename IdxT, + bool ARG_SORT, + short BLOCK_THREADS, +- short N_PER_THREAD> ++ short N_PER_THREAD, ++ typename CompareOp> + [[kernel, max_total_threads_per_threadgroup(BLOCK_THREADS)]] void mb_block_sort( + const device ValT* inp [[buffer(0)]], + device ValT* out_vals [[buffer(1)]], +@@ -493,7 +505,8 @@ template < + IdxT, + ARG_SORT, + BLOCK_THREADS, +- N_PER_THREAD>; ++ N_PER_THREAD, ++ CompareOp>; + + auto block_idx = elem_to_loc(tid.y, nc_shape, nc_strides, nc_dim); + inp += block_idx; +@@ -520,7 +533,8 @@ template < + typename IdxT, + bool ARG_SORT, + short BLOCK_THREADS, +- short N_PER_THREAD> ++ short N_PER_THREAD, ++ typename CompareOp> + [[kernel]] void mb_block_partition( + device IdxT* block_partitions [[buffer(0)]], + const device ValT* dev_vals [[buffer(1)]], +@@ -536,7 +550,8 @@ template < + IdxT, + ARG_SORT, + BLOCK_THREADS, +- N_PER_THREAD>; ++ N_PER_THREAD, ++ CompareOp>; + + block_partitions += tid.y * tgp_dims.x; + dev_vals += tid.y * size_sorted_axis; +@@ -573,7 +588,7 @@ template < + bool ARG_SORT, + short BLOCK_THREADS, + short N_PER_THREAD, +- typename CompareOp = LessThan> ++ typename CompareOp> + [[kernel, max_total_threads_per_threadgroup(BLOCK_THREADS)]] void + mb_block_merge( + const device IdxT* block_partitions [[buffer(0)]], +diff --git a/mlx/backend/metal/kernels/sort.metal b/mlx/backend/metal/kernels/sort.metal +index 7f198d55..f1ec796c 100644 +--- a/mlx/backend/metal/kernels/sort.metal ++++ b/mlx/backend/metal/kernels/sort.metal +@@ -6,24 +6,29 @@ + #include "mlx/backend/metal/kernels/utils.h" + #include "mlx/backend/metal/kernels/sort.h" + ++#define instantiate_comparator(name, type) \ ++ name ++ + #define instantiate_block_sort( \ +- name, itname, itype, otname, otype, arg_sort, bn, tn) \ +- instantiate_kernel("c" #name "_" #itname "_" #otname "_bn" #bn "_tn" #tn, \ +- block_sort, itype, otype, arg_sort, bn, tn) \ +- instantiate_kernel("nc" #name "_" #itname "_" #otname "_bn" #bn "_tn" #tn, \ +- block_sort_nc, itype, otype, arg_sort, bn, tn) ++ name, itname, itype, otname, otype, arg_sort, bn, tn, vcomparator) \ ++ instantiate_kernel("c" #name "_" #itname "_" #otname "_bn" #bn "_tn" #tn "_comp" #vcomparator, \ ++ block_sort, itype, otype, arg_sort, bn, tn, instantiate_comparator(vcomparator, itype)) \ ++ instantiate_kernel("nc" #name "_" #itname "_" #otname "_bn" #bn "_tn" #tn "_comp" #vcomparator, \ ++ block_sort_nc, itype, otype, arg_sort, bn, tn, instantiate_comparator(vcomparator, itype)) + +-#define instantiate_arg_block_sort_base(itname, itype, bn, tn) \ ++#define instantiate_arg_block_sort_base(itname, itype, bn, tn, comparator) \ + instantiate_block_sort( \ +- arg_block_sort, itname, itype, uint32, uint32_t, true, bn, tn) ++ arg_block_sort, itname, itype, uint32, uint32_t, true, bn, tn, comparator) + +-#define instantiate_block_sort_base(itname, itype, bn, tn) \ ++#define instantiate_block_sort_base(itname, itype, bn, tn, comparator) \ + instantiate_block_sort( \ +- _block_sort, itname, itype, itname, itype, false, bn, tn) ++ _block_sort, itname, itype, itname, itype, false, bn, tn, comparator) + + #define instantiate_block_sort_tn(itname, itype, bn) \ +- instantiate_block_sort_base(itname, itype, bn, 4) \ +- instantiate_arg_block_sort_base(itname, itype, bn, 4) ++ instantiate_block_sort_base(itname, itype, bn, 4, LessThan) \ ++ instantiate_block_sort_base(itname, itype, bn, 4, GreaterThan) \ ++ instantiate_arg_block_sort_base(itname, itype, bn, 4, LessThan) \ ++ instantiate_arg_block_sort_base(itname, itype, bn, 4, GreaterThan) + + #define instantiate_block_sort_bn(itname, itype) \ + instantiate_block_sort_tn(itname, itype, 32) \ +@@ -52,16 +57,17 @@ instantiate_block_sort_long(uint64, uint64_t) + instantiate_block_sort_long(int64, int64_t) + + #define instantiate_multi_block_sort( \ +- vtname, vtype, itname, itype, arg_sort, bn, tn) \ +- instantiate_kernel("sort_mbsort_" #vtname "_" #itname "_bn" #bn "_tn" #tn, \ +- mb_block_sort, vtype, itype, arg_sort, bn, tn) \ +- instantiate_kernel("partition_mbsort_" #vtname "_" #itname "_bn" #bn "_tn" #tn, \ +- mb_block_partition, vtype, itype, arg_sort, bn, tn) \ +- instantiate_kernel("merge_mbsort_" #vtname "_" #itname "_bn" #bn "_tn" #tn, \ +- mb_block_merge, vtype, itype, arg_sort, bn, tn) ++ vtname, vtype, itname, itype, arg_sort, bn, tn, vcomparator) \ ++ instantiate_kernel("sort_mbsort_" #vtname "_" #itname "_bn" #bn "_tn" #tn "_comp" #vcomparator, \ ++ mb_block_sort, vtype, itype, arg_sort, bn, tn, instantiate_comparator(vcomparator, vtype)) \ ++ instantiate_kernel("partition_mbsort_" #vtname "_" #itname "_bn" #bn "_tn" #tn "_comp" #vcomparator, \ ++ mb_block_partition, vtype, itype, arg_sort, bn, tn, instantiate_comparator(vcomparator, vtype)) \ ++ instantiate_kernel("merge_mbsort_" #vtname "_" #itname "_bn" #bn "_tn" #tn "_comp" #vcomparator, \ ++ mb_block_merge, vtype, itype, arg_sort, bn, tn, instantiate_comparator(vcomparator, vtype)) + + #define instantiate_multi_block_sort_base(vtname, vtype) \ +- instantiate_multi_block_sort(vtname, vtype, uint32, uint32_t, true, 512, 4) ++ instantiate_multi_block_sort(vtname, vtype, uint32, uint32_t, true, 512, 4, LessThan) \ ++ instantiate_multi_block_sort(vtname, vtype, uint32, uint32_t, true, 512, 4, GreaterThan) + + instantiate_multi_block_sort_base(uint8, uint8_t) + instantiate_multi_block_sort_base(uint16, uint16_t) +@@ -74,7 +80,8 @@ instantiate_multi_block_sort_base(float32, float) + instantiate_multi_block_sort_base(bfloat16, bfloat16_t) + + #define instantiate_multi_block_sort_long(vtname, vtype) \ +- instantiate_multi_block_sort(vtname, vtype, uint32, uint32_t, true, 256, 4) ++ instantiate_multi_block_sort(vtname, vtype, uint32, uint32_t, true, 256, 4, LessThan) \ ++ instantiate_multi_block_sort(vtname, vtype, uint32, uint32_t, true, 256, 4, GreaterThan) + + instantiate_multi_block_sort_long(uint64, uint64_t) + instantiate_multi_block_sort_long(int64, int64_t) // clang-format on +diff --git a/mlx/backend/metal/make_compiled_preamble.sh b/mlx/backend/metal/make_compiled_preamble.sh +index 425cd8d7..a48d0080 100644 +--- a/mlx/backend/metal/make_compiled_preamble.sh ++++ b/mlx/backend/metal/make_compiled_preamble.sh +@@ -5,18 +5,21 @@ + # + # Copyright © 2023-24 Apple Inc. + +-OUTPUT_DIR=$1 +-CC=$2 +-SRC_DIR=$3 +-SRC_FILE=$4 +-CFLAGS=$5 +-SRC_NAME=$(basename -- "${SRC_FILE}") +-JIT_INCLUDES=${SRC_DIR}/mlx/backend/metal/kernels/jit +-INPUT_FILE=${SRC_DIR}/mlx/backend/metal/kernels/${SRC_FILE}.h +-OUTPUT_FILE=${OUTPUT_DIR}/${SRC_NAME}.cpp ++OUTPUT_FILE=$1 ++INPUT_FILE=$2 ++CFLAGS=$3 + +-mkdir -p "$OUTPUT_DIR" +-CONTENT=$($CC -I"$SRC_DIR" -I"$JIT_INCLUDES" -DMLX_METAL_JIT -E -P "$INPUT_FILE" $CFLAGS 2>/dev/null) ++KERNEL_DIR=$(dirname $(realpath $INPUT_FILE)) ++# echo "KERNEL_DIR: $KERNEL_DIR" ++SRC_DIR=$(echo "$KERNEL_DIR" | cut -d'/' -f-$(($(echo "$KERNEL_DIR" | awk -F'/' '{print NF}')-4))) ++# echo "SRC_DIR: $SRC_DIR" ++JIT_INCLUDES="$KERNEL_DIR/jit" ++# echo "JIT_INCLUDES: $JIT_INCLUDES" ++ ++SRC_BASENAME=$(basename -- "${INPUT_FILE}") ++SRC_NAME="${SRC_BASENAME%.*}" ++ ++CONTENT=$(clang -I"$SRC_DIR" -I"$JIT_INCLUDES" -DMLX_METAL_JIT -E -P "$INPUT_FILE" $CFLAGS 2>/dev/null) + + cat << EOF > "$OUTPUT_FILE" + namespace mlx::core::metal { +diff --git a/mlx/backend/metal/nojit_kernels.cpp b/mlx/backend/metal/nojit_kernels.cpp +index 8da14797..1098f918 100644 +--- a/mlx/backend/metal/nojit_kernels.cpp ++++ b/mlx/backend/metal/nojit_kernels.cpp +@@ -96,7 +96,8 @@ MTL::ComputePipelineState* get_sort_kernel( + const array&, + const array&, + int, +- int) { ++ int, ++ const std::string&) { + return d.get_kernel(kernel_name); + } + +@@ -106,7 +107,8 @@ MTL::ComputePipelineState* get_mb_sort_kernel( + const array&, + const array&, + int, +- int) { ++ int, ++ const std::string&) { + return d.get_kernel(kernel_name); + } + +diff --git a/mlx/backend/metal/sort.cpp b/mlx/backend/metal/sort.cpp +index 3c84022f..bca888f6 100644 +--- a/mlx/backend/metal/sort.cpp ++++ b/mlx/backend/metal/sort.cpp +@@ -6,7 +6,9 @@ + #include "mlx/backend/metal/device.h" + #include "mlx/backend/metal/kernels.h" + #include "mlx/backend/metal/utils.h" ++#include "mlx/dtype_utils.h", + #include "mlx/primitives.h" ++#include "mlx/sort.h" + + namespace mlx::core { + +@@ -20,7 +22,8 @@ void single_block_sort( + int axis, + int bn, + int tn, +- bool argsort) { ++ bool argsort, ++ ComparatorType comparator_) { + // Prepare shapes + int n_rows = in.size() / in.shape(axis); + +@@ -58,9 +61,26 @@ void single_block_sort( + kname << "arg"; + } + ++ std::string comparator; ++ switch (comparator_) { ++ case ComparatorType::LessThan: ++ comparator = "LessThan"; ++ break; ++ case ComparatorType::GreaterThan: ++ comparator = "GreaterThan"; ++ break; ++ } ++ + kname << "_block_sort_" << type_to_name(in) << "_" << type_to_name(out) +- << "_bn" << bn << "_tn" << tn; +- auto kernel = get_sort_kernel(d, kname.str(), in, out, bn, tn); ++ << "_bn" << bn << "_tn" << tn << "_comp" << comparator; ++ auto kernel = get_sort_kernel( ++ d, ++ kname.str(), ++ in, ++ out, ++ bn, ++ tn, ++ comparator + "<" + dtype_to_string(in.dtype()) + ">"); + + // Prepare command encoder + auto& compute_encoder = d.get_command_encoder(s.index); +@@ -120,7 +140,8 @@ void multi_block_sort( + int bn, + int tn, + int n_blocks, +- bool argsort) { ++ bool argsort, ++ ComparatorType comparator_) { + // Prepare shapes + int n_rows = in.size() / in.shape(axis); + +@@ -159,6 +180,16 @@ void multi_block_sort( + std::vector copies = { + dev_vals_0, dev_vals_1, dev_idxs_0, dev_idxs_1, block_partitions}; + ++ std::string comparator; ++ switch (comparator_) { ++ case ComparatorType::LessThan: ++ comparator = "LessThan"; ++ break; ++ case ComparatorType::GreaterThan: ++ comparator = "GreaterThan"; ++ break; ++ } ++ + // Prepare command encoder + auto& compute_encoder = d.get_command_encoder(s.index); + +@@ -166,9 +197,16 @@ void multi_block_sort( + { + std::ostringstream kname; + kname << "sort_mbsort_" << type_to_name(dev_vals_0) << "_" +- << type_to_name(dev_idxs_0) << "_bn" << bn << "_tn" << tn; +- auto kernel = +- get_mb_sort_kernel(d, kname.str(), dev_vals_0, dev_idxs_0, bn, tn); ++ << type_to_name(dev_idxs_0) << "_bn" << bn << "_tn" << tn << "_comp" ++ << comparator; ++ auto kernel = get_mb_sort_kernel( ++ d, ++ kname.str(), ++ dev_vals_0, ++ dev_idxs_0, ++ bn, ++ tn, ++ comparator + "<" + dtype_to_string(in.dtype()) + ">"); + compute_encoder.set_compute_pipeline_state(kernel); + + compute_encoder.set_input_array(in, 0); +@@ -206,10 +244,17 @@ void multi_block_sort( + { + std::ostringstream kname; + kname << "partition_mbsort_" << type_to_name(dev_vals_in) << "_" +- << type_to_name(dev_idxs_in) << "_bn" << bn << "_tn" << tn; +- +- auto kernel = +- get_mb_sort_kernel(d, kname.str(), dev_vals_0, dev_idxs_0, bn, tn); ++ << type_to_name(dev_idxs_in) << "_bn" << bn << "_tn" << tn ++ << "_comp" << comparator; ++ ++ auto kernel = get_mb_sort_kernel( ++ d, ++ kname.str(), ++ dev_vals_0, ++ dev_idxs_0, ++ bn, ++ tn, ++ comparator + "<" + dtype_to_string(in.dtype()) + ">"); + compute_encoder.set_compute_pipeline_state(kernel); + + compute_encoder.set_output_array(block_partitions, 0); +@@ -229,10 +274,17 @@ void multi_block_sort( + { + std::ostringstream kname; + kname << "merge_mbsort_" << type_to_name(dev_vals_in) << "_" +- << type_to_name(dev_idxs_in) << "_bn" << bn << "_tn" << tn; +- +- auto kernel = +- get_mb_sort_kernel(d, kname.str(), dev_vals_0, dev_idxs_0, bn, tn); ++ << type_to_name(dev_idxs_in) << "_bn" << bn << "_tn" << tn ++ << "_comp" << comparator; ++ ++ auto kernel = get_mb_sort_kernel( ++ d, ++ kname.str(), ++ dev_vals_0, ++ dev_idxs_0, ++ bn, ++ tn, ++ comparator + "<" + dtype_to_string(in.dtype()) + ">"); + compute_encoder.set_compute_pipeline_state(kernel); + + compute_encoder.set_input_array(block_partitions, 0); +@@ -277,7 +329,8 @@ void gpu_merge_sort( + const array& in, + array& out, + int axis_, +- bool argsort) { ++ bool argsort, ++ ComparatorType comparator) { + // Get size info + int axis = axis_ < 0 ? axis_ + in.ndim() : axis_; + int size_sorted_axis = in.shape(axis); +@@ -307,9 +360,10 @@ void gpu_merge_sort( + int n_blocks = (size_sorted_axis + n_per_block - 1) / n_per_block; + + if (n_blocks > 1) { +- return multi_block_sort(s, d, in, out, axis, bn, tn, n_blocks, argsort); ++ return multi_block_sort( ++ s, d, in, out, axis, bn, tn, n_blocks, argsort, comparator); + } else { +- return single_block_sort(s, d, in, out, axis, bn, tn, argsort); ++ return single_block_sort(s, d, in, out, axis, bn, tn, argsort, comparator); + } + } + +@@ -324,7 +378,7 @@ void ArgSort::eval_gpu(const std::vector& inputs, array& out) { + auto& d = metal::device(s.device); + auto& in = inputs[0]; + +- gpu_merge_sort(s, d, in, out, axis_, true); ++ gpu_merge_sort(s, d, in, out, axis_, true, comparator_); + } + + void Sort::eval_gpu(const std::vector& inputs, array& out) { +@@ -336,7 +390,7 @@ void Sort::eval_gpu(const std::vector& inputs, array& out) { + auto& d = metal::device(s.device); + auto& in = inputs[0]; + +- gpu_merge_sort(s, d, in, out, axis_, false); ++ gpu_merge_sort(s, d, in, out, axis_, false, comparator_); + } + + void ArgPartition::eval_gpu(const std::vector& inputs, array& out) { +@@ -349,7 +403,7 @@ void ArgPartition::eval_gpu(const std::vector& inputs, array& out) { + auto& d = metal::device(s.device); + auto& in = inputs[0]; + +- gpu_merge_sort(s, d, in, out, axis_, true); ++ gpu_merge_sort(s, d, in, out, axis_, true, comparator_); + } + + void Partition::eval_gpu(const std::vector& inputs, array& out) { +@@ -362,7 +416,7 @@ void Partition::eval_gpu(const std::vector& inputs, array& out) { + auto& d = metal::device(s.device); + auto& in = inputs[0]; + +- gpu_merge_sort(s, d, in, out, axis_, false); ++ gpu_merge_sort(s, d, in, out, axis_, false, comparator_); + } + + } // namespace mlx::core +diff --git a/mlx/backend/no_gpu/primitives.cpp b/mlx/backend/no_gpu/primitives.cpp +index 676a6e55..178eb23f 100644 +--- a/mlx/backend/no_gpu/primitives.cpp ++++ b/mlx/backend/no_gpu/primitives.cpp +@@ -95,7 +95,7 @@ NO_GPU(Partition) + NO_GPU(Power) + NO_GPU_MULTI(QRF) + NO_GPU(QuantizedMatmul) +-NO_GPU(RandomBits) ++NO_GPU() + NO_GPU(Real) + NO_GPU(Reduce) + NO_GPU(Reshape) +diff --git a/mlx/mlx.h b/mlx/mlx.h +index cef8d806..f8932873 100644 +--- a/mlx/mlx.h ++++ b/mlx/mlx.h +@@ -17,6 +17,7 @@ + #include "mlx/memory.h" + #include "mlx/ops.h" + #include "mlx/random.h" ++#include "mlx/sort.h" + #include "mlx/stream.h" + #include "mlx/transforms.h" + #include "mlx/utils.h" +diff --git a/mlx/ops.cpp b/mlx/ops.cpp +index a72c2bc8..04ac921f 100644 +--- a/mlx/ops.cpp ++++ b/mlx/ops.cpp +@@ -2205,13 +2205,20 @@ array argmax( + } + + /** Returns a sorted copy of the flattened array. */ +-array sort(const array& a, StreamOrDevice s /* = {} */) { ++array sort( ++ const array& a, ++ ComparatorType comparator, ++ StreamOrDevice s /* = {} */) { + int size = a.size(); +- return sort(reshape(a, {size}, s), 0, s); ++ return sort(reshape(a, {size}, s), 0, comparator, s); + } + + /** Returns a sorted copy of the array along a given axis. */ +-array sort(const array& a, int axis, StreamOrDevice s /* = {} */) { ++array sort( ++ const array& a, ++ int axis, ++ ComparatorType comparator, ++ StreamOrDevice s /* = {} */) { + // Check for valid axis + if (axis + static_cast(a.ndim()) < 0 || + axis >= static_cast(a.ndim())) { +@@ -2222,17 +2229,27 @@ array sort(const array& a, int axis, StreamOrDevice s /* = {} */) { + } + + return array( +- a.shape(), a.dtype(), std::make_shared(to_stream(s), axis), {a}); ++ a.shape(), ++ a.dtype(), ++ std::make_shared(to_stream(s), axis, comparator), ++ {a}); + } + + /** Returns indices that sort the flattened array. */ +-array argsort(const array& a, StreamOrDevice s /* = {} */) { ++array argsort( ++ const array& a, ++ ComparatorType comparator, ++ StreamOrDevice s /* = {} */) { + int size = a.size(); +- return argsort(reshape(a, {size}, s), 0, s); ++ return argsort(reshape(a, {size}, s), 0, comparator, s); + } + + /** Returns indices that sort the array along a given axis. */ +-array argsort(const array& a, int axis, StreamOrDevice s /* = {} */) { ++array argsort( ++ const array& a, ++ int axis, ++ ComparatorType comparator, ++ StreamOrDevice s /* = {} */) { + // Check for valid axis + if (axis + static_cast(a.ndim()) < 0 || + axis >= static_cast(a.ndim())) { +@@ -2243,16 +2260,23 @@ array argsort(const array& a, int axis, StreamOrDevice s /* = {} */) { + } + + return array( +- a.shape(), uint32, std::make_shared(to_stream(s), axis), {a}); ++ a.shape(), ++ uint32, ++ std::make_shared(to_stream(s), axis, comparator), ++ {a}); + } + + /** + * Returns a partitioned copy of the flattened array + * such that the smaller kth elements are first. + **/ +-array partition(const array& a, int kth, StreamOrDevice s /* = {} */) { ++array partition( ++ const array& a, ++ int kth, ++ ComparatorType comparator, ++ StreamOrDevice s /* = {} */) { + int size = a.size(); +- return partition(reshape(a, {size}, s), kth, 0, s); ++ return partition(reshape(a, {size}, s), kth, 0, comparator, s); + } + + /** +@@ -2263,6 +2287,7 @@ array partition( + const array& a, + int kth, + int axis, ++ ComparatorType comparator, + StreamOrDevice s /* = {} */) { + // Check for valid axis + if (axis + static_cast(a.ndim()) < 0 || +@@ -2283,7 +2308,7 @@ array partition( + return array( + a.shape(), + a.dtype(), +- std::make_shared(to_stream(s), kth_, axis_), ++ std::make_shared(to_stream(s), kth_, axis_, comparator), + {a}); + } + +@@ -2291,9 +2316,13 @@ array partition( + * Returns indices that partition the flattened array + * such that the smaller kth elements are first. + **/ +-array argpartition(const array& a, int kth, StreamOrDevice s /* = {} */) { ++array argpartition( ++ const array& a, ++ int kth, ++ ComparatorType comparator, ++ StreamOrDevice s /* = {} */) { + int size = a.size(); +- return argpartition(reshape(a, {size}, s), kth, 0, s); ++ return argpartition(reshape(a, {size}, s), kth, 0, comparator, s); + } + + /** +@@ -2304,6 +2333,7 @@ array argpartition( + const array& a, + int kth, + int axis, ++ ComparatorType comparator, + StreamOrDevice s /* = {} */) { + // Check for valid axis + if (axis + static_cast(a.ndim()) < 0 || +@@ -2324,7 +2354,7 @@ array argpartition( + return array( + a.shape(), + uint32, +- std::make_shared(to_stream(s), kth_, axis_), ++ std::make_shared(to_stream(s), kth_, axis_, comparator), + {a}); + } + +diff --git a/mlx/ops.h b/mlx/ops.h +index af3cdb5b..be762b8a 100644 +--- a/mlx/ops.h ++++ b/mlx/ops.h +@@ -6,6 +6,7 @@ + + #include "mlx/array.h" + #include "mlx/device.h" ++#include "mlx/sort.h" + #include "mlx/stream.h" + #include "mlx/utils.h" + +@@ -674,40 +675,98 @@ array argmax( + StreamOrDevice s = {}); + + /** Returns a sorted copy of the flattened array. */ +-array sort(const array& a, StreamOrDevice s = {}); ++array sort( ++ const array& a, ++ ComparatorType comparator = ComparatorType::LessThan, ++ StreamOrDevice s = {}); ++inline array sort(const array& a, StreamOrDevice s = {}) { ++ return sort(a, ComparatorType::LessThan, s); ++} + + /** Returns a sorted copy of the array along a given axis. */ +-array sort(const array& a, int axis, StreamOrDevice s = {}); ++array sort( ++ const array& a, ++ int axis, ++ ComparatorType comparator = ComparatorType::LessThan, ++ StreamOrDevice s = {}); ++inline array sort(const array& a, int axis, StreamOrDevice s = {}) { ++ return sort(a, axis, ComparatorType::LessThan, s); ++} + + /** Returns indices that sort the flattened array. */ +-array argsort(const array& a, StreamOrDevice s = {}); ++array argsort( ++ const array& a, ++ ComparatorType comparator = ComparatorType::LessThan, ++ StreamOrDevice s = {}); ++inline array argsort(const array& a, StreamOrDevice s = {}) { ++ return argsort(a, ComparatorType::LessThan, s); ++} + + /** Returns indices that sort the array along a given axis. */ +-array argsort(const array& a, int axis, StreamOrDevice s = {}); ++array argsort( ++ const array& a, ++ int axis, ++ ComparatorType comparator = ComparatorType::LessThan, ++ StreamOrDevice s = {}); ++inline array argsort(const array& a, int axis, StreamOrDevice s = {}) { ++ return argsort(a, axis, ComparatorType::LessThan, s); ++} + + /** + * Returns a partitioned copy of the flattened array + * such that the smaller kth elements are first. + **/ +-array partition(const array& a, int kth, StreamOrDevice s = {}); ++array partition( ++ const array& a, ++ int kth, ++ ComparatorType comparator = ComparatorType::LessThan, ++ StreamOrDevice s = {}); ++inline array partition(const array& a, int kth, StreamOrDevice s = {}) { ++ return partition(a, kth, ComparatorType::LessThan, s); ++} + + /** + * Returns a partitioned copy of the array along a given axis + * such that the smaller kth elements are first. + **/ +-array partition(const array& a, int kth, int axis, StreamOrDevice s = {}); ++array partition( ++ const array& a, ++ int kth, ++ int axis, ++ ComparatorType comparator = ComparatorType::LessThan, ++ StreamOrDevice s = {}); ++inline array ++partition(const array& a, int kth, int axis, StreamOrDevice s = {}) { ++ return partition(a, kth, axis, ComparatorType::LessThan, s); ++} + + /** + * Returns indices that partition the flattened array + * such that the smaller kth elements are first. + **/ +-array argpartition(const array& a, int kth, StreamOrDevice s = {}); ++array argpartition( ++ const array& a, ++ int kth, ++ ComparatorType comparator = ComparatorType::LessThan, ++ StreamOrDevice s = {}); ++inline array argpartition(const array& a, int kth, StreamOrDevice s = {}) { ++ return argpartition(a, kth, ComparatorType::LessThan, s); ++} + + /** + * Returns indices that partition the array along a given axis + * such that the smaller kth elements are first. + **/ +-array argpartition(const array& a, int kth, int axis, StreamOrDevice s = {}); ++array argpartition( ++ const array& a, ++ int kth, ++ int axis, ++ ComparatorType comparator = ComparatorType::LessThan, ++ StreamOrDevice s = {}); ++inline array ++argpartition(const array& a, int kth, int axis, StreamOrDevice s = {}) { ++ return argpartition(a, kth, axis, ComparatorType::LessThan, s); ++} + + /** Returns topk elements of the flattened array. */ + array topk(const array& a, int k, StreamOrDevice s = {}); +diff --git a/mlx/primitives.h b/mlx/primitives.h +index c0fbfc84..13d67201 100644 +--- a/mlx/primitives.h ++++ b/mlx/primitives.h +@@ -7,6 +7,7 @@ + #include "mlx/array.h" + #include "mlx/device.h" + #include "mlx/io/load.h" ++#include "mlx/sort.h" + #include "mlx/stream.h" + + #define DEFINE_VMAP() \ +@@ -322,8 +323,15 @@ class ArcTanh : public UnaryPrimitive { + + class ArgPartition : public UnaryPrimitive { + public: +- explicit ArgPartition(Stream stream, int kth, int axis) +- : UnaryPrimitive(stream), kth_(kth), axis_(axis) {} ++ explicit ArgPartition( ++ Stream stream, ++ int kth, ++ int axis, ++ ComparatorType comparator) ++ : UnaryPrimitive(stream), ++ kth_(kth), ++ axis_(axis), ++ comparator_(comparator) {} + + void eval_cpu(const std::vector& inputs, array& out) override; + void eval_gpu(const std::vector& inputs, array& out) override; +@@ -333,13 +341,14 @@ class ArgPartition : public UnaryPrimitive { + DEFINE_PRINT(ArgPartition) + DEFINE_INPUT_OUTPUT_SHAPE() + bool is_equivalent(const Primitive& other) const override; +- std::pair state() const { +- return {kth_, axis_}; ++ auto state() const { ++ return std::make_tuple(kth_, axis_, comparator_); + }; + + private: + int kth_; + int axis_; ++ ComparatorType comparator_; + }; + + class ArgReduce : public UnaryPrimitive { +@@ -371,8 +380,8 @@ class ArgReduce : public UnaryPrimitive { + + class ArgSort : public UnaryPrimitive { + public: +- explicit ArgSort(Stream stream, int axis) +- : UnaryPrimitive(stream), axis_(axis) {} ++ explicit ArgSort(Stream stream, int axis, ComparatorType comparator) ++ : UnaryPrimitive(stream), axis_(axis), comparator_(comparator) {} + + void eval_cpu(const std::vector& inputs, array& out) override; + void eval_gpu(const std::vector& inputs, array& out) override; +@@ -381,12 +390,13 @@ class ArgSort : public UnaryPrimitive { + DEFINE_PRINT(ArgSort) + DEFINE_INPUT_OUTPUT_SHAPE() + bool is_equivalent(const Primitive& other) const override; +- int state() const { +- return axis_; ++ auto state() const { ++ return std::make_pair(axis_, comparator_); + }; + + private: + int axis_; ++ ComparatorType comparator_; + }; + + class AsType : public UnaryPrimitive { +@@ -1530,8 +1540,15 @@ class Pad : public UnaryPrimitive { + + class Partition : public UnaryPrimitive { + public: +- explicit Partition(Stream stream, int kth, int axis) +- : UnaryPrimitive(stream), kth_(kth), axis_(axis) {} ++ explicit Partition( ++ Stream stream, ++ int kth, ++ int axis, ++ ComparatorType comparator) ++ : UnaryPrimitive(stream), ++ kth_(kth), ++ axis_(axis), ++ comparator_(comparator) {} + + void eval_cpu(const std::vector& inputs, array& out) override; + void eval_gpu(const std::vector& inputs, array& out) override; +@@ -1542,12 +1559,13 @@ class Partition : public UnaryPrimitive { + DEFINE_INPUT_OUTPUT_SHAPE() + bool is_equivalent(const Primitive& other) const override; + auto state() const { +- return std::make_pair(kth_, axis_); ++ return std::make_tuple(kth_, axis_, comparator_); + }; + + private: + int kth_; + int axis_; ++ ComparatorType comparator_; + }; + + class Power : public UnaryPrimitive { +@@ -2073,8 +2091,8 @@ class Softmax : public UnaryPrimitive { + + class Sort : public UnaryPrimitive { + public: +- explicit Sort(Stream stream, int axis) +- : UnaryPrimitive(stream), axis_(axis) {} ++ explicit Sort(Stream stream, int axis, ComparatorType comparator) ++ : UnaryPrimitive(stream), axis_(axis), comparator_(comparator) {} + + void eval_cpu(const std::vector& inputs, array& out) override; + void eval_gpu(const std::vector& inputs, array& out) override; +@@ -2085,11 +2103,12 @@ class Sort : public UnaryPrimitive { + DEFINE_INPUT_OUTPUT_SHAPE() + bool is_equivalent(const Primitive& other) const override; + auto state() const { +- return axis_; ++ return std::make_pair(axis_, comparator_); + } + + private: + int axis_; ++ ComparatorType comparator_; + }; + + class Split : public Primitive { +diff --git a/mlx/random.cpp b/mlx/random.cpp +index 6c6d1eb9..5a4745a5 100644 +--- a/mlx/random.cpp ++++ b/mlx/random.cpp +@@ -225,6 +225,23 @@ array normal( + return samples; + } + ++array normal( ++ const Shape& shape, ++ Dtype dtype, ++ const array& loc, ++ const array& scale, ++ const std::optional& key /*= nullopt */, ++ StreamOrDevice s /* = {} */) { ++ auto stream = to_stream(s); ++ auto low = above_minus_one_with_default(dtype); ++ auto high = array(1.0f, dtype); ++ auto samples = uniform(low, high, shape, dtype, key, stream); ++ samples = multiply(array(std::sqrt(2.0), dtype), erfinv(samples, stream), stream); ++ samples = multiply(astype(scale, dtype), samples, stream); ++ samples = add(astype(loc, dtype), samples, stream); ++ return samples; ++} ++ + array multivariate_normal( + const array& mean, + const array& cov, +diff --git a/mlx/random.h b/mlx/random.h +index 0dfdab7a..ebee726c 100644 +--- a/mlx/random.h ++++ b/mlx/random.h +@@ -134,6 +134,14 @@ inline array normal( + return normal(shape, float32, std::nullopt, std::nullopt, key, s); + } + ++array normal( ++ const Shape& shape, ++ Dtype dtype, ++ const array& loc, ++ const array& scale, ++ const std::optional& key = std::nullopt, ++ StreamOrDevice s = {}); ++ + /** Generate samples from a multivariate normal distribution. **/ + array multivariate_normal( + const array& mean, +diff --git a/mlx/sort.h b/mlx/sort.h +new file mode 100644 +index 00000000..0afc6b62 +--- /dev/null ++++ b/mlx/sort.h +@@ -0,0 +1,14 @@ ++// Copyright © 2023-2024 Apple Inc. ++ ++#pragma once ++ ++#include "mlx/array.h" ++ ++namespace mlx::core { ++ ++enum class ComparatorType { ++ LessThan, ++ GreaterThan, ++}; ++ ++} // namespace mlx::core +diff --git a/mlx/types/complex.h b/mlx/types/complex.h +index 51101cc9..344d80f8 100644 +--- a/mlx/types/complex.h ++++ b/mlx/types/complex.h +@@ -59,10 +59,11 @@ inline bool operator>(const complex64_t& a, const complex64_t& b) { + inline complex64_t operator%(complex64_t a, complex64_t b) { + auto real = a.real() - (b.real() * static_cast(a.real() / b.real())); + auto imag = a.imag() - (b.imag() * static_cast(a.imag() / b.imag())); +- if (real != 0 && ((real < 0) != (b.real() < 0))) +- real += b.real(); +- if (imag != 0 && ((imag < 0) != (b.imag() < 0))) +- imag += b.imag(); ++ /* Disable to match `stablehlo` implementation semantics */ ++ // if (real != 0 && ((real < 0) != (b.real() < 0))) ++ // real += b.real(); ++ // if (imag != 0 && ((imag < 0) != (b.imag() < 0))) ++ // imag += b.imag(); + return {real, imag}; + } + diff --git a/third_party/modules/mlx/20250518.0-eebe730/source.json b/third_party/modules/mlx/20250518.0-eebe730/source.json new file mode 100644 index 00000000..ad6540c6 --- /dev/null +++ b/third_party/modules/mlx/20250518.0-eebe730/source.json @@ -0,0 +1,15 @@ +{ + "strip_prefix": "mlx-eebe73001affcb424171e9d49657e508f70a9201", + "url": "https://github.com/ml-explore/mlx/archive/eebe73001affcb424171e9d49657e508f70a9201.tar.gz", + "integrity": "sha256-bRpLtmR9R/bSlzEQ403T8/K2t8r3PLISd5JWmB22cgQ=", + "overlay": { + "metal_cpp.bzl": "", + "jit.bzl": "", + "MODULE.bazel": "", + "BUILD.bazel": "" + }, + "patch_strip": 1, + "patches": { + "0001-Modify-Sort-And-Comparison-Kernels.patch": "" + } +} diff --git a/third_party/modules/mlx/metadata.json b/third_party/modules/mlx/metadata.json new file mode 100644 index 00000000..6945817e --- /dev/null +++ b/third_party/modules/mlx/metadata.json @@ -0,0 +1,17 @@ +{ + "homepage": "https://github.com/ml-explore/mlx", + "maintainers": [ + { + "email": "bzlmod@zml.ai", + "github": "zml", + "name": "ZML Engineering Team" + } + ], + "repository": [ + "github:ml-explore/mlx" + ], + "versions": [ + "20250518.0-eebe730" + ], + "yanked_versions": {} +} diff --git a/third_party/modules/xla/20250317.2-71c67e2/MODULE.bazel b/third_party/modules/xla/20250317.2-71c67e2/MODULE.bazel index 042d899c..5b6b999f 100644 --- a/third_party/modules/xla/20250317.2-71c67e2/MODULE.bazel +++ b/third_party/modules/xla/20250317.2-71c67e2/MODULE.bazel @@ -17,6 +17,10 @@ bazel_dep(name = "zlib", version = "1.2.13") bazel_dep(name = "re2", version = "2024-07-02.bcr.1", repo_name = "com_googlesource_code_re2") bazel_dep(name = "rules_license", version = "0.0.8") +# MLX PjRt Plugin Dependencies +bazel_dep(name = "mlx", version = "20250518.0-eebe730") +bazel_dep(name = "range-v3", version = "0.12.0") + tsl = use_extension("//:tsl.bzl", "tsl") use_repo(tsl, "tsl", "python_version_repo") @@ -28,9 +32,15 @@ use_repo( "local_config_cuda", "local_config_remote_execution", "local_config_rocm", + "local_config_sycl", "local_config_tensorrt", + "eigen_archive", + "farmhash_archive", + "ml_dtypes_py", "llvm-raw", "stablehlo", + "shardy", + "snappy", ) llvm_configure = use_extension("//:llvm.bzl", "llvm_configure") diff --git a/third_party/modules/xla/20250317.2-71c67e2/overlay/workspace.bzl b/third_party/modules/xla/20250317.2-71c67e2/overlay/workspace.bzl index 192e06f8..fa890885 100644 --- a/third_party/modules/xla/20250317.2-71c67e2/overlay/workspace.bzl +++ b/third_party/modules/xla/20250317.2-71c67e2/overlay/workspace.bzl @@ -1,9 +1,11 @@ load("//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls") load("//third_party/gpus:cuda_configure.bzl", "cuda_configure") load("//third_party/gpus:rocm_configure.bzl", "rocm_configure") +load("//third_party/gpus:sycl_configure.bzl", "sycl_configure") load("//third_party/llvm:workspace.bzl", llvm = "repo") load("//third_party/pybind11_bazel:workspace.bzl", pybind11_bazel = "repo") load("//third_party/stablehlo:workspace.bzl", stablehlo = "repo") +load("//third_party/shardy:workspace.bzl", shardy = "repo") load("//third_party/tensorrt:tensorrt_configure.bzl", "tensorrt_configure") load("//third_party/triton:workspace.bzl", triton = "repo") load("//tools/toolchains/remote:configure.bzl", "remote_execution_configure") @@ -12,11 +14,45 @@ def _xla_workspace_impl(mctx): cuda_configure(name = "local_config_cuda") remote_execution_configure(name = "local_config_remote_execution") rocm_configure(name = "local_config_rocm") + sycl_configure(name = "local_config_sycl") tensorrt_configure(name = "local_config_tensorrt") pybind11_bazel() triton() llvm("llvm-raw") stablehlo() + shardy() + tf_http_archive( + name = "eigen_archive", + build_file = "//third_party/eigen3:eigen_archive.BUILD", + sha256 = "1a432ccbd597ea7b9faa1557b1752328d6adc1a3db8969f6fe793ff704be3bf0", + strip_prefix = "eigen-4c38131a16803130b66266a912029504f2cf23cd", + urls = tf_mirror_urls("https://gitlab.com/libeigen/eigen/-/archive/4c38131a16803130b66266a912029504f2cf23cd/eigen-4c38131a16803130b66266a912029504f2cf23cd.tar.gz"), + ) + tf_http_archive( + name = "farmhash_archive", + build_file = "//third_party/farmhash:farmhash.BUILD", + sha256 = "18392cf0736e1d62ecbb8d695c31496b6507859e8c75541d7ad0ba092dc52115", + strip_prefix = "farmhash-0d859a811870d10f53a594927d0d0b97573ad06d", + urls = tf_mirror_urls("https://github.com/google/farmhash/archive/0d859a811870d10f53a594927d0d0b97573ad06d.tar.gz"), + ) + tf_http_archive( + name = "ml_dtypes_py", + build_file = "//third_party/py/ml_dtypes:ml_dtypes_py.BUILD", + link_files = { + "//third_party/py/ml_dtypes:ml_dtypes.BUILD": "ml_dtypes/BUILD.bazel", + }, + sha256 = "f6e5880666661351e6cd084ac4178ddc4dabcde7e9a73722981c0d1500cf5937", + strip_prefix = "ml_dtypes-00d98cd92ade342fef589c0470379abb27baebe9", + urls = tf_mirror_urls("https://github.com/jax-ml/ml_dtypes/archive/00d98cd92ade342fef589c0470379abb27baebe9/ml_dtypes-00d98cd92ade342fef589c0470379abb27baebe9.tar.gz"), + ) + tf_http_archive( + name = "snappy", + build_file = "//third_party:snappy.BUILD", + sha256 = "2e458b7017cd58dcf1469ab315389e85e7f445bd035188f2983f81fb19ecfb29", + strip_prefix = "snappy-984b191f0fefdeb17050b42a90b7625999c13b8d", + system_build_file = "//third_party/systemlibs:snappy.BUILD", + urls = tf_mirror_urls("https://github.com/google/snappy/archive/984b191f0fefdeb17050b42a90b7625999c13b8d.tar.gz"), + ) tf_http_archive( name = "com_github_grpc_grpc", sha256 = "b956598d8cbe168b5ee717b5dafa56563eb5201a947856a6688bbeac9cac4e1f", diff --git a/third_party/modules/xla/20250317.2-71c67e2/patches/0003-Added-MLX-PjRt-Plugin.patch b/third_party/modules/xla/20250317.2-71c67e2/patches/0003-Added-MLX-PjRt-Plugin.patch new file mode 100644 index 00000000..b8917ab7 --- /dev/null +++ b/third_party/modules/xla/20250317.2-71c67e2/patches/0003-Added-MLX-PjRt-Plugin.patch @@ -0,0 +1,4663 @@ +diff --git a/xla/pjrt/plugin/mlx/BUILD b/xla/pjrt/plugin/mlx/BUILD +new file mode 100644 +index 0000000000..ca86b466bf +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/BUILD +@@ -0,0 +1,201 @@ ++load("//xla:xla.bzl", "xla_cc_binary", "xla_cc_test") ++ ++package( ++ # copybara:uncomment default_applicable_licenses = ["//tensorflow:license"], ++ default_visibility = ["//visibility:public"], ++ licenses = ["notice"], ++) ++ ++cc_library( ++ name = "buffer", ++ srcs = ["buffer.cc"], ++ hdrs = ["buffer.h"], ++ deps = [ ++ ":logging", ++ ":utils", ++ "//xla:literal", ++ "//xla:shape_util", ++ "//xla:util", ++ "//xla/hlo/translate/hlo_to_mhlo:hlo_utils", ++ "//xla/hlo/translate/mhlo_to_hlo:literal_exporter", ++ "//xla/hlo/translate/mhlo_to_hlo:type_to_shape", ++ "//xla/pjrt:pjrt_client", ++ "//xla/pjrt:pjrt_compiler", ++ "//xla/pjrt:pjrt_future", ++ "@com_google_absl//absl/functional:any_invocable", ++ "@com_google_absl//absl/log", ++ "@com_google_absl//absl/status:statusor", ++ "@com_google_absl//absl/types:span", ++ "@llvm-project//llvm:Support", ++ "@llvm-project//mlir:AsmParser", ++ "@llvm-project//mlir:IR", ++ "@llvm-project//mlir:Support", ++ ], ++) ++ ++ ++cc_library( ++ name = "executable", ++ srcs = ["executable.cc"], ++ hdrs = ["executable.h"], ++ deps = [ ++ "@range-v3", ++ "@mlx//:mlx", ++ ":buffer", ++ ":logging", ++ ":utils", ++ "//xla/hlo/builder:xla_computation", ++ "//xla/hlo/translate:stablehlo", ++ "//xla/mlir/utils:error_util", ++ "//xla/mlir_hlo:mhlo_passes", ++ "//xla/mlir_hlo:stablehlo_extension_passes", ++ # "//xla/mlir/utils:type_util", ++ "//xla/pjrt:mlir_to_hlo", ++ "//xla/pjrt:pjrt_client", ++ "//xla/pjrt:pjrt_executable", ++ "//xla/pjrt:pjrt_future", ++ "//xla/service:computation_placer_hdr", ++ "@com_google_absl//absl/log", ++ "@com_google_absl//absl/status", ++ "@com_google_absl//absl/status:statusor", ++ "@llvm-project//llvm:Support", ++ "@llvm-project//mlir:AsmParser", ++ "@llvm-project//mlir:BytecodeWriter", ++ "@llvm-project//mlir:DataLayoutInterfaces", ++ "@llvm-project//mlir:FuncDialect", ++ "@llvm-project//mlir:IR", ++ "@llvm-project//mlir:Parser", ++ "@llvm-project//mlir:Pass", ++ "@llvm-project//mlir:Support", ++ "@llvm-project//mlir:Transforms", ++ "@stablehlo//:reference_api", ++ "@stablehlo//:stablehlo_passes_optimization", ++ "@tsl//tsl/platform:statusor", ++ ], ++) ++ ++cc_library( ++ name = "device", ++ srcs = ["device.cc"], ++ hdrs = ["device.h"], ++ deps = [ ++ ":logging", ++ "//xla:util", ++ "//xla/pjrt:pjrt_client", ++ "//xla/pjrt:pjrt_common", ++ "//xla/pjrt:pjrt_compiler", ++ "//xla/pjrt:pjrt_device_description", ++ "@com_google_absl//absl/status:statusor", ++ "@com_google_absl//absl/strings:str_format", ++ "@com_google_absl//absl/strings:string_view", ++ "@com_google_absl//absl/types:span", ++ "@mlx//:mlx", ++ ], ++) ++ ++cc_library( ++ name = "client_cpp_pjrt", ++ srcs = [ ++ "client_cpp_pjrt.cc", ++ ], ++ hdrs = [ ++ "client_cpp_pjrt.h", ++ ], ++ deps = [ ++ ":buffer", ++ ":device", ++ ":executable", ++ ":logging", ++ ":utils", ++ "//xla:literal", ++ "//xla:shape_util", ++ "//xla:util", ++ "//xla/pjrt:host_memory_spaces", ++ "//xla/pjrt:pjrt_client", ++ "//xla/pjrt:pjrt_common", ++ "//xla/pjrt:pjrt_compiler", ++ "//xla/tsl/framework:allocator", ++ "@com_google_absl//absl/status:statusor", ++ "@com_google_absl//absl/strings:str_format", ++ "@com_google_absl//absl/strings:string_view", ++ "@com_google_absl//absl/types:span", ++ "@mlx//:mlx", ++ "@tsl//tsl/platform:fingerprint", ++ "@tsl//tsl/platform:statusor" ++ ], ++) ++ ++cc_library( ++ name = "client_c_pjrt", ++ srcs = [ ++ "client_c_pjrt.cc", ++ ], ++ hdrs = ["client_c_pjrt.h"], ++ deps = [ ++ ":client_cpp_pjrt", ++ "//xla/pjrt:pjrt_client", ++ "//xla/pjrt/c:pjrt_c_api_hdrs", ++ "//xla/pjrt/c:pjrt_c_api_layouts_extension_hdrs", ++ "//xla/pjrt/c:pjrt_c_api_wrapper_impl", ++ "@com_google_absl//absl/status", ++ "@com_google_absl//absl/strings:string_view", ++ ], ++ alwayslink = 1, ++) ++ ++cc_library( ++ name = "logging", ++ srcs = ["logging.cc"], ++ hdrs = ["logging.h"], ++ deps = [ ++ "@com_google_absl//absl/base:log_severity", ++ "@com_google_absl//absl/log", ++ "@com_google_absl//absl/log:globals", ++ "@llvm-project//llvm:Support", ++ "@llvm-project//mlir:AsmParser", ++ "@llvm-project//mlir:DataLayoutInterfaces", ++ "@llvm-project//mlir:IR", ++ ], ++) ++ ++cc_library( ++ name = "utils", ++ srcs = ["utils.cc"], ++ hdrs = ["utils.h"], ++ deps = [ ++ "@com_google_absl//absl/status", ++ "@com_google_absl//absl/status:statusor", ++ "@com_google_absl//absl/types:span", ++ "@llvm-project//llvm:Support", ++ "@llvm-project//mlir:IR", ++ "@llvm-project//mlir:Support", ++ "@mlx//:mlx", ++ "@tsl//tsl/platform:statusor", ++ "//xla/mlir/utils:type_util", ++ "//xla:literal", ++ "//xla:shape_util", ++ "//xla:util", ++ ++ ], ++) ++ ++xla_cc_test( ++ name = "plugin_pjrt_test", ++ srcs = ["plugin_pjrt_test.cc"], ++ deps = [ ++ ":client_c_pjrt", ++ ":client_cpp_pjrt", ++ "//xla/pjrt:pjrt_client_test_common", ++ "//xla/pjrt/c:pjrt_c_api_test_common", ++ "//xla/pjrt/c:pjrt_c_api_wrapper_impl", ++ "@com_google_googletest//:gtest_main", ++ ], ++) ++ ++xla_cc_binary( ++ name = "stablehlo_mlx_plugin.so", ++ linkshared = True, ++ deps = [ ++ ":client_c_pjrt", ++ ], ++) +\ No newline at end of file +diff --git a/xla/pjrt/plugin/mlx/buffer.cc b/xla/pjrt/plugin/mlx/buffer.cc +new file mode 100644 +index 0000000000..8bb2a335fe +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/buffer.cc +@@ -0,0 +1,372 @@ ++/* Copyright 2024 The OpenXLA Authors. ++Licensed under the Apache License, Version 2.0 (the "License"); ++you may not use this file except in compliance with the License. ++You may obtain a copy of the License at ++ http://www.apache.org/licenses/LICENSE-2.0 ++Unless required by applicable law or agreed to in writing, software ++distributed under the License is distributed on an "AS IS" BASIS, ++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++See the License for the specific language governing permissions and ++limitations under the License. ++==============================================================================*/ ++ ++#include "xla/pjrt/plugin/mlx/buffer.h" ++ ++#include ++#include ++#include ++#include ++#include ++ ++#include "absl/functional/any_invocable.h" ++#include "absl/log/log.h" ++#include "absl/status/statusor.h" ++#include "absl/types/span.h" ++#include "llvm/Support/Casting.h" ++#include "llvm/Support/raw_ostream.h" ++#include "mlir/AsmParser/AsmParser.h" ++#include "mlir/IR/Attributes.h" ++#include "mlir/IR/Builders.h" ++#include "mlir/IR/BuiltinAttributes.h" ++#include "mlir/IR/BuiltinTypeInterfaces.h" ++#include "mlir/IR/MLIRContext.h" ++#include "mlx/mlx.h" ++#include "mlir/Support/DebugStringHelper.h" ++#include "xla/hlo/translate/hlo_to_mhlo/hlo_utils.h" ++#include "xla/hlo/translate/mhlo_to_hlo/literal_exporter.h" ++#include "xla/hlo/translate/mhlo_to_hlo/type_to_shape.h" ++#include "xla/literal.h" ++#include "xla/pjrt/pjrt_client.h" ++#include "xla/pjrt/pjrt_compiler.h" ++#include "xla/pjrt/pjrt_future.h" ++#include "xla/pjrt/plugin/mlx/logging.h" ++#include "xla/pjrt/plugin/mlx/utils.h" ++#include "xla/shape.h" ++#include "xla/util.h" ++ ++namespace mx = mlx::core; ++namespace mlir::stablehlo { ++ ++using xla::MutableLiteralBase; ++using xla::PjRtBuffer; ++using xla::PjRtClient; ++using xla::PjRtDevice; ++using xla::PjRtFuture; ++using xla::PjRtMemorySpace; ++using xla::PjRtPlatformId; ++using xla::Shape; ++ ++#define UNIMPLEMENTED(name) \ ++ xla::Unimplemented("MlxPjrtBuffer::" #name " is not implemented") ++ ++class MlxPjrtBuffer : public PjRtBuffer { ++ public: ++ MlxPjrtBuffer(mx::array array, const Shape& shape, ++ PjRtMemorySpace* memory_space) ++ : xla::PjRtBuffer(), ++ context_(), ++ buffer_(), ++ array_(array), ++ shape_(shape), ++ memory_space_(memory_space) { ++ TRACE_ME_MEMBER; ++ } ++ ++ class MlirClonedExternalReference : public ExternalReference { ++ public: ++ explicit MlirClonedExternalReference(PjRtBuffer* buffer, ++ PjRtMemorySpace* memory_space) ++ : buffer_() { ++ TRACE_ME_MEMBER; ++ auto mlir_buffer = GetAttributeFromBuffer(buffer); ++ if (!mlir_buffer.ok()) { ++ LOG(ERROR) << "Could not get attribute from buffer: " ++ << mlir_buffer.status(); ++ } ++ ++ auto array = GetArrayFromBuffer(buffer); ++ if (!array.ok()) { ++ LOG(ERROR) << "Could not get attribute from buffer: " << array.status(); ++ } ++ buffer_ = CreateMlirBufferFromAttribute( ++ array.value(), mlir_buffer.value(), memory_space); ++ data_ptr_ = (void*)mlir_buffer.value().getRawData().data(); ++ } ++ ++ private: ++ std::unique_ptr buffer_; ++ }; ++ ++ // All buffers are managed by the MLIR Context ++ ~MlxPjrtBuffer() override = default; ++ ++ MlxPjrtBuffer(const MlxPjrtBuffer&) = delete; ++ MlxPjrtBuffer(MlxPjrtBuffer&&) = delete; ++ MlxPjrtBuffer& operator=(const MlxPjrtBuffer&) = delete; ++ MlxPjrtBuffer& operator=(MlxPjrtBuffer&&) = delete; ++ ++ static std::unique_ptr CreateFromLiteral( ++ mx::array array, const xla::LiteralSlice& literal, ++ xla::PjRtMemorySpace* memory_space) { ++ // TRACE_ME; ++ LOG(INFO) << "CreateFromLiteral: " << literal.ToString() << "\n"; ++ auto buffer = ++ std::make_unique(array, literal.shape(), memory_space); ++ LOG(INFO) << "CreateFromLiteral -> " << (void*)buffer.get() << "\n"; ++ mlir::Builder builder(&buffer->context_); ++ auto attr = xla::CreateDenseElementsAttrFromLiteral(literal, builder); ++ if (!attr.ok()) { ++ LOG(ERROR) << "Could not create dense elements attr from literal: " ++ << attr.status(); ++ return nullptr; ++ } ++ buffer->buffer_ = attr.value(); ++ return buffer; ++ } ++ ++ static std::unique_ptr CreateFromAttribute( ++ mx::array array, DenseElementsAttr attr, ++ xla::PjRtMemorySpace* memory_space) { ++ // TRACE_ME; ++ ++ // MLIR type to xla shape: ++ Shape shape = xla::TypeToShape(attr.getType()); ++ auto buffer = std::make_unique(array, shape, memory_space); ++ buffer->buffer_ = CloneIntoContext(attr, buffer->context_); ++ LOG(INFO) << "CreateFromAttribute(" << ToString(attr) << ") -> " ++ << (void*)buffer.get() << "\n"; ++ return buffer; ++ } ++ ++ const Shape& on_device_shape() const override { ++ TRACE_ME_MEMBER; ++ return shape_; ++ } ++ absl::StatusOr logical_on_device_shape() override { ++ TRACE_ME_MEMBER; ++ return shape_; ++ } ++ ++ PjRtPlatformId platform_id() const { ++ TRACE_ME_MEMBER; ++ return client()->platform_id(); ++ } ++ absl::string_view platform_name() const { ++ TRACE_ME_MEMBER; ++ return client()->platform_name(); ++ } ++ ++ bool IsEmptyTuple() const { ++ TRACE_ME_MEMBER; ++ return shape_.IsTuple() && shape_.tuple_shapes().empty(); ++ } ++ ++ // Buffer knows device + client per older design, should only need ++ // memory_space. ++ PjRtMemorySpace* memory_space() const override { ++ TRACE_ME_MEMBER; ++ return memory_space_; ++ } ++ PjRtDevice* device() const override { ++ TRACE_ME_MEMBER; ++ return memory_space_->devices().front(); ++ } ++ PjRtClient* client() const override { ++ TRACE_ME_MEMBER; ++ return memory_space_->client(); ++ } ++ ++ absl::StatusOr> ++ AcquireExternalReference() override { ++ TRACE_ME_MEMBER; ++ return std::make_unique(this, memory_space_); ++ } ++ ++ xla::PjRtFuture<> ToLiteral(xla::MutableLiteralBase* literal) override { ++ TRACE_ME_MEMBER; ++ if (IsEmptyTuple()) { ++ return PjRtFuture<>( ++ xla::InvalidArgument("ToLiteral called on empty tuple")); ++ } ++ ++ absl::StatusOr to_copy = ++ mhlo::CreateLiteralFromAttribute(buffer_, {}); ++ if (!to_copy.ok()) return PjRtFuture<>(to_copy.status()); ++ ++ // Synchronous! To make async, make the buffer, start the copy, and return a ++ // future that is ready when the copy is done. ++ auto status = literal->CopyFrom(to_copy.value()); ++ if (!status.ok()) return PjRtFuture<>(status); ++ return PjRtFuture<>(absl::OkStatus()); ++ } ++ ++ PjRtFuture<> LazyToLiteral( ++ absl::AnyInvocable() &&> generator) ++ override { ++ TRACE_ME_MEMBER; ++ auto buffer = std::move(generator)(); ++ if (!buffer.ok()) return PjRtFuture<>(buffer.status()); ++ return ToLiteral(buffer.value()); ++ } ++ ++ absl::StatusOr GetOnDeviceSizeInBytes() const override { ++ // This is needed by AcquireExternalReference, for framework figuring out ++ // how to read the underlying buffer data. ++ TRACE_ME_MEMBER; ++ if (!buffer_) return 0; ++ return buffer_.getRawData().size(); ++ } ++ ++ PjRtFuture<> CopyRawToHost(void* dst, int64_t offset, ++ int64_t transfer_size) override { ++ TRACE_ME_MEMBER; ++ return PjRtFuture<>(UNIMPLEMENTED(CopyRawToHost)); ++ } ++ ++ absl::StatusOr> ++ ReleaseDeviceMemoryOwnership(bool wait_for_operations_to_complete) override { ++ TRACE_ME_MEMBER; ++ auto external_ref = AcquireExternalReference(); ++ Delete(); ++ return external_ref; ++ } ++ ++ // Remove the buffer if deleted. ++ // Note: deleted and uninitialized appear the same in this scenario. ++ // Consider changing to mlir::NoneType when deleted. ++ void Delete() override { ++ TRACE_ME_MEMBER; ++ buffer_ = {}; ++ } ++ ++ bool IsDeleted() override { ++ TRACE_ME_MEMBER; ++ return !buffer_; ++ } ++ ++ absl::StatusOr> CopyToDevice( ++ xla::PjRtDevice* dst_device) { ++ TRACE_ME_MEMBER; ++ return CopyToMemorySpace( ++ dst_device->default_memory_space().value_or(nullptr)); ++ } ++ ++ absl::StatusOr> CopyToMemorySpace( ++ xla::PjRtMemorySpace* dst_memory_space) override { ++ TRACE_ME_MEMBER; ++ return CreateMlirBufferFromAttribute(array_, buffer_, dst_memory_space); ++ } ++ ++ void CopyToRemoteDevice( ++ xla::PjRtFuture serialized_descriptor, ++ xla::PjRtBuffer::RemoteSendCallback on_done) override { ++ TRACE_ME_MEMBER; ++ on_done(UNIMPLEMENTED(CopyToRemoteDevice), false); ++ } ++ ++ ++ xla::PjRtFuture<> GetReadyFuture() override { ++ TRACE_ME_MEMBER; ++ LOG(INFO) << "GetReadyFuture(" << (void*)this << ")\n"; ++ // Synchronous! To make async, have the device make a buffer with a ready ++ // future that is ready when the computation is done / buffer is ready. ++ return PjRtFuture<>(absl::OkStatus()); ++ } ++ ++ bool IsOnCpu() const override { ++ // If buffer is on CPU, it will be shared with framework via ++ // GetExternalReference, lse it is copied back to host. ++ // Since we are using reference interpreter, we are running on CPU in a ++ // shared memory space. ++ TRACE_ME_MEMBER; ++ return false; ++ } ++ ++ mlir::DenseElementsAttr GetBufferAttribute() const { return buffer_; } ++ ++ mx::array GetArray() const { return array_; } ++ ++ private: ++ MLIRContext context_; ++ mlir::DenseElementsAttr buffer_; ++ mx::array array_; ++ ++ xla::Shape shape_; ++ PjRtMemorySpace* memory_space_; ++}; ++ ++std::unique_ptr CreateMlirBufferFromMlxArray( ++ mx::array array, xla::PjRtMemorySpace* memory_space) { ++ // TRACE_ME; ++ std::vector span_shape(array.shape().begin(), array.shape().end()); ++ auto shape = xla::ShapeUtil::MakeShape( ++ utils::dtype::asXlaPrimitiveType(array.dtype()), ++ absl::Span(span_shape.data(), span_shape.size())); ++ auto literal = xla::BorrowingLiteral( ++ reinterpret_cast(array.data()), shape); ++ return MlxPjrtBuffer::CreateFromLiteral(array, literal, memory_space); ++} ++ ++std::unique_ptr CreateMlirBufferFromLiteral( ++ const xla::LiteralSlice& literal, xla::PjRtMemorySpace* memory_space) { ++ // TRACE_ME; ++ auto maybe_array = utils::array::fromHostLiteral(literal); ++ return MlxPjrtBuffer::CreateFromLiteral(maybe_array.value(), literal, ++ memory_space); ++} ++ ++std::unique_ptr CreateMlirBufferFromAttribute( ++ mx::array array, DenseElementsAttr attr, ++ xla::PjRtMemorySpace* memory_space) { ++ // TRACE_ME; ++ return MlxPjrtBuffer::CreateFromAttribute(array, attr, memory_space); ++} ++ ++std::unique_ptr CreateMlirBufferUninitialized( ++ const xla::Shape& shape, PjRtMemorySpace* memory_space) { ++ // TRACE_ME; ++ // TODO (@cryptodeal): C API doesn't implement this, but ++ // we'll want to ensure when the Buffer is initialized, ++ // the resulting array is correct shape/dtype. ++ ++ // Init empty array of dtype ++ auto array = mx::array({}); ++ return std::make_unique(array, shape, memory_space); ++} ++ ++absl::StatusOr GetAttributeFromBuffer( ++ xla::PjRtBuffer* buffer) { ++ // TRACE_ME; ++ if (buffer == nullptr || buffer->IsDeleted()) { ++ return xla::InvalidArgument("Buffer is null or deleted"); ++ } ++ auto mlir_buffer = dynamic_cast(buffer); ++ if (mlir_buffer == nullptr) { ++ return xla::InvalidArgument("Buffer is not a MlxPjrtBuffer"); ++ } ++ LOG(INFO) << "GetAttributeFromBuffer(" << (void*)buffer << ") -> " ++ << ToString(mlir_buffer->GetBufferAttribute()) << "\n"; ++ return mlir_buffer->GetBufferAttribute(); ++} ++ ++absl::StatusOr GetArrayFromBuffer(xla::PjRtBuffer* buffer) { ++ // TRACE_ME; ++ if (buffer == nullptr || buffer->IsDeleted()) { ++ return xla::InvalidArgument("Buffer is null or deleted"); ++ } ++ auto mlir_buffer = dynamic_cast(buffer); ++ if (mlir_buffer == nullptr) { ++ return xla::InvalidArgument("Buffer is not a MlxPjrtBuffer"); ++ } ++ ++ return mlir_buffer->GetArray(); ++} ++ ++DenseElementsAttr CloneIntoContext(DenseElementsAttr attr, ++ MLIRContext& context) { ++ Type type = mlir::parseType(mlir::debugString(attr.getType()), &context); ++ return DenseElementsAttr::getFromRawBuffer(llvm::cast(type), ++ attr.getRawData()); ++} ++ ++} // namespace mlir::stablehlo +\ No newline at end of file +diff --git a/xla/pjrt/plugin/mlx/buffer.h b/xla/pjrt/plugin/mlx/buffer.h +new file mode 100644 +index 0000000000..1740f3c779 +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/buffer.h +@@ -0,0 +1,43 @@ ++/* Copyright 2024 The OpenXLA Authors. ++Licensed under the Apache License, Version 2.0 (the "License"); ++you may not use this file except in compliance with the License. ++You may obtain a copy of the License at ++ http://www.apache.org/licenses/LICENSE-2.0 ++Unless required by applicable law or agreed to in writing, software ++distributed under the License is distributed on an "AS IS" BASIS, ++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++See the License for the specific language governing permissions and ++limitations under the License. ++==============================================================================*/ ++ ++#ifndef XLA_PJRT_PLUGIN_STABLEHLO_MLX_BUFFER_H_ ++#define XLA_PJRT_PLUGIN_STABLEHLO_MLX_BUFFER_H_ ++ ++#include ++ ++#include "mlir/IR/BuiltinAttributes.h" ++#include "mlx/mlx.h" ++#include "xla/literal.h" ++#include "xla/pjrt/pjrt_client.h" ++ ++namespace mx = mlx::core; ++ ++namespace mlir::stablehlo { ++std::unique_ptr CreateMlirBufferFromMlxArray( ++ mx::array array, xla::PjRtMemorySpace* memory_space); ++std::unique_ptr CreateMlirBufferFromLiteral( ++ const xla::LiteralSlice& literal, xla::PjRtMemorySpace* memory_space); ++std::unique_ptr CreateMlirBufferFromAttribute( ++ mx::array array, DenseElementsAttr attribute, ++ xla::PjRtMemorySpace* memory_space); ++std::unique_ptr CreateMlirBufferUninitialized( ++ const xla::Shape& shape, xla::PjRtMemorySpace* memory_space); ++absl::StatusOr GetAttributeFromBuffer( ++ xla::PjRtBuffer* buffer); ++absl::StatusOr GetArrayFromBuffer(xla::PjRtBuffer* buffer); ++DenseElementsAttr CloneIntoContext(DenseElementsAttr attr, ++ MLIRContext& context); ++ ++} // namespace mlir::stablehlo ++ ++#endif // XLA_PJRT_PLUGIN_STABLEHLO_MLX_BUFFER_H_ +\ No newline at end of file +diff --git a/xla/pjrt/plugin/mlx/client_c_pjrt.cc b/xla/pjrt/plugin/mlx/client_c_pjrt.cc +new file mode 100644 +index 0000000000..4ce81dcc46 +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/client_c_pjrt.cc +@@ -0,0 +1,72 @@ ++/* Copyright 2024 The OpenXLA Authors. ++Licensed under the Apache License, Version 2.0 (the "License"); ++you may not use this file except in compliance with the License. ++You may obtain a copy of the License at ++ http://www.apache.org/licenses/LICENSE-2.0 ++Unless required by applicable law or agreed to in writing, software ++distributed under the License is distributed on an "AS IS" BASIS, ++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++See the License for the specific language governing permissions and ++limitations under the License. ++==============================================================================*/ ++ ++#include "xla/pjrt/plugin/mlx/client_c_pjrt.h" ++ ++#include ++#include ++#include ++ ++#include "absl/status/status.h" ++#include "absl/strings/string_view.h" ++#include "xla/pjrt/c/pjrt_c_api.h" ++#include "xla/pjrt/c/pjrt_c_api_layouts_extension.h" ++#include "xla/pjrt/c/pjrt_c_api_wrapper_impl.h" ++#include "xla/pjrt/pjrt_client.h" ++#include "xla/pjrt/plugin/mlx/client_cpp_pjrt.h" ++ ++namespace mlir::stablehlo { ++ ++using xla::PjRtClient; ++ ++std::unique_ptr GetPluginPjRtClient() { ++ return CreateStablehloMlxPjrtClient(); ++} ++ ++// Create my client ++PJRT_Error* PJRT_StablehloMlxClient_Create(PJRT_Client_Create_Args* args) { ++ std::unique_ptr client = GetPluginPjRtClient(); ++ args->client = pjrt::CreateWrapperClient(std::move(client)); ++ // printf("Creating PJRT Client from client\n"); ++ return nullptr; ++} ++ ++PJRT_Error* PJRT_StablehloMlxExecuteContext_Create( ++ PJRT_ExecuteContext_Create_Args* args) { ++ return new PJRT_Error{absl::UnimplementedError( ++ "ExecuteContext not supported for client execution.")}; ++} ++ ++PJRT_Error* PJRT_StablehloMlxDeviceTopology_Create( ++ PJRT_TopologyDescription_Create_Args* args) { ++ return new PJRT_Error{absl::UnimplementedError( ++ "Topology not supported for client compilation.")}; ++} ++ ++} // namespace mlir::stablehlo ++ ++const PJRT_Api* GetPjrtApi() { ++ // printf("C++ Calling GetPjrtApi"); ++ static PJRT_Layouts_Extension layouts_extension = ++ pjrt::CreateLayoutsExtension(nullptr); ++ ++ static const PJRT_Api pjrt_api = pjrt::CreatePjrtApi( ++ mlir::stablehlo::PJRT_StablehloMlxClient_Create, ++ mlir::stablehlo::PJRT_StablehloMlxExecuteContext_Create, ++ mlir::stablehlo::PJRT_StablehloMlxDeviceTopology_Create, ++ pjrt::PJRT_Plugin_Initialize_NoOp, ++ reinterpret_cast(&layouts_extension), ++ pjrt::PJRT_Plugin_Attributes_Xla); ++ ++ // printf("stablehlo_mlx client called GetPjrtApi\n"); ++ return &pjrt_api; ++} +\ No newline at end of file +diff --git a/xla/pjrt/plugin/mlx/client_c_pjrt.h b/xla/pjrt/plugin/mlx/client_c_pjrt.h +new file mode 100644 +index 0000000000..cc56626953 +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/client_c_pjrt.h +@@ -0,0 +1,24 @@ ++/* Copyright 2024 The OpenXLA Authors. ++Licensed under the Apache License, Version 2.0 (the "License"); ++you may not use this file except in compliance with the License. ++You may obtain a copy of the License at ++ http://www.apache.org/licenses/LICENSE-2.0 ++Unless required by applicable law or agreed to in writing, software ++distributed under the License is distributed on an "AS IS" BASIS, ++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++See the License for the specific language governing permissions and ++limitations under the License. ++==============================================================================*/ ++ ++#ifndef XLA_PJRT_PLUGIN_STABLEHLO_MLX_CLIENT_C_PJRT_H_ ++#define XLA_PJRT_PLUGIN_STABLEHLO_MLX_CLIENT_C_PJRT_H_ ++ ++#include "xla/pjrt/c/pjrt_c_api.h" ++ ++extern "C" { ++ ++// Does not pass ownership of returned PJRT_Api* to caller. ++const PJRT_Api* GetPjrtApi(); ++} ++ ++#endif // XLA_PJRT_PLUGIN_STABLEHLO_MLX_CLIENT_C_PJRT_H_ +\ No newline at end of file +diff --git a/xla/pjrt/plugin/mlx/client_cpp_pjrt.cc b/xla/pjrt/plugin/mlx/client_cpp_pjrt.cc +new file mode 100644 +index 0000000000..93a6705d99 +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/client_cpp_pjrt.cc +@@ -0,0 +1,302 @@ ++/* Copyright 2024 The OpenXLA Authors. ++Licensed under the Apache License, Version 2.0 (the "License"); ++you may not use this file except in compliance with the License. ++You may obtain a copy of the License at ++ http://www.apache.org/licenses/LICENSE-2.0 ++Unless required by applicable law or agreed to in writing, software ++distributed under the License is distributed on an "AS IS" BASIS, ++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++See the License for the specific language governing permissions and ++limitations under the License. ++==============================================================================*/ ++ ++#include "xla/pjrt/plugin/mlx/client_cpp_pjrt.h" ++ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "absl/status/statusor.h" ++#include "absl/strings/str_format.h" ++#include "absl/strings/string_view.h" ++#include "absl/types/span.h" ++#include "mlx/mlx.h" ++#include "xla/literal.h" ++#include "xla/pjrt/host_memory_spaces.h" ++#include "xla/pjrt/pjrt_client.h" ++#include "xla/pjrt/pjrt_common.h" ++#include "xla/pjrt/pjrt_compiler.h" ++#include "xla/pjrt/plugin/mlx/buffer.h" ++#include "xla/pjrt/plugin/mlx/device.h" ++#include "xla/pjrt/plugin/mlx/executable.h" ++#include "xla/pjrt/plugin/mlx/logging.h" ++#include "xla/pjrt/plugin/mlx/utils.h" ++#include "xla/shape_util.h" ++#include "xla/util.h" ++#include "tsl/platform/fingerprint.h" ++#include "tsl/platform/statusor.h" ++ ++namespace mx = mlx::core; ++ ++#define UNIMPLEMENTED(name) \ ++ xla::Unimplemented("MlxPjrtBuffer::" #name " is not implemented") ++ ++namespace mlir::stablehlo { ++ ++const xla::PjRtPlatformId kStablehloMlxBackendId = ++ tsl::Fingerprint64(kStablehloMlxBackendName); ++ ++class StablehloMlxPjrtClient : public xla::PjRtClient { ++ public: ++ StablehloMlxPjrtClient() ++ : xla::PjRtClient(), ++ owned_devices_(), ++ devices_(), ++ owned_memory_space_(), ++ memory_space_(nullptr) { ++ // Init device and memory space. ++ TRACE_ME_MEMBER; ++ owned_devices_.push_back(GetStablehloMlxDevice(this)); ++ devices_.push_back(owned_devices_.back().get()); ++ owned_memory_space_ = std::make_unique( ++ /*id=*/0, devices_.front()); ++ memory_space_ = owned_memory_space_.get(); ++ AttachStablehloMlxMemorySpace(devices_.front(), memory_space_); ++ } ++ ++ ~StablehloMlxPjrtClient() override { ++ // mx::metal::stop_capture(); ++ }; ++ ++ absl::string_view platform_name() const override { ++ TRACE_ME_MEMBER; ++ return kStablehloMlxBackendName; ++ } ++ int process_index() const override { ++ TRACE_ME_MEMBER; ++ return 0; ++ } ++ ++ int device_count() const override { ++ TRACE_ME_MEMBER; ++ return devices_.size(); ++ } ++ ++ int addressable_device_count() const override { ++ TRACE_ME_MEMBER; ++ return devices_.size(); ++ } ++ ++ absl::Span devices() const override { ++ TRACE_ME_MEMBER; ++ return devices_; ++ } ++ absl::Span addressable_devices() const override { ++ TRACE_ME_MEMBER; ++ return devices_; ++ } ++ ++ absl::Span memory_spaces() const override { ++ TRACE_ME_MEMBER; ++ return absl::MakeSpan(&memory_space_, 1); ++ } ++ ++ // Return an ID that identifies the platform via tsl fingerprint. ++ xla::PjRtPlatformId platform_id() const override { ++ TRACE_ME_MEMBER; ++ return kStablehloMlxBackendId; ++ } ++ ++ // Returns a string containing human-readable, platform-specific version ++ // info (e.g. the CUDA version on GPU or libtpu version on Cloud TPU). ++ absl::string_view platform_version() const override { ++ TRACE_ME_MEMBER; ++ return "StableHLO MLX v0.1"; ++ } ++ ++ ///////////// ++ // Device ++ ///////////// ++ ++ // Lookup any PjRtDevice for a given PjRtDevice::id(). ++ // TODO: Should this be a base class? I.e. why doesn't the base client have ++ // a vector a device pointers? ++ absl::StatusOr LookupDevice( ++ xla::PjRtGlobalDeviceId global_device_id) const override { ++ TRACE_ME_MEMBER; ++ for (auto device : devices_) { ++ if (device->global_device_id() == global_device_id) { ++ return device; ++ } ++ } ++ // TODO: This error should be a base class method since its used in tests. ++ return xla::InvalidArgument("No matching device found for device_id %d", ++ global_device_id.value()); ++ } ++ ++ absl::StatusOr LookupAddressableDevice( ++ xla::PjRtLocalDeviceId local_device_id) const override { ++ TRACE_ME_MEMBER; ++ ++ for (auto* device : addressable_devices()) { ++ if (local_device_id == device->local_device_id()) { ++ return device; ++ } ++ } ++ return xla::InvalidArgument( ++ "No matching device found for local_device_id %d", ++ local_device_id.value()); ++ } ++ ++ absl::StatusOr GetDefaultDeviceAssignment( ++ int num_replicas, int num_partitions) const override { ++ TRACE_ME_MEMBER; ++ xla::DeviceAssignment assignment(num_replicas, num_partitions); ++ for (int64_t i = 0; i < num_replicas; ++i) { ++ for (int64_t j = 0; j < num_partitions; ++j) { ++ auto idx = (i + (j * num_replicas)) % devices_.size(); ++ assignment(i, j) = devices_[idx]->global_device_id().value(); ++ } ++ } ++ return assignment; ++ } ++ ++ ///////////////// ++ // Buffer methods ++ ///////////////// ++ ++ absl::StatusOr> CreateErrorBuffer( ++ absl::Status error, const xla::Shape& shape, ++ xla::PjRtDevice* device) { ++ // Prefer memory space implementation, device holding buffer is ++ // deprecated. ++ return CreateErrorBuffer(error, shape, ++ device->default_memory_space().value_or(nullptr)); ++ } ++ ++ absl::StatusOr> CreateErrorBuffer( ++ absl::Status error, const xla::Shape& shape, ++ xla::PjRtMemorySpace* memory) { ++ return UNIMPLEMENTED(CreateErrorBuffer); ++ } ++ ++ absl::StatusOr> CreateUninitializedBuffer( ++ const xla::Shape& shape, xla::PjRtDevice* device) { ++ TRACE_ME_MEMBER; ++ return CreateMlirBufferUninitialized( ++ shape, device->default_memory_space().value_or(nullptr)); ++ } ++ ++ absl::StatusOr> ++ CreateBuffersForAsyncHostToDevice(absl::Span shapes, ++ xla::PjRtDevice* device) { ++ TRACE_ME_MEMBER; ++ return CreateBuffersForAsyncHostToDevice( ++ shapes, device->default_memory_space().value_or(nullptr)); ++ } ++ ++ absl::StatusOr> ++ CreateBuffersForAsyncHostToDevice( ++ absl::Span shapes, ++ xla::PjRtMemorySpace* memory_space) { ++ TRACE_ME_MEMBER; ++ return UNIMPLEMENTED(CreateBuffersForAsyncHostToDevice); ++ } ++ ++ absl::StatusOr> BufferFromHostBuffer( ++ const void* data, xla::PrimitiveType type, absl::Span dims, ++ std::optional> byte_strides, ++ HostBufferSemantics host_buffer_semantics, ++ absl::AnyInvocable on_done_with_host_buffer, ++ xla::PjRtDevice* device) { ++ TRACE_ME_MEMBER; ++ return BufferFromHostBuffer( ++ data, type, dims, byte_strides, host_buffer_semantics, ++ std::move(on_done_with_host_buffer), ++ device->default_memory_space().value_or(nullptr), nullptr); ++ } ++ ++ absl::StatusOr> BufferFromHostBuffer( ++ const void* data, xla::PrimitiveType type, absl::Span dims, ++ std::optional> byte_strides, ++ HostBufferSemantics host_buffer_semantics, ++ absl::AnyInvocable on_done_with_host_buffer, ++ xla::PjRtDevice* device, const xla::Layout* device_layout) { ++ TRACE_ME_MEMBER; ++ return BufferFromHostBuffer( ++ data, type, dims, byte_strides, host_buffer_semantics, ++ std::move(on_done_with_host_buffer), ++ device->default_memory_space().value_or(nullptr), device_layout); ++ } ++ ++ absl::StatusOr> BufferFromHostBuffer( ++ const void* data, xla::PrimitiveType type, absl::Span dims, ++ std::optional> byte_strides, ++ HostBufferSemantics host_buffer_semantics, ++ absl::AnyInvocable on_done_with_host_buffer, ++ xla::PjRtMemorySpace* memory_space, ++ const xla::Layout* device_layout) { ++ TRACE_ME_MEMBER; ++ TF_ASSIGN_OR_RETURN( ++ mx::array array_buffer, ++ utils::array::fromHostBuffer(data, dims, byte_strides, type)); ++ auto buffer = CreateMlirBufferFromMlxArray(array_buffer, memory_space); ++ if (on_done_with_host_buffer) { ++ // If host is awaiting the result, must call this function. ++ std::move(on_done_with_host_buffer)(); ++ on_done_with_host_buffer = nullptr; ++ } ++ return buffer; ++ } ++ ++ absl::StatusOr> BufferFromHostLiteral( ++ const xla::LiteralSlice& literal, xla::PjRtDevice* device) { ++ TRACE_ME_MEMBER; ++ return CreateMlirBufferFromLiteral( ++ literal, device->default_memory_space().value_or(nullptr)); ++ } ++ ++ absl::StatusOr> BufferFromHostLiteral( ++ const xla::LiteralSlice& literal, ++ xla::PjRtMemorySpace* memory_space) { ++ TRACE_ME_MEMBER; ++ return CreateMlirBufferFromLiteral(literal, memory_space); ++ } ++ ++ /////////// ++ // Compile ++ /////////// ++ absl::StatusOr> Compile( ++ mlir::ModuleOp module, xla::CompileOptions options) override { ++ TRACE_ME_MEMBER; ++ return mlir::stablehlo::StablehloMlxCompile( ++ module, GetDefaultDeviceAssignment(1, devices_.size()).value(), this); ++ } ++ ++ // Compile `computation` with given `options`. ++ absl::StatusOr> Compile( ++ const xla::XlaComputation& computation, ++ xla::CompileOptions options) override { ++ TRACE_ME_MEMBER; ++ return mlir::stablehlo::StablehloMlxCompile( ++ computation.proto(), ++ GetDefaultDeviceAssignment(1, devices_.size()).value(), this); ++ } ++ ++ private: ++ std::vector> owned_devices_; ++ std::vector devices_; ++ std::unique_ptr owned_memory_space_; ++ xla::PjRtMemorySpace* memory_space_; ++}; // end class ++ ++std::unique_ptr CreateStablehloMlxPjrtClient() { ++ // mx::metal::start_capture("mlx_trace.gputrace"); ++ SetupLogLevelFromEnv(); ++ return std::make_unique(); ++} ++ ++} // namespace mlir::stablehlo +\ No newline at end of file +diff --git a/xla/pjrt/plugin/mlx/client_cpp_pjrt.h b/xla/pjrt/plugin/mlx/client_cpp_pjrt.h +new file mode 100644 +index 0000000000..8c90942f0c +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/client_cpp_pjrt.h +@@ -0,0 +1,30 @@ ++/* Copyright 2024 The OpenXLA Authors. ++Licensed under the Apache License, Version 2.0 (the "License"); ++you may not use this file except in compliance with the License. ++You may obtain a copy of the License at ++ http://www.apache.org/licenses/LICENSE-2.0 ++Unless required by applicable law or agreed to in writing, software ++distributed under the License is distributed on an "AS IS" BASIS, ++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++See the License for the specific language governing permissions and ++limitations under the License. ++==============================================================================*/ ++ ++#ifndef XLA_PJRT_PLUGIN_STABLEHLO_MLX_CLIENT_CPP_PJRT_H_ ++#define XLA_PJRT_PLUGIN_STABLEHLO_MLX_CLIENT_CPP_PJRT_H_ ++ ++#include ++ ++#include "xla/pjrt/pjrt_client.h" ++#include "xla/pjrt/pjrt_compiler.h" ++ ++namespace mlir::stablehlo { ++ ++inline constexpr char kStablehloMlxBackendName[] = "stablehlo_mlx"; ++extern const xla::PjRtPlatformId kStablehloMlxBackendId; ++ ++std::unique_ptr CreateStablehloMlxPjrtClient(); ++ ++} // namespace mlir::stablehlo ++ ++#endif // XLA_PJRT_PLUGIN_STABLEHLO_MLX_CLIENT_CPP_PJRT_H_ +\ No newline at end of file +diff --git a/xla/pjrt/plugin/mlx/device.cc b/xla/pjrt/plugin/mlx/device.cc +new file mode 100644 +index 0000000000..816ef734bf +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/device.cc +@@ -0,0 +1,213 @@ ++/* Copyright 2024 The OpenXLA Authors. ++ ++ ++Licensed under the Apache License, Version 2.0 (the "License"); ++you may not use this file except in compliance with the License. ++You may obtain a copy of the License at ++ http://www.apache.org/licenses/LICENSE-2.0 ++Unless required by applicable law or agreed to in writing, software ++distributed under the License is distributed on an "AS IS" BASIS, ++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++See the License for the specific language governing permissions and ++limitations under the License. ++==============================================================================*/ ++ ++#include "mlx/mlx.h" ++ ++#include "xla/pjrt/plugin/mlx/device.h" ++ ++#include "absl/status/statusor.h" ++#include "absl/strings/str_format.h" ++#include "absl/strings/string_view.h" ++#include "absl/types/span.h" ++#include "xla/pjrt/pjrt_client.h" ++#include "xla/pjrt/pjrt_common.h" ++#include "xla/pjrt/pjrt_compiler.h" ++#include "xla/pjrt/pjrt_device_description.h" ++#include "xla/pjrt/plugin/mlx/logging.h" ++#include "xla/util.h" ++ ++#define UNIMPLEMENTED(name) \ ++ xla::Unimplemented("StablehloMlxDevice::" #name " is not implemented") ++ ++namespace mlir::stablehlo { ++ ++using xla::LiteralSlice; ++using xla::MutableBorrowingLiteral; ++using xla::PjRtClient; ++using xla::PjRtDevice; ++using xla::PjRtDeviceAttribute; ++using xla::PjRtDeviceDescription; ++using xla::PjRtGlobalDeviceId; ++using xla::PjRtLocalDeviceId; ++using xla::PjRtLocalHardwareId; ++using xla::PjRtMemorySpace; ++using xla::ScopedAsyncTrackingEvent; ++using xla::Unimplemented; ++ ++namespace mx = mlx::core; ++ ++inline std::string mlxDeviceToString(const mx::Device& device) { ++ std::ostringstream os; ++ os << device; ++ return os.str(); ++} ++// Devices need a device description. ++class StablehloMlxDeviceDescription final : public PjRtDeviceDescription { ++ public: ++ explicit StablehloMlxDeviceDescription(int process_id, int local_device_id) ++ : id_(local_device_id), ++ process_index_(process_id), ++ local_hardware_id_(local_device_id), ++ debug_string_(mlxDeviceToString(mx::default_device())), ++ to_string_(absl::StrFormat("%s(id=%d,pid=%d)", debug_string_, ++ id_.value(), process_index_)) { ++ TRACE_ME_MEMBER; ++ } ++ ++ int id() const override { ++ TRACE_ME_MEMBER; ++ return id_.value(); ++ } ++ int process_index() const override { ++ TRACE_ME_MEMBER; ++ return process_index_; ++ } ++ int local_hardware_id() const { ++ TRACE_ME_MEMBER; ++ return local_hardware_id_; ++ } ++ ++ absl::string_view device_kind() const override { ++ TRACE_ME_MEMBER; ++ return DebugString(); ++ } ++ ++ absl::string_view DebugString() const override { ++ TRACE_ME_MEMBER; ++ return debug_string_; ++ } ++ ++ absl::string_view ToString() const override { ++ TRACE_ME_MEMBER; ++ return to_string_; ++ } ++ ++ const absl::flat_hash_map& Attributes() ++ const override { ++ TRACE_ME_MEMBER; ++ return attributes_; ++ } ++ ++ private: ++ PjRtGlobalDeviceId id_; ++ int process_index_; ++ int local_hardware_id_; ++ std::string debug_string_; ++ std::string to_string_; ++ absl::flat_hash_map attributes_ = {}; ++}; ++ ++// Clients need devices, and clients own the devices. ++class StablehloMlxDevice : public PjRtDevice { ++ public: ++ explicit StablehloMlxDevice(PjRtClient* client) ++ : PjRtDevice(), client_(client), description_(0, 0) { ++ TRACE_ME_MEMBER; ++ } ++ ++ absl::string_view DebugString() const override { ++ TRACE_ME_MEMBER; ++ return "StablehloMlxDevice"; ++ } ++ ++ PjRtLocalDeviceId local_device_id() const override { ++ TRACE_ME_MEMBER; ++ return PjRtLocalDeviceId(local_hardware_id().value()); ++ } ++ ++ PjRtLocalHardwareId local_hardware_id() const override { ++ TRACE_ME_MEMBER; ++ return PjRtLocalHardwareId(description_.local_hardware_id()); ++ } ++ ++ PjRtClient* client() const override { ++ TRACE_ME_MEMBER; ++ return client_; ++ } ++ ++ bool IsAddressable() const override { ++ TRACE_ME_MEMBER; ++ return process_index() == client()->process_index(); ++ } ++ ++ absl::Status TransferToInfeed(const LiteralSlice& literal) override { ++ TRACE_ME_MEMBER; ++ return UNIMPLEMENTED(TransferToInfeed); ++ } ++ ++ absl::Status TransferFromOutfeed(MutableBorrowingLiteral literal) override { ++ TRACE_ME_MEMBER; ++ return UNIMPLEMENTED(TransferFromOutfeed); ++ } ++ ++ void AttachDefaultMemorySpace(PjRtMemorySpace* memory_space) { ++ TRACE_ME_MEMBER; ++ memory_space_ = memory_space; ++ } ++ ++ absl::Span memory_spaces() const override { ++ TRACE_ME_MEMBER; ++ return absl::MakeSpan(&memory_space_, 1); ++ } ++ ++ absl::StatusOr default_memory_space() const override { ++ TRACE_ME_MEMBER; ++ if (!memory_space_) ++ return absl::InternalError("Plugin memory space unset."); ++ ++ return memory_space_; ++ } ++ ++ std::unique_ptr CreateAsyncTrackingEvent( ++ absl::string_view description) const override { ++ TRACE_ME_MEMBER; ++ LOG(FATAL) << "Plugin does not implement CreateAsyncTrackingEvent."; ++ return nullptr; ++ } ++ ++ const PjRtDeviceDescription& description() const override { ++ TRACE_ME_MEMBER; ++ return description_; ++ } ++ ++ private: ++ PjRtClient* client_; ++ PjRtMemorySpace* memory_space_; // unpinned memory owned by client ++ StablehloMlxDeviceDescription description_; ++}; ++ ++// Device Description ++std::unique_ptr GetStablehloMlxDeviceDescription( ++ int process_id, int local_device_id) { ++ return std::make_unique(process_id, ++ local_device_id); ++} ++ ++// Mlx Device ++std::unique_ptr GetStablehloMlxDevice(PjRtClient* client) { ++ return std::make_unique(client); ++} ++ ++void AttachStablehloMlxMemorySpace(PjRtDevice* device, ++ PjRtMemorySpace* memory_space) { ++ auto stablehlo_device = dynamic_cast(device); ++ if (stablehlo_device == nullptr) { ++ LOG(FATAL) << "Plugin cannot attach memory space to device of kind " ++ << device->device_kind(); ++ return; ++ } ++ stablehlo_device->AttachDefaultMemorySpace(memory_space); ++} ++ ++} // namespace mlir::stablehlo +\ No newline at end of file +diff --git a/xla/pjrt/plugin/mlx/device.h b/xla/pjrt/plugin/mlx/device.h +new file mode 100644 +index 0000000000..ef42968a3a +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/device.h +@@ -0,0 +1,36 @@ ++/* Copyright 2024 The OpenXLA Authors. ++Licensed under the Apache License, Version 2.0 (the "License"); ++you may not use this file except in compliance with the License. ++You may obtain a copy of the License at ++ http://www.apache.org/licenses/LICENSE-2.0 ++Unless required by applicable law or agreed to in writing, software ++distributed under the License is distributed on an "AS IS" BASIS, ++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++See the License for the specific language governing permissions and ++limitations under the License. ++==============================================================================*/ ++ ++#ifndef XLA_PJRT_PLUGIN_STABLEHLO_MLX_DEVICE_H_ ++#define XLA_PJRT_PLUGIN_STABLEHLO_MLX_DEVICE_H_ ++ ++#include ++ ++#include "xla/pjrt/pjrt_client.h" ++#include "xla/pjrt/pjrt_compiler.h" ++#include "xla/pjrt/pjrt_device_description.h" ++ ++namespace mlir::stablehlo { ++ ++// Device Description ++std::unique_ptr GetStablehloMlxDeviceDescription( ++ int process_id, int local_device_id); ++ ++// MLX Device ++std::unique_ptr GetStablehloMlxDevice(xla::PjRtClient* client); ++ ++void AttachStablehloMlxMemorySpace(xla::PjRtDevice* device, ++ xla::PjRtMemorySpace* memory_space); ++ ++} // namespace mlir::stablehlo ++ ++#endif // XLA_PJRT_PLUGIN_STABLEHLO_MLX_DEVICE_H_ +\ No newline at end of file +diff --git a/xla/pjrt/plugin/mlx/executable.cc b/xla/pjrt/plugin/mlx/executable.cc +new file mode 100644 +index 0000000000..8823b98305 +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/executable.cc +@@ -0,0 +1,2431 @@ ++/* Copyright 2024 The OpenXLA Authors. ++Licensed under the Apache License, Version 2.0 (the "License"); ++you may not use this file except in compliance with the License. ++You may obtain a copy of the License at ++ http://www.apache.org/licenses/LICENSE-2.0 ++Unless required by applicable law or agreed to in writing, software ++distributed under the License is distributed on an "AS IS" BASIS, ++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++See the License for the specific language governing permissions and ++limitations under the License. ++==============================================================================*/ ++ ++#include "xla/pjrt/plugin/mlx/executable.h" ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++// TODO(@cryptodeal): might need to update `BUILD` ++#include "mlir/IR/Visitors.h" ++#include "llvm/ADT/TypeSwitch.h" ++#include "llvm/ADT/Hashing.h" ++ ++#include "mlx/mlx.h" ++#include "absl/log/log.h" ++#include "absl/status/status.h" ++#include "absl/status/statusor.h" ++#include "mlir/Bytecode/BytecodeWriter.h" ++#include "mlir/Dialect/Func/IR/FuncOps.h" ++#include "mlir/IR/AsmState.h" ++#include "mlir/IR/BuiltinAttributes.h" ++#include "mlir/IR/MLIRContext.h" ++#include "mlir/IR/Operation.h" ++#include "mlir/IR/OwningOpRef.h" ++#include "mlir/Interfaces/DataLayoutInterfaces.h" ++#include "mlir/Pass/PassManager.h" ++#include "mlir/Support/LLVM.h" ++#include "stablehlo/reference/Api.h" ++#include "stablehlo/transforms/optimization/Passes.h" ++#include "xla/hlo/translate/stablehlo.h" ++#include "xla/mlir/utils/error_util.h" ++#include "xla/mlir/utils/type_util.h" ++#include "xla/mlir_hlo/mhlo/transforms/passes.h" ++#include "xla/mlir_hlo/stablehlo_ext/transforms/passes.h" ++#include "xla/pjrt/pjrt_client.h" ++#include "xla/pjrt/pjrt_executable.h" ++#include "xla/pjrt/pjrt_future.h" ++#include "xla/pjrt/plugin/mlx/buffer.h" ++#include "xla/pjrt/plugin/mlx/logging.h" ++#include "xla/pjrt/plugin/mlx/utils.h" ++#include "xla/service/computation_placer.h" ++#include "tsl/platform/status.h" ++#include "tsl/platform/statusor.h" ++ ++#define DEBUG_TYPE "stablehlo-pjrt" ++ ++namespace mx = mlx::core; ++ ++typedef absl::StatusOr StatusOrArray; ++typedef absl::StatusOr> StatusOrArrays; ++typedef absl::StatusOr StatusOrInt; ++ ++// ZML supports up to 8 dimensions ++auto index_space(const std::vector& dims) { ++ using namespace ranges; ++ // `MAX_RANK` for zml is 8 ++ std::vector used_dims(8, 1); ++ std::copy(dims.begin(), dims.end(), used_dims.begin()); ++ return views::cartesian_product( ++ views::indices(used_dims[0]), views::indices(used_dims[1]), ++ views::indices(used_dims[2]), views::indices(used_dims[3]), ++ views::indices(used_dims[4]), views::indices(used_dims[5]), ++ views::indices(used_dims[6]), views::indices(used_dims[7])); ++} ++ ++template >>> ++std::vector to_vector(Tuple&& tuple) { ++ return std::apply( ++ [](auto&&... elems) { ++ return std::vector{std::forward(elems)...}; ++ }, ++ std::forward(tuple)); ++} ++ ++template >>> ++std::vector to_vector(Tuple&& tuple, size_t size) { ++ std::vector result = to_vector(tuple); ++ result.resize(size); ++ return result; ++} ++ ++namespace mlir::stablehlo { ++ ++#define UNIMPLEMENTED(name) \ ++ xla::Unimplemented("MlxPjRtBuffer::" #name " is not implemented") ++ ++using xla::DeviceAssignment; ++using xla::PjRtBuffer; ++using xla::PjRtClient; ++using xla::PjRtDevice; ++using xla::PjRtFuture; ++using xla::PjRtLoadedExecutable; ++using xla::PjRtMemorySpace; ++ ++void resolveFuncDeps(ModuleOp mod, std::string func_name, Tree& dep_tree, ++ Node* parent = nullptr) { ++ auto new_node = dep_tree.insert(func_name, parent); ++ auto func = mod.lookupSymbol(func_name); ++ auto& block = func.front(); ++ std::set func_names; ++ for (auto op : block.getOps()) { ++ std::string callee_name = op.getCallee().str(); ++ func_names.insert(callee_name); ++ resolveFuncDeps(mod, callee_name, dep_tree, new_node); ++ } ++} ++ ++absl::Status validateModule(ModuleOp mod) { ++ absl::Status status; ++ // TODO: some ops should have additional checks (e.g. `stablehlo::ReduceOp`, ++ // `stablehlo::SortOp`, etc.) where we need additional checks at compile time ++ // to ensure that we're correctly mapping to the corresponding MLX op. ++ mod.walk([&status](Operation* op) { ++ // Series of checks to ensure we support the module ++ return llvm::TypeSwitch(op) ++ .Case([&status, &op](auto o) { ++ // Validate all input types are compatible with mlx ++ for (auto operand : o->getOperands()) { ++ auto primitive_type = xla::ConvertMlirTypeToPrimitiveType( ++ mlir::cast(operand.getType()).getElementType()); ++ switch (primitive_type) { ++ case xla::PrimitiveType::PRED: ++ case xla::PrimitiveType::U8: ++ case xla::PrimitiveType::S8: ++ case xla::PrimitiveType::U16: ++ case xla::PrimitiveType::S16: ++ case xla::PrimitiveType::U32: ++ case xla::PrimitiveType::S32: ++ case xla::PrimitiveType::U64: ++ case xla::PrimitiveType::S64: ++ case xla::PrimitiveType::F16: ++ case xla::PrimitiveType::BF16: ++ case xla::PrimitiveType::F32: ++ case xla::PrimitiveType::C64: ++ break; ++ default: { ++ std::cout << "Unsupported type: " ++ << xla::PrimitiveType_Name(primitive_type).c_str() ++ << std::endl; ++ status = absl::UnimplementedError( ++ absl::StrCat("Unsupported type: ", ToString(op))); ++ return WalkResult::interrupt(); ++ } ++ } ++ } ++ ++ // Validate all result types are compatible with mlx ++ for (auto result : o->getResults()) { ++ auto primitive_type = xla::ConvertMlirTypeToPrimitiveType( ++ mlir::cast(result.getType()).getElementType()); ++ switch (primitive_type) { ++ case xla::PrimitiveType::PRED: ++ case xla::PrimitiveType::U8: ++ case xla::PrimitiveType::S8: ++ case xla::PrimitiveType::U16: ++ case xla::PrimitiveType::S16: ++ case xla::PrimitiveType::U32: ++ case xla::PrimitiveType::S32: ++ case xla::PrimitiveType::U64: ++ case xla::PrimitiveType::S64: ++ case xla::PrimitiveType::F16: ++ case xla::PrimitiveType::BF16: ++ case xla::PrimitiveType::F32: ++ case xla::PrimitiveType::C64: ++ break; ++ default: { ++ std::cout << "Unsupported type: " ++ << xla::PrimitiveType_Name(primitive_type).c_str() ++ << std::endl; ++ status = absl::UnimplementedError( ++ absl::StrCat("Unsupported type: ", ToString(op))); ++ return WalkResult::interrupt(); ++ } ++ } ++ } ++ return mlir::WalkResult::advance(); ++ }) ++ // TODO(@cryptodeal): These ops were supported prior to `mlx::compile` ++ // need to modify the implementation to ensure no use of `mlx::eval` and ++ // ensure that `mlx::random::key` is being propogated correctly. ++ .Case( ++ [&status, &op](auto o) { ++ status = absl::UnimplementedError( ++ absl::StrCat("Unsupported op: ", ToString(op))); ++ return WalkResult::interrupt(); ++ }) ++ .Default([&status, &op](auto o) { ++ status = absl::UnimplementedError( ++ absl::StrCat("Unsupported op: ", ToString(op))); ++ return WalkResult::interrupt(); ++ }); ++ }); ++ return status; ++} ++ ++mlir::OwningOpRef cloneIntoContext(ModuleOp module, ++ MLIRContext& context) { ++ // Clone the module into the context. MHLO->StableHLO just in case. ++ PassManager pm(module->getContext()); ++ pm.addPass(mhlo::createHloLegalizeToStablehloPass()); ++ if (failed(pm.run(module))) { ++ LOG(ERROR) << "Failed to convert MHLO to StableHLO"; ++ return nullptr; ++ } ++ ++ std::string bytecode; ++ llvm::raw_string_ostream os(bytecode); ++ mlir::BytecodeWriterConfig config; ++ mlir::OwningOpRef cloned = module.clone(); ++ if (mlir::failed(mlir::writeBytecodeToFile(*cloned, os, config))) { ++ LOG(ERROR) << "Failed to write bytecode to string\n"; ++ return nullptr; ++ } ++ return *parseStablehloModule(bytecode, context); ++} ++ ++LogicalResult decomposeChloToStablehlo(ModuleOp module) { ++ PassManager pm(module->getContext()); ++ stablehlo_ext::createChloLegalizeToStablehloPipeline(pm); ++ if (failed(pm.run(module))) { ++ return module->emitError() << "Failed to recompose CHLO"; ++ } ++ return success(); ++} ++ ++std::optional getUnsupportedOp(ModuleOp module) { ++ std::optional unsupported_op(std::nullopt); ++ module.walk([&unsupported_op](Operation* op) { ++ auto cc = llvm::dyn_cast(op); ++ if (cc) { ++ unsupported_op = op; ++ return WalkResult::interrupt(); ++ } ++ return WalkResult::advance(); ++ }); ++ return unsupported_op; ++} ++ ++LogicalResult runHardwareIndependentOptimizations(ModuleOp module) { ++ PassManager pm(module->getContext()); ++ pm.addNestedPass( ++ stablehlo::createStablehloAggressiveFolderPass()); ++ pm.addNestedPass( ++ stablehlo::createStablehloAggressiveSimplificationPass()); ++ if (failed(pm.run(module))) { ++ return module->emitError() ++ << "Failed to run hardware independent optimizations"; ++ } ++ return success(); ++} ++ ++mlir::ShapedType getResultType(ModuleOp module, mx::Dtype dtype, ++ SmallVector shape) { ++ switch (dtype) { ++ case mx::bfloat16: ++ return RankedTensorType::get(ArrayRef(shape), ++ BFloat16Type::get(module.getContext())); ++ case mx::float16: ++ return RankedTensorType::get(ArrayRef(shape), ++ Float16Type::get(module.getContext())); ++ case mx::int8: ++ return RankedTensorType::get( ++ ArrayRef(shape), ++ IntegerType::get(module.getContext(), 8, ++ mlir::IntegerType::Signless)); ++ case mx::uint8: ++ return RankedTensorType::get( ++ ArrayRef(shape), ++ IntegerType::get(module.getContext(), 8, ++ mlir::IntegerType::Unsigned)); ++ case mx::int16: ++ return RankedTensorType::get( ++ ArrayRef(shape), ++ IntegerType::get(module.getContext(), 16, ++ mlir::IntegerType::Signless)); ++ case mx::uint16: ++ return RankedTensorType::get( ++ ArrayRef(shape), ++ IntegerType::get(module.getContext(), 16, ++ mlir::IntegerType::Unsigned)); ++ case mx::int32: ++ return RankedTensorType::get( ++ ArrayRef(shape), ++ IntegerType::get(module.getContext(), 32, ++ mlir::IntegerType::Signless)); ++ case mx::uint32: ++ return RankedTensorType::get( ++ ArrayRef(shape), ++ IntegerType::get(module.getContext(), 32, ++ mlir::IntegerType::Unsigned)); ++ case mx::int64: ++ return RankedTensorType::get( ++ ArrayRef(shape), ++ IntegerType::get(module.getContext(), 64, ++ mlir::IntegerType::Signless)); ++ case mx::uint64: ++ return RankedTensorType::get( ++ ArrayRef(shape), ++ IntegerType::get(module.getContext(), 64, ++ mlir::IntegerType::Unsigned)); ++ case mx::complex64: ++ return RankedTensorType::get( ++ ArrayRef(shape), ++ ComplexType::get(Float32Type::get(module.getContext()))); ++ default: ++ return RankedTensorType::get(ArrayRef(shape), ++ Float32Type::get(module.getContext())); ++ } ++} ++ ++// Returns true if the given `ReduceOp` matches ++// `zml.Tensor.argMax` op implementation. ++bool isArgMaxReduce(stablehlo::ReduceOp& reduce_op) { ++ Block& body_block = reduce_op.getBody().front(); ++ unsigned op_count = 0; ++ // TODO (@cryptodeal): This is a lazy verification; maybe check operands ++ // for each operation in the block. ++ for (auto& op : body_block.getOperations()) { ++ switch (op_count) { ++ case 0: { ++ auto compare_op = llvm::dyn_cast(op); ++ if (compare_op && ++ compare_op.getComparisonDirection() == ComparisonDirection::GT) { ++ break; ++ } else ++ return false; ++ } ++ case 1: { ++ auto compare_op = llvm::dyn_cast(op); ++ if (compare_op && ++ compare_op.getComparisonDirection() == ComparisonDirection::NE) { ++ break; ++ } else ++ return false; ++ } ++ case 2: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ case 3: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ case 4: { ++ auto compare_op = llvm::dyn_cast(op); ++ if (compare_op && ++ compare_op.getComparisonDirection() == ComparisonDirection::EQ) { ++ break; ++ } else ++ return false; ++ } ++ case 5: { ++ auto compare_op = llvm::dyn_cast(op); ++ if (compare_op && ++ compare_op.getComparisonDirection() == ComparisonDirection::LT) { ++ break; ++ } else ++ return false; ++ } ++ case 6: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ case 7: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ case 8: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ case 9: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ default: ++ return false; ++ } ++ op_count++; ++ } ++ return true; ++} ++ ++// Returns true if the given `ReduceOp` matches ++// `zml.Tensor.sum` op implementation. ++bool isSumReduce(stablehlo::ReduceOp& reduce_op) { ++ Block& body_block = reduce_op.getBody().front(); ++ unsigned op_count = 0; ++ // TODO (@cryptodeal): This is a lazy verification; maybe check operands ++ // for each operation in the block. ++ for (auto& op : body_block.getOperations()) { ++ switch (op_count) { ++ case 0: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ case 1: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ } ++ op_count++; ++ } ++ return true; ++} ++ ++// Returns true if the given `ReduceOp` matches ++// `zml.Tensor.max` op implementation. ++bool isMaxReduce(stablehlo::ReduceOp& reduce_op) { ++ Block& body_block = reduce_op.getBody().front(); ++ unsigned op_count = 0; ++ // TODO (@cryptodeal): This is a lazy verification; maybe check operands ++ // for each operation in the block. ++ for (auto& op : body_block.getOperations()) { ++ switch (op_count) { ++ case 0: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ case 1: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ } ++ op_count++; ++ } ++ return true; ++} ++ ++// Returns true if the given `ReduceOp` matches ++// `zml.Tensor.max` op implementation. ++bool isMinReduce(stablehlo::ReduceOp& reduce_op) { ++ Block& body_block = reduce_op.getBody().front(); ++ unsigned op_count = 0; ++ // TODO (@cryptodeal): This is a lazy verification; maybe check operands ++ // for each operation in the block. ++ for (auto& op : body_block.getOperations()) { ++ switch (op_count) { ++ case 0: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ case 1: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ } ++ op_count++; ++ } ++ return true; ++} ++ ++// Returns true if the given `ReduceOp` matches ++// `zml.Tensor.any` op implementation. ++bool isAnyReduce(stablehlo::ReduceOp& reduce_op) { ++ Block& body_block = reduce_op.getBody().front(); ++ unsigned op_count = 0; ++ // TODO (@cryptodeal): This is a lazy verification; maybe check operands ++ // for each operation in the block. ++ for (auto& op : body_block.getOperations()) { ++ switch (op_count) { ++ case 0: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ case 1: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ } ++ op_count++; ++ } ++ return true; ++} ++ ++absl::StatusOr getScatterType(mlir::Block& block) { ++ unsigned op_count = 0; ++ utils::ScatterType scatter_type = utils::ScatterType::Replace; ++ for (const auto& op : block.getOperations()) { ++ switch (op_count) { ++ case 0: ++ if (llvm::dyn_cast(op)) { ++ return scatter_type; ++ } ++ if (llvm::dyn_cast(op)) { ++ scatter_type = utils::ScatterType::Add; ++ } else if (llvm::dyn_cast(op)) { ++ scatter_type = utils::ScatterType::Prod; ++ } else if (llvm::dyn_cast(op)) { ++ scatter_type = utils::ScatterType::Prod; ++ } else if (llvm::dyn_cast(op)) { ++ scatter_type = utils::ScatterType::Max; ++ } else if (llvm::dyn_cast(op)) { ++ scatter_type = utils::ScatterType::Min; ++ } else { ++ return xla::Internal("Unsupported comparator: %s", ToString(block)); ++ } ++ break; ++ case 1: ++ if (llvm::dyn_cast(op)) { ++ return scatter_type; ++ } ++ default: ++ return xla::Internal("Unsupported comparator: %s", ToString(block)); ++ } ++ } ++} ++ ++absl::StatusOr> getSortInfo( ++ mlir::Block& block) { ++ unsigned op_count = 0; ++ mx::ComparatorType comparator = mx::ComparatorType::LessThan; ++ unsigned input_idx = 0; ++ for (const auto& op : block.getOperations()) { ++ switch (op_count) { ++ case 0: { ++ if (auto compare_op = llvm::dyn_cast(op)) { ++ switch (compare_op.getComparisonDirection()) { ++ case ComparisonDirection::GT: { ++ auto lhs = llvm::dyn_cast(compare_op.getLhs()); ++ auto rhs = llvm::dyn_cast(compare_op.getRhs()); ++ if (lhs && rhs) { ++ unsigned lhs_idx = static_cast(lhs.getArgNumber()); ++ unsigned rhs_idx = static_cast(rhs.getArgNumber()); ++ if (rhs_idx - 1 == lhs_idx) { ++ comparator = mx::ComparatorType::GreaterThan; ++ input_idx = lhs_idx == 0 ? 0 : lhs_idx / 2; ++ break; ++ } ++ } ++ return xla::Internal("Unsupported comparator: %s", ++ ToString(block)); ++ } ++ case ComparisonDirection::LT: { ++ auto lhs = llvm::dyn_cast(compare_op.getLhs()); ++ auto rhs = llvm::dyn_cast(compare_op.getRhs()); ++ if (lhs && rhs) { ++ unsigned lhs_idx = static_cast(lhs.getArgNumber()); ++ unsigned rhs_idx = static_cast(rhs.getArgNumber()); ++ if (rhs_idx - 1 == lhs_idx) { ++ comparator = mx::ComparatorType::LessThan; ++ input_idx = lhs_idx == 0 ? 0 : lhs_idx / 2; ++ break; ++ } ++ } ++ return xla::Internal("Unsupported comparator: %s", ++ ToString(block)); ++ } ++ default: ++ return xla::Internal("Unsupported comparator: %s", ++ ToString(block)); ++ } ++ break; ++ } ++ } ++ case 1: ++ if (llvm::dyn_cast(op)) break; ++ default: ++ return xla::Internal("Unsupported comparator: %s", ToString(block)); ++ } ++ op_count++; ++ } ++ return std::make_pair(input_idx, comparator); ++} ++ ++mx::array getOperandArray( ++ const Value& operand, ++ const std::unordered_map>& block_ctx, ++ const std::vector& inputs) { ++ // check if operand is the result of a previous operation ++ if (auto defining_op = operand.getDefiningOp()) { ++ if (auto search = block_ctx.find(defining_op); search != block_ctx.end()) { ++ for (auto i = 0; i < defining_op->getNumResults(); i++) { ++ Value maybe_res = defining_op->getResult(i); ++ if (maybe_res == operand) { ++ return search->second[i]; ++ } ++ } ++ } ++ } else { ++ return inputs[operand.cast().getArgNumber()]; ++ } ++} ++ ++std::pair getValueInfo(Value v) { ++ auto val_type = mlir::cast(v.getType()); ++ return std::make_pair(*utils::dtype::fromMlirType(val_type.getElementType()), ++ std::vector(val_type.getShape().begin(), ++ val_type.getShape().end())); ++} ++ ++std::pair getTypeInfo(Type v) { ++ auto val_type = mlir::cast(v); ++ return std::make_pair(*utils::dtype::fromMlirType(val_type.getElementType()), ++ std::vector(val_type.getShape().begin(), ++ val_type.getShape().end())); ++} ++ ++CallOpInfo getCallOpInfo(ModuleOp mod, func::FuncOp o) { ++ auto func_type = o.getFunctionType(); ++ ++ std::vector> input_types; ++ for (auto arg : func_type.getInputs()) { ++ input_types.emplace_back(getTypeInfo(arg)); ++ } ++ std::vector> result_types; ++ for (auto result : func_type.getResults()) { ++ result_types.emplace_back(getTypeInfo(result)); ++ } ++ return CallOpInfo(input_types, result_types, o.getName().str()); ++} ++ ++CallOpInfo getCallOpInfo(ModuleOp mod, func::CallOp o) { ++ std::vector> input_types; ++ for (auto arg : o.getOperands()) { ++ input_types.emplace_back(getValueInfo(arg)); ++ } ++ std::vector> result_types; ++ for (auto result : o.getResults()) { ++ result_types.emplace_back(getValueInfo(result)); ++ } ++ auto func = mod.lookupSymbol(o.getCallee()); ++ return CallOpInfo(input_types, result_types, func.getName().str()); ++} ++ ++IotaOpInfo getIotaOpInfo(stablehlo::IotaOp o) { ++ std::pair result_info = getValueInfo(o.getResult()); ++ auto iota_dim = o.getIotaDimension(); ++ std::vector dimensions; ++ for (auto i = 0; i < dimensions.size(); i++) { ++ dimensions.push_back(i != iota_dim ? 1 : result_info.second[i]); ++ } ++ return IotaOpInfo(std::vector>{}, ++ {result_info}, dimensions, result_info.first, iota_dim, ++ result_info.second); ++} ++ ++CbrtOpInfo getCbrtOpInfo(stablehlo::CbrtOp o) { ++ return CbrtOpInfo({getValueInfo(o.getOperand())}, ++ {getValueInfo(o.getResult())}); ++} ++ ++BroadcastInDimOpInfo getBroadcastInDimOpInfo(stablehlo::BroadcastInDimOp o) { ++ auto result_info = getValueInfo(o.getResult()); ++ return BroadcastInDimOpInfo( ++ {getValueInfo(o.getOperand())}, {result_info}, ++ std::vector(o.getBroadcastDimensions().begin(), ++ o.getBroadcastDimensions().end()), ++ result_info.second); ++} ++ ++DotGeneralOpInfo getDotGeneralOpInfo(stablehlo::DotGeneralOp o) { ++ auto dot_dim_nums = o.getDotDimensionNumbers(); ++ std::vector lhs_batch_dims( ++ dot_dim_nums.getLhsBatchingDimensions().begin(), ++ dot_dim_nums.getLhsBatchingDimensions().end()); ++ std::vector rhs_batch_dims( ++ dot_dim_nums.getRhsBatchingDimensions().begin(), ++ dot_dim_nums.getRhsBatchingDimensions().end()); ++ std::vector lhs_contract_dims( ++ dot_dim_nums.getLhsContractingDimensions().begin(), ++ dot_dim_nums.getLhsContractingDimensions().end()); ++ std::vector rhs_contract_dims( ++ dot_dim_nums.getRhsContractingDimensions().begin(), ++ dot_dim_nums.getRhsContractingDimensions().end()); ++ return DotGeneralOpInfo({getValueInfo(o.getLhs()), getValueInfo(o.getRhs())}, ++ {getValueInfo(o.getResult())}, lhs_batch_dims, ++ rhs_batch_dims, lhs_contract_dims, rhs_contract_dims); ++} ++ ++GatherOpInfo getGatherOpInfo(stablehlo::GatherOp o) { ++ std::vector> operand_info; ++ for (auto operand : o.getOperands()) { ++ operand_info.emplace_back(getValueInfo(operand)); ++ } ++ operand_info.emplace_back(getValueInfo(o.getStartIndices())); ++ auto result_info = getValueInfo(o.getResult()); ++ auto dim_nums = o.getDimensionNumbers(); ++ std::vector offset_dims( ++ o.getDimensionNumbers().getOffsetDims().begin(), ++ o.getDimensionNumbers().getOffsetDims().end()); ++ std::vector collapsed_slice_dims( ++ o.getDimensionNumbers().getCollapsedSliceDims().begin(), ++ o.getDimensionNumbers().getCollapsedSliceDims().end()); ++ std::vector operand_batching_dims( ++ o.getDimensionNumbers().getOperandBatchingDims().begin(), ++ o.getDimensionNumbers().getOperandBatchingDims().end()); ++ std::vector start_indices_batching_dims( ++ o.getDimensionNumbers().getStartIndicesBatchingDims().begin(), ++ o.getDimensionNumbers().getStartIndicesBatchingDims().end()); ++ std::vector start_index_map( ++ o.getDimensionNumbers().getStartIndexMap().begin(), ++ o.getDimensionNumbers().getStartIndexMap().end()); ++ auto index_vector_dim = o.getDimensionNumbers().getIndexVectorDim(); ++ std::vector slice_sizes(o.getSliceSizes().begin(), ++ o.getSliceSizes().end()); ++ return GatherOpInfo(operand_info, {getValueInfo(o.getResult())}, ++ collapsed_slice_dims, index_vector_dim, offset_dims, ++ operand_batching_dims, result_info.second, slice_sizes, ++ start_index_map, start_indices_batching_dims); ++} ++ ++ScatterOpInfo getScatterOpInfo(stablehlo::ScatterOp o) { ++ std::vector> inputs_info; ++ for (auto input : o.getInputs()) { ++ inputs_info.emplace_back(getValueInfo(input)); ++ } ++ for (auto update : o.getUpdates()) { ++ inputs_info.emplace_back(getValueInfo(update)); ++ } ++ inputs_info.emplace_back(getValueInfo(o.getScatterIndices())); ++ std::vector>> results_info; ++ for (auto result : o.getResults()) { ++ results_info.emplace_back(getValueInfo(result)); ++ } ++ // Get scatter dimension numbers ++ auto scatter_dim_nums = o.getScatterDimensionNumbers(); ++ std::vector update_window_dims( ++ scatter_dim_nums.getUpdateWindowDims().begin(), ++ scatter_dim_nums.getUpdateWindowDims().end()); ++ std::vector inserted_window_dims( ++ scatter_dim_nums.getInsertedWindowDims().begin(), ++ scatter_dim_nums.getInsertedWindowDims().end()); ++ std::vector input_batching_dims( ++ scatter_dim_nums.getInputBatchingDims().begin(), ++ scatter_dim_nums.getInputBatchingDims().end()); ++ ++ std::vector scatter_indices_batching_dims( ++ scatter_dim_nums.getScatterIndicesBatchingDims().begin(), ++ scatter_dim_nums.getScatterIndicesBatchingDims().end()); ++ ++ std::vector scatter_dims_to_operand_dims( ++ scatter_dim_nums.getScatterDimsToOperandDims().begin(), ++ scatter_dim_nums.getScatterDimsToOperandDims().end()); ++ ++ auto index_vector_dim = scatter_dim_nums.getIndexVectorDim(); ++ // Get update computation ++ mlir::Block& update_computation = o.getUpdateComputation().front(); ++ ++ // TODO(@cryptodeal): need to add a check in `Compile` ++ utils::ScatterType scatter_type = *getScatterType(update_computation); ++ return ScatterOpInfo( ++ inputs_info, results_info, index_vector_dim, input_batching_dims, ++ inserted_window_dims, scatter_dims_to_operand_dims, ++ scatter_indices_batching_dims, scatter_type, update_window_dims); ++} ++ ++PadOpInfo getPadOpInfo(stablehlo::PadOp o) { ++ auto result_info = getValueInfo(o.getResult()); ++ std::vector edge_pad_low(o.getEdgePaddingLow().begin(), ++ o.getEdgePaddingLow().end()); ++ std::vector interior_pad(o.getInteriorPadding().begin(), ++ o.getInteriorPadding().end()); ++ return PadOpInfo( ++ {getValueInfo(o.getOperand()), getValueInfo(o.getPaddingValue())}, ++ {result_info}, result_info.second, edge_pad_low, interior_pad); ++} ++ ++void compileCallOp(OpLookup& op_lookup, ModuleOp mod, func::FuncOp o) { ++ CallOpInfo call_op_info = getCallOpInfo(mod, o); ++ if (auto search = op_lookup.call.find(call_op_info); ++ search == op_lookup.call.end()) { ++ std::cout << "Compiling CallOp: " << o.getName().str().c_str() << std::endl; ++ auto& block = o.front(); ++ auto call_op_func = [&op_lookup, &block, mod, ++ call_op_info](const std::vector& inputs) { ++ std::unordered_map> block_ctx; ++ std::vector res; ++ for (Operation& op : block.getOperations()) { ++ res = ++ llvm::TypeSwitch>(&op) ++ .Case([&block_ctx, ++ &inputs](auto o) { ++ std::vector res; ++ for (Value val : o.getOperands()) { ++ res.emplace_back(getOperandArray(val, block_ctx, inputs)); ++ } ++ return res; ++ }) ++ .Case([&block_ctx, &inputs, mod, ++ &op_lookup](auto o) { ++ auto call_op_info = getCallOpInfo(mod, o); ++ std::vector operands; ++ for (Value val : o.getOperands()) { ++ operands.emplace_back( ++ getOperandArray(val, block_ctx, inputs)); ++ } ++ return op_lookup.call.find(call_op_info)->second(operands); ++ }) ++ // Handle StableHLO nullary ops ++ .Case([](auto o) { ++ return std::vector{ ++ *utils::array::fromDenseElementsAttr( ++ mlir::cast(o.getValue()))}; ++ }) ++ .Case([&op_lookup](auto o) { ++ return op_lookup.iota.find(getIotaOpInfo(o))->second({}); ++ }) ++ // .Case([](auto o) {}) ++ /* ++ .Case([](auto o) {}) ++ Deprecated see: ++ https://github.com/openxla/stablehlo/issues/2340 ++ https://github.com/openxla/stablehlo/pull/2283 ++ */ ++ // Handle StableHLO unary elementwise op ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::abs( ++ getOperandArray(o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs, ++ &op_lookup](auto o) { ++ return op_lookup.cbrt.find(getCbrtOpInfo(o)) ++ ->second( ++ {getOperandArray(o.getOperand(), block_ctx, inputs)}); ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::ceil( ++ getOperandArray(o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::astype( ++ getOperandArray(o.getOperand(), block_ctx, inputs), ++ *utils::dtype::fromMlirType( ++ o.getResult().getType().getElementType()))}; ++ }) ++ // .Case([](auto o) {}) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::cos( ++ getOperandArray(o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::exp( ++ getOperandArray(o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::expm1( ++ getOperandArray(o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::floor( ++ getOperandArray(o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::imag( ++ getOperandArray(o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::isfinite( ++ getOperandArray(o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::log( ++ getOperandArray(o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::log1p( ++ getOperandArray(o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::sigmoid( ++ getOperandArray(o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::logical_not( ++ getOperandArray(o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::negative( ++ getOperandArray(o.getOperand(), block_ctx, inputs))}; ++ }) ++ // .Case([](auto o) {}) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::real( ++ getOperandArray(o.getOperand(), block_ctx, inputs))}; ++ }) ++ // TODO(@cryptodeal): `stablehlo::RoundOp` does not match with ++ // the mlx metal implementation ++ // .Case([](auto o) {}) ++ .Case( ++ [&block_ctx, &inputs](auto o) { ++ return std::vector{mx::round( ++ getOperandArray(o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::rsqrt( ++ getOperandArray(o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::sign( ++ getOperandArray(o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::sin( ++ getOperandArray(o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::sqrt( ++ getOperandArray(o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::tan( ++ getOperandArray(o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::tanh( ++ getOperandArray(o.getOperand(), block_ctx, inputs))}; ++ }) ++ // Handle StableHLO binary elementwise ops ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{ ++ mx::add(getOperandArray(o.getLhs(), block_ctx, inputs), ++ getOperandArray(o.getRhs(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::arctan2( ++ getOperandArray(o.getLhs(), block_ctx, inputs), ++ getOperandArray(o.getRhs(), block_ctx, inputs))}; ++ }) ++ // TODO(@cryptodeal): implement complex op ++ // .Case([&block_ctx](auto ++ // op) {}) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::divide( ++ getOperandArray(o.getLhs(), block_ctx, inputs), ++ getOperandArray(o.getRhs(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::maximum( ++ getOperandArray(o.getLhs(), block_ctx, inputs), ++ getOperandArray(o.getRhs(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::minimum( ++ getOperandArray(o.getLhs(), block_ctx, inputs), ++ getOperandArray(o.getRhs(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::multiply( ++ getOperandArray(o.getLhs(), block_ctx, inputs), ++ getOperandArray(o.getRhs(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::power( ++ getOperandArray(o.getLhs(), block_ctx, inputs), ++ getOperandArray(o.getRhs(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::remainder( ++ getOperandArray(o.getLhs(), block_ctx, inputs), ++ getOperandArray(o.getRhs(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::left_shift( ++ getOperandArray(o.getLhs(), block_ctx, inputs), ++ getOperandArray(o.getRhs(), block_ctx, inputs))}; ++ }) ++ .Case( ++ [&block_ctx, &inputs](auto o) { ++ /** ++ * Per Metal Spec: ++ * For the right-shift operator, if E1 has an unsigned ++ * type or if E1 has a signed type and a nonnegative ++ * value, the vacated bits are filled with zeros. If E1 ++ * has a signed type and a negative value, the vacated ++ * bits are filled with ones. ++ */ ++ auto lhs = getOperandArray(o.getLhs(), block_ctx, inputs); ++ auto rhs = getOperandArray(o.getRhs(), block_ctx, inputs); ++ auto target_dtype = rhs.dtype(); ++ switch (lhs.dtype().size()) { ++ case 1: ++ target_dtype = mx::int8; ++ break; ++ case 2: ++ target_dtype = mx::int16; ++ break; ++ case 4: ++ target_dtype = mx::int32; ++ break; ++ case 8: ++ target_dtype = mx::int64; ++ break; ++ } ++ return std::vector{mx::view( ++ mx::right_shift(mx::view(lhs, target_dtype), ++ mx::astype(rhs, target_dtype)), ++ lhs.dtype())}; ++ }) ++ .Case( ++ [&block_ctx, &inputs](auto o) { ++ // Ensures that we bitcast to `uint` type before ++ // performing the right shift. Should ensure that vacated ++ // bits are zero populated. ++ auto lhs = getOperandArray(o.getLhs(), block_ctx, inputs); ++ auto rhs = getOperandArray(o.getRhs(), block_ctx, inputs); ++ auto target_dtype = rhs.dtype(); ++ switch (lhs.dtype().size()) { ++ case 1: ++ target_dtype = mx::uint8; ++ break; ++ case 2: ++ target_dtype = mx::uint16; ++ break; ++ case 4: ++ target_dtype = mx::uint32; ++ break; ++ case 8: ++ target_dtype = mx::uint64; ++ break; ++ } ++ return std::vector{mx::view( ++ mx::right_shift(mx::view(lhs, target_dtype), ++ mx::astype(rhs, target_dtype)), ++ lhs.dtype())}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::subtract( ++ getOperandArray(o.getLhs(), block_ctx, inputs), ++ getOperandArray(o.getRhs(), block_ctx, inputs))}; ++ }) ++ // Handle StableHLO binary logical elementwise ops ++ .Case([&block_ctx, &inputs](auto o) { ++ auto lhs = getOperandArray(o.getLhs(), block_ctx, inputs); ++ auto rhs = getOperandArray(o.getRhs(), block_ctx, inputs); ++ switch (mx::kindof(lhs.dtype())) { ++ case mx::Dtype::Kind::b: ++ return std::vector{mx::logical_and(lhs, rhs)}; ++ default: ++ return std::vector{mx::bitwise_and(lhs, rhs)}; ++ } ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ auto lhs = getOperandArray(o.getLhs(), block_ctx, inputs); ++ auto rhs = getOperandArray(o.getRhs(), block_ctx, inputs); ++ switch (mx::kindof(lhs.dtype())) { ++ case mx::Dtype::Kind::b: ++ return std::vector{mx::logical_or(lhs, rhs)}; ++ default: ++ return std::vector{mx::bitwise_or(lhs, rhs)}; ++ } ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::bitwise_xor( ++ getOperandArray(o.getLhs(), block_ctx, inputs), ++ getOperandArray(o.getRhs(), block_ctx, inputs))}; ++ }) ++ // Handle StableHLO communication ops ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ ++ // Handle StableHLO parallelism related ops ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ ++ // Handle StableHLO control flow ops ++ // .Case([](auto o) {}) ++ // TODO(@cryptodeal): need custom kernels in order to ++ // remove the need to call `mx::eval` for the following ops ++ // .Case([](auto o)) ++ // .Case([](auto o)) ++ // .Case([](auto o)) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ .Case([&block_ctx, &inputs](auto o) { ++ auto operand = ++ getOperandArray(o.getOperand(0), block_ctx, inputs); ++ if (isArgMaxReduce(o)) { ++ auto axis = static_cast(o.getDimensions()[0]); ++ auto indices = mx::argmax(operand, axis); ++ ++ mx::Dtype result_type = *utils::dtype::fromMlirType( ++ mlir::cast(o.getResults()[1].getType()) ++ .getElementType()); ++ return std::vector{ ++ mx::take(operand, indices, axis), ++ mx::astype(indices, result_type)}; ++ } ++ if (isSumReduce(o)) { ++ return std::vector{mx::sum( ++ operand, static_cast(o.getDimensions()[0]))}; ++ } ++ if (isMaxReduce(o)) { ++ return std::vector{mx::max( ++ operand, static_cast(o.getDimensions()[0]))}; ++ } ++ if (isMinReduce(o)) { ++ return std::vector{mx::min( ++ operand, static_cast(o.getDimensions()[0]))}; ++ } ++ // TODO(@cryptodeal): further cleanup module validation ++ // done via call to `Compile` to check for valid reduce types. ++ ++ // For now, we assume `mx::any` if none of the above ++ ++ // if (isAnyReduce(o)) { ++ return std::vector{ ++ mx::any(operand, static_cast(o.getDimensions()[0]))}; ++ //} ++ }) ++ // Handle StableHLO tuple ops ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ .Case([&block_ctx, &inputs](auto o) { ++ auto lhs = getOperandArray(o.getLhs(), block_ctx, inputs); ++ auto rhs = getOperandArray(o.getRhs(), block_ctx, inputs); ++ switch (o.getComparisonDirection()) { ++ case ComparisonDirection::NE: ++ return std::vector{mx::not_equal(lhs, rhs)}; ++ case ComparisonDirection::GE: ++ return std::vector{ ++ mx::greater_equal(lhs, rhs)}; ++ case ComparisonDirection::GT: ++ return std::vector{mx::greater(lhs, rhs)}; ++ case ComparisonDirection::LE: ++ return std::vector{mx::less_equal(lhs, rhs)}; ++ case ComparisonDirection::LT: ++ return std::vector{mx::less(lhs, rhs)}; ++ case ComparisonDirection::EQ: ++ return std::vector{mx::equal(lhs, rhs)}; ++ } ++ }) ++ // Handle StableHLO Slice ops ++ .Case([&block_ctx, &inputs](auto o) { ++ std::vector start_indices( ++ o.getStartIndices().begin(), o.getStartIndices().end()); ++ std::vector limit_indices( ++ o.getLimitIndices().begin(), o.getLimitIndices().end()); ++ std::vector strides(o.getStrides().begin(), ++ o.getStrides().end()); ++ return std::vector{mx::slice( ++ getOperandArray(o.getOperand(), block_ctx, inputs), ++ start_indices, limit_indices, strides)}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ auto operand = ++ getOperandArray(o.getOperand(), block_ctx, inputs); ++ std::vector indices; ++ for (auto val : o.getStartIndices()) { ++ indices.emplace_back( ++ getOperandArray(val, block_ctx, inputs)); ++ } ++ auto start_indices = mx::concatenate(indices); ++ std::vector axes(operand.ndim()); ++ std::iota(axes.begin(), axes.end(), 0); ++ std::vector slice_sizes(o.getSliceSizes().begin(), ++ o.getSliceSizes().end()); ++ return std::vector{ ++ mx::slice(operand, start_indices, axes, slice_sizes)}; ++ }) ++ .Case( ++ [&block_ctx, &inputs](auto o) { ++ auto operand = ++ getOperandArray(o.getOperand(), block_ctx, inputs); ++ auto update = ++ getOperandArray(o.getUpdate(), block_ctx, inputs); ++ std::vector indices; ++ for (auto val : o.getStartIndices()) { ++ indices.emplace_back( ++ getOperandArray(val, block_ctx, inputs)); ++ } ++ ++ auto start_indices = mx::concatenate(indices); ++ std::vector axes(operand.ndim()); ++ std::iota(axes.begin(), axes.end(), 0); ++ return std::vector{mx::slice_update( ++ operand, update, start_indices, axes)}; ++ }) ++ // Handle StableHLO Other ops ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ .Case( ++ [&block_ctx, &inputs](auto o) { ++ auto result_type = *utils::dtype::fromMlirType( ++ mlir::cast(o.getResult().getType()) ++ .getElementType()); ++ return std::vector{mx::view( ++ getOperandArray(o.getOperand(), block_ctx, inputs), ++ result_type)}; ++ }) ++ /* ++ .Case([](auto o) {}) ++ Deprecated see: ++ https://github.com/openxla/stablehlo/issues/2340 ++ https://github.com/openxla/stablehlo/pull/2283 ++ */ ++ .Case([&block_ctx, &inputs, ++ &op_lookup](auto o) { ++ return op_lookup.broadcast_in_dim ++ .find(getBroadcastInDimOpInfo(o)) ++ ->second( ++ {getOperandArray(o.getOperand(), block_ctx, inputs)}); ++ }) ++ // .Case([](auto o) {}) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::linalg::cholesky( ++ getOperandArray(o.getOperand(), block_ctx, inputs), ++ o.getLower())}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::clip( ++ getOperandArray(o.getOperand(), block_ctx, inputs), ++ getOperandArray(o.getMin(), block_ctx, inputs), ++ getOperandArray(o.getMax(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ std::vector inputs; ++ for (auto val : o.getOperands()) { ++ inputs.emplace_back( ++ getOperandArray(val, block_ctx, inputs)); ++ } ++ return std::vector{ ++ mx::concatenate(inputs, o.getDimension())}; ++ }) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ /* ++ .Case([](auto o) {}) ++ Deprecated see: ++ https://github.com/openxla/stablehlo/issues/2340 ++ https://github.com/openxla/stablehlo/pull/2283 ++ */ ++ .Case( ++ [&block_ctx, &inputs, &op_lookup](auto o) { ++ return op_lookup.dot_general.find(getDotGeneralOpInfo(o)) ++ ->second( ++ {getOperandArray(o.getLhs(), block_ctx, inputs), ++ getOperandArray(o.getRhs(), block_ctx, inputs)}); ++ }) ++ /* ++ .Case([](auto op) {}) ++ .Case([](auto op) {}) ++ Deprecated see: ++ https://github.com/openxla/stablehlo/issues/2340 ++ https://github.com/openxla/stablehlo/pull/2283 ++ */ ++ // .Case([](auto op) {}) ++ // TODO(@cryptodeal): definitely room for optimization here ++ .Case([&block_ctx, &inputs, ++ &op_lookup](auto o) { ++ return op_lookup.gather.find(getGatherOpInfo(o)) ++ ->second( ++ {getOperandArray(o.getOperand(), block_ctx, inputs), ++ getOperandArray(o.getStartIndices(), block_ctx, ++ inputs)}); ++ }) ++ .Case( ++ [&block_ctx, &inputs](auto o) { ++ return std::vector{mx::array( ++ (getOperandArray(o.getOperand(), block_ctx, inputs)) ++ .shape(o.getDimension()))}; ++ }) ++ // .Case([](auto o) {}) ++ .Case([&block_ctx, &inputs](auto o) { ++ auto result_type = ++ mlir::cast(o.getResult().getType()); ++ return std::vector{mx::reshape( ++ getOperandArray(o.getOperand(), block_ctx, inputs), ++ std::vector(result_type.getShape().begin(), ++ result_type.getShape().end()))}; ++ }) ++ // .Case([](auto o) {}) ++ .Case( ++ [&block_ctx, &inputs, &op_lookup](auto o) { ++ std::vector operands; ++ for (auto val : o.getInputs()) { ++ operands.emplace_back( ++ getOperandArray(val, block_ctx, inputs)); ++ } ++ // Add updates to input list ++ for (auto val : o.getUpdates()) { ++ operands.emplace_back( ++ getOperandArray(val, block_ctx, inputs)); ++ } ++ // Add scatter indices to input list ++ operands.emplace_back(getOperandArray( ++ o.getScatterIndices(), block_ctx, inputs)); ++ return op_lookup.scatter.find(getScatterOpInfo(o)) ++ ->second(operands); ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::where( ++ getOperandArray(o.getOperand(0), block_ctx, inputs), ++ getOperandArray(o.getOperand(1), block_ctx, inputs), ++ getOperandArray(o.getOperand(2), block_ctx, inputs))}; ++ }) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ .Case([&block_ctx, &inputs](auto o) { ++ std::vector inputs; ++ for (auto val : o.getInputs()) { ++ inputs.emplace_back( ++ getOperandArray(val, block_ctx, inputs)); ++ } ++ int dim = static_cast(o.getDimension()); ++ // Get update computation ++ mlir::Block& comparator = o.getComparator().front(); ++ auto [index, comparator_type] = *getSortInfo(comparator); ++ auto indices = ++ mx::argsort(inputs[index], dim, comparator_type); ++ std::vector res; ++ for (auto in : inputs) { ++ res.push_back(mx::gather(in, indices, {dim}, {1})); ++ } ++ return res; ++ }) ++ // .Case([](auto o) {}) ++ // TODO (@cryptodeal): lots of room for optimization here ++ .Case([&block_ctx, &inputs, ++ &op_lookup](auto o) { ++ return op_lookup.pad.find(getPadOpInfo(o)) ++ ->second( ++ {getOperandArray(o.getOperand(), block_ctx, inputs), ++ getOperandArray(o.getPaddingValue(), block_ctx, ++ inputs)}); ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ std::vector axes(o.getPermutation().begin(), ++ o.getPermutation().end()); ++ return std::vector{mx::transpose( ++ getOperandArray(o.getOperand(), block_ctx, inputs), ++ axes)}; ++ }) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // Need to modify such that `rng.state` is held outside of scope ++ // of `mx::compile` ++ // TODO(@cryptodeal): re-enable once we've handled ensuring ++ // state is handled correctly ++ // .Case([&block_ctx](auto o) { ++ // auto rng_func = [&o](const std::vector& inputs) ++ // { ++ // auto a = inputs[0]; ++ // auto b = inputs[1]; ++ // auto result_type = ++ // mlir::cast(o.getResult().getType()); ++ // std::vector ++ // shape(result_type.getShape().begin(), ++ // result_type.getShape().end()); ++ ++ // switch (o.getRngDistribution()) { ++ // case RngDistribution::UNIFORM: ++ // return std::vector{ ++ // mx::random::uniform(a, b, shape, a.dtype())}; ++ // default: ++ // return std::vector{ ++ // mx::random::normal(shape, a.dtype(), a, b)}; ++ // } ++ // }; ++ // return mx::compile(rng_func)( ++ // {getOperandArray(o.getA(), block_ctx), ++ // getOperandArray(o.getB(), block_ctx)}); ++ // }) ++ // .Case([&block_ctx](auto o) { ++ // auto rng_bit_gen_func = [&o](const std::vector& ++ // inputs) { ++ ++ // }; ++ // return mx::compile(rng_bit_gen_func)( ++ // {getOperandArray(o.getA(), block_ctx), ++ // getOperandArray(o.getB(), block_ctx)}); ++ // }) ++ // Handle StableHLO Quantize ops ++ // .Case([](auto op) {}) ++ // .Case([](auto op) {}) ++ // .Case([](auto op) {}) ++ /* ++ .Case([](auto o) {}) ++ Deprecated see: ++ https://github.com/openxla/stablehlo/issues/2340 ++ https://github.com/openxla/stablehlo/pull/2283 ++ */ ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ ++ .Default([&op](auto o) { ++ // Shouldn't be possible to reach this point, return ++ // empty vector. ++ return std::vector{}; ++ }); ++ block_ctx.emplace(&op, res); ++ } ++ return res; ++ }; ++ op_lookup.call.emplace(call_op_info, mx::compile(call_op_func)); ++ } ++} ++ ++void compileIotaOp(OpLookup& op_lookup, stablehlo::IotaOp o) { ++ IotaOpInfo iota_op_info = getIotaOpInfo(o); ++ auto dimensions = std::get<0>(iota_op_info.captures); ++ auto dtype = std::get<1>(iota_op_info.captures); ++ auto iota_dim = std::get<2>(iota_op_info.captures); ++ auto shape = std::get<3>(iota_op_info.captures); ++ if (auto search = op_lookup.iota.find(iota_op_info); ++ search == op_lookup.iota.end()) { ++ std::cout << "Compiling IotaOp" << std::endl; ++ auto iota_func = [dimensions, dtype, iota_dim, ++ shape](const std::vector& inputs) { ++ auto init_arange = ++ mx::arange(static_cast(shape[iota_dim]), dtype); ++ if (dimensions.size()) { ++ init_arange = mx::reshape(init_arange, dimensions); ++ } ++ return std::vector{mx::broadcast_to(init_arange, shape)}; ++ }; ++ op_lookup.iota.emplace(iota_op_info, mx::compile(iota_func)); ++ } ++} ++ ++void compileCbrtOp(OpLookup& op_lookup, stablehlo::CbrtOp o) { ++ CbrtOpInfo cbrt_op_info = getCbrtOpInfo(o); ++ if (auto search = op_lookup.cbrt.find(cbrt_op_info); ++ search == op_lookup.cbrt.end()) { ++ std::cout << "Compiling CbrtOp" << std::endl; ++ ++ auto cbrt_func = [](const std::vector& inputs) { ++ return std::vector{mx::power( ++ inputs[0], ++ mx::full(inputs[0].shape(), 1 / 3, inputs[0].dtype()))}; ++ }; ++ op_lookup.cbrt.emplace(cbrt_op_info, mx::compile(cbrt_func)); ++ } ++} ++ ++void compileBroadcastInDimOp(OpLookup& op_lookup, ++ stablehlo::BroadcastInDimOp o) { ++ BroadcastInDimOpInfo broadcast_in_dim_op_info = getBroadcastInDimOpInfo(o); ++ auto broadcast_dims = std::get<0>(broadcast_in_dim_op_info.captures); ++ auto target_shape = std::get<1>(broadcast_in_dim_op_info.captures); ++ if (auto search = op_lookup.broadcast_in_dim.find(broadcast_in_dim_op_info); ++ search == op_lookup.broadcast_in_dim.end()) { ++ std::cout << "Compiling BroadcastInDimOp" << std::endl; ++ ++ auto broadcast_in_dim_func = [broadcast_dims, target_shape]( ++ const std::vector& inputs) { ++ auto operand = inputs[0]; ++ ++ // potentially reshape to pad with ones, which allows ++ // broadcasting ++ auto min_rank = std::min(operand.ndim(), target_shape.size()); ++ if (min_rank == operand.ndim()) { ++ std::vector padded_shape = operand.shape(); ++ for (auto i = 0; i < target_shape.size(); i++) { ++ if (auto search = ++ std::find(broadcast_dims.begin(), broadcast_dims.end(), i); ++ search == broadcast_dims.end()) { ++ padded_shape.insert(padded_shape.begin() + i, 1); ++ } ++ } ++ operand = mx::reshape(operand, padded_shape); ++ } ++ return std::vector{mx::broadcast_to(operand, target_shape)}; ++ }; ++ op_lookup.broadcast_in_dim.emplace(broadcast_in_dim_op_info, ++ mx::compile(broadcast_in_dim_func)); ++ } ++} ++ ++void compileDotGeneralOp(OpLookup& op_lookup, stablehlo::DotGeneralOp o) { ++ DotGeneralOpInfo dot_general_op_info = getDotGeneralOpInfo(o); ++ auto lhs_batch_dims = std::get<0>(dot_general_op_info.captures); ++ auto rhs_batch_dims = std::get<1>(dot_general_op_info.captures); ++ auto lhs_contract_dims = std::get<2>(dot_general_op_info.captures); ++ auto rhs_contract_dims = std::get<3>(dot_general_op_info.captures); ++ if (auto search = op_lookup.dot_general.find(dot_general_op_info); ++ search == op_lookup.dot_general.end()) { ++ std::cout << "Compiling DotGeneralOp" << std::endl; ++ ++ auto dot_general_func = [lhs_batch_dims, rhs_batch_dims, lhs_contract_dims, ++ rhs_contract_dims]( ++ const std::vector& inputs) { ++ auto lhs = inputs[0]; ++ auto rhs = inputs[1]; ++ ++ int dim_count = 0; ++ auto getDimChar = [&dim_count]() -> char { return 'a' + dim_count++; }; ++ ++ std::unordered_map lhs_batch_map; ++ std::unordered_map rhs_batch_map; ++ std::unordered_map lhs_contract_map; ++ std::unordered_map rhs_contract_map; ++ ++ std::string res_subscript; ++ for (auto i = 0; i < lhs_batch_dims.size(); ++i) { ++ auto dim_char = getDimChar(); ++ res_subscript = res_subscript + dim_char; ++ lhs_batch_map.emplace(lhs_batch_dims[i], dim_char); ++ rhs_batch_map.emplace(rhs_batch_dims[i], dim_char); ++ } ++ ++ for (auto i = 0; i < lhs_contract_dims.size(); ++i) { ++ auto dim_char = getDimChar(); ++ lhs_contract_map.emplace(lhs_contract_dims[i], dim_char); ++ rhs_contract_map.emplace(rhs_contract_dims[i], dim_char); ++ } ++ ++ std::string lhs_subscript; ++ for (auto i = 0; i < lhs.ndim(); ++i) { ++ if (auto match = lhs_batch_map.find(i); match != lhs_batch_map.end()) { ++ lhs_subscript = lhs_subscript + match->second; ++ } else if (auto match = lhs_contract_map.find(i); ++ match != lhs_contract_map.end()) { ++ lhs_subscript = lhs_subscript + match->second; ++ } else { ++ auto dim_char = getDimChar(); ++ res_subscript = res_subscript + dim_char; ++ lhs_subscript = lhs_subscript + dim_char; ++ } ++ } ++ ++ std::string rhs_subscript; ++ for (auto i = 0; i < rhs.ndim(); ++i) { ++ if (auto match = rhs_batch_map.find(i); match != rhs_batch_map.end()) { ++ rhs_subscript = rhs_subscript + match->second; ++ } else if (auto match = rhs_contract_map.find(i); ++ match != rhs_contract_map.end()) { ++ rhs_subscript = rhs_subscript + match->second; ++ } else { ++ auto dim_char = getDimChar(); ++ res_subscript = res_subscript + dim_char; ++ rhs_subscript = rhs_subscript + dim_char; ++ } ++ } ++ ++ return std::vector{mx::einsum( ++ lhs_subscript + "," + rhs_subscript + "->" + res_subscript, inputs)}; ++ }; ++ op_lookup.dot_general.emplace(dot_general_op_info, ++ mx::compile(dot_general_func)); ++ } ++} ++ ++void compileGatherOp(OpLookup& op_lookup, stablehlo::GatherOp o) { ++ GatherOpInfo gather_op_info = getGatherOpInfo(o); ++ auto collapsed_slice_dims = std::get<0>(gather_op_info.captures); ++ auto index_vector_dim = std::get<1>(gather_op_info.captures); ++ auto offset_dims = std::get<2>(gather_op_info.captures); ++ auto operand_batching_dims = std::get<3>(gather_op_info.captures); ++ auto result_shape = std::get<4>(gather_op_info.captures); ++ auto slice_sizes = std::get<5>(gather_op_info.captures); ++ auto start_index_map = std::get<6>(gather_op_info.captures); ++ auto start_indices_batching_dims = std::get<7>(gather_op_info.captures); ++ ++ if (auto search = op_lookup.gather.find(gather_op_info); ++ search == op_lookup.gather.end()) { ++ std::cout << "Compiling GatherOp" << std::endl; ++ auto gather_func = [collapsed_slice_dims, index_vector_dim, offset_dims, ++ operand_batching_dims, result_shape, slice_sizes, ++ start_index_map, start_indices_batching_dims]( ++ const std::vector& inputs) { ++ auto operand = inputs[0]; ++ auto start_indices = inputs[1]; ++ ++ // Calculate batch dims ++ std::vector batch_dims; ++ for (int64_t i = 0; i < result_shape.size(); i++) { ++ if (std::find(offset_dims.begin(), offset_dims.end(), i) == ++ offset_dims.end()) { ++ batch_dims.emplace_back(static_cast(i)); ++ } ++ } ++ ++ std::vector operand_indices; ++ ++ // Iterate over result index space, populating result ++ for (const auto result_index_tuple : index_space(result_shape)) { ++ std::vector result_index = ++ to_vector(result_index_tuple, result_shape.size()); ++ ++ std::vector batch_index(batch_dims.size()); ++ for (unsigned i = 0; i < batch_dims.size(); i++) { ++ batch_index[i] = result_index[batch_dims[i]]; ++ } ++ ++ // Slice start index for the current batch ++ std::vector sin_start(start_indices.ndim()); ++ std::vector sin_stop(start_indices.ndim()); ++ unsigned batch_idx_count = 0; ++ ++ for (auto i = 0; i < start_indices.ndim(); i++) { ++ if (index_vector_dim == static_cast(i)) { ++ sin_start[i] = 0; ++ sin_stop[i] = start_indices.shape(i) + 1; ++ continue; ++ } ++ sin_start[i] = batch_index[batch_idx_count++]; ++ sin_stop[i] = sin_start[i] + 1; ++ } ++ mx::array start_index = ++ mx::flatten(mx::slice(start_indices, sin_start, sin_stop)); ++ ++ // Compute full start index ++ mx::array full_start_index = ++ mx::zeros({static_cast(operand.ndim())}, mx::int32); ++ for (auto d_start = 0; d_start < start_index_map.size(); d_start++) { ++ int d_operand = start_index_map[d_start]; ++ auto index_scalar = mx::slice(start_index, {d_start}, {d_start + 1}); ++ full_start_index = mx::slice_update( ++ full_start_index, ++ mx::clip(index_scalar, mx::array({0}, mx::int32), ++ mx::array({operand.shape(d_operand) - ++ static_cast(slice_sizes[d_operand])}, ++ mx::int32)), ++ {d_operand}, {d_operand + 1}); ++ } ++ ++ // Compute full batching index ++ std::vector full_batching_index(operand.ndim(), 0); ++ for (auto i_batching = 0; i_batching < operand_batching_dims.size(); ++ i_batching++) { ++ auto d_operand = ++ static_cast(operand_batching_dims[i_batching]); ++ auto d_start = ++ static_cast(start_indices_batching_dims[i_batching]); ++ full_batching_index[d_operand] = batch_index ++ [d_start - ++ (static_cast(d_start) < index_vector_dim ? 0 : 1)]; ++ } ++ ++ // Compute offset index ++ std::vector offset_index(offset_dims.size()); ++ for (unsigned i = 0; i < offset_dims.size(); i++) { ++ offset_index[i] = result_index[offset_dims[i]]; ++ } ++ ++ // Compute full offset index ++ std::vector full_offset_index(operand.ndim(), 0); ++ unsigned offset_index_count = 0; ++ for (unsigned i = 0; i < full_offset_index.size(); i++) { ++ if (std::find(operand_batching_dims.begin(), ++ operand_batching_dims.end(), static_cast(i)) != ++ operand_batching_dims.end() || ++ std::find(collapsed_slice_dims.begin(), ++ collapsed_slice_dims.end(), static_cast(i)) != ++ collapsed_slice_dims.end()) { ++ continue; ++ } ++ full_offset_index[i] = offset_index[offset_index_count++]; ++ } ++ ++ operand_indices.emplace_back( ++ full_start_index + ++ mx::array(full_batching_index.data(), ++ {static_cast(full_batching_index.size())}, ++ mx::int32) + ++ mx::array(full_offset_index.data(), ++ {static_cast(full_offset_index.size())}, ++ mx::int32)); ++ } ++ std::vector gather_slice_sizes(operand.ndim(), 1); ++ std::vector gather_axes(operand.ndim()); ++ std::iota(gather_axes.begin(), gather_axes.end(), 0); ++ return std::vector{ ++ mx::reshape(mx::gather(operand, ++ mx::split(mx::stack(operand_indices, 1), ++ operand_indices[0].shape(0), 0), ++ gather_axes, gather_slice_sizes), ++ result_shape)}; ++ }; ++ ++ op_lookup.gather.emplace(gather_op_info, mx::compile(gather_func)); ++ } ++} ++ ++int32_t computeProduct(const std::vector& dims) { ++ return std::accumulate(dims.begin(), dims.end(), 1, std::multiplies()); ++} ++ ++ ++mx::array computeGatherIndices( ++ const mx::array& start_indices, ++ const std::vector& start_index_map, ++ const std::vector& operand_shape, ++ const std::vector& slice_sizes, ++ const std::vector& operand_batching_dims, ++ const std::vector& start_indices_batching_dims, ++ int64_t index_vector_dim) { ++ ++ // Validate input dimensions ++ if (start_indices.ndim() == 0) { ++ throw std::invalid_argument("start_indices must have at least one dimension."); ++ } ++ ++ // Compute the shape of start_indices ++ std::vector start_indices_shape = start_indices.shape(); ++ ++ // Compute the number of index vectors ++ int32_t index_vector_size = static_cast(start_indices_shape[index_vector_dim]); ++ std::vector outer_shape = start_indices_shape; ++ outer_shape.erase(outer_shape.begin() + index_vector_dim); ++ int32_t num_index_vectors = computeProduct(outer_shape); ++ ++ // Reshape start_indices to a 2D tensor: [num_index_vectors, index_vector_size] ++ mx::array reshaped_start_indices = mx::reshape(start_indices, {num_index_vectors, index_vector_size}); ++ ++ // Initialize full_start_indices with zeros: [num_index_vectors, operand_rank] ++ int32_t operand_rank = static_cast(operand_shape.size()); ++ mx::array full_start_indices = mx::zeros({num_index_vectors, operand_rank}, mx::int32); ++ ++ // Populate full_start_indices using start_index_map ++ for (size_t i = 0; i < start_index_map.size(); ++i) { ++ int32_t operand_dim = start_index_map[i]; ++ // Extract the i-th column from reshaped_start_indices ++ mx::array indices_column = mx::slice(reshaped_start_indices, {0, static_cast(i)}, ++ {num_index_vectors, static_cast(i + 1)}); ++ // Clip indices to ensure they are within bounds ++ int32_t max_index = operand_shape[operand_dim] - slice_sizes[operand_dim]; ++ mx::array clipped_indices = mx::clip(indices_column, mx::array({0}, mx::int32), ++ mx::array({max_index}, mx::int32)); ++ // Update the corresponding column in full_start_indices ++ full_start_indices = mx::slice_update(full_start_indices, clipped_indices, ++ {0, operand_dim}, ++ {num_index_vectors, operand_dim + 1}); ++ } ++ ++ return full_start_indices; ++} ++ ++void _compileGatherOp(OpLookup& op_lookup, stablehlo::GatherOp o) { ++ GatherOpInfo gather_op_info = getGatherOpInfo(o); ++ auto collapsed_slice_dims = std::get<0>(gather_op_info.captures); ++ auto index_vector_dim = std::get<1>(gather_op_info.captures); ++ auto offset_dims = std::get<2>(gather_op_info.captures); ++ auto operand_batching_dims = std::get<3>(gather_op_info.captures); ++ auto result_shape = std::get<4>(gather_op_info.captures); ++ auto slice_sizes = std::get<5>(gather_op_info.captures); ++ auto start_index_map = std::get<6>(gather_op_info.captures); ++ auto start_indices_batching_dims = std::get<7>(gather_op_info.captures); ++ ++ if (auto search = op_lookup.gather.find(gather_op_info); ++ search == op_lookup.gather.end()) { ++ std::cout << "Compiling GatherOp" << std::endl; ++ auto gather_func = ++ [collapsed_slice_dims, index_vector_dim, offset_dims, ++ operand_batching_dims, result_shape, slice_sizes, start_index_map, ++ start_indices_batching_dims](const std::vector& inputs) { ++ auto operand = inputs[0]; ++ auto start_indices = inputs[1]; ++ ++ // Calculate batch dims ++ std::vector batch_dims; ++ for (int64_t i = 0; i < result_shape.size(); i++) { ++ if (std::find(offset_dims.begin(), offset_dims.end(), i) == ++ offset_dims.end()) { ++ batch_dims.emplace_back(static_cast(i)); ++ } ++ } ++ ++ // Iterate over result index space, populating result ++ for (const auto result_index_tuple : index_space(result_shape)) { ++ std::vector result_index = ++ to_vector(result_index_tuple, result_shape.size()); ++ ++ std::vector batch_index(batch_dims.size()); ++ for (unsigned i = 0; i < batch_dims.size(); i++) { ++ batch_index[i] = result_index[batch_dims[i]]; ++ } ++ } ++ }; ++ } ++} ++ ++void compileScatterOp(OpLookup& op_lookup, stablehlo::ScatterOp o) { ++ ScatterOpInfo scatter_op_info = getScatterOpInfo(o); ++ auto index_vector_dim = std::get<0>(scatter_op_info.captures); ++ auto input_batching_dims = std::get<1>(scatter_op_info.captures); ++ auto inserted_window_dims = std::get<2>(scatter_op_info.captures); ++ auto scatter_dims_to_operand_dims = std::get<3>(scatter_op_info.captures); ++ auto scatter_indices_batching_dims = std::get<4>(scatter_op_info.captures); ++ auto scatter_type = std::get<5>(scatter_op_info.captures); ++ auto update_window_dims = std::get<6>(scatter_op_info.captures); ++ if (auto search = op_lookup.scatter.find(scatter_op_info); ++ search == op_lookup.scatter.end()) { ++ std::cout << "Compiling ScatterOp" << std::endl; ++ auto scatter_func = [index_vector_dim, input_batching_dims, ++ inserted_window_dims, scatter_dims_to_operand_dims, ++ scatter_indices_batching_dims, scatter_type, ++ update_window_dims](const std::vector& in) { ++ auto input_count = (in.size() - 1) / 2; ++ std::vector inputs(in.begin(), in.begin() + input_count); ++ std::vector updates(in.begin() + input_count, in.end() - 1); ++ auto scatter_indices = in[in.size() - 1]; ++ ++ std::vector input_indices; ++ std::vector update_indices; ++ ++ // iterate over updates[0] index space ++ for (const auto update_index_tuple : index_space(updates[0].shape())) { ++ std::vector update_index = to_vector( ++ update_index_tuple, static_cast(updates[0].ndim())); ++ ++ // Calculate update scatter dims ++ std::vector update_scatter_dims; ++ for (auto i = 0; i < updates[0].ndim(); i++) { ++ if (std::find(update_window_dims.begin(), update_window_dims.end(), ++ static_cast(i)) == update_window_dims.end()) { ++ update_scatter_dims.emplace_back(static_cast(i)); ++ } ++ } ++ ++ // Calculate update scatter index ++ std::vector update_scatter_index(update_scatter_dims.size()); ++ for (auto i = 0; i < update_scatter_dims.size(); i++) { ++ update_scatter_index[i] = update_index[update_scatter_dims[i]]; ++ } ++ ++ // Slice start index ++ std::vector sin_start = update_scatter_index; ++ std::vector sin_stop = update_scatter_index; ++ if (index_vector_dim < scatter_indices.ndim()) { ++ sin_start.insert(sin_start.begin() + index_vector_dim, 0); ++ sin_stop.insert(sin_stop.begin() + index_vector_dim, ++ scatter_indices.shape(index_vector_dim)); ++ } ++ for (auto& d : sin_stop) d += 1; ++ mx::array start_index = ++ mx::flatten(mx::slice(scatter_indices, sin_start, sin_stop)); ++ ++ // Compute full start index ++ mx::array full_start_index = ++ mx::zeros({static_cast(inputs[0].ndim())}, mx::int32); ++ for (auto i = 0; i < scatter_dims_to_operand_dims.size(); i++) { ++ auto d_input = static_cast(scatter_dims_to_operand_dims[i]); ++ full_start_index = mx::slice_update( ++ full_start_index, mx::slice(start_index, {i}, {i + 1}), {d_input}, ++ {d_input + 1}); ++ } ++ ++ // Compute full batching index ++ mx::array full_batching_index = ++ mx::zeros({static_cast(inputs[0].ndim())}, mx::int32); ++ for (auto i = 0; i < input_batching_dims.size(); i++) { ++ int32_t d_input = input_batching_dims[i]; ++ int32_t d_start = scatter_indices_batching_dims[i]; ++ full_batching_index = mx::slice_update( ++ full_batching_index, ++ mx::array( ++ {update_scatter_index[d_start - ++ (d_start < index_vector_dim ? 0 : 1)]}), ++ {d_input}, {d_input + 1}); ++ } ++ ++ // Compute update window index ++ std::vector update_window_index(update_window_dims.size()); ++ for (auto i = 0; i < update_window_dims.size(); i++) { ++ update_window_index[i] = update_index[update_window_dims[i]]; ++ } ++ ++ // Compute full window index ++ mx::array full_window_index = ++ mx::zeros({static_cast(update_window_index.size() + ++ inserted_window_dims.size() + ++ input_batching_dims.size())}, ++ mx::int32); ++ unsigned update_window_index_count = 0; ++ for (int32_t i = 0; i < full_window_index.size(); i++) { ++ if (std::find(inserted_window_dims.begin(), ++ inserted_window_dims.end(), ++ i) != inserted_window_dims.end() || ++ std::find(input_batching_dims.begin(), input_batching_dims.end(), ++ i) != input_batching_dims.end()) { ++ continue; ++ } ++ full_window_index = mx::slice_update( ++ full_window_index, ++ mx::array({update_window_index[update_window_index_count++]}), ++ {i}, {i + 1}); ++ } ++ ++ // Compute result index ++ mx::array result_index = ++ full_start_index + full_batching_index + full_window_index; ++ ++ // TODO (@cryptodeal): rework this so we can still leverage ++ // `mx::compile` OR ensure this cannot happen. ++ // if (mx::sum( ++ // result_index >= ++ // mx::array(reinterpret_cast( ++ // result_shape.data()), ++ // {static_cast(result_shape.size())})) ++ // .item()) { ++ // continue; ++ // } ++ ++ input_indices.push_back(result_index); ++ update_indices.push_back(mx::array( ++ update_index.data(), {static_cast(updates[0].ndim())})); ++ } ++ if (update_indices.empty()) { ++ return inputs; ++ } ++ std::vector scatter_axes(inputs[0].ndim()); ++ std::iota(scatter_axes.begin(), scatter_axes.end(), 0); ++ std::vector gather_axes(updates[0].ndim()); ++ std::iota(gather_axes.begin(), gather_axes.end(), 0); ++ std::vector gather_slice_sizes(updates[0].ndim(), 1); ++ std::vector res; ++ auto result_indices = ++ mx::split(mx::stack(input_indices, 1), input_indices[0].shape(0), 0); ++ auto gather_indices = mx::split(mx::stack(update_indices, 1), ++ update_indices[0].shape(0), 0); ++ auto idx_shape = gather_indices[0].shape(); ++ std::vector update_shape(inputs[0].ndim() + idx_shape.size(), 1); ++ for (auto i = 0; i < idx_shape.size(); ++i) { ++ update_shape[i] = idx_shape[i]; ++ } ++ for (auto i = 0; i < inputs.size(); ++i) { ++ auto update_vals = ++ mx::reshape(mx::gather(updates[i], gather_indices, gather_axes, ++ gather_slice_sizes), ++ update_shape); ++ switch (scatter_type) { ++ case utils::ScatterType::Replace: ++ res.push_back(mx::scatter(inputs[i], result_indices, update_vals, ++ scatter_axes)); ++ break; ++ case utils::ScatterType::Add: ++ res.push_back(mx::scatter_add(inputs[i], result_indices, ++ update_vals, scatter_axes)); ++ break; ++ case utils::ScatterType::Prod: ++ res.push_back(mx::scatter_prod(inputs[i], result_indices, ++ update_vals, scatter_axes)); ++ break; ++ case utils::ScatterType::Max: ++ res.push_back(mx::scatter_max(inputs[i], result_indices, ++ update_vals, scatter_axes)); ++ break; ++ case utils::ScatterType::Min: ++ res.push_back(mx::scatter_min(inputs[i], result_indices, ++ update_vals, scatter_axes)); ++ break; ++ } ++ } ++ return res; ++ }; ++ op_lookup.scatter.emplace(scatter_op_info, mx::compile(scatter_func)); ++ } ++} ++void compilePadOp(OpLookup& op_lookup, stablehlo::PadOp o) { ++ PadOpInfo pad_op_info = getPadOpInfo(o); ++ auto result_shape = std::get<0>(pad_op_info.captures); ++ auto edge_pad_low = std::get<1>(pad_op_info.captures); ++ auto interior_pad = std::get<2>(pad_op_info.captures); ++ if (auto search = op_lookup.pad.find(pad_op_info); ++ search == op_lookup.pad.end()) { ++ std::cout << "Compiling PadOp" << std::endl; ++ auto pad_func = [result_shape, edge_pad_low, ++ interior_pad](const std::vector& inputs) { ++ auto operand = inputs[0]; ++ auto padding_value = inputs[1]; ++ auto edge_padding_low = ++ mx::array(edge_pad_low.data(), ++ {static_cast(edge_pad_low.size())}, mx::int32); ++ auto interior_padding = ++ mx::array(interior_pad.data(), ++ {static_cast(edge_pad_low.size())}, mx::int32); ++ ++ // initialize array full of padding values ++ mx::array result = mx::full(result_shape, padding_value); ++ std::vector scatter_indices; ++ std::vector gather_indices; ++ // assign values to correct indices in result ++ for (const auto operand_index_tuple : index_space(operand.shape())) { ++ std::vector operand_index = ++ to_vector(operand_index_tuple, static_cast(operand.ndim())); ++ mx::array result_index = ++ edge_padding_low + ++ mx::array(operand_index.data(), ++ {static_cast(operand_index.size())}, mx::int32) * ++ (interior_padding + ++ mx::full(interior_padding.shape(), 1)); ++ scatter_indices.push_back(result_index); ++ gather_indices.push_back(mx::array( ++ operand_index.data(), {static_cast(operand.ndim())})); ++ } ++ std::vector axes(operand.ndim()); ++ std::iota(axes.begin(), axes.end(), 0); ++ std::vector gather_slice_sizes(operand.ndim(), 1); ++ auto target_indices = mx::split(mx::stack(scatter_indices, 1), ++ scatter_indices[0].shape(0), 0); ++ auto input_indices = mx::split(mx::stack(gather_indices, 1), ++ gather_indices[0].shape(0), 0); ++ auto idx_shape = input_indices[0].shape(); ++ std::vector update_shape(operand.ndim() + idx_shape.size(), 1); ++ for (auto i = 0; i < idx_shape.size(); ++i) { ++ update_shape[i] = idx_shape[i]; ++ } ++ auto update_vals = ++ mx::reshape(mx::gather(operand, input_indices, axes, ++ std::vector(operand.ndim(), 1)), ++ update_shape); ++ return std::vector{ ++ mx::scatter(result, target_indices, update_vals, axes)}; ++ }; ++ op_lookup.pad.emplace(pad_op_info, mx::compile(pad_func)); ++ } ++} ++ ++void compileModule(ModuleOp mod, OpLookup& op_lookup) { ++ // the top level block to compile ++ // auto main = mod.lookupSymbol("main"); ++ ++ Tree dep_tree; ++ resolveFuncDeps(mod, "main", dep_tree); ++ ++ // walk each function and pre-compile lookup table ++ // for all StableHLO op variants ++ for (auto op_name : dep_tree.getAllFnNames()) { ++ auto func = mod.lookupSymbol(op_name); ++ auto& block = func.front(); ++ for (auto& op : block.getOperations()) { ++ llvm::TypeSwitch(&op) ++ .Case( ++ [&op_lookup](auto o) { compileIotaOp(op_lookup, o); }) ++ .Case( ++ [&op_lookup](auto o) { compileCbrtOp(op_lookup, o); }) ++ .Case( ++ [&op_lookup](auto o) { compileBroadcastInDimOp(op_lookup, o); }) ++ .Case( ++ [&op_lookup](auto o) { compileDotGeneralOp(op_lookup, o); }) ++ .Case( ++ [&op_lookup](auto o) { compileGatherOp(op_lookup, o); }) ++ .Case( ++ [&op_lookup](auto o) { compileScatterOp(op_lookup, o); }) ++ .Case( ++ [&op_lookup](auto o) { compilePadOp(op_lookup, o); }) ++ .Default([](auto o) {}); ++ } ++ } ++ ++ while (true) { ++ auto compilable_fns = dep_tree.getCompilableFns(); ++ if (compilable_fns.empty()) { ++ break; ++ } ++ for (auto fn_name : compilable_fns) { ++ std::cout << fn_name.c_str() << std::endl; ++ auto func = mod.lookupSymbol(fn_name); ++ compileCallOp(op_lookup, mod, func); ++ dep_tree.remove(fn_name); ++ } ++ } ++} ++ ++class MlirLoadedExecutable : public PjRtLoadedExecutable { ++ public: ++ MlirLoadedExecutable(ModuleOp module, DeviceAssignment assignment, ++ absl::Span devices, ++ PjRtClient* client) ++ : PjRtLoadedExecutable(), ++ name_("MlirLoadedExecutable"), ++ assignment_(assignment), ++ devices_(client->devices()), ++ client_(client), ++ context_(), ++ module_(cloneIntoContext(module, context_)) { ++ TRACE_ME_MEMBER; ++ auto main = module.lookupSymbol("main"); ++ std::cout << "Started call to `mlx::compile`" << std::endl << std::endl; ++ compileModule(module, op_lookup_); ++ // try { ++ auto& main_block = main.front(); ++ compiled_module_ = ++ op_lookup_.call.find(getCallOpInfo(module, main))->second; ++ ++ // force compilation by running w zeroed inputs ++ std::vector tmp_inputs; ++ for (auto arg : main_block.getArguments()) { ++ auto shaped_type = mlir::cast(arg.getType()); ++ tmp_inputs.push_back( ++ mx::zeros(std::vector(shaped_type.getShape().begin(), ++ shaped_type.getShape().end()), ++ *utils::dtype::fromMlirType(shaped_type.getElementType()))); ++ } ++ compiled_module_(tmp_inputs); ++ std::cout << "Finished call to `mlx::compile`" << std::endl << std::endl; ++ // } catch (std::exception& e) { ++ // std::cout << "Error in `mlx::compile`" << std::endl; ++ // } ++ } ++ ++ static absl::StatusOr> Compile( ++ ModuleOp module, DeviceAssignment assignment, ++ absl::Span devices, PjRtClient* client) { ++ // TRACE_ME; ++ ++ mlir::BaseScopedDiagnosticHandler diagnostic_handler(module->getContext()); ++ if (failed(decomposeChloToStablehlo(module))) { ++ return diagnostic_handler.ConsumeStatus(); ++ } ++ ++ // Simplify the graph using available HWI passes. ++ if (failed(runHardwareIndependentOptimizations(module))) { ++ return diagnostic_handler.ConsumeStatus(); ++ } ++ ++ auto module_status = validateModule(module); ++ if (!module_status.ok()) { ++ return module_status; ++ } ++ ++ // std::cout << ToString(module).c_str() << std::endl << std::endl; ++ ++ auto executable = std::make_unique(module, assignment, ++ devices, client); ++ ++ return executable; ++ } ++ ++ PjRtClient* client() const override { ++ TRACE_ME_MEMBER; ++ return client_; ++ } ++ ++ const DeviceAssignment& device_assignment() const override { ++ TRACE_ME_MEMBER; ++ return assignment_; ++ } ++ ++ absl::Span ++ addressable_device_logical_ids() const override { ++ TRACE_ME_MEMBER; ++ LOG_UNIMPLEMENTED(addressable_device_logical_ids); ++ return {}; ++ } ++ ++ absl::Span addressable_devices() const override { ++ TRACE_ME_MEMBER; ++ return devices_; ++ } ++ ++ // Helper function to get default mem from device. ++ PjRtMemorySpace* get_default_memory_space() const { ++ TRACE_ME_MEMBER; ++ return devices_[0]->default_memory_space().value_or(nullptr); ++ } ++ ++ // absl::StatusOr> evalModule( ++ // ModuleOp& module, const SmallVector& inputs) { ++ // TRACE_ME_MEMBER; ++ // std::vector block_arguments; ++ // for (auto input : inputs) { ++ // TF_ASSIGN_OR_RETURN(auto input_arr, ++ // utils::array::fromDenseElementsAttr(input)); ++ // block_arguments.emplace_back(input_arr); ++ // } ++ ++ // // std::cout << std::endl; ++ // // module.dump(); ++ // // std::cout << std::endl; ++ // auto main = module.lookupSymbol("main"); ++ // std::unordered_map< ++ // mlir::Block*, ++ // std::tuple, ++ // std::unordered_map>>> ++ // block_ctx; ++ // TF_ASSIGN_OR_RETURN(auto mlx_res, evalBlock(module, main.front(), ++ // block_arguments, block_ctx)); ++ ++ // for (auto& a : mlx_res) { ++ // a = mx::contiguous(a); ++ // } ++ // mx::eval(mlx_res); ++ // return mlx_res; ++ // } ++ ++ absl::StatusOr ExecuteWithMlxInterpreter( ++ absl::Span argument_handles, ModuleOp module, ++ PjRtDevice* device, bool fill_future) { ++ TRACE_ME_MEMBER; ++ SmallVector input_attrs; ++ for (auto* arg : argument_handles) { ++ TF_ASSIGN_OR_RETURN(auto mlirArg, GetAttributeFromBuffer(arg)); ++ auto mlirArgInModuleContext = ++ CloneIntoContext(mlirArg, *module->getContext()); ++ input_attrs.push_back(mlirArgInModuleContext); ++ } ++ // LOG(INFO) << "EvalModule:\n" << ToString(module) << "\n"; ++ // LOG(INFO) << "Inputs: " << ToString(inputs) << "\n"; ++ // TF_ASSIGN_OR_RETURN(auto mlx_res, evalModule(module, inputs)); ++ ++ std::vector inputs; ++ for (auto& attr : input_attrs) { ++ TF_ASSIGN_OR_RETURN(auto input, ++ utils::array::fromDenseElementsAttr(attr)); ++ inputs.push_back(input); ++ } ++ ++ auto mlx_res = compiled_module_(inputs); ++ mx::eval(mlx_res); ++ ++ // LOG(INFO) << "Results: " << ToString(result.value()) << "\n"; ++ ++ // Naive memory space selection, only using CPU global memory. ++ PjRtMemorySpace* memory_space = ++ device->default_memory_space().value_or(nullptr); ++ std::vector> buffer_results; ++ for (auto i = 0; i < mlx_res.size(); i++) { ++ buffer_results.push_back( ++ CreateMlirBufferFromMlxArray(mlx_res[i], memory_space)); ++ } ++ ++ std::optional> future; ++ if (fill_future) { ++ // Synchronous! To make async, this would need to return a future that ++ // is ready when the computation is done. ++ future = PjRtFuture<>(absl::OkStatus()); ++ } ++ return PjRtLoadedExecutable::Result{future, std::move(buffer_results)}; ++ } ++ ++ absl::StatusOr>>> Execute( ++ absl::Span> argument_handles, ++ const xla::ExecuteOptions& options, ++ std::optional>>& returned_futures) override { ++ TRACE_ME_MEMBER; ++ if (argument_handles.size() != 1) { ++ // One arg handle per device. ++ return absl::InvalidArgumentError( ++ "MlirLoadedExecutable::Execute only supports a single argument " ++ "vector"); ++ } ++ ++ // Single device, synchronous, can always use 0. ++ PjRtDevice* device = devices_[0]; ++ bool fill_future = returned_futures.has_value(); ++ TF_ASSIGN_OR_RETURN( ++ PjRtLoadedExecutable::Result result, ++ ExecuteWithMlxInterpreter(argument_handles[0], module_.get(), device, ++ fill_future)); ++ std::vector>> results; ++ results.push_back(std::move(result.buffers)); ++ if (returned_futures.has_value()) { ++ returned_futures->push_back(std::move(result.future.value())); ++ } ++ return results; ++ } ++ ++ absl::StatusOr>> ExecuteSharded( ++ absl::Span argument_handles, PjRtDevice* device, ++ const xla::ExecuteOptions& options, ++ std::optional>& returned_future, bool fill_future) override { ++ TRACE_ME_MEMBER; ++ // Synchronous! To make async, have the device make a buffer with a ready ++ // future that is ready when the computation is done / buffer is ready. ++ TF_ASSIGN_OR_RETURN( ++ PjRtLoadedExecutable::Result result, ++ ExecuteWithMlxInterpreter(argument_handles, module_.get(), device, ++ fill_future)); ++ if (returned_future.has_value() && fill_future) { ++ returned_future = std::move(result.future); ++ } ++ return std::move(result.buffers); ++ } ++ ++ absl::StatusOr>> ExecutePortable( ++ absl::Span argument_handles, PjRtDevice* device, ++ const xla::ExecuteOptions& options, ++ std::optional>& returned_future, bool fill_future) override { ++ TRACE_ME_MEMBER; ++ // Synchronous! To make async, have the device make a buffer with a ready ++ // future that is ready when the computation is done / buffer is ready. ++ TF_ASSIGN_OR_RETURN( ++ PjRtLoadedExecutable::Result result, ++ ExecuteWithMlxInterpreter(argument_handles, module_.get(), device, ++ fill_future)); ++ if (returned_future.has_value() && fill_future) { ++ returned_future = std::move(result.future); ++ } ++ return std::move(result.buffers); ++ } ++ ++ void Delete() override { ++ TRACE_ME_MEMBER; ++ module_.release(); ++ module_ = nullptr; ++ } ++ bool IsDeleted() override { ++ TRACE_ME_MEMBER; ++ return !module_; ++ } ++ ++ // PjRtExecutable API. ++ int num_replicas() const override { ++ TRACE_ME_MEMBER; ++ return assignment_.replica_count(); ++ } ++ int num_partitions() const override { ++ TRACE_ME_MEMBER; ++ return assignment_.computation_count(); ++ } ++ int64_t SizeOfGeneratedCodeInBytes() const override { ++ // No generated code.. so just return 1. ++ TRACE_ME_MEMBER; ++ return 1; ++ } ++ absl::string_view name() const override { ++ TRACE_ME_MEMBER; ++ return name_; ++ } ++ ++ absl::StatusOr>> GetHloModules() ++ const override { ++ // TODO: This shouldn't be needed for an MLIR plugin, its only used in the ++ // JAX layer for determining output sharding, which exists on the mlir ++ // module. ++ TRACE_ME_MEMBER; ++ auto moduleClone = llvm::cast(module_.get()->clone()); ++ TF_ASSIGN_OR_RETURN(auto hlo_module, ++ xla::ConvertStablehloToHlo(moduleClone)); ++ return std::vector>{std::move(hlo_module)}; ++ } ++ ++ absl::StatusOr>> ++ GetOutputMemoryKinds() const override { ++ TRACE_ME_MEMBER; ++ return UNIMPLEMENTED(GetOutputMemoryKinds); ++ } ++ ++ absl::StatusOr FingerprintExecutable() const override { ++ TRACE_ME_MEMBER; ++ return UNIMPLEMENTED(FingerprintExecutable); ++ } ++ ++ private: ++ std::string name_; ++ DeviceAssignment assignment_; ++ absl::Span devices_; ++ PjRtClient* client_; ++ std::function(const std::vector&)> ++ compiled_module_; ++ OpLookup op_lookup_; ++ ++ // MLIR ++ MLIRContext context_; ++ mlir::OwningOpRef module_; ++}; ++ ++absl::StatusOr> StablehloMlxCompile( ++ mlir::ModuleOp module, DeviceAssignment assignment, PjRtClient* client) { ++ // TRACE_ME; ++ return MlirLoadedExecutable::Compile(module, assignment, client->devices(), ++ client); ++} ++ ++absl::StatusOr> StablehloMlxCompile( ++ xla::XlaComputation const& computation, xla::DeviceAssignment assignment, ++ xla::PjRtClient* client) { ++ // TRACE_ME; ++ MLIRContext context; ++ TF_ASSIGN_OR_RETURN(auto module, ++ ConvertHloToStablehlo(context, &computation.proto())); ++ return StablehloMlxCompile(module.get(), assignment, client); ++} ++ ++} // namespace mlir::stablehlo +diff --git a/xla/pjrt/plugin/mlx/executable.h b/xla/pjrt/plugin/mlx/executable.h +new file mode 100644 +index 0000000000..e579af4e1f +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/executable.h +@@ -0,0 +1,244 @@ ++/* Copyright 2024 The OpenXLA Authors. ++Licensed under the Apache License, Version 2.0 (the "License"); ++you may not use this file except in compliance with the License. ++You may obtain a copy of the License at ++ http://www.apache.org/licenses/LICENSE-2.0 ++Unless required by applicable law or agreed to in writing, software ++distributed under the License is distributed on an "AS IS" BASIS, ++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++See the License for the specific language governing permissions and ++limitations under the License. ++==============================================================================*/ ++ ++#ifndef XLA_PJRT_PLUGIN_STABLEHLO_MLX_EXECUTABLE_H_ ++#define XLA_PJRT_PLUGIN_STABLEHLO_MLX_EXECUTABLE_H_ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "mlir/IR/BuiltinOps.h" ++#include "mlx/mlx.h" ++#include "xla/pjrt/plugin/mlx/utils.h" ++#include "xla/hlo/builder/xla_computation.h" ++#include "xla/pjrt/pjrt_client.h" ++#include "xla/service/computation_placer.h" ++ ++namespace mlir::stablehlo { ++ ++struct Node { ++ std::string name; ++ std::vector> children; ++ ++ Node(std::string name) : name(name) {} ++}; ++ ++class Tree { ++ public: ++ std::unique_ptr root; ++ ++ Tree() : root(nullptr) {} ++ ++ Node* insert(std::string name, Node* parent = nullptr) { ++ auto new_node = std::make_unique(name); ++ if (!root) { ++ root = std::move(new_node); ++ return root.get(); ++ } else if (parent) { ++ parent->children.push_back(std::move(new_node)); ++ return parent->children.back().get(); ++ } ++ } ++ ++ void remove(const std::string& name) { ++ if (!root) return; ++ ++ // Special case: root node matches ++ if (root->name == name) { ++ root.reset(); // deletes entire tree ++ return; ++ } ++ ++ remove_helper(root, name); ++ } ++ ++ std::set getAllFnNames() { ++ std::set fn_names; ++ if (!root) return fn_names; ++ ++ std::function traverse = [&](Node* node) { ++ fn_names.insert(node->name); ++ for (auto& child : node->children) { ++ traverse(child.get()); ++ } ++ }; ++ traverse(root.get()); ++ return fn_names; ++ } ++ ++ std::unordered_set getCompilableFns() { ++ std::unordered_set compilable_fns; ++ if (!root) return compilable_fns; ++ ++ std::function traverse = [&](Node* node) { ++ if (node->children.empty()) { ++ compilable_fns.insert(node->name); ++ } else { ++ for (auto& child : node->children) { ++ traverse(child.get()); ++ } ++ } ++ }; ++ traverse(root.get()); ++ return compilable_fns; ++ } ++ ++ private: ++ void remove_helper(std::unique_ptr& current, const std::string& name) { ++ if (!current) return; ++ ++ auto& children = current->children; ++ ++ // Remove matching children ++ children.erase(std::remove_if(children.begin(), children.end(), ++ [&](std::unique_ptr& child) { ++ return child->name == name; ++ }), ++ children.end()); ++ ++ // Recurse on remaining children ++ for (auto& child : children) { ++ remove_helper(child, name); ++ } ++ } ++}; ++ ++template ++class CompiledOpInfo { ++ public: ++ std::tuple captures; ++ std::vector> input_metadata; ++ std::vector> output_metadata; ++ ++ CompiledOpInfo(std::vector inputs, ++ std::vector> outputs, ++ T... capture_values); ++ ++ CompiledOpInfo(std::vector> inputs, ++ std::vector> outputs, ++ T... capture_values); ++ ++ bool operator==(const CompiledOpInfo& other) const { ++ return captures == other.captures && ++ input_metadata == other.input_metadata && ++ output_metadata == other.output_metadata; ++ } ++}; ++ ++template ++CompiledOpInfo::CompiledOpInfo( ++ std::vector inputs, ++ std::vector> outputs, T... capture_values) ++ : captures(std::make_tuple(capture_values...)), output_metadata(outputs) { ++ input_metadata.reserve(inputs.size()); ++ for (const auto& input : inputs) { ++ input_metadata.push_back(std::make_pair(input.dtype(), input.shape())); ++ } ++} ++ ++template ++CompiledOpInfo::CompiledOpInfo( ++ std::vector> inputs, ++ std::vector> outputs, T... capture_values) ++ : captures(std::make_tuple(capture_values...)), ++ input_metadata(inputs), ++ output_metadata(outputs) {} ++ ++using CompiledOpSign = ++ std::function(const std::vector&)>; ++ ++using CallOpInfo = CompiledOpInfo; ++ ++// stablehlo::IotaOp - Captures: (dimensions, dtype, iota_dim, result_shape) ++using IotaOpInfo = CompiledOpInfo, mx::Dtype, int64_t, ++ std::vector>; ++ ++// stablehlo::CbrtOp - Captures: () ++using CbrtOpInfo = CompiledOpInfo<>; ++ ++// stablehlo::BroadcastInDimOp - Captures: (broadcast_dims, target_shape) ++using BroadcastInDimOpInfo = ++ CompiledOpInfo, std::vector>; ++ ++// stablehlo::DotGeneralOp - Captures: (lhs_batch_dims, rhs_batch_dims, ++// lhs_contract_dims, rhs_contract_dims) ++using DotGeneralOpInfo = ++ CompiledOpInfo, std::vector, ++ std::vector, std::vector>; ++ ++// stablehlo::GatherOp - Captures: (collapsed_slice_dims, index_vector_dim, ++// offset_dims, operand_batching_dims, result_shape, slice_sizes, ++// start_index_map, start_indices_batching_dims) ++using GatherOpInfo = CompiledOpInfo, int64_t, ++ std::vector, std::vector, ++ std::vector, std::vector, ++ std::vector, std::vector>; ++ ++// stablehlo::ScatterOp - Captures: (index_vector_dim, input_batching_dims, ++// inserted_window_dims, scatter_dims_to_operand_dims, ++// scatter_indices_batching_dims, scatter_type, update_window_dims) ++using ScatterOpInfo = ++ CompiledOpInfo, std::vector, ++ std::vector, std::vector, ++ utils::ScatterType, std::vector>; ++ ++// stablehlo::PadOp - Captures: (result_shape, edge_pad_low, interior_pad) ++using PadOpInfo = CompiledOpInfo, std::vector, ++ std::vector>; ++ ++absl::StatusOr> StablehloMlxCompile( ++ mlir::ModuleOp module, xla::DeviceAssignment assignment, ++ xla::PjRtClient* client); ++ ++absl::StatusOr> StablehloMlxCompile( ++ xla::XlaComputation const& computation, xla::DeviceAssignment assignment, ++ xla::PjRtClient* client); ++ ++} // namespace mlir::stablehlo ++ ++template ++struct std::hash> { ++ std::size_t operator()( ++ const mlir::stablehlo::CompiledOpInfo& info) const noexcept { ++ std::size_t seed = 0; ++ hash_tuple(seed, info.captures); ++ for (const auto& meta : info.input_metadata) { ++ hash_combine(seed, std::hash{}(meta.first)); ++ hash_combine(seed, std::hash>{}(meta.second)); ++ } ++ for (const auto& meta : info.output_metadata) { ++ hash_combine(seed, std::hash{}(meta.first)); ++ hash_combine(seed, std::hash>{}(meta.second)); ++ } ++ return seed; ++ } ++}; ++ ++namespace mlir::stablehlo { ++struct OpLookup { ++ std::unordered_map, CompiledOpSign> call; ++ std::unordered_map iota; ++ std::unordered_map cbrt; ++ std::unordered_map broadcast_in_dim; ++ std::unordered_map dot_general; ++ std::unordered_map gather; ++ std::unordered_map scatter; ++ std::unordered_map pad; ++}; ++ ++}; // namespace mlir::stablehlo ++ ++#endif // XLA_PJRT_PLUGIN_STABLEHLO_MLX_EXECUTABLE_H_ +\ No newline at end of file +diff --git a/xla/pjrt/plugin/mlx/logging.cc b/xla/pjrt/plugin/mlx/logging.cc +new file mode 100644 +index 0000000000..3f21f67baf +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/logging.cc +@@ -0,0 +1,89 @@ ++/* Copyright 2024 The OpenXLA Authors. ++Licensed under the Apache License, Version 2.0 (the "License"); ++you may not use this file except in compliance with the License. ++You may obtain a copy of the License at ++ http://www.apache.org/licenses/LICENSE-2.0 ++Unless required by applicable law or agreed to in writing, software ++distributed under the License is distributed on an "AS IS" BASIS, ++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++See the License for the specific language governing permissions and ++limitations under the License. ++==============================================================================*/ ++ ++#include "xla/pjrt/plugin/mlx/logging.h" ++ ++#include ++ ++#include "absl/base/log_severity.h" ++#include "absl/log/globals.h" ++#include "llvm/Support/raw_ostream.h" ++#include "mlir/IR/Attributes.h" ++#include "mlir/IR/BuiltinAttributes.h" ++#include "mlir/IR/Operation.h" ++ ++namespace mlir::stablehlo { ++ ++std::string ToString(mlir::Attribute attr) { ++ std::string out; ++ llvm::raw_string_ostream os(out); ++ attr.print(os); ++ return out; ++} ++ ++std::string ToString(SmallVector attrs) { ++ std::string out; ++ llvm::raw_string_ostream os(out); ++ os << "["; ++ bool first = true; ++ for (auto attr : attrs) { ++ if (!first) os << ", "; ++ first = false; ++ attr.print(os); ++ } ++ os << "]"; ++ return out; ++} ++std::string ToString(Operation& op) { ++ std::string out; ++ llvm::raw_string_ostream os(out); ++ op.print(os); ++ return out; ++} ++ ++std::string ToString(Operation* op) { ++ std::string out; ++ llvm::raw_string_ostream os(out); ++ os << *op; ++ return out; ++} ++ ++std::string ToString(mlir::Value value) { ++ std::string out; ++ llvm::raw_string_ostream os(out); ++ value.print(os); ++ return out; ++} ++ ++std::string ToString(mlir::Block& block) { ++ std::string out; ++ llvm::raw_string_ostream os(out); ++ block.print(os); ++ return out; ++} ++ ++void SetupLogLevelFromEnv() { ++ absl::SetMinLogLevel(absl::LogSeverityAtLeast::kError); ++ const char* log_env = std::getenv("PJRT_LOG_LEVEL"); ++ if (!log_env) return; ++ if (strcmp(log_env, "INFO") == 0) { ++ absl::SetMinLogLevel(absl::LogSeverityAtLeast::kInfo); ++ } else if (strcmp(log_env, "WARNING") == 0) { ++ absl::SetMinLogLevel(absl::LogSeverityAtLeast::kWarning); ++ } else if (strcmp(log_env, "ERROR") == 0) { ++ absl::SetMinLogLevel(absl::LogSeverityAtLeast::kError); ++ } else { ++ LOG(ERROR) << "Invalid PJRT_LOG_LEVEL: " << log_env; ++ } ++} ++ ++} // namespace mlir::stablehlo +\ No newline at end of file +diff --git a/xla/pjrt/plugin/mlx/logging.h b/xla/pjrt/plugin/mlx/logging.h +new file mode 100644 +index 0000000000..762dfabbcc +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/logging.h +@@ -0,0 +1,42 @@ ++/* Copyright 2024 The OpenXLA Authors. ++Licensed under the Apache License, Version 2.0 (the "License"); ++you may not use this file except in compliance with the License. ++You may obtain a copy of the License at ++ http://www.apache.org/licenses/LICENSE-2.0 ++Unless required by applicable law or agreed to in writing, software ++distributed under the License is distributed on an "AS IS" BASIS, ++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++See the License for the specific language governing permissions and ++limitations under the License. ++==============================================================================*/ ++ ++#ifndef XLA_PJRT_PLUGIN_STABLEHLO_MLX_LOGGING_H_ ++#define XLA_PJRT_PLUGIN_STABLEHLO_MLX_LOGGING_H_ ++ ++// This file has some joint logging to allow LOG and VLOG to play well with ++// MLIR data structures ++ ++#include "absl/log/log.h" ++#include "mlir/IR/Attributes.h" ++#include "mlir/IR/BuiltinAttributes.h" ++ ++#define LOG_UNIMPLEMENTED(name) \ ++ LOG(ERROR) << "MlxPjRtBuffer::" #name " is not implemented" ++ ++#define TRACE_ME LOG(INFO) << __func__; ++ ++#define TRACE_ME_MEMBER LOG(INFO) << __func__ << "(" << (void*)this << ")\n"; ++ ++namespace mlir::stablehlo { ++std::string ToString(mlir::Attribute attr); ++std::string ToString(llvm::SmallVector attrs); ++std::string ToString(Operation* op); ++std::string ToString(Operation& op); ++std::string ToString(mlir::Value op); ++std::string ToString(mlir::Block& block); ++ ++// Looks for `PJRT_LOG_LEVEL = INFO|WARNING|ERROR` in env variables. ++void SetupLogLevelFromEnv(); ++} // namespace mlir::stablehlo ++ ++#endif // XLA_PJRT_PLUGIN_STABLEHLO_MLX_LOGGING_H_ +\ No newline at end of file +diff --git a/xla/pjrt/plugin/mlx/plugin_pjrt_test.cc b/xla/pjrt/plugin/mlx/plugin_pjrt_test.cc +new file mode 100644 +index 0000000000..d56a22565f +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/plugin_pjrt_test.cc +@@ -0,0 +1,33 @@ ++/* Copyright 2022 The OpenXLA Authors. ++Licensed under the Apache License, Version 2.0 (the "License"); ++you may not use this file except in compliance with the License. ++You may obtain a copy of the License at ++ http://www.apache.org/licenses/LICENSE-2.0 ++Unless required by applicable law or agreed to in writing, software ++distributed under the License is distributed on an "AS IS" BASIS, ++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++See the License for the specific language governing permissions and ++limitations under the License. ++==============================================================================*/ ++ ++#include "xla/pjrt/c/pjrt_c_api_test.h" ++#include "xla/pjrt/c/pjrt_c_api_wrapper_impl.h" ++#include "xla/pjrt/pjrt_client_test.h" ++#include "xla/pjrt/plugin/mlx/client_c_pjrt.h" ++#include "xla/pjrt/plugin/mlx/client_cpp_pjrt.h" ++ ++namespace pjrt { ++namespace { ++ ++const bool kUnused = ++ (RegisterPjRtCApiTestFactory([]() { return GetPjrtApi(); }, ++ /*platform_name=*/"stablehlo_mlx"), ++ true); ++ ++const bool kUnused2 = (xla::RegisterTestClientFactory([]() { ++ return mlir::stablehlo::CreateStablehloMlxPjrtClient(); ++ }), ++ true); ++ ++} // namespace ++} // namespace pjrt +\ No newline at end of file +diff --git a/xla/pjrt/plugin/mlx/utils.cc b/xla/pjrt/plugin/mlx/utils.cc +new file mode 100644 +index 0000000000..5fcfc318ab +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/utils.cc +@@ -0,0 +1,292 @@ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "xla/pjrt/plugin/mlx/utils.h" ++#include "absl/status/status.h" ++#include "absl/status/statusor.h" ++#include "absl/types/span.h" ++#include "llvm/ADT/TypeSwitch.h" ++#include "mlir/IR/BuiltinAttributes.h" ++#include "mlir/IR/Types.h" ++#include "mlx/mlx.h" ++#include "tsl/platform/statusor.h" ++#include "xla/mlir/utils/type_util.h" ++#include "xla/shape_util.h" ++ ++#include "xla/literal.h" ++#include "xla/util.h" ++ ++namespace mx = mlx::core; ++ ++std::tuple shapeAndBytes( ++ absl::Span dims, ++ std::optional> byte_strides, mx::Dtype dtype) { ++ mx::Shape shape(dims.size()); ++ mx::Strides strides; ++ mx::Shape raw_buffer_shape = {shape.size() && !byte_strides.has_value() ? 1 ++ : 0}; ++ for (auto i = 0; i < dims.size(); ++i) { ++ shape[i] = static_cast(dims[i]); ++ if (byte_strides.has_value()) { ++ auto stride_value = byte_strides.value()[i]; ++ strides.push_back(stride_value / dtype.size()); ++ if (strides[i] != 0) { ++ raw_buffer_shape[0] = std::max( ++ raw_buffer_shape[0], static_cast(stride_value) * shape[i]); ++ } ++ } else { ++ raw_buffer_shape[0] *= static_cast(dims[i]); ++ } ++ } ++ if (!byte_strides.has_value()) { ++ raw_buffer_shape[0] *= dtype.size(); ++ } else if (raw_buffer_shape[0] == 0 and ++ std::find(dims.begin(), dims.end(), 0) == dims.end()) { ++ raw_buffer_shape[0] += dtype.size(); ++ } ++ return std::make_tuple(shape, raw_buffer_shape, strides); ++} ++ ++namespace utils { ++ ++namespace dtype { ++xla::PrimitiveType asXlaPrimitiveType(mx::Dtype dtype) { ++ switch (dtype) { ++ case mx::bool_: ++ return xla::PrimitiveType::PRED; ++ case mx::uint8: ++ return xla::PrimitiveType::U8; ++ case mx::uint16: ++ return xla::PrimitiveType::U16; ++ case mx::uint32: ++ return xla::PrimitiveType::U32; ++ case mx::uint64: ++ return xla::PrimitiveType::U64; ++ case mx::int8: ++ return xla::PrimitiveType::S8; ++ case mx::int16: ++ return xla::PrimitiveType::S16; ++ case mx::int32: ++ return xla::PrimitiveType::S32; ++ case mx::int64: ++ return xla::PrimitiveType::S64; ++ case mx::float16: ++ return xla::PrimitiveType::F16; ++ case mx::float32: ++ return xla::PrimitiveType::F32; ++ case mx::bfloat16: ++ return xla::PrimitiveType::BF16; ++ case mx::complex64: ++ return xla::PrimitiveType::C64; ++ } ++} ++ ++absl::StatusOr fromXlaPrimitiveType(xla::PrimitiveType dtype) { ++ switch (dtype) { ++ case xla::PrimitiveType::PRED: ++ return mx::bool_; ++ case xla::PrimitiveType::BF16: ++ return mx::bfloat16; ++ case xla::PrimitiveType::F16: ++ return mx::float16; ++ case xla::PrimitiveType::F32: ++ return mx::float32; ++ case xla::PrimitiveType::S8: ++ return mx::int8; ++ case xla::PrimitiveType::S16: ++ return mx::int16; ++ case xla::PrimitiveType::S32: ++ return mx::int32; ++ case xla::PrimitiveType::S64: ++ return mx::int64; ++ case xla::PrimitiveType::U8: ++ return mx::uint8; ++ case xla::PrimitiveType::U16: ++ return mx::uint16; ++ case xla::PrimitiveType::U32: ++ return mx::uint32; ++ case xla::PrimitiveType::U64: ++ return mx::uint64; ++ case xla::PrimitiveType::C64: ++ return mx::complex64; ++ default: ++ return absl::InvalidArgumentError("Unsupported type"); ++ } ++} ++ ++absl::StatusOr fromMlirType(mlir::Type type) { ++ auto primitive_type = xla::ConvertMlirTypeToPrimitiveType(type); ++ if (primitive_type == xla::PrimitiveType::PRIMITIVE_TYPE_INVALID) { ++ return xla::Internal("Unsupported type: %s", ++ xla::PrimitiveType_Name(primitive_type)); ++ } ++ return fromXlaPrimitiveType(primitive_type); ++} ++ ++int mlirTypeByteWidth(mlir::Type type) { ++ return xla::primitive_util::ByteWidth( ++ xla::ConvertMlirTypeToPrimitiveType(type)); ++} ++} // namespace dtype ++ ++namespace array { ++absl::StatusOr fromHostBuffer( ++ const void* data, absl::Span dims, ++ std::optional> byte_strides, ++ xla::PrimitiveType type) { ++ TF_ASSIGN_OR_RETURN(mx::Dtype dtype, dtype::fromXlaPrimitiveType(type)); ++ auto [shape, raw_buffer_shape, strides] = ++ shapeAndBytes(dims, byte_strides, dtype); ++ mx::array typed_view = mx::view(mx::array(reinterpret_cast(data), ++ raw_buffer_shape, mx::uint8), ++ dtype); ++ if (byte_strides.has_value()) { ++ auto res = mx::contiguous(mx::as_strided(typed_view, shape, strides, 0)); ++ res.eval(); ++ return res; ++ } else { ++ auto res = mx::reshape(typed_view, shape); ++ res.eval(); ++ return res; ++ } ++} ++ ++absl::StatusOr fromHostLiteral(const xla::LiteralSlice& literal) { ++ return fromHostBuffer(literal.untyped_data(), literal.shape().dimensions(), ++ std::nullopt, literal.shape().element_type()); ++} ++ ++template ++absl::StatusOr fromDenseElementsAttr(mlir::DenseElementsAttr attr) { ++ auto attr_type = attr.getType(); ++ // convert to mlx shape ++ std::vector shape(attr_type.getShape().begin(), ++ attr_type.getShape().end()); ++ ++ // handle splat values ++ if (attr.isSplat()) { ++ if constexpr (std::is_same::value) { ++ return mx::full( ++ shape, static_cast(attr.getSplatValue())); ++ } else if constexpr (std::is_same::value) { ++ return mx::full( ++ shape, static_cast(attr.getSplatValue())); ++ } else if constexpr (std::is_same>::value) { ++ return mx::full( ++ shape, static_cast(attr.getSplatValue())); ++ } else { ++ return mx::full(shape, attr.getSplatValue()); ++ } ++ } ++ ++ // handle non-splat values that are expanded to fit shape ++ auto it = attr.getValues(); ++ auto buffer = llvm::to_vector(it); ++ if (attr.size() != attr.getNumElements()) { ++ if constexpr (std::is_same::value) { ++ return mx::full( ++ shape, ++ mx::array(reinterpret_cast(buffer.data()), ++ {static_cast(attr.size())})); ++ } else if constexpr (std::is_same::value) { ++ return mx::full( ++ shape, ++ mx::array(reinterpret_cast(buffer.data()), ++ {static_cast(attr.size())})); ++ } else if constexpr (std::is_same>::value) { ++ return mx::full( ++ shape, ++ mx::array(reinterpret_cast(buffer.data()), ++ {static_cast(attr.size())})); ++ } else { ++ return mx::full(shape, mx::array(buffer.begin(), ++ {static_cast(attr.size())})); ++ } ++ } ++ ++ if constexpr (std::is_same::value) { ++ return mx::array(reinterpret_cast(buffer.data()), ++ shape); ++ } else if constexpr (std::is_same::value) { ++ return mx::array(reinterpret_cast(buffer.data()), ++ shape); ++ } else if constexpr (std::is_same>::value) { ++ return mx::array(reinterpret_cast(buffer.data()), ++ shape); ++ } else { ++ return mx::array(buffer.begin(), shape); ++ } ++} ++ ++absl::StatusOr fromDenseElementsAttr(mlir::DenseElementsAttr attr) { ++ auto element_type = ++ xla::ConvertMlirTypeToPrimitiveType(attr.getType().getElementType()); ++ switch (element_type) { ++ case xla::PrimitiveType::PRED: ++ return fromDenseElementsAttr(attr); ++ case xla::PrimitiveType::U8: ++ return fromDenseElementsAttr(attr); ++ case xla::PrimitiveType::S8: ++ return fromDenseElementsAttr(attr); ++ case xla::PrimitiveType::U16: ++ return fromDenseElementsAttr(attr); ++ case xla::PrimitiveType::S16: ++ return fromDenseElementsAttr(attr); ++ case xla::PrimitiveType::U32: ++ return fromDenseElementsAttr(attr); ++ case xla::PrimitiveType::S32: ++ return fromDenseElementsAttr(attr); ++ case xla::PrimitiveType::U64: ++ return fromDenseElementsAttr(attr); ++ case xla::PrimitiveType::S64: ++ return fromDenseElementsAttr(attr); ++ case xla::PrimitiveType::F16: ++ return fromDenseElementsAttr(attr); ++ case xla::PrimitiveType::BF16: ++ return fromDenseElementsAttr(attr); ++ case xla::PrimitiveType::F32: ++ return fromDenseElementsAttr(attr); ++ case xla::PrimitiveType::C64: ++ return fromDenseElementsAttr>(attr); ++ default: ++ return xla::Internal("Unsupported type: %s", ++ xla::PrimitiveType_Name(element_type)); ++ } ++} ++ ++absl::StatusOr fromOperand( ++ mlir::Value operand, absl::Span block_args, ++ const std::unordered_map>& ++ transient_buffers, ++ const std::unordered_map& init_values) {} ++ ++absl::StatusOr fromOperand( ++ mlir::Value operand, absl::Span block_args, ++ const std::unordered_map>& ++ transient_buffers) { ++ return fromOperand(operand, block_args, transient_buffers, {}); ++} ++ ++} // namespace array ++ ++void printVector(const std::string& name, const std::vector& vec, ++ bool indent) { ++ if (indent) { ++ std::cout << "\t"; ++ } ++ std::cout << name.c_str() << ": { "; ++ for (auto i = 0; i < vec.size(); i++) { ++ std::cout << std::to_string(vec[i]); ++ if (i < vec.size() - 1) { ++ std::cout << ", "; ++ } ++ } ++ std::cout << " }" << std::endl; ++} ++} // namespace utils +diff --git a/xla/pjrt/plugin/mlx/utils.h b/xla/pjrt/plugin/mlx/utils.h +new file mode 100644 +index 0000000000..1a6d85c060 +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/utils.h +@@ -0,0 +1,129 @@ ++/* Copyright 2024 The OpenXLA Authors. ++Licensed under the Apache License, Version 2.0 (the "License"); ++you may not use this file except in compliance with the License. ++You may obtain a copy of the License at ++ http://www.apache.org/licenses/LICENSE-2.0 ++Unless required by applicable law or agreed to in writing, software ++distributed under the License is distributed on an "AS IS" BASIS, ++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++See the License for the specific language governing permissions and ++limitations under the License. ++==============================================================================*/ ++ ++#ifndef XLA_PJRT_PLUGIN_STABLEHLO_MLX_UTILS_H_ ++#define XLA_PJRT_PLUGIN_STABLEHLO_MLX_UTILS_H_ ++ ++#include ++#include ++#include ++#include ++#include ++ ++#include "absl/status/status.h" ++#include "absl/status/statusor.h" ++#include "absl/types/span.h" ++#include "mlir/IR/BuiltinAttributes.h" ++#include "mlir/IR/Types.h" ++#include "mlx/mlx.h" ++#include "xla/literal.h" ++#include "xla/util.h" ++ ++namespace mx = mlx::core; ++ ++namespace utils { ++ ++enum ScatterType { ++ Replace, ++ Add, ++ Prod, ++ Max, ++ Min, ++}; ++namespace dtype { ++xla::PrimitiveType asXlaPrimitiveType(mx::Dtype dtype); ++ ++absl::StatusOr fromXlaPrimitiveType(xla::PrimitiveType dtype); ++ ++absl::StatusOr fromMlirType(mlir::Type type); ++ ++int mlirTypeByteWidth(mlir::Type type); ++} // namespace dtype ++ ++namespace array { ++absl::StatusOr fromHostBuffer( ++ const void* data, absl::Span dims, ++ std::optional> byte_strides, ++ xla::PrimitiveType type); ++ ++absl::StatusOr fromHostLiteral(const xla::LiteralSlice& literal); ++ ++absl::StatusOr fromDenseElementsAttr(mlir::DenseElementsAttr attr); ++ ++absl::StatusOr fromOperand( ++ mlir::Value operand, absl::Span block_args, ++ const std::unordered_map>& ++ transient_buffers, ++ const std::unordered_map& init_values); ++ ++absl::StatusOr fromOperand( ++ mlir::Value operand, absl::Span block_args, ++ const std::unordered_map>& ++ transient_buffers); ++} // namespace array ++ ++void printVector(const std::string& name, const std::vector& vec, ++ bool indent = false); ++} // namespace utils ++ ++// Helper function to hash a tuple ++template ++inline void hash_combine(std::size_t& seed, const T& val) { ++ seed ^= std::hash{}(val) + 0x9e3779b9 + (seed << 6) + (seed >> 2); ++} ++ ++template ++inline ++ typename std::enable_if::value, void>::type ++ hash_tuple(std::size_t&, const Tuple&) {} ++ ++template ++ inline typename std::enable_if < ++ Index::value, void>::type hash_tuple( ++ std::size_t& seed, const Tuple& tuple) { ++ hash_combine(seed, std::get(tuple)); ++ hash_tuple(seed, tuple); ++} ++ ++template <> ++struct std::hash { ++ std::size_t operator()(const utils::ScatterType& value) const noexcept { ++ return std::hash::type>{}( ++ static_cast::type>(value)); ++ } ++}; ++ ++template ++struct std::hash> { ++ std::size_t operator()(const std::vector& vec) const { ++ std::size_t seed = 0; ++ for (const T& elem : vec) { ++ seed ^= std::hash{}(elem) + 0x9e3779b9 + (seed << 6) + (seed >> 2); ++ } ++ return seed; ++ } ++}; ++ ++template <> ++struct std::hash { ++ std::size_t operator()(const mx::Dtype& dtype) const noexcept { ++ std::size_t seed = 0; ++ hash_combine(seed, std::hash{}(dtype.size())); ++ hash_combine(seed, ++ std::hash::type>{}( ++ static_cast::type>( ++ dtype.val()))); ++ return seed; ++ } ++}; ++ ++#endif // XLA_PJRT_PLUGIN_STABLEHLO_MLX_UTILS_H_ +\ No newline at end of file diff --git a/third_party/modules/xla/20250317.2-71c67e2/source.json b/third_party/modules/xla/20250317.2-71c67e2/source.json index 9d579abc..2a9d52ba 100644 --- a/third_party/modules/xla/20250317.2-71c67e2/source.json +++ b/third_party/modules/xla/20250317.2-71c67e2/source.json @@ -1,16 +1,17 @@ { - "strip_prefix": "xla-71c67e2a4f40267115a0d4ea7c36748bbe7e750e", - "url": "https://github.com/openxla/xla/archive/71c67e2a4f40267115a0d4ea7c36748bbe7e750e.tar.gz", - "integrity": "sha256-j6D1MC7+WsbZ+Ve3hPmDlCzZX1yV6RIP2BWDgQcbcYc=", - "overlay": { - "llvm.bzl": "", - "MODULE.bazel": "", - "tsl.bzl": "", - "workspace.bzl": "" - }, - "patch_strip": 1, - "patches": { - "0001-bazel-migration-to-bazel-8.1.1.patch": "", - "0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch": "" - } + "strip_prefix": "xla-71c67e2a4f40267115a0d4ea7c36748bbe7e750e", + "url": "https://github.com/openxla/xla/archive/71c67e2a4f40267115a0d4ea7c36748bbe7e750e.tar.gz", + "integrity": "sha256-j6D1MC7+WsbZ+Ve3hPmDlCzZX1yV6RIP2BWDgQcbcYc=", + "overlay": { + "llvm.bzl": "", + "MODULE.bazel": "", + "tsl.bzl": "", + "workspace.bzl": "" + }, + "patch_strip": 1, + "patches": { + "0001-bazel-migration-to-bazel-8.1.1.patch": "", + "0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch": "", + "0003-Added-MLX-PjRt-Plugin.patch": "" + } } diff --git a/zml/context.zig b/zml/context.zig index 03de62a3..9f239b9e 100644 --- a/zml/context.zig +++ b/zml/context.zig @@ -120,7 +120,7 @@ pub const Context = struct { self.* = undefined; } - const prefered_targets = [_]Target{ .tpu, .neuron, .cuda, .rocm, .cpu }; + const prefered_targets = [_]Target{ .tpu, .neuron, .cuda, .rocm, .cpu, .mlx }; /// Automatically selects the best Platform loaded in the current Context. /// diff --git a/zml/platform.zig b/zml/platform.zig index ec4ae4c6..4ae4b06d 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -99,6 +99,7 @@ const _CreateOptions = struct { rocm: struct {} = .{}, tpu: struct {} = .{}, neuron: struct {} = .{}, + mlx: struct {} = .{}, pub const Cuda = struct { allocator: Allocator = .{ .bfc = .{} }, From 6f7c941274a7483832aef6a5ee4b2ced38003036 Mon Sep 17 00:00:00 2001 From: James Deal Date: Wed, 21 May 2025 23:55:36 -0500 Subject: [PATCH 02/13] remove custom_call FFI for print --- zml/context.zig | 4 +++- zml/tensor.zig | 4 ++++ zml/testing.zig | 6 ++++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/zml/context.zig b/zml/context.zig index 9f239b9e..31c2006b 100644 --- a/zml/context.zig +++ b/zml/context.zig @@ -177,7 +177,9 @@ pub const Context = struct { return error.NoDevicesFound; } - try CustomCall.registerZmlCustomCalls(p); + if (target != .mlx) { + try CustomCall.registerZmlCustomCalls(p); + } self.platforms.set(target, p); return p; diff --git a/zml/tensor.zig b/zml/tensor.zig index ae06a776..66ef46dd 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -2426,6 +2426,7 @@ pub const Tensor = struct { test gatherSlices { const zml = @import("zml.zig"); const platform = zml.testing.env(); + try zml.testing.mlxSkipTest(platform); const Local = struct { pub fn _gatherSlices(self: Tensor, slice_shape: Shape, indices: Tensor, opts: GatherOpts) Tensor { @@ -2893,6 +2894,9 @@ pub const Tensor = struct { test argsort { const zml = @import("zml.zig"); const platform = zml.testing.env(); + // TODO (@cryptodeal): Fix this for MLX PjRt Plugin Implementation. + // Currently segfaults; worked prior to rebase on latest XLA, MLX, and ZML. + try zml.testing.mlxSkipTest(platform); const Local = struct { pub fn _argsort(x: Tensor, axis_: u3, opts: ArgSortOpts) Tensor { diff --git a/zml/testing.zig b/zml/testing.zig index 8946f78b..6583520e 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -11,6 +11,12 @@ const log = std.log.scoped(.@"zml/testing"); var _platform: ?zml.Platform = null; +pub fn mlxSkipTest(platform: zml.Platform) !void { + if (platform.target == .mlx) { + return error.SkipZigTest; + } +} + pub fn env() zml.Platform { if (!builtin.is_test) @compileError("Cannot use zml.testing.env outside of a test block"); if (_platform == null) { From 74fae48bf8c2f5ba41188b92500024ee16e58685 Mon Sep 17 00:00:00 2001 From: James Deal Date: Thu, 22 May 2025 23:00:12 -0500 Subject: [PATCH 03/13] Clean up test output; comment failing tests as TODOs --- .../patches/0003-Added-MLX-PjRt-Plugin.patch | 309 +++++++++--------- zml/module.zig | 2 + zml/nn.zig | 4 + zml/ops.zig | 5 + zml/tensor.zig | 11 +- zml/torch.zig | 1 + 6 files changed, 177 insertions(+), 155 deletions(-) diff --git a/third_party/modules/xla/20250317.2-71c67e2/patches/0003-Added-MLX-PjRt-Plugin.patch b/third_party/modules/xla/20250317.2-71c67e2/patches/0003-Added-MLX-PjRt-Plugin.patch index b8917ab7..6e9d4535 100644 --- a/third_party/modules/xla/20250317.2-71c67e2/patches/0003-Added-MLX-PjRt-Plugin.patch +++ b/third_party/modules/xla/20250317.2-71c67e2/patches/0003-Added-MLX-PjRt-Plugin.patch @@ -208,10 +208,10 @@ index 0000000000..ca86b466bf \ No newline at end of file diff --git a/xla/pjrt/plugin/mlx/buffer.cc b/xla/pjrt/plugin/mlx/buffer.cc new file mode 100644 -index 0000000000..8bb2a335fe +index 0000000000..c4e7d410ac --- /dev/null +++ b/xla/pjrt/plugin/mlx/buffer.cc -@@ -0,0 +1,372 @@ +@@ -0,0 +1,371 @@ +/* Copyright 2024 The OpenXLA Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. @@ -283,7 +283,7 @@ index 0000000000..8bb2a335fe + array_(array), + shape_(shape), + memory_space_(memory_space) { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + } + + class MlirClonedExternalReference : public ExternalReference { @@ -291,7 +291,7 @@ index 0000000000..8bb2a335fe + explicit MlirClonedExternalReference(PjRtBuffer* buffer, + PjRtMemorySpace* memory_space) + : buffer_() { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + auto mlir_buffer = GetAttributeFromBuffer(buffer); + if (!mlir_buffer.ok()) { + LOG(ERROR) << "Could not get attribute from buffer: " @@ -353,51 +353,51 @@ index 0000000000..8bb2a335fe + } + + const Shape& on_device_shape() const override { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + return shape_; + } + absl::StatusOr logical_on_device_shape() override { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + return shape_; + } + + PjRtPlatformId platform_id() const { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + return client()->platform_id(); + } + absl::string_view platform_name() const { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + return client()->platform_name(); + } + + bool IsEmptyTuple() const { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + return shape_.IsTuple() && shape_.tuple_shapes().empty(); + } + + // Buffer knows device + client per older design, should only need + // memory_space. + PjRtMemorySpace* memory_space() const override { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + return memory_space_; + } + PjRtDevice* device() const override { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + return memory_space_->devices().front(); + } + PjRtClient* client() const override { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + return memory_space_->client(); + } + + absl::StatusOr> + AcquireExternalReference() override { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + return std::make_unique(this, memory_space_); + } + + xla::PjRtFuture<> ToLiteral(xla::MutableLiteralBase* literal) override { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + if (IsEmptyTuple()) { + return PjRtFuture<>( + xla::InvalidArgument("ToLiteral called on empty tuple")); @@ -417,7 +417,7 @@ index 0000000000..8bb2a335fe + PjRtFuture<> LazyToLiteral( + absl::AnyInvocable() &&> generator) + override { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + auto buffer = std::move(generator)(); + if (!buffer.ok()) return PjRtFuture<>(buffer.status()); + return ToLiteral(buffer.value()); @@ -426,20 +426,20 @@ index 0000000000..8bb2a335fe + absl::StatusOr GetOnDeviceSizeInBytes() const override { + // This is needed by AcquireExternalReference, for framework figuring out + // how to read the underlying buffer data. -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + if (!buffer_) return 0; + return buffer_.getRawData().size(); + } + + PjRtFuture<> CopyRawToHost(void* dst, int64_t offset, + int64_t transfer_size) override { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + return PjRtFuture<>(UNIMPLEMENTED(CopyRawToHost)); + } + + absl::StatusOr> + ReleaseDeviceMemoryOwnership(bool wait_for_operations_to_complete) override { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + auto external_ref = AcquireExternalReference(); + Delete(); + return external_ref; @@ -449,38 +449,37 @@ index 0000000000..8bb2a335fe + // Note: deleted and uninitialized appear the same in this scenario. + // Consider changing to mlir::NoneType when deleted. + void Delete() override { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + buffer_ = {}; + } + + bool IsDeleted() override { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + return !buffer_; + } + + absl::StatusOr> CopyToDevice( + xla::PjRtDevice* dst_device) { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + return CopyToMemorySpace( + dst_device->default_memory_space().value_or(nullptr)); + } + + absl::StatusOr> CopyToMemorySpace( + xla::PjRtMemorySpace* dst_memory_space) override { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + return CreateMlirBufferFromAttribute(array_, buffer_, dst_memory_space); + } + + void CopyToRemoteDevice( + xla::PjRtFuture serialized_descriptor, + xla::PjRtBuffer::RemoteSendCallback on_done) override { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + on_done(UNIMPLEMENTED(CopyToRemoteDevice), false); + } + -+ + xla::PjRtFuture<> GetReadyFuture() override { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + LOG(INFO) << "GetReadyFuture(" << (void*)this << ")\n"; + // Synchronous! To make async, have the device make a buffer with a ready + // future that is ready when the computation is done / buffer is ready. @@ -492,7 +491,7 @@ index 0000000000..8bb2a335fe + // GetExternalReference, lse it is copied back to host. + // Since we are using reference interpreter, we are running on CPU in a + // shared memory space. -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + return false; + } + @@ -747,10 +746,10 @@ index 0000000000..cc56626953 \ No newline at end of file diff --git a/xla/pjrt/plugin/mlx/client_cpp_pjrt.cc b/xla/pjrt/plugin/mlx/client_cpp_pjrt.cc new file mode 100644 -index 0000000000..93a6705d99 +index 0000000000..9bdf4095b5 --- /dev/null +++ b/xla/pjrt/plugin/mlx/client_cpp_pjrt.cc -@@ -0,0 +1,302 @@ +@@ -0,0 +1,298 @@ +/* Copyright 2024 The OpenXLA Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. @@ -811,7 +810,7 @@ index 0000000000..93a6705d99 + owned_memory_space_(), + memory_space_(nullptr) { + // Init device and memory space. -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + owned_devices_.push_back(GetStablehloMlxDevice(this)); + devices_.push_back(owned_devices_.back().get()); + owned_memory_space_ = std::make_unique( @@ -825,48 +824,48 @@ index 0000000000..93a6705d99 + }; + + absl::string_view platform_name() const override { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + return kStablehloMlxBackendName; + } + int process_index() const override { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + return 0; + } + + int device_count() const override { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + return devices_.size(); + } + + int addressable_device_count() const override { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + return devices_.size(); + } + + absl::Span devices() const override { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + return devices_; + } + absl::Span addressable_devices() const override { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + return devices_; + } + + absl::Span memory_spaces() const override { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + return absl::MakeSpan(&memory_space_, 1); + } + + // Return an ID that identifies the platform via tsl fingerprint. + xla::PjRtPlatformId platform_id() const override { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + return kStablehloMlxBackendId; + } + + // Returns a string containing human-readable, platform-specific version + // info (e.g. the CUDA version on GPU or libtpu version on Cloud TPU). + absl::string_view platform_version() const override { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + return "StableHLO MLX v0.1"; + } + @@ -879,7 +878,7 @@ index 0000000000..93a6705d99 + // a vector a device pointers? + absl::StatusOr LookupDevice( + xla::PjRtGlobalDeviceId global_device_id) const override { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + for (auto device : devices_) { + if (device->global_device_id() == global_device_id) { + return device; @@ -892,7 +891,7 @@ index 0000000000..93a6705d99 + + absl::StatusOr LookupAddressableDevice( + xla::PjRtLocalDeviceId local_device_id) const override { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + + for (auto* device : addressable_devices()) { + if (local_device_id == device->local_device_id()) { @@ -906,7 +905,7 @@ index 0000000000..93a6705d99 + + absl::StatusOr GetDefaultDeviceAssignment( + int num_replicas, int num_partitions) const override { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + xla::DeviceAssignment assignment(num_replicas, num_partitions); + for (int64_t i = 0; i < num_replicas; ++i) { + for (int64_t j = 0; j < num_partitions; ++j) { @@ -922,8 +921,7 @@ index 0000000000..93a6705d99 + ///////////////// + + absl::StatusOr> CreateErrorBuffer( -+ absl::Status error, const xla::Shape& shape, -+ xla::PjRtDevice* device) { ++ absl::Status error, const xla::Shape& shape, xla::PjRtDevice* device) { + // Prefer memory space implementation, device holding buffer is + // deprecated. + return CreateErrorBuffer(error, shape, @@ -938,7 +936,7 @@ index 0000000000..93a6705d99 + + absl::StatusOr> CreateUninitializedBuffer( + const xla::Shape& shape, xla::PjRtDevice* device) { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + return CreateMlirBufferUninitialized( + shape, device->default_memory_space().value_or(nullptr)); + } @@ -946,16 +944,15 @@ index 0000000000..93a6705d99 + absl::StatusOr> + CreateBuffersForAsyncHostToDevice(absl::Span shapes, + xla::PjRtDevice* device) { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + return CreateBuffersForAsyncHostToDevice( + shapes, device->default_memory_space().value_or(nullptr)); + } + + absl::StatusOr> -+ CreateBuffersForAsyncHostToDevice( -+ absl::Span shapes, -+ xla::PjRtMemorySpace* memory_space) { -+ TRACE_ME_MEMBER; ++ CreateBuffersForAsyncHostToDevice(absl::Span shapes, ++ xla::PjRtMemorySpace* memory_space) { ++ // TRACE_ME_MEMBER; + return UNIMPLEMENTED(CreateBuffersForAsyncHostToDevice); + } + @@ -965,7 +962,7 @@ index 0000000000..93a6705d99 + HostBufferSemantics host_buffer_semantics, + absl::AnyInvocable on_done_with_host_buffer, + xla::PjRtDevice* device) { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + return BufferFromHostBuffer( + data, type, dims, byte_strides, host_buffer_semantics, + std::move(on_done_with_host_buffer), @@ -978,7 +975,7 @@ index 0000000000..93a6705d99 + HostBufferSemantics host_buffer_semantics, + absl::AnyInvocable on_done_with_host_buffer, + xla::PjRtDevice* device, const xla::Layout* device_layout) { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + return BufferFromHostBuffer( + data, type, dims, byte_strides, host_buffer_semantics, + std::move(on_done_with_host_buffer), @@ -990,9 +987,8 @@ index 0000000000..93a6705d99 + std::optional> byte_strides, + HostBufferSemantics host_buffer_semantics, + absl::AnyInvocable on_done_with_host_buffer, -+ xla::PjRtMemorySpace* memory_space, -+ const xla::Layout* device_layout) { -+ TRACE_ME_MEMBER; ++ xla::PjRtMemorySpace* memory_space, const xla::Layout* device_layout) { ++ // TRACE_ME_MEMBER; + TF_ASSIGN_OR_RETURN( + mx::array array_buffer, + utils::array::fromHostBuffer(data, dims, byte_strides, type)); @@ -1007,15 +1003,14 @@ index 0000000000..93a6705d99 + + absl::StatusOr> BufferFromHostLiteral( + const xla::LiteralSlice& literal, xla::PjRtDevice* device) { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + return CreateMlirBufferFromLiteral( + literal, device->default_memory_space().value_or(nullptr)); + } + + absl::StatusOr> BufferFromHostLiteral( -+ const xla::LiteralSlice& literal, -+ xla::PjRtMemorySpace* memory_space) { -+ TRACE_ME_MEMBER; ++ const xla::LiteralSlice& literal, xla::PjRtMemorySpace* memory_space) { ++ // TRACE_ME_MEMBER; + return CreateMlirBufferFromLiteral(literal, memory_space); + } + @@ -1024,7 +1019,7 @@ index 0000000000..93a6705d99 + /////////// + absl::StatusOr> Compile( + mlir::ModuleOp module, xla::CompileOptions options) override { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + return mlir::stablehlo::StablehloMlxCompile( + module, GetDefaultDeviceAssignment(1, devices_.size()).value(), this); + } @@ -1033,7 +1028,7 @@ index 0000000000..93a6705d99 + absl::StatusOr> Compile( + const xla::XlaComputation& computation, + xla::CompileOptions options) override { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + return mlir::stablehlo::StablehloMlxCompile( + computation.proto(), + GetDefaultDeviceAssignment(1, devices_.size()).value(), this); @@ -1093,7 +1088,7 @@ index 0000000000..8c90942f0c \ No newline at end of file diff --git a/xla/pjrt/plugin/mlx/device.cc b/xla/pjrt/plugin/mlx/device.cc new file mode 100644 -index 0000000000..816ef734bf +index 0000000000..6984da5b7f --- /dev/null +++ b/xla/pjrt/plugin/mlx/device.cc @@ -0,0 +1,213 @@ @@ -1161,40 +1156,40 @@ index 0000000000..816ef734bf + debug_string_(mlxDeviceToString(mx::default_device())), + to_string_(absl::StrFormat("%s(id=%d,pid=%d)", debug_string_, + id_.value(), process_index_)) { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + } + + int id() const override { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + return id_.value(); + } + int process_index() const override { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + return process_index_; + } + int local_hardware_id() const { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + return local_hardware_id_; + } + + absl::string_view device_kind() const override { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + return DebugString(); + } + + absl::string_view DebugString() const override { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + return debug_string_; + } + + absl::string_view ToString() const override { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + return to_string_; + } + + const absl::flat_hash_map& Attributes() + const override { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + return attributes_; + } + @@ -1212,56 +1207,56 @@ index 0000000000..816ef734bf + public: + explicit StablehloMlxDevice(PjRtClient* client) + : PjRtDevice(), client_(client), description_(0, 0) { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + } + + absl::string_view DebugString() const override { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + return "StablehloMlxDevice"; + } + + PjRtLocalDeviceId local_device_id() const override { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + return PjRtLocalDeviceId(local_hardware_id().value()); + } + + PjRtLocalHardwareId local_hardware_id() const override { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + return PjRtLocalHardwareId(description_.local_hardware_id()); + } + + PjRtClient* client() const override { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + return client_; + } + + bool IsAddressable() const override { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + return process_index() == client()->process_index(); + } + + absl::Status TransferToInfeed(const LiteralSlice& literal) override { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + return UNIMPLEMENTED(TransferToInfeed); + } + + absl::Status TransferFromOutfeed(MutableBorrowingLiteral literal) override { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + return UNIMPLEMENTED(TransferFromOutfeed); + } + + void AttachDefaultMemorySpace(PjRtMemorySpace* memory_space) { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + memory_space_ = memory_space; + } + + absl::Span memory_spaces() const override { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + return absl::MakeSpan(&memory_space_, 1); + } + + absl::StatusOr default_memory_space() const override { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + if (!memory_space_) + return absl::InternalError("Plugin memory space unset."); + @@ -1270,13 +1265,13 @@ index 0000000000..816ef734bf + + std::unique_ptr CreateAsyncTrackingEvent( + absl::string_view description) const override { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + LOG(FATAL) << "Plugin does not implement CreateAsyncTrackingEvent."; + return nullptr; + } + + const PjRtDeviceDescription& description() const override { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + return description_; + } + @@ -1356,10 +1351,10 @@ index 0000000000..ef42968a3a \ No newline at end of file diff --git a/xla/pjrt/plugin/mlx/executable.cc b/xla/pjrt/plugin/mlx/executable.cc new file mode 100644 -index 0000000000..8823b98305 +index 0000000000..1305486a0d --- /dev/null +++ b/xla/pjrt/plugin/mlx/executable.cc -@@ -0,0 +1,2431 @@ +@@ -0,0 +1,2437 @@ +/* Copyright 2024 The OpenXLA Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. @@ -1661,10 +1656,10 @@ index 0000000000..8823b98305 + switch (dtype) { + case mx::bfloat16: + return RankedTensorType::get(ArrayRef(shape), -+ BFloat16Type::get(module.getContext())); ++ BFloat16Type::get(module.getContext())); + case mx::float16: + return RankedTensorType::get(ArrayRef(shape), -+ Float16Type::get(module.getContext())); ++ Float16Type::get(module.getContext())); + case mx::int8: + return RankedTensorType::get( + ArrayRef(shape), @@ -2181,7 +2176,8 @@ index 0000000000..8823b98305 + CallOpInfo call_op_info = getCallOpInfo(mod, o); + if (auto search = op_lookup.call.find(call_op_info); + search == op_lookup.call.end()) { -+ std::cout << "Compiling CallOp: " << o.getName().str().c_str() << std::endl; ++ // std::cout << "Compiling CallOp: " << o.getName().str().c_str() << ++ // std::endl; + auto& block = o.front(); + auto call_op_func = [&op_lookup, &block, mod, + call_op_info](const std::vector& inputs) { @@ -2829,7 +2825,7 @@ index 0000000000..8823b98305 + auto shape = std::get<3>(iota_op_info.captures); + if (auto search = op_lookup.iota.find(iota_op_info); + search == op_lookup.iota.end()) { -+ std::cout << "Compiling IotaOp" << std::endl; ++ // std::cout << "Compiling IotaOp" << std::endl; + auto iota_func = [dimensions, dtype, iota_dim, + shape](const std::vector& inputs) { + auto init_arange = @@ -2847,7 +2843,7 @@ index 0000000000..8823b98305 + CbrtOpInfo cbrt_op_info = getCbrtOpInfo(o); + if (auto search = op_lookup.cbrt.find(cbrt_op_info); + search == op_lookup.cbrt.end()) { -+ std::cout << "Compiling CbrtOp" << std::endl; ++ // std::cout << "Compiling CbrtOp" << std::endl; + + auto cbrt_func = [](const std::vector& inputs) { + return std::vector{mx::power( @@ -2865,7 +2861,7 @@ index 0000000000..8823b98305 + auto target_shape = std::get<1>(broadcast_in_dim_op_info.captures); + if (auto search = op_lookup.broadcast_in_dim.find(broadcast_in_dim_op_info); + search == op_lookup.broadcast_in_dim.end()) { -+ std::cout << "Compiling BroadcastInDimOp" << std::endl; ++ // std::cout << "Compiling BroadcastInDimOp" << std::endl; + + auto broadcast_in_dim_func = [broadcast_dims, target_shape]( + const std::vector& inputs) { @@ -2900,7 +2896,7 @@ index 0000000000..8823b98305 + auto rhs_contract_dims = std::get<3>(dot_general_op_info.captures); + if (auto search = op_lookup.dot_general.find(dot_general_op_info); + search == op_lookup.dot_general.end()) { -+ std::cout << "Compiling DotGeneralOp" << std::endl; ++ // std::cout << "Compiling DotGeneralOp" << std::endl; + + auto dot_general_func = [lhs_batch_dims, rhs_batch_dims, lhs_contract_dims, + rhs_contract_dims]( @@ -2979,7 +2975,7 @@ index 0000000000..8823b98305 + + if (auto search = op_lookup.gather.find(gather_op_info); + search == op_lookup.gather.end()) { -+ std::cout << "Compiling GatherOp" << std::endl; ++ // std::cout << "Compiling GatherOp" << std::endl; + auto gather_func = [collapsed_slice_dims, index_vector_dim, offset_dims, + operand_batching_dims, result_shape, slice_sizes, + start_index_map, start_indices_batching_dims]( @@ -3099,57 +3095,62 @@ index 0000000000..8823b98305 +} + +int32_t computeProduct(const std::vector& dims) { -+ return std::accumulate(dims.begin(), dims.end(), 1, std::multiplies()); ++ return std::accumulate(dims.begin(), dims.end(), 1, ++ std::multiplies()); +} + -+ +mx::array computeGatherIndices( -+ const mx::array& start_indices, -+ const std::vector& start_index_map, ++ const mx::array& start_indices, const std::vector& start_index_map, + const std::vector& operand_shape, + const std::vector& slice_sizes, + const std::vector& operand_batching_dims, + const std::vector& start_indices_batching_dims, + int64_t index_vector_dim) { ++ // Validate input dimensions ++ if (start_indices.ndim() == 0) { ++ throw std::invalid_argument( ++ "start_indices must have at least one dimension."); ++ } + -+ // Validate input dimensions -+ if (start_indices.ndim() == 0) { -+ throw std::invalid_argument("start_indices must have at least one dimension."); -+ } ++ // Compute the shape of start_indices ++ std::vector start_indices_shape = start_indices.shape(); + -+ // Compute the shape of start_indices -+ std::vector start_indices_shape = start_indices.shape(); ++ // Compute the number of index vectors ++ int32_t index_vector_size = ++ static_cast(start_indices_shape[index_vector_dim]); ++ std::vector outer_shape = start_indices_shape; ++ outer_shape.erase(outer_shape.begin() + index_vector_dim); ++ int32_t num_index_vectors = computeProduct(outer_shape); + -+ // Compute the number of index vectors -+ int32_t index_vector_size = static_cast(start_indices_shape[index_vector_dim]); -+ std::vector outer_shape = start_indices_shape; -+ outer_shape.erase(outer_shape.begin() + index_vector_dim); -+ int32_t num_index_vectors = computeProduct(outer_shape); ++ // Reshape start_indices to a 2D tensor: [num_index_vectors, ++ // index_vector_size] ++ mx::array reshaped_start_indices = ++ mx::reshape(start_indices, {num_index_vectors, index_vector_size}); + -+ // Reshape start_indices to a 2D tensor: [num_index_vectors, index_vector_size] -+ mx::array reshaped_start_indices = mx::reshape(start_indices, {num_index_vectors, index_vector_size}); ++ // Initialize full_start_indices with zeros: [num_index_vectors, operand_rank] ++ int32_t operand_rank = static_cast(operand_shape.size()); ++ mx::array full_start_indices = ++ mx::zeros({num_index_vectors, operand_rank}, mx::int32); + -+ // Initialize full_start_indices with zeros: [num_index_vectors, operand_rank] -+ int32_t operand_rank = static_cast(operand_shape.size()); -+ mx::array full_start_indices = mx::zeros({num_index_vectors, operand_rank}, mx::int32); ++ // Populate full_start_indices using start_index_map ++ for (size_t i = 0; i < start_index_map.size(); ++i) { ++ int32_t operand_dim = start_index_map[i]; ++ // Extract the i-th column from reshaped_start_indices ++ mx::array indices_column = ++ mx::slice(reshaped_start_indices, {0, static_cast(i)}, ++ {num_index_vectors, static_cast(i + 1)}); ++ // Clip indices to ensure they are within bounds ++ int32_t max_index = operand_shape[operand_dim] - slice_sizes[operand_dim]; ++ mx::array clipped_indices = ++ mx::clip(indices_column, mx::array({0}, mx::int32), ++ mx::array({max_index}, mx::int32)); ++ // Update the corresponding column in full_start_indices ++ full_start_indices = ++ mx::slice_update(full_start_indices, clipped_indices, {0, operand_dim}, ++ {num_index_vectors, operand_dim + 1}); ++ } + -+ // Populate full_start_indices using start_index_map -+ for (size_t i = 0; i < start_index_map.size(); ++i) { -+ int32_t operand_dim = start_index_map[i]; -+ // Extract the i-th column from reshaped_start_indices -+ mx::array indices_column = mx::slice(reshaped_start_indices, {0, static_cast(i)}, -+ {num_index_vectors, static_cast(i + 1)}); -+ // Clip indices to ensure they are within bounds -+ int32_t max_index = operand_shape[operand_dim] - slice_sizes[operand_dim]; -+ mx::array clipped_indices = mx::clip(indices_column, mx::array({0}, mx::int32), -+ mx::array({max_index}, mx::int32)); -+ // Update the corresponding column in full_start_indices -+ full_start_indices = mx::slice_update(full_start_indices, clipped_indices, -+ {0, operand_dim}, -+ {num_index_vectors, operand_dim + 1}); -+ } -+ -+ return full_start_indices; ++ return full_start_indices; +} + +void _compileGatherOp(OpLookup& op_lookup, stablehlo::GatherOp o) { @@ -3165,7 +3166,7 @@ index 0000000000..8823b98305 + + if (auto search = op_lookup.gather.find(gather_op_info); + search == op_lookup.gather.end()) { -+ std::cout << "Compiling GatherOp" << std::endl; ++ // std::cout << "Compiling GatherOp" << std::endl; + auto gather_func = + [collapsed_slice_dims, index_vector_dim, offset_dims, + operand_batching_dims, result_shape, slice_sizes, start_index_map, @@ -3207,7 +3208,7 @@ index 0000000000..8823b98305 + auto update_window_dims = std::get<6>(scatter_op_info.captures); + if (auto search = op_lookup.scatter.find(scatter_op_info); + search == op_lookup.scatter.end()) { -+ std::cout << "Compiling ScatterOp" << std::endl; ++ // std::cout << "Compiling ScatterOp" << std::endl; + auto scatter_func = [index_vector_dim, input_batching_dims, + inserted_window_dims, scatter_dims_to_operand_dims, + scatter_indices_batching_dims, scatter_type, @@ -3380,7 +3381,7 @@ index 0000000000..8823b98305 + auto interior_pad = std::get<2>(pad_op_info.captures); + if (auto search = op_lookup.pad.find(pad_op_info); + search == op_lookup.pad.end()) { -+ std::cout << "Compiling PadOp" << std::endl; ++ // std::cout << "Compiling PadOp" << std::endl; + auto pad_func = [result_shape, edge_pad_low, + interior_pad](const std::vector& inputs) { + auto operand = inputs[0]; @@ -3471,7 +3472,7 @@ index 0000000000..8823b98305 + break; + } + for (auto fn_name : compilable_fns) { -+ std::cout << fn_name.c_str() << std::endl; ++ // std::cout << fn_name.c_str() << std::endl; + auto func = mod.lookupSymbol(fn_name); + compileCallOp(op_lookup, mod, func); + dep_tree.remove(fn_name); @@ -3491,9 +3492,9 @@ index 0000000000..8823b98305 + client_(client), + context_(), + module_(cloneIntoContext(module, context_)) { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + auto main = module.lookupSymbol("main"); -+ std::cout << "Started call to `mlx::compile`" << std::endl << std::endl; ++ // std::cout << "Started call to `mlx::compile`" << std::endl << std::endl; + compileModule(module, op_lookup_); + // try { + auto& main_block = main.front(); @@ -3510,7 +3511,7 @@ index 0000000000..8823b98305 + *utils::dtype::fromMlirType(shaped_type.getElementType()))); + } + compiled_module_(tmp_inputs); -+ std::cout << "Finished call to `mlx::compile`" << std::endl << std::endl; ++ // std::cout << "Finished call to `mlx::compile`" << std::endl << std::endl; + // } catch (std::exception& e) { + // std::cout << "Error in `mlx::compile`" << std::endl; + // } @@ -3545,30 +3546,30 @@ index 0000000000..8823b98305 + } + + PjRtClient* client() const override { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + return client_; + } + + const DeviceAssignment& device_assignment() const override { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + return assignment_; + } + + absl::Span + addressable_device_logical_ids() const override { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + LOG_UNIMPLEMENTED(addressable_device_logical_ids); + return {}; + } + + absl::Span addressable_devices() const override { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + return devices_; + } + + // Helper function to get default mem from device. + PjRtMemorySpace* get_default_memory_space() const { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + return devices_[0]->default_memory_space().value_or(nullptr); + } + @@ -3604,7 +3605,7 @@ index 0000000000..8823b98305 + absl::StatusOr ExecuteWithMlxInterpreter( + absl::Span argument_handles, ModuleOp module, + PjRtDevice* device, bool fill_future) { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + SmallVector input_attrs; + for (auto* arg : argument_handles) { + TF_ASSIGN_OR_RETURN(auto mlirArg, GetAttributeFromBuffer(arg)); @@ -3650,7 +3651,7 @@ index 0000000000..8823b98305 + absl::Span> argument_handles, + const xla::ExecuteOptions& options, + std::optional>>& returned_futures) override { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + if (argument_handles.size() != 1) { + // One arg handle per device. + return absl::InvalidArgumentError( @@ -3677,7 +3678,7 @@ index 0000000000..8823b98305 + absl::Span argument_handles, PjRtDevice* device, + const xla::ExecuteOptions& options, + std::optional>& returned_future, bool fill_future) override { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + // Synchronous! To make async, have the device make a buffer with a ready + // future that is ready when the computation is done / buffer is ready. + TF_ASSIGN_OR_RETURN( @@ -3694,7 +3695,7 @@ index 0000000000..8823b98305 + absl::Span argument_handles, PjRtDevice* device, + const xla::ExecuteOptions& options, + std::optional>& returned_future, bool fill_future) override { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + // Synchronous! To make async, have the device make a buffer with a ready + // future that is ready when the computation is done / buffer is ready. + TF_ASSIGN_OR_RETURN( @@ -3708,31 +3709,31 @@ index 0000000000..8823b98305 + } + + void Delete() override { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + module_.release(); + module_ = nullptr; + } + bool IsDeleted() override { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + return !module_; + } + + // PjRtExecutable API. + int num_replicas() const override { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + return assignment_.replica_count(); + } + int num_partitions() const override { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + return assignment_.computation_count(); + } + int64_t SizeOfGeneratedCodeInBytes() const override { + // No generated code.. so just return 1. -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + return 1; + } + absl::string_view name() const override { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + return name_; + } + @@ -3741,7 +3742,7 @@ index 0000000000..8823b98305 + // TODO: This shouldn't be needed for an MLIR plugin, its only used in the + // JAX layer for determining output sharding, which exists on the mlir + // module. -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + auto moduleClone = llvm::cast(module_.get()->clone()); + TF_ASSIGN_OR_RETURN(auto hlo_module, + xla::ConvertStablehloToHlo(moduleClone)); @@ -3750,12 +3751,12 @@ index 0000000000..8823b98305 + + absl::StatusOr>> + GetOutputMemoryKinds() const override { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + return UNIMPLEMENTED(GetOutputMemoryKinds); + } + + absl::StatusOr FingerprintExecutable() const override { -+ TRACE_ME_MEMBER; ++ // TRACE_ME_MEMBER; + return UNIMPLEMENTED(FingerprintExecutable); + } + diff --git a/zml/module.zig b/zml/module.zig index 798b81cf..c0bc9d1b 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -975,6 +975,7 @@ pub const FnCache = std.AutoHashMapUnmanaged(FnKey, MlirFn); pub const FnKey = struct { fn_ptr: *const anyopaque, input_hash: u64 }; test FnCache { + // TODO(@cryptodeal): Passing; need to implement MLX PjRt plugin cache const zml = @import("zml.zig"); const platform = zml.testing.env(); @@ -1035,6 +1036,7 @@ test FnCache { } test "FnCache with mixed integer/tensor" { + // TODO(@cryptodeal): Passing; need to implement MLX PjRt plugin cache const zml = @import("zml.zig"); const platform = zml.testing.env(); diff --git a/zml/nn.zig b/zml/nn.zig index c6a8560a..1c54365e 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -1095,6 +1095,7 @@ pub fn sdpaChunk(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) PartialSoft } test sdpaMemEfficient { + // TODO(@cryptodeal): MLX PjRt plugin implement `stablehlo::RngOp` const platform = zml.testing.env(); const allocator = std.testing.allocator; @@ -1153,6 +1154,7 @@ test sdpaMemEfficient { } test "sdpaMemEfficient transposed" { + // TODO(@cryptodeal): MLX PjRt plugin implement `stablehlo::RngOp` const platform = zml.testing.env(); const allocator = std.testing.allocator; @@ -1248,6 +1250,7 @@ pub fn sampleTokens(activations: Tensor, opts: SamplingStrategy, rng: Tensor.Rng } test sampleTokens { + // TODO(@cryptodeal): MLX PjRt plugin implement `stablehlo::RandomBitGeneratorOp` const platform = zml.testing.env(); const allocator = std.testing.allocator; @@ -1371,6 +1374,7 @@ fn fixupLogits(logits: Tensor, opts: DynamicSamplingStrategy) [2]Tensor { } test sampleTokensDynamic { + // TODO(@cryptodeal): MLX PjRt plugin implement `stablehlo::ReduceWindowOp` const platform = zml.testing.env(); const allocator = std.testing.allocator; diff --git a/zml/ops.zig b/zml/ops.zig index 1765888a..0eae345b 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -72,6 +72,7 @@ pub fn while_( } test "simple while" { + // TODO(@cryptodeal): MLX PjRt plugin implement `stablehlo::WhileOp` const CountInts = struct { step: Tensor, end: Tensor, @@ -349,6 +350,7 @@ pub fn for_(comptime func: anytype, blk_ctx: BlockSign(func).BlkCtx, num_steps_: } test for_ { + // TODO(@cryptodeal): MLX PjRt plugin implement `stablehlo::WhileOp` const Squares = struct { const Squares = @This(); @@ -389,6 +391,7 @@ test for_ { } test "nested for" { + // TODO(@cryptodeal): MLX PjRt plugin implement `stablehlo::WhileOp` const OuterProd = struct { const OuterProd = @This(); @@ -482,6 +485,7 @@ pub fn if_( } test "if" { + // TODO(@cryptodeal): MLX PjRt plugin implement `stablehlo::IfOp` const zml = @import("zml.zig"); const platform = zml.testing.env(); const allocator = std.testing.allocator; @@ -544,6 +548,7 @@ pub fn case( } test "case" { + // TODO(@cryptodeal): MLX PjRt plugin implement `stablehlo::CaseOp` const zml = @import("zml.zig"); const platform = zml.testing.env(); diff --git a/zml/tensor.zig b/zml/tensor.zig index 66ef46dd..ffabcdb9 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -615,6 +615,7 @@ pub const Tensor = struct { } test uniform { + // TODO(@cryptodeal): MLX PjRt plugin implement `stablehlo::RandomBitGeneratorOp` const zml = @import("zml.zig"); const Stats = struct { const Stats = @This(); @@ -708,6 +709,7 @@ pub const Tensor = struct { } test gumbel { + // TODO(@cryptodeal): MLX PjRt plugin implement `stablehlo::RandomBitGeneratorOp` const zml = @import("zml.zig"); const Stats = struct { const Stats = @This(); @@ -1375,6 +1377,7 @@ pub const Tensor = struct { } test cumulativeSum { + // TODO(@cryptodeal): MLX PjRt plugin implement `stablehlo::ReduceWindowOp` const zml = @import("zml.zig"); const platform = zml.testing.env(); @@ -1607,6 +1610,7 @@ pub const Tensor = struct { } test slice { + // TODO(@cryptodeal): fix test case for MLX PjRt plugin const zml = @import("zml.zig"); const platform = zml.testing.env(); @@ -2426,6 +2430,7 @@ pub const Tensor = struct { test gatherSlices { const zml = @import("zml.zig"); const platform = zml.testing.env(); + // TODO(@cryptodeal): MLX PjRt plugin calculate gather indices in batches try zml.testing.mlxSkipTest(platform); const Local = struct { @@ -2831,6 +2836,7 @@ pub const Tensor = struct { } test argMax { + // TODO(@cryptodeal): fix test case for MLX PjRt plugin const zml = @import("zml.zig"); const platform = zml.testing.env(); const allocator = std.testing.allocator; @@ -2894,7 +2900,7 @@ pub const Tensor = struct { test argsort { const zml = @import("zml.zig"); const platform = zml.testing.env(); - // TODO (@cryptodeal): Fix this for MLX PjRt Plugin Implementation. + // TODO(@cryptodeal): Fix this for MLX PjRt Plugin Implementation. // Currently segfaults; worked prior to rebase on latest XLA, MLX, and ZML. try zml.testing.mlxSkipTest(platform); @@ -3726,6 +3732,7 @@ pub const Tensor = struct { } test cartesianProduct { + // TODO(@cryptodeal): fix test case for MLX PjRt plugin const zml = @import("zml.zig"); const client = zml.testing.env(); @@ -3897,6 +3904,7 @@ fn getComparisonType(ctx: mlir.Context, dtype: DataType) dialect.stablehlo.Compa } test "Tensor.maxPool1d" { + // TODO(@cryptodeal): MLX PjRt plugin implement `stablehlo::ReduceWindowOp` const zml = @import("zml.zig"); const platform = zml.testing.env(); @@ -3933,6 +3941,7 @@ test "Tensor.maxPool1d" { } test "Tensor.maxPool2d" { + // TODO(@cryptodeal): MLX PjRt plugin implement `stablehlo::ReduceWindowOp` const zml = @import("zml.zig"); const platform = zml.testing.env(); diff --git a/zml/torch.zig b/zml/torch.zig index 94c9693c..400390e1 100644 --- a/zml/torch.zig +++ b/zml/torch.zig @@ -241,6 +241,7 @@ pub fn meshgrid(comptime N: u3, vectors: [N]Tensor, indexing: MeshgridIndexing) } test meshgrid { + // TODO(@cryptodeal): fix test case for MLX PjRt plugin const platform = zml.testing.env(); const x = try zml.Buffer.fromSlice(platform, .{6}, &[_]i32{ 0, 1, 2, 3, 4, 5 }); From 3f7d5fcfed25f7a55a91d7de88da357a7f66d98f Mon Sep 17 00:00:00 2001 From: James Deal Date: Thu, 22 May 2025 23:27:02 -0500 Subject: [PATCH 04/13] Add tests for `shiftLeft`, `shiftRightArithmetic`, `shiftRightLogical`, `logical` (OR) ops --- zml/tensor.zig | 81 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/zml/tensor.zig b/zml/tensor.zig index ffabcdb9..c592adcd 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -386,16 +386,64 @@ pub const Tensor = struct { return binaryOp(@src(), "shiftLeft", dialect.stablehlo.shift_left)(self, other); } + test shiftLeft { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + const lhs: [3]i64 = .{ -1, 0, 1 }; + const rhs: [3]i64 = .{ 1, 2, 3 }; + const expected: [3]i64 = .{ -2, 0, 8 }; + + const input = try zml.Buffer.fromSlice(platform, .{3}, &lhs); + const shift_by = try zml.Buffer.fromSlice(platform, .{3}, &rhs); + + const output = try zml.testing.compileAndCall(platform, Tensor.shiftLeft, .{ input, shift_by }); + + try zml.testing.expectClose(zml.HostBuffer.fromSlice(.{3}, &expected), output, 1e-4); + } + /// Returns a Tensor containing the element-wise arithmetic right-shift operation of 'self' by 'other'. pub fn shiftRightArithmetic(self: Tensor, other: Tensor) Tensor { return binaryOp(@src(), "shiftRightArithmetic", dialect.stablehlo.shift_right_arithmetic)(self, other); } + test shiftRightArithmetic { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + const lhs: [3]i64 = .{ -1, 0, 8 }; + const rhs: [3]i64 = .{ 1, 2, 3 }; + const expected: [3]i64 = .{ -1, 0, 1 }; + + const input = try zml.Buffer.fromSlice(platform, .{3}, &lhs); + const shift_by = try zml.Buffer.fromSlice(platform, .{3}, &rhs); + + const output = try zml.testing.compileAndCall(platform, Tensor.shiftRightArithmetic, .{ input, shift_by }); + + try zml.testing.expectClose(zml.HostBuffer.fromSlice(.{3}, &expected), output, 1e-4); + } + /// Returns a Tensor containing the element-wise logical right-shift operation of 'self' by 'other'. pub fn shiftRightLogical(self: Tensor, other: Tensor) Tensor { return binaryOp(@src(), "shiftRightLogical", dialect.stablehlo.shift_right_logical)(self, other); } + test shiftRightLogical { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + const lhs: [3]i64 = .{ -1, 0, 8 }; + const rhs: [3]i64 = .{ 1, 2, 3 }; + const expected: [3]i64 = .{ 9223372036854775807, 0, 1 }; + + const input = try zml.Buffer.fromSlice(platform, .{3}, &lhs); + const shift_by = try zml.Buffer.fromSlice(platform, .{3}, &rhs); + + const output = try zml.testing.compileAndCall(platform, Tensor.shiftRightLogical, .{ input, shift_by }); + + try zml.testing.expectClose(zml.HostBuffer.fromSlice(.{3}, &expected), output, 1e-4); + } + /// Returns the Cholesky decomposition of the input Tensor. /// /// 'lower' controls the form of the output Tensor. The output will be lower-triangular if 'lower' is true @@ -1035,6 +1083,39 @@ pub const Tensor = struct { }; } + test logical { + const zml = @import("zml.zig"); + const Local = struct { + pub fn call(comptime logical_op: LogicalOp) type { + return struct { + fn _logical(a: zml.Tensor, b: zml.Tensor) zml.Tensor { + return a.logical(logical_op, b); + } + + pub fn testCase(comptime T: type, platform: zml.Platform, lhs: [3]T, rhs: [3]T, expected: [3]T) !void { + const actual = try zml.testing.compileAndCall(platform, _logical, .{ try zml.Buffer.fromSlice(platform, .{3}, &lhs), try zml.Buffer.fromSlice(platform, .{3}, &rhs) }); + return zml.testing.expectClose(zml.HostBuffer.fromArray(&expected), actual, 1e-4); + } + }; + } + }.call; + + const platform = zml.testing.env(); + inline for (.{ + .{ i8, .{ 127, -128, -128 }, .{ 0, 127, -128 }, .{ 127, -1, -128 } }, + .{ u8, .{ 0, 127, 255 }, .{ 255, 255, 255 }, .{ 255, 255, 255 } }, + .{ i16, .{ 32767, -32768, -32768 }, .{ 0, 32767, -32768 }, .{ 32767, -1, -32768 } }, + .{ u16, .{ 0, 32767, 65535 }, .{ 65535, 65535, 65535 }, .{ 65535, 65535, 65535 } }, + .{ i32, .{ 2147483647, -2147483648, -2147483648 }, .{ 0, 2147483647, -2147483648 }, .{ 2147483647, -1, -2147483648 } }, + .{ u32, .{ 0, 2147483647, 4294967295 }, .{ 4294967295, 4294967295, 4294967295 }, .{ 4294967295, 4294967295, 4294967295 } }, + .{ i64, .{ 9223372036854775807, -9223372036854775808, -9223372036854775808 }, .{ 0, 9223372036854775807, -9223372036854775808 }, .{ 9223372036854775807, -1, -9223372036854775808 } }, + .{ u64, .{ 0, 9223372036854775807, 18446744073709551615 }, .{ 18446744073709551615, 18446744073709551615, 18446744073709551615 }, .{ 18446744073709551615, 18446744073709551615, 18446744073709551615 } }, + }) |testcase| { + const T, const lhs, const rhs, const expected = testcase; + try Local(.OR).testCase(T, platform, lhs, rhs, expected); + } + } + /// Returns a Tensor containing the element-wise floor operation of the input Tensor. pub fn floor(self: Tensor) Tensor { const loc = self.getContext().mlirCtx().location(@src()); From ae770c802e1e4f6fe3cc9b38eee157e19ffc91d0 Mon Sep 17 00:00:00 2001 From: James Deal Date: Sat, 24 May 2025 21:32:34 -0500 Subject: [PATCH 05/13] Fix failing tests for `slice`, `meshgrid`, `cartesianProduct` --- .../patches/0003-Added-MLX-PjRt-Plugin.patch | 19 ++++++++++++++++--- zml/tensor.zig | 2 -- zml/torch.zig | 1 - 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/third_party/modules/xla/20250317.2-71c67e2/patches/0003-Added-MLX-PjRt-Plugin.patch b/third_party/modules/xla/20250317.2-71c67e2/patches/0003-Added-MLX-PjRt-Plugin.patch index 6e9d4535..39575d67 100644 --- a/third_party/modules/xla/20250317.2-71c67e2/patches/0003-Added-MLX-PjRt-Plugin.patch +++ b/third_party/modules/xla/20250317.2-71c67e2/patches/0003-Added-MLX-PjRt-Plugin.patch @@ -1351,10 +1351,10 @@ index 0000000000..ef42968a3a \ No newline at end of file diff --git a/xla/pjrt/plugin/mlx/executable.cc b/xla/pjrt/plugin/mlx/executable.cc new file mode 100644 -index 0000000000..1305486a0d +index 0000000000..c45e9a2e14 --- /dev/null +++ b/xla/pjrt/plugin/mlx/executable.cc -@@ -0,0 +1,2437 @@ +@@ -0,0 +1,2450 @@ +/* Copyright 2024 The OpenXLA Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. @@ -3495,6 +3495,8 @@ index 0000000000..1305486a0d + // TRACE_ME_MEMBER; + auto main = module.lookupSymbol("main"); + // std::cout << "Started call to `mlx::compile`" << std::endl << std::endl; ++ // module.dump(); ++ // std::cout << std::endl; + compileModule(module, op_lookup_); + // try { + auto& main_block = main.front(); @@ -3618,14 +3620,25 @@ index 0000000000..1305486a0d + // TF_ASSIGN_OR_RETURN(auto mlx_res, evalModule(module, inputs)); + + std::vector inputs; ++ unsigned in_count = 0; + for (auto& attr : input_attrs) { -+ TF_ASSIGN_OR_RETURN(auto input, ++ TF_ASSIGN_OR_RETURN(mx::array input, + utils::array::fromDenseElementsAttr(attr)); ++ // std::cout << "inputs[" << std::to_string(in_count++) << "]: " << input ++ // << std::endl; + inputs.push_back(input); + } + + auto mlx_res = compiled_module_(inputs); ++ for (auto& a : mlx_res) { ++ a = mx::contiguous(a); ++ } + mx::eval(mlx_res); ++ // unsigned out_count = 0; ++ // for (auto& a : mlx_res) { ++ // std::cout << "outputs[" << std::to_string(out_count++) << "]: " << a ++ // << std::endl; ++ // } + + // LOG(INFO) << "Results: " << ToString(result.value()) << "\n"; + diff --git a/zml/tensor.zig b/zml/tensor.zig index c592adcd..2854d9b5 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -1691,7 +1691,6 @@ pub const Tensor = struct { } test slice { - // TODO(@cryptodeal): fix test case for MLX PjRt plugin const zml = @import("zml.zig"); const platform = zml.testing.env(); @@ -3813,7 +3812,6 @@ pub const Tensor = struct { } test cartesianProduct { - // TODO(@cryptodeal): fix test case for MLX PjRt plugin const zml = @import("zml.zig"); const client = zml.testing.env(); diff --git a/zml/torch.zig b/zml/torch.zig index 400390e1..94c9693c 100644 --- a/zml/torch.zig +++ b/zml/torch.zig @@ -241,7 +241,6 @@ pub fn meshgrid(comptime N: u3, vectors: [N]Tensor, indexing: MeshgridIndexing) } test meshgrid { - // TODO(@cryptodeal): fix test case for MLX PjRt plugin const platform = zml.testing.env(); const x = try zml.Buffer.fromSlice(platform, .{6}, &[_]i32{ 0, 1, 2, 3, 4, 5 }); From 802e5ca7efba3e16c2f6eadd6d5ebacae740559c Mon Sep 17 00:00:00 2001 From: James Deal Date: Tue, 27 May 2025 23:30:35 -0500 Subject: [PATCH 06/13] Optimize `stablehlo::gather` implementation via batched indice calculation --- .../patches/0003-Added-MLX-PjRt-Plugin.patch | 311 +++++++----------- zml/tensor.zig | 2 - 2 files changed, 119 insertions(+), 194 deletions(-) diff --git a/third_party/modules/xla/20250317.2-71c67e2/patches/0003-Added-MLX-PjRt-Plugin.patch b/third_party/modules/xla/20250317.2-71c67e2/patches/0003-Added-MLX-PjRt-Plugin.patch index 39575d67..94880758 100644 --- a/third_party/modules/xla/20250317.2-71c67e2/patches/0003-Added-MLX-PjRt-Plugin.patch +++ b/third_party/modules/xla/20250317.2-71c67e2/patches/0003-Added-MLX-PjRt-Plugin.patch @@ -1351,10 +1351,10 @@ index 0000000000..ef42968a3a \ No newline at end of file diff --git a/xla/pjrt/plugin/mlx/executable.cc b/xla/pjrt/plugin/mlx/executable.cc new file mode 100644 -index 0000000000..c45e9a2e14 +index 0000000000..3c36b94b82 --- /dev/null +++ b/xla/pjrt/plugin/mlx/executable.cc -@@ -0,0 +1,2450 @@ +@@ -0,0 +1,2377 @@ +/* Copyright 2024 The OpenXLA Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. @@ -2984,222 +2984,149 @@ index 0000000000..c45e9a2e14 + auto start_indices = inputs[1]; + + // Calculate batch dims -+ std::vector batch_dims; ++ std::vector batch_dims_vec; + for (int64_t i = 0; i < result_shape.size(); i++) { + if (std::find(offset_dims.begin(), offset_dims.end(), i) == + offset_dims.end()) { -+ batch_dims.emplace_back(static_cast(i)); ++ batch_dims_vec.emplace_back(static_cast(i)); + } + } ++ mx::array batch_dims = mx::array( ++ batch_dims_vec.data(), {static_cast(batch_dims_vec.size())}); ++ mx::array start_idx_map = ++ mx::array(start_index_map.data(), ++ {static_cast(start_index_map.size())}); ++ mx::array op_batching_dims = ++ mx::array(operand_batching_dims.data(), ++ {static_cast(operand_batching_dims.size())}); + -+ std::vector operand_indices; ++ // Calculate result indices ++ std::vector meshgrid_arrays; ++ for (const auto& dim : result_shape) { ++ meshgrid_arrays.push_back(mx::arange(dim)); ++ } ++ mx::array result_indices = ++ mx::reshape(mx::stack(mx::meshgrid(meshgrid_arrays, false, "ij"), -1), ++ {-1, static_cast(result_shape.size())}); + -+ // Iterate over result index space, populating result -+ for (const auto result_index_tuple : index_space(result_shape)) { -+ std::vector result_index = -+ to_vector(result_index_tuple, result_shape.size()); ++ // Initialize operand indices with zeros ++ mx::array operand_indices = mx::zeros( ++ {result_indices.shape(0), static_cast(operand.ndim())}, ++ mx::int32); + -+ std::vector batch_index(batch_dims.size()); -+ for (unsigned i = 0; i < batch_dims.size(); i++) { -+ batch_index[i] = result_index[batch_dims[i]]; ++ // Slice batch indices ++ mx::array batch_indices = mx::take(result_indices, batch_dims, 1); ++ if (batch_indices.size()) { ++ // Calculate start indices ++ std::vector tmp_indices = ++ mx::split(batch_indices, batch_indices.shape(1), 1); ++ for (auto& idx : tmp_indices) { ++ idx = mx::flatten(idx); + } + -+ // Slice start index for the current batch -+ std::vector sin_start(start_indices.ndim()); -+ std::vector sin_stop(start_indices.ndim()); -+ unsigned batch_idx_count = 0; -+ -+ for (auto i = 0; i < start_indices.ndim(); i++) { -+ if (index_vector_dim == static_cast(i)) { -+ sin_start[i] = 0; -+ sin_stop[i] = start_indices.shape(i) + 1; -+ continue; -+ } -+ sin_start[i] = batch_index[batch_idx_count++]; -+ sin_stop[i] = sin_start[i] + 1; ++ std::vector tmp_axes; ++ for (auto i = 0; i < start_indices.ndim(); ++i) { ++ if (i == index_vector_dim) continue; ++ tmp_axes.push_back(i); ++ } ++ std::vector tmp_slice_sizes(start_indices.ndim(), 1); ++ if (index_vector_dim < tmp_slice_sizes.size()) { ++ tmp_slice_sizes[index_vector_dim] = ++ start_indices.shape(index_vector_dim); + } -+ mx::array start_index = -+ mx::flatten(mx::slice(start_indices, sin_start, sin_stop)); + -+ // Compute full start index -+ mx::array full_start_index = -+ mx::zeros({static_cast(operand.ndim())}, mx::int32); -+ for (auto d_start = 0; d_start < start_index_map.size(); d_start++) { ++ mx::array batch_start_indices = mx::reshape( ++ mx::gather(start_indices, tmp_indices, tmp_axes, tmp_slice_sizes), ++ {result_indices.shape(0), -1}); ++ ++ // Seed with full start indices ++ std::vector max; ++ for (auto d_start = 0; d_start < start_index_map.size(); ++d_start) { + int d_operand = start_index_map[d_start]; -+ auto index_scalar = mx::slice(start_index, {d_start}, {d_start + 1}); -+ full_start_index = mx::slice_update( -+ full_start_index, -+ mx::clip(index_scalar, mx::array({0}, mx::int32), -+ mx::array({operand.shape(d_operand) - -+ static_cast(slice_sizes[d_operand])}, -+ mx::int32)), -+ {d_operand}, {d_operand + 1}); ++ max.push_back(operand.shape(d_operand) - slice_sizes[d_operand]); + } + -+ // Compute full batching index -+ std::vector full_batching_index(operand.ndim(), 0); -+ for (auto i_batching = 0; i_batching < operand_batching_dims.size(); -+ i_batching++) { -+ auto d_operand = -+ static_cast(operand_batching_dims[i_batching]); -+ auto d_start = -+ static_cast(start_indices_batching_dims[i_batching]); -+ full_batching_index[d_operand] = batch_index -+ [d_start - -+ (static_cast(d_start) < index_vector_dim ? 0 : 1)]; -+ } -+ -+ // Compute offset index -+ std::vector offset_index(offset_dims.size()); -+ for (unsigned i = 0; i < offset_dims.size(); i++) { -+ offset_index[i] = result_index[offset_dims[i]]; -+ } -+ -+ // Compute full offset index -+ std::vector full_offset_index(operand.ndim(), 0); -+ unsigned offset_index_count = 0; -+ for (unsigned i = 0; i < full_offset_index.size(); i++) { -+ if (std::find(operand_batching_dims.begin(), -+ operand_batching_dims.end(), static_cast(i)) != -+ operand_batching_dims.end() || -+ std::find(collapsed_slice_dims.begin(), -+ collapsed_slice_dims.end(), static_cast(i)) != -+ collapsed_slice_dims.end()) { -+ continue; -+ } -+ full_offset_index[i] = offset_index[offset_index_count++]; -+ } -+ -+ operand_indices.emplace_back( -+ full_start_index + -+ mx::array(full_batching_index.data(), -+ {static_cast(full_batching_index.size())}, -+ mx::int32) + -+ mx::array(full_offset_index.data(), -+ {static_cast(full_offset_index.size())}, -+ mx::int32)); ++ operand_indices = mx::put_along_axis( ++ operand_indices, ++ mx::broadcast_to(start_idx_map, ++ {operand_indices.shape(0), ++ static_cast(start_index_map.size())}), ++ mx::clip( ++ mx::take(batch_start_indices, ++ mx::arange(static_cast(start_index_map.size())), ++ 1), ++ mx::zeros(batch_start_indices.shape(), mx::int32), ++ mx::array(max.data(), {static_cast(max.size())})), ++ 1); + } -+ std::vector gather_slice_sizes(operand.ndim(), 1); ++ ++ // Calculate full batching indices ++ mx::array i_batching = ++ mx::arange(static_cast(operand_batching_dims.size())); ++ mx::array d_operands = mx::take(op_batching_dims, i_batching); ++ mx::array d_starts = mx::take( ++ mx::array(start_indices_batching_dims.data(), ++ {static_cast(start_indices_batching_dims.size())}), ++ i_batching); ++ auto tmp_batch_start_indices = mx::subtract( ++ d_starts, mx::where(mx::less(d_starts, mx::full(d_starts.shape(), ++ index_vector_dim)), ++ mx::zeros(d_starts.shape(), mx::int32), ++ mx::ones(d_starts.shape(), mx::int32))); ++ ++ // Add batching indices ++ operand_indices = mx::scatter_add_axis( ++ operand_indices, ++ mx::broadcast_to(d_operands, ++ {operand_indices.shape(0), d_operands.shape(0)}), ++ mx::take(batch_indices, tmp_batch_start_indices, 1), 1); ++ ++ // Calculate offset indices ++ mx::array offset_indices = ++ mx::take(result_indices, ++ mx::array(offset_dims.data(), ++ {static_cast(offset_dims.size())}), ++ 1); ++ ++ std::vector result_offset_idx; ++ for (unsigned i = 0; i < operand_indices.shape(1); i++) { ++ if (std::find(operand_batching_dims.begin(), ++ operand_batching_dims.end(), ++ static_cast(i)) != operand_batching_dims.end() || ++ std::find(collapsed_slice_dims.begin(), collapsed_slice_dims.end(), ++ static_cast(i)) != collapsed_slice_dims.end()) { ++ continue; ++ } ++ result_offset_idx.push_back(i); ++ } ++ ++ // Add offset indices ++ operand_indices = mx::scatter_add_axis( ++ operand_indices, ++ mx::broadcast_to( ++ mx::array(result_offset_idx.data(), ++ {static_cast(result_offset_idx.size())}), ++ {operand_indices.shape(0), ++ static_cast(result_offset_idx.size())}), ++ offset_indices, 1); ++ + std::vector gather_axes(operand.ndim()); + std::iota(gather_axes.begin(), gather_axes.end(), 0); -+ return std::vector{ -+ mx::reshape(mx::gather(operand, -+ mx::split(mx::stack(operand_indices, 1), -+ operand_indices[0].shape(0), 0), -+ gather_axes, gather_slice_sizes), -+ result_shape)}; ++ return std::vector{mx::reshape( ++ mx::gather(operand, ++ mx::split(operand_indices, operand_indices.shape(1), 1), ++ gather_axes, std::vector(operand.ndim(), 1)), ++ result_shape)}; + }; + + op_lookup.gather.emplace(gather_op_info, mx::compile(gather_func)); + } +} + -+int32_t computeProduct(const std::vector& dims) { -+ return std::accumulate(dims.begin(), dims.end(), 1, -+ std::multiplies()); -+} -+ -+mx::array computeGatherIndices( -+ const mx::array& start_indices, const std::vector& start_index_map, -+ const std::vector& operand_shape, -+ const std::vector& slice_sizes, -+ const std::vector& operand_batching_dims, -+ const std::vector& start_indices_batching_dims, -+ int64_t index_vector_dim) { -+ // Validate input dimensions -+ if (start_indices.ndim() == 0) { -+ throw std::invalid_argument( -+ "start_indices must have at least one dimension."); -+ } -+ -+ // Compute the shape of start_indices -+ std::vector start_indices_shape = start_indices.shape(); -+ -+ // Compute the number of index vectors -+ int32_t index_vector_size = -+ static_cast(start_indices_shape[index_vector_dim]); -+ std::vector outer_shape = start_indices_shape; -+ outer_shape.erase(outer_shape.begin() + index_vector_dim); -+ int32_t num_index_vectors = computeProduct(outer_shape); -+ -+ // Reshape start_indices to a 2D tensor: [num_index_vectors, -+ // index_vector_size] -+ mx::array reshaped_start_indices = -+ mx::reshape(start_indices, {num_index_vectors, index_vector_size}); -+ -+ // Initialize full_start_indices with zeros: [num_index_vectors, operand_rank] -+ int32_t operand_rank = static_cast(operand_shape.size()); -+ mx::array full_start_indices = -+ mx::zeros({num_index_vectors, operand_rank}, mx::int32); -+ -+ // Populate full_start_indices using start_index_map -+ for (size_t i = 0; i < start_index_map.size(); ++i) { -+ int32_t operand_dim = start_index_map[i]; -+ // Extract the i-th column from reshaped_start_indices -+ mx::array indices_column = -+ mx::slice(reshaped_start_indices, {0, static_cast(i)}, -+ {num_index_vectors, static_cast(i + 1)}); -+ // Clip indices to ensure they are within bounds -+ int32_t max_index = operand_shape[operand_dim] - slice_sizes[operand_dim]; -+ mx::array clipped_indices = -+ mx::clip(indices_column, mx::array({0}, mx::int32), -+ mx::array({max_index}, mx::int32)); -+ // Update the corresponding column in full_start_indices -+ full_start_indices = -+ mx::slice_update(full_start_indices, clipped_indices, {0, operand_dim}, -+ {num_index_vectors, operand_dim + 1}); -+ } -+ -+ return full_start_indices; -+} -+ -+void _compileGatherOp(OpLookup& op_lookup, stablehlo::GatherOp o) { -+ GatherOpInfo gather_op_info = getGatherOpInfo(o); -+ auto collapsed_slice_dims = std::get<0>(gather_op_info.captures); -+ auto index_vector_dim = std::get<1>(gather_op_info.captures); -+ auto offset_dims = std::get<2>(gather_op_info.captures); -+ auto operand_batching_dims = std::get<3>(gather_op_info.captures); -+ auto result_shape = std::get<4>(gather_op_info.captures); -+ auto slice_sizes = std::get<5>(gather_op_info.captures); -+ auto start_index_map = std::get<6>(gather_op_info.captures); -+ auto start_indices_batching_dims = std::get<7>(gather_op_info.captures); -+ -+ if (auto search = op_lookup.gather.find(gather_op_info); -+ search == op_lookup.gather.end()) { -+ // std::cout << "Compiling GatherOp" << std::endl; -+ auto gather_func = -+ [collapsed_slice_dims, index_vector_dim, offset_dims, -+ operand_batching_dims, result_shape, slice_sizes, start_index_map, -+ start_indices_batching_dims](const std::vector& inputs) { -+ auto operand = inputs[0]; -+ auto start_indices = inputs[1]; -+ -+ // Calculate batch dims -+ std::vector batch_dims; -+ for (int64_t i = 0; i < result_shape.size(); i++) { -+ if (std::find(offset_dims.begin(), offset_dims.end(), i) == -+ offset_dims.end()) { -+ batch_dims.emplace_back(static_cast(i)); -+ } -+ } -+ -+ // Iterate over result index space, populating result -+ for (const auto result_index_tuple : index_space(result_shape)) { -+ std::vector result_index = -+ to_vector(result_index_tuple, result_shape.size()); -+ -+ std::vector batch_index(batch_dims.size()); -+ for (unsigned i = 0; i < batch_dims.size(); i++) { -+ batch_index[i] = result_index[batch_dims[i]]; -+ } -+ } -+ }; -+ } -+} -+ +void compileScatterOp(OpLookup& op_lookup, stablehlo::ScatterOp o) { + ScatterOpInfo scatter_op_info = getScatterOpInfo(o); -+ auto index_vector_dim = std::get<0>(scatter_op_info.captures); ++ auto index_vector_dim = ++ static_cast(std::get<0>(scatter_op_info.captures)); + auto input_batching_dims = std::get<1>(scatter_op_info.captures); + auto inserted_window_dims = std::get<2>(scatter_op_info.captures); + auto scatter_dims_to_operand_dims = std::get<3>(scatter_op_info.captures); diff --git a/zml/tensor.zig b/zml/tensor.zig index 2854d9b5..a048ff4c 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -2510,8 +2510,6 @@ pub const Tensor = struct { test gatherSlices { const zml = @import("zml.zig"); const platform = zml.testing.env(); - // TODO(@cryptodeal): MLX PjRt plugin calculate gather indices in batches - try zml.testing.mlxSkipTest(platform); const Local = struct { pub fn _gatherSlices(self: Tensor, slice_shape: Shape, indices: Tensor, opts: GatherOpts) Tensor { From 7e209d3174585e703195e39ec4e973f4d4bd953a Mon Sep 17 00:00:00 2001 From: James Deal Date: Sat, 31 May 2025 15:02:07 -0500 Subject: [PATCH 07/13] Optimize `scatter`/`pad` implementations; fix `argsort` --- .../patches/0003-Added-MLX-PjRt-Plugin.patch | 309 +++++++++--------- zml/tensor.zig | 3 - 2 files changed, 151 insertions(+), 161 deletions(-) diff --git a/third_party/modules/xla/20250317.2-71c67e2/patches/0003-Added-MLX-PjRt-Plugin.patch b/third_party/modules/xla/20250317.2-71c67e2/patches/0003-Added-MLX-PjRt-Plugin.patch index 94880758..410ea7e1 100644 --- a/third_party/modules/xla/20250317.2-71c67e2/patches/0003-Added-MLX-PjRt-Plugin.patch +++ b/third_party/modules/xla/20250317.2-71c67e2/patches/0003-Added-MLX-PjRt-Plugin.patch @@ -1351,10 +1351,10 @@ index 0000000000..ef42968a3a \ No newline at end of file diff --git a/xla/pjrt/plugin/mlx/executable.cc b/xla/pjrt/plugin/mlx/executable.cc new file mode 100644 -index 0000000000..3c36b94b82 +index 0000000000..cf551b2dc7 --- /dev/null +++ b/xla/pjrt/plugin/mlx/executable.cc -@@ -0,0 +1,2377 @@ +@@ -0,0 +1,2370 @@ +/* Copyright 2024 The OpenXLA Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. @@ -2713,9 +2713,9 @@ index 0000000000..3c36b94b82 + // .Case([](auto o) {}) + // .Case([](auto o) {}) + .Case([&block_ctx, &inputs](auto o) { -+ std::vector inputs; ++ std::vector tmp_inputs; + for (auto val : o.getInputs()) { -+ inputs.emplace_back( ++ tmp_inputs.emplace_back( + getOperandArray(val, block_ctx, inputs)); + } + int dim = static_cast(o.getDimension()); @@ -2723,10 +2723,11 @@ index 0000000000..3c36b94b82 + mlir::Block& comparator = o.getComparator().front(); + auto [index, comparator_type] = *getSortInfo(comparator); + auto indices = -+ mx::argsort(inputs[index], dim, comparator_type); ++ mx::argsort(tmp_inputs[index], dim, comparator_type); + std::vector res; -+ for (auto in : inputs) { -+ res.push_back(mx::gather(in, indices, {dim}, {1})); ++ mx::Shape slice_sizes(tmp_inputs[0].ndim(), 1); ++ for (auto in : tmp_inputs) { ++ res.push_back(mx::gather(in, indices, dim, slice_sizes)); + } + return res; + }) @@ -3144,154 +3145,158 @@ index 0000000000..3c36b94b82 + std::vector inputs(in.begin(), in.begin() + input_count); + std::vector updates(in.begin() + input_count, in.end() - 1); + auto scatter_indices = in[in.size() - 1]; -+ -+ std::vector input_indices; -+ std::vector update_indices; -+ -+ // iterate over updates[0] index space -+ for (const auto update_index_tuple : index_space(updates[0].shape())) { -+ std::vector update_index = to_vector( -+ update_index_tuple, static_cast(updates[0].ndim())); -+ -+ // Calculate update scatter dims -+ std::vector update_scatter_dims; -+ for (auto i = 0; i < updates[0].ndim(); i++) { -+ if (std::find(update_window_dims.begin(), update_window_dims.end(), -+ static_cast(i)) == update_window_dims.end()) { -+ update_scatter_dims.emplace_back(static_cast(i)); -+ } ++ // Calculate update scatter dims ++ std::vector update_scatter_dims; ++ for (auto i = 0; i < updates[0].ndim(); i++) { ++ if (std::find(update_window_dims.begin(), update_window_dims.end(), ++ static_cast(i)) == update_window_dims.end()) { ++ update_scatter_dims.emplace_back(static_cast(i)); + } ++ } + -+ // Calculate update scatter index -+ std::vector update_scatter_index(update_scatter_dims.size()); -+ for (auto i = 0; i < update_scatter_dims.size(); i++) { -+ update_scatter_index[i] = update_index[update_scatter_dims[i]]; -+ } ++ // Calculate update indices ++ std::vector meshgrid_arrays; ++ for (const auto& dim : updates[0].shape()) { ++ meshgrid_arrays.push_back(mx::arange(dim)); ++ } ++ mx::array update_indices = ++ mx::reshape(mx::stack(mx::meshgrid(meshgrid_arrays, false, "ij"), -1), ++ {-1, static_cast(updates[0].ndim())}); + -+ // Slice start index -+ std::vector sin_start = update_scatter_index; -+ std::vector sin_stop = update_scatter_index; -+ if (index_vector_dim < scatter_indices.ndim()) { -+ sin_start.insert(sin_start.begin() + index_vector_dim, 0); -+ sin_stop.insert(sin_stop.begin() + index_vector_dim, -+ scatter_indices.shape(index_vector_dim)); -+ } -+ for (auto& d : sin_stop) d += 1; -+ mx::array start_index = -+ mx::flatten(mx::slice(scatter_indices, sin_start, sin_stop)); ++ // Calculate update scatter index ++ mx::array update_scatter_indices = mx::take( ++ update_indices, ++ mx::array(update_scatter_dims.data(), ++ {static_cast(update_scatter_dims.size())}), ++ 1); + -+ // Compute full start index -+ mx::array full_start_index = -+ mx::zeros({static_cast(inputs[0].ndim())}, mx::int32); -+ for (auto i = 0; i < scatter_dims_to_operand_dims.size(); i++) { -+ auto d_input = static_cast(scatter_dims_to_operand_dims[i]); -+ full_start_index = mx::slice_update( -+ full_start_index, mx::slice(start_index, {i}, {i + 1}), {d_input}, -+ {d_input + 1}); -+ } ++ // Initialize result indices with zeros ++ auto result_indices = mx::zeros( ++ {update_indices.shape(0), static_cast(inputs[0].ndim())}, ++ mx::int32); + -+ // Compute full batching index -+ mx::array full_batching_index = -+ mx::zeros({static_cast(inputs[0].ndim())}, mx::int32); -+ for (auto i = 0; i < input_batching_dims.size(); i++) { -+ int32_t d_input = input_batching_dims[i]; -+ int32_t d_start = scatter_indices_batching_dims[i]; -+ full_batching_index = mx::slice_update( -+ full_batching_index, ++ // Gather start indices; populating the corresponding ++ // result indices. ++ auto gather_dim_count = index_vector_dim < scatter_indices.ndim() ++ ? update_scatter_dims.size() + 1 ++ : update_scatter_dims.size(); ++ std::vector tmp_gather_axes(update_scatter_dims.size()); ++ std::iota(tmp_gather_axes.begin(), tmp_gather_axes.end(), 0); ++ std::vector gather_slice_sizes(gather_dim_count, 1); ++ if (index_vector_dim < scatter_indices.ndim()) { ++ gather_slice_sizes[index_vector_dim] = ++ scatter_indices.shape(index_vector_dim); ++ } ++ mx::array start_indices = ++ mx::reshape(mx::gather(scatter_indices, ++ mx::split(update_scatter_indices, ++ update_scatter_indices.shape(1), 1), ++ tmp_gather_axes, gather_slice_sizes), ++ {update_indices.shape(0), -1}); ++ ++ // Seed with full start indices ++ result_indices = mx::put_along_axis( ++ result_indices, ++ mx::broadcast_to( + mx::array( -+ {update_scatter_index[d_start - -+ (d_start < index_vector_dim ? 0 : 1)]}), -+ {d_input}, {d_input + 1}); ++ scatter_dims_to_operand_dims.data(), ++ {static_cast(scatter_dims_to_operand_dims.size())}), ++ {result_indices.shape(0), ++ static_cast(scatter_dims_to_operand_dims.size())}), ++ start_indices, 1); ++ ++ // Calculate full batching indices; accumulating to the ++ // corresponding result indices. ++ auto d_input = ++ mx::array(input_batching_dims.data(), ++ {static_cast(input_batching_dims.size())}); ++ auto d_start = mx::array( ++ scatter_indices_batching_dims.data(), ++ {static_cast(scatter_indices_batching_dims.size())}); ++ result_indices = mx::scatter_add_axis( ++ result_indices, ++ mx::broadcast_to(d_input, ++ {result_indices.shape(0), d_input.shape(0)}), ++ mx::take(update_scatter_indices, ++ mx::subtract( ++ d_start, ++ mx::where(mx::less(d_start, mx::full(d_start.shape(), ++ index_vector_dim)), ++ mx::zeros(d_start.shape(), mx::int32), ++ mx::ones(d_start.shape(), mx::int32))), ++ 1), ++ 1); ++ ++ // Calculate update window indices ++ auto update_window_indices = ++ mx::take(update_indices, ++ mx::array(update_window_dims.data(), ++ {static_cast(update_window_dims.size())}), ++ 1); ++ ++ // Calculate full window indices; accumulating to the ++ // corresponding result indices. ++ std::vector tmp_indices; ++ auto full_window_idx_count = static_cast( ++ update_window_indices.shape(1) + inserted_window_dims.size() + ++ input_batching_dims.size()); ++ for (auto i = 0; i < full_window_idx_count; ++i) { ++ if (std::find(inserted_window_dims.begin(), inserted_window_dims.end(), ++ i) != inserted_window_dims.end() || ++ std::find(input_batching_dims.begin(), input_batching_dims.end(), ++ i) != input_batching_dims.end()) { ++ continue; + } -+ -+ // Compute update window index -+ std::vector update_window_index(update_window_dims.size()); -+ for (auto i = 0; i < update_window_dims.size(); i++) { -+ update_window_index[i] = update_index[update_window_dims[i]]; -+ } -+ -+ // Compute full window index -+ mx::array full_window_index = -+ mx::zeros({static_cast(update_window_index.size() + -+ inserted_window_dims.size() + -+ input_batching_dims.size())}, -+ mx::int32); -+ unsigned update_window_index_count = 0; -+ for (int32_t i = 0; i < full_window_index.size(); i++) { -+ if (std::find(inserted_window_dims.begin(), -+ inserted_window_dims.end(), -+ i) != inserted_window_dims.end() || -+ std::find(input_batching_dims.begin(), input_batching_dims.end(), -+ i) != input_batching_dims.end()) { -+ continue; -+ } -+ full_window_index = mx::slice_update( -+ full_window_index, -+ mx::array({update_window_index[update_window_index_count++]}), -+ {i}, {i + 1}); -+ } -+ -+ // Compute result index -+ mx::array result_index = -+ full_start_index + full_batching_index + full_window_index; -+ -+ // TODO (@cryptodeal): rework this so we can still leverage -+ // `mx::compile` OR ensure this cannot happen. -+ // if (mx::sum( -+ // result_index >= -+ // mx::array(reinterpret_cast( -+ // result_shape.data()), -+ // {static_cast(result_shape.size())})) -+ // .item()) { -+ // continue; -+ // } -+ -+ input_indices.push_back(result_index); -+ update_indices.push_back(mx::array( -+ update_index.data(), {static_cast(updates[0].ndim())})); -+ } -+ if (update_indices.empty()) { -+ return inputs; ++ tmp_indices.push_back(i); + } ++ result_indices = mx::scatter_add_axis( ++ result_indices, ++ mx::broadcast_to( ++ mx::array(tmp_indices.data(), ++ {static_cast(tmp_indices.size())}), ++ {result_indices.shape(0), ++ static_cast(tmp_indices.size())}), ++ update_window_indices, 1); ++ + std::vector scatter_axes(inputs[0].ndim()); + std::iota(scatter_axes.begin(), scatter_axes.end(), 0); + std::vector gather_axes(updates[0].ndim()); + std::iota(gather_axes.begin(), gather_axes.end(), 0); -+ std::vector gather_slice_sizes(updates[0].ndim(), 1); + std::vector res; -+ auto result_indices = -+ mx::split(mx::stack(input_indices, 1), input_indices[0].shape(0), 0); -+ auto gather_indices = mx::split(mx::stack(update_indices, 1), -+ update_indices[0].shape(0), 0); -+ auto idx_shape = gather_indices[0].shape(); -+ std::vector update_shape(inputs[0].ndim() + idx_shape.size(), 1); -+ for (auto i = 0; i < idx_shape.size(); ++i) { -+ update_shape[i] = idx_shape[i]; -+ } ++ ++ auto tmp_gather_indices = ++ mx::split(update_indices, update_indices.shape(1), 1); ++ auto tmp_scatter_indices = ++ mx::split(result_indices, result_indices.shape(1), 1); ++ ++ std::vector update_shape( ++ inputs[0].ndim() + tmp_scatter_indices[0].ndim(), 1); ++ update_shape[0] = update_indices.shape(0); ++ std::vector tmp_gather_slice_sizes(updates[0].ndim(), 1); + for (auto i = 0; i < inputs.size(); ++i) { + auto update_vals = -+ mx::reshape(mx::gather(updates[i], gather_indices, gather_axes, -+ gather_slice_sizes), ++ mx::reshape(mx::gather(updates[i], tmp_gather_indices, gather_axes, ++ tmp_gather_slice_sizes), + update_shape); + switch (scatter_type) { + case utils::ScatterType::Replace: -+ res.push_back(mx::scatter(inputs[i], result_indices, update_vals, -+ scatter_axes)); ++ res.push_back(mx::scatter(inputs[i], tmp_scatter_indices, ++ update_vals, scatter_axes)); + break; + case utils::ScatterType::Add: -+ res.push_back(mx::scatter_add(inputs[i], result_indices, ++ res.push_back(mx::scatter_add(inputs[i], tmp_scatter_indices, + update_vals, scatter_axes)); + break; + case utils::ScatterType::Prod: -+ res.push_back(mx::scatter_prod(inputs[i], result_indices, ++ res.push_back(mx::scatter_prod(inputs[i], tmp_scatter_indices, + update_vals, scatter_axes)); + break; + case utils::ScatterType::Max: -+ res.push_back(mx::scatter_max(inputs[i], result_indices, ++ res.push_back(mx::scatter_max(inputs[i], tmp_scatter_indices, + update_vals, scatter_axes)); + break; + case utils::ScatterType::Min: -+ res.push_back(mx::scatter_min(inputs[i], result_indices, ++ res.push_back(mx::scatter_min(inputs[i], tmp_scatter_indices, + update_vals, scatter_axes)); + break; + } @@ -3301,6 +3306,7 @@ index 0000000000..3c36b94b82 + op_lookup.scatter.emplace(scatter_op_info, mx::compile(scatter_func)); + } +} ++ +void compilePadOp(OpLookup& op_lookup, stablehlo::PadOp o) { + PadOpInfo pad_op_info = getPadOpInfo(o); + auto result_shape = std::get<0>(pad_op_info.captures); @@ -3320,42 +3326,29 @@ index 0000000000..3c36b94b82 + mx::array(interior_pad.data(), + {static_cast(edge_pad_low.size())}, mx::int32); + ++ // Calculate operand indices ++ std::vector meshgrid_arrays; ++ for (const auto& dim : operand.shape()) { ++ meshgrid_arrays.push_back(mx::arange(dim)); ++ } ++ mx::array operand_indices = ++ mx::reshape(mx::stack(mx::meshgrid(meshgrid_arrays, false, "ij"), -1), ++ {-1, static_cast(operand.ndim())}); ++ ++ mx::array result_indices = ++ edge_padding_low + ++ operand_indices * (interior_padding + mx::array(1, mx::int32)); ++ + // initialize array full of padding values + mx::array result = mx::full(result_shape, padding_value); -+ std::vector scatter_indices; -+ std::vector gather_indices; -+ // assign values to correct indices in result -+ for (const auto operand_index_tuple : index_space(operand.shape())) { -+ std::vector operand_index = -+ to_vector(operand_index_tuple, static_cast(operand.ndim())); -+ mx::array result_index = -+ edge_padding_low + -+ mx::array(operand_index.data(), -+ {static_cast(operand_index.size())}, mx::int32) * -+ (interior_padding + -+ mx::full(interior_padding.shape(), 1)); -+ scatter_indices.push_back(result_index); -+ gather_indices.push_back(mx::array( -+ operand_index.data(), {static_cast(operand.ndim())})); -+ } + std::vector axes(operand.ndim()); + std::iota(axes.begin(), axes.end(), 0); -+ std::vector gather_slice_sizes(operand.ndim(), 1); -+ auto target_indices = mx::split(mx::stack(scatter_indices, 1), -+ scatter_indices[0].shape(0), 0); -+ auto input_indices = mx::split(mx::stack(gather_indices, 1), -+ gather_indices[0].shape(0), 0); -+ auto idx_shape = input_indices[0].shape(); -+ std::vector update_shape(operand.ndim() + idx_shape.size(), 1); -+ for (auto i = 0; i < idx_shape.size(); ++i) { -+ update_shape[i] = idx_shape[i]; -+ } -+ auto update_vals = -+ mx::reshape(mx::gather(operand, input_indices, axes, -+ std::vector(operand.ndim(), 1)), -+ update_shape); -+ return std::vector{ -+ mx::scatter(result, target_indices, update_vals, axes)}; ++ return std::vector{mx::scatter( ++ result, mx::split(result_indices, result_indices.shape(1), 1), ++ mx::gather(operand, ++ mx::split(operand_indices, operand_indices.shape(1), 1), ++ axes, std::vector(operand.ndim(), 1)), ++ axes)}; + }; + op_lookup.pad.emplace(pad_op_info, mx::compile(pad_func)); + } diff --git a/zml/tensor.zig b/zml/tensor.zig index a048ff4c..c997dcaa 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -2978,9 +2978,6 @@ pub const Tensor = struct { test argsort { const zml = @import("zml.zig"); const platform = zml.testing.env(); - // TODO(@cryptodeal): Fix this for MLX PjRt Plugin Implementation. - // Currently segfaults; worked prior to rebase on latest XLA, MLX, and ZML. - try zml.testing.mlxSkipTest(platform); const Local = struct { pub fn _argsort(x: Tensor, axis_: u3, opts: ArgSortOpts) Tensor { From 07e0e0fe69995eea6e8e3abf43ced735e3d541c3 Mon Sep 17 00:00:00 2001 From: James Deal Date: Wed, 4 Jun 2025 00:13:53 -0500 Subject: [PATCH 08/13] Fix unit tests; WIP debug llama --- examples/MODULE.bazel | 5 + examples/MODULE.bazel.lock | 5 +- examples/llama/llama.zig | 1 + examples/llama/main.zig | 15 +- .../mlx/20250518.0-eebe730/MODULE.bazel | 3 +- .../xla/20250527.0-cb67f2f/MODULE.bazel | 12 +- .../overlay/workspace_private.bzl | 25 +- .../patches/0004-Added-MLX-PjRt-Plugin.patch | 4558 +++++++++++++++++ .../xla/20250527.0-cb67f2f/source.json | 31 +- zml/testing.zig | 3 +- 10 files changed, 4628 insertions(+), 30 deletions(-) create mode 100644 third_party/modules/xla/20250527.0-cb67f2f/patches/0004-Added-MLX-PjRt-Plugin.patch diff --git a/examples/MODULE.bazel b/examples/MODULE.bazel index e207c9fb..d7c2c910 100644 --- a/examples/MODULE.bazel +++ b/examples/MODULE.bazel @@ -1,5 +1,10 @@ module(name = "examples") +# local_path_override( +# module_name = "xla", +# path = "../../xla", +# ) + bazel_dep(name = "aspect_bazel_lib", version = "2.14.0") bazel_dep(name = "bazel_skylib", version = "1.7.1") bazel_dep(name = "platforms", version = "0.0.11") diff --git a/examples/MODULE.bazel.lock b/examples/MODULE.bazel.lock index 854c8dbf..7b0a6eb4 100644 --- a/examples/MODULE.bazel.lock +++ b/examples/MODULE.bazel.lock @@ -7,7 +7,6 @@ "https://bazel-registry.zml.ai/modules/abseil-cpp/20230802.0.bcr.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/abseil-cpp/20230802.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/abseil-cpp/20230802.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/abseil-cpp/20240116.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/abseil-cpp/20240116.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/abseil-cpp/20240116.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/abseil-cpp/20240722.0.bcr.2/MODULE.bazel": "not found", @@ -51,6 +50,7 @@ "https://bazel-registry.zml.ai/modules/buildifier_prebuilt/6.4.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/buildifier_prebuilt/7.3.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/buildozer/7.1.2/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/fmt/11.2.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/google_benchmark/1.8.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/googletest/1.11.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/googletest/1.14.0.bcr.1/MODULE.bazel": "not found", @@ -186,7 +186,6 @@ "https://bcr.bazel.build/modules/abseil-cpp/20230802.0.bcr.1/MODULE.bazel": "1c8cec495288dccd14fdae6e3f95f772c1c91857047a098fad772034264cc8cb", "https://bcr.bazel.build/modules/abseil-cpp/20230802.0/MODULE.bazel": "d253ae36a8bd9ee3c5955384096ccb6baf16a1b1e93e858370da0a3b94f77c16", "https://bcr.bazel.build/modules/abseil-cpp/20230802.1/MODULE.bazel": "fa92e2eb41a04df73cdabeec37107316f7e5272650f81d6cc096418fe647b915", - "https://bcr.bazel.build/modules/abseil-cpp/20240116.0/MODULE.bazel": "98dc378d64c12a4e4741ad3362f87fb737ee6a0886b2d90c3cdbb4d93ea3e0bf", "https://bcr.bazel.build/modules/abseil-cpp/20240116.1/MODULE.bazel": "37bcdb4440fbb61df6a1c296ae01b327f19e9bb521f9b8e26ec854b6f97309ed", "https://bcr.bazel.build/modules/abseil-cpp/20240116.2/MODULE.bazel": "73939767a4686cd9a520d16af5ab440071ed75cec1a876bf2fcfaf1f71987a16", "https://bcr.bazel.build/modules/abseil-cpp/20240722.0.bcr.2/MODULE.bazel": "c3661b44c9d3f17f0b65ffb544896aaeb89127398ea867537babac18133a002a", @@ -238,6 +237,8 @@ "https://bcr.bazel.build/modules/buildifier_prebuilt/7.3.1/source.json": "55153a5e6ca9c8a7e266c4b46b951e8a010d25ec6062bc35d5d4f89925796bad", "https://bcr.bazel.build/modules/buildozer/7.1.2/MODULE.bazel": "2e8dd40ede9c454042645fd8d8d0cd1527966aa5c919de86661e62953cd73d84", "https://bcr.bazel.build/modules/buildozer/7.1.2/source.json": "c9028a501d2db85793a6996205c8de120944f50a0d570438fcae0457a5f9d1f8", + "https://bcr.bazel.build/modules/fmt/11.2.0/MODULE.bazel": "bb2cef47f22163e23aa39ff7812cfe8cec85adb6337493fe38d1fa039133ca5c", + "https://bcr.bazel.build/modules/fmt/11.2.0/source.json": "a2c3846c868dbb75e3793e14c7bb9afbb0e34d3967c9a22b335df71eb2ff1481", "https://bcr.bazel.build/modules/google_benchmark/1.8.2/MODULE.bazel": "a70cf1bba851000ba93b58ae2f6d76490a9feb74192e57ab8e8ff13c34ec50cb", "https://bcr.bazel.build/modules/googletest/1.11.0/MODULE.bazel": "3a83f095183f66345ca86aa13c58b59f9f94a2f81999c093d4eeaa2d262d12f4", "https://bcr.bazel.build/modules/googletest/1.14.0.bcr.1/MODULE.bazel": "22c31a561553727960057361aa33bf20fb2e98584bc4fec007906e27053f80c6", diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index 8314a307..ee9c852f 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -51,6 +51,7 @@ pub const LlamaLM = struct { .freq_base = config.rope_theta, .scaling = config.rope_scaling, }; + self.model.norm.eps = config.rms_norm_eps; for (self.model.layers) |*layer| { layer.self_attn.num_heads = self.model.num_heads; layer.self_attn.num_kv_heads = self.model.num_kv_heads; diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 0b121e65..d8148e85 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -216,16 +216,19 @@ pub fn asyncMain() !void { var context = try zml.Context.init(); defer context.deinit(); - const compilation_options = zml.CompilationOptions{ - .xla_dump_to = "/tmp/zml/llama", - .sharding_enabled = res.args.sharding orelse true, - }; - // initialize ZML platform with optional create options // eg: --create-options='{"cuda":{"allocator":{"bfc":{"memory_fraction": 0.99}}}}' const create_opts_json = res.args.@"create-options" orelse "{}"; const create_opts = try std.json.parseFromSlice(zml.Platform.CreateOptions, allocator, create_opts_json, .{}); - const platform = context.autoPlatform(create_opts.value).withCompilationOptions(compilation_options); + + const platform = blk: { + var auto_platform = context.autoPlatform(create_opts.value); + const compilation_options: zml.CompilationOptions = if (auto_platform.target == .mlx) .{} else .{ + .xla_dump_to = "/tmp/zml/llama", + .sharding_enabled = res.args.sharding orelse true, + }; + break :blk auto_platform.withCompilationOptions(compilation_options); + }; create_opts.deinit(); context.printAvailablePlatforms(platform); diff --git a/third_party/modules/mlx/20250518.0-eebe730/MODULE.bazel b/third_party/modules/mlx/20250518.0-eebe730/MODULE.bazel index afd89f51..c24f5489 100644 --- a/third_party/modules/mlx/20250518.0-eebe730/MODULE.bazel +++ b/third_party/modules/mlx/20250518.0-eebe730/MODULE.bazel @@ -4,8 +4,7 @@ module( compatibility_level = 1, ) -bazel_dep(name = "rules_apple", version = "3.17.0", repo_name = "build_bazel_rules_apple") +bazel_dep(name = "rules_apple", version = "3.22.0", repo_name = "build_bazel_rules_apple") bazel_dep(name = "fmt", version = "11.2.0") -bazel_dep(name = "range-v3", version = "0.12.0") metal_cpp = use_extension("//:metal_cpp.bzl", "metal_cpp") use_repo(metal_cpp, "metal_cpp") diff --git a/third_party/modules/xla/20250527.0-cb67f2f/MODULE.bazel b/third_party/modules/xla/20250527.0-cb67f2f/MODULE.bazel index 74c9eaad..434d3365 100644 --- a/third_party/modules/xla/20250527.0-cb67f2f/MODULE.bazel +++ b/third_party/modules/xla/20250527.0-cb67f2f/MODULE.bazel @@ -8,7 +8,6 @@ bazel_dep(name = "platforms", version = "0.0.8") bazel_dep(name = "bazel_skylib", version = "1.5.0") bazel_dep(name = "rules_cc", version = "0.0.17") bazel_dep(name = "rules_apple", version = "3.22.0", repo_name = "build_bazel_rules_apple") -bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "com_google_absl") bazel_dep(name = "rules_python", version = "0.39.0") bazel_dep(name = "rules_proto", version = "6.0.0-rc1") bazel_dep(name = "rules_java", version = "7.3.2") @@ -19,17 +18,28 @@ bazel_dep(name = "rules_license", version = "0.0.8") bazel_dep(name = "rules_shell", version = "0.4.1") bazel_dep(name = "bazel_features", version = "1.25.0", repo_name = "proto_bazel_features") +# MLX PjRt Plugin Dependencies +bazel_dep(name = "mlx", version = "20250518.0-eebe730") + workspace_private = use_extension("//:workspace_private.bzl", "workspace_private") use_repo( workspace_private, "com_github_grpc_grpc", "com_google_protobuf", + "com_google_absl", "local_config_cuda", "local_config_remote_execution", "local_config_rocm", + "local_config_sycl", "local_config_tensorrt", "python_version_repo", "tsl", + "eigen_archive", + "farmhash_archive", + "highwayhash", + "ml_dtypes_py", + "shardy", + "snappy", ) workspace_public = use_extension("//:xla.bzl", "xla") diff --git a/third_party/modules/xla/20250527.0-cb67f2f/overlay/workspace_private.bzl b/third_party/modules/xla/20250527.0-cb67f2f/overlay/workspace_private.bzl index b234d122..4d229862 100644 --- a/third_party/modules/xla/20250527.0-cb67f2f/overlay/workspace_private.bzl +++ b/third_party/modules/xla/20250527.0-cb67f2f/overlay/workspace_private.bzl @@ -1,22 +1,41 @@ +load("//third_party/absl:workspace.bzl", absl = "repo") load("//third_party/gpus:cuda_configure.bzl", "cuda_configure") load("//third_party/gpus:rocm_configure.bzl", "rocm_configure") -load("//third_party/llvm:workspace.bzl", llvm = "repo") +load("//third_party/gpus:sycl_configure.bzl", "sycl_configure") +load("//third_party/eigen3:workspace.bzl", eigen3 = "repo") +load("//third_party/farmhash:workspace.bzl", farmhash = "repo") +load("//third_party/highwayhash:workspace.bzl", highwayhash = "repo") +load("//third_party/py/ml_dtypes:workspace.bzl", ml_dtypes = "repo") load("//third_party/py:python_repo.bzl", "python_repository") load("//third_party/pybind11_bazel:workspace.bzl", pybind11_bazel = "repo") -load("//third_party/stablehlo:workspace.bzl", stablehlo = "repo") +load("//third_party/shardy:workspace.bzl", shardy = "repo") load("//third_party/tensorrt:tensorrt_configure.bzl", "tensorrt_configure") -load("//third_party/triton:workspace.bzl", triton = "repo") load("//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls") load("//third_party:repo.bzl", "tf_vendored") load("//tools/toolchains/remote:configure.bzl", "remote_execution_configure") def _workspace_private_impl(mctx): + absl() cuda_configure(name = "local_config_cuda") remote_execution_configure(name = "local_config_remote_execution") rocm_configure(name = "local_config_rocm") + sycl_configure(name = "local_config_sycl") + shardy() tensorrt_configure(name = "local_config_tensorrt") tf_vendored(name = "tsl", relpath = "third_party/tsl") pybind11_bazel() + eigen3() + farmhash() + highwayhash() + ml_dtypes() + tf_http_archive( + name = "snappy", + build_file = "//third_party:snappy.BUILD", + sha256 = "2e458b7017cd58dcf1469ab315389e85e7f445bd035188f2983f81fb19ecfb29", + strip_prefix = "snappy-984b191f0fefdeb17050b42a90b7625999c13b8d", + system_build_file = "//third_party/systemlibs:snappy.BUILD", + urls = tf_mirror_urls("https://github.com/google/snappy/archive/984b191f0fefdeb17050b42a90b7625999c13b8d.tar.gz"), + ) tf_http_archive( name = "com_github_grpc_grpc", sha256 = "b956598d8cbe168b5ee717b5dafa56563eb5201a947856a6688bbeac9cac4e1f", diff --git a/third_party/modules/xla/20250527.0-cb67f2f/patches/0004-Added-MLX-PjRt-Plugin.patch b/third_party/modules/xla/20250527.0-cb67f2f/patches/0004-Added-MLX-PjRt-Plugin.patch new file mode 100644 index 00000000..ca883653 --- /dev/null +++ b/third_party/modules/xla/20250527.0-cb67f2f/patches/0004-Added-MLX-PjRt-Plugin.patch @@ -0,0 +1,4558 @@ +diff --git a/xla/pjrt/plugin/mlx/BUILD b/xla/pjrt/plugin/mlx/BUILD +new file mode 100644 +index 0000000000..f7417e1e4b +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/BUILD +@@ -0,0 +1,201 @@ ++load("//xla:xla.default.bzl", "xla_cc_binary", "xla_cc_test") ++ ++package( ++ # copybara:uncomment default_applicable_licenses = ["//tensorflow:license"], ++ default_visibility = ["//visibility:public"], ++ licenses = ["notice"], ++) ++ ++cc_library( ++ name = "buffer", ++ srcs = ["buffer.cc"], ++ hdrs = ["buffer.h"], ++ deps = [ ++ ":logging", ++ ":utils", ++ "//xla:literal", ++ "//xla:shape_util", ++ "//xla:util", ++ "//xla/hlo/translate/hlo_to_mhlo:hlo_utils", ++ "//xla/hlo/translate/mhlo_to_hlo:literal_exporter", ++ "//xla/hlo/translate/mhlo_to_hlo:type_to_shape", ++ "//xla/pjrt:pjrt_client", ++ "//xla/pjrt:pjrt_compiler", ++ "//xla/pjrt:pjrt_future", ++ "@com_google_absl//absl/functional:any_invocable", ++ "@com_google_absl//absl/log", ++ "@com_google_absl//absl/status:statusor", ++ "@com_google_absl//absl/types:span", ++ "@llvm-project//llvm:Support", ++ "@llvm-project//mlir:AsmParser", ++ "@llvm-project//mlir:IR", ++ "@llvm-project//mlir:Support", ++ ], ++) ++ ++ ++cc_library( ++ name = "executable", ++ srcs = ["executable.cc"], ++ hdrs = ["executable.h"], ++ deps = [ ++ "@mlx//:mlx", ++ ":buffer", ++ ":logging", ++ ":utils", ++ "//xla/hlo/builder:xla_computation", ++ "//xla/hlo/translate:stablehlo", ++ "//xla/mlir/utils:error_util", ++ "//xla/mlir_hlo:mhlo_passes", ++ "//xla/mlir_hlo:stablehlo_extension_passes", ++ # "//xla/mlir/utils:type_util", ++ "//xla/pjrt:mlir_to_hlo", ++ "//xla/pjrt:pjrt_client", ++ "//xla/pjrt:pjrt_executable", ++ "//xla/pjrt:pjrt_future", ++ "//xla/service:computation_placer_hdr", ++ "@com_google_absl//absl/log", ++ "@com_google_absl//absl/status", ++ "@com_google_absl//absl/status:statusor", ++ "@llvm-project//llvm:Support", ++ "@llvm-project//mlir:AsmParser", ++ "@llvm-project//mlir:BytecodeWriter", ++ "@llvm-project//mlir:DataLayoutInterfaces", ++ "@llvm-project//mlir:FuncDialect", ++ "@llvm-project//mlir:IR", ++ "@llvm-project//mlir:Parser", ++ "@llvm-project//mlir:Pass", ++ "@llvm-project//mlir:Support", ++ "@llvm-project//mlir:Transforms", ++ "@stablehlo//:reference_api", ++ "@stablehlo//:stablehlo_passes_optimization", ++ "@tsl//tsl/platform:statusor", ++ ], ++) ++ ++cc_library( ++ name = "device", ++ srcs = ["device.cc"], ++ hdrs = ["device.h"], ++ deps = [ ++ ":logging", ++ "//xla:util", ++ "//xla/pjrt:pjrt_client", ++ "//xla/pjrt:pjrt_common", ++ "//xla/pjrt:pjrt_compiler", ++ "//xla/pjrt:pjrt_device_description", ++ "@com_google_absl//absl/status:statusor", ++ "@com_google_absl//absl/strings:str_format", ++ "@com_google_absl//absl/strings:string_view", ++ "@com_google_absl//absl/types:span", ++ "@mlx//:mlx", ++ ], ++) ++ ++cc_library( ++ name = "client_cpp_pjrt", ++ srcs = [ ++ "client_cpp_pjrt.cc", ++ ], ++ hdrs = [ ++ "client_cpp_pjrt.h", ++ ], ++ deps = [ ++ # Deps used in example plugin ++ "//xla/pjrt:pjrt_client", ++ "//xla/pjrt:pjrt_compiler", ++ "@com_google_absl//absl/strings:string_view", ++ "@com_google_absl//absl/types:span", ++ "@tsl//tsl/platform:fingerprint", ++ # Deps used for mlx plugin ++ ":buffer", ++ ":device", ++ ":executable", ++ ":logging", ++ ":utils", ++ "//xla:literal", ++ "//xla:shape_util", ++ "//xla:util", ++ "//xla/pjrt:host_memory_spaces", ++ "//xla/pjrt:pjrt_common", ++ "//xla/tsl/framework:allocator", ++ "@com_google_absl//absl/status:statusor", ++ "@com_google_absl//absl/strings:str_format", ++ "@mlx//:mlx", ++ "@tsl//tsl/platform:statusor" ++ ], ++) ++ ++cc_library( ++ name = "client_c_pjrt", ++ srcs = [ ++ "client_c_pjrt.cc", ++ ], ++ hdrs = ["client_c_pjrt.h"], ++ deps = [ ++ ":client_cpp_pjrt", ++ "//xla/pjrt:pjrt_client", ++ "//xla/pjrt/c:pjrt_c_api_hdrs", ++ "//xla/pjrt/c:pjrt_c_api_layouts_extension_hdrs", ++ "//xla/pjrt/c:pjrt_c_api_wrapper_impl", ++ "@com_google_absl//absl/status", ++ ], ++ alwayslink = 1, ++) ++ ++cc_library( ++ name = "logging", ++ srcs = ["logging.cc"], ++ hdrs = ["logging.h"], ++ deps = [ ++ "@com_google_absl//absl/base:log_severity", ++ "@com_google_absl//absl/log", ++ "@com_google_absl//absl/log:globals", ++ "@llvm-project//llvm:Support", ++ "@llvm-project//mlir:AsmParser", ++ "@llvm-project//mlir:DataLayoutInterfaces", ++ "@llvm-project//mlir:IR", ++ ], ++) ++ ++cc_library( ++ name = "utils", ++ srcs = ["utils.cc"], ++ hdrs = ["utils.h"], ++ deps = [ ++ "@com_google_absl//absl/status", ++ "@com_google_absl//absl/status:statusor", ++ "@com_google_absl//absl/types:span", ++ "@llvm-project//llvm:Support", ++ "@llvm-project//mlir:IR", ++ "@llvm-project//mlir:Support", ++ "@mlx//:mlx", ++ "@tsl//tsl/platform:statusor", ++ "//xla/mlir/utils:type_util", ++ "//xla:literal", ++ "//xla:shape_util", ++ "//xla:util", ++ ++ ], ++) ++ ++xla_cc_test( ++ name = "plugin_pjrt_test", ++ srcs = ["plugin_pjrt_test.cc"], ++ deps = [ ++ ":client_c_pjrt", ++ ":client_cpp_pjrt", ++ "//xla/pjrt:pjrt_client_test_common", ++ "//xla/pjrt/c:pjrt_c_api_test_common", ++ "//xla/pjrt/c:pjrt_c_api_wrapper_impl", ++ "@com_google_googletest//:gtest_main", ++ ], ++) ++ ++xla_cc_binary( ++ name = "stablehlo_mlx_plugin.so", ++ linkshared = True, ++ deps = [ ++ ":client_c_pjrt", ++ ], ++) +\ No newline at end of file +diff --git a/xla/pjrt/plugin/mlx/buffer.cc b/xla/pjrt/plugin/mlx/buffer.cc +new file mode 100644 +index 0000000000..c4e7d410ac +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/buffer.cc +@@ -0,0 +1,371 @@ ++/* Copyright 2024 The OpenXLA Authors. ++Licensed under the Apache License, Version 2.0 (the "License"); ++you may not use this file except in compliance with the License. ++You may obtain a copy of the License at ++ http://www.apache.org/licenses/LICENSE-2.0 ++Unless required by applicable law or agreed to in writing, software ++distributed under the License is distributed on an "AS IS" BASIS, ++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++See the License for the specific language governing permissions and ++limitations under the License. ++==============================================================================*/ ++ ++#include "xla/pjrt/plugin/mlx/buffer.h" ++ ++#include ++#include ++#include ++#include ++#include ++ ++#include "absl/functional/any_invocable.h" ++#include "absl/log/log.h" ++#include "absl/status/statusor.h" ++#include "absl/types/span.h" ++#include "llvm/Support/Casting.h" ++#include "llvm/Support/raw_ostream.h" ++#include "mlir/AsmParser/AsmParser.h" ++#include "mlir/IR/Attributes.h" ++#include "mlir/IR/Builders.h" ++#include "mlir/IR/BuiltinAttributes.h" ++#include "mlir/IR/BuiltinTypeInterfaces.h" ++#include "mlir/IR/MLIRContext.h" ++#include "mlx/mlx.h" ++#include "mlir/Support/DebugStringHelper.h" ++#include "xla/hlo/translate/hlo_to_mhlo/hlo_utils.h" ++#include "xla/hlo/translate/mhlo_to_hlo/literal_exporter.h" ++#include "xla/hlo/translate/mhlo_to_hlo/type_to_shape.h" ++#include "xla/literal.h" ++#include "xla/pjrt/pjrt_client.h" ++#include "xla/pjrt/pjrt_compiler.h" ++#include "xla/pjrt/pjrt_future.h" ++#include "xla/pjrt/plugin/mlx/logging.h" ++#include "xla/pjrt/plugin/mlx/utils.h" ++#include "xla/shape.h" ++#include "xla/util.h" ++ ++namespace mx = mlx::core; ++namespace mlir::stablehlo { ++ ++using xla::MutableLiteralBase; ++using xla::PjRtBuffer; ++using xla::PjRtClient; ++using xla::PjRtDevice; ++using xla::PjRtFuture; ++using xla::PjRtMemorySpace; ++using xla::PjRtPlatformId; ++using xla::Shape; ++ ++#define UNIMPLEMENTED(name) \ ++ xla::Unimplemented("MlxPjrtBuffer::" #name " is not implemented") ++ ++class MlxPjrtBuffer : public PjRtBuffer { ++ public: ++ MlxPjrtBuffer(mx::array array, const Shape& shape, ++ PjRtMemorySpace* memory_space) ++ : xla::PjRtBuffer(), ++ context_(), ++ buffer_(), ++ array_(array), ++ shape_(shape), ++ memory_space_(memory_space) { ++ // TRACE_ME_MEMBER; ++ } ++ ++ class MlirClonedExternalReference : public ExternalReference { ++ public: ++ explicit MlirClonedExternalReference(PjRtBuffer* buffer, ++ PjRtMemorySpace* memory_space) ++ : buffer_() { ++ // TRACE_ME_MEMBER; ++ auto mlir_buffer = GetAttributeFromBuffer(buffer); ++ if (!mlir_buffer.ok()) { ++ LOG(ERROR) << "Could not get attribute from buffer: " ++ << mlir_buffer.status(); ++ } ++ ++ auto array = GetArrayFromBuffer(buffer); ++ if (!array.ok()) { ++ LOG(ERROR) << "Could not get attribute from buffer: " << array.status(); ++ } ++ buffer_ = CreateMlirBufferFromAttribute( ++ array.value(), mlir_buffer.value(), memory_space); ++ data_ptr_ = (void*)mlir_buffer.value().getRawData().data(); ++ } ++ ++ private: ++ std::unique_ptr buffer_; ++ }; ++ ++ // All buffers are managed by the MLIR Context ++ ~MlxPjrtBuffer() override = default; ++ ++ MlxPjrtBuffer(const MlxPjrtBuffer&) = delete; ++ MlxPjrtBuffer(MlxPjrtBuffer&&) = delete; ++ MlxPjrtBuffer& operator=(const MlxPjrtBuffer&) = delete; ++ MlxPjrtBuffer& operator=(MlxPjrtBuffer&&) = delete; ++ ++ static std::unique_ptr CreateFromLiteral( ++ mx::array array, const xla::LiteralSlice& literal, ++ xla::PjRtMemorySpace* memory_space) { ++ // TRACE_ME; ++ LOG(INFO) << "CreateFromLiteral: " << literal.ToString() << "\n"; ++ auto buffer = ++ std::make_unique(array, literal.shape(), memory_space); ++ LOG(INFO) << "CreateFromLiteral -> " << (void*)buffer.get() << "\n"; ++ mlir::Builder builder(&buffer->context_); ++ auto attr = xla::CreateDenseElementsAttrFromLiteral(literal, builder); ++ if (!attr.ok()) { ++ LOG(ERROR) << "Could not create dense elements attr from literal: " ++ << attr.status(); ++ return nullptr; ++ } ++ buffer->buffer_ = attr.value(); ++ return buffer; ++ } ++ ++ static std::unique_ptr CreateFromAttribute( ++ mx::array array, DenseElementsAttr attr, ++ xla::PjRtMemorySpace* memory_space) { ++ // TRACE_ME; ++ ++ // MLIR type to xla shape: ++ Shape shape = xla::TypeToShape(attr.getType()); ++ auto buffer = std::make_unique(array, shape, memory_space); ++ buffer->buffer_ = CloneIntoContext(attr, buffer->context_); ++ LOG(INFO) << "CreateFromAttribute(" << ToString(attr) << ") -> " ++ << (void*)buffer.get() << "\n"; ++ return buffer; ++ } ++ ++ const Shape& on_device_shape() const override { ++ // TRACE_ME_MEMBER; ++ return shape_; ++ } ++ absl::StatusOr logical_on_device_shape() override { ++ // TRACE_ME_MEMBER; ++ return shape_; ++ } ++ ++ PjRtPlatformId platform_id() const { ++ // TRACE_ME_MEMBER; ++ return client()->platform_id(); ++ } ++ absl::string_view platform_name() const { ++ // TRACE_ME_MEMBER; ++ return client()->platform_name(); ++ } ++ ++ bool IsEmptyTuple() const { ++ // TRACE_ME_MEMBER; ++ return shape_.IsTuple() && shape_.tuple_shapes().empty(); ++ } ++ ++ // Buffer knows device + client per older design, should only need ++ // memory_space. ++ PjRtMemorySpace* memory_space() const override { ++ // TRACE_ME_MEMBER; ++ return memory_space_; ++ } ++ PjRtDevice* device() const override { ++ // TRACE_ME_MEMBER; ++ return memory_space_->devices().front(); ++ } ++ PjRtClient* client() const override { ++ // TRACE_ME_MEMBER; ++ return memory_space_->client(); ++ } ++ ++ absl::StatusOr> ++ AcquireExternalReference() override { ++ // TRACE_ME_MEMBER; ++ return std::make_unique(this, memory_space_); ++ } ++ ++ xla::PjRtFuture<> ToLiteral(xla::MutableLiteralBase* literal) override { ++ // TRACE_ME_MEMBER; ++ if (IsEmptyTuple()) { ++ return PjRtFuture<>( ++ xla::InvalidArgument("ToLiteral called on empty tuple")); ++ } ++ ++ absl::StatusOr to_copy = ++ mhlo::CreateLiteralFromAttribute(buffer_, {}); ++ if (!to_copy.ok()) return PjRtFuture<>(to_copy.status()); ++ ++ // Synchronous! To make async, make the buffer, start the copy, and return a ++ // future that is ready when the copy is done. ++ auto status = literal->CopyFrom(to_copy.value()); ++ if (!status.ok()) return PjRtFuture<>(status); ++ return PjRtFuture<>(absl::OkStatus()); ++ } ++ ++ PjRtFuture<> LazyToLiteral( ++ absl::AnyInvocable() &&> generator) ++ override { ++ // TRACE_ME_MEMBER; ++ auto buffer = std::move(generator)(); ++ if (!buffer.ok()) return PjRtFuture<>(buffer.status()); ++ return ToLiteral(buffer.value()); ++ } ++ ++ absl::StatusOr GetOnDeviceSizeInBytes() const override { ++ // This is needed by AcquireExternalReference, for framework figuring out ++ // how to read the underlying buffer data. ++ // TRACE_ME_MEMBER; ++ if (!buffer_) return 0; ++ return buffer_.getRawData().size(); ++ } ++ ++ PjRtFuture<> CopyRawToHost(void* dst, int64_t offset, ++ int64_t transfer_size) override { ++ // TRACE_ME_MEMBER; ++ return PjRtFuture<>(UNIMPLEMENTED(CopyRawToHost)); ++ } ++ ++ absl::StatusOr> ++ ReleaseDeviceMemoryOwnership(bool wait_for_operations_to_complete) override { ++ // TRACE_ME_MEMBER; ++ auto external_ref = AcquireExternalReference(); ++ Delete(); ++ return external_ref; ++ } ++ ++ // Remove the buffer if deleted. ++ // Note: deleted and uninitialized appear the same in this scenario. ++ // Consider changing to mlir::NoneType when deleted. ++ void Delete() override { ++ // TRACE_ME_MEMBER; ++ buffer_ = {}; ++ } ++ ++ bool IsDeleted() override { ++ // TRACE_ME_MEMBER; ++ return !buffer_; ++ } ++ ++ absl::StatusOr> CopyToDevice( ++ xla::PjRtDevice* dst_device) { ++ // TRACE_ME_MEMBER; ++ return CopyToMemorySpace( ++ dst_device->default_memory_space().value_or(nullptr)); ++ } ++ ++ absl::StatusOr> CopyToMemorySpace( ++ xla::PjRtMemorySpace* dst_memory_space) override { ++ // TRACE_ME_MEMBER; ++ return CreateMlirBufferFromAttribute(array_, buffer_, dst_memory_space); ++ } ++ ++ void CopyToRemoteDevice( ++ xla::PjRtFuture serialized_descriptor, ++ xla::PjRtBuffer::RemoteSendCallback on_done) override { ++ // TRACE_ME_MEMBER; ++ on_done(UNIMPLEMENTED(CopyToRemoteDevice), false); ++ } ++ ++ xla::PjRtFuture<> GetReadyFuture() override { ++ // TRACE_ME_MEMBER; ++ LOG(INFO) << "GetReadyFuture(" << (void*)this << ")\n"; ++ // Synchronous! To make async, have the device make a buffer with a ready ++ // future that is ready when the computation is done / buffer is ready. ++ return PjRtFuture<>(absl::OkStatus()); ++ } ++ ++ bool IsOnCpu() const override { ++ // If buffer is on CPU, it will be shared with framework via ++ // GetExternalReference, lse it is copied back to host. ++ // Since we are using reference interpreter, we are running on CPU in a ++ // shared memory space. ++ // TRACE_ME_MEMBER; ++ return false; ++ } ++ ++ mlir::DenseElementsAttr GetBufferAttribute() const { return buffer_; } ++ ++ mx::array GetArray() const { return array_; } ++ ++ private: ++ MLIRContext context_; ++ mlir::DenseElementsAttr buffer_; ++ mx::array array_; ++ ++ xla::Shape shape_; ++ PjRtMemorySpace* memory_space_; ++}; ++ ++std::unique_ptr CreateMlirBufferFromMlxArray( ++ mx::array array, xla::PjRtMemorySpace* memory_space) { ++ // TRACE_ME; ++ std::vector span_shape(array.shape().begin(), array.shape().end()); ++ auto shape = xla::ShapeUtil::MakeShape( ++ utils::dtype::asXlaPrimitiveType(array.dtype()), ++ absl::Span(span_shape.data(), span_shape.size())); ++ auto literal = xla::BorrowingLiteral( ++ reinterpret_cast(array.data()), shape); ++ return MlxPjrtBuffer::CreateFromLiteral(array, literal, memory_space); ++} ++ ++std::unique_ptr CreateMlirBufferFromLiteral( ++ const xla::LiteralSlice& literal, xla::PjRtMemorySpace* memory_space) { ++ // TRACE_ME; ++ auto maybe_array = utils::array::fromHostLiteral(literal); ++ return MlxPjrtBuffer::CreateFromLiteral(maybe_array.value(), literal, ++ memory_space); ++} ++ ++std::unique_ptr CreateMlirBufferFromAttribute( ++ mx::array array, DenseElementsAttr attr, ++ xla::PjRtMemorySpace* memory_space) { ++ // TRACE_ME; ++ return MlxPjrtBuffer::CreateFromAttribute(array, attr, memory_space); ++} ++ ++std::unique_ptr CreateMlirBufferUninitialized( ++ const xla::Shape& shape, PjRtMemorySpace* memory_space) { ++ // TRACE_ME; ++ // TODO (@cryptodeal): C API doesn't implement this, but ++ // we'll want to ensure when the Buffer is initialized, ++ // the resulting array is correct shape/dtype. ++ ++ // Init empty array of dtype ++ auto array = mx::array({}); ++ return std::make_unique(array, shape, memory_space); ++} ++ ++absl::StatusOr GetAttributeFromBuffer( ++ xla::PjRtBuffer* buffer) { ++ // TRACE_ME; ++ if (buffer == nullptr || buffer->IsDeleted()) { ++ return xla::InvalidArgument("Buffer is null or deleted"); ++ } ++ auto mlir_buffer = dynamic_cast(buffer); ++ if (mlir_buffer == nullptr) { ++ return xla::InvalidArgument("Buffer is not a MlxPjrtBuffer"); ++ } ++ LOG(INFO) << "GetAttributeFromBuffer(" << (void*)buffer << ") -> " ++ << ToString(mlir_buffer->GetBufferAttribute()) << "\n"; ++ return mlir_buffer->GetBufferAttribute(); ++} ++ ++absl::StatusOr GetArrayFromBuffer(xla::PjRtBuffer* buffer) { ++ // TRACE_ME; ++ if (buffer == nullptr || buffer->IsDeleted()) { ++ return xla::InvalidArgument("Buffer is null or deleted"); ++ } ++ auto mlir_buffer = dynamic_cast(buffer); ++ if (mlir_buffer == nullptr) { ++ return xla::InvalidArgument("Buffer is not a MlxPjrtBuffer"); ++ } ++ ++ return mlir_buffer->GetArray(); ++} ++ ++DenseElementsAttr CloneIntoContext(DenseElementsAttr attr, ++ MLIRContext& context) { ++ Type type = mlir::parseType(mlir::debugString(attr.getType()), &context); ++ return DenseElementsAttr::getFromRawBuffer(llvm::cast(type), ++ attr.getRawData()); ++} ++ ++} // namespace mlir::stablehlo +\ No newline at end of file +diff --git a/xla/pjrt/plugin/mlx/buffer.h b/xla/pjrt/plugin/mlx/buffer.h +new file mode 100644 +index 0000000000..1740f3c779 +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/buffer.h +@@ -0,0 +1,43 @@ ++/* Copyright 2024 The OpenXLA Authors. ++Licensed under the Apache License, Version 2.0 (the "License"); ++you may not use this file except in compliance with the License. ++You may obtain a copy of the License at ++ http://www.apache.org/licenses/LICENSE-2.0 ++Unless required by applicable law or agreed to in writing, software ++distributed under the License is distributed on an "AS IS" BASIS, ++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++See the License for the specific language governing permissions and ++limitations under the License. ++==============================================================================*/ ++ ++#ifndef XLA_PJRT_PLUGIN_STABLEHLO_MLX_BUFFER_H_ ++#define XLA_PJRT_PLUGIN_STABLEHLO_MLX_BUFFER_H_ ++ ++#include ++ ++#include "mlir/IR/BuiltinAttributes.h" ++#include "mlx/mlx.h" ++#include "xla/literal.h" ++#include "xla/pjrt/pjrt_client.h" ++ ++namespace mx = mlx::core; ++ ++namespace mlir::stablehlo { ++std::unique_ptr CreateMlirBufferFromMlxArray( ++ mx::array array, xla::PjRtMemorySpace* memory_space); ++std::unique_ptr CreateMlirBufferFromLiteral( ++ const xla::LiteralSlice& literal, xla::PjRtMemorySpace* memory_space); ++std::unique_ptr CreateMlirBufferFromAttribute( ++ mx::array array, DenseElementsAttr attribute, ++ xla::PjRtMemorySpace* memory_space); ++std::unique_ptr CreateMlirBufferUninitialized( ++ const xla::Shape& shape, xla::PjRtMemorySpace* memory_space); ++absl::StatusOr GetAttributeFromBuffer( ++ xla::PjRtBuffer* buffer); ++absl::StatusOr GetArrayFromBuffer(xla::PjRtBuffer* buffer); ++DenseElementsAttr CloneIntoContext(DenseElementsAttr attr, ++ MLIRContext& context); ++ ++} // namespace mlir::stablehlo ++ ++#endif // XLA_PJRT_PLUGIN_STABLEHLO_MLX_BUFFER_H_ +\ No newline at end of file +diff --git a/xla/pjrt/plugin/mlx/client_c_pjrt.cc b/xla/pjrt/plugin/mlx/client_c_pjrt.cc +new file mode 100644 +index 0000000000..6a0fcbcbcf +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/client_c_pjrt.cc +@@ -0,0 +1,66 @@ ++/* Copyright 2024 The OpenXLA Authors. ++Licensed under the Apache License, Version 2.0 (the "License"); ++you may not use this file except in compliance with the License. ++You may obtain a copy of the License at ++ http://www.apache.org/licenses/LICENSE-2.0 ++Unless required by applicable law or agreed to in writing, software ++distributed under the License is distributed on an "AS IS" BASIS, ++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++See the License for the specific language governing permissions and ++limitations under the License. ++==============================================================================*/ ++ ++#include "xla/pjrt/plugin/mlx/client_c_pjrt.h" ++ ++#include ++#include ++#include ++ ++#include "absl/status/status.h" ++#include "xla/pjrt/c/pjrt_c_api.h" ++#include "xla/pjrt/c/pjrt_c_api_layouts_extension.h" ++#include "xla/pjrt/c/pjrt_c_api_wrapper_impl.h" ++#include "xla/pjrt/pjrt_client.h" ++#include "xla/pjrt/plugin/mlx/client_cpp_pjrt.h" ++ ++namespace mlir::stablehlo { ++ ++// Create my C++ client. Called by the C API and is the glue between the C API ++// and the C++ API. ++PJRT_Error* PJRT_StablehloMlxClient_Create(PJRT_Client_Create_Args* args) { ++ std::unique_ptr client = CreateStablehloMlxPjrtClient(); ++ args->client = pjrt::CreateWrapperClient(std::move(client)); ++ // printf("Creating PJRT Client from client\n"); ++ return nullptr; ++} ++ ++PJRT_Error* PJRT_StablehloMlxExecuteContext_Create( ++ PJRT_ExecuteContext_Create_Args* args) { ++ return new PJRT_Error{absl::UnimplementedError( ++ "ExecuteContext not supported for StablehloMlx execution.")}; ++} ++ ++PJRT_Error* PJRT_StablehloMlxDeviceTopology_Create( ++ PJRT_TopologyDescription_Create_Args* args) { ++ return new PJRT_Error{absl::UnimplementedError( ++ "Topology not supported for StablehloMlx compilation.")}; ++} ++ ++} // namespace mlir::stablehlo ++ ++const PJRT_Api* GetPjrtApi() { ++ // printf("C++ Calling GetPjrtApi"); ++ static PJRT_Layouts_Extension layouts_extension = ++ pjrt::CreateLayoutsExtension(nullptr); ++ ++ static const PJRT_Api pjrt_api = pjrt::CreatePjrtApi( ++ mlir::stablehlo::PJRT_StablehloMlxClient_Create, ++ mlir::stablehlo::PJRT_StablehloMlxExecuteContext_Create, ++ mlir::stablehlo::PJRT_StablehloMlxDeviceTopology_Create, ++ pjrt::PJRT_Plugin_Initialize_NoOp, ++ reinterpret_cast(&layouts_extension), ++ pjrt::PJRT_Plugin_Attributes_Xla); ++ ++ // printf("stablehlo_mlx client called GetPjrtApi\n"); ++ return &pjrt_api; ++} +\ No newline at end of file +diff --git a/xla/pjrt/plugin/mlx/client_c_pjrt.h b/xla/pjrt/plugin/mlx/client_c_pjrt.h +new file mode 100644 +index 0000000000..cc56626953 +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/client_c_pjrt.h +@@ -0,0 +1,24 @@ ++/* Copyright 2024 The OpenXLA Authors. ++Licensed under the Apache License, Version 2.0 (the "License"); ++you may not use this file except in compliance with the License. ++You may obtain a copy of the License at ++ http://www.apache.org/licenses/LICENSE-2.0 ++Unless required by applicable law or agreed to in writing, software ++distributed under the License is distributed on an "AS IS" BASIS, ++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++See the License for the specific language governing permissions and ++limitations under the License. ++==============================================================================*/ ++ ++#ifndef XLA_PJRT_PLUGIN_STABLEHLO_MLX_CLIENT_C_PJRT_H_ ++#define XLA_PJRT_PLUGIN_STABLEHLO_MLX_CLIENT_C_PJRT_H_ ++ ++#include "xla/pjrt/c/pjrt_c_api.h" ++ ++extern "C" { ++ ++// Does not pass ownership of returned PJRT_Api* to caller. ++const PJRT_Api* GetPjrtApi(); ++} ++ ++#endif // XLA_PJRT_PLUGIN_STABLEHLO_MLX_CLIENT_C_PJRT_H_ +\ No newline at end of file +diff --git a/xla/pjrt/plugin/mlx/client_cpp_pjrt.cc b/xla/pjrt/plugin/mlx/client_cpp_pjrt.cc +new file mode 100644 +index 0000000000..fa5ece7d81 +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/client_cpp_pjrt.cc +@@ -0,0 +1,298 @@ ++/* Copyright 2024 The OpenXLA Authors. ++Licensed under the Apache License, Version 2.0 (the "License"); ++you may not use this file except in compliance with the License. ++You may obtain a copy of the License at ++ http://www.apache.org/licenses/LICENSE-2.0 ++Unless required by applicable law or agreed to in writing, software ++distributed under the License is distributed on an "AS IS" BASIS, ++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++See the License for the specific language governing permissions and ++limitations under the License. ++==============================================================================*/ ++ ++#include "xla/pjrt/plugin/mlx/client_cpp_pjrt.h" ++ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "absl/status/statusor.h" ++#include "absl/strings/str_format.h" ++#include "absl/strings/string_view.h" ++#include "absl/types/span.h" ++#include "mlx/mlx.h" ++#include "xla/literal.h" ++#include "xla/pjrt/host_memory_spaces.h" ++#include "xla/pjrt/pjrt_client.h" ++#include "xla/pjrt/pjrt_common.h" ++#include "xla/pjrt/pjrt_compiler.h" ++#include "xla/pjrt/plugin/mlx/buffer.h" ++#include "xla/pjrt/plugin/mlx/device.h" ++#include "xla/pjrt/plugin/mlx/executable.h" ++#include "xla/pjrt/plugin/mlx/logging.h" ++#include "xla/pjrt/plugin/mlx/utils.h" ++#include "xla/shape_util.h" ++#include "xla/util.h" ++#include "tsl/platform/fingerprint.h" ++#include "tsl/platform/statusor.h" ++ ++namespace mx = mlx::core; ++ ++#define UNIMPLEMENTED(name) \ ++ xla::Unimplemented("MlxPjrtBuffer::" #name " is not implemented") ++ ++namespace mlir::stablehlo { ++ ++const xla::PjRtPlatformId kStablehloMlxBackendId = ++ tsl::Fingerprint64(kStablehloMlxBackendName); ++ ++class StablehloMlxPjrtClient : public xla::PjRtClient { ++ public: ++ StablehloMlxPjrtClient() ++ : xla::PjRtClient(), ++ owned_devices_(), ++ devices_(), ++ owned_memory_space_(), ++ memory_space_(nullptr) { ++ // Init device and memory space. ++ // TRACE_ME_MEMBER; ++ owned_devices_.push_back(GetStablehloMlxDevice(this)); ++ devices_.push_back(owned_devices_.back().get()); ++ owned_memory_space_ = std::make_unique( ++ /*id=*/0, devices_.front()); ++ memory_space_ = owned_memory_space_.get(); ++ AttachStablehloMlxMemorySpace(devices_.front(), memory_space_); ++ } ++ ++ ~StablehloMlxPjrtClient() override { ++ // mx::metal::stop_capture(); ++ }; ++ ++ absl::string_view platform_name() const override { ++ // TRACE_ME_MEMBER; ++ return kStablehloMlxBackendName; ++ } ++ int process_index() const override { ++ // TRACE_ME_MEMBER; ++ return 0; ++ } ++ ++ int device_count() const override { ++ // TRACE_ME_MEMBER; ++ return devices_.size(); ++ } ++ ++ int addressable_device_count() const override { ++ // TRACE_ME_MEMBER; ++ return devices_.size(); ++ } ++ ++ absl::Span devices() const override { ++ // TRACE_ME_MEMBER; ++ return devices_; ++ } ++ absl::Span addressable_devices() const override { ++ // TRACE_ME_MEMBER; ++ return devices_; ++ } ++ ++ absl::Span memory_spaces() const override { ++ // TRACE_ME_MEMBER; ++ return absl::MakeSpan(&memory_space_, 1); ++ } ++ ++ // Return an ID that identifies the platform via tsl fingerprint. ++ xla::PjRtPlatformId platform_id() const override { ++ // TRACE_ME_MEMBER; ++ return kStablehloMlxBackendId; ++ } ++ ++ // Returns a string containing human-readable, platform-specific version ++ // info (e.g. the CUDA version on GPU or libtpu version on Cloud TPU). ++ absl::string_view platform_version() const override { ++ // TRACE_ME_MEMBER; ++ return "StableHLO MLX v0.1"; ++ } ++ ++ ///////////// ++ // Device ++ ///////////// ++ ++ // Lookup any PjRtDevice for a given PjRtDevice::id(). ++ // TODO: Should this be a base class? I.e. why doesn't the base client have ++ // a vector a device pointers? ++ absl::StatusOr LookupDevice( ++ xla::PjRtGlobalDeviceId global_device_id) const override { ++ // TRACE_ME_MEMBER; ++ for (auto device : devices_) { ++ if (device->global_device_id() == global_device_id) { ++ return device; ++ } ++ } ++ // TODO: This error should be a base class method since its used in tests. ++ return xla::InvalidArgument("No matching device found for device_id %d", ++ global_device_id.value()); ++ } ++ ++ absl::StatusOr LookupAddressableDevice( ++ xla::PjRtLocalDeviceId local_device_id) const override { ++ // TRACE_ME_MEMBER; ++ ++ for (auto* device : addressable_devices()) { ++ if (local_device_id == device->local_device_id()) { ++ return device; ++ } ++ } ++ return xla::InvalidArgument( ++ "No matching device found for local_device_id %d", ++ local_device_id.value()); ++ } ++ ++ absl::StatusOr GetDefaultDeviceAssignment( ++ int num_replicas, int num_partitions) const override { ++ // TRACE_ME_MEMBER; ++ xla::DeviceAssignment assignment(num_replicas, num_partitions); ++ for (int64_t i = 0; i < num_replicas; ++i) { ++ for (int64_t j = 0; j < num_partitions; ++j) { ++ auto idx = (i + (j * num_replicas)) % devices_.size(); ++ assignment(i, j) = devices_[idx]->global_device_id().value(); ++ } ++ } ++ return assignment; ++ } ++ ++ ///////////////// ++ // Buffer methods ++ ///////////////// ++ ++ absl::StatusOr> CreateErrorBuffer( ++ absl::Status error, const xla::Shape& shape, xla::PjRtDevice* device) { ++ // Prefer memory space implementation, device holding buffer is ++ // deprecated. ++ return CreateErrorBuffer(error, shape, ++ device->default_memory_space().value_or(nullptr)); ++ } ++ ++ absl::StatusOr> CreateErrorBuffer( ++ absl::Status error, const xla::Shape& shape, ++ xla::PjRtMemorySpace* memory) { ++ return UNIMPLEMENTED(CreateErrorBuffer); ++ } ++ ++ absl::StatusOr> CreateUninitializedBuffer( ++ const xla::Shape& shape, xla::PjRtDevice* device) { ++ // TRACE_ME_MEMBER; ++ return CreateMlirBufferUninitialized( ++ shape, device->default_memory_space().value_or(nullptr)); ++ } ++ ++ absl::StatusOr> ++ CreateBuffersForAsyncHostToDevice(absl::Span shapes, ++ xla::PjRtDevice* device) { ++ // TRACE_ME_MEMBER; ++ return CreateBuffersForAsyncHostToDevice( ++ shapes, device->default_memory_space().value_or(nullptr)); ++ } ++ ++ absl::StatusOr> ++ CreateBuffersForAsyncHostToDevice(absl::Span shapes, ++ xla::PjRtMemorySpace* memory_space) { ++ // TRACE_ME_MEMBER; ++ return UNIMPLEMENTED(CreateBuffersForAsyncHostToDevice); ++ } ++ ++ absl::StatusOr> BufferFromHostBuffer( ++ const void* data, xla::PrimitiveType type, absl::Span dims, ++ std::optional> byte_strides, ++ HostBufferSemantics host_buffer_semantics, ++ absl::AnyInvocable on_done_with_host_buffer, ++ xla::PjRtDevice* device) { ++ // TRACE_ME_MEMBER; ++ return BufferFromHostBuffer( ++ data, type, dims, byte_strides, host_buffer_semantics, ++ std::move(on_done_with_host_buffer), ++ device->default_memory_space().value_or(nullptr), nullptr); ++ } ++ ++ absl::StatusOr> BufferFromHostBuffer( ++ const void* data, xla::PrimitiveType type, absl::Span dims, ++ std::optional> byte_strides, ++ HostBufferSemantics host_buffer_semantics, ++ absl::AnyInvocable on_done_with_host_buffer, ++ xla::PjRtDevice* device, const xla::Layout* device_layout) { ++ // TRACE_ME_MEMBER; ++ return BufferFromHostBuffer( ++ data, type, dims, byte_strides, host_buffer_semantics, ++ std::move(on_done_with_host_buffer), ++ device->default_memory_space().value_or(nullptr), device_layout); ++ } ++ ++ absl::StatusOr> BufferFromHostBuffer( ++ const void* data, xla::PrimitiveType type, absl::Span dims, ++ std::optional> byte_strides, ++ HostBufferSemantics host_buffer_semantics, ++ absl::AnyInvocable on_done_with_host_buffer, ++ xla::PjRtMemorySpace* memory_space, const xla::Layout* device_layout) { ++ // TRACE_ME_MEMBER; ++ TF_ASSIGN_OR_RETURN( ++ mx::array array_buffer, ++ utils::array::fromHostBuffer(data, dims, byte_strides, type)); ++ auto buffer = CreateMlirBufferFromMlxArray(array_buffer, memory_space); ++ if (on_done_with_host_buffer) { ++ // If host is awaiting the result, must call this function. ++ std::move(on_done_with_host_buffer)(); ++ on_done_with_host_buffer = nullptr; ++ } ++ return buffer; ++ } ++ ++ absl::StatusOr> BufferFromHostLiteral( ++ const xla::LiteralSlice& literal, xla::PjRtDevice* device) { ++ // TRACE_ME_MEMBER; ++ return CreateMlirBufferFromLiteral( ++ literal, device->default_memory_space().value_or(nullptr)); ++ } ++ ++ absl::StatusOr> BufferFromHostLiteral( ++ const xla::LiteralSlice& literal, xla::PjRtMemorySpace* memory_space) { ++ // TRACE_ME_MEMBER; ++ return CreateMlirBufferFromLiteral(literal, memory_space); ++ } ++ ++ /////////// ++ // Compile ++ /////////// ++ absl::StatusOr> CompileAndLoad( ++ mlir::ModuleOp module, xla::CompileOptions options) override { ++ // TRACE_ME_MEMBER; ++ return mlir::stablehlo::StablehloMlxCompile( ++ module, GetDefaultDeviceAssignment(1, devices_.size()).value(), this); ++ } ++ ++ // Compile `computation` with given `options`. ++ absl::StatusOr> CompileAndLoad( ++ const xla::XlaComputation& computation, ++ xla::CompileOptions options) override { ++ // TRACE_ME_MEMBER; ++ return mlir::stablehlo::StablehloMlxCompile( ++ computation.proto(), ++ GetDefaultDeviceAssignment(1, devices_.size()).value(), this); ++ } ++ ++ private: ++ std::vector> owned_devices_; ++ std::vector devices_; ++ std::unique_ptr owned_memory_space_; ++ xla::PjRtMemorySpace* memory_space_; ++}; // end class ++ ++std::unique_ptr CreateStablehloMlxPjrtClient() { ++ // mx::metal::start_capture("mlx_trace.gputrace"); ++ SetupLogLevelFromEnv(); ++ return std::make_unique(); ++} ++ ++} // namespace mlir::stablehlo +\ No newline at end of file +diff --git a/xla/pjrt/plugin/mlx/client_cpp_pjrt.h b/xla/pjrt/plugin/mlx/client_cpp_pjrt.h +new file mode 100644 +index 0000000000..8c90942f0c +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/client_cpp_pjrt.h +@@ -0,0 +1,30 @@ ++/* Copyright 2024 The OpenXLA Authors. ++Licensed under the Apache License, Version 2.0 (the "License"); ++you may not use this file except in compliance with the License. ++You may obtain a copy of the License at ++ http://www.apache.org/licenses/LICENSE-2.0 ++Unless required by applicable law or agreed to in writing, software ++distributed under the License is distributed on an "AS IS" BASIS, ++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++See the License for the specific language governing permissions and ++limitations under the License. ++==============================================================================*/ ++ ++#ifndef XLA_PJRT_PLUGIN_STABLEHLO_MLX_CLIENT_CPP_PJRT_H_ ++#define XLA_PJRT_PLUGIN_STABLEHLO_MLX_CLIENT_CPP_PJRT_H_ ++ ++#include ++ ++#include "xla/pjrt/pjrt_client.h" ++#include "xla/pjrt/pjrt_compiler.h" ++ ++namespace mlir::stablehlo { ++ ++inline constexpr char kStablehloMlxBackendName[] = "stablehlo_mlx"; ++extern const xla::PjRtPlatformId kStablehloMlxBackendId; ++ ++std::unique_ptr CreateStablehloMlxPjrtClient(); ++ ++} // namespace mlir::stablehlo ++ ++#endif // XLA_PJRT_PLUGIN_STABLEHLO_MLX_CLIENT_CPP_PJRT_H_ +\ No newline at end of file +diff --git a/xla/pjrt/plugin/mlx/device.cc b/xla/pjrt/plugin/mlx/device.cc +new file mode 100644 +index 0000000000..6984da5b7f +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/device.cc +@@ -0,0 +1,213 @@ ++/* Copyright 2024 The OpenXLA Authors. ++ ++ ++Licensed under the Apache License, Version 2.0 (the "License"); ++you may not use this file except in compliance with the License. ++You may obtain a copy of the License at ++ http://www.apache.org/licenses/LICENSE-2.0 ++Unless required by applicable law or agreed to in writing, software ++distributed under the License is distributed on an "AS IS" BASIS, ++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++See the License for the specific language governing permissions and ++limitations under the License. ++==============================================================================*/ ++ ++#include "mlx/mlx.h" ++ ++#include "xla/pjrt/plugin/mlx/device.h" ++ ++#include "absl/status/statusor.h" ++#include "absl/strings/str_format.h" ++#include "absl/strings/string_view.h" ++#include "absl/types/span.h" ++#include "xla/pjrt/pjrt_client.h" ++#include "xla/pjrt/pjrt_common.h" ++#include "xla/pjrt/pjrt_compiler.h" ++#include "xla/pjrt/pjrt_device_description.h" ++#include "xla/pjrt/plugin/mlx/logging.h" ++#include "xla/util.h" ++ ++#define UNIMPLEMENTED(name) \ ++ xla::Unimplemented("StablehloMlxDevice::" #name " is not implemented") ++ ++namespace mlir::stablehlo { ++ ++using xla::LiteralSlice; ++using xla::MutableBorrowingLiteral; ++using xla::PjRtClient; ++using xla::PjRtDevice; ++using xla::PjRtDeviceAttribute; ++using xla::PjRtDeviceDescription; ++using xla::PjRtGlobalDeviceId; ++using xla::PjRtLocalDeviceId; ++using xla::PjRtLocalHardwareId; ++using xla::PjRtMemorySpace; ++using xla::ScopedAsyncTrackingEvent; ++using xla::Unimplemented; ++ ++namespace mx = mlx::core; ++ ++inline std::string mlxDeviceToString(const mx::Device& device) { ++ std::ostringstream os; ++ os << device; ++ return os.str(); ++} ++// Devices need a device description. ++class StablehloMlxDeviceDescription final : public PjRtDeviceDescription { ++ public: ++ explicit StablehloMlxDeviceDescription(int process_id, int local_device_id) ++ : id_(local_device_id), ++ process_index_(process_id), ++ local_hardware_id_(local_device_id), ++ debug_string_(mlxDeviceToString(mx::default_device())), ++ to_string_(absl::StrFormat("%s(id=%d,pid=%d)", debug_string_, ++ id_.value(), process_index_)) { ++ // TRACE_ME_MEMBER; ++ } ++ ++ int id() const override { ++ // TRACE_ME_MEMBER; ++ return id_.value(); ++ } ++ int process_index() const override { ++ // TRACE_ME_MEMBER; ++ return process_index_; ++ } ++ int local_hardware_id() const { ++ // TRACE_ME_MEMBER; ++ return local_hardware_id_; ++ } ++ ++ absl::string_view device_kind() const override { ++ // TRACE_ME_MEMBER; ++ return DebugString(); ++ } ++ ++ absl::string_view DebugString() const override { ++ // TRACE_ME_MEMBER; ++ return debug_string_; ++ } ++ ++ absl::string_view ToString() const override { ++ // TRACE_ME_MEMBER; ++ return to_string_; ++ } ++ ++ const absl::flat_hash_map& Attributes() ++ const override { ++ // TRACE_ME_MEMBER; ++ return attributes_; ++ } ++ ++ private: ++ PjRtGlobalDeviceId id_; ++ int process_index_; ++ int local_hardware_id_; ++ std::string debug_string_; ++ std::string to_string_; ++ absl::flat_hash_map attributes_ = {}; ++}; ++ ++// Clients need devices, and clients own the devices. ++class StablehloMlxDevice : public PjRtDevice { ++ public: ++ explicit StablehloMlxDevice(PjRtClient* client) ++ : PjRtDevice(), client_(client), description_(0, 0) { ++ // TRACE_ME_MEMBER; ++ } ++ ++ absl::string_view DebugString() const override { ++ // TRACE_ME_MEMBER; ++ return "StablehloMlxDevice"; ++ } ++ ++ PjRtLocalDeviceId local_device_id() const override { ++ // TRACE_ME_MEMBER; ++ return PjRtLocalDeviceId(local_hardware_id().value()); ++ } ++ ++ PjRtLocalHardwareId local_hardware_id() const override { ++ // TRACE_ME_MEMBER; ++ return PjRtLocalHardwareId(description_.local_hardware_id()); ++ } ++ ++ PjRtClient* client() const override { ++ // TRACE_ME_MEMBER; ++ return client_; ++ } ++ ++ bool IsAddressable() const override { ++ // TRACE_ME_MEMBER; ++ return process_index() == client()->process_index(); ++ } ++ ++ absl::Status TransferToInfeed(const LiteralSlice& literal) override { ++ // TRACE_ME_MEMBER; ++ return UNIMPLEMENTED(TransferToInfeed); ++ } ++ ++ absl::Status TransferFromOutfeed(MutableBorrowingLiteral literal) override { ++ // TRACE_ME_MEMBER; ++ return UNIMPLEMENTED(TransferFromOutfeed); ++ } ++ ++ void AttachDefaultMemorySpace(PjRtMemorySpace* memory_space) { ++ // TRACE_ME_MEMBER; ++ memory_space_ = memory_space; ++ } ++ ++ absl::Span memory_spaces() const override { ++ // TRACE_ME_MEMBER; ++ return absl::MakeSpan(&memory_space_, 1); ++ } ++ ++ absl::StatusOr default_memory_space() const override { ++ // TRACE_ME_MEMBER; ++ if (!memory_space_) ++ return absl::InternalError("Plugin memory space unset."); ++ ++ return memory_space_; ++ } ++ ++ std::unique_ptr CreateAsyncTrackingEvent( ++ absl::string_view description) const override { ++ // TRACE_ME_MEMBER; ++ LOG(FATAL) << "Plugin does not implement CreateAsyncTrackingEvent."; ++ return nullptr; ++ } ++ ++ const PjRtDeviceDescription& description() const override { ++ // TRACE_ME_MEMBER; ++ return description_; ++ } ++ ++ private: ++ PjRtClient* client_; ++ PjRtMemorySpace* memory_space_; // unpinned memory owned by client ++ StablehloMlxDeviceDescription description_; ++}; ++ ++// Device Description ++std::unique_ptr GetStablehloMlxDeviceDescription( ++ int process_id, int local_device_id) { ++ return std::make_unique(process_id, ++ local_device_id); ++} ++ ++// Mlx Device ++std::unique_ptr GetStablehloMlxDevice(PjRtClient* client) { ++ return std::make_unique(client); ++} ++ ++void AttachStablehloMlxMemorySpace(PjRtDevice* device, ++ PjRtMemorySpace* memory_space) { ++ auto stablehlo_device = dynamic_cast(device); ++ if (stablehlo_device == nullptr) { ++ LOG(FATAL) << "Plugin cannot attach memory space to device of kind " ++ << device->device_kind(); ++ return; ++ } ++ stablehlo_device->AttachDefaultMemorySpace(memory_space); ++} ++ ++} // namespace mlir::stablehlo +\ No newline at end of file +diff --git a/xla/pjrt/plugin/mlx/device.h b/xla/pjrt/plugin/mlx/device.h +new file mode 100644 +index 0000000000..ef42968a3a +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/device.h +@@ -0,0 +1,36 @@ ++/* Copyright 2024 The OpenXLA Authors. ++Licensed under the Apache License, Version 2.0 (the "License"); ++you may not use this file except in compliance with the License. ++You may obtain a copy of the License at ++ http://www.apache.org/licenses/LICENSE-2.0 ++Unless required by applicable law or agreed to in writing, software ++distributed under the License is distributed on an "AS IS" BASIS, ++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++See the License for the specific language governing permissions and ++limitations under the License. ++==============================================================================*/ ++ ++#ifndef XLA_PJRT_PLUGIN_STABLEHLO_MLX_DEVICE_H_ ++#define XLA_PJRT_PLUGIN_STABLEHLO_MLX_DEVICE_H_ ++ ++#include ++ ++#include "xla/pjrt/pjrt_client.h" ++#include "xla/pjrt/pjrt_compiler.h" ++#include "xla/pjrt/pjrt_device_description.h" ++ ++namespace mlir::stablehlo { ++ ++// Device Description ++std::unique_ptr GetStablehloMlxDeviceDescription( ++ int process_id, int local_device_id); ++ ++// MLX Device ++std::unique_ptr GetStablehloMlxDevice(xla::PjRtClient* client); ++ ++void AttachStablehloMlxMemorySpace(xla::PjRtDevice* device, ++ xla::PjRtMemorySpace* memory_space); ++ ++} // namespace mlir::stablehlo ++ ++#endif // XLA_PJRT_PLUGIN_STABLEHLO_MLX_DEVICE_H_ +\ No newline at end of file +diff --git a/xla/pjrt/plugin/mlx/executable.cc b/xla/pjrt/plugin/mlx/executable.cc +new file mode 100644 +index 0000000000..c1ceac0c7c +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/executable.cc +@@ -0,0 +1,2337 @@ ++/* Copyright 2024 The OpenXLA Authors. ++Licensed under the Apache License, Version 2.0 (the "License"); ++you may not use this file except in compliance with the License. ++You may obtain a copy of the License at ++ http://www.apache.org/licenses/LICENSE-2.0 ++Unless required by applicable law or agreed to in writing, software ++distributed under the License is distributed on an "AS IS" BASIS, ++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++See the License for the specific language governing permissions and ++limitations under the License. ++==============================================================================*/ ++ ++#include "xla/pjrt/plugin/mlx/executable.h" ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++// TODO(@cryptodeal): might need to update `BUILD` ++#include "mlir/IR/Visitors.h" ++#include "llvm/ADT/TypeSwitch.h" ++#include "llvm/ADT/Hashing.h" ++ ++#include "mlx/mlx.h" ++#include "absl/log/log.h" ++#include "absl/status/status.h" ++#include "absl/status/statusor.h" ++#include "mlir/Bytecode/BytecodeWriter.h" ++#include "mlir/Dialect/Func/IR/FuncOps.h" ++#include "mlir/IR/AsmState.h" ++#include "mlir/IR/BuiltinAttributes.h" ++#include "mlir/IR/MLIRContext.h" ++#include "mlir/IR/Operation.h" ++#include "mlir/IR/OwningOpRef.h" ++#include "mlir/Interfaces/DataLayoutInterfaces.h" ++#include "mlir/Pass/PassManager.h" ++#include "mlir/Support/LLVM.h" ++#include "stablehlo/reference/Api.h" ++#include "stablehlo/transforms/optimization/Passes.h" ++#include "xla/hlo/translate/stablehlo.h" ++#include "xla/mlir/utils/error_util.h" ++#include "xla/mlir/utils/type_util.h" ++#include "xla/mlir_hlo/mhlo/transforms/passes.h" ++#include "xla/mlir_hlo/stablehlo_ext/transforms/passes.h" ++#include "xla/pjrt/pjrt_client.h" ++#include "xla/pjrt/pjrt_executable.h" ++#include "xla/pjrt/pjrt_future.h" ++#include "xla/pjrt/plugin/mlx/buffer.h" ++#include "xla/pjrt/plugin/mlx/logging.h" ++#include "xla/pjrt/plugin/mlx/utils.h" ++#include "xla/service/computation_placer.h" ++#include "tsl/platform/status.h" ++#include "tsl/platform/statusor.h" ++ ++#define DEBUG_TYPE "stablehlo-pjrt" ++ ++namespace mx = mlx::core; ++ ++typedef absl::StatusOr StatusOrArray; ++typedef absl::StatusOr> StatusOrArrays; ++typedef absl::StatusOr StatusOrInt; ++ ++namespace mlir::stablehlo { ++ ++#define UNIMPLEMENTED(name) \ ++ xla::Unimplemented("MlxPjRtBuffer::" #name " is not implemented") ++ ++using xla::DeviceAssignment; ++using xla::PjRtBuffer; ++using xla::PjRtClient; ++using xla::PjRtDevice; ++using xla::PjRtFuture; ++using xla::PjRtLoadedExecutable; ++using xla::PjRtMemorySpace; ++ ++void resolveFuncDeps(ModuleOp mod, std::string func_name, Tree& dep_tree, ++ Node* parent = nullptr) { ++ auto new_node = dep_tree.insert(func_name, parent); ++ auto func = mod.lookupSymbol(func_name); ++ auto& block = func.front(); ++ std::set func_names; ++ for (auto op : block.getOps()) { ++ std::string callee_name = op.getCallee().str(); ++ func_names.insert(callee_name); ++ resolveFuncDeps(mod, callee_name, dep_tree, new_node); ++ } ++} ++ ++absl::Status validateModule(ModuleOp mod) { ++ absl::Status status; ++ // TODO: some ops should have additional checks (e.g. `stablehlo::ReduceOp`, ++ // `stablehlo::SortOp`, etc.) where we need additional checks at compile time ++ // to ensure that we're correctly mapping to the corresponding MLX op. ++ mod.walk([&status](Operation* op) { ++ // Series of checks to ensure we support the module ++ return llvm::TypeSwitch(op) ++ .Case([&status, &op](auto o) { ++ // Validate all input types are compatible with mlx ++ for (auto operand : o->getOperands()) { ++ auto primitive_type = xla::ConvertMlirTypeToPrimitiveType( ++ mlir::cast(operand.getType()).getElementType()); ++ switch (primitive_type) { ++ case xla::PrimitiveType::PRED: ++ case xla::PrimitiveType::U8: ++ case xla::PrimitiveType::S8: ++ case xla::PrimitiveType::U16: ++ case xla::PrimitiveType::S16: ++ case xla::PrimitiveType::U32: ++ case xla::PrimitiveType::S32: ++ case xla::PrimitiveType::U64: ++ case xla::PrimitiveType::S64: ++ case xla::PrimitiveType::F16: ++ case xla::PrimitiveType::BF16: ++ case xla::PrimitiveType::F32: ++ case xla::PrimitiveType::C64: ++ break; ++ default: { ++ std::cout << "Unsupported type: " ++ << xla::PrimitiveType_Name(primitive_type).c_str() ++ << std::endl; ++ status = absl::UnimplementedError( ++ absl::StrCat("Unsupported type: ", ToString(op))); ++ return WalkResult::interrupt(); ++ } ++ } ++ } ++ ++ // Validate all result types are compatible with mlx ++ for (auto result : o->getResults()) { ++ auto primitive_type = xla::ConvertMlirTypeToPrimitiveType( ++ mlir::cast(result.getType()).getElementType()); ++ switch (primitive_type) { ++ case xla::PrimitiveType::PRED: ++ case xla::PrimitiveType::U8: ++ case xla::PrimitiveType::S8: ++ case xla::PrimitiveType::U16: ++ case xla::PrimitiveType::S16: ++ case xla::PrimitiveType::U32: ++ case xla::PrimitiveType::S32: ++ case xla::PrimitiveType::U64: ++ case xla::PrimitiveType::S64: ++ case xla::PrimitiveType::F16: ++ case xla::PrimitiveType::BF16: ++ case xla::PrimitiveType::F32: ++ case xla::PrimitiveType::C64: ++ break; ++ default: { ++ std::cout << "Unsupported type: " ++ << xla::PrimitiveType_Name(primitive_type).c_str() ++ << std::endl; ++ status = absl::UnimplementedError( ++ absl::StrCat("Unsupported type: ", ToString(op))); ++ return WalkResult::interrupt(); ++ } ++ } ++ } ++ return mlir::WalkResult::advance(); ++ }) ++ // TODO(@cryptodeal): These ops were supported prior to `mlx::compile` ++ // need to modify the implementation to ensure no use of `mlx::eval` and ++ // ensure that `mlx::random::key` is being propogated correctly. ++ .Case( ++ [&status, &op](auto o) { ++ status = absl::UnimplementedError( ++ absl::StrCat("Unsupported op: ", ToString(op))); ++ return WalkResult::interrupt(); ++ }) ++ .Default([&status, &op](auto o) { ++ status = absl::UnimplementedError( ++ absl::StrCat("Unsupported op: ", ToString(op))); ++ return WalkResult::interrupt(); ++ }); ++ }); ++ return status; ++} ++ ++mlir::OwningOpRef cloneIntoContext(ModuleOp module, ++ MLIRContext& context) { ++ // Clone the module into the context. MHLO->StableHLO just in case. ++ PassManager pm(module->getContext()); ++ pm.addPass(mhlo::createHloLegalizeToStablehloPass()); ++ if (failed(pm.run(module))) { ++ LOG(ERROR) << "Failed to convert MHLO to StableHLO"; ++ return nullptr; ++ } ++ ++ std::string bytecode; ++ llvm::raw_string_ostream os(bytecode); ++ mlir::BytecodeWriterConfig config; ++ mlir::OwningOpRef cloned = module.clone(); ++ if (mlir::failed(mlir::writeBytecodeToFile(*cloned, os, config))) { ++ LOG(ERROR) << "Failed to write bytecode to string\n"; ++ return nullptr; ++ } ++ return *parseStablehloModule(bytecode, context); ++} ++ ++LogicalResult decomposeChloToStablehlo(ModuleOp module) { ++ PassManager pm(module->getContext()); ++ stablehlo_ext::createChloLegalizeToStablehloPipeline(pm); ++ if (failed(pm.run(module))) { ++ return module->emitError() << "Failed to recompose CHLO"; ++ } ++ return success(); ++} ++ ++std::optional getUnsupportedOp(ModuleOp module) { ++ std::optional unsupported_op(std::nullopt); ++ module.walk([&unsupported_op](Operation* op) { ++ auto cc = llvm::dyn_cast(op); ++ if (cc) { ++ unsupported_op = op; ++ return WalkResult::interrupt(); ++ } ++ return WalkResult::advance(); ++ }); ++ return unsupported_op; ++} ++ ++LogicalResult runHardwareIndependentOptimizations(ModuleOp module) { ++ PassManager pm(module->getContext()); ++ pm.addNestedPass( ++ stablehlo::createStablehloAggressiveFolderPass()); ++ pm.addNestedPass( ++ stablehlo::createStablehloAggressiveSimplificationPass()); ++ if (failed(pm.run(module))) { ++ return module->emitError() ++ << "Failed to run hardware independent optimizations"; ++ } ++ return success(); ++} ++ ++mlir::ShapedType getResultType(ModuleOp module, mx::Dtype dtype, ++ SmallVector shape) { ++ switch (dtype) { ++ case mx::bfloat16: ++ return RankedTensorType::get(ArrayRef(shape), ++ BFloat16Type::get(module.getContext())); ++ case mx::float16: ++ return RankedTensorType::get(ArrayRef(shape), ++ Float16Type::get(module.getContext())); ++ case mx::int8: ++ return RankedTensorType::get( ++ ArrayRef(shape), ++ IntegerType::get(module.getContext(), 8, ++ mlir::IntegerType::Signless)); ++ case mx::uint8: ++ return RankedTensorType::get( ++ ArrayRef(shape), ++ IntegerType::get(module.getContext(), 8, ++ mlir::IntegerType::Unsigned)); ++ case mx::int16: ++ return RankedTensorType::get( ++ ArrayRef(shape), ++ IntegerType::get(module.getContext(), 16, ++ mlir::IntegerType::Signless)); ++ case mx::uint16: ++ return RankedTensorType::get( ++ ArrayRef(shape), ++ IntegerType::get(module.getContext(), 16, ++ mlir::IntegerType::Unsigned)); ++ case mx::int32: ++ return RankedTensorType::get( ++ ArrayRef(shape), ++ IntegerType::get(module.getContext(), 32, ++ mlir::IntegerType::Signless)); ++ case mx::uint32: ++ return RankedTensorType::get( ++ ArrayRef(shape), ++ IntegerType::get(module.getContext(), 32, ++ mlir::IntegerType::Unsigned)); ++ case mx::int64: ++ return RankedTensorType::get( ++ ArrayRef(shape), ++ IntegerType::get(module.getContext(), 64, ++ mlir::IntegerType::Signless)); ++ case mx::uint64: ++ return RankedTensorType::get( ++ ArrayRef(shape), ++ IntegerType::get(module.getContext(), 64, ++ mlir::IntegerType::Unsigned)); ++ case mx::complex64: ++ return RankedTensorType::get( ++ ArrayRef(shape), ++ ComplexType::get(Float32Type::get(module.getContext()))); ++ default: ++ return RankedTensorType::get(ArrayRef(shape), ++ Float32Type::get(module.getContext())); ++ } ++} ++ ++// Returns true if the given `ReduceOp` matches ++// `zml.Tensor.argMax` op implementation. ++bool isArgMaxReduce(stablehlo::ReduceOp& reduce_op) { ++ Block& body_block = reduce_op.getBody().front(); ++ unsigned op_count = 0; ++ // TODO (@cryptodeal): This is a lazy verification; maybe check operands ++ // for each operation in the block. ++ for (auto& op : body_block.getOperations()) { ++ switch (op_count) { ++ case 0: { ++ auto compare_op = llvm::dyn_cast(op); ++ if (compare_op && ++ compare_op.getComparisonDirection() == ComparisonDirection::GT) { ++ break; ++ } else ++ return false; ++ } ++ case 1: { ++ auto compare_op = llvm::dyn_cast(op); ++ if (compare_op && ++ compare_op.getComparisonDirection() == ComparisonDirection::NE) { ++ break; ++ } else ++ return false; ++ } ++ case 2: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ case 3: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ case 4: { ++ auto compare_op = llvm::dyn_cast(op); ++ if (compare_op && ++ compare_op.getComparisonDirection() == ComparisonDirection::EQ) { ++ break; ++ } else ++ return false; ++ } ++ case 5: { ++ auto compare_op = llvm::dyn_cast(op); ++ if (compare_op && ++ compare_op.getComparisonDirection() == ComparisonDirection::LT) { ++ break; ++ } else ++ return false; ++ } ++ case 6: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ case 7: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ case 8: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ case 9: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ default: ++ return false; ++ } ++ op_count++; ++ } ++ return true; ++} ++ ++// Returns true if the given `ReduceOp` matches ++// `zml.Tensor.sum` op implementation. ++bool isSumReduce(stablehlo::ReduceOp& reduce_op) { ++ Block& body_block = reduce_op.getBody().front(); ++ unsigned op_count = 0; ++ // TODO (@cryptodeal): This is a lazy verification; maybe check operands ++ // for each operation in the block. ++ for (auto& op : body_block.getOperations()) { ++ switch (op_count) { ++ case 0: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ case 1: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ } ++ op_count++; ++ } ++ return true; ++} ++ ++// Returns true if the given `ReduceOp` matches ++// `zml.Tensor.max` op implementation. ++bool isMaxReduce(stablehlo::ReduceOp& reduce_op) { ++ Block& body_block = reduce_op.getBody().front(); ++ unsigned op_count = 0; ++ // TODO (@cryptodeal): This is a lazy verification; maybe check operands ++ // for each operation in the block. ++ for (auto& op : body_block.getOperations()) { ++ switch (op_count) { ++ case 0: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ case 1: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ } ++ op_count++; ++ } ++ return true; ++} ++ ++// Returns true if the given `ReduceOp` matches ++// `zml.Tensor.max` op implementation. ++bool isMinReduce(stablehlo::ReduceOp& reduce_op) { ++ Block& body_block = reduce_op.getBody().front(); ++ unsigned op_count = 0; ++ // TODO (@cryptodeal): This is a lazy verification; maybe check operands ++ // for each operation in the block. ++ for (auto& op : body_block.getOperations()) { ++ switch (op_count) { ++ case 0: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ case 1: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ } ++ op_count++; ++ } ++ return true; ++} ++ ++// Returns true if the given `ReduceOp` matches ++// `zml.Tensor.any` op implementation. ++bool isAnyReduce(stablehlo::ReduceOp& reduce_op) { ++ Block& body_block = reduce_op.getBody().front(); ++ unsigned op_count = 0; ++ // TODO (@cryptodeal): This is a lazy verification; maybe check operands ++ // for each operation in the block. ++ for (auto& op : body_block.getOperations()) { ++ switch (op_count) { ++ case 0: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ case 1: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ } ++ op_count++; ++ } ++ return true; ++} ++ ++absl::StatusOr getScatterType(mlir::Block& block) { ++ unsigned op_count = 0; ++ utils::ScatterType scatter_type = utils::ScatterType::Replace; ++ for (const auto& op : block.getOperations()) { ++ switch (op_count) { ++ case 0: ++ if (llvm::dyn_cast(op)) { ++ return scatter_type; ++ } ++ if (llvm::dyn_cast(op)) { ++ scatter_type = utils::ScatterType::Add; ++ } else if (llvm::dyn_cast(op)) { ++ scatter_type = utils::ScatterType::Prod; ++ } else if (llvm::dyn_cast(op)) { ++ scatter_type = utils::ScatterType::Prod; ++ } else if (llvm::dyn_cast(op)) { ++ scatter_type = utils::ScatterType::Max; ++ } else if (llvm::dyn_cast(op)) { ++ scatter_type = utils::ScatterType::Min; ++ } else { ++ return xla::Internal("Unsupported comparator: %s", ToString(block)); ++ } ++ break; ++ case 1: ++ if (llvm::dyn_cast(op)) { ++ return scatter_type; ++ } ++ default: ++ return xla::Internal("Unsupported comparator: %s", ToString(block)); ++ } ++ } ++} ++ ++absl::StatusOr> getSortInfo( ++ mlir::Block& block) { ++ unsigned op_count = 0; ++ mx::ComparatorType comparator = mx::ComparatorType::LessThan; ++ unsigned input_idx = 0; ++ for (const auto& op : block.getOperations()) { ++ switch (op_count) { ++ case 0: { ++ if (auto compare_op = llvm::dyn_cast(op)) { ++ switch (compare_op.getComparisonDirection()) { ++ case ComparisonDirection::GT: { ++ auto lhs = llvm::dyn_cast(compare_op.getLhs()); ++ auto rhs = llvm::dyn_cast(compare_op.getRhs()); ++ if (lhs && rhs) { ++ unsigned lhs_idx = static_cast(lhs.getArgNumber()); ++ unsigned rhs_idx = static_cast(rhs.getArgNumber()); ++ if (rhs_idx - 1 == lhs_idx) { ++ comparator = mx::ComparatorType::GreaterThan; ++ input_idx = lhs_idx == 0 ? 0 : lhs_idx / 2; ++ break; ++ } ++ } ++ return xla::Internal("Unsupported comparator: %s", ++ ToString(block)); ++ } ++ case ComparisonDirection::LT: { ++ auto lhs = llvm::dyn_cast(compare_op.getLhs()); ++ auto rhs = llvm::dyn_cast(compare_op.getRhs()); ++ if (lhs && rhs) { ++ unsigned lhs_idx = static_cast(lhs.getArgNumber()); ++ unsigned rhs_idx = static_cast(rhs.getArgNumber()); ++ if (rhs_idx - 1 == lhs_idx) { ++ comparator = mx::ComparatorType::LessThan; ++ input_idx = lhs_idx == 0 ? 0 : lhs_idx / 2; ++ break; ++ } ++ } ++ return xla::Internal("Unsupported comparator: %s", ++ ToString(block)); ++ } ++ default: ++ return xla::Internal("Unsupported comparator: %s", ++ ToString(block)); ++ } ++ break; ++ } ++ } ++ case 1: ++ if (llvm::dyn_cast(op)) break; ++ default: ++ return xla::Internal("Unsupported comparator: %s", ToString(block)); ++ } ++ op_count++; ++ } ++ return std::make_pair(input_idx, comparator); ++} ++ ++mx::array getOperandArray( ++ const Value& operand, ++ const std::unordered_map>& block_ctx, ++ const std::vector& inputs) { ++ // check if operand is the result of a previous operation ++ if (auto defining_op = operand.getDefiningOp()) { ++ if (auto search = block_ctx.find(defining_op); search != block_ctx.end()) { ++ for (auto i = 0; i < defining_op->getNumResults(); i++) { ++ Value maybe_res = defining_op->getResult(i); ++ if (maybe_res == operand) { ++ return search->second[i]; ++ } ++ } ++ } ++ } else { ++ return inputs[llvm::cast(operand).getArgNumber()]; ++ } ++} ++ ++std::pair getValueInfo(Value v) { ++ auto val_type = mlir::cast(v.getType()); ++ return std::make_pair(*utils::dtype::fromMlirType(val_type.getElementType()), ++ std::vector(val_type.getShape().begin(), ++ val_type.getShape().end())); ++} ++ ++std::pair getTypeInfo(Type v) { ++ auto val_type = mlir::cast(v); ++ return std::make_pair(*utils::dtype::fromMlirType(val_type.getElementType()), ++ std::vector(val_type.getShape().begin(), ++ val_type.getShape().end())); ++} ++ ++CallOpInfo getCallOpInfo(ModuleOp mod, func::FuncOp o) { ++ auto func_type = o.getFunctionType(); ++ ++ std::vector> input_types; ++ for (auto arg : func_type.getInputs()) { ++ input_types.emplace_back(getTypeInfo(arg)); ++ } ++ std::vector> result_types; ++ for (auto result : func_type.getResults()) { ++ result_types.emplace_back(getTypeInfo(result)); ++ } ++ return CallOpInfo(input_types, result_types, o.getName().str()); ++} ++ ++CallOpInfo getCallOpInfo(ModuleOp mod, func::CallOp o) { ++ std::vector> input_types; ++ for (auto arg : o.getOperands()) { ++ input_types.emplace_back(getValueInfo(arg)); ++ } ++ std::vector> result_types; ++ for (auto result : o.getResults()) { ++ result_types.emplace_back(getValueInfo(result)); ++ } ++ auto func = mod.lookupSymbol(o.getCallee()); ++ return CallOpInfo(input_types, result_types, func.getName().str()); ++} ++ ++IotaOpInfo getIotaOpInfo(stablehlo::IotaOp o) { ++ std::pair result_info = getValueInfo(o.getResult()); ++ auto iota_dim = o.getIotaDimension(); ++ std::vector dimensions; ++ for (auto i = 0; i < dimensions.size(); i++) { ++ dimensions.push_back(i != iota_dim ? 1 : result_info.second[i]); ++ } ++ return IotaOpInfo(std::vector>{}, ++ {result_info}, dimensions, result_info.first, iota_dim, ++ result_info.second); ++} ++ ++CbrtOpInfo getCbrtOpInfo(stablehlo::CbrtOp o) { ++ return CbrtOpInfo({getValueInfo(o.getOperand())}, ++ {getValueInfo(o.getResult())}); ++} ++ ++BroadcastInDimOpInfo getBroadcastInDimOpInfo(stablehlo::BroadcastInDimOp o) { ++ auto result_info = getValueInfo(o.getResult()); ++ return BroadcastInDimOpInfo( ++ {getValueInfo(o.getOperand())}, {result_info}, ++ std::vector(o.getBroadcastDimensions().begin(), ++ o.getBroadcastDimensions().end()), ++ result_info.second); ++} ++ ++DotGeneralOpInfo getDotGeneralOpInfo(stablehlo::DotGeneralOp o) { ++ auto dot_dim_nums = o.getDotDimensionNumbers(); ++ std::vector lhs_batch_dims( ++ dot_dim_nums.getLhsBatchingDimensions().begin(), ++ dot_dim_nums.getLhsBatchingDimensions().end()); ++ std::vector rhs_batch_dims( ++ dot_dim_nums.getRhsBatchingDimensions().begin(), ++ dot_dim_nums.getRhsBatchingDimensions().end()); ++ std::vector lhs_contract_dims( ++ dot_dim_nums.getLhsContractingDimensions().begin(), ++ dot_dim_nums.getLhsContractingDimensions().end()); ++ std::vector rhs_contract_dims( ++ dot_dim_nums.getRhsContractingDimensions().begin(), ++ dot_dim_nums.getRhsContractingDimensions().end()); ++ return DotGeneralOpInfo({getValueInfo(o.getLhs()), getValueInfo(o.getRhs())}, ++ {getValueInfo(o.getResult())}, lhs_batch_dims, ++ rhs_batch_dims, lhs_contract_dims, rhs_contract_dims); ++} ++ ++GatherOpInfo getGatherOpInfo(stablehlo::GatherOp o) { ++ std::vector> operand_info; ++ for (auto operand : o.getOperands()) { ++ operand_info.emplace_back(getValueInfo(operand)); ++ } ++ operand_info.emplace_back(getValueInfo(o.getStartIndices())); ++ auto result_info = getValueInfo(o.getResult()); ++ auto dim_nums = o.getDimensionNumbers(); ++ std::vector offset_dims( ++ o.getDimensionNumbers().getOffsetDims().begin(), ++ o.getDimensionNumbers().getOffsetDims().end()); ++ std::vector collapsed_slice_dims( ++ o.getDimensionNumbers().getCollapsedSliceDims().begin(), ++ o.getDimensionNumbers().getCollapsedSliceDims().end()); ++ std::vector operand_batching_dims( ++ o.getDimensionNumbers().getOperandBatchingDims().begin(), ++ o.getDimensionNumbers().getOperandBatchingDims().end()); ++ std::vector start_indices_batching_dims( ++ o.getDimensionNumbers().getStartIndicesBatchingDims().begin(), ++ o.getDimensionNumbers().getStartIndicesBatchingDims().end()); ++ std::vector start_index_map( ++ o.getDimensionNumbers().getStartIndexMap().begin(), ++ o.getDimensionNumbers().getStartIndexMap().end()); ++ auto index_vector_dim = o.getDimensionNumbers().getIndexVectorDim(); ++ std::vector slice_sizes(o.getSliceSizes().begin(), ++ o.getSliceSizes().end()); ++ return GatherOpInfo(operand_info, {getValueInfo(o.getResult())}, ++ collapsed_slice_dims, index_vector_dim, offset_dims, ++ operand_batching_dims, result_info.second, slice_sizes, ++ start_index_map, start_indices_batching_dims); ++} ++ ++ScatterOpInfo getScatterOpInfo(stablehlo::ScatterOp o) { ++ std::vector> inputs_info; ++ for (auto input : o.getInputs()) { ++ inputs_info.emplace_back(getValueInfo(input)); ++ } ++ for (auto update : o.getUpdates()) { ++ inputs_info.emplace_back(getValueInfo(update)); ++ } ++ inputs_info.emplace_back(getValueInfo(o.getScatterIndices())); ++ std::vector>> results_info; ++ for (auto result : o.getResults()) { ++ results_info.emplace_back(getValueInfo(result)); ++ } ++ // Get scatter dimension numbers ++ auto scatter_dim_nums = o.getScatterDimensionNumbers(); ++ std::vector update_window_dims( ++ scatter_dim_nums.getUpdateWindowDims().begin(), ++ scatter_dim_nums.getUpdateWindowDims().end()); ++ std::vector inserted_window_dims( ++ scatter_dim_nums.getInsertedWindowDims().begin(), ++ scatter_dim_nums.getInsertedWindowDims().end()); ++ std::vector input_batching_dims( ++ scatter_dim_nums.getInputBatchingDims().begin(), ++ scatter_dim_nums.getInputBatchingDims().end()); ++ ++ std::vector scatter_indices_batching_dims( ++ scatter_dim_nums.getScatterIndicesBatchingDims().begin(), ++ scatter_dim_nums.getScatterIndicesBatchingDims().end()); ++ ++ std::vector scatter_dims_to_operand_dims( ++ scatter_dim_nums.getScatterDimsToOperandDims().begin(), ++ scatter_dim_nums.getScatterDimsToOperandDims().end()); ++ ++ auto index_vector_dim = scatter_dim_nums.getIndexVectorDim(); ++ // Get update computation ++ mlir::Block& update_computation = o.getUpdateComputation().front(); ++ ++ // TODO(@cryptodeal): need to add a check in `Compile` ++ utils::ScatterType scatter_type = *getScatterType(update_computation); ++ return ScatterOpInfo( ++ inputs_info, results_info, index_vector_dim, input_batching_dims, ++ inserted_window_dims, scatter_dims_to_operand_dims, ++ scatter_indices_batching_dims, scatter_type, update_window_dims); ++} ++ ++PadOpInfo getPadOpInfo(stablehlo::PadOp o) { ++ auto result_info = getValueInfo(o.getResult()); ++ std::vector edge_pad_low(o.getEdgePaddingLow().begin(), ++ o.getEdgePaddingLow().end()); ++ std::vector interior_pad(o.getInteriorPadding().begin(), ++ o.getInteriorPadding().end()); ++ return PadOpInfo( ++ {getValueInfo(o.getOperand()), getValueInfo(o.getPaddingValue())}, ++ {result_info}, result_info.second, edge_pad_low, interior_pad); ++} ++ ++void compileCallOp(OpLookup& op_lookup, ModuleOp mod, func::FuncOp o) { ++ CallOpInfo call_op_info = getCallOpInfo(mod, o); ++ if (auto search = op_lookup.call.find(call_op_info); ++ search == op_lookup.call.end()) { ++ // std::cout << "Compiling CallOp: " << o.getName().str().c_str() << ++ // std::endl; ++ auto& block = o.front(); ++ auto call_op_func = [&op_lookup, &block, mod, ++ call_op_info](const std::vector& inputs) { ++ std::unordered_map> block_ctx; ++ std::vector res; ++ for (Operation& op : block.getOperations()) { ++ res = ++ llvm::TypeSwitch>(&op) ++ .Case([&block_ctx, ++ &inputs](auto o) { ++ std::vector res; ++ for (Value val : o.getOperands()) { ++ res.emplace_back(getOperandArray(val, block_ctx, inputs)); ++ } ++ return res; ++ }) ++ .Case([&block_ctx, &inputs, mod, ++ &op_lookup](auto o) { ++ auto call_op_info = getCallOpInfo(mod, o); ++ std::vector operands; ++ for (Value val : o.getOperands()) { ++ operands.emplace_back( ++ getOperandArray(val, block_ctx, inputs)); ++ } ++ return op_lookup.call.find(call_op_info)->second(operands); ++ }) ++ // Handle StableHLO nullary ops ++ .Case([](auto o) { ++ return std::vector{ ++ *utils::array::fromDenseElementsAttr( ++ mlir::cast(o.getValue()))}; ++ }) ++ .Case([&op_lookup](auto o) { ++ return op_lookup.iota.find(getIotaOpInfo(o))->second({}); ++ }) ++ // .Case([](auto o) {}) ++ /* ++ .Case([](auto o) {}) ++ Deprecated see: ++ https://github.com/openxla/stablehlo/issues/2340 ++ https://github.com/openxla/stablehlo/pull/2283 ++ */ ++ // Handle StableHLO unary elementwise op ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::abs( ++ getOperandArray(o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs, ++ &op_lookup](auto o) { ++ return op_lookup.cbrt.find(getCbrtOpInfo(o)) ++ ->second( ++ {getOperandArray(o.getOperand(), block_ctx, inputs)}); ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::ceil( ++ getOperandArray(o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::astype( ++ getOperandArray(o.getOperand(), block_ctx, inputs), ++ *utils::dtype::fromMlirType( ++ o.getResult().getType().getElementType()))}; ++ }) ++ // .Case([](auto o) {}) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::cos( ++ getOperandArray(o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::exp( ++ getOperandArray(o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::expm1( ++ getOperandArray(o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::floor( ++ getOperandArray(o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::imag( ++ getOperandArray(o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::isfinite( ++ getOperandArray(o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::log( ++ getOperandArray(o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::log1p( ++ getOperandArray(o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::sigmoid( ++ getOperandArray(o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::logical_not( ++ getOperandArray(o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::negative( ++ getOperandArray(o.getOperand(), block_ctx, inputs))}; ++ }) ++ // .Case([](auto o) {}) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::real( ++ getOperandArray(o.getOperand(), block_ctx, inputs))}; ++ }) ++ // TODO(@cryptodeal): `stablehlo::RoundOp` does not match with ++ // the mlx metal implementation ++ // .Case([](auto o) {}) ++ .Case( ++ [&block_ctx, &inputs](auto o) { ++ return std::vector{mx::round( ++ getOperandArray(o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::rsqrt( ++ getOperandArray(o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::sign( ++ getOperandArray(o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::sin( ++ getOperandArray(o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::sqrt( ++ getOperandArray(o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::tan( ++ getOperandArray(o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::tanh( ++ getOperandArray(o.getOperand(), block_ctx, inputs))}; ++ }) ++ // Handle StableHLO binary elementwise ops ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{ ++ mx::add(getOperandArray(o.getLhs(), block_ctx, inputs), ++ getOperandArray(o.getRhs(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::arctan2( ++ getOperandArray(o.getLhs(), block_ctx, inputs), ++ getOperandArray(o.getRhs(), block_ctx, inputs))}; ++ }) ++ // TODO(@cryptodeal): implement complex op ++ // .Case([&block_ctx](auto ++ // op) {}) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::divide( ++ getOperandArray(o.getLhs(), block_ctx, inputs), ++ getOperandArray(o.getRhs(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::maximum( ++ getOperandArray(o.getLhs(), block_ctx, inputs), ++ getOperandArray(o.getRhs(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::minimum( ++ getOperandArray(o.getLhs(), block_ctx, inputs), ++ getOperandArray(o.getRhs(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::multiply( ++ getOperandArray(o.getLhs(), block_ctx, inputs), ++ getOperandArray(o.getRhs(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::power( ++ getOperandArray(o.getLhs(), block_ctx, inputs), ++ getOperandArray(o.getRhs(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::remainder( ++ getOperandArray(o.getLhs(), block_ctx, inputs), ++ getOperandArray(o.getRhs(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::left_shift( ++ getOperandArray(o.getLhs(), block_ctx, inputs), ++ getOperandArray(o.getRhs(), block_ctx, inputs))}; ++ }) ++ .Case( ++ [&block_ctx, &inputs](auto o) { ++ /** ++ * Per Metal Spec: ++ * For the right-shift operator, if E1 has an unsigned ++ * type or if E1 has a signed type and a nonnegative ++ * value, the vacated bits are filled with zeros. If E1 ++ * has a signed type and a negative value, the vacated ++ * bits are filled with ones. ++ */ ++ auto lhs = getOperandArray(o.getLhs(), block_ctx, inputs); ++ auto rhs = getOperandArray(o.getRhs(), block_ctx, inputs); ++ auto target_dtype = rhs.dtype(); ++ switch (lhs.dtype().size()) { ++ case 1: ++ target_dtype = mx::int8; ++ break; ++ case 2: ++ target_dtype = mx::int16; ++ break; ++ case 4: ++ target_dtype = mx::int32; ++ break; ++ case 8: ++ target_dtype = mx::int64; ++ break; ++ } ++ return std::vector{mx::view( ++ mx::right_shift(mx::view(lhs, target_dtype), ++ mx::astype(rhs, target_dtype)), ++ lhs.dtype())}; ++ }) ++ .Case( ++ [&block_ctx, &inputs](auto o) { ++ // Ensures that we bitcast to `uint` type before ++ // performing the right shift. Should ensure that vacated ++ // bits are zero populated. ++ auto lhs = getOperandArray(o.getLhs(), block_ctx, inputs); ++ auto rhs = getOperandArray(o.getRhs(), block_ctx, inputs); ++ auto target_dtype = rhs.dtype(); ++ switch (lhs.dtype().size()) { ++ case 1: ++ target_dtype = mx::uint8; ++ break; ++ case 2: ++ target_dtype = mx::uint16; ++ break; ++ case 4: ++ target_dtype = mx::uint32; ++ break; ++ case 8: ++ target_dtype = mx::uint64; ++ break; ++ } ++ return std::vector{mx::view( ++ mx::right_shift(mx::view(lhs, target_dtype), ++ mx::astype(rhs, target_dtype)), ++ lhs.dtype())}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::subtract( ++ getOperandArray(o.getLhs(), block_ctx, inputs), ++ getOperandArray(o.getRhs(), block_ctx, inputs))}; ++ }) ++ // Handle StableHLO binary logical elementwise ops ++ .Case([&block_ctx, &inputs](auto o) { ++ auto lhs = getOperandArray(o.getLhs(), block_ctx, inputs); ++ auto rhs = getOperandArray(o.getRhs(), block_ctx, inputs); ++ switch (mx::kindof(lhs.dtype())) { ++ case mx::Dtype::Kind::b: ++ return std::vector{mx::logical_and(lhs, rhs)}; ++ default: ++ return std::vector{mx::bitwise_and(lhs, rhs)}; ++ } ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ auto lhs = getOperandArray(o.getLhs(), block_ctx, inputs); ++ auto rhs = getOperandArray(o.getRhs(), block_ctx, inputs); ++ switch (mx::kindof(lhs.dtype())) { ++ case mx::Dtype::Kind::b: ++ return std::vector{mx::logical_or(lhs, rhs)}; ++ default: ++ return std::vector{mx::bitwise_or(lhs, rhs)}; ++ } ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::bitwise_xor( ++ getOperandArray(o.getLhs(), block_ctx, inputs), ++ getOperandArray(o.getRhs(), block_ctx, inputs))}; ++ }) ++ // Handle StableHLO communication ops ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ ++ // Handle StableHLO parallelism related ops ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ ++ // Handle StableHLO control flow ops ++ // .Case([](auto o) {}) ++ // TODO(@cryptodeal): need custom kernels in order to ++ // remove the need to call `mx::eval` for the following ops ++ // .Case([](auto o)) ++ // .Case([](auto o)) ++ // .Case([](auto o)) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ .Case([&block_ctx, &inputs](auto o) { ++ auto operand = ++ getOperandArray(o.getOperand(0), block_ctx, inputs); ++ if (isArgMaxReduce(o)) { ++ auto axis = static_cast(o.getDimensions()[0]); ++ auto indices = mx::argmax(operand, axis); ++ ++ mx::Dtype result_type = *utils::dtype::fromMlirType( ++ mlir::cast(o.getResults()[1].getType()) ++ .getElementType()); ++ return std::vector{ ++ mx::take(operand, indices, axis), ++ mx::astype(indices, result_type)}; ++ } ++ if (isSumReduce(o)) { ++ return std::vector{mx::sum( ++ operand, static_cast(o.getDimensions()[0]))}; ++ } ++ if (isMaxReduce(o)) { ++ return std::vector{mx::max( ++ operand, static_cast(o.getDimensions()[0]))}; ++ } ++ if (isMinReduce(o)) { ++ return std::vector{mx::min( ++ operand, static_cast(o.getDimensions()[0]))}; ++ } ++ // TODO(@cryptodeal): further cleanup module validation ++ // done via call to `Compile` to check for valid reduce types. ++ ++ // For now, we assume `mx::any` if none of the above ++ ++ // if (isAnyReduce(o)) { ++ return std::vector{ ++ mx::any(operand, static_cast(o.getDimensions()[0]))}; ++ //} ++ }) ++ // Handle StableHLO tuple ops ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ .Case([&block_ctx, &inputs](auto o) { ++ auto lhs = getOperandArray(o.getLhs(), block_ctx, inputs); ++ auto rhs = getOperandArray(o.getRhs(), block_ctx, inputs); ++ switch (o.getComparisonDirection()) { ++ case ComparisonDirection::NE: ++ return std::vector{mx::not_equal(lhs, rhs)}; ++ case ComparisonDirection::GE: ++ return std::vector{ ++ mx::greater_equal(lhs, rhs)}; ++ case ComparisonDirection::GT: ++ return std::vector{mx::greater(lhs, rhs)}; ++ case ComparisonDirection::LE: ++ return std::vector{mx::less_equal(lhs, rhs)}; ++ case ComparisonDirection::LT: ++ return std::vector{mx::less(lhs, rhs)}; ++ case ComparisonDirection::EQ: ++ return std::vector{mx::equal(lhs, rhs)}; ++ } ++ }) ++ // Handle StableHLO Slice ops ++ .Case([&block_ctx, &inputs](auto o) { ++ std::vector start_indices( ++ o.getStartIndices().begin(), o.getStartIndices().end()); ++ std::vector limit_indices( ++ o.getLimitIndices().begin(), o.getLimitIndices().end()); ++ std::vector strides(o.getStrides().begin(), ++ o.getStrides().end()); ++ return std::vector{mx::slice( ++ getOperandArray(o.getOperand(), block_ctx, inputs), ++ start_indices, limit_indices, strides)}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ auto operand = ++ getOperandArray(o.getOperand(), block_ctx, inputs); ++ std::vector indices; ++ for (auto val : o.getStartIndices()) { ++ indices.emplace_back( ++ getOperandArray(val, block_ctx, inputs)); ++ } ++ auto start_indices = mx::concatenate(indices); ++ std::vector axes(operand.ndim()); ++ std::iota(axes.begin(), axes.end(), 0); ++ std::vector slice_sizes(o.getSliceSizes().begin(), ++ o.getSliceSizes().end()); ++ return std::vector{ ++ mx::slice(operand, start_indices, axes, slice_sizes)}; ++ }) ++ .Case( ++ [&block_ctx, &inputs](auto o) { ++ auto operand = ++ getOperandArray(o.getOperand(), block_ctx, inputs); ++ auto update = ++ getOperandArray(o.getUpdate(), block_ctx, inputs); ++ std::vector indices; ++ for (auto val : o.getStartIndices()) { ++ indices.emplace_back( ++ getOperandArray(val, block_ctx, inputs)); ++ } ++ ++ auto start_indices = mx::concatenate(indices); ++ std::vector axes(operand.ndim()); ++ std::iota(axes.begin(), axes.end(), 0); ++ return std::vector{mx::slice_update( ++ operand, update, start_indices, axes)}; ++ }) ++ // Handle StableHLO Other ops ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ .Case( ++ [&block_ctx, &inputs](auto o) { ++ auto result_type = *utils::dtype::fromMlirType( ++ mlir::cast(o.getResult().getType()) ++ .getElementType()); ++ return std::vector{mx::view( ++ getOperandArray(o.getOperand(), block_ctx, inputs), ++ result_type)}; ++ }) ++ /* ++ .Case([](auto o) {}) ++ Deprecated see: ++ https://github.com/openxla/stablehlo/issues/2340 ++ https://github.com/openxla/stablehlo/pull/2283 ++ */ ++ .Case([&block_ctx, &inputs, ++ &op_lookup](auto o) { ++ return op_lookup.broadcast_in_dim ++ .find(getBroadcastInDimOpInfo(o)) ++ ->second( ++ {getOperandArray(o.getOperand(), block_ctx, inputs)}); ++ }) ++ // .Case([](auto o) {}) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::linalg::cholesky( ++ getOperandArray(o.getOperand(), block_ctx, inputs), ++ o.getLower())}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::clip( ++ getOperandArray(o.getOperand(), block_ctx, inputs), ++ getOperandArray(o.getMin(), block_ctx, inputs), ++ getOperandArray(o.getMax(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ std::vector inputs; ++ for (auto val : o.getOperands()) { ++ inputs.emplace_back( ++ getOperandArray(val, block_ctx, inputs)); ++ } ++ return std::vector{ ++ mx::concatenate(inputs, o.getDimension())}; ++ }) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ /* ++ .Case([](auto o) {}) ++ Deprecated see: ++ https://github.com/openxla/stablehlo/issues/2340 ++ https://github.com/openxla/stablehlo/pull/2283 ++ */ ++ .Case( ++ [&block_ctx, &inputs, &op_lookup](auto o) { ++ return op_lookup.dot_general.find(getDotGeneralOpInfo(o)) ++ ->second( ++ {getOperandArray(o.getLhs(), block_ctx, inputs), ++ getOperandArray(o.getRhs(), block_ctx, inputs)}); ++ }) ++ /* ++ .Case([](auto op) {}) ++ .Case([](auto op) {}) ++ Deprecated see: ++ https://github.com/openxla/stablehlo/issues/2340 ++ https://github.com/openxla/stablehlo/pull/2283 ++ */ ++ // .Case([](auto op) {}) ++ // TODO(@cryptodeal): definitely room for optimization here ++ .Case([&block_ctx, &inputs, ++ &op_lookup](auto o) { ++ return op_lookup.gather.find(getGatherOpInfo(o)) ++ ->second( ++ {getOperandArray(o.getOperand(), block_ctx, inputs), ++ getOperandArray(o.getStartIndices(), block_ctx, ++ inputs)}); ++ }) ++ .Case( ++ [&block_ctx, &inputs](auto o) { ++ return std::vector{mx::array( ++ (getOperandArray(o.getOperand(), block_ctx, inputs)) ++ .shape(o.getDimension()))}; ++ }) ++ // .Case([](auto o) {}) ++ .Case([&block_ctx, &inputs](auto o) { ++ auto result_type = ++ mlir::cast(o.getResult().getType()); ++ return std::vector{mx::reshape( ++ getOperandArray(o.getOperand(), block_ctx, inputs), ++ std::vector(result_type.getShape().begin(), ++ result_type.getShape().end()))}; ++ }) ++ // .Case([](auto o) {}) ++ .Case( ++ [&block_ctx, &inputs, &op_lookup](auto o) { ++ std::vector operands; ++ for (auto val : o.getInputs()) { ++ operands.emplace_back( ++ getOperandArray(val, block_ctx, inputs)); ++ } ++ // Add updates to input list ++ for (auto val : o.getUpdates()) { ++ operands.emplace_back( ++ getOperandArray(val, block_ctx, inputs)); ++ } ++ // Add scatter indices to input list ++ operands.emplace_back(getOperandArray( ++ o.getScatterIndices(), block_ctx, inputs)); ++ return op_lookup.scatter.find(getScatterOpInfo(o)) ++ ->second(operands); ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::where( ++ getOperandArray(o.getOperand(0), block_ctx, inputs), ++ getOperandArray(o.getOperand(1), block_ctx, inputs), ++ getOperandArray(o.getOperand(2), block_ctx, inputs))}; ++ }) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ .Case([&block_ctx, &inputs](auto o) { ++ std::vector tmp_inputs; ++ for (auto val : o.getInputs()) { ++ tmp_inputs.emplace_back( ++ getOperandArray(val, block_ctx, inputs)); ++ } ++ int dim = static_cast(o.getDimension()); ++ // Get update computation ++ mlir::Block& comparator = o.getComparator().front(); ++ auto [index, comparator_type] = *getSortInfo(comparator); ++ auto indices = ++ mx::argsort(tmp_inputs[index], dim, comparator_type); ++ std::vector res; ++ mx::Shape slice_sizes(tmp_inputs[0].ndim(), 1); ++ for (auto in : tmp_inputs) { ++ res.push_back(mx::gather(in, indices, dim, slice_sizes)); ++ } ++ return res; ++ }) ++ // .Case([](auto o) {}) ++ // TODO (@cryptodeal): lots of room for optimization here ++ .Case([&block_ctx, &inputs, ++ &op_lookup](auto o) { ++ return op_lookup.pad.find(getPadOpInfo(o)) ++ ->second( ++ {getOperandArray(o.getOperand(), block_ctx, inputs), ++ getOperandArray(o.getPaddingValue(), block_ctx, ++ inputs)}); ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ std::vector axes(o.getPermutation().begin(), ++ o.getPermutation().end()); ++ return std::vector{mx::transpose( ++ getOperandArray(o.getOperand(), block_ctx, inputs), ++ axes)}; ++ }) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // Need to modify such that `rng.state` is held outside of scope ++ // of `mx::compile` ++ // TODO(@cryptodeal): re-enable once we've handled ensuring ++ // state is handled correctly ++ // .Case([&block_ctx](auto o) { ++ // auto rng_func = [&o](const std::vector& inputs) ++ // { ++ // auto a = inputs[0]; ++ // auto b = inputs[1]; ++ // auto result_type = ++ // mlir::cast(o.getResult().getType()); ++ // std::vector ++ // shape(result_type.getShape().begin(), ++ // result_type.getShape().end()); ++ ++ // switch (o.getRngDistribution()) { ++ // case RngDistribution::UNIFORM: ++ // return std::vector{ ++ // mx::random::uniform(a, b, shape, a.dtype())}; ++ // default: ++ // return std::vector{ ++ // mx::random::normal(shape, a.dtype(), a, b)}; ++ // } ++ // }; ++ // return mx::compile(rng_func)( ++ // {getOperandArray(o.getA(), block_ctx), ++ // getOperandArray(o.getB(), block_ctx)}); ++ // }) ++ // .Case([&block_ctx](auto o) { ++ // auto rng_bit_gen_func = [&o](const std::vector& ++ // inputs) { ++ ++ // }; ++ // return mx::compile(rng_bit_gen_func)( ++ // {getOperandArray(o.getA(), block_ctx), ++ // getOperandArray(o.getB(), block_ctx)}); ++ // }) ++ // Handle StableHLO Quantize ops ++ // .Case([](auto op) {}) ++ // .Case([](auto op) {}) ++ // .Case([](auto op) {}) ++ /* ++ .Case([](auto o) {}) ++ Deprecated see: ++ https://github.com/openxla/stablehlo/issues/2340 ++ https://github.com/openxla/stablehlo/pull/2283 ++ */ ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ ++ .Default([&op](auto o) { ++ // Shouldn't be possible to reach this point, return ++ // empty vector. ++ return std::vector{}; ++ }); ++ block_ctx.emplace(&op, res); ++ } ++ return res; ++ }; ++ op_lookup.call.emplace(call_op_info, mx::compile(call_op_func)); ++ } ++} ++ ++void compileIotaOp(OpLookup& op_lookup, stablehlo::IotaOp o) { ++ IotaOpInfo iota_op_info = getIotaOpInfo(o); ++ auto dimensions = std::get<0>(iota_op_info.captures); ++ auto dtype = std::get<1>(iota_op_info.captures); ++ auto iota_dim = std::get<2>(iota_op_info.captures); ++ auto shape = std::get<3>(iota_op_info.captures); ++ if (auto search = op_lookup.iota.find(iota_op_info); ++ search == op_lookup.iota.end()) { ++ // std::cout << "Compiling IotaOp" << std::endl; ++ auto iota_func = [dimensions, dtype, iota_dim, ++ shape](const std::vector& inputs) { ++ auto init_arange = ++ mx::arange(static_cast(shape[iota_dim]), dtype); ++ if (dimensions.size()) { ++ init_arange = mx::reshape(init_arange, dimensions); ++ } ++ return std::vector{mx::broadcast_to(init_arange, shape)}; ++ }; ++ op_lookup.iota.emplace(iota_op_info, mx::compile(iota_func)); ++ } ++} ++ ++void compileCbrtOp(OpLookup& op_lookup, stablehlo::CbrtOp o) { ++ CbrtOpInfo cbrt_op_info = getCbrtOpInfo(o); ++ if (auto search = op_lookup.cbrt.find(cbrt_op_info); ++ search == op_lookup.cbrt.end()) { ++ // std::cout << "Compiling CbrtOp" << std::endl; ++ ++ auto cbrt_func = [](const std::vector& inputs) { ++ return std::vector{mx::power( ++ inputs[0], ++ mx::full(inputs[0].shape(), 1 / 3, inputs[0].dtype()))}; ++ }; ++ op_lookup.cbrt.emplace(cbrt_op_info, mx::compile(cbrt_func)); ++ } ++} ++ ++void compileBroadcastInDimOp(OpLookup& op_lookup, ++ stablehlo::BroadcastInDimOp o) { ++ BroadcastInDimOpInfo broadcast_in_dim_op_info = getBroadcastInDimOpInfo(o); ++ auto broadcast_dims = std::get<0>(broadcast_in_dim_op_info.captures); ++ auto target_shape = std::get<1>(broadcast_in_dim_op_info.captures); ++ if (auto search = op_lookup.broadcast_in_dim.find(broadcast_in_dim_op_info); ++ search == op_lookup.broadcast_in_dim.end()) { ++ // std::cout << "Compiling BroadcastInDimOp" << std::endl; ++ ++ auto broadcast_in_dim_func = [broadcast_dims, target_shape]( ++ const std::vector& inputs) { ++ auto operand = inputs[0]; ++ ++ // potentially reshape to pad with ones, which allows ++ // broadcasting ++ auto min_rank = std::min(operand.ndim(), target_shape.size()); ++ if (min_rank == operand.ndim()) { ++ std::vector padded_shape = operand.shape(); ++ for (auto i = 0; i < target_shape.size(); i++) { ++ if (auto search = ++ std::find(broadcast_dims.begin(), broadcast_dims.end(), i); ++ search == broadcast_dims.end()) { ++ padded_shape.insert(padded_shape.begin() + i, 1); ++ } ++ } ++ operand = mx::reshape(operand, padded_shape); ++ } ++ return std::vector{mx::broadcast_to(operand, target_shape)}; ++ }; ++ op_lookup.broadcast_in_dim.emplace(broadcast_in_dim_op_info, ++ mx::compile(broadcast_in_dim_func)); ++ } ++} ++ ++void compileDotGeneralOp(OpLookup& op_lookup, stablehlo::DotGeneralOp o) { ++ DotGeneralOpInfo dot_general_op_info = getDotGeneralOpInfo(o); ++ auto lhs_batch_dims = std::get<0>(dot_general_op_info.captures); ++ auto rhs_batch_dims = std::get<1>(dot_general_op_info.captures); ++ auto lhs_contract_dims = std::get<2>(dot_general_op_info.captures); ++ auto rhs_contract_dims = std::get<3>(dot_general_op_info.captures); ++ if (auto search = op_lookup.dot_general.find(dot_general_op_info); ++ search == op_lookup.dot_general.end()) { ++ // std::cout << "Compiling DotGeneralOp" << std::endl; ++ ++ auto dot_general_func = [lhs_batch_dims, rhs_batch_dims, lhs_contract_dims, ++ rhs_contract_dims]( ++ const std::vector& inputs) { ++ auto lhs = inputs[0]; ++ auto rhs = inputs[1]; ++ ++ int dim_count = 0; ++ auto getDimChar = [&dim_count]() -> char { return 'a' + dim_count++; }; ++ ++ std::unordered_map lhs_batch_map; ++ std::unordered_map rhs_batch_map; ++ std::unordered_map lhs_contract_map; ++ std::unordered_map rhs_contract_map; ++ ++ std::string res_subscript; ++ for (auto i = 0; i < lhs_batch_dims.size(); ++i) { ++ auto dim_char = getDimChar(); ++ res_subscript = res_subscript + dim_char; ++ lhs_batch_map.emplace(lhs_batch_dims[i], dim_char); ++ rhs_batch_map.emplace(rhs_batch_dims[i], dim_char); ++ } ++ ++ for (auto i = 0; i < lhs_contract_dims.size(); ++i) { ++ auto dim_char = getDimChar(); ++ lhs_contract_map.emplace(lhs_contract_dims[i], dim_char); ++ rhs_contract_map.emplace(rhs_contract_dims[i], dim_char); ++ } ++ ++ std::string lhs_subscript; ++ for (auto i = 0; i < lhs.ndim(); ++i) { ++ if (auto match = lhs_batch_map.find(i); match != lhs_batch_map.end()) { ++ lhs_subscript = lhs_subscript + match->second; ++ } else if (auto match = lhs_contract_map.find(i); ++ match != lhs_contract_map.end()) { ++ lhs_subscript = lhs_subscript + match->second; ++ } else { ++ auto dim_char = getDimChar(); ++ res_subscript = res_subscript + dim_char; ++ lhs_subscript = lhs_subscript + dim_char; ++ } ++ } ++ ++ std::string rhs_subscript; ++ for (auto i = 0; i < rhs.ndim(); ++i) { ++ if (auto match = rhs_batch_map.find(i); match != rhs_batch_map.end()) { ++ rhs_subscript = rhs_subscript + match->second; ++ } else if (auto match = rhs_contract_map.find(i); ++ match != rhs_contract_map.end()) { ++ rhs_subscript = rhs_subscript + match->second; ++ } else { ++ auto dim_char = getDimChar(); ++ res_subscript = res_subscript + dim_char; ++ rhs_subscript = rhs_subscript + dim_char; ++ } ++ } ++ ++ return std::vector{mx::einsum( ++ lhs_subscript + "," + rhs_subscript + "->" + res_subscript, inputs)}; ++ }; ++ op_lookup.dot_general.emplace(dot_general_op_info, ++ mx::compile(dot_general_func)); ++ } ++} ++ ++void compileGatherOp(OpLookup& op_lookup, stablehlo::GatherOp o) { ++ GatherOpInfo gather_op_info = getGatherOpInfo(o); ++ auto collapsed_slice_dims = std::get<0>(gather_op_info.captures); ++ auto index_vector_dim = std::get<1>(gather_op_info.captures); ++ auto offset_dims = std::get<2>(gather_op_info.captures); ++ auto operand_batching_dims = std::get<3>(gather_op_info.captures); ++ auto result_shape = std::get<4>(gather_op_info.captures); ++ auto slice_sizes = std::get<5>(gather_op_info.captures); ++ auto start_index_map = std::get<6>(gather_op_info.captures); ++ auto start_indices_batching_dims = std::get<7>(gather_op_info.captures); ++ ++ if (auto search = op_lookup.gather.find(gather_op_info); ++ search == op_lookup.gather.end()) { ++ // std::cout << "Compiling GatherOp" << std::endl; ++ auto gather_func = [collapsed_slice_dims, index_vector_dim, offset_dims, ++ operand_batching_dims, result_shape, slice_sizes, ++ start_index_map, start_indices_batching_dims]( ++ const std::vector& inputs) { ++ auto operand = inputs[0]; ++ auto start_indices = inputs[1]; ++ ++ // Calculate batch dims ++ std::vector batch_dims_vec; ++ for (int64_t i = 0; i < result_shape.size(); i++) { ++ if (std::find(offset_dims.begin(), offset_dims.end(), i) == ++ offset_dims.end()) { ++ batch_dims_vec.emplace_back(static_cast(i)); ++ } ++ } ++ mx::array batch_dims = mx::array( ++ batch_dims_vec.data(), {static_cast(batch_dims_vec.size())}); ++ mx::array start_idx_map = ++ mx::array(start_index_map.data(), ++ {static_cast(start_index_map.size())}); ++ mx::array op_batching_dims = ++ mx::array(operand_batching_dims.data(), ++ {static_cast(operand_batching_dims.size())}); ++ ++ // Calculate result indices ++ std::vector meshgrid_arrays; ++ for (const auto& dim : result_shape) { ++ meshgrid_arrays.push_back(mx::arange(dim)); ++ } ++ mx::array result_indices = ++ mx::reshape(mx::stack(mx::meshgrid(meshgrid_arrays, false, "ij"), -1), ++ {-1, static_cast(result_shape.size())}); ++ ++ // Initialize operand indices with zeros ++ mx::array operand_indices = mx::zeros( ++ {result_indices.shape(0), static_cast(operand.ndim())}, ++ mx::int32); ++ ++ // Slice batch indices ++ mx::array batch_indices = mx::take(result_indices, batch_dims, 1); ++ if (batch_indices.size()) { ++ // Calculate start indices ++ std::vector tmp_indices = ++ mx::split(batch_indices, batch_indices.shape(1), 1); ++ for (auto& idx : tmp_indices) { ++ idx = mx::flatten(idx); ++ } ++ ++ std::vector tmp_axes; ++ for (auto i = 0; i < start_indices.ndim(); ++i) { ++ if (i == index_vector_dim) continue; ++ tmp_axes.push_back(i); ++ } ++ std::vector tmp_slice_sizes(start_indices.ndim(), 1); ++ if (index_vector_dim < tmp_slice_sizes.size()) { ++ tmp_slice_sizes[index_vector_dim] = ++ start_indices.shape(index_vector_dim); ++ } ++ ++ mx::array batch_start_indices = mx::reshape( ++ mx::gather(start_indices, tmp_indices, tmp_axes, tmp_slice_sizes), ++ {result_indices.shape(0), -1}); ++ ++ // Seed with full start indices ++ std::vector max; ++ for (auto d_start = 0; d_start < start_index_map.size(); ++d_start) { ++ int d_operand = start_index_map[d_start]; ++ max.push_back(operand.shape(d_operand) - slice_sizes[d_operand]); ++ } ++ ++ operand_indices = mx::put_along_axis( ++ operand_indices, ++ mx::broadcast_to(start_idx_map, ++ {operand_indices.shape(0), ++ static_cast(start_index_map.size())}), ++ mx::clip( ++ mx::take(batch_start_indices, ++ mx::arange(static_cast(start_index_map.size())), ++ 1), ++ mx::zeros(batch_start_indices.shape(), mx::int32), ++ mx::array(max.data(), {static_cast(max.size())})), ++ 1); ++ } ++ ++ // Calculate full batching indices ++ mx::array i_batching = ++ mx::arange(static_cast(operand_batching_dims.size())); ++ mx::array d_operands = mx::take(op_batching_dims, i_batching); ++ mx::array d_starts = mx::take( ++ mx::array(start_indices_batching_dims.data(), ++ {static_cast(start_indices_batching_dims.size())}), ++ i_batching); ++ auto tmp_batch_start_indices = mx::subtract( ++ d_starts, mx::where(mx::less(d_starts, mx::full(d_starts.shape(), ++ index_vector_dim)), ++ mx::zeros(d_starts.shape(), mx::int32), ++ mx::ones(d_starts.shape(), mx::int32))); ++ ++ // Add batching indices ++ operand_indices = mx::scatter_add_axis( ++ operand_indices, ++ mx::broadcast_to(d_operands, ++ {operand_indices.shape(0), d_operands.shape(0)}), ++ mx::take(batch_indices, tmp_batch_start_indices, 1), 1); ++ ++ // Calculate offset indices ++ mx::array offset_indices = ++ mx::take(result_indices, ++ mx::array(offset_dims.data(), ++ {static_cast(offset_dims.size())}), ++ 1); ++ ++ std::vector result_offset_idx; ++ for (unsigned i = 0; i < operand_indices.shape(1); i++) { ++ if (std::find(operand_batching_dims.begin(), ++ operand_batching_dims.end(), ++ static_cast(i)) != operand_batching_dims.end() || ++ std::find(collapsed_slice_dims.begin(), collapsed_slice_dims.end(), ++ static_cast(i)) != collapsed_slice_dims.end()) { ++ continue; ++ } ++ result_offset_idx.push_back(i); ++ } ++ ++ // Add offset indices ++ operand_indices = mx::scatter_add_axis( ++ operand_indices, ++ mx::broadcast_to( ++ mx::array(result_offset_idx.data(), ++ {static_cast(result_offset_idx.size())}), ++ {operand_indices.shape(0), ++ static_cast(result_offset_idx.size())}), ++ offset_indices, 1); ++ ++ std::vector gather_axes(operand.ndim()); ++ std::iota(gather_axes.begin(), gather_axes.end(), 0); ++ return std::vector{mx::reshape( ++ mx::gather(operand, ++ mx::split(operand_indices, operand_indices.shape(1), 1), ++ gather_axes, std::vector(operand.ndim(), 1)), ++ result_shape)}; ++ }; ++ ++ op_lookup.gather.emplace(gather_op_info, mx::compile(gather_func)); ++ } ++} ++ ++void compileScatterOp(OpLookup& op_lookup, stablehlo::ScatterOp o) { ++ ScatterOpInfo scatter_op_info = getScatterOpInfo(o); ++ auto index_vector_dim = ++ static_cast(std::get<0>(scatter_op_info.captures)); ++ auto input_batching_dims = std::get<1>(scatter_op_info.captures); ++ auto inserted_window_dims = std::get<2>(scatter_op_info.captures); ++ auto scatter_dims_to_operand_dims = std::get<3>(scatter_op_info.captures); ++ auto scatter_indices_batching_dims = std::get<4>(scatter_op_info.captures); ++ auto scatter_type = std::get<5>(scatter_op_info.captures); ++ auto update_window_dims = std::get<6>(scatter_op_info.captures); ++ if (auto search = op_lookup.scatter.find(scatter_op_info); ++ search == op_lookup.scatter.end()) { ++ // std::cout << "Compiling ScatterOp" << std::endl; ++ auto scatter_func = [index_vector_dim, input_batching_dims, ++ inserted_window_dims, scatter_dims_to_operand_dims, ++ scatter_indices_batching_dims, scatter_type, ++ update_window_dims](const std::vector& in) { ++ auto input_count = (in.size() - 1) / 2; ++ std::vector inputs(in.begin(), in.begin() + input_count); ++ std::vector updates(in.begin() + input_count, in.end() - 1); ++ auto scatter_indices = in[in.size() - 1]; ++ // Calculate update scatter dims ++ std::vector update_scatter_dims; ++ for (auto i = 0; i < updates[0].ndim(); i++) { ++ if (std::find(update_window_dims.begin(), update_window_dims.end(), ++ static_cast(i)) == update_window_dims.end()) { ++ update_scatter_dims.emplace_back(static_cast(i)); ++ } ++ } ++ ++ // Calculate update indices ++ std::vector meshgrid_arrays; ++ for (const auto& dim : updates[0].shape()) { ++ meshgrid_arrays.push_back(mx::arange(dim)); ++ } ++ mx::array update_indices = ++ mx::reshape(mx::stack(mx::meshgrid(meshgrid_arrays, false, "ij"), -1), ++ {-1, static_cast(updates[0].ndim())}); ++ ++ // Calculate update scatter index ++ mx::array update_scatter_indices = mx::take( ++ update_indices, ++ mx::array(update_scatter_dims.data(), ++ {static_cast(update_scatter_dims.size())}), ++ 1); ++ ++ // Initialize result indices with zeros ++ auto result_indices = mx::zeros( ++ {update_indices.shape(0), static_cast(inputs[0].ndim())}, ++ mx::int32); ++ ++ // Gather start indices; populating the corresponding ++ // result indices. ++ auto gather_dim_count = index_vector_dim < scatter_indices.ndim() ++ ? update_scatter_dims.size() + 1 ++ : update_scatter_dims.size(); ++ std::vector tmp_gather_axes(update_scatter_dims.size()); ++ std::iota(tmp_gather_axes.begin(), tmp_gather_axes.end(), 0); ++ std::vector gather_slice_sizes(gather_dim_count, 1); ++ if (index_vector_dim < scatter_indices.ndim()) { ++ gather_slice_sizes[index_vector_dim] = ++ scatter_indices.shape(index_vector_dim); ++ } ++ mx::array start_indices = ++ mx::reshape(mx::gather(scatter_indices, ++ mx::split(update_scatter_indices, ++ update_scatter_indices.shape(1), 1), ++ tmp_gather_axes, gather_slice_sizes), ++ {update_indices.shape(0), -1}); ++ ++ // Seed with full start indices ++ result_indices = mx::put_along_axis( ++ result_indices, ++ mx::broadcast_to( ++ mx::array( ++ scatter_dims_to_operand_dims.data(), ++ {static_cast(scatter_dims_to_operand_dims.size())}), ++ {result_indices.shape(0), ++ static_cast(scatter_dims_to_operand_dims.size())}), ++ start_indices, 1); ++ ++ // Calculate full batching indices; accumulating to the ++ // corresponding result indices. ++ auto d_input = ++ mx::array(input_batching_dims.data(), ++ {static_cast(input_batching_dims.size())}); ++ auto d_start = mx::array( ++ scatter_indices_batching_dims.data(), ++ {static_cast(scatter_indices_batching_dims.size())}); ++ result_indices = mx::scatter_add_axis( ++ result_indices, ++ mx::broadcast_to(d_input, ++ {result_indices.shape(0), d_input.shape(0)}), ++ mx::take(update_scatter_indices, ++ mx::subtract( ++ d_start, ++ mx::where(mx::less(d_start, mx::full(d_start.shape(), ++ index_vector_dim)), ++ mx::zeros(d_start.shape(), mx::int32), ++ mx::ones(d_start.shape(), mx::int32))), ++ 1), ++ 1); ++ ++ // Calculate update window indices ++ auto update_window_indices = ++ mx::take(update_indices, ++ mx::array(update_window_dims.data(), ++ {static_cast(update_window_dims.size())}), ++ 1); ++ ++ // Calculate full window indices; accumulating to the ++ // corresponding result indices. ++ std::vector tmp_indices; ++ auto full_window_idx_count = static_cast( ++ update_window_indices.shape(1) + inserted_window_dims.size() + ++ input_batching_dims.size()); ++ for (auto i = 0; i < full_window_idx_count; ++i) { ++ if (std::find(inserted_window_dims.begin(), inserted_window_dims.end(), ++ i) != inserted_window_dims.end() || ++ std::find(input_batching_dims.begin(), input_batching_dims.end(), ++ i) != input_batching_dims.end()) { ++ continue; ++ } ++ tmp_indices.push_back(i); ++ } ++ result_indices = mx::scatter_add_axis( ++ result_indices, ++ mx::broadcast_to( ++ mx::array(tmp_indices.data(), ++ {static_cast(tmp_indices.size())}), ++ {result_indices.shape(0), ++ static_cast(tmp_indices.size())}), ++ update_window_indices, 1); ++ ++ std::vector scatter_axes(inputs[0].ndim()); ++ std::iota(scatter_axes.begin(), scatter_axes.end(), 0); ++ std::vector gather_axes(updates[0].ndim()); ++ std::iota(gather_axes.begin(), gather_axes.end(), 0); ++ std::vector res; ++ ++ auto tmp_gather_indices = ++ mx::split(update_indices, update_indices.shape(1), 1); ++ auto tmp_scatter_indices = ++ mx::split(result_indices, result_indices.shape(1), 1); ++ ++ std::vector update_shape( ++ inputs[0].ndim() + tmp_scatter_indices[0].ndim(), 1); ++ update_shape[0] = update_indices.shape(0); ++ std::vector tmp_gather_slice_sizes(updates[0].ndim(), 1); ++ for (auto i = 0; i < inputs.size(); ++i) { ++ auto update_vals = ++ mx::reshape(mx::gather(updates[i], tmp_gather_indices, gather_axes, ++ tmp_gather_slice_sizes), ++ update_shape); ++ switch (scatter_type) { ++ case utils::ScatterType::Replace: ++ res.push_back(mx::scatter(inputs[i], tmp_scatter_indices, ++ update_vals, scatter_axes)); ++ break; ++ case utils::ScatterType::Add: ++ res.push_back(mx::scatter_add(inputs[i], tmp_scatter_indices, ++ update_vals, scatter_axes)); ++ break; ++ case utils::ScatterType::Prod: ++ res.push_back(mx::scatter_prod(inputs[i], tmp_scatter_indices, ++ update_vals, scatter_axes)); ++ break; ++ case utils::ScatterType::Max: ++ res.push_back(mx::scatter_max(inputs[i], tmp_scatter_indices, ++ update_vals, scatter_axes)); ++ break; ++ case utils::ScatterType::Min: ++ res.push_back(mx::scatter_min(inputs[i], tmp_scatter_indices, ++ update_vals, scatter_axes)); ++ break; ++ } ++ } ++ return res; ++ }; ++ op_lookup.scatter.emplace(scatter_op_info, mx::compile(scatter_func)); ++ } ++} ++ ++void compilePadOp(OpLookup& op_lookup, stablehlo::PadOp o) { ++ PadOpInfo pad_op_info = getPadOpInfo(o); ++ auto result_shape = std::get<0>(pad_op_info.captures); ++ auto edge_pad_low = std::get<1>(pad_op_info.captures); ++ auto interior_pad = std::get<2>(pad_op_info.captures); ++ if (auto search = op_lookup.pad.find(pad_op_info); ++ search == op_lookup.pad.end()) { ++ // std::cout << "Compiling PadOp" << std::endl; ++ auto pad_func = [result_shape, edge_pad_low, ++ interior_pad](const std::vector& inputs) { ++ auto operand = inputs[0]; ++ auto padding_value = inputs[1]; ++ auto edge_padding_low = ++ mx::array(edge_pad_low.data(), ++ {static_cast(edge_pad_low.size())}, mx::int32); ++ auto interior_padding = ++ mx::array(interior_pad.data(), ++ {static_cast(edge_pad_low.size())}, mx::int32); ++ ++ // Calculate operand indices ++ std::vector meshgrid_arrays; ++ for (const auto& dim : operand.shape()) { ++ meshgrid_arrays.push_back(mx::arange(dim)); ++ } ++ mx::array operand_indices = ++ mx::reshape(mx::stack(mx::meshgrid(meshgrid_arrays, false, "ij"), -1), ++ {-1, static_cast(operand.ndim())}); ++ ++ mx::array result_indices = ++ edge_padding_low + ++ operand_indices * (interior_padding + mx::array(1, mx::int32)); ++ ++ // initialize array full of padding values ++ mx::array result = mx::full(result_shape, padding_value); ++ std::vector axes(operand.ndim()); ++ std::iota(axes.begin(), axes.end(), 0); ++ return std::vector{mx::scatter( ++ result, mx::split(result_indices, result_indices.shape(1), 1), ++ mx::gather(operand, ++ mx::split(operand_indices, operand_indices.shape(1), 1), ++ axes, std::vector(operand.ndim(), 1)), ++ axes)}; ++ }; ++ op_lookup.pad.emplace(pad_op_info, mx::compile(pad_func)); ++ } ++} ++ ++void compileModule(ModuleOp mod, OpLookup& op_lookup) { ++ // the top level block to compile ++ // auto main = mod.lookupSymbol("main"); ++ ++ Tree dep_tree; ++ resolveFuncDeps(mod, "main", dep_tree); ++ ++ // walk each function and pre-compile lookup table ++ // for all StableHLO op variants ++ for (auto op_name : dep_tree.getAllFnNames()) { ++ auto func = mod.lookupSymbol(op_name); ++ auto& block = func.front(); ++ for (auto& op : block.getOperations()) { ++ llvm::TypeSwitch(&op) ++ .Case( ++ [&op_lookup](auto o) { compileIotaOp(op_lookup, o); }) ++ .Case( ++ [&op_lookup](auto o) { compileCbrtOp(op_lookup, o); }) ++ .Case( ++ [&op_lookup](auto o) { compileBroadcastInDimOp(op_lookup, o); }) ++ .Case( ++ [&op_lookup](auto o) { compileDotGeneralOp(op_lookup, o); }) ++ .Case( ++ [&op_lookup](auto o) { compileGatherOp(op_lookup, o); }) ++ .Case( ++ [&op_lookup](auto o) { compileScatterOp(op_lookup, o); }) ++ .Case( ++ [&op_lookup](auto o) { compilePadOp(op_lookup, o); }) ++ .Default([](auto o) {}); ++ } ++ } ++ ++ while (true) { ++ auto compilable_fns = dep_tree.getCompilableFns(); ++ if (compilable_fns.empty()) { ++ break; ++ } ++ for (auto fn_name : compilable_fns) { ++ // std::cout << fn_name.c_str() << std::endl; ++ auto func = mod.lookupSymbol(fn_name); ++ compileCallOp(op_lookup, mod, func); ++ dep_tree.remove(fn_name); ++ } ++ } ++} ++ ++class MlirLoadedExecutable : public PjRtLoadedExecutable { ++ public: ++ MlirLoadedExecutable(ModuleOp module, DeviceAssignment assignment, ++ absl::Span devices, ++ PjRtClient* client) ++ : PjRtLoadedExecutable(), ++ name_("MlirLoadedExecutable"), ++ assignment_(assignment), ++ devices_(client->devices()), ++ client_(client), ++ context_(), ++ module_(cloneIntoContext(module, context_)) { ++ // TRACE_ME_MEMBER; ++ auto main = module.lookupSymbol("main"); ++ // std::cout << "Started call to `mlx::compile`" << std::endl << std::endl; ++ // module.dump(); ++ // std::cout << std::endl; ++ compileModule(module, op_lookup_); ++ // try { ++ auto& main_block = main.front(); ++ compiled_module_ = ++ op_lookup_.call.find(getCallOpInfo(module, main))->second; ++ ++ // force compilation by running w zeroed inputs ++ std::vector tmp_inputs; ++ for (auto arg : main_block.getArguments()) { ++ auto shaped_type = mlir::cast(arg.getType()); ++ tmp_inputs.push_back( ++ mx::zeros(std::vector(shaped_type.getShape().begin(), ++ shaped_type.getShape().end()), ++ *utils::dtype::fromMlirType(shaped_type.getElementType()))); ++ } ++ compiled_module_(tmp_inputs); ++ // std::cout << "Finished call to `mlx::compile`" << std::endl << std::endl; ++ // } catch (std::exception& e) { ++ // std::cout << "Error in `mlx::compile`" << std::endl; ++ // } ++ } ++ ++ static absl::StatusOr> CompileAndLoad( ++ ModuleOp module, DeviceAssignment assignment, ++ absl::Span devices, PjRtClient* client) { ++ // TRACE_ME; ++ ++ mlir::BaseScopedDiagnosticHandler diagnostic_handler(module->getContext()); ++ if (failed(decomposeChloToStablehlo(module))) { ++ return diagnostic_handler.ConsumeStatus(); ++ } ++ ++ // Simplify the graph using available HWI passes. ++ if (failed(runHardwareIndependentOptimizations(module))) { ++ return diagnostic_handler.ConsumeStatus(); ++ } ++ ++ auto module_status = validateModule(module); ++ if (!module_status.ok()) { ++ return module_status; ++ } ++ ++ // std::cout << ToString(module).c_str() << std::endl << std::endl; ++ ++ auto executable = std::make_unique(module, assignment, ++ devices, client); ++ ++ return executable; ++ } ++ ++ PjRtClient* client() const override { ++ // TRACE_ME_MEMBER; ++ return client_; ++ } ++ ++ const DeviceAssignment& device_assignment() const override { ++ // TRACE_ME_MEMBER; ++ return assignment_; ++ } ++ ++ absl::Span ++ addressable_device_logical_ids() const override { ++ // TRACE_ME_MEMBER; ++ LOG_UNIMPLEMENTED(addressable_device_logical_ids); ++ return {}; ++ } ++ ++ absl::Span addressable_devices() const override { ++ // TRACE_ME_MEMBER; ++ return devices_; ++ } ++ ++ // Helper function to get default mem from device. ++ PjRtMemorySpace* get_default_memory_space() const { ++ // TRACE_ME_MEMBER; ++ return devices_[0]->default_memory_space().value_or(nullptr); ++ } ++ ++ // absl::StatusOr> evalModule( ++ // ModuleOp& module, const SmallVector& inputs) { ++ // // TRACE_ME_MEMBER; ++ // std::vector block_arguments; ++ // for (auto input : inputs) { ++ // TF_ASSIGN_OR_RETURN(auto input_arr, ++ // utils::array::fromDenseElementsAttr(input)); ++ // block_arguments.emplace_back(input_arr); ++ // } ++ ++ // // std::cout << std::endl; ++ // // module.dump(); ++ // // std::cout << std::endl; ++ // auto main = module.lookupSymbol("main"); ++ // std::unordered_map< ++ // mlir::Block*, ++ // std::tuple, ++ // std::unordered_map>>> ++ // block_ctx; ++ // TF_ASSIGN_OR_RETURN(auto mlx_res, evalBlock(module, main.front(), ++ // block_arguments, block_ctx)); ++ ++ // for (auto& a : mlx_res) { ++ // a = mx::contiguous(a); ++ // } ++ // mx::eval(mlx_res); ++ // return mlx_res; ++ // } ++ ++ absl::StatusOr ExecuteWithMlxInterpreter( ++ absl::Span argument_handles, ModuleOp module, ++ PjRtDevice* device, bool fill_future) { ++ // TRACE_ME_MEMBER; ++ SmallVector input_attrs; ++ for (auto* arg : argument_handles) { ++ TF_ASSIGN_OR_RETURN(auto mlirArg, GetAttributeFromBuffer(arg)); ++ auto mlirArgInModuleContext = ++ CloneIntoContext(mlirArg, *module->getContext()); ++ input_attrs.push_back(mlirArgInModuleContext); ++ } ++ // LOG(INFO) << "EvalModule:\n" << ToString(module) << "\n"; ++ // LOG(INFO) << "Inputs: " << ToString(inputs) << "\n"; ++ // TF_ASSIGN_OR_RETURN(auto mlx_res, evalModule(module, inputs)); ++ ++ std::vector inputs; ++ // unsigned in_count = 0; ++ for (auto& attr : input_attrs) { ++ TF_ASSIGN_OR_RETURN(mx::array input, ++ utils::array::fromDenseElementsAttr(attr)); ++ // std::cout << "inputs[" << std::to_string(in_count++) << "]: " << input ++ // << std::endl; ++ inputs.push_back(input); ++ } ++ ++ auto mlx_res = compiled_module_(inputs); ++ for (auto& a : mlx_res) { ++ a = mx::contiguous(a); ++ } ++ mx::eval(mlx_res); ++ // unsigned out_count = 0; ++ // for (auto& a : mlx_res) { ++ // std::cout << "outputs[" << std::to_string(out_count++) << "]: " << a ++ // << std::endl; ++ // } ++ ++ // LOG(INFO) << "Results: " << ToString(result.value()) << "\n"; ++ ++ // Naive memory space selection, only using CPU global memory. ++ PjRtMemorySpace* memory_space = ++ device->default_memory_space().value_or(nullptr); ++ std::vector> buffer_results; ++ for (auto i = 0; i < mlx_res.size(); i++) { ++ buffer_results.push_back( ++ CreateMlirBufferFromMlxArray(mlx_res[i], memory_space)); ++ } ++ ++ std::optional> future; ++ if (fill_future) { ++ // Synchronous! To make async, this would need to return a future that ++ // is ready when the computation is done. ++ future = PjRtFuture<>(absl::OkStatus()); ++ } ++ return PjRtLoadedExecutable::Result{future, std::move(buffer_results)}; ++ } ++ ++ absl::StatusOr>>> Execute( ++ absl::Span> argument_handles, ++ const xla::ExecuteOptions& options, ++ std::optional>>& returned_futures) override { ++ // TRACE_ME_MEMBER; ++ if (argument_handles.size() != 1) { ++ // One arg handle per device. ++ return absl::InvalidArgumentError( ++ "MlirLoadedExecutable::Execute only supports a single argument " ++ "vector"); ++ } ++ ++ // Single device, synchronous, can always use 0. ++ PjRtDevice* device = devices_[0]; ++ bool fill_future = returned_futures.has_value(); ++ TF_ASSIGN_OR_RETURN( ++ PjRtLoadedExecutable::Result result, ++ ExecuteWithMlxInterpreter(argument_handles[0], module_.get(), device, ++ fill_future)); ++ std::vector>> results; ++ results.push_back(std::move(result.buffers)); ++ if (returned_futures.has_value()) { ++ returned_futures->push_back(std::move(result.future.value())); ++ } ++ return results; ++ } ++ ++ absl::StatusOr>> ExecuteSharded( ++ absl::Span argument_handles, PjRtDevice* device, ++ const xla::ExecuteOptions& options, ++ std::optional>& returned_future, bool fill_future) override { ++ // TRACE_ME_MEMBER; ++ // Synchronous! To make async, have the device make a buffer with a ready ++ // future that is ready when the computation is done / buffer is ready. ++ TF_ASSIGN_OR_RETURN( ++ PjRtLoadedExecutable::Result result, ++ ExecuteWithMlxInterpreter(argument_handles, module_.get(), device, ++ fill_future)); ++ if (returned_future.has_value() && fill_future) { ++ returned_future = std::move(result.future); ++ } ++ return std::move(result.buffers); ++ } ++ ++ absl::StatusOr>> ExecutePortable( ++ absl::Span argument_handles, PjRtDevice* device, ++ const xla::ExecuteOptions& options, ++ std::optional>& returned_future, bool fill_future) override { ++ // TRACE_ME_MEMBER; ++ // Synchronous! To make async, have the device make a buffer with a ready ++ // future that is ready when the computation is done / buffer is ready. ++ TF_ASSIGN_OR_RETURN( ++ PjRtLoadedExecutable::Result result, ++ ExecuteWithMlxInterpreter(argument_handles, module_.get(), device, ++ fill_future)); ++ if (returned_future.has_value() && fill_future) { ++ returned_future = std::move(result.future); ++ } ++ return std::move(result.buffers); ++ } ++ ++ void Delete() override { ++ // TRACE_ME_MEMBER; ++ module_.release(); ++ module_ = nullptr; ++ } ++ bool IsDeleted() override { ++ // TRACE_ME_MEMBER; ++ return !module_; ++ } ++ ++ // PjRtExecutable API. ++ int num_replicas() const override { ++ // TRACE_ME_MEMBER; ++ return assignment_.replica_count(); ++ } ++ int num_partitions() const override { ++ // TRACE_ME_MEMBER; ++ return assignment_.computation_count(); ++ } ++ int64_t SizeOfGeneratedCodeInBytes() const override { ++ // No generated code.. so just return 1. ++ // TRACE_ME_MEMBER; ++ return 1; ++ } ++ absl::string_view name() const override { ++ // TRACE_ME_MEMBER; ++ return name_; ++ } ++ ++ absl::StatusOr>> GetHloModules() ++ const override { ++ // TODO: This shouldn't be needed for an MLIR plugin, its only used in the ++ // JAX layer for determining output sharding, which exists on the mlir ++ // module. ++ // TRACE_ME_MEMBER; ++ auto moduleClone = llvm::cast(module_.get()->clone()); ++ TF_ASSIGN_OR_RETURN(auto hlo_module, ++ xla::ConvertStablehloToHlo(moduleClone)); ++ return std::vector>{std::move(hlo_module)}; ++ } ++ ++ absl::StatusOr>> ++ GetOutputMemoryKinds() const override { ++ // TRACE_ME_MEMBER; ++ return UNIMPLEMENTED(GetOutputMemoryKinds); ++ } ++ ++ absl::StatusOr FingerprintExecutable() const override { ++ // TRACE_ME_MEMBER; ++ return UNIMPLEMENTED(FingerprintExecutable); ++ } ++ ++ private: ++ std::string name_; ++ DeviceAssignment assignment_; ++ absl::Span devices_; ++ PjRtClient* client_; ++ std::function(const std::vector&)> ++ compiled_module_; ++ OpLookup op_lookup_; ++ ++ // MLIR ++ MLIRContext context_; ++ mlir::OwningOpRef module_; ++}; ++ ++absl::StatusOr> StablehloMlxCompile( ++ mlir::ModuleOp module, DeviceAssignment assignment, PjRtClient* client) { ++ // TRACE_ME; ++ return MlirLoadedExecutable::CompileAndLoad(module, assignment, ++ client->devices(), client); ++} ++ ++absl::StatusOr> StablehloMlxCompile( ++ xla::XlaComputation const& computation, xla::DeviceAssignment assignment, ++ xla::PjRtClient* client) { ++ // TRACE_ME; ++ MLIRContext context; ++ TF_ASSIGN_OR_RETURN(auto module, ++ ConvertHloToStablehlo(context, &computation.proto())); ++ return StablehloMlxCompile(module.get(), assignment, client); ++} ++ ++} // namespace mlir::stablehlo +diff --git a/xla/pjrt/plugin/mlx/executable.h b/xla/pjrt/plugin/mlx/executable.h +new file mode 100644 +index 0000000000..e579af4e1f +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/executable.h +@@ -0,0 +1,244 @@ ++/* Copyright 2024 The OpenXLA Authors. ++Licensed under the Apache License, Version 2.0 (the "License"); ++you may not use this file except in compliance with the License. ++You may obtain a copy of the License at ++ http://www.apache.org/licenses/LICENSE-2.0 ++Unless required by applicable law or agreed to in writing, software ++distributed under the License is distributed on an "AS IS" BASIS, ++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++See the License for the specific language governing permissions and ++limitations under the License. ++==============================================================================*/ ++ ++#ifndef XLA_PJRT_PLUGIN_STABLEHLO_MLX_EXECUTABLE_H_ ++#define XLA_PJRT_PLUGIN_STABLEHLO_MLX_EXECUTABLE_H_ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "mlir/IR/BuiltinOps.h" ++#include "mlx/mlx.h" ++#include "xla/pjrt/plugin/mlx/utils.h" ++#include "xla/hlo/builder/xla_computation.h" ++#include "xla/pjrt/pjrt_client.h" ++#include "xla/service/computation_placer.h" ++ ++namespace mlir::stablehlo { ++ ++struct Node { ++ std::string name; ++ std::vector> children; ++ ++ Node(std::string name) : name(name) {} ++}; ++ ++class Tree { ++ public: ++ std::unique_ptr root; ++ ++ Tree() : root(nullptr) {} ++ ++ Node* insert(std::string name, Node* parent = nullptr) { ++ auto new_node = std::make_unique(name); ++ if (!root) { ++ root = std::move(new_node); ++ return root.get(); ++ } else if (parent) { ++ parent->children.push_back(std::move(new_node)); ++ return parent->children.back().get(); ++ } ++ } ++ ++ void remove(const std::string& name) { ++ if (!root) return; ++ ++ // Special case: root node matches ++ if (root->name == name) { ++ root.reset(); // deletes entire tree ++ return; ++ } ++ ++ remove_helper(root, name); ++ } ++ ++ std::set getAllFnNames() { ++ std::set fn_names; ++ if (!root) return fn_names; ++ ++ std::function traverse = [&](Node* node) { ++ fn_names.insert(node->name); ++ for (auto& child : node->children) { ++ traverse(child.get()); ++ } ++ }; ++ traverse(root.get()); ++ return fn_names; ++ } ++ ++ std::unordered_set getCompilableFns() { ++ std::unordered_set compilable_fns; ++ if (!root) return compilable_fns; ++ ++ std::function traverse = [&](Node* node) { ++ if (node->children.empty()) { ++ compilable_fns.insert(node->name); ++ } else { ++ for (auto& child : node->children) { ++ traverse(child.get()); ++ } ++ } ++ }; ++ traverse(root.get()); ++ return compilable_fns; ++ } ++ ++ private: ++ void remove_helper(std::unique_ptr& current, const std::string& name) { ++ if (!current) return; ++ ++ auto& children = current->children; ++ ++ // Remove matching children ++ children.erase(std::remove_if(children.begin(), children.end(), ++ [&](std::unique_ptr& child) { ++ return child->name == name; ++ }), ++ children.end()); ++ ++ // Recurse on remaining children ++ for (auto& child : children) { ++ remove_helper(child, name); ++ } ++ } ++}; ++ ++template ++class CompiledOpInfo { ++ public: ++ std::tuple captures; ++ std::vector> input_metadata; ++ std::vector> output_metadata; ++ ++ CompiledOpInfo(std::vector inputs, ++ std::vector> outputs, ++ T... capture_values); ++ ++ CompiledOpInfo(std::vector> inputs, ++ std::vector> outputs, ++ T... capture_values); ++ ++ bool operator==(const CompiledOpInfo& other) const { ++ return captures == other.captures && ++ input_metadata == other.input_metadata && ++ output_metadata == other.output_metadata; ++ } ++}; ++ ++template ++CompiledOpInfo::CompiledOpInfo( ++ std::vector inputs, ++ std::vector> outputs, T... capture_values) ++ : captures(std::make_tuple(capture_values...)), output_metadata(outputs) { ++ input_metadata.reserve(inputs.size()); ++ for (const auto& input : inputs) { ++ input_metadata.push_back(std::make_pair(input.dtype(), input.shape())); ++ } ++} ++ ++template ++CompiledOpInfo::CompiledOpInfo( ++ std::vector> inputs, ++ std::vector> outputs, T... capture_values) ++ : captures(std::make_tuple(capture_values...)), ++ input_metadata(inputs), ++ output_metadata(outputs) {} ++ ++using CompiledOpSign = ++ std::function(const std::vector&)>; ++ ++using CallOpInfo = CompiledOpInfo; ++ ++// stablehlo::IotaOp - Captures: (dimensions, dtype, iota_dim, result_shape) ++using IotaOpInfo = CompiledOpInfo, mx::Dtype, int64_t, ++ std::vector>; ++ ++// stablehlo::CbrtOp - Captures: () ++using CbrtOpInfo = CompiledOpInfo<>; ++ ++// stablehlo::BroadcastInDimOp - Captures: (broadcast_dims, target_shape) ++using BroadcastInDimOpInfo = ++ CompiledOpInfo, std::vector>; ++ ++// stablehlo::DotGeneralOp - Captures: (lhs_batch_dims, rhs_batch_dims, ++// lhs_contract_dims, rhs_contract_dims) ++using DotGeneralOpInfo = ++ CompiledOpInfo, std::vector, ++ std::vector, std::vector>; ++ ++// stablehlo::GatherOp - Captures: (collapsed_slice_dims, index_vector_dim, ++// offset_dims, operand_batching_dims, result_shape, slice_sizes, ++// start_index_map, start_indices_batching_dims) ++using GatherOpInfo = CompiledOpInfo, int64_t, ++ std::vector, std::vector, ++ std::vector, std::vector, ++ std::vector, std::vector>; ++ ++// stablehlo::ScatterOp - Captures: (index_vector_dim, input_batching_dims, ++// inserted_window_dims, scatter_dims_to_operand_dims, ++// scatter_indices_batching_dims, scatter_type, update_window_dims) ++using ScatterOpInfo = ++ CompiledOpInfo, std::vector, ++ std::vector, std::vector, ++ utils::ScatterType, std::vector>; ++ ++// stablehlo::PadOp - Captures: (result_shape, edge_pad_low, interior_pad) ++using PadOpInfo = CompiledOpInfo, std::vector, ++ std::vector>; ++ ++absl::StatusOr> StablehloMlxCompile( ++ mlir::ModuleOp module, xla::DeviceAssignment assignment, ++ xla::PjRtClient* client); ++ ++absl::StatusOr> StablehloMlxCompile( ++ xla::XlaComputation const& computation, xla::DeviceAssignment assignment, ++ xla::PjRtClient* client); ++ ++} // namespace mlir::stablehlo ++ ++template ++struct std::hash> { ++ std::size_t operator()( ++ const mlir::stablehlo::CompiledOpInfo& info) const noexcept { ++ std::size_t seed = 0; ++ hash_tuple(seed, info.captures); ++ for (const auto& meta : info.input_metadata) { ++ hash_combine(seed, std::hash{}(meta.first)); ++ hash_combine(seed, std::hash>{}(meta.second)); ++ } ++ for (const auto& meta : info.output_metadata) { ++ hash_combine(seed, std::hash{}(meta.first)); ++ hash_combine(seed, std::hash>{}(meta.second)); ++ } ++ return seed; ++ } ++}; ++ ++namespace mlir::stablehlo { ++struct OpLookup { ++ std::unordered_map, CompiledOpSign> call; ++ std::unordered_map iota; ++ std::unordered_map cbrt; ++ std::unordered_map broadcast_in_dim; ++ std::unordered_map dot_general; ++ std::unordered_map gather; ++ std::unordered_map scatter; ++ std::unordered_map pad; ++}; ++ ++}; // namespace mlir::stablehlo ++ ++#endif // XLA_PJRT_PLUGIN_STABLEHLO_MLX_EXECUTABLE_H_ +\ No newline at end of file +diff --git a/xla/pjrt/plugin/mlx/logging.cc b/xla/pjrt/plugin/mlx/logging.cc +new file mode 100644 +index 0000000000..3f21f67baf +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/logging.cc +@@ -0,0 +1,89 @@ ++/* Copyright 2024 The OpenXLA Authors. ++Licensed under the Apache License, Version 2.0 (the "License"); ++you may not use this file except in compliance with the License. ++You may obtain a copy of the License at ++ http://www.apache.org/licenses/LICENSE-2.0 ++Unless required by applicable law or agreed to in writing, software ++distributed under the License is distributed on an "AS IS" BASIS, ++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++See the License for the specific language governing permissions and ++limitations under the License. ++==============================================================================*/ ++ ++#include "xla/pjrt/plugin/mlx/logging.h" ++ ++#include ++ ++#include "absl/base/log_severity.h" ++#include "absl/log/globals.h" ++#include "llvm/Support/raw_ostream.h" ++#include "mlir/IR/Attributes.h" ++#include "mlir/IR/BuiltinAttributes.h" ++#include "mlir/IR/Operation.h" ++ ++namespace mlir::stablehlo { ++ ++std::string ToString(mlir::Attribute attr) { ++ std::string out; ++ llvm::raw_string_ostream os(out); ++ attr.print(os); ++ return out; ++} ++ ++std::string ToString(SmallVector attrs) { ++ std::string out; ++ llvm::raw_string_ostream os(out); ++ os << "["; ++ bool first = true; ++ for (auto attr : attrs) { ++ if (!first) os << ", "; ++ first = false; ++ attr.print(os); ++ } ++ os << "]"; ++ return out; ++} ++std::string ToString(Operation& op) { ++ std::string out; ++ llvm::raw_string_ostream os(out); ++ op.print(os); ++ return out; ++} ++ ++std::string ToString(Operation* op) { ++ std::string out; ++ llvm::raw_string_ostream os(out); ++ os << *op; ++ return out; ++} ++ ++std::string ToString(mlir::Value value) { ++ std::string out; ++ llvm::raw_string_ostream os(out); ++ value.print(os); ++ return out; ++} ++ ++std::string ToString(mlir::Block& block) { ++ std::string out; ++ llvm::raw_string_ostream os(out); ++ block.print(os); ++ return out; ++} ++ ++void SetupLogLevelFromEnv() { ++ absl::SetMinLogLevel(absl::LogSeverityAtLeast::kError); ++ const char* log_env = std::getenv("PJRT_LOG_LEVEL"); ++ if (!log_env) return; ++ if (strcmp(log_env, "INFO") == 0) { ++ absl::SetMinLogLevel(absl::LogSeverityAtLeast::kInfo); ++ } else if (strcmp(log_env, "WARNING") == 0) { ++ absl::SetMinLogLevel(absl::LogSeverityAtLeast::kWarning); ++ } else if (strcmp(log_env, "ERROR") == 0) { ++ absl::SetMinLogLevel(absl::LogSeverityAtLeast::kError); ++ } else { ++ LOG(ERROR) << "Invalid PJRT_LOG_LEVEL: " << log_env; ++ } ++} ++ ++} // namespace mlir::stablehlo +\ No newline at end of file +diff --git a/xla/pjrt/plugin/mlx/logging.h b/xla/pjrt/plugin/mlx/logging.h +new file mode 100644 +index 0000000000..762dfabbcc +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/logging.h +@@ -0,0 +1,42 @@ ++/* Copyright 2024 The OpenXLA Authors. ++Licensed under the Apache License, Version 2.0 (the "License"); ++you may not use this file except in compliance with the License. ++You may obtain a copy of the License at ++ http://www.apache.org/licenses/LICENSE-2.0 ++Unless required by applicable law or agreed to in writing, software ++distributed under the License is distributed on an "AS IS" BASIS, ++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++See the License for the specific language governing permissions and ++limitations under the License. ++==============================================================================*/ ++ ++#ifndef XLA_PJRT_PLUGIN_STABLEHLO_MLX_LOGGING_H_ ++#define XLA_PJRT_PLUGIN_STABLEHLO_MLX_LOGGING_H_ ++ ++// This file has some joint logging to allow LOG and VLOG to play well with ++// MLIR data structures ++ ++#include "absl/log/log.h" ++#include "mlir/IR/Attributes.h" ++#include "mlir/IR/BuiltinAttributes.h" ++ ++#define LOG_UNIMPLEMENTED(name) \ ++ LOG(ERROR) << "MlxPjRtBuffer::" #name " is not implemented" ++ ++#define TRACE_ME LOG(INFO) << __func__; ++ ++#define TRACE_ME_MEMBER LOG(INFO) << __func__ << "(" << (void*)this << ")\n"; ++ ++namespace mlir::stablehlo { ++std::string ToString(mlir::Attribute attr); ++std::string ToString(llvm::SmallVector attrs); ++std::string ToString(Operation* op); ++std::string ToString(Operation& op); ++std::string ToString(mlir::Value op); ++std::string ToString(mlir::Block& block); ++ ++// Looks for `PJRT_LOG_LEVEL = INFO|WARNING|ERROR` in env variables. ++void SetupLogLevelFromEnv(); ++} // namespace mlir::stablehlo ++ ++#endif // XLA_PJRT_PLUGIN_STABLEHLO_MLX_LOGGING_H_ +\ No newline at end of file +diff --git a/xla/pjrt/plugin/mlx/plugin_pjrt_test.cc b/xla/pjrt/plugin/mlx/plugin_pjrt_test.cc +new file mode 100644 +index 0000000000..d56a22565f +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/plugin_pjrt_test.cc +@@ -0,0 +1,33 @@ ++/* Copyright 2022 The OpenXLA Authors. ++Licensed under the Apache License, Version 2.0 (the "License"); ++you may not use this file except in compliance with the License. ++You may obtain a copy of the License at ++ http://www.apache.org/licenses/LICENSE-2.0 ++Unless required by applicable law or agreed to in writing, software ++distributed under the License is distributed on an "AS IS" BASIS, ++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++See the License for the specific language governing permissions and ++limitations under the License. ++==============================================================================*/ ++ ++#include "xla/pjrt/c/pjrt_c_api_test.h" ++#include "xla/pjrt/c/pjrt_c_api_wrapper_impl.h" ++#include "xla/pjrt/pjrt_client_test.h" ++#include "xla/pjrt/plugin/mlx/client_c_pjrt.h" ++#include "xla/pjrt/plugin/mlx/client_cpp_pjrt.h" ++ ++namespace pjrt { ++namespace { ++ ++const bool kUnused = ++ (RegisterPjRtCApiTestFactory([]() { return GetPjrtApi(); }, ++ /*platform_name=*/"stablehlo_mlx"), ++ true); ++ ++const bool kUnused2 = (xla::RegisterTestClientFactory([]() { ++ return mlir::stablehlo::CreateStablehloMlxPjrtClient(); ++ }), ++ true); ++ ++} // namespace ++} // namespace pjrt +\ No newline at end of file +diff --git a/xla/pjrt/plugin/mlx/utils.cc b/xla/pjrt/plugin/mlx/utils.cc +new file mode 100644 +index 0000000000..5fcfc318ab +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/utils.cc +@@ -0,0 +1,292 @@ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "xla/pjrt/plugin/mlx/utils.h" ++#include "absl/status/status.h" ++#include "absl/status/statusor.h" ++#include "absl/types/span.h" ++#include "llvm/ADT/TypeSwitch.h" ++#include "mlir/IR/BuiltinAttributes.h" ++#include "mlir/IR/Types.h" ++#include "mlx/mlx.h" ++#include "tsl/platform/statusor.h" ++#include "xla/mlir/utils/type_util.h" ++#include "xla/shape_util.h" ++ ++#include "xla/literal.h" ++#include "xla/util.h" ++ ++namespace mx = mlx::core; ++ ++std::tuple shapeAndBytes( ++ absl::Span dims, ++ std::optional> byte_strides, mx::Dtype dtype) { ++ mx::Shape shape(dims.size()); ++ mx::Strides strides; ++ mx::Shape raw_buffer_shape = {shape.size() && !byte_strides.has_value() ? 1 ++ : 0}; ++ for (auto i = 0; i < dims.size(); ++i) { ++ shape[i] = static_cast(dims[i]); ++ if (byte_strides.has_value()) { ++ auto stride_value = byte_strides.value()[i]; ++ strides.push_back(stride_value / dtype.size()); ++ if (strides[i] != 0) { ++ raw_buffer_shape[0] = std::max( ++ raw_buffer_shape[0], static_cast(stride_value) * shape[i]); ++ } ++ } else { ++ raw_buffer_shape[0] *= static_cast(dims[i]); ++ } ++ } ++ if (!byte_strides.has_value()) { ++ raw_buffer_shape[0] *= dtype.size(); ++ } else if (raw_buffer_shape[0] == 0 and ++ std::find(dims.begin(), dims.end(), 0) == dims.end()) { ++ raw_buffer_shape[0] += dtype.size(); ++ } ++ return std::make_tuple(shape, raw_buffer_shape, strides); ++} ++ ++namespace utils { ++ ++namespace dtype { ++xla::PrimitiveType asXlaPrimitiveType(mx::Dtype dtype) { ++ switch (dtype) { ++ case mx::bool_: ++ return xla::PrimitiveType::PRED; ++ case mx::uint8: ++ return xla::PrimitiveType::U8; ++ case mx::uint16: ++ return xla::PrimitiveType::U16; ++ case mx::uint32: ++ return xla::PrimitiveType::U32; ++ case mx::uint64: ++ return xla::PrimitiveType::U64; ++ case mx::int8: ++ return xla::PrimitiveType::S8; ++ case mx::int16: ++ return xla::PrimitiveType::S16; ++ case mx::int32: ++ return xla::PrimitiveType::S32; ++ case mx::int64: ++ return xla::PrimitiveType::S64; ++ case mx::float16: ++ return xla::PrimitiveType::F16; ++ case mx::float32: ++ return xla::PrimitiveType::F32; ++ case mx::bfloat16: ++ return xla::PrimitiveType::BF16; ++ case mx::complex64: ++ return xla::PrimitiveType::C64; ++ } ++} ++ ++absl::StatusOr fromXlaPrimitiveType(xla::PrimitiveType dtype) { ++ switch (dtype) { ++ case xla::PrimitiveType::PRED: ++ return mx::bool_; ++ case xla::PrimitiveType::BF16: ++ return mx::bfloat16; ++ case xla::PrimitiveType::F16: ++ return mx::float16; ++ case xla::PrimitiveType::F32: ++ return mx::float32; ++ case xla::PrimitiveType::S8: ++ return mx::int8; ++ case xla::PrimitiveType::S16: ++ return mx::int16; ++ case xla::PrimitiveType::S32: ++ return mx::int32; ++ case xla::PrimitiveType::S64: ++ return mx::int64; ++ case xla::PrimitiveType::U8: ++ return mx::uint8; ++ case xla::PrimitiveType::U16: ++ return mx::uint16; ++ case xla::PrimitiveType::U32: ++ return mx::uint32; ++ case xla::PrimitiveType::U64: ++ return mx::uint64; ++ case xla::PrimitiveType::C64: ++ return mx::complex64; ++ default: ++ return absl::InvalidArgumentError("Unsupported type"); ++ } ++} ++ ++absl::StatusOr fromMlirType(mlir::Type type) { ++ auto primitive_type = xla::ConvertMlirTypeToPrimitiveType(type); ++ if (primitive_type == xla::PrimitiveType::PRIMITIVE_TYPE_INVALID) { ++ return xla::Internal("Unsupported type: %s", ++ xla::PrimitiveType_Name(primitive_type)); ++ } ++ return fromXlaPrimitiveType(primitive_type); ++} ++ ++int mlirTypeByteWidth(mlir::Type type) { ++ return xla::primitive_util::ByteWidth( ++ xla::ConvertMlirTypeToPrimitiveType(type)); ++} ++} // namespace dtype ++ ++namespace array { ++absl::StatusOr fromHostBuffer( ++ const void* data, absl::Span dims, ++ std::optional> byte_strides, ++ xla::PrimitiveType type) { ++ TF_ASSIGN_OR_RETURN(mx::Dtype dtype, dtype::fromXlaPrimitiveType(type)); ++ auto [shape, raw_buffer_shape, strides] = ++ shapeAndBytes(dims, byte_strides, dtype); ++ mx::array typed_view = mx::view(mx::array(reinterpret_cast(data), ++ raw_buffer_shape, mx::uint8), ++ dtype); ++ if (byte_strides.has_value()) { ++ auto res = mx::contiguous(mx::as_strided(typed_view, shape, strides, 0)); ++ res.eval(); ++ return res; ++ } else { ++ auto res = mx::reshape(typed_view, shape); ++ res.eval(); ++ return res; ++ } ++} ++ ++absl::StatusOr fromHostLiteral(const xla::LiteralSlice& literal) { ++ return fromHostBuffer(literal.untyped_data(), literal.shape().dimensions(), ++ std::nullopt, literal.shape().element_type()); ++} ++ ++template ++absl::StatusOr fromDenseElementsAttr(mlir::DenseElementsAttr attr) { ++ auto attr_type = attr.getType(); ++ // convert to mlx shape ++ std::vector shape(attr_type.getShape().begin(), ++ attr_type.getShape().end()); ++ ++ // handle splat values ++ if (attr.isSplat()) { ++ if constexpr (std::is_same::value) { ++ return mx::full( ++ shape, static_cast(attr.getSplatValue())); ++ } else if constexpr (std::is_same::value) { ++ return mx::full( ++ shape, static_cast(attr.getSplatValue())); ++ } else if constexpr (std::is_same>::value) { ++ return mx::full( ++ shape, static_cast(attr.getSplatValue())); ++ } else { ++ return mx::full(shape, attr.getSplatValue()); ++ } ++ } ++ ++ // handle non-splat values that are expanded to fit shape ++ auto it = attr.getValues(); ++ auto buffer = llvm::to_vector(it); ++ if (attr.size() != attr.getNumElements()) { ++ if constexpr (std::is_same::value) { ++ return mx::full( ++ shape, ++ mx::array(reinterpret_cast(buffer.data()), ++ {static_cast(attr.size())})); ++ } else if constexpr (std::is_same::value) { ++ return mx::full( ++ shape, ++ mx::array(reinterpret_cast(buffer.data()), ++ {static_cast(attr.size())})); ++ } else if constexpr (std::is_same>::value) { ++ return mx::full( ++ shape, ++ mx::array(reinterpret_cast(buffer.data()), ++ {static_cast(attr.size())})); ++ } else { ++ return mx::full(shape, mx::array(buffer.begin(), ++ {static_cast(attr.size())})); ++ } ++ } ++ ++ if constexpr (std::is_same::value) { ++ return mx::array(reinterpret_cast(buffer.data()), ++ shape); ++ } else if constexpr (std::is_same::value) { ++ return mx::array(reinterpret_cast(buffer.data()), ++ shape); ++ } else if constexpr (std::is_same>::value) { ++ return mx::array(reinterpret_cast(buffer.data()), ++ shape); ++ } else { ++ return mx::array(buffer.begin(), shape); ++ } ++} ++ ++absl::StatusOr fromDenseElementsAttr(mlir::DenseElementsAttr attr) { ++ auto element_type = ++ xla::ConvertMlirTypeToPrimitiveType(attr.getType().getElementType()); ++ switch (element_type) { ++ case xla::PrimitiveType::PRED: ++ return fromDenseElementsAttr(attr); ++ case xla::PrimitiveType::U8: ++ return fromDenseElementsAttr(attr); ++ case xla::PrimitiveType::S8: ++ return fromDenseElementsAttr(attr); ++ case xla::PrimitiveType::U16: ++ return fromDenseElementsAttr(attr); ++ case xla::PrimitiveType::S16: ++ return fromDenseElementsAttr(attr); ++ case xla::PrimitiveType::U32: ++ return fromDenseElementsAttr(attr); ++ case xla::PrimitiveType::S32: ++ return fromDenseElementsAttr(attr); ++ case xla::PrimitiveType::U64: ++ return fromDenseElementsAttr(attr); ++ case xla::PrimitiveType::S64: ++ return fromDenseElementsAttr(attr); ++ case xla::PrimitiveType::F16: ++ return fromDenseElementsAttr(attr); ++ case xla::PrimitiveType::BF16: ++ return fromDenseElementsAttr(attr); ++ case xla::PrimitiveType::F32: ++ return fromDenseElementsAttr(attr); ++ case xla::PrimitiveType::C64: ++ return fromDenseElementsAttr>(attr); ++ default: ++ return xla::Internal("Unsupported type: %s", ++ xla::PrimitiveType_Name(element_type)); ++ } ++} ++ ++absl::StatusOr fromOperand( ++ mlir::Value operand, absl::Span block_args, ++ const std::unordered_map>& ++ transient_buffers, ++ const std::unordered_map& init_values) {} ++ ++absl::StatusOr fromOperand( ++ mlir::Value operand, absl::Span block_args, ++ const std::unordered_map>& ++ transient_buffers) { ++ return fromOperand(operand, block_args, transient_buffers, {}); ++} ++ ++} // namespace array ++ ++void printVector(const std::string& name, const std::vector& vec, ++ bool indent) { ++ if (indent) { ++ std::cout << "\t"; ++ } ++ std::cout << name.c_str() << ": { "; ++ for (auto i = 0; i < vec.size(); i++) { ++ std::cout << std::to_string(vec[i]); ++ if (i < vec.size() - 1) { ++ std::cout << ", "; ++ } ++ } ++ std::cout << " }" << std::endl; ++} ++} // namespace utils +diff --git a/xla/pjrt/plugin/mlx/utils.h b/xla/pjrt/plugin/mlx/utils.h +new file mode 100644 +index 0000000000..1a6d85c060 +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/utils.h +@@ -0,0 +1,129 @@ ++/* Copyright 2024 The OpenXLA Authors. ++Licensed under the Apache License, Version 2.0 (the "License"); ++you may not use this file except in compliance with the License. ++You may obtain a copy of the License at ++ http://www.apache.org/licenses/LICENSE-2.0 ++Unless required by applicable law or agreed to in writing, software ++distributed under the License is distributed on an "AS IS" BASIS, ++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++See the License for the specific language governing permissions and ++limitations under the License. ++==============================================================================*/ ++ ++#ifndef XLA_PJRT_PLUGIN_STABLEHLO_MLX_UTILS_H_ ++#define XLA_PJRT_PLUGIN_STABLEHLO_MLX_UTILS_H_ ++ ++#include ++#include ++#include ++#include ++#include ++ ++#include "absl/status/status.h" ++#include "absl/status/statusor.h" ++#include "absl/types/span.h" ++#include "mlir/IR/BuiltinAttributes.h" ++#include "mlir/IR/Types.h" ++#include "mlx/mlx.h" ++#include "xla/literal.h" ++#include "xla/util.h" ++ ++namespace mx = mlx::core; ++ ++namespace utils { ++ ++enum ScatterType { ++ Replace, ++ Add, ++ Prod, ++ Max, ++ Min, ++}; ++namespace dtype { ++xla::PrimitiveType asXlaPrimitiveType(mx::Dtype dtype); ++ ++absl::StatusOr fromXlaPrimitiveType(xla::PrimitiveType dtype); ++ ++absl::StatusOr fromMlirType(mlir::Type type); ++ ++int mlirTypeByteWidth(mlir::Type type); ++} // namespace dtype ++ ++namespace array { ++absl::StatusOr fromHostBuffer( ++ const void* data, absl::Span dims, ++ std::optional> byte_strides, ++ xla::PrimitiveType type); ++ ++absl::StatusOr fromHostLiteral(const xla::LiteralSlice& literal); ++ ++absl::StatusOr fromDenseElementsAttr(mlir::DenseElementsAttr attr); ++ ++absl::StatusOr fromOperand( ++ mlir::Value operand, absl::Span block_args, ++ const std::unordered_map>& ++ transient_buffers, ++ const std::unordered_map& init_values); ++ ++absl::StatusOr fromOperand( ++ mlir::Value operand, absl::Span block_args, ++ const std::unordered_map>& ++ transient_buffers); ++} // namespace array ++ ++void printVector(const std::string& name, const std::vector& vec, ++ bool indent = false); ++} // namespace utils ++ ++// Helper function to hash a tuple ++template ++inline void hash_combine(std::size_t& seed, const T& val) { ++ seed ^= std::hash{}(val) + 0x9e3779b9 + (seed << 6) + (seed >> 2); ++} ++ ++template ++inline ++ typename std::enable_if::value, void>::type ++ hash_tuple(std::size_t&, const Tuple&) {} ++ ++template ++ inline typename std::enable_if < ++ Index::value, void>::type hash_tuple( ++ std::size_t& seed, const Tuple& tuple) { ++ hash_combine(seed, std::get(tuple)); ++ hash_tuple(seed, tuple); ++} ++ ++template <> ++struct std::hash { ++ std::size_t operator()(const utils::ScatterType& value) const noexcept { ++ return std::hash::type>{}( ++ static_cast::type>(value)); ++ } ++}; ++ ++template ++struct std::hash> { ++ std::size_t operator()(const std::vector& vec) const { ++ std::size_t seed = 0; ++ for (const T& elem : vec) { ++ seed ^= std::hash{}(elem) + 0x9e3779b9 + (seed << 6) + (seed >> 2); ++ } ++ return seed; ++ } ++}; ++ ++template <> ++struct std::hash { ++ std::size_t operator()(const mx::Dtype& dtype) const noexcept { ++ std::size_t seed = 0; ++ hash_combine(seed, std::hash{}(dtype.size())); ++ hash_combine(seed, ++ std::hash::type>{}( ++ static_cast::type>( ++ dtype.val()))); ++ return seed; ++ } ++}; ++ ++#endif // XLA_PJRT_PLUGIN_STABLEHLO_MLX_UTILS_H_ +\ No newline at end of file diff --git a/third_party/modules/xla/20250527.0-cb67f2f/source.json b/third_party/modules/xla/20250527.0-cb67f2f/source.json index 86bdd83a..ee8f90a3 100644 --- a/third_party/modules/xla/20250527.0-cb67f2f/source.json +++ b/third_party/modules/xla/20250527.0-cb67f2f/source.json @@ -1,17 +1,18 @@ { - "strip_prefix": "xla-cb67f2f7ce4787f63f5fc80dc5c30cd3dee8f4e3", - "url": "https://github.com/openxla/xla/archive/cb67f2f7ce4787f63f5fc80dc5c30cd3dee8f4e3.tar.gz", - "integrity": "sha256-SDAAOY6cjcCQ5e1JMob5G4+BYHk7spDb5zZEDrVeA4I=", - "overlay": { - "llvm.bzl": "", - "MODULE.bazel": "", - "workspace_private.bzl": "", - "xla.bzl": "" - }, - "patch_strip": 1, - "patches": { - "0001-bazel-migration-to-bazel-8.1.1.patch": "", - "0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch": "", - "0003-Revert-Add-optional-allowOtherDialects-field-to-stab.patch": "" - } + "strip_prefix": "xla-cb67f2f7ce4787f63f5fc80dc5c30cd3dee8f4e3", + "url": "https://github.com/openxla/xla/archive/cb67f2f7ce4787f63f5fc80dc5c30cd3dee8f4e3.tar.gz", + "integrity": "sha256-SDAAOY6cjcCQ5e1JMob5G4+BYHk7spDb5zZEDrVeA4I=", + "overlay": { + "llvm.bzl": "", + "MODULE.bazel": "", + "workspace_private.bzl": "", + "xla.bzl": "" + }, + "patch_strip": 1, + "patches": { + "0001-bazel-migration-to-bazel-8.1.1.patch": "", + "0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch": "", + "0003-Revert-Add-optional-allowOtherDialects-field-to-stab.patch": "", + "0004-Added-MLX-PjRt-Plugin.patch": "" + } } diff --git a/zml/testing.zig b/zml/testing.zig index 6583520e..4fb0d312 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -21,7 +21,8 @@ pub fn env() zml.Platform { if (!builtin.is_test) @compileError("Cannot use zml.testing.env outside of a test block"); if (_platform == null) { var ctx = zml.Context.init() catch unreachable; - _platform = ctx.autoPlatform(.{}).withCompilationOptions(.{ + var auto_platform = ctx.autoPlatform(.{}); + _platform = auto_platform.withCompilationOptions(if (auto_platform.target == .mlx) .{} else .{ .xla_dump_to = "/tmp/zml/tests/", .sharding_enabled = true, }); From 4a36b961fa93bdc22719c2579fd8a602279b27fb Mon Sep 17 00:00:00 2001 From: James Deal Date: Wed, 4 Jun 2025 23:31:47 -0500 Subject: [PATCH 09/13] Works to compile llama; pending optimization --- .../patches/0004-Added-MLX-PjRt-Plugin.patch | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/third_party/modules/xla/20250527.0-cb67f2f/patches/0004-Added-MLX-PjRt-Plugin.patch b/third_party/modules/xla/20250527.0-cb67f2f/patches/0004-Added-MLX-PjRt-Plugin.patch index ca883653..00156e7e 100644 --- a/third_party/modules/xla/20250527.0-cb67f2f/patches/0004-Added-MLX-PjRt-Plugin.patch +++ b/third_party/modules/xla/20250527.0-cb67f2f/patches/0004-Added-MLX-PjRt-Plugin.patch @@ -1,9 +1,9 @@ diff --git a/xla/pjrt/plugin/mlx/BUILD b/xla/pjrt/plugin/mlx/BUILD new file mode 100644 -index 0000000000..f7417e1e4b +index 0000000000..94bc83ea41 --- /dev/null +++ b/xla/pjrt/plugin/mlx/BUILD -@@ -0,0 +1,201 @@ +@@ -0,0 +1,205 @@ +load("//xla:xla.default.bzl", "xla_cc_binary", "xla_cc_test") + +package( @@ -54,8 +54,8 @@ index 0000000000..f7417e1e4b + "//xla/mlir/utils:error_util", + "//xla/mlir_hlo:mhlo_passes", + "//xla/mlir_hlo:stablehlo_extension_passes", -+ # "//xla/mlir/utils:type_util", -+ "//xla/pjrt:mlir_to_hlo", ++ "//xla/mlir/utils:type_util", ++ # "//xla/pjrt:mlir_to_hlo", + "//xla/pjrt:pjrt_client", + "//xla/pjrt:pjrt_executable", + "//xla/pjrt:pjrt_future", @@ -201,6 +201,10 @@ index 0000000000..f7417e1e4b +xla_cc_binary( + name = "stablehlo_mlx_plugin.so", + linkshared = True, ++ linkopts = [ ++ "-Wl,-exported_symbol,_GetPjrtApi", ++ "-Wl,-undefined,error", ++ ], + deps = [ + ":client_c_pjrt", + ], From 1d6ffc5a187d8acb7af8ae637d14789641ce5bec Mon Sep 17 00:00:00 2001 From: James Deal Date: Mon, 16 Jun 2025 23:49:08 -0500 Subject: [PATCH 10/13] Vendor Plugin performance fixes --- MODULE.bazel | 9 + examples/MODULE.bazel | 5 + .../mlx/20250605.0-c763fe1/MODULE.bazel | 10 + .../20250605.0-c763fe1/overlay/BUILD.bazel | 597 ++++++++ .../20250605.0-c763fe1/overlay/MODULE.bazel | 1 + .../mlx/20250605.0-c763fe1/overlay/jit.bzl | 28 + .../20250605.0-c763fe1/overlay/metal_cpp.bzl | 23 + ...1-Modify-Sort-And-Comparison-Kernels.patch | 1221 +++++++++++++++++ .../mlx/20250605.0-c763fe1/source.json | 15 + third_party/modules/mlx/metadata.json | 3 +- .../xla/20250527.0-cb67f2f/MODULE.bazel | 2 +- .../patches/0004-Added-MLX-PjRt-Plugin.patch | 104 +- 12 files changed, 1971 insertions(+), 47 deletions(-) create mode 100644 third_party/modules/mlx/20250605.0-c763fe1/MODULE.bazel create mode 100644 third_party/modules/mlx/20250605.0-c763fe1/overlay/BUILD.bazel create mode 120000 third_party/modules/mlx/20250605.0-c763fe1/overlay/MODULE.bazel create mode 100644 third_party/modules/mlx/20250605.0-c763fe1/overlay/jit.bzl create mode 100644 third_party/modules/mlx/20250605.0-c763fe1/overlay/metal_cpp.bzl create mode 100644 third_party/modules/mlx/20250605.0-c763fe1/patches/0001-Modify-Sort-And-Comparison-Kernels.patch create mode 100644 third_party/modules/mlx/20250605.0-c763fe1/source.json diff --git a/MODULE.bazel b/MODULE.bazel index 61572c88..8e2faf42 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -1,6 +1,15 @@ module( name = "zml", ) +# local_path_override( +# module_name = "xla", +# path = "../xla", +# ) + +# local_path_override( +# module_name = "mlx", +# path = "../mlx_master", +# ) bazel_dep(name = "abseil-cpp", version = "20240722.0.bcr.2") bazel_dep(name = "aspect_bazel_lib", version = "2.14.0") diff --git a/examples/MODULE.bazel b/examples/MODULE.bazel index d7c2c910..4ed306cc 100644 --- a/examples/MODULE.bazel +++ b/examples/MODULE.bazel @@ -5,6 +5,11 @@ module(name = "examples") # path = "../../xla", # ) +# local_path_override( +# module_name = "mlx", +# path = "../../mlx_master", +# ) + bazel_dep(name = "aspect_bazel_lib", version = "2.14.0") bazel_dep(name = "bazel_skylib", version = "1.7.1") bazel_dep(name = "platforms", version = "0.0.11") diff --git a/third_party/modules/mlx/20250605.0-c763fe1/MODULE.bazel b/third_party/modules/mlx/20250605.0-c763fe1/MODULE.bazel new file mode 100644 index 00000000..c81ba37f --- /dev/null +++ b/third_party/modules/mlx/20250605.0-c763fe1/MODULE.bazel @@ -0,0 +1,10 @@ +module( + name = "mlx", + version = "20250605.0-c763fe1", + compatibility_level = 1, +) + +bazel_dep(name = "rules_apple", version = "3.13.0", repo_name = "build_bazel_rules_apple") +bazel_dep(name = "fmt", version = "11.0.2") +metal_cpp = use_extension("metal_cpp.bzl", "metal_cpp") +use_repo(metal_cpp, "metal_cpp") diff --git a/third_party/modules/mlx/20250605.0-c763fe1/overlay/BUILD.bazel b/third_party/modules/mlx/20250605.0-c763fe1/overlay/BUILD.bazel new file mode 100644 index 00000000..1ea75697 --- /dev/null +++ b/third_party/modules/mlx/20250605.0-c763fe1/overlay/BUILD.bazel @@ -0,0 +1,597 @@ +load("@build_bazel_rules_apple//apple:resources.bzl", "apple_metal_library") +load("//:jit.bzl", "make_jit_source") + +filegroup( + name = "mlx_h", + srcs = glob(["mlx/*.h"], ["mlx/mlx.h"]), +) + +# Contains `mlx/types/*.h` headers +cc_library( + name = "types", + hdrs = glob(["mlx/types/*.h"]), +) + +cc_library( + name = "pocketfft", + hdrs = [ + "mlx/3rdparty/pocketfft.h", + ], +) + +# mlx/distributed/ring/CMakeLists.txt +cc_library( + name = "ring", + srcs = [ + "mlx/distributed/ring/no_ring.cpp", + + # Other sources (maybe reorganize dependencies?) + ":mlx_h", + "mlx/distributed/distributed.h", + ], + hdrs = [ + "mlx/distributed/ring/ring.h", + ], + deps = [ + ":types" + ], +) + +# mlx/distributed/mpi/CMakeLists.txt +cc_library( + name = "mpi", + srcs = [ + "mlx/distributed/mpi/no_mpi.cpp", + # Other sources (maybe reorganize dependencies?) + "mlx/distributed/distributed.h", + ":mlx_h", + ], + hdrs = [ + "mlx/distributed/mpi/mpi.h", + ], + deps = [ + ":types", + ] +) + +# mlx/distributed/CMakeLists.txt +cc_library( + name = "distributed", + srcs = [ + "mlx/distributed/primitives.cpp", + "mlx/distributed/ops.cpp", + "mlx/distributed/distributed.cpp", + + # Other sources (maybe reorganize dependencies?) + ":mlx_h", + ], + hdrs = [ + "mlx/distributed/distributed_impl.h", + "mlx/distributed/distributed.h", + "mlx/distributed/ops.h", + "mlx/distributed/primitives.h", + ], + deps = [ + ":io", + ":mpi", + ":ring", + ":types", + ] +) + +# mlx/io/CMakeLists.txt +cc_library( + name = "io", + srcs = [ + "mlx/io/load.cpp", + "mlx/io/no_safetensors.cpp", + "mlx/io/no_gguf.cpp", + + # Other sources (maybe reorganize dependencies?) + ":mlx_h", + ], + hdrs = [ + "mlx/io/load.h", + ], + deps = [ + ":types", + ], +) + +# mlx/backend/common/CMakeLists.txt +cc_library( + name = "backend_common", + srcs = [ + "mlx/backend/common/broadcasting.cpp", + "mlx/backend/common/compiled.cpp", + "mlx/backend/common/common.cpp", + "mlx/backend/common/load.cpp", + "mlx/backend/common/reduce.cpp", + "mlx/backend/common/slicing.cpp", + "mlx/backend/common/utils.cpp", + # Other sources (maybe reorganize dependencies?) + "mlx/backend/gpu/eval.h", + ":mlx_h", + ], + hdrs = [ + "mlx/backend/common/binary.h", + "mlx/backend/common/broadcasting.h", + "mlx/backend/common/buffer_cache.h", + "mlx/backend/common/compiled.h", + "mlx/backend/common/copy.h", + "mlx/backend/common/hadamard.h", + "mlx/backend/common/reduce.h", + "mlx/backend/common/slicing.h", + "mlx/backend/common/ternary.h", + "mlx/backend/common/utils.h", + ], + deps = [ + + ":io", + ":types", + ], +) + +# mlx/backend/cpu/CMakeLists.txt +genrule( + name = "compiled_preamble", + visibility = ["//visibility:public"], + srcs = ["mlx/backend/cpu/compiled_preamble.h"], + outs = ["mlx/backend/cpu/compiled_preamable.cpp"], + cmd = "bash $(location :mlx/backend/cpu/make_compiled_preamble.sh) $@ $(location mlx/backend/cpu/compiled_preamble.h)", + tools = [":mlx/backend/cpu/make_compiled_preamble.sh"], +) + +# TODO (@cryptodeal): finish bazel config for cpu backend +cc_library( + name = "backend_cpu", + linkopts = [ + "-framework", + "Accelerate", + ], + defines = [ + "MLX_USE_ACCELERATE", + "ACCELERATE_NEW_LAPACK" + ], + srcs = [ + "mlx/backend/cpu/arg_reduce.cpp", + "mlx/backend/cpu/binary.cpp", + "mlx/backend/cpu/conv.cpp", + "mlx/backend/cpu/copy.cpp", + "mlx/backend/cpu/eigh.cpp", + "mlx/backend/cpu/fft.cpp", + "mlx/backend/cpu/hadamard.cpp", + "mlx/backend/cpu/matmul.cpp", + "mlx/backend/cpu/gemms/cblas.cpp", + "mlx/backend/cpu/masked_mm.cpp", + "mlx/backend/cpu/primitives.cpp", + "mlx/backend/cpu/quantized.cpp", + "mlx/backend/cpu/reduce.cpp", + "mlx/backend/cpu/scan.cpp", + "mlx/backend/cpu/select.cpp", + "mlx/backend/cpu/softmax.cpp", + "mlx/backend/cpu/sort.cpp", + "mlx/backend/cpu/threefry.cpp", + "mlx/backend/cpu/indexing.cpp", + "mlx/backend/cpu/luf.cpp", + "mlx/backend/cpu/qrf.cpp", + "mlx/backend/cpu/svd.cpp", + "mlx/backend/cpu/inverse.cpp", + "mlx/backend/cpu/cholesky.cpp", + "mlx/backend/cpu/unary.cpp", + # Build with Accelerate + "mlx/backend/cpu/gemms/bnns.cpp", + # Not Building for IOS + "mlx/backend/cpu/compiled.cpp", + "mlx/backend/cpu/jit_compiler.cpp", + # Additional Header Sources + "mlx/backend/cpu/simd/simd.h", + "mlx/backend/cpu/simd/math.h", + "mlx/backend/cpu/simd/type.h", + "mlx/backend/cpu/simd/base_simd.h", + "mlx/backend/cpu/simd/accelerate_simd.h", + "mlx/backend/cpu/simd/accelerate_fp16_simd.h", + # generated `compiled_preamble.cpp` + ":compiled_preamble", + + # Other sources (maybe reorganize dependencies?) + ":mlx_h", + "mlx/io/load.h", + ], + hdrs = [ + "mlx/backend/cpu/arange.h", + "mlx/backend/cpu/binary_ops.h", + "mlx/backend/cpu/binary_two.h", + "mlx/backend/cpu/binary.h", + "mlx/backend/cpu/compiled_preamble.h", + "mlx/backend/cpu/copy.h", + "mlx/backend/cpu/gemm.h", + "mlx/backend/cpu/jit_compiler.h", + "mlx/backend/cpu/lapack.h", + "mlx/backend/cpu/slicing.h", + "mlx/backend/cpu/ternary.h", + "mlx/backend/cpu/threefry.h", + "mlx/backend/cpu/unary_ops.h", + "mlx/backend/cpu/unary.h", + ], + deps = [ + "@fmt", + ":backend_common", + ":pocketfft", + ":types", + ], +) + +# mlx/backend/no_cpu/CMakeLists.txt +cc_library( + name = "backend_no_cpu", + srcs = [ + "mlx/backend/no_cpu/available.cpp", + "mlx/backend/no_cpu/primitives.cpp", + "mlx/backend/cpu/eval.cpp", + "mlx/backend/cpu/encoder.cpp", + "mlx/backend/no_cpu/compiled.cpp", + # Other sources (maybe reorganize dependencies?) + "mlx/backend/cpu/available.h", + "mlx/backend/cpu/encoder.h", + "mlx/backend/cpu/eval.h", + "mlx/backend/gpu/eval.h", + ":mlx_h", + ], + deps = [ + ":distributed", + ":io", + # ":backend_common", + ":types", + ], +) + +# mlx/backend/metal/CMakeLists.txt +make_jit_source( + name = "utils", + deps = [ + "mlx/backend/metal/kernels/jit/bf16.h", + "mlx/backend/metal/kernels/metal_3_0/bf16.h", + "mlx/backend/metal/kernels/metal_3_1/bf16.h", + "mlx/backend/metal/kernels/bf16_math.h", + "mlx/backend/metal/kernels/complex.h", + "mlx/backend/metal/kernels/defines.h", + ], +) + +make_jit_source( + name = "unary_ops", + deps = [ + "mlx/backend/metal/kernels/erf.h", + "mlx/backend/metal/kernels/expm1f.h", + ], +) + +make_jit_source(name = "binary_ops") + +make_jit_source(name = "ternary_ops") + +make_jit_source( + name = "reduce_utils", + deps = [ + "mlx/backend/metal/kernels/atomic.h", + "mlx/backend/metal/kernels/reduction/ops.h" + ], +) + +make_jit_source( + name = "scatter", + deps = [ + "mlx/backend/metal/kernels/indexing.h", + ], +) + +make_jit_source( + name = "gather", + deps = [ + "mlx/backend/metal/kernels/indexing.h", + ], +) + +make_jit_source(name = "gather_axis") + +make_jit_source(name = "scatter_axis") + +make_jit_source(name = "hadamard") + +cc_library( + name = "backend_metal", + linkopts = [ + "-framework Metal", + "-framework Foundation", + "-framework QuartzCore", + ], + # Use for metal debugging/traces + # local_defines = [ + # "MLX_METAL_DEBUG", + # ], + srcs = [ + "mlx/backend/metal/allocator.cpp", + "mlx/backend/metal/binary.cpp", + "mlx/backend/metal/compiled.cpp", + "mlx/backend/metal/conv.cpp", + "mlx/backend/metal/copy.cpp", + "mlx/backend/metal/custom_kernel.cpp", + "mlx/backend/metal/distributed.cpp", + "mlx/backend/metal/device.cpp", + "mlx/backend/metal/event.cpp", + "mlx/backend/metal/eval.cpp", + "mlx/backend/metal/fence.cpp", + "mlx/backend/metal/fft.cpp", + "mlx/backend/metal/hadamard.cpp", + "mlx/backend/metal/indexing.cpp", + "mlx/backend/metal/logsumexp.cpp", + "mlx/backend/metal/matmul.cpp", + "mlx/backend/metal/scaled_dot_product_attention.cpp", + "mlx/backend/metal/metal.cpp", + "mlx/backend/metal/primitives.cpp", + "mlx/backend/metal/quantized.cpp", + "mlx/backend/metal/normalization.cpp", + "mlx/backend/metal/rope.cpp", + "mlx/backend/metal/scan.cpp", + "mlx/backend/metal/slicing.cpp", + "mlx/backend/metal/softmax.cpp", + "mlx/backend/metal/sort.cpp", + "mlx/backend/metal/reduce.cpp", + "mlx/backend/metal/ternary.cpp", + "mlx/backend/metal/unary.cpp", + "mlx/backend/metal/resident.cpp", + "mlx/backend/metal/utils.cpp", + "mlx/backend/metal/nojit_kernels.cpp", + # Additional Header Sources + "mlx/backend/metal/jit/indexing.h", + "mlx/backend/metal/kernels/defines.h", + "mlx/backend/metal/kernels/steel/conv/params.h", + "mlx/backend/metal/kernels/steel/gemm/params.h", + "mlx/backend/metal/kernels/steel/attn/params.h", + "mlx/backend/metal/jit/includes.h", + # Generate METAL_PATH define + ":generate_metal_path", + # Generated sources + ":utils", + ":unary_ops", + ":binary_ops", + ":ternary_ops", + ":reduce_utils", + ":scatter", + ":gather", + ":gather_axis", + ":scatter_axis", + ":hadamard", + + # Other sources (maybe reorganize dependencies?) + ":mlx_h", + ], + hdrs = [ + "mlx/backend/metal/allocator.h", + "mlx/backend/metal/binary.h", + "mlx/backend/metal/device.h", + "mlx/backend/metal/kernels.h", + "mlx/backend/metal/matmul.h", + "mlx/backend/metal/metal.h", + "mlx/backend/metal/reduce.h", + "mlx/backend/metal/resident.h", + "mlx/backend/metal/ternary.h", + "mlx/backend/metal/unary.h", + "mlx/backend/metal/utils.h", + ], + deps = [ + "@metal_cpp//:metal_cpp", + "@fmt", + ":backend_common", + ":backend_gpu", + ":distributed", + ":pocketfft", + ], +) + +genrule( + name = "generate_metal_path", + outs = [ + "mlx/backend/metal/metal_path.h", + ], + cmd = "echo \"#define METAL_PATH \\\"`realpath $(location //:mlx_metallib)`\\\"\" > $@", + tools = [ + "//:mlx_metallib", + ], +) + +# TODO (@cryptodeal): find a better solution than `copts` includes +apple_metal_library( + name = "mlx_metallib", + visibility = ["//visibility:public"], + copts = [ + "-target", + "air64-apple-macosx15.4.1", + "-Iexternal/mlx+", + "-Iexternal/mlx+/mlx/backend/metal/kernels/metal_3_1", + # "-I.", + # "-Imlx/backend/metal/kernels/metal_3_1", + "-Wall", + "-Wextra", + "-fno-fast-math", + "-Wno-c++17-extensions", + # Need for Metal GPU trace + # "-gline-tables-only", + # "-frecord-sources", + ], + srcs = [ + "mlx/backend/metal/kernels/arg_reduce.metal", + "mlx/backend/metal/kernels/conv.metal", + "mlx/backend/metal/kernels/gemv.metal", + "mlx/backend/metal/kernels/layer_norm.metal", + "mlx/backend/metal/kernels/random.metal", + "mlx/backend/metal/kernels/rms_norm.metal", + "mlx/backend/metal/kernels/rope.metal", + "mlx/backend/metal/kernels/scaled_dot_product_attention.metal", + "mlx/backend/metal/kernels/fence.metal", + "mlx/backend/metal/kernels/steel/attn/kernels/steel_attention.metal", + # Disable JIT kernels (for default ops) + "mlx/backend/metal/kernels/arange.metal", + "mlx/backend/metal/kernels/binary.metal", + "mlx/backend/metal/kernels/binary_two.metal", + "mlx/backend/metal/kernels/copy.metal", + "mlx/backend/metal/kernels/fft.metal", + "mlx/backend/metal/kernels/reduce.metal", + "mlx/backend/metal/kernels/quantized.metal", + "mlx/backend/metal/kernels/scan.metal", + "mlx/backend/metal/kernels/softmax.metal", + "mlx/backend/metal/kernels/sort.metal", + "mlx/backend/metal/kernels/ternary.metal", + "mlx/backend/metal/kernels/unary.metal", + "mlx/backend/metal/kernels/steel/conv/kernels/steel_conv.metal", + "mlx/backend/metal/kernels/steel/conv/kernels/steel_conv_general.metal", + "mlx/backend/metal/kernels/steel/gemm/kernels/steel_gemm_fused.metal", + "mlx/backend/metal/kernels/steel/gemm/kernels/steel_gemm_masked.metal", + "mlx/backend/metal/kernels/steel/gemm/kernels/steel_gemm_splitk.metal", + "mlx/backend/metal/kernels/gemv_masked.metal", + ], + hdrs = [ + "mlx/backend/metal/kernels/metal_3_1/bf16.h", + "mlx/backend/metal/kernels/metal_3_0/bf16.h", + "mlx/backend/metal/kernels/bf16_math.h", + "mlx/backend/metal/kernels/complex.h", + "mlx/backend/metal/kernels/defines.h", + "mlx/backend/metal/kernels/erf.h", + "mlx/backend/metal/kernels/expm1f.h", + "mlx/backend/metal/kernels/utils.h", + # `conv` headers + "mlx/backend/metal/kernels/steel/conv/params.h", + # `scaled_dot_product_attention` headers + "mlx/backend/metal/kernels/sdpa_vector.h", + # Steel Headers + "mlx/backend/metal/kernels/steel/defines.h", + "mlx/backend/metal/kernels/steel/utils.h", + "mlx/backend/metal/kernels/steel/conv/conv.h", + "mlx/backend/metal/kernels/steel/conv/loader.h", + "mlx/backend/metal/kernels/steel/conv/loaders/loader_channel_l.h", + "mlx/backend/metal/kernels/steel/conv/loaders/loader_channel_n.h", + "mlx/backend/metal/kernels/steel/conv/loaders/loader_general.h", + "mlx/backend/metal/kernels/steel/conv/kernels/steel_conv.h", + "mlx/backend/metal/kernels/steel/conv/kernels/steel_conv_general.h", + "mlx/backend/metal/kernels/steel/gemm/gemm.h", + "mlx/backend/metal/kernels/steel/gemm/mma.h", + "mlx/backend/metal/kernels/steel/gemm/loader.h", + "mlx/backend/metal/kernels/steel/gemm/params.h", + "mlx/backend/metal/kernels/steel/gemm/transforms.h", + "mlx/backend/metal/kernels/steel/gemm/kernels/steel_gemm_fused.h", + "mlx/backend/metal/kernels/steel/gemm/kernels/steel_gemm_masked.h", + "mlx/backend/metal/kernels/steel/gemm/kernels/steel_gemm_splitk.h", + "mlx/backend/metal/kernels/steel/utils/type_traits.h", + "mlx/backend/metal/kernels/steel/utils/integral_constant.h", + # Steel Attn Headers (not repeating previously listed) + "mlx/backend/metal/kernels/steel/attn/attn.h", + "mlx/backend/metal/kernels/steel/attn/loader.h", + "mlx/backend/metal/kernels/steel/attn/mma.h", + "mlx/backend/metal/kernels/steel/attn/params.h", + "mlx/backend/metal/kernels/steel/attn/transforms.h", + "mlx/backend/metal/kernels/steel/attn/kernels/steel_attention.h", + # `arange` headers + "mlx/backend/metal/kernels/arange.h", + # `binary` headers + "mlx/backend/metal/kernels/binary.h", + "mlx/backend/metal/kernels/binary_ops.h", + # `binary_two` headers + "mlx/backend/metal/kernels/binary_two.h", + # `copy` headers + "mlx/backend/metal/kernels/copy.h", + # `fft` headers + "mlx/backend/metal/kernels/fft.h", + "mlx/backend/metal/kernels/fft/radix.h", + "mlx/backend/metal/kernels/fft/readwrite.h", + # `reduce` headers + "mlx/backend/metal/kernels/reduce.h", + "mlx/backend/metal/kernels/atomic.h", + "mlx/backend/metal/kernels/reduction/ops.h", + "mlx/backend/metal/kernels/reduction/reduce_init.h", + "mlx/backend/metal/kernels/reduction/reduce_all.h", + "mlx/backend/metal/kernels/reduction/reduce_col.h", + "mlx/backend/metal/kernels/reduction/reduce_row.h", + # `quantized` headers + "mlx/backend/metal/kernels/quantized.h", + # `scan` headers + "mlx/backend/metal/kernels/scan.h", + # `softmax` headers + "mlx/backend/metal/kernels/softmax.h", + # `sort` headers + "mlx/backend/metal/kernels/sort.h", + # `ternary` headers + "mlx/backend/metal/kernels/ternary.h", + "mlx/backend/metal/kernels/ternary_ops.h", + # `unary` headers + "mlx/backend/metal/kernels/unary.h", + "mlx/backend/metal/kernels/unary_ops.h", + # `gemv` headers + "mlx/backend/metal/kernels/gemv_masked.h", + ], + out = "mlx.metallib", +) + +cc_library( + name = "backend_gpu", + srcs = [ + "mlx/backend/gpu/copy.cpp", + "mlx/backend/gpu/primitives.cpp", + "mlx/backend/gpu/slicing.cpp", + # Other sources (maybe reorganize dependencies?) + ":mlx_h", + ], + hdrs = [ + "mlx/backend/gpu/available.h", + "mlx/backend/gpu/copy.h", + "mlx/backend/gpu/eval.h", + "mlx/backend/gpu/slicing.h", + ], + deps = [ + ":backend_common", + ], +) + +cc_library( + name = "mlx", + visibility = ["//visibility:public"], + data = [ + ":mlx_metallib", + ], + srcs = [ + "mlx/allocator.cpp", + "mlx/array.cpp", + "mlx/compile.cpp", + "mlx/device.cpp", + "mlx/dtype.cpp", + "mlx/dtype_utils.cpp", + "mlx/export.cpp", + "mlx/einsum.cpp", + "mlx/fast.cpp", + "mlx/fft.cpp", + "mlx/ops.cpp", + "mlx/graph_utils.cpp", + "mlx/primitives.cpp", + "mlx/random.cpp", + "mlx/scheduler.cpp", + "mlx/transforms.cpp", + "mlx/utils.cpp", + "mlx/linalg.cpp", + "mlx/backend/metal/metal.h", + ":mlx_h", + ], + hdrs = [ + "mlx/mlx.h", + ], + deps = [ + ":io", + ":distributed", + ":backend_common", + ":backend_metal", + ":backend_gpu", + ":backend_no_cpu", + # TODO (@cryptodeal): enable cpu backend once changes to `sort`, and + # comparison ops have been made for parity with metal backend changes. + # ":backend_cpu", + ], +) diff --git a/third_party/modules/mlx/20250605.0-c763fe1/overlay/MODULE.bazel b/third_party/modules/mlx/20250605.0-c763fe1/overlay/MODULE.bazel new file mode 120000 index 00000000..9b599e3a --- /dev/null +++ b/third_party/modules/mlx/20250605.0-c763fe1/overlay/MODULE.bazel @@ -0,0 +1 @@ +../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/mlx/20250605.0-c763fe1/overlay/jit.bzl b/third_party/modules/mlx/20250605.0-c763fe1/overlay/jit.bzl new file mode 100644 index 00000000..ca877e9b --- /dev/null +++ b/third_party/modules/mlx/20250605.0-c763fe1/overlay/jit.bzl @@ -0,0 +1,28 @@ +def make_jit_source(name, deps = [], **kwargs): + """ + Bazel equivalent of the CMake make_jit_source function. + Generates a C++ source file from a Metal header file by running a script. + + Args: + name: The Metal header file to process. + deps: Additional dependencies required for processing. + kwargs: Additional arguments to pass to the native.genrule function. + """ + src_name = name.split("/")[-1] # Extract the filename + output_file = "mlx/backend/metal/jit/{}.cpp".format(src_name) + + native.genrule( + name = name, + srcs = ["mlx/backend/metal/kernels/{}.h".format(src_name)] + deps, + outs = [output_file], + cmd = ( + "bash $(location :mlx/backend/metal/make_compiled_preamble.sh) $@ $(location mlx/backend/metal/kernels/{}.h)" + ).format(name), + tools = [":mlx/backend/metal/make_compiled_preamble.sh"], + **kwargs, + ) + + # native.cc_library( + # name = name + "_jit_lib", + # srcs = [output_file], + # ) diff --git a/third_party/modules/mlx/20250605.0-c763fe1/overlay/metal_cpp.bzl b/third_party/modules/mlx/20250605.0-c763fe1/overlay/metal_cpp.bzl new file mode 100644 index 00000000..64874afe --- /dev/null +++ b/third_party/modules/mlx/20250605.0-c763fe1/overlay/metal_cpp.bzl @@ -0,0 +1,23 @@ +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +def _metal_cpp_impl(mctx): + http_archive( + name = "metal_cpp", + build_file_content = """\ +cc_library( + name = "metal_cpp", + hdrs = glob(["metal-cpp/**/*.hpp"]), + strip_include_prefix = "metal-cpp", + visibility = ["//visibility:public"], +) +""", + sha256 = "0433df1e0ab13c2b0becbd78665071e3fa28381e9714a3fce28a497892b8a184", + url = "https://developer.apple.com/metal/cpp/files/metal-cpp_macOS15_iOS18.zip", + ) + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = "all", + root_module_direct_dev_deps = [], + ) +metal_cpp = module_extension( + implementation = _metal_cpp_impl, +) diff --git a/third_party/modules/mlx/20250605.0-c763fe1/patches/0001-Modify-Sort-And-Comparison-Kernels.patch b/third_party/modules/mlx/20250605.0-c763fe1/patches/0001-Modify-Sort-And-Comparison-Kernels.patch new file mode 100644 index 00000000..823a14db --- /dev/null +++ b/third_party/modules/mlx/20250605.0-c763fe1/patches/0001-Modify-Sort-And-Comparison-Kernels.patch @@ -0,0 +1,1221 @@ +diff --git a/.bazelrc b/.bazelrc +new file mode 100644 +index 000000000..562cc0ce7 +--- /dev/null ++++ b/.bazelrc +@@ -0,0 +1,6 @@ ++build --cxxopt='-std=c++17' ++ ++# Sets the MLX debug flag (for capturing Metal traces) ++# build --define=MLX_METAL_DEBUG=ON ++# run --define=MLX_METAL_DEBUG=ON ++ +diff --git a/.bazelversion b/.bazelversion +new file mode 100644 +index 000000000..2b0aa2121 +--- /dev/null ++++ b/.bazelversion +@@ -0,0 +1 @@ ++8.2.1 +diff --git a/.gitignore b/.gitignore +index 43629548d..0b74c60d8 100644 +--- a/.gitignore ++++ b/.gitignore +@@ -1,3 +1,8 @@ ++# Bazel symlinks ++bazel-* ++ ++# macOS ++.DS_Store + # Byte-compiled / optimized / DLL files + __pycache__/ + *.py[cod] +diff --git a/examples/cpp/BUILD.bazel b/examples/cpp/BUILD.bazel +new file mode 100644 +index 000000000..28ed3894b +--- /dev/null ++++ b/examples/cpp/BUILD.bazel +@@ -0,0 +1,23 @@ ++cc_binary( ++ name = "tutorial", ++ srcs = ["tutorial.cpp"], ++ deps = ["@mlx//:mlx"] ++) ++ ++cc_binary( ++ name = "linear_regression", ++ srcs = [ ++ "linear_regression.cpp", ++ "timer.h", ++ ], ++ deps = ["@mlx//:mlx"] ++) ++ ++cc_binary( ++ name = "logistic_regression", ++ srcs = [ ++ "linear_regression.cpp", ++ "timer.h", ++ ], ++ deps = ["@mlx//:mlx"] ++) +diff --git a/mlx/backend/cpu/make_compiled_preamble.sh b/mlx/backend/cpu/make_compiled_preamble.sh +index 88b4c4615..ca567d83e 100644 +--- a/mlx/backend/cpu/make_compiled_preamble.sh ++++ b/mlx/backend/cpu/make_compiled_preamble.sh +@@ -7,13 +7,17 @@ + + + OUTPUT_FILE=$1 +-GCC=$2 +-SRCDIR=$3 +-CLANG=$4 +-ARCH=$5 ++HDR_FILE=$(realpath $2) + +-if [ "$CLANG" = "TRUE" ]; then +- read -r -d '' INCLUDES <<- EOM ++ARCH=$(uname -m) ++ ++CPUDIR=$(dirname $HDR_FILE) ++# echo "CPUDIR: $CPUDIR" ++SRCDIR=$(echo "$CPUDIR" | cut -d'/' -f-$(($(echo "$CPUDIR" | awk -F'/' '{print NF}')-3))) ++# echo "SRCDIR: $SRCDIR" ++ ++ ++read -r -d '' INCLUDES <<- EOM + #include + #include + #include +@@ -23,11 +27,8 @@ if [ "$CLANG" = "TRUE" ]; then + #endif + EOM + CC_FLAGS="-arch ${ARCH} -nobuiltininc -nostdinc" +-else +-CC_FLAGS="-std=c++17" +-fi + +-CONTENT=$($GCC $CC_FLAGS -I "$SRCDIR" -E -P "$SRCDIR/mlx/backend/cpu/compiled_preamble.h" 2>/dev/null) ++CONTENT=$(clang $CC_FLAGS -I "$SRCDIR" -E -P "$HDR_FILE" 2>/dev/null) + + cat << EOF > "$OUTPUT_FILE" + const char* get_kernel_preamble() { +diff --git a/mlx/backend/metal/device.cpp b/mlx/backend/metal/device.cpp +index ebc3cc77f..7bdca47d7 100644 +--- a/mlx/backend/metal/device.cpp ++++ b/mlx/backend/metal/device.cpp +@@ -10,6 +10,7 @@ + + #include "mlx/backend/metal/device.h" + #include "mlx/backend/metal/metal.h" ++#include "mlx/backend/metal/metal_path.h" + #include "mlx/backend/metal/utils.h" + #include "mlx/utils.h" + +diff --git a/mlx/backend/metal/jit_kernels.cpp b/mlx/backend/metal/jit_kernels.cpp +index 5206c9b54..6678e7acb 100644 +--- a/mlx/backend/metal/jit_kernels.cpp ++++ b/mlx/backend/metal/jit_kernels.cpp +@@ -328,7 +328,8 @@ MTL::ComputePipelineState* get_sort_kernel( + const array& in, + const array& out, + int bn, +- int tn) { ++ int tn, ++ const std::string& comparator) { + std::string lib_name = kernel_name.substr(kernel_name.find("_") + 1); + auto lib = d.get_library(lib_name, [&]() { + std::ostringstream kernel_source; +@@ -345,7 +346,8 @@ MTL::ComputePipelineState* get_sort_kernel( + out_type, + bool_string, + bn, +- tn); ++ tn, ++ comparator); + kernel_source << get_template_definition( + "n" + func_string + lib_name, + "block_sort_nc", +@@ -353,7 +355,8 @@ MTL::ComputePipelineState* get_sort_kernel( + out_type, + bool_string, + bn, +- tn); ++ tn, ++ comparator); + } + return kernel_source.str(); + }); +@@ -366,7 +369,8 @@ MTL::ComputePipelineState* get_mb_sort_kernel( + const array& in, + const array& idx, + int bn, +- int tn) { ++ int tn, ++ const std::string& comparator) { + std::string lib_name = kernel_name.substr(kernel_name.find("_") + 1); + auto lib = d.get_library(lib_name, [&]() { + std::ostringstream kernel_source; +@@ -383,7 +387,8 @@ MTL::ComputePipelineState* get_mb_sort_kernel( + get_type_string(idx.dtype()), + "true", + bn, +- tn); ++ tn, ++ comparator); + } + return kernel_source.str(); + }); +diff --git a/mlx/backend/metal/kernels.h b/mlx/backend/metal/kernels.h +index 6d8864385..ff2dd14be 100644 +--- a/mlx/backend/metal/kernels.h ++++ b/mlx/backend/metal/kernels.h +@@ -79,7 +79,8 @@ MTL::ComputePipelineState* get_sort_kernel( + const array& in, + const array& out, + int bn, +- int tn); ++ int tn, ++ const std::string& comparator); + + MTL::ComputePipelineState* get_mb_sort_kernel( + metal::Device& d, +@@ -87,7 +88,8 @@ MTL::ComputePipelineState* get_mb_sort_kernel( + const array& in, + const array& idx, + int bn, +- int tn); ++ int tn, ++ const std::string& comparator); + + MTL::ComputePipelineState* get_reduce_init_kernel( + metal::Device& d, +diff --git a/mlx/backend/metal/kernels/binary_ops.h b/mlx/backend/metal/kernels/binary_ops.h +index 4aaf2b4da..2c6e27b18 100644 +--- a/mlx/backend/metal/kernels/binary_ops.h ++++ b/mlx/backend/metal/kernels/binary_ops.h +@@ -37,29 +37,14 @@ struct Divide { + return x / y; + } + }; +- + struct Remainder { + template +- metal::enable_if_t & !metal::is_signed_v, T> +- operator()(T x, T y) { ++ metal::enable_if_t, T> operator()(T x, T y) { + return x % y; + } + template +- metal::enable_if_t & metal::is_signed_v, T> +- operator()(T x, T y) { +- auto r = x % y; +- if (r != 0 && (r < 0 != y < 0)) { +- r += y; +- } +- return r; +- } +- template + metal::enable_if_t, T> operator()(T x, T y) { +- T r = fmod(x, y); +- if (r != 0 && (r < 0 != y < 0)) { +- r += y; +- } +- return r; ++ return fmod(x, y); + } + template <> + complex64_t operator()(complex64_t x, complex64_t y) { +diff --git a/mlx/backend/metal/kernels/complex.h b/mlx/backend/metal/kernels/complex.h +index c88002cb3..54fc8d520 100644 +--- a/mlx/backend/metal/kernels/complex.h ++++ b/mlx/backend/metal/kernels/complex.h +@@ -142,11 +142,12 @@ constexpr complex64_t operator/(float a, complex64_t b) { + constexpr complex64_t operator%(complex64_t a, complex64_t b) { + auto real = a.real - (b.real * static_cast(a.real / b.real)); + auto imag = a.imag - (b.imag * static_cast(a.imag / b.imag)); +- if (real != 0 && (real < 0 != b.real < 0)) { +- real += b.real; +- } +- if (imag != 0 && (imag < 0 != b.imag < 0)) { +- imag += b.imag; +- } ++ /* Disable to match `stablehlo` implementation semantics */ ++ // if (real != 0 && (real < 0 != b.real < 0)) { ++ // real += b.real; ++ // } ++ // if (imag != 0 && (imag < 0 != b.imag < 0)) { ++ // imag += b.imag; ++ // } + return {real, imag}; + } +diff --git a/mlx/backend/metal/kernels/sort.h b/mlx/backend/metal/kernels/sort.h +index b067150d8..a24e429fd 100644 +--- a/mlx/backend/metal/kernels/sort.h ++++ b/mlx/backend/metal/kernels/sort.h +@@ -28,6 +28,15 @@ struct LessThan { + } + }; + ++template ++struct GreaterThan { ++ static constexpr constant T init = Limits::min; ++ ++ METAL_FUNC bool operator()(T a, T b) { ++ return a > b; ++ } ++}; ++ + template < + typename ValT, + typename IdxT, +@@ -216,7 +225,7 @@ template < + bool ARG_SORT, + short BLOCK_THREADS, + short N_PER_THREAD, +- typename CompareOp = LessThan> ++ typename CompareOp> + struct KernelMergeSort { + using ValT = T; + using IdxT = uint; +@@ -278,7 +287,8 @@ template < + typename U, + bool ARG_SORT, + short BLOCK_THREADS, +- short N_PER_THREAD> ++ short N_PER_THREAD, ++ typename CompareOp = LessThan> + [[kernel, max_total_threads_per_threadgroup(BLOCK_THREADS)]] void block_sort( + const device T* inp [[buffer(0)]], + device U* out [[buffer(1)]], +@@ -290,7 +300,7 @@ template < + uint3 tid [[threadgroup_position_in_grid]], + uint3 lid [[thread_position_in_threadgroup]]) { + using sort_kernel = +- KernelMergeSort; ++ KernelMergeSort; + using ValT = typename sort_kernel::ValT; + using IdxT = typename sort_kernel::IdxT; + +@@ -333,7 +343,8 @@ template < + typename U, + bool ARG_SORT, + short BLOCK_THREADS, +- short N_PER_THREAD> ++ short N_PER_THREAD, ++ typename CompareOp> + [[kernel, max_total_threads_per_threadgroup(BLOCK_THREADS)]] void block_sort_nc( + const device T* inp [[buffer(0)]], + device U* out [[buffer(1)]], +@@ -347,7 +358,7 @@ template < + uint3 tid [[threadgroup_position_in_grid]], + uint3 lid [[thread_position_in_threadgroup]]) { + using sort_kernel = +- KernelMergeSort; ++ KernelMergeSort; + using ValT = typename sort_kernel::ValT; + using IdxT = typename sort_kernel::IdxT; + +@@ -394,7 +405,7 @@ template < + bool ARG_SORT, + short BLOCK_THREADS, + short N_PER_THREAD, +- typename CompareOp = LessThan> ++ typename CompareOp> + struct KernelMultiBlockMergeSort { + using block_merge_sort_t = BlockMergeSort< + ValT, +@@ -476,7 +487,8 @@ template < + typename IdxT, + bool ARG_SORT, + short BLOCK_THREADS, +- short N_PER_THREAD> ++ short N_PER_THREAD, ++ typename CompareOp> + [[kernel, max_total_threads_per_threadgroup(BLOCK_THREADS)]] void mb_block_sort( + const device ValT* inp [[buffer(0)]], + device ValT* out_vals [[buffer(1)]], +@@ -493,7 +505,8 @@ template < + IdxT, + ARG_SORT, + BLOCK_THREADS, +- N_PER_THREAD>; ++ N_PER_THREAD, ++ CompareOp>; + + auto block_idx = elem_to_loc(tid.y, nc_shape, nc_strides, nc_dim); + inp += block_idx; +@@ -520,7 +533,8 @@ template < + typename IdxT, + bool ARG_SORT, + short BLOCK_THREADS, +- short N_PER_THREAD> ++ short N_PER_THREAD, ++ typename CompareOp> + [[kernel]] void mb_block_partition( + device IdxT* block_partitions [[buffer(0)]], + const device ValT* dev_vals [[buffer(1)]], +@@ -536,7 +550,8 @@ template < + IdxT, + ARG_SORT, + BLOCK_THREADS, +- N_PER_THREAD>; ++ N_PER_THREAD, ++ CompareOp>; + + block_partitions += tid.y * tgp_dims.x; + dev_vals += tid.y * size_sorted_axis; +@@ -573,7 +588,7 @@ template < + bool ARG_SORT, + short BLOCK_THREADS, + short N_PER_THREAD, +- typename CompareOp = LessThan> ++ typename CompareOp> + [[kernel, max_total_threads_per_threadgroup(BLOCK_THREADS)]] void + mb_block_merge( + const device IdxT* block_partitions [[buffer(0)]], +diff --git a/mlx/backend/metal/kernels/sort.metal b/mlx/backend/metal/kernels/sort.metal +index 7f198d55d..f1ec796c9 100644 +--- a/mlx/backend/metal/kernels/sort.metal ++++ b/mlx/backend/metal/kernels/sort.metal +@@ -6,24 +6,29 @@ + #include "mlx/backend/metal/kernels/utils.h" + #include "mlx/backend/metal/kernels/sort.h" + ++#define instantiate_comparator(name, type) \ ++ name ++ + #define instantiate_block_sort( \ +- name, itname, itype, otname, otype, arg_sort, bn, tn) \ +- instantiate_kernel("c" #name "_" #itname "_" #otname "_bn" #bn "_tn" #tn, \ +- block_sort, itype, otype, arg_sort, bn, tn) \ +- instantiate_kernel("nc" #name "_" #itname "_" #otname "_bn" #bn "_tn" #tn, \ +- block_sort_nc, itype, otype, arg_sort, bn, tn) ++ name, itname, itype, otname, otype, arg_sort, bn, tn, vcomparator) \ ++ instantiate_kernel("c" #name "_" #itname "_" #otname "_bn" #bn "_tn" #tn "_comp" #vcomparator, \ ++ block_sort, itype, otype, arg_sort, bn, tn, instantiate_comparator(vcomparator, itype)) \ ++ instantiate_kernel("nc" #name "_" #itname "_" #otname "_bn" #bn "_tn" #tn "_comp" #vcomparator, \ ++ block_sort_nc, itype, otype, arg_sort, bn, tn, instantiate_comparator(vcomparator, itype)) + +-#define instantiate_arg_block_sort_base(itname, itype, bn, tn) \ ++#define instantiate_arg_block_sort_base(itname, itype, bn, tn, comparator) \ + instantiate_block_sort( \ +- arg_block_sort, itname, itype, uint32, uint32_t, true, bn, tn) ++ arg_block_sort, itname, itype, uint32, uint32_t, true, bn, tn, comparator) + +-#define instantiate_block_sort_base(itname, itype, bn, tn) \ ++#define instantiate_block_sort_base(itname, itype, bn, tn, comparator) \ + instantiate_block_sort( \ +- _block_sort, itname, itype, itname, itype, false, bn, tn) ++ _block_sort, itname, itype, itname, itype, false, bn, tn, comparator) + + #define instantiate_block_sort_tn(itname, itype, bn) \ +- instantiate_block_sort_base(itname, itype, bn, 4) \ +- instantiate_arg_block_sort_base(itname, itype, bn, 4) ++ instantiate_block_sort_base(itname, itype, bn, 4, LessThan) \ ++ instantiate_block_sort_base(itname, itype, bn, 4, GreaterThan) \ ++ instantiate_arg_block_sort_base(itname, itype, bn, 4, LessThan) \ ++ instantiate_arg_block_sort_base(itname, itype, bn, 4, GreaterThan) + + #define instantiate_block_sort_bn(itname, itype) \ + instantiate_block_sort_tn(itname, itype, 32) \ +@@ -52,16 +57,17 @@ instantiate_block_sort_long(uint64, uint64_t) + instantiate_block_sort_long(int64, int64_t) + + #define instantiate_multi_block_sort( \ +- vtname, vtype, itname, itype, arg_sort, bn, tn) \ +- instantiate_kernel("sort_mbsort_" #vtname "_" #itname "_bn" #bn "_tn" #tn, \ +- mb_block_sort, vtype, itype, arg_sort, bn, tn) \ +- instantiate_kernel("partition_mbsort_" #vtname "_" #itname "_bn" #bn "_tn" #tn, \ +- mb_block_partition, vtype, itype, arg_sort, bn, tn) \ +- instantiate_kernel("merge_mbsort_" #vtname "_" #itname "_bn" #bn "_tn" #tn, \ +- mb_block_merge, vtype, itype, arg_sort, bn, tn) ++ vtname, vtype, itname, itype, arg_sort, bn, tn, vcomparator) \ ++ instantiate_kernel("sort_mbsort_" #vtname "_" #itname "_bn" #bn "_tn" #tn "_comp" #vcomparator, \ ++ mb_block_sort, vtype, itype, arg_sort, bn, tn, instantiate_comparator(vcomparator, vtype)) \ ++ instantiate_kernel("partition_mbsort_" #vtname "_" #itname "_bn" #bn "_tn" #tn "_comp" #vcomparator, \ ++ mb_block_partition, vtype, itype, arg_sort, bn, tn, instantiate_comparator(vcomparator, vtype)) \ ++ instantiate_kernel("merge_mbsort_" #vtname "_" #itname "_bn" #bn "_tn" #tn "_comp" #vcomparator, \ ++ mb_block_merge, vtype, itype, arg_sort, bn, tn, instantiate_comparator(vcomparator, vtype)) + + #define instantiate_multi_block_sort_base(vtname, vtype) \ +- instantiate_multi_block_sort(vtname, vtype, uint32, uint32_t, true, 512, 4) ++ instantiate_multi_block_sort(vtname, vtype, uint32, uint32_t, true, 512, 4, LessThan) \ ++ instantiate_multi_block_sort(vtname, vtype, uint32, uint32_t, true, 512, 4, GreaterThan) + + instantiate_multi_block_sort_base(uint8, uint8_t) + instantiate_multi_block_sort_base(uint16, uint16_t) +@@ -74,7 +80,8 @@ instantiate_multi_block_sort_base(float32, float) + instantiate_multi_block_sort_base(bfloat16, bfloat16_t) + + #define instantiate_multi_block_sort_long(vtname, vtype) \ +- instantiate_multi_block_sort(vtname, vtype, uint32, uint32_t, true, 256, 4) ++ instantiate_multi_block_sort(vtname, vtype, uint32, uint32_t, true, 256, 4, LessThan) \ ++ instantiate_multi_block_sort(vtname, vtype, uint32, uint32_t, true, 256, 4, GreaterThan) + + instantiate_multi_block_sort_long(uint64, uint64_t) + instantiate_multi_block_sort_long(int64, int64_t) // clang-format on +diff --git a/mlx/backend/metal/make_compiled_preamble.sh b/mlx/backend/metal/make_compiled_preamble.sh +index 425cd8d70..a48d00808 100644 +--- a/mlx/backend/metal/make_compiled_preamble.sh ++++ b/mlx/backend/metal/make_compiled_preamble.sh +@@ -5,18 +5,21 @@ + # + # Copyright © 2023-24 Apple Inc. + +-OUTPUT_DIR=$1 +-CC=$2 +-SRC_DIR=$3 +-SRC_FILE=$4 +-CFLAGS=$5 +-SRC_NAME=$(basename -- "${SRC_FILE}") +-JIT_INCLUDES=${SRC_DIR}/mlx/backend/metal/kernels/jit +-INPUT_FILE=${SRC_DIR}/mlx/backend/metal/kernels/${SRC_FILE}.h +-OUTPUT_FILE=${OUTPUT_DIR}/${SRC_NAME}.cpp ++OUTPUT_FILE=$1 ++INPUT_FILE=$2 ++CFLAGS=$3 + +-mkdir -p "$OUTPUT_DIR" +-CONTENT=$($CC -I"$SRC_DIR" -I"$JIT_INCLUDES" -DMLX_METAL_JIT -E -P "$INPUT_FILE" $CFLAGS 2>/dev/null) ++KERNEL_DIR=$(dirname $(realpath $INPUT_FILE)) ++# echo "KERNEL_DIR: $KERNEL_DIR" ++SRC_DIR=$(echo "$KERNEL_DIR" | cut -d'/' -f-$(($(echo "$KERNEL_DIR" | awk -F'/' '{print NF}')-4))) ++# echo "SRC_DIR: $SRC_DIR" ++JIT_INCLUDES="$KERNEL_DIR/jit" ++# echo "JIT_INCLUDES: $JIT_INCLUDES" ++ ++SRC_BASENAME=$(basename -- "${INPUT_FILE}") ++SRC_NAME="${SRC_BASENAME%.*}" ++ ++CONTENT=$(clang -I"$SRC_DIR" -I"$JIT_INCLUDES" -DMLX_METAL_JIT -E -P "$INPUT_FILE" $CFLAGS 2>/dev/null) + + cat << EOF > "$OUTPUT_FILE" + namespace mlx::core::metal { +diff --git a/mlx/backend/metal/nojit_kernels.cpp b/mlx/backend/metal/nojit_kernels.cpp +index 8da147971..1098f9188 100644 +--- a/mlx/backend/metal/nojit_kernels.cpp ++++ b/mlx/backend/metal/nojit_kernels.cpp +@@ -96,7 +96,8 @@ MTL::ComputePipelineState* get_sort_kernel( + const array&, + const array&, + int, +- int) { ++ int, ++ const std::string&) { + return d.get_kernel(kernel_name); + } + +@@ -106,7 +107,8 @@ MTL::ComputePipelineState* get_mb_sort_kernel( + const array&, + const array&, + int, +- int) { ++ int, ++ const std::string&) { + return d.get_kernel(kernel_name); + } + +diff --git a/mlx/backend/metal/sort.cpp b/mlx/backend/metal/sort.cpp +index 3c84022f2..bca888f66 100644 +--- a/mlx/backend/metal/sort.cpp ++++ b/mlx/backend/metal/sort.cpp +@@ -6,7 +6,9 @@ + #include "mlx/backend/metal/device.h" + #include "mlx/backend/metal/kernels.h" + #include "mlx/backend/metal/utils.h" ++#include "mlx/dtype_utils.h", + #include "mlx/primitives.h" ++#include "mlx/sort.h" + + namespace mlx::core { + +@@ -20,7 +22,8 @@ void single_block_sort( + int axis, + int bn, + int tn, +- bool argsort) { ++ bool argsort, ++ ComparatorType comparator_) { + // Prepare shapes + int n_rows = in.size() / in.shape(axis); + +@@ -58,9 +61,26 @@ void single_block_sort( + kname << "arg"; + } + ++ std::string comparator; ++ switch (comparator_) { ++ case ComparatorType::LessThan: ++ comparator = "LessThan"; ++ break; ++ case ComparatorType::GreaterThan: ++ comparator = "GreaterThan"; ++ break; ++ } ++ + kname << "_block_sort_" << type_to_name(in) << "_" << type_to_name(out) +- << "_bn" << bn << "_tn" << tn; +- auto kernel = get_sort_kernel(d, kname.str(), in, out, bn, tn); ++ << "_bn" << bn << "_tn" << tn << "_comp" << comparator; ++ auto kernel = get_sort_kernel( ++ d, ++ kname.str(), ++ in, ++ out, ++ bn, ++ tn, ++ comparator + "<" + dtype_to_string(in.dtype()) + ">"); + + // Prepare command encoder + auto& compute_encoder = d.get_command_encoder(s.index); +@@ -120,7 +140,8 @@ void multi_block_sort( + int bn, + int tn, + int n_blocks, +- bool argsort) { ++ bool argsort, ++ ComparatorType comparator_) { + // Prepare shapes + int n_rows = in.size() / in.shape(axis); + +@@ -159,6 +180,16 @@ void multi_block_sort( + std::vector copies = { + dev_vals_0, dev_vals_1, dev_idxs_0, dev_idxs_1, block_partitions}; + ++ std::string comparator; ++ switch (comparator_) { ++ case ComparatorType::LessThan: ++ comparator = "LessThan"; ++ break; ++ case ComparatorType::GreaterThan: ++ comparator = "GreaterThan"; ++ break; ++ } ++ + // Prepare command encoder + auto& compute_encoder = d.get_command_encoder(s.index); + +@@ -166,9 +197,16 @@ void multi_block_sort( + { + std::ostringstream kname; + kname << "sort_mbsort_" << type_to_name(dev_vals_0) << "_" +- << type_to_name(dev_idxs_0) << "_bn" << bn << "_tn" << tn; +- auto kernel = +- get_mb_sort_kernel(d, kname.str(), dev_vals_0, dev_idxs_0, bn, tn); ++ << type_to_name(dev_idxs_0) << "_bn" << bn << "_tn" << tn << "_comp" ++ << comparator; ++ auto kernel = get_mb_sort_kernel( ++ d, ++ kname.str(), ++ dev_vals_0, ++ dev_idxs_0, ++ bn, ++ tn, ++ comparator + "<" + dtype_to_string(in.dtype()) + ">"); + compute_encoder.set_compute_pipeline_state(kernel); + + compute_encoder.set_input_array(in, 0); +@@ -206,10 +244,17 @@ void multi_block_sort( + { + std::ostringstream kname; + kname << "partition_mbsort_" << type_to_name(dev_vals_in) << "_" +- << type_to_name(dev_idxs_in) << "_bn" << bn << "_tn" << tn; +- +- auto kernel = +- get_mb_sort_kernel(d, kname.str(), dev_vals_0, dev_idxs_0, bn, tn); ++ << type_to_name(dev_idxs_in) << "_bn" << bn << "_tn" << tn ++ << "_comp" << comparator; ++ ++ auto kernel = get_mb_sort_kernel( ++ d, ++ kname.str(), ++ dev_vals_0, ++ dev_idxs_0, ++ bn, ++ tn, ++ comparator + "<" + dtype_to_string(in.dtype()) + ">"); + compute_encoder.set_compute_pipeline_state(kernel); + + compute_encoder.set_output_array(block_partitions, 0); +@@ -229,10 +274,17 @@ void multi_block_sort( + { + std::ostringstream kname; + kname << "merge_mbsort_" << type_to_name(dev_vals_in) << "_" +- << type_to_name(dev_idxs_in) << "_bn" << bn << "_tn" << tn; +- +- auto kernel = +- get_mb_sort_kernel(d, kname.str(), dev_vals_0, dev_idxs_0, bn, tn); ++ << type_to_name(dev_idxs_in) << "_bn" << bn << "_tn" << tn ++ << "_comp" << comparator; ++ ++ auto kernel = get_mb_sort_kernel( ++ d, ++ kname.str(), ++ dev_vals_0, ++ dev_idxs_0, ++ bn, ++ tn, ++ comparator + "<" + dtype_to_string(in.dtype()) + ">"); + compute_encoder.set_compute_pipeline_state(kernel); + + compute_encoder.set_input_array(block_partitions, 0); +@@ -277,7 +329,8 @@ void gpu_merge_sort( + const array& in, + array& out, + int axis_, +- bool argsort) { ++ bool argsort, ++ ComparatorType comparator) { + // Get size info + int axis = axis_ < 0 ? axis_ + in.ndim() : axis_; + int size_sorted_axis = in.shape(axis); +@@ -307,9 +360,10 @@ void gpu_merge_sort( + int n_blocks = (size_sorted_axis + n_per_block - 1) / n_per_block; + + if (n_blocks > 1) { +- return multi_block_sort(s, d, in, out, axis, bn, tn, n_blocks, argsort); ++ return multi_block_sort( ++ s, d, in, out, axis, bn, tn, n_blocks, argsort, comparator); + } else { +- return single_block_sort(s, d, in, out, axis, bn, tn, argsort); ++ return single_block_sort(s, d, in, out, axis, bn, tn, argsort, comparator); + } + } + +@@ -324,7 +378,7 @@ void ArgSort::eval_gpu(const std::vector& inputs, array& out) { + auto& d = metal::device(s.device); + auto& in = inputs[0]; + +- gpu_merge_sort(s, d, in, out, axis_, true); ++ gpu_merge_sort(s, d, in, out, axis_, true, comparator_); + } + + void Sort::eval_gpu(const std::vector& inputs, array& out) { +@@ -336,7 +390,7 @@ void Sort::eval_gpu(const std::vector& inputs, array& out) { + auto& d = metal::device(s.device); + auto& in = inputs[0]; + +- gpu_merge_sort(s, d, in, out, axis_, false); ++ gpu_merge_sort(s, d, in, out, axis_, false, comparator_); + } + + void ArgPartition::eval_gpu(const std::vector& inputs, array& out) { +@@ -349,7 +403,7 @@ void ArgPartition::eval_gpu(const std::vector& inputs, array& out) { + auto& d = metal::device(s.device); + auto& in = inputs[0]; + +- gpu_merge_sort(s, d, in, out, axis_, true); ++ gpu_merge_sort(s, d, in, out, axis_, true, comparator_); + } + + void Partition::eval_gpu(const std::vector& inputs, array& out) { +@@ -362,7 +416,7 @@ void Partition::eval_gpu(const std::vector& inputs, array& out) { + auto& d = metal::device(s.device); + auto& in = inputs[0]; + +- gpu_merge_sort(s, d, in, out, axis_, false); ++ gpu_merge_sort(s, d, in, out, axis_, false, comparator_); + } + + } // namespace mlx::core +diff --git a/mlx/backend/no_gpu/primitives.cpp b/mlx/backend/no_gpu/primitives.cpp +index 409aa2c89..6d49f1c60 100644 +--- a/mlx/backend/no_gpu/primitives.cpp ++++ b/mlx/backend/no_gpu/primitives.cpp +@@ -112,7 +112,7 @@ NO_GPU(Partition) + NO_GPU(Power) + NO_GPU_MULTI(QRF) + NO_GPU(QuantizedMatmul) +-NO_GPU(RandomBits) ++NO_GPU() + NO_GPU(Real) + NO_GPU(Reduce) + NO_GPU(Reshape) +diff --git a/mlx/mlx.h b/mlx/mlx.h +index cef8d806d..f8932873c 100644 +--- a/mlx/mlx.h ++++ b/mlx/mlx.h +@@ -17,6 +17,7 @@ + #include "mlx/memory.h" + #include "mlx/ops.h" + #include "mlx/random.h" ++#include "mlx/sort.h" + #include "mlx/stream.h" + #include "mlx/transforms.h" + #include "mlx/utils.h" +diff --git a/mlx/ops.cpp b/mlx/ops.cpp +index 9602f667a..82bc8883c 100644 +--- a/mlx/ops.cpp ++++ b/mlx/ops.cpp +@@ -2205,13 +2205,20 @@ array argmax( + } + + /** Returns a sorted copy of the flattened array. */ +-array sort(const array& a, StreamOrDevice s /* = {} */) { ++array sort( ++ const array& a, ++ ComparatorType comparator, ++ StreamOrDevice s /* = {} */) { + int size = a.size(); +- return sort(reshape(a, {size}, s), 0, s); ++ return sort(reshape(a, {size}, s), 0, comparator, s); + } + + /** Returns a sorted copy of the array along a given axis. */ +-array sort(const array& a, int axis, StreamOrDevice s /* = {} */) { ++array sort( ++ const array& a, ++ int axis, ++ ComparatorType comparator, ++ StreamOrDevice s /* = {} */) { + // Check for valid axis + if (axis + static_cast(a.ndim()) < 0 || + axis >= static_cast(a.ndim())) { +@@ -2222,17 +2229,27 @@ array sort(const array& a, int axis, StreamOrDevice s /* = {} */) { + } + + return array( +- a.shape(), a.dtype(), std::make_shared(to_stream(s), axis), {a}); ++ a.shape(), ++ a.dtype(), ++ std::make_shared(to_stream(s), axis, comparator), ++ {a}); + } + + /** Returns indices that sort the flattened array. */ +-array argsort(const array& a, StreamOrDevice s /* = {} */) { ++array argsort( ++ const array& a, ++ ComparatorType comparator, ++ StreamOrDevice s /* = {} */) { + int size = a.size(); +- return argsort(reshape(a, {size}, s), 0, s); ++ return argsort(reshape(a, {size}, s), 0, comparator, s); + } + + /** Returns indices that sort the array along a given axis. */ +-array argsort(const array& a, int axis, StreamOrDevice s /* = {} */) { ++array argsort( ++ const array& a, ++ int axis, ++ ComparatorType comparator, ++ StreamOrDevice s /* = {} */) { + // Check for valid axis + if (axis + static_cast(a.ndim()) < 0 || + axis >= static_cast(a.ndim())) { +@@ -2243,16 +2260,23 @@ array argsort(const array& a, int axis, StreamOrDevice s /* = {} */) { + } + + return array( +- a.shape(), uint32, std::make_shared(to_stream(s), axis), {a}); ++ a.shape(), ++ uint32, ++ std::make_shared(to_stream(s), axis, comparator), ++ {a}); + } + + /** + * Returns a partitioned copy of the flattened array + * such that the smaller kth elements are first. + **/ +-array partition(const array& a, int kth, StreamOrDevice s /* = {} */) { ++array partition( ++ const array& a, ++ int kth, ++ ComparatorType comparator, ++ StreamOrDevice s /* = {} */) { + int size = a.size(); +- return partition(reshape(a, {size}, s), kth, 0, s); ++ return partition(reshape(a, {size}, s), kth, 0, comparator, s); + } + + /** +@@ -2263,6 +2287,7 @@ array partition( + const array& a, + int kth, + int axis, ++ ComparatorType comparator, + StreamOrDevice s /* = {} */) { + // Check for valid axis + if (axis + static_cast(a.ndim()) < 0 || +@@ -2283,7 +2308,7 @@ array partition( + return array( + a.shape(), + a.dtype(), +- std::make_shared(to_stream(s), kth_, axis_), ++ std::make_shared(to_stream(s), kth_, axis_, comparator), + {a}); + } + +@@ -2291,9 +2316,13 @@ array partition( + * Returns indices that partition the flattened array + * such that the smaller kth elements are first. + **/ +-array argpartition(const array& a, int kth, StreamOrDevice s /* = {} */) { ++array argpartition( ++ const array& a, ++ int kth, ++ ComparatorType comparator, ++ StreamOrDevice s /* = {} */) { + int size = a.size(); +- return argpartition(reshape(a, {size}, s), kth, 0, s); ++ return argpartition(reshape(a, {size}, s), kth, 0, comparator, s); + } + + /** +@@ -2304,6 +2333,7 @@ array argpartition( + const array& a, + int kth, + int axis, ++ ComparatorType comparator, + StreamOrDevice s /* = {} */) { + // Check for valid axis + if (axis + static_cast(a.ndim()) < 0 || +@@ -2324,7 +2354,7 @@ array argpartition( + return array( + a.shape(), + uint32, +- std::make_shared(to_stream(s), kth_, axis_), ++ std::make_shared(to_stream(s), kth_, axis_, comparator), + {a}); + } + +diff --git a/mlx/ops.h b/mlx/ops.h +index af3cdb5bd..be762b8a6 100644 +--- a/mlx/ops.h ++++ b/mlx/ops.h +@@ -6,6 +6,7 @@ + + #include "mlx/array.h" + #include "mlx/device.h" ++#include "mlx/sort.h" + #include "mlx/stream.h" + #include "mlx/utils.h" + +@@ -674,40 +675,98 @@ array argmax( + StreamOrDevice s = {}); + + /** Returns a sorted copy of the flattened array. */ +-array sort(const array& a, StreamOrDevice s = {}); ++array sort( ++ const array& a, ++ ComparatorType comparator = ComparatorType::LessThan, ++ StreamOrDevice s = {}); ++inline array sort(const array& a, StreamOrDevice s = {}) { ++ return sort(a, ComparatorType::LessThan, s); ++} + + /** Returns a sorted copy of the array along a given axis. */ +-array sort(const array& a, int axis, StreamOrDevice s = {}); ++array sort( ++ const array& a, ++ int axis, ++ ComparatorType comparator = ComparatorType::LessThan, ++ StreamOrDevice s = {}); ++inline array sort(const array& a, int axis, StreamOrDevice s = {}) { ++ return sort(a, axis, ComparatorType::LessThan, s); ++} + + /** Returns indices that sort the flattened array. */ +-array argsort(const array& a, StreamOrDevice s = {}); ++array argsort( ++ const array& a, ++ ComparatorType comparator = ComparatorType::LessThan, ++ StreamOrDevice s = {}); ++inline array argsort(const array& a, StreamOrDevice s = {}) { ++ return argsort(a, ComparatorType::LessThan, s); ++} + + /** Returns indices that sort the array along a given axis. */ +-array argsort(const array& a, int axis, StreamOrDevice s = {}); ++array argsort( ++ const array& a, ++ int axis, ++ ComparatorType comparator = ComparatorType::LessThan, ++ StreamOrDevice s = {}); ++inline array argsort(const array& a, int axis, StreamOrDevice s = {}) { ++ return argsort(a, axis, ComparatorType::LessThan, s); ++} + + /** + * Returns a partitioned copy of the flattened array + * such that the smaller kth elements are first. + **/ +-array partition(const array& a, int kth, StreamOrDevice s = {}); ++array partition( ++ const array& a, ++ int kth, ++ ComparatorType comparator = ComparatorType::LessThan, ++ StreamOrDevice s = {}); ++inline array partition(const array& a, int kth, StreamOrDevice s = {}) { ++ return partition(a, kth, ComparatorType::LessThan, s); ++} + + /** + * Returns a partitioned copy of the array along a given axis + * such that the smaller kth elements are first. + **/ +-array partition(const array& a, int kth, int axis, StreamOrDevice s = {}); ++array partition( ++ const array& a, ++ int kth, ++ int axis, ++ ComparatorType comparator = ComparatorType::LessThan, ++ StreamOrDevice s = {}); ++inline array ++partition(const array& a, int kth, int axis, StreamOrDevice s = {}) { ++ return partition(a, kth, axis, ComparatorType::LessThan, s); ++} + + /** + * Returns indices that partition the flattened array + * such that the smaller kth elements are first. + **/ +-array argpartition(const array& a, int kth, StreamOrDevice s = {}); ++array argpartition( ++ const array& a, ++ int kth, ++ ComparatorType comparator = ComparatorType::LessThan, ++ StreamOrDevice s = {}); ++inline array argpartition(const array& a, int kth, StreamOrDevice s = {}) { ++ return argpartition(a, kth, ComparatorType::LessThan, s); ++} + + /** + * Returns indices that partition the array along a given axis + * such that the smaller kth elements are first. + **/ +-array argpartition(const array& a, int kth, int axis, StreamOrDevice s = {}); ++array argpartition( ++ const array& a, ++ int kth, ++ int axis, ++ ComparatorType comparator = ComparatorType::LessThan, ++ StreamOrDevice s = {}); ++inline array ++argpartition(const array& a, int kth, int axis, StreamOrDevice s = {}) { ++ return argpartition(a, kth, axis, ComparatorType::LessThan, s); ++} + + /** Returns topk elements of the flattened array. */ + array topk(const array& a, int k, StreamOrDevice s = {}); +diff --git a/mlx/primitives.h b/mlx/primitives.h +index cc60bcfb9..2b79c0d67 100644 +--- a/mlx/primitives.h ++++ b/mlx/primitives.h +@@ -7,6 +7,7 @@ + #include "mlx/array.h" + #include "mlx/device.h" + #include "mlx/io/load.h" ++#include "mlx/sort.h" + #include "mlx/stream.h" + + #define DEFINE_VMAP() \ +@@ -322,8 +323,15 @@ class ArcTanh : public UnaryPrimitive { + + class ArgPartition : public UnaryPrimitive { + public: +- explicit ArgPartition(Stream stream, int kth, int axis) +- : UnaryPrimitive(stream), kth_(kth), axis_(axis) {} ++ explicit ArgPartition( ++ Stream stream, ++ int kth, ++ int axis, ++ ComparatorType comparator) ++ : UnaryPrimitive(stream), ++ kth_(kth), ++ axis_(axis), ++ comparator_(comparator) {} + + void eval_cpu(const std::vector& inputs, array& out) override; + void eval_gpu(const std::vector& inputs, array& out) override; +@@ -333,13 +341,14 @@ class ArgPartition : public UnaryPrimitive { + DEFINE_PRINT(ArgPartition) + DEFINE_INPUT_OUTPUT_SHAPE() + bool is_equivalent(const Primitive& other) const override; +- std::pair state() const { +- return {kth_, axis_}; ++ auto state() const { ++ return std::make_tuple(kth_, axis_, comparator_); + }; + + private: + int kth_; + int axis_; ++ ComparatorType comparator_; + }; + + class ArgReduce : public UnaryPrimitive { +@@ -371,8 +380,8 @@ class ArgReduce : public UnaryPrimitive { + + class ArgSort : public UnaryPrimitive { + public: +- explicit ArgSort(Stream stream, int axis) +- : UnaryPrimitive(stream), axis_(axis) {} ++ explicit ArgSort(Stream stream, int axis, ComparatorType comparator) ++ : UnaryPrimitive(stream), axis_(axis), comparator_(comparator) {} + + void eval_cpu(const std::vector& inputs, array& out) override; + void eval_gpu(const std::vector& inputs, array& out) override; +@@ -381,12 +390,13 @@ class ArgSort : public UnaryPrimitive { + DEFINE_PRINT(ArgSort) + DEFINE_INPUT_OUTPUT_SHAPE() + bool is_equivalent(const Primitive& other) const override; +- int state() const { +- return axis_; ++ auto state() const { ++ return std::make_pair(axis_, comparator_); + }; + + private: + int axis_; ++ ComparatorType comparator_; + }; + + class AsType : public UnaryPrimitive { +@@ -1531,8 +1541,15 @@ class Pad : public UnaryPrimitive { + + class Partition : public UnaryPrimitive { + public: +- explicit Partition(Stream stream, int kth, int axis) +- : UnaryPrimitive(stream), kth_(kth), axis_(axis) {} ++ explicit Partition( ++ Stream stream, ++ int kth, ++ int axis, ++ ComparatorType comparator) ++ : UnaryPrimitive(stream), ++ kth_(kth), ++ axis_(axis), ++ comparator_(comparator) {} + + void eval_cpu(const std::vector& inputs, array& out) override; + void eval_gpu(const std::vector& inputs, array& out) override; +@@ -1543,12 +1560,13 @@ class Partition : public UnaryPrimitive { + DEFINE_INPUT_OUTPUT_SHAPE() + bool is_equivalent(const Primitive& other) const override; + auto state() const { +- return std::make_pair(kth_, axis_); ++ return std::make_tuple(kth_, axis_, comparator_); + }; + + private: + int kth_; + int axis_; ++ ComparatorType comparator_; + }; + + class Power : public UnaryPrimitive { +@@ -2074,8 +2092,8 @@ class Softmax : public UnaryPrimitive { + + class Sort : public UnaryPrimitive { + public: +- explicit Sort(Stream stream, int axis) +- : UnaryPrimitive(stream), axis_(axis) {} ++ explicit Sort(Stream stream, int axis, ComparatorType comparator) ++ : UnaryPrimitive(stream), axis_(axis), comparator_(comparator) {} + + void eval_cpu(const std::vector& inputs, array& out) override; + void eval_gpu(const std::vector& inputs, array& out) override; +@@ -2086,11 +2104,12 @@ class Sort : public UnaryPrimitive { + DEFINE_INPUT_OUTPUT_SHAPE() + bool is_equivalent(const Primitive& other) const override; + auto state() const { +- return axis_; ++ return std::make_pair(axis_, comparator_); + } + + private: + int axis_; ++ ComparatorType comparator_; + }; + + class Split : public Primitive { +diff --git a/mlx/random.cpp b/mlx/random.cpp +index 6c6d1eb95..5a4745a52 100644 +--- a/mlx/random.cpp ++++ b/mlx/random.cpp +@@ -225,6 +225,23 @@ array normal( + return samples; + } + ++array normal( ++ const Shape& shape, ++ Dtype dtype, ++ const array& loc, ++ const array& scale, ++ const std::optional& key /*= nullopt */, ++ StreamOrDevice s /* = {} */) { ++ auto stream = to_stream(s); ++ auto low = above_minus_one_with_default(dtype); ++ auto high = array(1.0f, dtype); ++ auto samples = uniform(low, high, shape, dtype, key, stream); ++ samples = multiply(array(std::sqrt(2.0), dtype), erfinv(samples, stream), stream); ++ samples = multiply(astype(scale, dtype), samples, stream); ++ samples = add(astype(loc, dtype), samples, stream); ++ return samples; ++} ++ + array multivariate_normal( + const array& mean, + const array& cov, +diff --git a/mlx/random.h b/mlx/random.h +index 0dfdab7a1..ebee726c8 100644 +--- a/mlx/random.h ++++ b/mlx/random.h +@@ -134,6 +134,14 @@ inline array normal( + return normal(shape, float32, std::nullopt, std::nullopt, key, s); + } + ++array normal( ++ const Shape& shape, ++ Dtype dtype, ++ const array& loc, ++ const array& scale, ++ const std::optional& key = std::nullopt, ++ StreamOrDevice s = {}); ++ + /** Generate samples from a multivariate normal distribution. **/ + array multivariate_normal( + const array& mean, +diff --git a/mlx/sort.h b/mlx/sort.h +new file mode 100644 +index 000000000..0afc6b62b +--- /dev/null ++++ b/mlx/sort.h +@@ -0,0 +1,14 @@ ++// Copyright © 2023-2024 Apple Inc. ++ ++#pragma once ++ ++#include "mlx/array.h" ++ ++namespace mlx::core { ++ ++enum class ComparatorType { ++ LessThan, ++ GreaterThan, ++}; ++ ++} // namespace mlx::core +diff --git a/mlx/types/complex.h b/mlx/types/complex.h +index 51101cc97..344d80f87 100644 +--- a/mlx/types/complex.h ++++ b/mlx/types/complex.h +@@ -59,10 +59,11 @@ inline bool operator>(const complex64_t& a, const complex64_t& b) { + inline complex64_t operator%(complex64_t a, complex64_t b) { + auto real = a.real() - (b.real() * static_cast(a.real() / b.real())); + auto imag = a.imag() - (b.imag() * static_cast(a.imag() / b.imag())); +- if (real != 0 && ((real < 0) != (b.real() < 0))) +- real += b.real(); +- if (imag != 0 && ((imag < 0) != (b.imag() < 0))) +- imag += b.imag(); ++ /* Disable to match `stablehlo` implementation semantics */ ++ // if (real != 0 && ((real < 0) != (b.real() < 0))) ++ // real += b.real(); ++ // if (imag != 0 && ((imag < 0) != (b.imag() < 0))) ++ // imag += b.imag(); + return {real, imag}; + } + diff --git a/third_party/modules/mlx/20250605.0-c763fe1/source.json b/third_party/modules/mlx/20250605.0-c763fe1/source.json new file mode 100644 index 00000000..d15eec9a --- /dev/null +++ b/third_party/modules/mlx/20250605.0-c763fe1/source.json @@ -0,0 +1,15 @@ +{ + "strip_prefix": "mlx-c763fe1be0f1158e0f53f6f6a28f56d69b1c7fe8", + "url": "https://github.com/ml-explore/mlx/archive/c763fe1be0f1158e0f53f6f6a28f56d69b1c7fe8.tar.gz", + "integrity": "sha256-vE091y6Qc9r0cw/dU2OCCDosZzCXZfTv0tp7Zi2YqYE=", + "overlay": { + "metal_cpp.bzl": "", + "jit.bzl": "", + "MODULE.bazel": "", + "BUILD.bazel": "" + }, + "patch_strip": 1, + "patches": { + "0001-Modify-Sort-And-Comparison-Kernels.patch": "" + } +} diff --git a/third_party/modules/mlx/metadata.json b/third_party/modules/mlx/metadata.json index 6945817e..805d7f05 100644 --- a/third_party/modules/mlx/metadata.json +++ b/third_party/modules/mlx/metadata.json @@ -11,7 +11,8 @@ "github:ml-explore/mlx" ], "versions": [ - "20250518.0-eebe730" + "20250518.0-eebe730", + "20250605.0-c763fe1" ], "yanked_versions": {} } diff --git a/third_party/modules/xla/20250527.0-cb67f2f/MODULE.bazel b/third_party/modules/xla/20250527.0-cb67f2f/MODULE.bazel index 434d3365..2cd155e8 100644 --- a/third_party/modules/xla/20250527.0-cb67f2f/MODULE.bazel +++ b/third_party/modules/xla/20250527.0-cb67f2f/MODULE.bazel @@ -19,7 +19,7 @@ bazel_dep(name = "rules_shell", version = "0.4.1") bazel_dep(name = "bazel_features", version = "1.25.0", repo_name = "proto_bazel_features") # MLX PjRt Plugin Dependencies -bazel_dep(name = "mlx", version = "20250518.0-eebe730") +bazel_dep(name = "mlx", version = "20250605.0-c763fe1") workspace_private = use_extension("//:workspace_private.bzl", "workspace_private") use_repo( diff --git a/third_party/modules/xla/20250527.0-cb67f2f/patches/0004-Added-MLX-PjRt-Plugin.patch b/third_party/modules/xla/20250527.0-cb67f2f/patches/0004-Added-MLX-PjRt-Plugin.patch index 00156e7e..a47c5373 100644 --- a/third_party/modules/xla/20250527.0-cb67f2f/patches/0004-Added-MLX-PjRt-Plugin.patch +++ b/third_party/modules/xla/20250527.0-cb67f2f/patches/0004-Added-MLX-PjRt-Plugin.patch @@ -212,10 +212,10 @@ index 0000000000..94bc83ea41 \ No newline at end of file diff --git a/xla/pjrt/plugin/mlx/buffer.cc b/xla/pjrt/plugin/mlx/buffer.cc new file mode 100644 -index 0000000000..c4e7d410ac +index 0000000000..750a793c1f --- /dev/null +++ b/xla/pjrt/plugin/mlx/buffer.cc -@@ -0,0 +1,371 @@ +@@ -0,0 +1,366 @@ +/* Copyright 2024 The OpenXLA Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. @@ -324,13 +324,11 @@ index 0000000000..c4e7d410ac + MlxPjrtBuffer& operator=(MlxPjrtBuffer&&) = delete; + + static std::unique_ptr CreateFromLiteral( -+ mx::array array, const xla::LiteralSlice& literal, ++ const mx::array& array, const xla::LiteralSlice& literal, + xla::PjRtMemorySpace* memory_space) { -+ // TRACE_ME; -+ LOG(INFO) << "CreateFromLiteral: " << literal.ToString() << "\n"; ++ TRACE_ME; + auto buffer = + std::make_unique(array, literal.shape(), memory_space); -+ LOG(INFO) << "CreateFromLiteral -> " << (void*)buffer.get() << "\n"; + mlir::Builder builder(&buffer->context_); + auto attr = xla::CreateDenseElementsAttrFromLiteral(literal, builder); + if (!attr.ok()) { @@ -345,14 +343,12 @@ index 0000000000..c4e7d410ac + static std::unique_ptr CreateFromAttribute( + mx::array array, DenseElementsAttr attr, + xla::PjRtMemorySpace* memory_space) { -+ // TRACE_ME; ++ TRACE_ME; + + // MLIR type to xla shape: + Shape shape = xla::TypeToShape(attr.getType()); + auto buffer = std::make_unique(array, shape, memory_space); + buffer->buffer_ = CloneIntoContext(attr, buffer->context_); -+ LOG(INFO) << "CreateFromAttribute(" << ToString(attr) << ") -> " -+ << (void*)buffer.get() << "\n"; + return buffer; + } + @@ -369,6 +365,7 @@ index 0000000000..c4e7d410ac + // TRACE_ME_MEMBER; + return client()->platform_id(); + } ++ + absl::string_view platform_name() const { + // TRACE_ME_MEMBER; + return client()->platform_name(); @@ -455,6 +452,7 @@ index 0000000000..c4e7d410ac + void Delete() override { + // TRACE_ME_MEMBER; + buffer_ = {}; ++ array_ = mx::array({}); // Reset the array to empty. + } + + bool IsDeleted() override { @@ -484,7 +482,6 @@ index 0000000000..c4e7d410ac + + xla::PjRtFuture<> GetReadyFuture() override { + // TRACE_ME_MEMBER; -+ LOG(INFO) << "GetReadyFuture(" << (void*)this << ")\n"; + // Synchronous! To make async, have the device make a buffer with a ready + // future that is ready when the computation is done / buffer is ready. + return PjRtFuture<>(absl::OkStatus()); @@ -501,7 +498,7 @@ index 0000000000..c4e7d410ac + + mlir::DenseElementsAttr GetBufferAttribute() const { return buffer_; } + -+ mx::array GetArray() const { return array_; } ++ mx::array& GetArray() { return array_; } + + private: + MLIRContext context_; @@ -513,8 +510,8 @@ index 0000000000..c4e7d410ac +}; + +std::unique_ptr CreateMlirBufferFromMlxArray( -+ mx::array array, xla::PjRtMemorySpace* memory_space) { -+ // TRACE_ME; ++ const mx::array& array, xla::PjRtMemorySpace* memory_space) { ++ TRACE_ME; + std::vector span_shape(array.shape().begin(), array.shape().end()); + auto shape = xla::ShapeUtil::MakeShape( + utils::dtype::asXlaPrimitiveType(array.dtype()), @@ -526,7 +523,7 @@ index 0000000000..c4e7d410ac + +std::unique_ptr CreateMlirBufferFromLiteral( + const xla::LiteralSlice& literal, xla::PjRtMemorySpace* memory_space) { -+ // TRACE_ME; ++ TRACE_ME; + auto maybe_array = utils::array::fromHostLiteral(literal); + return MlxPjrtBuffer::CreateFromLiteral(maybe_array.value(), literal, + memory_space); @@ -535,13 +532,13 @@ index 0000000000..c4e7d410ac +std::unique_ptr CreateMlirBufferFromAttribute( + mx::array array, DenseElementsAttr attr, + xla::PjRtMemorySpace* memory_space) { -+ // TRACE_ME; ++ TRACE_ME; + return MlxPjrtBuffer::CreateFromAttribute(array, attr, memory_space); +} + +std::unique_ptr CreateMlirBufferUninitialized( + const xla::Shape& shape, PjRtMemorySpace* memory_space) { -+ // TRACE_ME; ++ TRACE_ME; + // TODO (@cryptodeal): C API doesn't implement this, but + // we'll want to ensure when the Buffer is initialized, + // the resulting array is correct shape/dtype. @@ -553,7 +550,7 @@ index 0000000000..c4e7d410ac + +absl::StatusOr GetAttributeFromBuffer( + xla::PjRtBuffer* buffer) { -+ // TRACE_ME; ++ TRACE_ME; + if (buffer == nullptr || buffer->IsDeleted()) { + return xla::InvalidArgument("Buffer is null or deleted"); + } @@ -561,13 +558,11 @@ index 0000000000..c4e7d410ac + if (mlir_buffer == nullptr) { + return xla::InvalidArgument("Buffer is not a MlxPjrtBuffer"); + } -+ LOG(INFO) << "GetAttributeFromBuffer(" << (void*)buffer << ") -> " -+ << ToString(mlir_buffer->GetBufferAttribute()) << "\n"; + return mlir_buffer->GetBufferAttribute(); +} + +absl::StatusOr GetArrayFromBuffer(xla::PjRtBuffer* buffer) { -+ // TRACE_ME; ++ TRACE_ME; + if (buffer == nullptr || buffer->IsDeleted()) { + return xla::InvalidArgument("Buffer is null or deleted"); + } @@ -590,7 +585,7 @@ index 0000000000..c4e7d410ac \ No newline at end of file diff --git a/xla/pjrt/plugin/mlx/buffer.h b/xla/pjrt/plugin/mlx/buffer.h new file mode 100644 -index 0000000000..1740f3c779 +index 0000000000..da37e97645 --- /dev/null +++ b/xla/pjrt/plugin/mlx/buffer.h @@ -0,0 +1,43 @@ @@ -620,7 +615,7 @@ index 0000000000..1740f3c779 + +namespace mlir::stablehlo { +std::unique_ptr CreateMlirBufferFromMlxArray( -+ mx::array array, xla::PjRtMemorySpace* memory_space); ++ const mx::array& array, xla::PjRtMemorySpace* memory_space); +std::unique_ptr CreateMlirBufferFromLiteral( + const xla::LiteralSlice& literal, xla::PjRtMemorySpace* memory_space); +std::unique_ptr CreateMlirBufferFromAttribute( @@ -1349,10 +1344,10 @@ index 0000000000..ef42968a3a \ No newline at end of file diff --git a/xla/pjrt/plugin/mlx/executable.cc b/xla/pjrt/plugin/mlx/executable.cc new file mode 100644 -index 0000000000..c1ceac0c7c +index 0000000000..02aaf61559 --- /dev/null +++ b/xla/pjrt/plugin/mlx/executable.cc -@@ -0,0 +1,2337 @@ +@@ -0,0 +1,2356 @@ +/* Copyright 2024 The OpenXLA Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. @@ -3379,10 +3374,11 @@ index 0000000000..c1ceac0c7c + module_(cloneIntoContext(module, context_)) { + // TRACE_ME_MEMBER; + auto main = module.lookupSymbol("main"); ++ + // std::cout << "Started call to `mlx::compile`" << std::endl << std::endl; + // module.dump(); + // std::cout << std::endl; -+ compileModule(module, op_lookup_); ++ compileModule(module_.get(), op_lookup_); + // try { + auto& main_block = main.front(); + compiled_module_ = @@ -3407,7 +3403,7 @@ index 0000000000..c1ceac0c7c + static absl::StatusOr> CompileAndLoad( + ModuleOp module, DeviceAssignment assignment, + absl::Span devices, PjRtClient* client) { -+ // TRACE_ME; ++ TRACE_ME; + + mlir::BaseScopedDiagnosticHandler diagnostic_handler(module->getContext()); + if (failed(decomposeChloToStablehlo(module))) { @@ -3424,7 +3420,17 @@ index 0000000000..c1ceac0c7c + return module_status; + } + -+ // std::cout << ToString(module).c_str() << std::endl << std::endl; ++ // auto device_info = mx::metal::device_info(); ++ // for (const auto& [key, value] : device_info) { ++ // std::cout << key << ": "; ++ ++ // std::visit([](const auto& v) { std::cout << v; }, value); ++ ++ // std::cout << std::endl; ++ // } ++ ++ // std::cout << ToString(module).c_str() << std::endl << std::endl << ++ // std::endl; + + auto executable = std::make_unique(module, assignment, + devices, client); @@ -3493,32 +3499,31 @@ index 0000000000..c1ceac0c7c + absl::Span argument_handles, ModuleOp module, + PjRtDevice* device, bool fill_future) { + // TRACE_ME_MEMBER; -+ SmallVector input_attrs; -+ for (auto* arg : argument_handles) { -+ TF_ASSIGN_OR_RETURN(auto mlirArg, GetAttributeFromBuffer(arg)); -+ auto mlirArgInModuleContext = -+ CloneIntoContext(mlirArg, *module->getContext()); -+ input_attrs.push_back(mlirArgInModuleContext); -+ } ++ // SmallVector input_attrs; ++ // for (auto* arg : argument_handles) { ++ // TF_ASSIGN_OR_RETURN(auto mlirArg, GetAttributeFromBuffer(arg)); ++ // auto mlirArgInModuleContext = ++ // CloneIntoContext(mlirArg, *module->getContext()); ++ // input_attrs.push_back(mlirArgInModuleContext); ++ // } + // LOG(INFO) << "EvalModule:\n" << ToString(module) << "\n"; + // LOG(INFO) << "Inputs: " << ToString(inputs) << "\n"; + // TF_ASSIGN_OR_RETURN(auto mlx_res, evalModule(module, inputs)); + + std::vector inputs; + // unsigned in_count = 0; -+ for (auto& attr : input_attrs) { -+ TF_ASSIGN_OR_RETURN(mx::array input, -+ utils::array::fromDenseElementsAttr(attr)); ++ for (auto& buffer : argument_handles) { ++ TF_ASSIGN_OR_RETURN(mx::array input, GetArrayFromBuffer(buffer)); + // std::cout << "inputs[" << std::to_string(in_count++) << "]: " << input + // << std::endl; + inputs.push_back(input); + } -+ + auto mlx_res = compiled_module_(inputs); + for (auto& a : mlx_res) { + a = mx::contiguous(a); + } + mx::eval(mlx_res); ++ + // unsigned out_count = 0; + // for (auto& a : mlx_res) { + // std::cout << "outputs[" << std::to_string(out_count++) << "]: " << a @@ -3530,10 +3535,11 @@ index 0000000000..c1ceac0c7c + // Naive memory space selection, only using CPU global memory. + PjRtMemorySpace* memory_space = + device->default_memory_space().value_or(nullptr); -+ std::vector> buffer_results; ++ ++ std::vector> buffer_results(mlx_res.size()); + for (auto i = 0; i < mlx_res.size(); i++) { -+ buffer_results.push_back( -+ CreateMlirBufferFromMlxArray(mlx_res[i], memory_space)); ++ buffer_results[i] = ++ CreateMlirBufferFromMlxArray(mlx_res[i], memory_space); + } + + std::optional> future; @@ -3542,6 +3548,7 @@ index 0000000000..c1ceac0c7c + // is ready when the computation is done. + future = PjRtFuture<>(absl::OkStatus()); + } ++ + return PjRtLoadedExecutable::Result{future, std::move(buffer_results)}; + } + @@ -3606,11 +3613,15 @@ index 0000000000..c1ceac0c7c + return std::move(result.buffers); + } + ++ // TODO (@cryptodeal): Implement `SerializeExecutable` ++ // absl::StatusOr SerializeExecutable() {} ++ + void Delete() override { + // TRACE_ME_MEMBER; + module_.release(); + module_ = nullptr; + } ++ + bool IsDeleted() override { + // TRACE_ME_MEMBER; + return !module_; @@ -3621,15 +3632,18 @@ index 0000000000..c1ceac0c7c + // TRACE_ME_MEMBER; + return assignment_.replica_count(); + } ++ + int num_partitions() const override { + // TRACE_ME_MEMBER; + return assignment_.computation_count(); + } ++ + int64_t SizeOfGeneratedCodeInBytes() const override { + // No generated code.. so just return 1. + // TRACE_ME_MEMBER; + return 1; + } ++ + absl::string_view name() const override { + // TRACE_ME_MEMBER; + return name_; @@ -3674,7 +3688,7 @@ index 0000000000..c1ceac0c7c + +absl::StatusOr> StablehloMlxCompile( + mlir::ModuleOp module, DeviceAssignment assignment, PjRtClient* client) { -+ // TRACE_ME; ++ TRACE_ME; + return MlirLoadedExecutable::CompileAndLoad(module, assignment, + client->devices(), client); +} @@ -3682,7 +3696,7 @@ index 0000000000..c1ceac0c7c +absl::StatusOr> StablehloMlxCompile( + xla::XlaComputation const& computation, xla::DeviceAssignment assignment, + xla::PjRtClient* client) { -+ // TRACE_ME; ++ TRACE_ME; + MLIRContext context; + TF_ASSIGN_OR_RETURN(auto module, + ConvertHloToStablehlo(context, &computation.proto())); @@ -4039,7 +4053,7 @@ index 0000000000..3f21f67baf \ No newline at end of file diff --git a/xla/pjrt/plugin/mlx/logging.h b/xla/pjrt/plugin/mlx/logging.h new file mode 100644 -index 0000000000..762dfabbcc +index 0000000000..58f296ab2f --- /dev/null +++ b/xla/pjrt/plugin/mlx/logging.h @@ -0,0 +1,42 @@ @@ -4068,7 +4082,7 @@ index 0000000000..762dfabbcc +#define LOG_UNIMPLEMENTED(name) \ + LOG(ERROR) << "MlxPjRtBuffer::" #name " is not implemented" + -+#define TRACE_ME LOG(INFO) << __func__; ++#define TRACE_ME LOG(INFO) << __func__ << "\n"; + +#define TRACE_ME_MEMBER LOG(INFO) << __func__ << "(" << (void*)this << ")\n"; + From 408300a70c4724a5fed61f1ab5eab29f296b1411 Mon Sep 17 00:00:00 2001 From: James Deal Date: Tue, 15 Jul 2025 23:59:21 -0500 Subject: [PATCH 11/13] Update vendored xla mlx plugin to latest --- .../20250605.0-c763fe1/overlay/BUILD.bazel | 1 + ...1-Modify-Sort-And-Comparison-Kernels.patch | 1252 +++++ .../patches/0004-Added-MLX-PjRt-Plugin.patch | 4119 ++++++++--------- 3 files changed, 3128 insertions(+), 2244 deletions(-) diff --git a/third_party/modules/mlx/20250605.0-c763fe1/overlay/BUILD.bazel b/third_party/modules/mlx/20250605.0-c763fe1/overlay/BUILD.bazel index 1ea75697..c53b4afa 100644 --- a/third_party/modules/mlx/20250605.0-c763fe1/overlay/BUILD.bazel +++ b/third_party/modules/mlx/20250605.0-c763fe1/overlay/BUILD.bazel @@ -371,6 +371,7 @@ cc_library( "mlx/backend/metal/allocator.h", "mlx/backend/metal/binary.h", "mlx/backend/metal/device.h", + "mlx/backend/metal/thread_safety.h", "mlx/backend/metal/kernels.h", "mlx/backend/metal/matmul.h", "mlx/backend/metal/metal.h", diff --git a/third_party/modules/mlx/20250605.0-c763fe1/patches/0001-Modify-Sort-And-Comparison-Kernels.patch b/third_party/modules/mlx/20250605.0-c763fe1/patches/0001-Modify-Sort-And-Comparison-Kernels.patch index 823a14db..f7c27b51 100644 --- a/third_party/modules/mlx/20250605.0-c763fe1/patches/0001-Modify-Sort-And-Comparison-Kernels.patch +++ b/third_party/modules/mlx/20250605.0-c763fe1/patches/0001-Modify-Sort-And-Comparison-Kernels.patch @@ -112,6 +112,106 @@ index ebc3cc77f..7bdca47d7 100644 #include "mlx/backend/metal/utils.h" #include "mlx/utils.h" +diff --git a/mlx/backend/metal/eval.cpp b/mlx/backend/metal/eval.cpp +index 49783200a..fd9caf147 100644 +--- a/mlx/backend/metal/eval.cpp ++++ b/mlx/backend/metal/eval.cpp +@@ -4,12 +4,15 @@ + #include "mlx/backend/gpu/available.h" + #include "mlx/backend/gpu/eval.h" + #include "mlx/backend/metal/device.h" ++#include "mlx/backend/metal/thread_safety.h" + #include "mlx/backend/metal/utils.h" + #include "mlx/primitives.h" + #include "mlx/scheduler.h" + + namespace mlx::core::gpu { + ++std::mutex metal_operation_mutex; ++ + bool is_available() { + return true; + } +@@ -30,6 +33,7 @@ inline void check_error(MTL::CommandBuffer* cbuf) { + } + + void eval(array& arr) { ++ std::lock_guard lock(metal_operation_mutex); + auto pool = metal::new_scoped_memory_pool(); + auto s = arr.primitive().stream(); + auto& d = metal::device(s.device); +@@ -78,6 +82,7 @@ void eval(array& arr) { + } + + void finalize(Stream s) { ++ std::lock_guard lock(metal_operation_mutex); + auto pool = metal::new_scoped_memory_pool(); + auto& d = metal::device(s.device); + auto cb = d.get_command_buffer(s.index); +@@ -88,6 +93,7 @@ void finalize(Stream s) { + } + + void synchronize(Stream s) { ++ std::lock_guard lock(metal_operation_mutex); + auto pool = metal::new_scoped_memory_pool(); + auto& d = metal::device(s.device); + auto cb = d.get_command_buffer(s.index); +diff --git a/mlx/backend/metal/event.cpp b/mlx/backend/metal/event.cpp +index eb7f1b58a..5c9cc3808 100644 +--- a/mlx/backend/metal/event.cpp ++++ b/mlx/backend/metal/event.cpp +@@ -2,6 +2,7 @@ + + #include "mlx/event.h" + #include "mlx/backend/metal/device.h" ++#include "mlx/backend/metal/thread_safety.h" + #include "mlx/scheduler.h" + + namespace mlx::core { +@@ -27,6 +28,7 @@ void Event::wait(Stream stream) { + if (stream.device == Device::cpu) { + scheduler::enqueue(stream, [*this]() mutable { wait(); }); + } else { ++ std::lock_guard lock(gpu::metal_operation_mutex); + auto& d = metal::device(stream.device); + d.end_encoding(stream.index); + auto command_buffer = d.get_command_buffer(stream.index); +@@ -41,6 +43,7 @@ void Event::signal(Stream stream) { + static_cast(event_.get())->setSignaledValue(value()); + }); + } else { ++ std::lock_guard lock(gpu::metal_operation_mutex); + auto& d = metal::device(stream.device); + d.end_encoding(stream.index); + auto command_buffer = d.get_command_buffer(stream.index); +diff --git a/mlx/backend/metal/fence.cpp b/mlx/backend/metal/fence.cpp +index 5abdf7309..e1bfdaa9a 100644 +--- a/mlx/backend/metal/fence.cpp ++++ b/mlx/backend/metal/fence.cpp +@@ -1,6 +1,7 @@ + // Copyright © 2024 Apple Inc. + #include "mlx/fence.h" + #include "mlx/backend/metal/device.h" ++#include "mlx/backend/metal/thread_safety.h" + #include "mlx/scheduler.h" + #include "mlx/utils.h" + +@@ -68,6 +69,7 @@ void Fence::wait(Stream stream, const array& x) { + return; + } + ++ std::lock_guard lock(gpu::metal_operation_mutex); + auto& d = metal::device(stream.device); + auto idx = stream.index; + +@@ -116,6 +118,7 @@ void Fence::update(Stream stream, const array& x) { + return; + } + ++ std::lock_guard lock(gpu::metal_operation_mutex); + auto& d = metal::device(stream.device); + auto idx = stream.index; + if (!f.use_fast) { diff --git a/mlx/backend/metal/jit_kernels.cpp b/mlx/backend/metal/jit_kernels.cpp index 5206c9b54..6678e7acb 100644 --- a/mlx/backend/metal/jit_kernels.cpp @@ -719,6 +819,20 @@ index 3c84022f2..bca888f66 100644 } } // namespace mlx::core +diff --git a/mlx/backend/metal/thread_safety.h b/mlx/backend/metal/thread_safety.h +new file mode 100644 +index 000000000..d4724a654 +--- /dev/null ++++ b/mlx/backend/metal/thread_safety.h +@@ -0,0 +1,7 @@ ++#pragma once ++ ++#include ++ ++namespace mlx::core::gpu { ++extern std::mutex metal_operation_mutex; ++} +\ No newline at end of file diff --git a/mlx/backend/no_gpu/primitives.cpp b/mlx/backend/no_gpu/primitives.cpp index 409aa2c89..6d49f1c60 100644 --- a/mlx/backend/no_gpu/primitives.cpp @@ -1219,3 +1333,1141 @@ index 51101cc97..344d80f87 100644 return {real, imag}; } +diff --git a/test.patch b/test.patch +new file mode 100644 +index 000000000..dae2293ba +--- /dev/null ++++ b/test.patch +@@ -0,0 +1,1132 @@ ++diff --git a/.gitignore b/.gitignore ++index 43629548..0b74c60d 100644 ++--- a/.gitignore +++++ b/.gitignore ++@@ -1,3 +1,8 @@ +++# Bazel symlinks +++bazel-* +++ +++# macOS +++.DS_Store ++ # Byte-compiled / optimized / DLL files ++ __pycache__/ ++ *.py[cod] ++diff --git a/mlx/backend/metal/device.cpp b/mlx/backend/metal/device.cpp ++index ebc3cc77..7bdca47d 100644 ++--- a/mlx/backend/metal/device.cpp +++++ b/mlx/backend/metal/device.cpp ++@@ -10,6 +10,7 @@ ++ ++ #include "mlx/backend/metal/device.h" ++ #include "mlx/backend/metal/metal.h" +++#include "mlx/backend/metal/metal_path.h" ++ #include "mlx/backend/metal/utils.h" ++ #include "mlx/utils.h" ++ ++diff --git a/mlx/backend/metal/jit_kernels.cpp b/mlx/backend/metal/jit_kernels.cpp ++index 5206c9b5..6678e7ac 100644 ++--- a/mlx/backend/metal/jit_kernels.cpp +++++ b/mlx/backend/metal/jit_kernels.cpp ++@@ -328,7 +328,8 @@ MTL::ComputePipelineState* get_sort_kernel( ++ const array& in, ++ const array& out, ++ int bn, ++- int tn) { +++ int tn, +++ const std::string& comparator) { ++ std::string lib_name = kernel_name.substr(kernel_name.find("_") + 1); ++ auto lib = d.get_library(lib_name, [&]() { ++ std::ostringstream kernel_source; ++@@ -345,7 +346,8 @@ MTL::ComputePipelineState* get_sort_kernel( ++ out_type, ++ bool_string, ++ bn, ++- tn); +++ tn, +++ comparator); ++ kernel_source << get_template_definition( ++ "n" + func_string + lib_name, ++ "block_sort_nc", ++@@ -353,7 +355,8 @@ MTL::ComputePipelineState* get_sort_kernel( ++ out_type, ++ bool_string, ++ bn, ++- tn); +++ tn, +++ comparator); ++ } ++ return kernel_source.str(); ++ }); ++@@ -366,7 +369,8 @@ MTL::ComputePipelineState* get_mb_sort_kernel( ++ const array& in, ++ const array& idx, ++ int bn, ++- int tn) { +++ int tn, +++ const std::string& comparator) { ++ std::string lib_name = kernel_name.substr(kernel_name.find("_") + 1); ++ auto lib = d.get_library(lib_name, [&]() { ++ std::ostringstream kernel_source; ++@@ -383,7 +387,8 @@ MTL::ComputePipelineState* get_mb_sort_kernel( ++ get_type_string(idx.dtype()), ++ "true", ++ bn, ++- tn); +++ tn, +++ comparator); ++ } ++ return kernel_source.str(); ++ }); ++diff --git a/mlx/backend/metal/kernels.h b/mlx/backend/metal/kernels.h ++index 6d886438..ff2dd14b 100644 ++--- a/mlx/backend/metal/kernels.h +++++ b/mlx/backend/metal/kernels.h ++@@ -79,7 +79,8 @@ MTL::ComputePipelineState* get_sort_kernel( ++ const array& in, ++ const array& out, ++ int bn, ++- int tn); +++ int tn, +++ const std::string& comparator); ++ ++ MTL::ComputePipelineState* get_mb_sort_kernel( ++ metal::Device& d, ++@@ -87,7 +88,8 @@ MTL::ComputePipelineState* get_mb_sort_kernel( ++ const array& in, ++ const array& idx, ++ int bn, ++- int tn); +++ int tn, +++ const std::string& comparator); ++ ++ MTL::ComputePipelineState* get_reduce_init_kernel( ++ metal::Device& d, ++diff --git a/mlx/backend/metal/kernels/binary_ops.h b/mlx/backend/metal/kernels/binary_ops.h ++index 4aaf2b4d..2c6e27b1 100644 ++--- a/mlx/backend/metal/kernels/binary_ops.h +++++ b/mlx/backend/metal/kernels/binary_ops.h ++@@ -37,29 +37,14 @@ struct Divide { ++ return x / y; ++ } ++ }; ++- ++ struct Remainder { ++ template ++- metal::enable_if_t & !metal::is_signed_v, T> ++- operator()(T x, T y) { +++ metal::enable_if_t, T> operator()(T x, T y) { ++ return x % y; ++ } ++ template ++- metal::enable_if_t & metal::is_signed_v, T> ++- operator()(T x, T y) { ++- auto r = x % y; ++- if (r != 0 && (r < 0 != y < 0)) { ++- r += y; ++- } ++- return r; ++- } ++- template ++ metal::enable_if_t, T> operator()(T x, T y) { ++- T r = fmod(x, y); ++- if (r != 0 && (r < 0 != y < 0)) { ++- r += y; ++- } ++- return r; +++ return fmod(x, y); ++ } ++ template <> ++ complex64_t operator()(complex64_t x, complex64_t y) { ++diff --git a/mlx/backend/metal/kernels/complex.h b/mlx/backend/metal/kernels/complex.h ++index c88002cb..54fc8d52 100644 ++--- a/mlx/backend/metal/kernels/complex.h +++++ b/mlx/backend/metal/kernels/complex.h ++@@ -142,11 +142,12 @@ constexpr complex64_t operator/(float a, complex64_t b) { ++ constexpr complex64_t operator%(complex64_t a, complex64_t b) { ++ auto real = a.real - (b.real * static_cast(a.real / b.real)); ++ auto imag = a.imag - (b.imag * static_cast(a.imag / b.imag)); ++- if (real != 0 && (real < 0 != b.real < 0)) { ++- real += b.real; ++- } ++- if (imag != 0 && (imag < 0 != b.imag < 0)) { ++- imag += b.imag; ++- } +++ /* Disable to match `stablehlo` implementation semantics */ +++ // if (real != 0 && (real < 0 != b.real < 0)) { +++ // real += b.real; +++ // } +++ // if (imag != 0 && (imag < 0 != b.imag < 0)) { +++ // imag += b.imag; +++ // } ++ return {real, imag}; ++ } ++diff --git a/mlx/backend/metal/kernels/sort.h b/mlx/backend/metal/kernels/sort.h ++index b067150d..a24e429f 100644 ++--- a/mlx/backend/metal/kernels/sort.h +++++ b/mlx/backend/metal/kernels/sort.h ++@@ -28,6 +28,15 @@ struct LessThan { ++ } ++ }; ++ +++template +++struct GreaterThan { +++ static constexpr constant T init = Limits::min; +++ +++ METAL_FUNC bool operator()(T a, T b) { +++ return a > b; +++ } +++}; +++ ++ template < ++ typename ValT, ++ typename IdxT, ++@@ -216,7 +225,7 @@ template < ++ bool ARG_SORT, ++ short BLOCK_THREADS, ++ short N_PER_THREAD, ++- typename CompareOp = LessThan> +++ typename CompareOp> ++ struct KernelMergeSort { ++ using ValT = T; ++ using IdxT = uint; ++@@ -278,7 +287,8 @@ template < ++ typename U, ++ bool ARG_SORT, ++ short BLOCK_THREADS, ++- short N_PER_THREAD> +++ short N_PER_THREAD, +++ typename CompareOp = LessThan> ++ [[kernel, max_total_threads_per_threadgroup(BLOCK_THREADS)]] void block_sort( ++ const device T* inp [[buffer(0)]], ++ device U* out [[buffer(1)]], ++@@ -290,7 +300,7 @@ template < ++ uint3 tid [[threadgroup_position_in_grid]], ++ uint3 lid [[thread_position_in_threadgroup]]) { ++ using sort_kernel = ++- KernelMergeSort; +++ KernelMergeSort; ++ using ValT = typename sort_kernel::ValT; ++ using IdxT = typename sort_kernel::IdxT; ++ ++@@ -333,7 +343,8 @@ template < ++ typename U, ++ bool ARG_SORT, ++ short BLOCK_THREADS, ++- short N_PER_THREAD> +++ short N_PER_THREAD, +++ typename CompareOp> ++ [[kernel, max_total_threads_per_threadgroup(BLOCK_THREADS)]] void block_sort_nc( ++ const device T* inp [[buffer(0)]], ++ device U* out [[buffer(1)]], ++@@ -347,7 +358,7 @@ template < ++ uint3 tid [[threadgroup_position_in_grid]], ++ uint3 lid [[thread_position_in_threadgroup]]) { ++ using sort_kernel = ++- KernelMergeSort; +++ KernelMergeSort; ++ using ValT = typename sort_kernel::ValT; ++ using IdxT = typename sort_kernel::IdxT; ++ ++@@ -394,7 +405,7 @@ template < ++ bool ARG_SORT, ++ short BLOCK_THREADS, ++ short N_PER_THREAD, ++- typename CompareOp = LessThan> +++ typename CompareOp> ++ struct KernelMultiBlockMergeSort { ++ using block_merge_sort_t = BlockMergeSort< ++ ValT, ++@@ -476,7 +487,8 @@ template < ++ typename IdxT, ++ bool ARG_SORT, ++ short BLOCK_THREADS, ++- short N_PER_THREAD> +++ short N_PER_THREAD, +++ typename CompareOp> ++ [[kernel, max_total_threads_per_threadgroup(BLOCK_THREADS)]] void mb_block_sort( ++ const device ValT* inp [[buffer(0)]], ++ device ValT* out_vals [[buffer(1)]], ++@@ -493,7 +505,8 @@ template < ++ IdxT, ++ ARG_SORT, ++ BLOCK_THREADS, ++- N_PER_THREAD>; +++ N_PER_THREAD, +++ CompareOp>; ++ ++ auto block_idx = elem_to_loc(tid.y, nc_shape, nc_strides, nc_dim); ++ inp += block_idx; ++@@ -520,7 +533,8 @@ template < ++ typename IdxT, ++ bool ARG_SORT, ++ short BLOCK_THREADS, ++- short N_PER_THREAD> +++ short N_PER_THREAD, +++ typename CompareOp> ++ [[kernel]] void mb_block_partition( ++ device IdxT* block_partitions [[buffer(0)]], ++ const device ValT* dev_vals [[buffer(1)]], ++@@ -536,7 +550,8 @@ template < ++ IdxT, ++ ARG_SORT, ++ BLOCK_THREADS, ++- N_PER_THREAD>; +++ N_PER_THREAD, +++ CompareOp>; ++ ++ block_partitions += tid.y * tgp_dims.x; ++ dev_vals += tid.y * size_sorted_axis; ++@@ -573,7 +588,7 @@ template < ++ bool ARG_SORT, ++ short BLOCK_THREADS, ++ short N_PER_THREAD, ++- typename CompareOp = LessThan> +++ typename CompareOp> ++ [[kernel, max_total_threads_per_threadgroup(BLOCK_THREADS)]] void ++ mb_block_merge( ++ const device IdxT* block_partitions [[buffer(0)]], ++diff --git a/mlx/backend/metal/kernels/sort.metal b/mlx/backend/metal/kernels/sort.metal ++index 7f198d55..f1ec796c 100644 ++--- a/mlx/backend/metal/kernels/sort.metal +++++ b/mlx/backend/metal/kernels/sort.metal ++@@ -6,24 +6,29 @@ ++ #include "mlx/backend/metal/kernels/utils.h" ++ #include "mlx/backend/metal/kernels/sort.h" ++ +++#define instantiate_comparator(name, type) \ +++ name +++ ++ #define instantiate_block_sort( \ ++- name, itname, itype, otname, otype, arg_sort, bn, tn) \ ++- instantiate_kernel("c" #name "_" #itname "_" #otname "_bn" #bn "_tn" #tn, \ ++- block_sort, itype, otype, arg_sort, bn, tn) \ ++- instantiate_kernel("nc" #name "_" #itname "_" #otname "_bn" #bn "_tn" #tn, \ ++- block_sort_nc, itype, otype, arg_sort, bn, tn) +++ name, itname, itype, otname, otype, arg_sort, bn, tn, vcomparator) \ +++ instantiate_kernel("c" #name "_" #itname "_" #otname "_bn" #bn "_tn" #tn "_comp" #vcomparator, \ +++ block_sort, itype, otype, arg_sort, bn, tn, instantiate_comparator(vcomparator, itype)) \ +++ instantiate_kernel("nc" #name "_" #itname "_" #otname "_bn" #bn "_tn" #tn "_comp" #vcomparator, \ +++ block_sort_nc, itype, otype, arg_sort, bn, tn, instantiate_comparator(vcomparator, itype)) ++ ++-#define instantiate_arg_block_sort_base(itname, itype, bn, tn) \ +++#define instantiate_arg_block_sort_base(itname, itype, bn, tn, comparator) \ ++ instantiate_block_sort( \ ++- arg_block_sort, itname, itype, uint32, uint32_t, true, bn, tn) +++ arg_block_sort, itname, itype, uint32, uint32_t, true, bn, tn, comparator) ++ ++-#define instantiate_block_sort_base(itname, itype, bn, tn) \ +++#define instantiate_block_sort_base(itname, itype, bn, tn, comparator) \ ++ instantiate_block_sort( \ ++- _block_sort, itname, itype, itname, itype, false, bn, tn) +++ _block_sort, itname, itype, itname, itype, false, bn, tn, comparator) ++ ++ #define instantiate_block_sort_tn(itname, itype, bn) \ ++- instantiate_block_sort_base(itname, itype, bn, 4) \ ++- instantiate_arg_block_sort_base(itname, itype, bn, 4) +++ instantiate_block_sort_base(itname, itype, bn, 4, LessThan) \ +++ instantiate_block_sort_base(itname, itype, bn, 4, GreaterThan) \ +++ instantiate_arg_block_sort_base(itname, itype, bn, 4, LessThan) \ +++ instantiate_arg_block_sort_base(itname, itype, bn, 4, GreaterThan) ++ ++ #define instantiate_block_sort_bn(itname, itype) \ ++ instantiate_block_sort_tn(itname, itype, 32) \ ++@@ -52,16 +57,17 @@ instantiate_block_sort_long(uint64, uint64_t) ++ instantiate_block_sort_long(int64, int64_t) ++ ++ #define instantiate_multi_block_sort( \ ++- vtname, vtype, itname, itype, arg_sort, bn, tn) \ ++- instantiate_kernel("sort_mbsort_" #vtname "_" #itname "_bn" #bn "_tn" #tn, \ ++- mb_block_sort, vtype, itype, arg_sort, bn, tn) \ ++- instantiate_kernel("partition_mbsort_" #vtname "_" #itname "_bn" #bn "_tn" #tn, \ ++- mb_block_partition, vtype, itype, arg_sort, bn, tn) \ ++- instantiate_kernel("merge_mbsort_" #vtname "_" #itname "_bn" #bn "_tn" #tn, \ ++- mb_block_merge, vtype, itype, arg_sort, bn, tn) +++ vtname, vtype, itname, itype, arg_sort, bn, tn, vcomparator) \ +++ instantiate_kernel("sort_mbsort_" #vtname "_" #itname "_bn" #bn "_tn" #tn "_comp" #vcomparator, \ +++ mb_block_sort, vtype, itype, arg_sort, bn, tn, instantiate_comparator(vcomparator, vtype)) \ +++ instantiate_kernel("partition_mbsort_" #vtname "_" #itname "_bn" #bn "_tn" #tn "_comp" #vcomparator, \ +++ mb_block_partition, vtype, itype, arg_sort, bn, tn, instantiate_comparator(vcomparator, vtype)) \ +++ instantiate_kernel("merge_mbsort_" #vtname "_" #itname "_bn" #bn "_tn" #tn "_comp" #vcomparator, \ +++ mb_block_merge, vtype, itype, arg_sort, bn, tn, instantiate_comparator(vcomparator, vtype)) ++ ++ #define instantiate_multi_block_sort_base(vtname, vtype) \ ++- instantiate_multi_block_sort(vtname, vtype, uint32, uint32_t, true, 512, 4) +++ instantiate_multi_block_sort(vtname, vtype, uint32, uint32_t, true, 512, 4, LessThan) \ +++ instantiate_multi_block_sort(vtname, vtype, uint32, uint32_t, true, 512, 4, GreaterThan) ++ ++ instantiate_multi_block_sort_base(uint8, uint8_t) ++ instantiate_multi_block_sort_base(uint16, uint16_t) ++@@ -74,7 +80,8 @@ instantiate_multi_block_sort_base(float32, float) ++ instantiate_multi_block_sort_base(bfloat16, bfloat16_t) ++ ++ #define instantiate_multi_block_sort_long(vtname, vtype) \ ++- instantiate_multi_block_sort(vtname, vtype, uint32, uint32_t, true, 256, 4) +++ instantiate_multi_block_sort(vtname, vtype, uint32, uint32_t, true, 256, 4, LessThan) \ +++ instantiate_multi_block_sort(vtname, vtype, uint32, uint32_t, true, 256, 4, GreaterThan) ++ ++ instantiate_multi_block_sort_long(uint64, uint64_t) ++ instantiate_multi_block_sort_long(int64, int64_t) // clang-format on ++diff --git a/mlx/backend/metal/make_compiled_preamble.sh b/mlx/backend/metal/make_compiled_preamble.sh ++index 425cd8d7..a48d0080 100644 ++--- a/mlx/backend/metal/make_compiled_preamble.sh +++++ b/mlx/backend/metal/make_compiled_preamble.sh ++@@ -5,18 +5,21 @@ ++ # ++ # Copyright © 2023-24 Apple Inc. ++ ++-OUTPUT_DIR=$1 ++-CC=$2 ++-SRC_DIR=$3 ++-SRC_FILE=$4 ++-CFLAGS=$5 ++-SRC_NAME=$(basename -- "${SRC_FILE}") ++-JIT_INCLUDES=${SRC_DIR}/mlx/backend/metal/kernels/jit ++-INPUT_FILE=${SRC_DIR}/mlx/backend/metal/kernels/${SRC_FILE}.h ++-OUTPUT_FILE=${OUTPUT_DIR}/${SRC_NAME}.cpp +++OUTPUT_FILE=$1 +++INPUT_FILE=$2 +++CFLAGS=$3 ++ ++-mkdir -p "$OUTPUT_DIR" ++-CONTENT=$($CC -I"$SRC_DIR" -I"$JIT_INCLUDES" -DMLX_METAL_JIT -E -P "$INPUT_FILE" $CFLAGS 2>/dev/null) +++KERNEL_DIR=$(dirname $(realpath $INPUT_FILE)) +++# echo "KERNEL_DIR: $KERNEL_DIR" +++SRC_DIR=$(echo "$KERNEL_DIR" | cut -d'/' -f-$(($(echo "$KERNEL_DIR" | awk -F'/' '{print NF}')-4))) +++# echo "SRC_DIR: $SRC_DIR" +++JIT_INCLUDES="$KERNEL_DIR/jit" +++# echo "JIT_INCLUDES: $JIT_INCLUDES" +++ +++SRC_BASENAME=$(basename -- "${INPUT_FILE}") +++SRC_NAME="${SRC_BASENAME%.*}" +++ +++CONTENT=$(clang -I"$SRC_DIR" -I"$JIT_INCLUDES" -DMLX_METAL_JIT -E -P "$INPUT_FILE" $CFLAGS 2>/dev/null) ++ ++ cat << EOF > "$OUTPUT_FILE" ++ namespace mlx::core::metal { ++diff --git a/mlx/backend/metal/nojit_kernels.cpp b/mlx/backend/metal/nojit_kernels.cpp ++index 8da14797..1098f918 100644 ++--- a/mlx/backend/metal/nojit_kernels.cpp +++++ b/mlx/backend/metal/nojit_kernels.cpp ++@@ -96,7 +96,8 @@ MTL::ComputePipelineState* get_sort_kernel( ++ const array&, ++ const array&, ++ int, ++- int) { +++ int, +++ const std::string&) { ++ return d.get_kernel(kernel_name); ++ } ++ ++@@ -106,7 +107,8 @@ MTL::ComputePipelineState* get_mb_sort_kernel( ++ const array&, ++ const array&, ++ int, ++- int) { +++ int, +++ const std::string&) { ++ return d.get_kernel(kernel_name); ++ } ++ ++diff --git a/mlx/backend/metal/sort.cpp b/mlx/backend/metal/sort.cpp ++index 3c84022f..bca888f6 100644 ++--- a/mlx/backend/metal/sort.cpp +++++ b/mlx/backend/metal/sort.cpp ++@@ -6,7 +6,9 @@ ++ #include "mlx/backend/metal/device.h" ++ #include "mlx/backend/metal/kernels.h" ++ #include "mlx/backend/metal/utils.h" +++#include "mlx/dtype_utils.h", ++ #include "mlx/primitives.h" +++#include "mlx/sort.h" ++ ++ namespace mlx::core { ++ ++@@ -20,7 +22,8 @@ void single_block_sort( ++ int axis, ++ int bn, ++ int tn, ++- bool argsort) { +++ bool argsort, +++ ComparatorType comparator_) { ++ // Prepare shapes ++ int n_rows = in.size() / in.shape(axis); ++ ++@@ -58,9 +61,26 @@ void single_block_sort( ++ kname << "arg"; ++ } ++ +++ std::string comparator; +++ switch (comparator_) { +++ case ComparatorType::LessThan: +++ comparator = "LessThan"; +++ break; +++ case ComparatorType::GreaterThan: +++ comparator = "GreaterThan"; +++ break; +++ } +++ ++ kname << "_block_sort_" << type_to_name(in) << "_" << type_to_name(out) ++- << "_bn" << bn << "_tn" << tn; ++- auto kernel = get_sort_kernel(d, kname.str(), in, out, bn, tn); +++ << "_bn" << bn << "_tn" << tn << "_comp" << comparator; +++ auto kernel = get_sort_kernel( +++ d, +++ kname.str(), +++ in, +++ out, +++ bn, +++ tn, +++ comparator + "<" + dtype_to_string(in.dtype()) + ">"); ++ ++ // Prepare command encoder ++ auto& compute_encoder = d.get_command_encoder(s.index); ++@@ -120,7 +140,8 @@ void multi_block_sort( ++ int bn, ++ int tn, ++ int n_blocks, ++- bool argsort) { +++ bool argsort, +++ ComparatorType comparator_) { ++ // Prepare shapes ++ int n_rows = in.size() / in.shape(axis); ++ ++@@ -159,6 +180,16 @@ void multi_block_sort( ++ std::vector copies = { ++ dev_vals_0, dev_vals_1, dev_idxs_0, dev_idxs_1, block_partitions}; ++ +++ std::string comparator; +++ switch (comparator_) { +++ case ComparatorType::LessThan: +++ comparator = "LessThan"; +++ break; +++ case ComparatorType::GreaterThan: +++ comparator = "GreaterThan"; +++ break; +++ } +++ ++ // Prepare command encoder ++ auto& compute_encoder = d.get_command_encoder(s.index); ++ ++@@ -166,9 +197,16 @@ void multi_block_sort( ++ { ++ std::ostringstream kname; ++ kname << "sort_mbsort_" << type_to_name(dev_vals_0) << "_" ++- << type_to_name(dev_idxs_0) << "_bn" << bn << "_tn" << tn; ++- auto kernel = ++- get_mb_sort_kernel(d, kname.str(), dev_vals_0, dev_idxs_0, bn, tn); +++ << type_to_name(dev_idxs_0) << "_bn" << bn << "_tn" << tn << "_comp" +++ << comparator; +++ auto kernel = get_mb_sort_kernel( +++ d, +++ kname.str(), +++ dev_vals_0, +++ dev_idxs_0, +++ bn, +++ tn, +++ comparator + "<" + dtype_to_string(in.dtype()) + ">"); ++ compute_encoder.set_compute_pipeline_state(kernel); ++ ++ compute_encoder.set_input_array(in, 0); ++@@ -206,10 +244,17 @@ void multi_block_sort( ++ { ++ std::ostringstream kname; ++ kname << "partition_mbsort_" << type_to_name(dev_vals_in) << "_" ++- << type_to_name(dev_idxs_in) << "_bn" << bn << "_tn" << tn; ++- ++- auto kernel = ++- get_mb_sort_kernel(d, kname.str(), dev_vals_0, dev_idxs_0, bn, tn); +++ << type_to_name(dev_idxs_in) << "_bn" << bn << "_tn" << tn +++ << "_comp" << comparator; +++ +++ auto kernel = get_mb_sort_kernel( +++ d, +++ kname.str(), +++ dev_vals_0, +++ dev_idxs_0, +++ bn, +++ tn, +++ comparator + "<" + dtype_to_string(in.dtype()) + ">"); ++ compute_encoder.set_compute_pipeline_state(kernel); ++ ++ compute_encoder.set_output_array(block_partitions, 0); ++@@ -229,10 +274,17 @@ void multi_block_sort( ++ { ++ std::ostringstream kname; ++ kname << "merge_mbsort_" << type_to_name(dev_vals_in) << "_" ++- << type_to_name(dev_idxs_in) << "_bn" << bn << "_tn" << tn; ++- ++- auto kernel = ++- get_mb_sort_kernel(d, kname.str(), dev_vals_0, dev_idxs_0, bn, tn); +++ << type_to_name(dev_idxs_in) << "_bn" << bn << "_tn" << tn +++ << "_comp" << comparator; +++ +++ auto kernel = get_mb_sort_kernel( +++ d, +++ kname.str(), +++ dev_vals_0, +++ dev_idxs_0, +++ bn, +++ tn, +++ comparator + "<" + dtype_to_string(in.dtype()) + ">"); ++ compute_encoder.set_compute_pipeline_state(kernel); ++ ++ compute_encoder.set_input_array(block_partitions, 0); ++@@ -277,7 +329,8 @@ void gpu_merge_sort( ++ const array& in, ++ array& out, ++ int axis_, ++- bool argsort) { +++ bool argsort, +++ ComparatorType comparator) { ++ // Get size info ++ int axis = axis_ < 0 ? axis_ + in.ndim() : axis_; ++ int size_sorted_axis = in.shape(axis); ++@@ -307,9 +360,10 @@ void gpu_merge_sort( ++ int n_blocks = (size_sorted_axis + n_per_block - 1) / n_per_block; ++ ++ if (n_blocks > 1) { ++- return multi_block_sort(s, d, in, out, axis, bn, tn, n_blocks, argsort); +++ return multi_block_sort( +++ s, d, in, out, axis, bn, tn, n_blocks, argsort, comparator); ++ } else { ++- return single_block_sort(s, d, in, out, axis, bn, tn, argsort); +++ return single_block_sort(s, d, in, out, axis, bn, tn, argsort, comparator); ++ } ++ } ++ ++@@ -324,7 +378,7 @@ void ArgSort::eval_gpu(const std::vector& inputs, array& out) { ++ auto& d = metal::device(s.device); ++ auto& in = inputs[0]; ++ ++- gpu_merge_sort(s, d, in, out, axis_, true); +++ gpu_merge_sort(s, d, in, out, axis_, true, comparator_); ++ } ++ ++ void Sort::eval_gpu(const std::vector& inputs, array& out) { ++@@ -336,7 +390,7 @@ void Sort::eval_gpu(const std::vector& inputs, array& out) { ++ auto& d = metal::device(s.device); ++ auto& in = inputs[0]; ++ ++- gpu_merge_sort(s, d, in, out, axis_, false); +++ gpu_merge_sort(s, d, in, out, axis_, false, comparator_); ++ } ++ ++ void ArgPartition::eval_gpu(const std::vector& inputs, array& out) { ++@@ -349,7 +403,7 @@ void ArgPartition::eval_gpu(const std::vector& inputs, array& out) { ++ auto& d = metal::device(s.device); ++ auto& in = inputs[0]; ++ ++- gpu_merge_sort(s, d, in, out, axis_, true); +++ gpu_merge_sort(s, d, in, out, axis_, true, comparator_); ++ } ++ ++ void Partition::eval_gpu(const std::vector& inputs, array& out) { ++@@ -362,7 +416,7 @@ void Partition::eval_gpu(const std::vector& inputs, array& out) { ++ auto& d = metal::device(s.device); ++ auto& in = inputs[0]; ++ ++- gpu_merge_sort(s, d, in, out, axis_, false); +++ gpu_merge_sort(s, d, in, out, axis_, false, comparator_); ++ } ++ ++ } // namespace mlx::core ++diff --git a/mlx/backend/no_gpu/primitives.cpp b/mlx/backend/no_gpu/primitives.cpp ++index 676a6e55..178eb23f 100644 ++--- a/mlx/backend/no_gpu/primitives.cpp +++++ b/mlx/backend/no_gpu/primitives.cpp ++@@ -95,7 +95,7 @@ NO_GPU(Partition) ++ NO_GPU(Power) ++ NO_GPU_MULTI(QRF) ++ NO_GPU(QuantizedMatmul) ++-NO_GPU(RandomBits) +++NO_GPU() ++ NO_GPU(Real) ++ NO_GPU(Reduce) ++ NO_GPU(Reshape) ++diff --git a/mlx/mlx.h b/mlx/mlx.h ++index cef8d806..f8932873 100644 ++--- a/mlx/mlx.h +++++ b/mlx/mlx.h ++@@ -17,6 +17,7 @@ ++ #include "mlx/memory.h" ++ #include "mlx/ops.h" ++ #include "mlx/random.h" +++#include "mlx/sort.h" ++ #include "mlx/stream.h" ++ #include "mlx/transforms.h" ++ #include "mlx/utils.h" ++diff --git a/mlx/ops.cpp b/mlx/ops.cpp ++index a72c2bc8..04ac921f 100644 ++--- a/mlx/ops.cpp +++++ b/mlx/ops.cpp ++@@ -2205,13 +2205,20 @@ array argmax( ++ } ++ ++ /** Returns a sorted copy of the flattened array. */ ++-array sort(const array& a, StreamOrDevice s /* = {} */) { +++array sort( +++ const array& a, +++ ComparatorType comparator, +++ StreamOrDevice s /* = {} */) { ++ int size = a.size(); ++- return sort(reshape(a, {size}, s), 0, s); +++ return sort(reshape(a, {size}, s), 0, comparator, s); ++ } ++ ++ /** Returns a sorted copy of the array along a given axis. */ ++-array sort(const array& a, int axis, StreamOrDevice s /* = {} */) { +++array sort( +++ const array& a, +++ int axis, +++ ComparatorType comparator, +++ StreamOrDevice s /* = {} */) { ++ // Check for valid axis ++ if (axis + static_cast(a.ndim()) < 0 || ++ axis >= static_cast(a.ndim())) { ++@@ -2222,17 +2229,27 @@ array sort(const array& a, int axis, StreamOrDevice s /* = {} */) { ++ } ++ ++ return array( ++- a.shape(), a.dtype(), std::make_shared(to_stream(s), axis), {a}); +++ a.shape(), +++ a.dtype(), +++ std::make_shared(to_stream(s), axis, comparator), +++ {a}); ++ } ++ ++ /** Returns indices that sort the flattened array. */ ++-array argsort(const array& a, StreamOrDevice s /* = {} */) { +++array argsort( +++ const array& a, +++ ComparatorType comparator, +++ StreamOrDevice s /* = {} */) { ++ int size = a.size(); ++- return argsort(reshape(a, {size}, s), 0, s); +++ return argsort(reshape(a, {size}, s), 0, comparator, s); ++ } ++ ++ /** Returns indices that sort the array along a given axis. */ ++-array argsort(const array& a, int axis, StreamOrDevice s /* = {} */) { +++array argsort( +++ const array& a, +++ int axis, +++ ComparatorType comparator, +++ StreamOrDevice s /* = {} */) { ++ // Check for valid axis ++ if (axis + static_cast(a.ndim()) < 0 || ++ axis >= static_cast(a.ndim())) { ++@@ -2243,16 +2260,23 @@ array argsort(const array& a, int axis, StreamOrDevice s /* = {} */) { ++ } ++ ++ return array( ++- a.shape(), uint32, std::make_shared(to_stream(s), axis), {a}); +++ a.shape(), +++ uint32, +++ std::make_shared(to_stream(s), axis, comparator), +++ {a}); ++ } ++ ++ /** ++ * Returns a partitioned copy of the flattened array ++ * such that the smaller kth elements are first. ++ **/ ++-array partition(const array& a, int kth, StreamOrDevice s /* = {} */) { +++array partition( +++ const array& a, +++ int kth, +++ ComparatorType comparator, +++ StreamOrDevice s /* = {} */) { ++ int size = a.size(); ++- return partition(reshape(a, {size}, s), kth, 0, s); +++ return partition(reshape(a, {size}, s), kth, 0, comparator, s); ++ } ++ ++ /** ++@@ -2263,6 +2287,7 @@ array partition( ++ const array& a, ++ int kth, ++ int axis, +++ ComparatorType comparator, ++ StreamOrDevice s /* = {} */) { ++ // Check for valid axis ++ if (axis + static_cast(a.ndim()) < 0 || ++@@ -2283,7 +2308,7 @@ array partition( ++ return array( ++ a.shape(), ++ a.dtype(), ++- std::make_shared(to_stream(s), kth_, axis_), +++ std::make_shared(to_stream(s), kth_, axis_, comparator), ++ {a}); ++ } ++ ++@@ -2291,9 +2316,13 @@ array partition( ++ * Returns indices that partition the flattened array ++ * such that the smaller kth elements are first. ++ **/ ++-array argpartition(const array& a, int kth, StreamOrDevice s /* = {} */) { +++array argpartition( +++ const array& a, +++ int kth, +++ ComparatorType comparator, +++ StreamOrDevice s /* = {} */) { ++ int size = a.size(); ++- return argpartition(reshape(a, {size}, s), kth, 0, s); +++ return argpartition(reshape(a, {size}, s), kth, 0, comparator, s); ++ } ++ ++ /** ++@@ -2304,6 +2333,7 @@ array argpartition( ++ const array& a, ++ int kth, ++ int axis, +++ ComparatorType comparator, ++ StreamOrDevice s /* = {} */) { ++ // Check for valid axis ++ if (axis + static_cast(a.ndim()) < 0 || ++@@ -2324,7 +2354,7 @@ array argpartition( ++ return array( ++ a.shape(), ++ uint32, ++- std::make_shared(to_stream(s), kth_, axis_), +++ std::make_shared(to_stream(s), kth_, axis_, comparator), ++ {a}); ++ } ++ ++diff --git a/mlx/ops.h b/mlx/ops.h ++index af3cdb5b..be762b8a 100644 ++--- a/mlx/ops.h +++++ b/mlx/ops.h ++@@ -6,6 +6,7 @@ ++ ++ #include "mlx/array.h" ++ #include "mlx/device.h" +++#include "mlx/sort.h" ++ #include "mlx/stream.h" ++ #include "mlx/utils.h" ++ ++@@ -674,40 +675,98 @@ array argmax( ++ StreamOrDevice s = {}); ++ ++ /** Returns a sorted copy of the flattened array. */ ++-array sort(const array& a, StreamOrDevice s = {}); +++array sort( +++ const array& a, +++ ComparatorType comparator = ComparatorType::LessThan, +++ StreamOrDevice s = {}); +++inline array sort(const array& a, StreamOrDevice s = {}) { +++ return sort(a, ComparatorType::LessThan, s); +++} ++ ++ /** Returns a sorted copy of the array along a given axis. */ ++-array sort(const array& a, int axis, StreamOrDevice s = {}); +++array sort( +++ const array& a, +++ int axis, +++ ComparatorType comparator = ComparatorType::LessThan, +++ StreamOrDevice s = {}); +++inline array sort(const array& a, int axis, StreamOrDevice s = {}) { +++ return sort(a, axis, ComparatorType::LessThan, s); +++} ++ ++ /** Returns indices that sort the flattened array. */ ++-array argsort(const array& a, StreamOrDevice s = {}); +++array argsort( +++ const array& a, +++ ComparatorType comparator = ComparatorType::LessThan, +++ StreamOrDevice s = {}); +++inline array argsort(const array& a, StreamOrDevice s = {}) { +++ return argsort(a, ComparatorType::LessThan, s); +++} ++ ++ /** Returns indices that sort the array along a given axis. */ ++-array argsort(const array& a, int axis, StreamOrDevice s = {}); +++array argsort( +++ const array& a, +++ int axis, +++ ComparatorType comparator = ComparatorType::LessThan, +++ StreamOrDevice s = {}); +++inline array argsort(const array& a, int axis, StreamOrDevice s = {}) { +++ return argsort(a, axis, ComparatorType::LessThan, s); +++} ++ ++ /** ++ * Returns a partitioned copy of the flattened array ++ * such that the smaller kth elements are first. ++ **/ ++-array partition(const array& a, int kth, StreamOrDevice s = {}); +++array partition( +++ const array& a, +++ int kth, +++ ComparatorType comparator = ComparatorType::LessThan, +++ StreamOrDevice s = {}); +++inline array partition(const array& a, int kth, StreamOrDevice s = {}) { +++ return partition(a, kth, ComparatorType::LessThan, s); +++} ++ ++ /** ++ * Returns a partitioned copy of the array along a given axis ++ * such that the smaller kth elements are first. ++ **/ ++-array partition(const array& a, int kth, int axis, StreamOrDevice s = {}); +++array partition( +++ const array& a, +++ int kth, +++ int axis, +++ ComparatorType comparator = ComparatorType::LessThan, +++ StreamOrDevice s = {}); +++inline array +++partition(const array& a, int kth, int axis, StreamOrDevice s = {}) { +++ return partition(a, kth, axis, ComparatorType::LessThan, s); +++} ++ ++ /** ++ * Returns indices that partition the flattened array ++ * such that the smaller kth elements are first. ++ **/ ++-array argpartition(const array& a, int kth, StreamOrDevice s = {}); +++array argpartition( +++ const array& a, +++ int kth, +++ ComparatorType comparator = ComparatorType::LessThan, +++ StreamOrDevice s = {}); +++inline array argpartition(const array& a, int kth, StreamOrDevice s = {}) { +++ return argpartition(a, kth, ComparatorType::LessThan, s); +++} ++ ++ /** ++ * Returns indices that partition the array along a given axis ++ * such that the smaller kth elements are first. ++ **/ ++-array argpartition(const array& a, int kth, int axis, StreamOrDevice s = {}); +++array argpartition( +++ const array& a, +++ int kth, +++ int axis, +++ ComparatorType comparator = ComparatorType::LessThan, +++ StreamOrDevice s = {}); +++inline array +++argpartition(const array& a, int kth, int axis, StreamOrDevice s = {}) { +++ return argpartition(a, kth, axis, ComparatorType::LessThan, s); +++} ++ ++ /** Returns topk elements of the flattened array. */ ++ array topk(const array& a, int k, StreamOrDevice s = {}); ++diff --git a/mlx/primitives.h b/mlx/primitives.h ++index c0fbfc84..13d67201 100644 ++--- a/mlx/primitives.h +++++ b/mlx/primitives.h ++@@ -7,6 +7,7 @@ ++ #include "mlx/array.h" ++ #include "mlx/device.h" ++ #include "mlx/io/load.h" +++#include "mlx/sort.h" ++ #include "mlx/stream.h" ++ ++ #define DEFINE_VMAP() \ ++@@ -322,8 +323,15 @@ class ArcTanh : public UnaryPrimitive { ++ ++ class ArgPartition : public UnaryPrimitive { ++ public: ++- explicit ArgPartition(Stream stream, int kth, int axis) ++- : UnaryPrimitive(stream), kth_(kth), axis_(axis) {} +++ explicit ArgPartition( +++ Stream stream, +++ int kth, +++ int axis, +++ ComparatorType comparator) +++ : UnaryPrimitive(stream), +++ kth_(kth), +++ axis_(axis), +++ comparator_(comparator) {} ++ ++ void eval_cpu(const std::vector& inputs, array& out) override; ++ void eval_gpu(const std::vector& inputs, array& out) override; ++@@ -333,13 +341,14 @@ class ArgPartition : public UnaryPrimitive { ++ DEFINE_PRINT(ArgPartition) ++ DEFINE_INPUT_OUTPUT_SHAPE() ++ bool is_equivalent(const Primitive& other) const override; ++- std::pair state() const { ++- return {kth_, axis_}; +++ auto state() const { +++ return std::make_tuple(kth_, axis_, comparator_); ++ }; ++ ++ private: ++ int kth_; ++ int axis_; +++ ComparatorType comparator_; ++ }; ++ ++ class ArgReduce : public UnaryPrimitive { ++@@ -371,8 +380,8 @@ class ArgReduce : public UnaryPrimitive { ++ ++ class ArgSort : public UnaryPrimitive { ++ public: ++- explicit ArgSort(Stream stream, int axis) ++- : UnaryPrimitive(stream), axis_(axis) {} +++ explicit ArgSort(Stream stream, int axis, ComparatorType comparator) +++ : UnaryPrimitive(stream), axis_(axis), comparator_(comparator) {} ++ ++ void eval_cpu(const std::vector& inputs, array& out) override; ++ void eval_gpu(const std::vector& inputs, array& out) override; ++@@ -381,12 +390,13 @@ class ArgSort : public UnaryPrimitive { ++ DEFINE_PRINT(ArgSort) ++ DEFINE_INPUT_OUTPUT_SHAPE() ++ bool is_equivalent(const Primitive& other) const override; ++- int state() const { ++- return axis_; +++ auto state() const { +++ return std::make_pair(axis_, comparator_); ++ }; ++ ++ private: ++ int axis_; +++ ComparatorType comparator_; ++ }; ++ ++ class AsType : public UnaryPrimitive { ++@@ -1530,8 +1540,15 @@ class Pad : public UnaryPrimitive { ++ ++ class Partition : public UnaryPrimitive { ++ public: ++- explicit Partition(Stream stream, int kth, int axis) ++- : UnaryPrimitive(stream), kth_(kth), axis_(axis) {} +++ explicit Partition( +++ Stream stream, +++ int kth, +++ int axis, +++ ComparatorType comparator) +++ : UnaryPrimitive(stream), +++ kth_(kth), +++ axis_(axis), +++ comparator_(comparator) {} ++ ++ void eval_cpu(const std::vector& inputs, array& out) override; ++ void eval_gpu(const std::vector& inputs, array& out) override; ++@@ -1542,12 +1559,13 @@ class Partition : public UnaryPrimitive { ++ DEFINE_INPUT_OUTPUT_SHAPE() ++ bool is_equivalent(const Primitive& other) const override; ++ auto state() const { ++- return std::make_pair(kth_, axis_); +++ return std::make_tuple(kth_, axis_, comparator_); ++ }; ++ ++ private: ++ int kth_; ++ int axis_; +++ ComparatorType comparator_; ++ }; ++ ++ class Power : public UnaryPrimitive { ++@@ -2073,8 +2091,8 @@ class Softmax : public UnaryPrimitive { ++ ++ class Sort : public UnaryPrimitive { ++ public: ++- explicit Sort(Stream stream, int axis) ++- : UnaryPrimitive(stream), axis_(axis) {} +++ explicit Sort(Stream stream, int axis, ComparatorType comparator) +++ : UnaryPrimitive(stream), axis_(axis), comparator_(comparator) {} ++ ++ void eval_cpu(const std::vector& inputs, array& out) override; ++ void eval_gpu(const std::vector& inputs, array& out) override; ++@@ -2085,11 +2103,12 @@ class Sort : public UnaryPrimitive { ++ DEFINE_INPUT_OUTPUT_SHAPE() ++ bool is_equivalent(const Primitive& other) const override; ++ auto state() const { ++- return axis_; +++ return std::make_pair(axis_, comparator_); ++ } ++ ++ private: ++ int axis_; +++ ComparatorType comparator_; ++ }; ++ ++ class Split : public Primitive { ++diff --git a/mlx/random.cpp b/mlx/random.cpp ++index 6c6d1eb9..5a4745a5 100644 ++--- a/mlx/random.cpp +++++ b/mlx/random.cpp ++@@ -225,6 +225,23 @@ array normal( ++ return samples; ++ } ++ +++array normal( +++ const Shape& shape, +++ Dtype dtype, +++ const array& loc, +++ const array& scale, +++ const std::optional& key /*= nullopt */, +++ StreamOrDevice s /* = {} */) { +++ auto stream = to_stream(s); +++ auto low = above_minus_one_with_default(dtype); +++ auto high = array(1.0f, dtype); +++ auto samples = uniform(low, high, shape, dtype, key, stream); +++ samples = multiply(array(std::sqrt(2.0), dtype), erfinv(samples, stream), stream); +++ samples = multiply(astype(scale, dtype), samples, stream); +++ samples = add(astype(loc, dtype), samples, stream); +++ return samples; +++} +++ ++ array multivariate_normal( ++ const array& mean, ++ const array& cov, ++diff --git a/mlx/random.h b/mlx/random.h ++index 0dfdab7a..ebee726c 100644 ++--- a/mlx/random.h +++++ b/mlx/random.h ++@@ -134,6 +134,14 @@ inline array normal( ++ return normal(shape, float32, std::nullopt, std::nullopt, key, s); ++ } ++ +++array normal( +++ const Shape& shape, +++ Dtype dtype, +++ const array& loc, +++ const array& scale, +++ const std::optional& key = std::nullopt, +++ StreamOrDevice s = {}); +++ ++ /** Generate samples from a multivariate normal distribution. **/ ++ array multivariate_normal( ++ const array& mean, ++diff --git a/mlx/sort.h b/mlx/sort.h ++new file mode 100644 ++index 00000000..0afc6b62 ++--- /dev/null +++++ b/mlx/sort.h ++@@ -0,0 +1,14 @@ +++// Copyright © 2023-2024 Apple Inc. +++ +++#pragma once +++ +++#include "mlx/array.h" +++ +++namespace mlx::core { +++ +++enum class ComparatorType { +++ LessThan, +++ GreaterThan, +++}; +++ +++} // namespace mlx::core ++diff --git a/mlx/types/complex.h b/mlx/types/complex.h ++index 51101cc9..344d80f8 100644 ++--- a/mlx/types/complex.h +++++ b/mlx/types/complex.h ++@@ -59,10 +59,11 @@ inline bool operator>(const complex64_t& a, const complex64_t& b) { ++ inline complex64_t operator%(complex64_t a, complex64_t b) { ++ auto real = a.real() - (b.real() * static_cast(a.real() / b.real())); ++ auto imag = a.imag() - (b.imag() * static_cast(a.imag() / b.imag())); ++- if (real != 0 && ((real < 0) != (b.real() < 0))) ++- real += b.real(); ++- if (imag != 0 && ((imag < 0) != (b.imag() < 0))) ++- imag += b.imag(); +++ /* Disable to match `stablehlo` implementation semantics */ +++ // if (real != 0 && ((real < 0) != (b.real() < 0))) +++ // real += b.real(); +++ // if (imag != 0 && ((imag < 0) != (b.imag() < 0))) +++ // imag += b.imag(); ++ return {real, imag}; ++ } ++ diff --git a/third_party/modules/xla/20250527.0-cb67f2f/patches/0004-Added-MLX-PjRt-Plugin.patch b/third_party/modules/xla/20250527.0-cb67f2f/patches/0004-Added-MLX-PjRt-Plugin.patch index a47c5373..6eeb2ca1 100644 --- a/third_party/modules/xla/20250527.0-cb67f2f/patches/0004-Added-MLX-PjRt-Plugin.patch +++ b/third_party/modules/xla/20250527.0-cb67f2f/patches/0004-Added-MLX-PjRt-Plugin.patch @@ -1,9 +1,9 @@ diff --git a/xla/pjrt/plugin/mlx/BUILD b/xla/pjrt/plugin/mlx/BUILD new file mode 100644 -index 0000000000..94bc83ea41 +index 0000000000..8255851b7f --- /dev/null +++ b/xla/pjrt/plugin/mlx/BUILD -@@ -0,0 +1,205 @@ +@@ -0,0 +1,243 @@ +load("//xla:xla.default.bzl", "xla_cc_binary", "xla_cc_test") + +package( @@ -39,6 +39,45 @@ index 0000000000..94bc83ea41 + ], +) + ++cc_library( ++ name = "compile", ++ srcs = ["compile.cc"], ++ hdrs = ["compile.h"], ++ deps = [ ++ "@mlx//:mlx", ++ ":buffer", ++ ":logging", ++ ":utils", ++ "//xla/hlo/builder:xla_computation", ++ "//xla/hlo/translate:stablehlo", ++ "//xla/mlir/utils:error_util", ++ "//xla/mlir_hlo:mhlo_passes", ++ "//xla/mlir_hlo:stablehlo_extension_passes", ++ "//xla/mlir/utils:type_util", ++ # "//xla/pjrt:mlir_to_hlo", ++ "//xla/pjrt:pjrt_client", ++ "//xla/pjrt:pjrt_executable", ++ "//xla/pjrt:pjrt_future", ++ "//xla/service:computation_placer_hdr", ++ "@com_google_absl//absl/log", ++ "@com_google_absl//absl/status", ++ "@com_google_absl//absl/status:statusor", ++ "@llvm-project//llvm:Support", ++ "@llvm-project//mlir:AsmParser", ++ "@llvm-project//mlir:BytecodeWriter", ++ "@llvm-project//mlir:DataLayoutInterfaces", ++ "@llvm-project//mlir:FuncDialect", ++ "@llvm-project//mlir:IR", ++ "@llvm-project//mlir:Parser", ++ "@llvm-project//mlir:Pass", ++ "@llvm-project//mlir:Support", ++ "@llvm-project//mlir:Transforms", ++ "@stablehlo//:reference_api", ++ "@stablehlo//:stablehlo_passes_optimization", ++ "@tsl//tsl/platform:statusor", ++ ], ++) ++ + +cc_library( + name = "executable", @@ -47,6 +86,7 @@ index 0000000000..94bc83ea41 + deps = [ + "@mlx//:mlx", + ":buffer", ++ ":compile", + ":logging", + ":utils", + "//xla/hlo/builder:xla_computation", @@ -169,19 +209,17 @@ index 0000000000..94bc83ea41 + srcs = ["utils.cc"], + hdrs = ["utils.h"], + deps = [ -+ "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/types:span", -+ "@llvm-project//llvm:Support", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Support", + "@mlx//:mlx", -+ "@tsl//tsl/platform:statusor", ++ "@stablehlo//:reference_api", + "//xla/mlir/utils:type_util", + "//xla:literal", + "//xla:shape_util", + "//xla:util", -+ ++ ":logging", + ], +) + @@ -212,10 +250,10 @@ index 0000000000..94bc83ea41 \ No newline at end of file diff --git a/xla/pjrt/plugin/mlx/buffer.cc b/xla/pjrt/plugin/mlx/buffer.cc new file mode 100644 -index 0000000000..750a793c1f +index 0000000000..f87671b3d5 --- /dev/null +++ b/xla/pjrt/plugin/mlx/buffer.cc -@@ -0,0 +1,366 @@ +@@ -0,0 +1,372 @@ +/* Copyright 2024 The OpenXLA Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. @@ -324,7 +362,7 @@ index 0000000000..750a793c1f + MlxPjrtBuffer& operator=(MlxPjrtBuffer&&) = delete; + + static std::unique_ptr CreateFromLiteral( -+ const mx::array& array, const xla::LiteralSlice& literal, ++ mx::array array, const xla::LiteralSlice& literal, + xla::PjRtMemorySpace* memory_space) { + TRACE_ME; + auto buffer = @@ -452,12 +490,12 @@ index 0000000000..750a793c1f + void Delete() override { + // TRACE_ME_MEMBER; + buffer_ = {}; -+ array_ = mx::array({}); // Reset the array to empty. ++ is_deleted_ = true; + } + + bool IsDeleted() override { + // TRACE_ME_MEMBER; -+ return !buffer_; ++ return !buffer_ && is_deleted_; + } + + absl::StatusOr> CopyToDevice( @@ -498,11 +536,12 @@ index 0000000000..750a793c1f + + mlir::DenseElementsAttr GetBufferAttribute() const { return buffer_; } + -+ mx::array& GetArray() { return array_; } ++ mx::array GetArray() { return array_; } + + private: + MLIRContext context_; + mlir::DenseElementsAttr buffer_; ++ bool is_deleted_ = false; + mx::array array_; + + xla::Shape shape_; @@ -510,14 +549,13 @@ index 0000000000..750a793c1f +}; + +std::unique_ptr CreateMlirBufferFromMlxArray( -+ const mx::array& array, xla::PjRtMemorySpace* memory_space) { ++ mx::array array, xla::PjRtMemorySpace* memory_space) { + TRACE_ME; + std::vector span_shape(array.shape().begin(), array.shape().end()); + auto shape = xla::ShapeUtil::MakeShape( + utils::dtype::asXlaPrimitiveType(array.dtype()), + absl::Span(span_shape.data(), span_shape.size())); -+ auto literal = xla::BorrowingLiteral( -+ reinterpret_cast(array.data()), shape); ++ auto literal = xla::BorrowingLiteral(array.data(), shape); + return MlxPjrtBuffer::CreateFromLiteral(array, literal, memory_space); +} + @@ -555,6 +593,9 @@ index 0000000000..750a793c1f + return xla::InvalidArgument("Buffer is null or deleted"); + } + auto mlir_buffer = dynamic_cast(buffer); ++ if (mlir_buffer->IsDeleted()) { ++ return xla::InvalidArgument("Buffer is deleted"); ++ } + if (mlir_buffer == nullptr) { + return xla::InvalidArgument("Buffer is not a MlxPjrtBuffer"); + } @@ -567,6 +608,9 @@ index 0000000000..750a793c1f + return xla::InvalidArgument("Buffer is null or deleted"); + } + auto mlir_buffer = dynamic_cast(buffer); ++ if (mlir_buffer->IsDeleted()) { ++ return xla::InvalidArgument("Buffer is deleted"); ++ } + if (mlir_buffer == nullptr) { + return xla::InvalidArgument("Buffer is not a MlxPjrtBuffer"); + } @@ -585,7 +629,7 @@ index 0000000000..750a793c1f \ No newline at end of file diff --git a/xla/pjrt/plugin/mlx/buffer.h b/xla/pjrt/plugin/mlx/buffer.h new file mode 100644 -index 0000000000..da37e97645 +index 0000000000..1740f3c779 --- /dev/null +++ b/xla/pjrt/plugin/mlx/buffer.h @@ -0,0 +1,43 @@ @@ -615,7 +659,7 @@ index 0000000000..da37e97645 + +namespace mlir::stablehlo { +std::unique_ptr CreateMlirBufferFromMlxArray( -+ const mx::array& array, xla::PjRtMemorySpace* memory_space); ++ mx::array array, xla::PjRtMemorySpace* memory_space); +std::unique_ptr CreateMlirBufferFromLiteral( + const xla::LiteralSlice& literal, xla::PjRtMemorySpace* memory_space); +std::unique_ptr CreateMlirBufferFromAttribute( @@ -739,10 +783,10 @@ index 0000000000..cc56626953 \ No newline at end of file diff --git a/xla/pjrt/plugin/mlx/client_cpp_pjrt.cc b/xla/pjrt/plugin/mlx/client_cpp_pjrt.cc new file mode 100644 -index 0000000000..fa5ece7d81 +index 0000000000..4c64c20b7b --- /dev/null +++ b/xla/pjrt/plugin/mlx/client_cpp_pjrt.cc -@@ -0,0 +1,298 @@ +@@ -0,0 +1,299 @@ +/* Copyright 2024 The OpenXLA Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. @@ -810,6 +854,7 @@ index 0000000000..fa5ece7d81 + /*id=*/0, devices_.front()); + memory_space_ = owned_memory_space_.get(); + AttachStablehloMlxMemorySpace(devices_.front(), memory_space_); ++ // mx::disable_compile(); + } + + ~StablehloMlxPjrtClient() override { @@ -1079,6 +1124,1347 @@ index 0000000000..8c90942f0c + +#endif // XLA_PJRT_PLUGIN_STABLEHLO_MLX_CLIENT_CPP_PJRT_H_ \ No newline at end of file +diff --git a/xla/pjrt/plugin/mlx/compile.cc b/xla/pjrt/plugin/mlx/compile.cc +new file mode 100644 +index 0000000000..889ad0fffe +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/compile.cc +@@ -0,0 +1,1297 @@ ++#include "xla/pjrt/plugin/mlx/compile.h" ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++// TODO(@cryptodeal): might need to update `BUILD` ++#include "mlir/IR/Visitors.h" ++ ++#include "mlx/mlx.h" ++#include "mlx/compile_impl.h" ++#include "absl/log/log.h" ++#include "absl/status/status.h" ++#include "absl/status/statusor.h" ++#include "llvm/ADT/TypeSwitch.h" ++#include "mlir/Bytecode/BytecodeWriter.h" ++#include "mlir/Dialect/Func/IR/FuncOps.h" ++#include "mlir/IR/AsmState.h" ++#include "mlir/IR/BuiltinAttributes.h" ++#include "mlir/IR/MLIRContext.h" ++#include "mlir/IR/Operation.h" ++#include "mlir/IR/OperationSupport.h" ++#include "mlir/IR/OwningOpRef.h" ++#include "mlir/Interfaces/DataLayoutInterfaces.h" ++#include "mlir/Pass/PassManager.h" ++#include "mlir/Support/LLVM.h" ++#include "stablehlo/reference/Api.h" ++#include "stablehlo/transforms/optimization/Passes.h" ++#include "xla/hlo/translate/stablehlo.h" ++#include "xla/mlir/utils/error_util.h" ++#include "xla/mlir/utils/type_util.h" ++#include "xla/mlir_hlo/mhlo/transforms/passes.h" ++#include "xla/mlir_hlo/stablehlo_ext/transforms/passes.h" ++#include "xla/pjrt/pjrt_client.h" ++#include "xla/pjrt/pjrt_executable.h" ++#include "xla/pjrt/pjrt_future.h" ++#include "xla/pjrt/plugin/mlx/buffer.h" ++#include "xla/pjrt/plugin/mlx/logging.h" ++#include "xla/pjrt/plugin/mlx/utils.h" ++#include "xla/service/computation_placer.h" ++#include "tsl/platform/status.h" ++#include "tsl/platform/statusor.h" ++ ++namespace mx = mlx::core; ++ ++namespace compile { ++ ++namespace detail { ++// cbrt implementation ++std::vector cbrt(const std::vector& inputs) { ++ return std::vector{mx::power( ++ inputs[0], mx::full(inputs[0].shape(), 1 / 3, inputs[0].dtype()))}; ++} ++ ++std::vector func(mlir::ModuleOp mod, mlir::func::FuncOp op, ++ const std::vector& inputs, ++ bool recursive_compile) { ++ auto& block = op.front(); ++ std::unordered_map> block_ctx; ++ std::vector res; ++ for (mlir::Operation& op : block.getOperations()) { ++ res = ++ llvm::TypeSwitch>(&op) ++ .Case( ++ [&block_ctx, &inputs](auto o) { ++ // std::cout << "return op:" << std::endl; ++ std::vector res; ++ for (mlir::Value val : o.getOperands()) { ++ res.emplace_back(utils::array::fromOperand(val, block_ctx, ++ inputs, false)); ++ } ++ return res; ++ }) ++ .Case( ++ [&block_ctx, &inputs, &mod, recursive_compile](auto o) { ++ std::vector operands; ++ for (mlir::Value val : o.getOperands()) { ++ operands.emplace_back(utils::array::fromOperand( ++ val, block_ctx, inputs, false)); ++ } ++ auto func_op = ++ mod.lookupSymbol(o.getCallee()); ++ // std::cout << "Calling function: " << ++ // func_op.getName().str() ++ // << std::endl; ++ if (recursive_compile) { ++ return compile::func(mod, func_op)(operands); ++ } else { ++ return func(mod, func_op, operands, false); ++ } ++ }) ++ // Handle StableHLO nullary ops ++ .Case([](auto o) { ++ return std::vector{ ++ *utils::array::fromDenseElementsAttr( ++ mlir::cast(o.getValue()))}; ++ }) ++ .Case([](auto o) { ++ auto result_type = ++ mlir::cast(o.getResult().getType()); ++ mx::Shape result_shape(result_type.getShape().begin(), ++ result_type.getShape().end()); ++ auto iota_dim = o.getIotaDimension(); ++ mx::Dtype dtype = ++ *utils::dtype::fromMlirType(result_type.getElementType()); ++ mx::Shape broadcast_dims(result_shape.size(), 1); ++ broadcast_dims[iota_dim] = result_shape[iota_dim]; ++ auto init_arange = mx::arange( ++ static_cast(result_shape[iota_dim]), dtype); ++ if (broadcast_dims.size()) { ++ init_arange = mx::reshape(init_arange, broadcast_dims); ++ } ++ return std::vector{ ++ mx::broadcast_to(init_arange, result_shape)}; ++ }) ++ // .Case([](auto o) {}) ++ /* ++ .Case([](auto o) {}) ++ Deprecated see: ++ https://github.com/openxla/stablehlo/issues/2340 ++ https://github.com/openxla/stablehlo/pull/2283 ++ */ ++ // Handle StableHLO unary elementwise op ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::abs(utils::array::fromOperand( ++ o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return mx::compile(cbrt)({utils::array::fromOperand( ++ o.getOperand(), block_ctx, inputs)}); ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::ceil(utils::array::fromOperand( ++ o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::astype( ++ utils::array::fromOperand(o.getOperand(), block_ctx, inputs), ++ *utils::dtype::fromMlirType( ++ o.getResult().getType().getElementType()))}; ++ }) ++ // .Case([](auto o) {}) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::cos(utils::array::fromOperand( ++ o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::exp(utils::array::fromOperand( ++ o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::expm1(utils::array::fromOperand( ++ o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::floor(utils::array::fromOperand( ++ o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::imag(utils::array::fromOperand( ++ o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{ ++ mx::isfinite(utils::array::fromOperand(o.getOperand(), ++ block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::log(utils::array::fromOperand( ++ o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::log1p(utils::array::fromOperand( ++ o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{ ++ mx::sigmoid(utils::array::fromOperand(o.getOperand(), ++ block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{ ++ mx::logical_not(utils::array::fromOperand( ++ o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{ ++ mx::negative(utils::array::fromOperand(o.getOperand(), ++ block_ctx, inputs))}; ++ }) ++ // .Case([](auto o) {}) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::real(utils::array::fromOperand( ++ o.getOperand(), block_ctx, inputs))}; ++ }) ++ // TODO(@cryptodeal): `mlir::stablehlo::RoundOp` does not match with ++ // the mlx metal implementation ++ // .Case([](auto o) {}) ++ .Case([&block_ctx, ++ &inputs](auto o) { ++ return std::vector{mx::round(utils::array::fromOperand( ++ o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::rsqrt(utils::array::fromOperand( ++ o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::sign(utils::array::fromOperand( ++ o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::sin(utils::array::fromOperand( ++ o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::sqrt(utils::array::fromOperand( ++ o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::tan(utils::array::fromOperand( ++ o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::tanh(utils::array::fromOperand( ++ o.getOperand(), block_ctx, inputs))}; ++ }) ++ // Handle StableHLO binary elementwise ops ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::add( ++ utils::array::fromOperand(o.getLhs(), block_ctx, inputs), ++ utils::array::fromOperand(o.getRhs(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::arctan2( ++ utils::array::fromOperand(o.getLhs(), block_ctx, inputs), ++ utils::array::fromOperand(o.getRhs(), block_ctx, inputs))}; ++ }) ++ // TODO(@cryptodeal): implement complex op ++ // .Case([&block_ctx](auto ++ // op) {}) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::divide( ++ utils::array::fromOperand(o.getLhs(), block_ctx, inputs), ++ utils::array::fromOperand(o.getRhs(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::maximum( ++ utils::array::fromOperand(o.getLhs(), block_ctx, inputs), ++ utils::array::fromOperand(o.getRhs(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::minimum( ++ utils::array::fromOperand(o.getLhs(), block_ctx, inputs), ++ utils::array::fromOperand(o.getRhs(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::multiply( ++ utils::array::fromOperand(o.getLhs(), block_ctx, inputs), ++ utils::array::fromOperand(o.getRhs(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::power( ++ utils::array::fromOperand(o.getLhs(), block_ctx, inputs), ++ utils::array::fromOperand(o.getRhs(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::remainder( ++ utils::array::fromOperand(o.getLhs(), block_ctx, inputs), ++ utils::array::fromOperand(o.getRhs(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::left_shift( ++ utils::array::fromOperand(o.getLhs(), block_ctx, inputs), ++ utils::array::fromOperand(o.getRhs(), block_ctx, inputs))}; ++ }) ++ .Case( ++ [&block_ctx, &inputs](auto o) { ++ /** ++ * Per Metal Spec: ++ * For the right-shift operator, if E1 has an unsigned ++ * type or if E1 has a signed type and a nonnegative ++ * value, the vacated bits are filled with zeros. If E1 ++ * has a signed type and a negative value, the vacated ++ * bits are filled with ones. ++ */ ++ auto lhs = ++ utils::array::fromOperand(o.getLhs(), block_ctx, inputs); ++ auto rhs = ++ utils::array::fromOperand(o.getRhs(), block_ctx, inputs); ++ auto target_dtype = rhs.dtype(); ++ switch (lhs.dtype().size()) { ++ case 1: ++ target_dtype = mx::int8; ++ break; ++ case 2: ++ target_dtype = mx::int16; ++ break; ++ case 4: ++ target_dtype = mx::int32; ++ break; ++ case 8: ++ target_dtype = mx::int64; ++ break; ++ } ++ return std::vector{ ++ mx::view(mx::right_shift(mx::view(lhs, target_dtype), ++ mx::astype(rhs, target_dtype)), ++ lhs.dtype())}; ++ }) ++ .Case( ++ [&block_ctx, &inputs](auto o) { ++ // Ensures that we bitcast to `uint` type before ++ // performing the right shift. Should ensure that vacated ++ // bits are zero populated. ++ auto lhs = ++ utils::array::fromOperand(o.getLhs(), block_ctx, inputs); ++ auto rhs = ++ utils::array::fromOperand(o.getRhs(), block_ctx, inputs); ++ auto target_dtype = rhs.dtype(); ++ switch (lhs.dtype().size()) { ++ case 1: ++ target_dtype = mx::uint8; ++ break; ++ case 2: ++ target_dtype = mx::uint16; ++ break; ++ case 4: ++ target_dtype = mx::uint32; ++ break; ++ case 8: ++ target_dtype = mx::uint64; ++ break; ++ } ++ return std::vector{ ++ mx::view(mx::right_shift(mx::view(lhs, target_dtype), ++ mx::astype(rhs, target_dtype)), ++ lhs.dtype())}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::subtract( ++ utils::array::fromOperand(o.getLhs(), block_ctx, inputs), ++ utils::array::fromOperand(o.getRhs(), block_ctx, inputs))}; ++ }) ++ // Handle StableHLO binary logical elementwise ops ++ .Case([&block_ctx, &inputs](auto o) { ++ auto lhs = ++ utils::array::fromOperand(o.getLhs(), block_ctx, inputs); ++ auto rhs = ++ utils::array::fromOperand(o.getRhs(), block_ctx, inputs); ++ switch (mx::kindof(lhs.dtype())) { ++ case mx::Dtype::Kind::b: ++ return std::vector{mx::logical_and(lhs, rhs)}; ++ default: ++ return std::vector{mx::bitwise_and(lhs, rhs)}; ++ } ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ auto lhs = ++ utils::array::fromOperand(o.getLhs(), block_ctx, inputs); ++ auto rhs = ++ utils::array::fromOperand(o.getRhs(), block_ctx, inputs); ++ switch (mx::kindof(lhs.dtype())) { ++ case mx::Dtype::Kind::b: ++ return std::vector{mx::logical_or(lhs, rhs)}; ++ default: ++ return std::vector{mx::bitwise_or(lhs, rhs)}; ++ } ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::bitwise_xor( ++ utils::array::fromOperand(o.getLhs(), block_ctx, inputs), ++ utils::array::fromOperand(o.getRhs(), block_ctx, inputs))}; ++ }) ++ // Handle StableHLO communication ops ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ ++ // Handle StableHLO parallelism related ops ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ ++ // Handle StableHLO control flow ops ++ // .Case([](auto o) {}) ++ // TODO(@cryptodeal): need custom kernels in order to ++ // remove the need to call `mx::eval` for the following ops ++ // .Case([](auto o)) ++ // .Case([](auto o)) ++ // .Case([](auto o)) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ .Case([&block_ctx, &inputs](auto o) { ++ auto operand = ++ utils::array::fromOperand(o.getOperand(0), block_ctx, inputs); ++ if (utils::isArgMaxReduce(o)) { ++ // std::cout << "ArgMaxReduce op: " << std::endl; ++ auto axis = static_cast(o.getDimensions()[0]); ++ auto indices = mx::argmax(operand, axis); ++ ++ mx::Dtype result_type = *utils::dtype::fromMlirType( ++ mlir::cast(o.getResults()[1].getType()) ++ .getElementType()); ++ return std::vector{mx::take(operand, indices, axis), ++ mx::astype(indices, result_type)}; ++ } ++ if (utils::isSumReduce(o)) { ++ // std::cout << "SumReduce op: " << std::endl; ++ return std::vector{ ++ mx::sum(operand, static_cast(o.getDimensions()[0]))}; ++ } ++ if (utils::isMaxReduce(o)) { ++ // std::cout << "MaxReduce op: " << std::endl; ++ return std::vector{ ++ mx::max(operand, static_cast(o.getDimensions()[0]))}; ++ } ++ if (utils::isMinReduce(o)) { ++ // std::cout << "MinReduce op: " << std::endl; ++ return std::vector{ ++ mx::min(operand, static_cast(o.getDimensions()[0]))}; ++ } ++ // TODO(@cryptodeal): further cleanup module validation ++ // done via call to `Compile` to check for valid reduce types. ++ ++ // For now, we assume `mx::any` if none of the above ++ ++ // if (isAnyReduce(o)) { ++ // std::cout << "Unhandled reduce op: " << std::endl; ++ return std::vector{ ++ mx::any(operand, static_cast(o.getDimensions()[0]))}; ++ //} ++ }) ++ // Handle StableHLO tuple ops ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ .Case([&block_ctx, &inputs](auto o) { ++ auto lhs = ++ utils::array::fromOperand(o.getLhs(), block_ctx, inputs); ++ auto rhs = ++ utils::array::fromOperand(o.getRhs(), block_ctx, inputs); ++ switch (o.getComparisonDirection()) { ++ case mlir::stablehlo::ComparisonDirection::NE: ++ return std::vector{mx::not_equal(lhs, rhs)}; ++ case mlir::stablehlo::ComparisonDirection::GE: ++ return std::vector{mx::greater_equal(lhs, rhs)}; ++ case mlir::stablehlo::ComparisonDirection::GT: ++ return std::vector{mx::greater(lhs, rhs)}; ++ case mlir::stablehlo::ComparisonDirection::LE: ++ return std::vector{mx::less_equal(lhs, rhs)}; ++ case mlir::stablehlo::ComparisonDirection::LT: ++ return std::vector{mx::less(lhs, rhs)}; ++ case mlir::stablehlo::ComparisonDirection::EQ: ++ return std::vector{mx::equal(lhs, rhs)}; ++ } ++ }) ++ // Handle StableHLO Slice ops ++ .Case([&block_ctx, &inputs](auto o) { ++ std::vector start_indices(o.getStartIndices().begin(), ++ o.getStartIndices().end()); ++ std::vector limit_indices(o.getLimitIndices().begin(), ++ o.getLimitIndices().end()); ++ std::vector strides(o.getStrides().begin(), ++ o.getStrides().end()); ++ return std::vector{mx::slice( ++ utils::array::fromOperand(o.getOperand(), block_ctx, inputs), ++ start_indices, limit_indices, strides)}; ++ }) ++ .Case([&block_ctx, ++ &inputs](auto o) { ++ auto operand = ++ utils::array::fromOperand(o.getOperand(), block_ctx, inputs); ++ std::vector indices; ++ for (auto val : o.getStartIndices()) { ++ indices.emplace_back( ++ utils::array::fromOperand(val, block_ctx, inputs)); ++ } ++ auto start_indices = mx::concatenate(indices); ++ std::vector axes(operand.ndim()); ++ std::iota(axes.begin(), axes.end(), 0); ++ std::vector slice_sizes(o.getSliceSizes().begin(), ++ o.getSliceSizes().end()); ++ return std::vector{ ++ mx::slice(operand, start_indices, axes, slice_sizes)}; ++ }) ++ .Case([&block_ctx, ++ &inputs](auto o) { ++ auto operand = ++ utils::array::fromOperand(o.getOperand(), block_ctx, inputs); ++ auto update = ++ utils::array::fromOperand(o.getUpdate(), block_ctx, inputs); ++ std::vector indices; ++ for (auto val : o.getStartIndices()) { ++ indices.emplace_back( ++ utils::array::fromOperand(val, block_ctx, inputs)); ++ } ++ ++ auto start_indices = mx::concatenate(indices); ++ std::vector axes(operand.ndim()); ++ std::iota(axes.begin(), axes.end(), 0); ++ return std::vector{ ++ mx::slice_update(operand, update, start_indices, axes)}; ++ }) ++ // Handle StableHLO Other ops ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ .Case([&block_ctx, ++ &inputs](auto o) { ++ auto result_type = *utils::dtype::fromMlirType( ++ mlir::cast(o.getResult().getType()) ++ .getElementType()); ++ return std::vector{mx::view( ++ utils::array::fromOperand(o.getOperand(), block_ctx, inputs), ++ result_type)}; ++ }) ++ /* ++ .Case([](auto o) {}) ++ Deprecated see: ++ https://github.com/openxla/stablehlo/issues/2340 ++ https://github.com/openxla/stablehlo/pull/2283 ++ */ ++ .Case([&block_ctx, ++ &inputs](auto o) { ++ auto result_type = ++ mlir::cast(o.getResult().getType()); ++ mx::Shape result_shape(result_type.getShape().begin(), ++ result_type.getShape().end()); ++ std::vector broadcast_dims( ++ o.getBroadcastDimensions().begin(), ++ o.getBroadcastDimensions().end()); ++ auto operand = ++ utils::array::fromOperand(o.getOperand(), block_ctx, inputs); ++ ++ // potentially reshape to pad with ones, which allows ++ // broadcasting ++ auto min_rank = std::min(operand.ndim(), result_shape.size()); ++ if (min_rank == operand.ndim()) { ++ mx::Shape padded_shape = operand.shape(); ++ for (auto i = 0; i < result_shape.size(); i++) { ++ if (auto search = std::find(broadcast_dims.begin(), ++ broadcast_dims.end(), i); ++ search == broadcast_dims.end()) { ++ padded_shape.insert(padded_shape.begin() + i, 1); ++ } ++ } ++ operand = mx::reshape(operand, padded_shape); ++ } ++ return std::vector{ ++ mx::broadcast_to(operand, result_shape)}; ++ }) ++ // .Case([](auto o) {}) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::linalg::cholesky( ++ utils::array::fromOperand(o.getOperand(), block_ctx, inputs), ++ o.getLower())}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::clip( ++ utils::array::fromOperand(o.getOperand(), block_ctx, inputs), ++ utils::array::fromOperand(o.getMin(), block_ctx, inputs), ++ utils::array::fromOperand(o.getMax(), block_ctx, inputs))}; ++ }) ++ .Case( ++ [&block_ctx, &inputs](auto o) { ++ std::vector inputs; ++ for (auto val : o.getOperands()) { ++ inputs.emplace_back( ++ utils::array::fromOperand(val, block_ctx, inputs)); ++ } ++ return std::vector{ ++ mx::concatenate(inputs, o.getDimension())}; ++ }) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ /* ++ .Case([](auto o) {}) ++ Deprecated see: ++ https://github.com/openxla/stablehlo/issues/2340 ++ https://github.com/openxla/stablehlo/pull/2283 ++ */ ++ .Case([&block_ctx, &inputs](auto o) { ++ return compile::dotGeneral(o)( ++ {utils::array::fromOperand(o.getLhs(), block_ctx, inputs), ++ utils::array::fromOperand(o.getRhs(), block_ctx, inputs)}); ++ }) ++ /* ++ .Case([](auto op) {}) ++ .Case([](auto op) {}) ++ Deprecated see: ++ https://github.com/openxla/stablehlo/issues/2340 ++ https://github.com/openxla/stablehlo/pull/2283 ++ */ ++ // .Case([](auto op) {}) ++ // TODO(@cryptodeal): definitely room for optimization here ++ .Case([&block_ctx, &inputs](auto o) { ++ return compile::gather(o)( ++ {utils::array::fromOperand(o.getOperand(), block_ctx, inputs), ++ utils::array::fromOperand(o.getStartIndices(), block_ctx, ++ inputs)}); ++ }) ++ .Case([&block_ctx, ++ &inputs](auto o) { ++ return std::vector{mx::array( ++ (utils::array::fromOperand(o.getOperand(), block_ctx, inputs)) ++ .shape(o.getDimension()))}; ++ }) ++ // .Case([](auto o) {}) ++ .Case([&block_ctx, &inputs](auto o) { ++ auto result_type = ++ mlir::cast(o.getResult().getType()); ++ return std::vector{mx::reshape( ++ utils::array::fromOperand(o.getOperand(), block_ctx, inputs), ++ std::vector(result_type.getShape().begin(), ++ result_type.getShape().end()))}; ++ }) ++ // .Case([](auto o) {}) ++ .Case([&block_ctx, &inputs](auto o) { ++ std::vector operands; ++ for (auto val : o.getInputs()) { ++ operands.emplace_back( ++ utils::array::fromOperand(val, block_ctx, inputs)); ++ } ++ // Add updates to input list ++ for (auto val : o.getUpdates()) { ++ operands.emplace_back( ++ utils::array::fromOperand(val, block_ctx, inputs)); ++ } ++ // Add scatter indices to input list ++ operands.emplace_back(utils::array::fromOperand( ++ o.getScatterIndices(), block_ctx, inputs)); ++ return compile::scatter(o)(operands); ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::where( ++ utils::array::fromOperand(o.getOperand(0), block_ctx, inputs), ++ utils::array::fromOperand(o.getOperand(1), block_ctx, inputs), ++ utils::array::fromOperand(o.getOperand(2), block_ctx, ++ inputs))}; ++ }) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ .Case([&block_ctx, &inputs](auto o) { ++ std::vector tmp_inputs; ++ for (auto val : o.getInputs()) { ++ tmp_inputs.emplace_back( ++ utils::array::fromOperand(val, block_ctx, inputs)); ++ } ++ int dim = static_cast(o.getDimension()); ++ // Get update computation ++ mlir::Block& comparator = o.getComparator().front(); ++ auto [index, comparator_type] = *utils::getSortInfo(comparator); ++ auto indices = ++ mx::argsort(tmp_inputs[index], dim, comparator_type); ++ std::vector res; ++ mx::Shape slice_sizes(tmp_inputs[0].ndim(), 1); ++ for (auto in : tmp_inputs) { ++ res.emplace_back(mx::reshape( ++ mx::gather(in, indices, dim, slice_sizes), in.shape())); ++ } ++ return res; ++ }) ++ // .Case([](auto o) {}) ++ // TODO (@cryptodeal): lots of room for optimization here ++ .Case([&block_ctx, &inputs](auto o) { ++ return compile::pad(o)( ++ {utils::array::fromOperand(o.getOperand(), block_ctx, inputs), ++ utils::array::fromOperand(o.getPaddingValue(), block_ctx, ++ inputs)}); ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ std::vector axes(o.getPermutation().begin(), ++ o.getPermutation().end()); ++ return std::vector{mx::transpose( ++ utils::array::fromOperand(o.getOperand(), block_ctx, inputs), ++ axes)}; ++ }) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // Need to modify such that `rng.state` is held outside of scope ++ // of `mx::compile` ++ // TODO(@cryptodeal): re-enable once we've handled ensuring ++ // state is handled correctly ++ // .Case([&block_ctx](auto o) { ++ // auto rng_func = [&o](const std::vector& inputs) ++ // { ++ // auto a = inputs[0]; ++ // auto b = inputs[1]; ++ // auto result_type = ++ // mlir::cast(o.getResult().getType()); ++ // std::vector ++ // shape(result_type.getShape().begin(), ++ // result_type.getShape().end()); ++ ++ // switch (o.getRngDistribution()) { ++ // case RngDistribution::UNIFORM: ++ // return std::vector{ ++ // mx::random::uniform(a, b, shape, a.dtype())}; ++ // default: ++ // return std::vector{ ++ // mx::random::normal(shape, a.dtype(), a, b)}; ++ // } ++ // }; ++ // return mx::compile(rng_func)( ++ // {utils::array::fromOperand(o.getA(), block_ctx), ++ // utils::array::fromOperand(o.getB(), block_ctx)}); ++ // }) ++ // .Case([&block_ctx](auto o) { ++ // auto rng_bit_gen_func = [&o](const std::vector& ++ // inputs) { ++ ++ // }; ++ // return mx::compile(rng_bit_gen_func)( ++ // {utils::array::fromOperand(o.getA(), block_ctx), ++ // utils::array::fromOperand(o.getB(), block_ctx)}); ++ // }) ++ // Handle StableHLO Quantize ops ++ // .Case([](auto op) {}) ++ // .Case([](auto op) {}) ++ // .Case([](auto op) {}) ++ /* ++ .Case([](auto o) {}) ++ Deprecated see: ++ https://github.com/openxla/stablehlo/issues/2340 ++ https://github.com/openxla/stablehlo/pull/2283 ++ */ ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ ++ .Default([&op](auto o) { ++ // Shouldn't be possible to reach this point, return ++ // empty vector. ++ return std::vector{}; ++ }); ++ block_ctx.emplace(&op, res); ++ } ++ return res; ++} ++} // namespace detail ++ ++MlxFunc dotGeneral(mlir::stablehlo::DotGeneralOp op) { ++ auto dot_dim_nums = op.getDotDimensionNumbers(); ++ std::vector lhs_batch_dims( ++ dot_dim_nums.getLhsBatchingDimensions().begin(), ++ dot_dim_nums.getLhsBatchingDimensions().end()); ++ std::vector rhs_batch_dims( ++ dot_dim_nums.getRhsBatchingDimensions().begin(), ++ dot_dim_nums.getRhsBatchingDimensions().end()); ++ std::vector lhs_contract_dims( ++ dot_dim_nums.getLhsContractingDimensions().begin(), ++ dot_dim_nums.getLhsContractingDimensions().end()); ++ std::vector rhs_contract_dims( ++ dot_dim_nums.getRhsContractingDimensions().begin(), ++ dot_dim_nums.getRhsContractingDimensions().end()); ++ ++ MlxFunc fun = [lhs_batch_dims = std::move(lhs_batch_dims), ++ rhs_batch_dims = std::move(rhs_batch_dims), ++ lhs_contract_dims = std::move(lhs_contract_dims), ++ rhs_contract_dims = std::move(rhs_contract_dims)]( ++ const std::vector& inputs) { ++ auto lhs = inputs[0]; ++ auto rhs = inputs[1]; ++ ++ int dim_count = 0; ++ auto getDimChar = [&dim_count]() -> char { return 'a' + dim_count++; }; ++ ++ std::unordered_map lhs_batch_map; ++ std::unordered_map rhs_batch_map; ++ std::unordered_map lhs_contract_map; ++ std::unordered_map rhs_contract_map; ++ ++ std::string res_subscript; ++ for (auto i = 0; i < lhs_batch_dims.size(); ++i) { ++ auto dim_char = getDimChar(); ++ res_subscript = res_subscript + dim_char; ++ lhs_batch_map.emplace(lhs_batch_dims[i], dim_char); ++ rhs_batch_map.emplace(rhs_batch_dims[i], dim_char); ++ } ++ ++ for (auto i = 0; i < lhs_contract_dims.size(); ++i) { ++ auto dim_char = getDimChar(); ++ lhs_contract_map.emplace(lhs_contract_dims[i], dim_char); ++ rhs_contract_map.emplace(rhs_contract_dims[i], dim_char); ++ } ++ ++ std::string lhs_subscript; ++ for (auto i = 0; i < lhs.ndim(); ++i) { ++ if (auto match = lhs_batch_map.find(i); match != lhs_batch_map.end()) { ++ lhs_subscript = lhs_subscript + match->second; ++ } else if (auto match = lhs_contract_map.find(i); ++ match != lhs_contract_map.end()) { ++ lhs_subscript = lhs_subscript + match->second; ++ } else { ++ auto dim_char = getDimChar(); ++ res_subscript = res_subscript + dim_char; ++ lhs_subscript = lhs_subscript + dim_char; ++ } ++ } ++ ++ std::string rhs_subscript; ++ for (auto i = 0; i < rhs.ndim(); ++i) { ++ if (auto match = rhs_batch_map.find(i); match != rhs_batch_map.end()) { ++ rhs_subscript = rhs_subscript + match->second; ++ } else if (auto match = rhs_contract_map.find(i); ++ match != rhs_contract_map.end()) { ++ rhs_subscript = rhs_subscript + match->second; ++ } else { ++ auto dim_char = getDimChar(); ++ res_subscript = res_subscript + dim_char; ++ rhs_subscript = rhs_subscript + dim_char; ++ } ++ } ++ ++ return std::vector{mx::einsum( ++ lhs_subscript + "," + rhs_subscript + "->" + res_subscript, inputs)}; ++ }; ++ return mx::compile(std::move(fun)); ++} ++ ++MlxFunc gather(mlir::stablehlo::GatherOp op) { ++ auto dim_nums = op.getDimensionNumbers(); ++ std::vector offset_dims(dim_nums.getOffsetDims().begin(), ++ dim_nums.getOffsetDims().end()); ++ std::vector collapsed_slice_dims( ++ dim_nums.getCollapsedSliceDims().begin(), ++ dim_nums.getCollapsedSliceDims().end()); ++ std::vector operand_batching_dims( ++ dim_nums.getOperandBatchingDims().begin(), ++ dim_nums.getOperandBatchingDims().end()); ++ std::vector start_indices_batching_dims( ++ dim_nums.getStartIndicesBatchingDims().begin(), ++ dim_nums.getStartIndicesBatchingDims().end()); ++ std::vector start_index_map(dim_nums.getStartIndexMap().begin(), ++ dim_nums.getStartIndexMap().end()); ++ auto index_vector_dim = dim_nums.getIndexVectorDim(); ++ std::vector slice_sizes(op.getSliceSizes().begin(), ++ op.getSliceSizes().end()); ++ MlxFunc fun = [collapsed_slice_dims = std::move(collapsed_slice_dims), ++ index_vector_dim, offset_dims = std::move(offset_dims), ++ operand_batching_dims = std::move(operand_batching_dims), ++ slice_sizes = std::move(slice_sizes), ++ start_index_map = std::move(start_index_map), ++ start_indices_batching_dims = ++ std::move(start_indices_batching_dims)]( ++ const std::vector& inputs) { ++ auto operand = inputs[0]; ++ auto start_indices = inputs[1]; ++ ++ std::vector batch_dim_sizes = start_indices.shape(); ++ if (index_vector_dim < batch_dim_sizes.size()) { ++ batch_dim_sizes.erase(batch_dim_sizes.begin() + index_vector_dim); ++ } ++ ++ std::vector offset_dim_sizes; ++ for (auto i = 0; i < slice_sizes.size(); ++i) { ++ if (std::find(collapsed_slice_dims.begin(), collapsed_slice_dims.end(), ++ i) != collapsed_slice_dims.end() || ++ std::find(operand_batching_dims.begin(), operand_batching_dims.end(), ++ i) != operand_batching_dims.end()) { ++ continue; ++ } ++ offset_dim_sizes.push_back(slice_sizes[i]); ++ } ++ ++ // Calculate result shape and batch dims ++ std::vector result_shape(batch_dim_sizes.size() + ++ offset_dim_sizes.size()); ++ std::vector batch_dims_vec(batch_dim_sizes.size()); ++ auto batch_dims_count = 0; ++ auto offset_dims_count = 0; ++ for (auto i = 0; i < result_shape.size(); ++i) { ++ if (auto search = std::find(offset_dims.begin(), offset_dims.end(), i); ++ search == offset_dims.end()) { ++ batch_dims_vec[batch_dims_count] = i; ++ result_shape[i] = batch_dim_sizes[batch_dims_count++]; ++ } else { ++ result_shape[i] = offset_dim_sizes[offset_dims_count++]; ++ } ++ } ++ ++ mx::array batch_dims = mx::array( ++ batch_dims_vec.data(), {static_cast(batch_dims_vec.size())}); ++ mx::array start_idx_map = mx::array( ++ start_index_map.data(), {static_cast(start_index_map.size())}); ++ mx::array op_batching_dims = ++ mx::array(operand_batching_dims.data(), ++ {static_cast(operand_batching_dims.size())}); ++ ++ // Calculate result indices ++ std::vector meshgrid_arrays; ++ for (const auto& dim : result_shape) { ++ meshgrid_arrays.push_back(mx::arange(dim)); ++ } ++ mx::array result_indices = ++ mx::reshape(mx::stack(mx::meshgrid(meshgrid_arrays, false, "ij"), -1), ++ {-1, static_cast(result_shape.size())}); ++ ++ // Initialize operand indices with zeros ++ mx::array operand_indices = mx::zeros( ++ {result_indices.shape(0), static_cast(operand.ndim())}, ++ mx::int32); ++ ++ // Slice batch indices ++ mx::array batch_indices = mx::take(result_indices, batch_dims, 1); ++ ++ if (batch_indices.size()) { ++ // Calculate start indices ++ std::vector tmp_indices = ++ mx::split(batch_indices, batch_indices.shape(1), 1); ++ for (auto& idx : tmp_indices) { ++ idx = mx::flatten(idx); ++ } ++ ++ std::vector tmp_axes; ++ for (auto i = 0; i < start_indices.ndim(); ++i) { ++ if (i == index_vector_dim) continue; ++ tmp_axes.push_back(i); ++ } ++ std::vector tmp_slice_sizes(start_indices.ndim(), 1); ++ if (index_vector_dim < tmp_slice_sizes.size()) { ++ tmp_slice_sizes[index_vector_dim] = ++ start_indices.shape(index_vector_dim); ++ } ++ ++ mx::array batch_start_indices = mx::reshape( ++ mx::gather(start_indices, tmp_indices, tmp_axes, tmp_slice_sizes), ++ {result_indices.shape(0), -1}); ++ ++ // Seed with full start indices ++ std::vector max; ++ for (auto d_start = 0; d_start < start_index_map.size(); ++d_start) { ++ int d_operand = start_index_map[d_start]; ++ max.push_back(operand.shape(d_operand) - slice_sizes[d_operand]); ++ } ++ ++ operand_indices = mx::put_along_axis( ++ operand_indices, ++ mx::broadcast_to(start_idx_map, ++ {operand_indices.shape(0), ++ static_cast(start_index_map.size())}), ++ mx::clip( ++ mx::take(batch_start_indices, ++ mx::arange(static_cast(start_index_map.size())), 1), ++ mx::zeros(batch_start_indices.shape(), mx::int32), ++ mx::array(max.data(), {static_cast(max.size())})), ++ 1); ++ } ++ ++ // Calculate full batching indices ++ mx::array i_batching = ++ mx::arange(static_cast(operand_batching_dims.size())); ++ mx::array d_operands = mx::take(op_batching_dims, i_batching); ++ mx::array d_starts = mx::take( ++ mx::array(start_indices_batching_dims.data(), ++ {static_cast(start_indices_batching_dims.size())}), ++ i_batching); ++ auto tmp_batch_start_indices = mx::subtract( ++ d_starts, mx::where(mx::less(d_starts, mx::full(d_starts.shape(), ++ index_vector_dim)), ++ mx::zeros(d_starts.shape(), mx::int32), ++ mx::ones(d_starts.shape(), mx::int32))); ++ ++ // Add batching indices ++ operand_indices = mx::scatter_add_axis( ++ operand_indices, ++ mx::broadcast_to(d_operands, ++ {operand_indices.shape(0), d_operands.shape(0)}), ++ mx::take(batch_indices, tmp_batch_start_indices, 1), 1); ++ ++ // Calculate offset indices ++ mx::array offset_indices = ++ mx::take(result_indices, ++ mx::array(offset_dims.data(), ++ {static_cast(offset_dims.size())}), ++ 1); ++ ++ std::vector result_offset_idx; ++ for (unsigned i = 0; i < operand_indices.shape(1); i++) { ++ if (std::find(operand_batching_dims.begin(), operand_batching_dims.end(), ++ static_cast(i)) != operand_batching_dims.end() || ++ std::find(collapsed_slice_dims.begin(), collapsed_slice_dims.end(), ++ static_cast(i)) != collapsed_slice_dims.end()) { ++ continue; ++ } ++ result_offset_idx.push_back(i); ++ } ++ ++ // Add offset indices ++ operand_indices = mx::scatter_add_axis( ++ operand_indices, ++ mx::broadcast_to( ++ mx::array(result_offset_idx.data(), ++ {static_cast(result_offset_idx.size())}), ++ {operand_indices.shape(0), ++ static_cast(result_offset_idx.size())}), ++ offset_indices, 1); ++ ++ std::vector gather_axes(operand.ndim()); ++ std::iota(gather_axes.begin(), gather_axes.end(), 0); ++ return std::vector{mx::reshape( ++ mx::gather(operand, ++ mx::split(operand_indices, operand_indices.shape(1), 1), ++ gather_axes, std::vector(operand.ndim(), 1)), ++ result_shape)}; ++ }; ++ return mx::compile(fun); ++} ++ ++MlxFunc pad(mlir::stablehlo::PadOp op) { ++ std::vector edge_pad_low(op.getEdgePaddingLow().begin(), ++ op.getEdgePaddingLow().end()); ++ std::vector edge_pad_high(op.getEdgePaddingHigh().begin(), ++ op.getEdgePaddingHigh().end()); ++ std::vector interior_pad(op.getInteriorPadding().begin(), ++ op.getInteriorPadding().end()); ++ ++ MlxFunc fun = [edge_pad_low = std::move(edge_pad_low), ++ edge_pad_high = std::move(edge_pad_high), ++ interior_pad = std::move(interior_pad)]( ++ const std::vector& inputs) { ++ auto operand = inputs[0]; ++ auto padding_value = inputs[1]; ++ ++ std::vector result_shape(operand.ndim()); ++ for (auto i = 0; i < result_shape.size(); ++i) { ++ result_shape[i] = operand.shape(i) + edge_pad_low[i] + ++ std::max(operand.shape(i) - 1, 0) * interior_pad[i] + ++ edge_pad_high[i]; ++ } ++ ++ auto edge_padding_low = ++ mx::array(edge_pad_low.data(), ++ {static_cast(edge_pad_low.size())}, mx::int32); ++ auto interior_padding = ++ mx::array(interior_pad.data(), ++ {static_cast(edge_pad_low.size())}, mx::int32); ++ ++ // Calculate operand indices ++ std::vector meshgrid_arrays; ++ for (const auto& dim : operand.shape()) { ++ meshgrid_arrays.push_back(mx::arange(dim)); ++ } ++ mx::array operand_indices = ++ mx::reshape(mx::stack(mx::meshgrid(meshgrid_arrays, false, "ij"), -1), ++ {-1, static_cast(operand.ndim())}); ++ ++ mx::array result_indices = ++ edge_padding_low + ++ operand_indices * (interior_padding + mx::array(1, mx::int32)); ++ ++ // initialize array full of padding values ++ mx::array result = mx::full(result_shape, padding_value); ++ std::vector axes(operand.ndim()); ++ std::iota(axes.begin(), axes.end(), 0); ++ return std::vector{mx::scatter( ++ result, mx::split(result_indices, result_indices.shape(1), 1), ++ mx::gather(operand, ++ mx::split(operand_indices, operand_indices.shape(1), 1), ++ axes, std::vector(operand.ndim(), 1)), ++ axes)}; ++ }; ++ return mx::compile(fun); ++} ++ ++MlxFunc scatter(mlir::stablehlo::ScatterOp op) { ++ // Get scatter dimension numbers ++ auto scatter_dim_nums = op.getScatterDimensionNumbers(); ++ auto index_vector_dim = scatter_dim_nums.getIndexVectorDim(); ++ std::vector input_batching_dims( ++ scatter_dim_nums.getInputBatchingDims().begin(), ++ scatter_dim_nums.getInputBatchingDims().end()); ++ std::vector inserted_window_dims( ++ scatter_dim_nums.getInsertedWindowDims().begin(), ++ scatter_dim_nums.getInsertedWindowDims().end()); ++ std::vector scatter_dims_to_operand_dims( ++ scatter_dim_nums.getScatterDimsToOperandDims().begin(), ++ scatter_dim_nums.getScatterDimsToOperandDims().end()); ++ std::vector scatter_indices_batching_dims( ++ scatter_dim_nums.getScatterIndicesBatchingDims().begin(), ++ scatter_dim_nums.getScatterIndicesBatchingDims().end()); ++ std::vector update_window_dims( ++ scatter_dim_nums.getUpdateWindowDims().begin(), ++ scatter_dim_nums.getUpdateWindowDims().end()); ++ mlir::Block& update_computation = op.getUpdateComputation().front(); ++ // TODO(@cryptodeal): need to experiment to see how `computeHash` ++ // handles hashing `update_computation` property. ++ utils::ScatterType scatter_type = *utils::getScatterType(update_computation); ++ ++ // mlir::Block& update_computation = o.getUpdateComputation().front(); ++ // utils::ScatterType scatter_type = *getScatterType(update_computation); ++ ++ MlxFunc fun = [index_vector_dim, ++ input_batching_dims = std::move(input_batching_dims), ++ inserted_window_dims = std::move(inserted_window_dims), ++ scatter_dims_to_operand_dims = ++ std::move(scatter_dims_to_operand_dims), ++ scatter_indices_batching_dims = ++ std::move(scatter_indices_batching_dims), ++ scatter_type, ++ update_window_dims = std::move(update_window_dims)]( ++ const std::vector& in) { ++ auto input_count = (in.size() - 1) / 2; ++ std::vector inputs(in.begin(), in.begin() + input_count); ++ std::vector updates(in.begin() + input_count, in.end() - 1); ++ auto scatter_indices = in[in.size() - 1]; ++ // Calculate update scatter dims ++ std::vector update_scatter_dims; ++ for (auto i = 0; i < updates[0].ndim(); i++) { ++ if (std::find(update_window_dims.begin(), update_window_dims.end(), ++ static_cast(i)) == update_window_dims.end()) { ++ update_scatter_dims.emplace_back(static_cast(i)); ++ } ++ } ++ ++ // Calculate update indices ++ std::vector meshgrid_arrays; ++ for (const auto& dim : updates[0].shape()) { ++ meshgrid_arrays.push_back(mx::arange(dim)); ++ } ++ mx::array update_indices = ++ mx::reshape(mx::stack(mx::meshgrid(meshgrid_arrays, false, "ij"), -1), ++ {-1, static_cast(updates[0].ndim())}); ++ ++ // Calculate update scatter index ++ mx::array update_scatter_indices = ++ mx::take(update_indices, ++ mx::array(update_scatter_dims.data(), ++ {static_cast(update_scatter_dims.size())}), ++ 1); ++ ++ // Initialize result indices with zeros ++ auto result_indices = mx::zeros( ++ {update_indices.shape(0), static_cast(inputs[0].ndim())}, ++ mx::int32); ++ ++ // Gather start indices; populating the corresponding ++ // result indices. ++ auto gather_dim_count = index_vector_dim < scatter_indices.ndim() ++ ? update_scatter_dims.size() + 1 ++ : update_scatter_dims.size(); ++ std::vector tmp_gather_axes(update_scatter_dims.size()); ++ std::iota(tmp_gather_axes.begin(), tmp_gather_axes.end(), 0); ++ std::vector gather_slice_sizes(gather_dim_count, 1); ++ if (index_vector_dim < scatter_indices.ndim()) { ++ gather_slice_sizes[index_vector_dim] = ++ scatter_indices.shape(index_vector_dim); ++ } ++ mx::array start_indices = ++ mx::reshape(mx::gather(scatter_indices, ++ mx::split(update_scatter_indices, ++ update_scatter_indices.shape(1), 1), ++ tmp_gather_axes, gather_slice_sizes), ++ {update_indices.shape(0), -1}); ++ ++ // Seed with full start indices ++ result_indices = mx::put_along_axis( ++ result_indices, ++ mx::broadcast_to( ++ mx::array( ++ scatter_dims_to_operand_dims.data(), ++ {static_cast(scatter_dims_to_operand_dims.size())}), ++ {result_indices.shape(0), ++ static_cast(scatter_dims_to_operand_dims.size())}), ++ start_indices, 1); ++ ++ // Calculate full batching indices; accumulating to the ++ // corresponding result indices. ++ auto d_input = ++ mx::array(input_batching_dims.data(), ++ {static_cast(input_batching_dims.size())}); ++ auto d_start = ++ mx::array(scatter_indices_batching_dims.data(), ++ {static_cast(scatter_indices_batching_dims.size())}); ++ result_indices = mx::scatter_add_axis( ++ result_indices, ++ mx::broadcast_to(d_input, {result_indices.shape(0), d_input.shape(0)}), ++ mx::take(update_scatter_indices, ++ mx::subtract( ++ d_start, ++ mx::where(mx::less(d_start, mx::full(d_start.shape(), ++ index_vector_dim)), ++ mx::zeros(d_start.shape(), mx::int32), ++ mx::ones(d_start.shape(), mx::int32))), ++ 1), ++ 1); ++ ++ // Calculate update window indices ++ auto update_window_indices = ++ mx::take(update_indices, ++ mx::array(update_window_dims.data(), ++ {static_cast(update_window_dims.size())}), ++ 1); ++ ++ // Calculate full window indices; accumulating to the ++ // corresponding result indices. ++ std::vector tmp_indices; ++ auto full_window_idx_count = static_cast( ++ update_window_indices.shape(1) + inserted_window_dims.size() + ++ input_batching_dims.size()); ++ for (auto i = 0; i < full_window_idx_count; ++i) { ++ if (std::find(inserted_window_dims.begin(), inserted_window_dims.end(), ++ i) != inserted_window_dims.end() || ++ std::find(input_batching_dims.begin(), input_batching_dims.end(), ++ i) != input_batching_dims.end()) { ++ continue; ++ } ++ tmp_indices.push_back(i); ++ } ++ result_indices = mx::scatter_add_axis( ++ result_indices, ++ mx::broadcast_to(mx::array(tmp_indices.data(), ++ {static_cast(tmp_indices.size())}), ++ {result_indices.shape(0), ++ static_cast(tmp_indices.size())}), ++ update_window_indices, 1); ++ ++ std::vector scatter_axes(inputs[0].ndim()); ++ std::iota(scatter_axes.begin(), scatter_axes.end(), 0); ++ std::vector gather_axes(updates[0].ndim()); ++ std::iota(gather_axes.begin(), gather_axes.end(), 0); ++ std::vector res; ++ ++ auto tmp_gather_indices = ++ mx::split(update_indices, update_indices.shape(1), 1); ++ auto tmp_scatter_indices = ++ mx::split(result_indices, result_indices.shape(1), 1); ++ ++ std::vector update_shape( ++ inputs[0].ndim() + tmp_scatter_indices[0].ndim(), 1); ++ update_shape[0] = update_indices.shape(0); ++ std::vector tmp_gather_slice_sizes(updates[0].ndim(), 1); ++ for (auto i = 0; i < inputs.size(); ++i) { ++ auto update_vals = ++ mx::reshape(mx::gather(updates[i], tmp_gather_indices, gather_axes, ++ tmp_gather_slice_sizes), ++ update_shape); ++ switch (scatter_type) { ++ case utils::ScatterType::Replace: ++ res.push_back(mx::scatter(inputs[i], tmp_scatter_indices, update_vals, ++ scatter_axes)); ++ break; ++ case utils::ScatterType::Add: ++ res.push_back(mx::scatter_add(inputs[i], tmp_scatter_indices, ++ update_vals, scatter_axes)); ++ break; ++ case utils::ScatterType::Prod: ++ res.push_back(mx::scatter_prod(inputs[i], tmp_scatter_indices, ++ update_vals, scatter_axes)); ++ break; ++ case utils::ScatterType::Max: ++ res.push_back(mx::scatter_max(inputs[i], tmp_scatter_indices, ++ update_vals, scatter_axes)); ++ break; ++ case utils::ScatterType::Min: ++ res.push_back(mx::scatter_min(inputs[i], tmp_scatter_indices, ++ update_vals, scatter_axes)); ++ break; ++ } ++ } ++ return res; ++ }; ++ return mx::compile(fun); ++} ++ ++MlxFunc func(mlir::ModuleOp mod, mlir::func::FuncOp op) { ++ MlxFunc fun = [func_info = std::shared_ptr( ++ new utils::FuncInfo(mod, op.getName().str()))]( ++ const std::vector& inputs) mutable { ++ auto mod = func_info->mod_.get(); ++ auto op = mod.lookupSymbol(func_info->name_); ++ return detail::func(mod, op, inputs); ++ }; ++ return mx::compile(std::move(fun)); ++} ++ ++} // namespace compile +\ No newline at end of file +diff --git a/xla/pjrt/plugin/mlx/compile.h b/xla/pjrt/plugin/mlx/compile.h +new file mode 100644 +index 0000000000..af5cc7e342 +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/compile.h +@@ -0,0 +1,30 @@ ++#ifndef XLA_PJRT_PLUGIN_STABLEHLO_MLX_COMPILE_H_ ++#define XLA_PJRT_PLUGIN_STABLEHLO_MLX_COMPILE_H_ ++ ++#include ++ ++#include "mlx/mlx.h" ++#include "mlir/IR/Operation.h" ++ ++#include "stablehlo/reference/Api.h" ++#include "mlir/IR/Operation.h" ++#include "mlir/Dialect/Func/IR/FuncOps.h" ++#include "xla/pjrt/plugin/mlx/utils.h" ++ ++namespace mx = mlx::core; ++ ++namespace compile { ++namespace detail { ++std::vector cbrt(const std::vector& inputs); ++std::vector func(mlir::ModuleOp mod, mlir::func::FuncOp op, ++ const std::vector& inputs, ++ bool recursive_compile = true); ++} // namespace detail ++MlxFunc dotGeneral(mlir::stablehlo::DotGeneralOp op); ++MlxFunc gather(mlir::stablehlo::GatherOp op); ++MlxFunc pad(mlir::stablehlo::PadOp op); ++MlxFunc scatter(mlir::stablehlo::ScatterOp op); ++MlxFunc func(mlir::ModuleOp mod, mlir::func::FuncOp op); ++} // namespace compile ++ ++#endif // XLA_PJRT_PLUGIN_STABLEHLO_MLX_COMPILE_H_ +\ No newline at end of file diff --git a/xla/pjrt/plugin/mlx/device.cc b/xla/pjrt/plugin/mlx/device.cc new file mode 100644 index 0000000000..6984da5b7f @@ -1344,10 +2730,10 @@ index 0000000000..ef42968a3a \ No newline at end of file diff --git a/xla/pjrt/plugin/mlx/executable.cc b/xla/pjrt/plugin/mlx/executable.cc new file mode 100644 -index 0000000000..02aaf61559 +index 0000000000..dfdfb33f87 --- /dev/null +++ b/xla/pjrt/plugin/mlx/executable.cc -@@ -0,0 +1,2356 @@ +@@ -0,0 +1,507 @@ +/* Copyright 2024 The OpenXLA Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. @@ -1371,24 +2757,24 @@ index 0000000000..02aaf61559 +#include +#include +#include ++#include +#include +#include + +// TODO(@cryptodeal): might need to update `BUILD` +#include "mlir/IR/Visitors.h" -+#include "llvm/ADT/TypeSwitch.h" -+#include "llvm/ADT/Hashing.h" -+ +#include "mlx/mlx.h" +#include "absl/log/log.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" ++#include "llvm/ADT/TypeSwitch.h" +#include "mlir/Bytecode/BytecodeWriter.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" +#include "mlir/IR/AsmState.h" +#include "mlir/IR/BuiltinAttributes.h" +#include "mlir/IR/MLIRContext.h" +#include "mlir/IR/Operation.h" ++#include "mlir/IR/OperationSupport.h" +#include "mlir/IR/OwningOpRef.h" +#include "mlir/Interfaces/DataLayoutInterfaces.h" +#include "mlir/Pass/PassManager.h" @@ -1404,6 +2790,7 @@ index 0000000000..02aaf61559 +#include "xla/pjrt/pjrt_executable.h" +#include "xla/pjrt/pjrt_future.h" +#include "xla/pjrt/plugin/mlx/buffer.h" ++#include "xla/pjrt/plugin/mlx/compile.h" +#include "xla/pjrt/plugin/mlx/logging.h" +#include "xla/pjrt/plugin/mlx/utils.h" +#include "xla/service/computation_placer.h" @@ -1413,11 +2800,6 @@ index 0000000000..02aaf61559 +#define DEBUG_TYPE "stablehlo-pjrt" + +namespace mx = mlx::core; -+ -+typedef absl::StatusOr StatusOrArray; -+typedef absl::StatusOr> StatusOrArrays; -+typedef absl::StatusOr StatusOrInt; -+ +namespace mlir::stablehlo { + +#define UNIMPLEMENTED(name) \ @@ -1431,24 +2813,53 @@ index 0000000000..02aaf61559 +using xla::PjRtLoadedExecutable; +using xla::PjRtMemorySpace; + -+void resolveFuncDeps(ModuleOp mod, std::string func_name, Tree& dep_tree, -+ Node* parent = nullptr) { -+ auto new_node = dep_tree.insert(func_name, parent); -+ auto func = mod.lookupSymbol(func_name); -+ auto& block = func.front(); -+ std::set func_names; -+ for (auto op : block.getOps()) { -+ std::string callee_name = op.getCallee().str(); -+ func_names.insert(callee_name); -+ resolveFuncDeps(mod, callee_name, dep_tree, new_node); ++mlir::OwningOpRef cloneIntoContext(ModuleOp module, ++ MLIRContext& context) { ++ // Clone the module into the context. MHLO->StableHLO just in case. ++ PassManager pm(module->getContext()); ++ pm.addPass(mhlo::createHloLegalizeToStablehloPass()); ++ if (failed(pm.run(module))) { ++ LOG(ERROR) << "Failed to convert MHLO to StableHLO"; ++ return nullptr; + } ++ ++ std::string bytecode; ++ llvm::raw_string_ostream os(bytecode); ++ mlir::BytecodeWriterConfig config; ++ mlir::OwningOpRef cloned = module.clone(); ++ if (mlir::failed(mlir::writeBytecodeToFile(*cloned, os, config))) { ++ LOG(ERROR) << "Failed to write bytecode to string\n"; ++ return nullptr; ++ } ++ return *parseStablehloModule(bytecode, context); ++} ++ ++LogicalResult decomposeChloToStablehlo(ModuleOp module) { ++ PassManager pm(module->getContext()); ++ stablehlo_ext::createChloLegalizeToStablehloPipeline(pm); ++ if (failed(pm.run(module))) { ++ return module->emitError() << "Failed to recompose CHLO"; ++ } ++ return success(); ++} ++ ++LogicalResult runHardwareIndependentOptimizations(ModuleOp module) { ++ PassManager pm(module->getContext()); ++ pm.addNestedPass( ++ stablehlo::createStablehloTargetIndependentOptimizationPass()); ++ if (failed(pm.run(module))) { ++ return module->emitError() ++ << "Failed to run hardware independent optimizations"; ++ } ++ return success(); +} + +absl::Status validateModule(ModuleOp mod) { + absl::Status status; + // TODO: some ops should have additional checks (e.g. `stablehlo::ReduceOp`, -+ // `stablehlo::SortOp`, etc.) where we need additional checks at compile time -+ // to ensure that we're correctly mapping to the corresponding MLX op. ++ // `stablehlo::SortOp`, `stablehlo::ScatterOp`, etc.) where we need additional ++ // checks at compile time to ensure that we're correctly mapping to the ++ // corresponding MLX op. + mod.walk([&status](Operation* op) { + // Series of checks to ensure we support the module + return llvm::TypeSwitch(op) @@ -1539,13 +2950,6 @@ index 0000000000..02aaf61559 + // TODO(@cryptodeal): These ops were supported prior to `mlx::compile` + // need to modify the implementation to ensure no use of `mlx::eval` and + // ensure that `mlx::random::key` is being propogated correctly. -+ .Case( -+ [&status, &op](auto o) { -+ status = absl::UnimplementedError( -+ absl::StrCat("Unsupported op: ", ToString(op))); -+ return WalkResult::interrupt(); -+ }) + .Default([&status, &op](auto o) { + status = absl::UnimplementedError( + absl::StrCat("Unsupported op: ", ToString(op))); @@ -1555,1811 +2959,6 @@ index 0000000000..02aaf61559 + return status; +} + -+mlir::OwningOpRef cloneIntoContext(ModuleOp module, -+ MLIRContext& context) { -+ // Clone the module into the context. MHLO->StableHLO just in case. -+ PassManager pm(module->getContext()); -+ pm.addPass(mhlo::createHloLegalizeToStablehloPass()); -+ if (failed(pm.run(module))) { -+ LOG(ERROR) << "Failed to convert MHLO to StableHLO"; -+ return nullptr; -+ } -+ -+ std::string bytecode; -+ llvm::raw_string_ostream os(bytecode); -+ mlir::BytecodeWriterConfig config; -+ mlir::OwningOpRef cloned = module.clone(); -+ if (mlir::failed(mlir::writeBytecodeToFile(*cloned, os, config))) { -+ LOG(ERROR) << "Failed to write bytecode to string\n"; -+ return nullptr; -+ } -+ return *parseStablehloModule(bytecode, context); -+} -+ -+LogicalResult decomposeChloToStablehlo(ModuleOp module) { -+ PassManager pm(module->getContext()); -+ stablehlo_ext::createChloLegalizeToStablehloPipeline(pm); -+ if (failed(pm.run(module))) { -+ return module->emitError() << "Failed to recompose CHLO"; -+ } -+ return success(); -+} -+ -+std::optional getUnsupportedOp(ModuleOp module) { -+ std::optional unsupported_op(std::nullopt); -+ module.walk([&unsupported_op](Operation* op) { -+ auto cc = llvm::dyn_cast(op); -+ if (cc) { -+ unsupported_op = op; -+ return WalkResult::interrupt(); -+ } -+ return WalkResult::advance(); -+ }); -+ return unsupported_op; -+} -+ -+LogicalResult runHardwareIndependentOptimizations(ModuleOp module) { -+ PassManager pm(module->getContext()); -+ pm.addNestedPass( -+ stablehlo::createStablehloAggressiveFolderPass()); -+ pm.addNestedPass( -+ stablehlo::createStablehloAggressiveSimplificationPass()); -+ if (failed(pm.run(module))) { -+ return module->emitError() -+ << "Failed to run hardware independent optimizations"; -+ } -+ return success(); -+} -+ -+mlir::ShapedType getResultType(ModuleOp module, mx::Dtype dtype, -+ SmallVector shape) { -+ switch (dtype) { -+ case mx::bfloat16: -+ return RankedTensorType::get(ArrayRef(shape), -+ BFloat16Type::get(module.getContext())); -+ case mx::float16: -+ return RankedTensorType::get(ArrayRef(shape), -+ Float16Type::get(module.getContext())); -+ case mx::int8: -+ return RankedTensorType::get( -+ ArrayRef(shape), -+ IntegerType::get(module.getContext(), 8, -+ mlir::IntegerType::Signless)); -+ case mx::uint8: -+ return RankedTensorType::get( -+ ArrayRef(shape), -+ IntegerType::get(module.getContext(), 8, -+ mlir::IntegerType::Unsigned)); -+ case mx::int16: -+ return RankedTensorType::get( -+ ArrayRef(shape), -+ IntegerType::get(module.getContext(), 16, -+ mlir::IntegerType::Signless)); -+ case mx::uint16: -+ return RankedTensorType::get( -+ ArrayRef(shape), -+ IntegerType::get(module.getContext(), 16, -+ mlir::IntegerType::Unsigned)); -+ case mx::int32: -+ return RankedTensorType::get( -+ ArrayRef(shape), -+ IntegerType::get(module.getContext(), 32, -+ mlir::IntegerType::Signless)); -+ case mx::uint32: -+ return RankedTensorType::get( -+ ArrayRef(shape), -+ IntegerType::get(module.getContext(), 32, -+ mlir::IntegerType::Unsigned)); -+ case mx::int64: -+ return RankedTensorType::get( -+ ArrayRef(shape), -+ IntegerType::get(module.getContext(), 64, -+ mlir::IntegerType::Signless)); -+ case mx::uint64: -+ return RankedTensorType::get( -+ ArrayRef(shape), -+ IntegerType::get(module.getContext(), 64, -+ mlir::IntegerType::Unsigned)); -+ case mx::complex64: -+ return RankedTensorType::get( -+ ArrayRef(shape), -+ ComplexType::get(Float32Type::get(module.getContext()))); -+ default: -+ return RankedTensorType::get(ArrayRef(shape), -+ Float32Type::get(module.getContext())); -+ } -+} -+ -+// Returns true if the given `ReduceOp` matches -+// `zml.Tensor.argMax` op implementation. -+bool isArgMaxReduce(stablehlo::ReduceOp& reduce_op) { -+ Block& body_block = reduce_op.getBody().front(); -+ unsigned op_count = 0; -+ // TODO (@cryptodeal): This is a lazy verification; maybe check operands -+ // for each operation in the block. -+ for (auto& op : body_block.getOperations()) { -+ switch (op_count) { -+ case 0: { -+ auto compare_op = llvm::dyn_cast(op); -+ if (compare_op && -+ compare_op.getComparisonDirection() == ComparisonDirection::GT) { -+ break; -+ } else -+ return false; -+ } -+ case 1: { -+ auto compare_op = llvm::dyn_cast(op); -+ if (compare_op && -+ compare_op.getComparisonDirection() == ComparisonDirection::NE) { -+ break; -+ } else -+ return false; -+ } -+ case 2: -+ if (llvm::dyn_cast(op)) { -+ break; -+ } else -+ return false; -+ case 3: -+ if (llvm::dyn_cast(op)) { -+ break; -+ } else -+ return false; -+ case 4: { -+ auto compare_op = llvm::dyn_cast(op); -+ if (compare_op && -+ compare_op.getComparisonDirection() == ComparisonDirection::EQ) { -+ break; -+ } else -+ return false; -+ } -+ case 5: { -+ auto compare_op = llvm::dyn_cast(op); -+ if (compare_op && -+ compare_op.getComparisonDirection() == ComparisonDirection::LT) { -+ break; -+ } else -+ return false; -+ } -+ case 6: -+ if (llvm::dyn_cast(op)) { -+ break; -+ } else -+ return false; -+ case 7: -+ if (llvm::dyn_cast(op)) { -+ break; -+ } else -+ return false; -+ case 8: -+ if (llvm::dyn_cast(op)) { -+ break; -+ } else -+ return false; -+ case 9: -+ if (llvm::dyn_cast(op)) { -+ break; -+ } else -+ return false; -+ default: -+ return false; -+ } -+ op_count++; -+ } -+ return true; -+} -+ -+// Returns true if the given `ReduceOp` matches -+// `zml.Tensor.sum` op implementation. -+bool isSumReduce(stablehlo::ReduceOp& reduce_op) { -+ Block& body_block = reduce_op.getBody().front(); -+ unsigned op_count = 0; -+ // TODO (@cryptodeal): This is a lazy verification; maybe check operands -+ // for each operation in the block. -+ for (auto& op : body_block.getOperations()) { -+ switch (op_count) { -+ case 0: -+ if (llvm::dyn_cast(op)) { -+ break; -+ } else -+ return false; -+ case 1: -+ if (llvm::dyn_cast(op)) { -+ break; -+ } else -+ return false; -+ } -+ op_count++; -+ } -+ return true; -+} -+ -+// Returns true if the given `ReduceOp` matches -+// `zml.Tensor.max` op implementation. -+bool isMaxReduce(stablehlo::ReduceOp& reduce_op) { -+ Block& body_block = reduce_op.getBody().front(); -+ unsigned op_count = 0; -+ // TODO (@cryptodeal): This is a lazy verification; maybe check operands -+ // for each operation in the block. -+ for (auto& op : body_block.getOperations()) { -+ switch (op_count) { -+ case 0: -+ if (llvm::dyn_cast(op)) { -+ break; -+ } else -+ return false; -+ case 1: -+ if (llvm::dyn_cast(op)) { -+ break; -+ } else -+ return false; -+ } -+ op_count++; -+ } -+ return true; -+} -+ -+// Returns true if the given `ReduceOp` matches -+// `zml.Tensor.max` op implementation. -+bool isMinReduce(stablehlo::ReduceOp& reduce_op) { -+ Block& body_block = reduce_op.getBody().front(); -+ unsigned op_count = 0; -+ // TODO (@cryptodeal): This is a lazy verification; maybe check operands -+ // for each operation in the block. -+ for (auto& op : body_block.getOperations()) { -+ switch (op_count) { -+ case 0: -+ if (llvm::dyn_cast(op)) { -+ break; -+ } else -+ return false; -+ case 1: -+ if (llvm::dyn_cast(op)) { -+ break; -+ } else -+ return false; -+ } -+ op_count++; -+ } -+ return true; -+} -+ -+// Returns true if the given `ReduceOp` matches -+// `zml.Tensor.any` op implementation. -+bool isAnyReduce(stablehlo::ReduceOp& reduce_op) { -+ Block& body_block = reduce_op.getBody().front(); -+ unsigned op_count = 0; -+ // TODO (@cryptodeal): This is a lazy verification; maybe check operands -+ // for each operation in the block. -+ for (auto& op : body_block.getOperations()) { -+ switch (op_count) { -+ case 0: -+ if (llvm::dyn_cast(op)) { -+ break; -+ } else -+ return false; -+ case 1: -+ if (llvm::dyn_cast(op)) { -+ break; -+ } else -+ return false; -+ } -+ op_count++; -+ } -+ return true; -+} -+ -+absl::StatusOr getScatterType(mlir::Block& block) { -+ unsigned op_count = 0; -+ utils::ScatterType scatter_type = utils::ScatterType::Replace; -+ for (const auto& op : block.getOperations()) { -+ switch (op_count) { -+ case 0: -+ if (llvm::dyn_cast(op)) { -+ return scatter_type; -+ } -+ if (llvm::dyn_cast(op)) { -+ scatter_type = utils::ScatterType::Add; -+ } else if (llvm::dyn_cast(op)) { -+ scatter_type = utils::ScatterType::Prod; -+ } else if (llvm::dyn_cast(op)) { -+ scatter_type = utils::ScatterType::Prod; -+ } else if (llvm::dyn_cast(op)) { -+ scatter_type = utils::ScatterType::Max; -+ } else if (llvm::dyn_cast(op)) { -+ scatter_type = utils::ScatterType::Min; -+ } else { -+ return xla::Internal("Unsupported comparator: %s", ToString(block)); -+ } -+ break; -+ case 1: -+ if (llvm::dyn_cast(op)) { -+ return scatter_type; -+ } -+ default: -+ return xla::Internal("Unsupported comparator: %s", ToString(block)); -+ } -+ } -+} -+ -+absl::StatusOr> getSortInfo( -+ mlir::Block& block) { -+ unsigned op_count = 0; -+ mx::ComparatorType comparator = mx::ComparatorType::LessThan; -+ unsigned input_idx = 0; -+ for (const auto& op : block.getOperations()) { -+ switch (op_count) { -+ case 0: { -+ if (auto compare_op = llvm::dyn_cast(op)) { -+ switch (compare_op.getComparisonDirection()) { -+ case ComparisonDirection::GT: { -+ auto lhs = llvm::dyn_cast(compare_op.getLhs()); -+ auto rhs = llvm::dyn_cast(compare_op.getRhs()); -+ if (lhs && rhs) { -+ unsigned lhs_idx = static_cast(lhs.getArgNumber()); -+ unsigned rhs_idx = static_cast(rhs.getArgNumber()); -+ if (rhs_idx - 1 == lhs_idx) { -+ comparator = mx::ComparatorType::GreaterThan; -+ input_idx = lhs_idx == 0 ? 0 : lhs_idx / 2; -+ break; -+ } -+ } -+ return xla::Internal("Unsupported comparator: %s", -+ ToString(block)); -+ } -+ case ComparisonDirection::LT: { -+ auto lhs = llvm::dyn_cast(compare_op.getLhs()); -+ auto rhs = llvm::dyn_cast(compare_op.getRhs()); -+ if (lhs && rhs) { -+ unsigned lhs_idx = static_cast(lhs.getArgNumber()); -+ unsigned rhs_idx = static_cast(rhs.getArgNumber()); -+ if (rhs_idx - 1 == lhs_idx) { -+ comparator = mx::ComparatorType::LessThan; -+ input_idx = lhs_idx == 0 ? 0 : lhs_idx / 2; -+ break; -+ } -+ } -+ return xla::Internal("Unsupported comparator: %s", -+ ToString(block)); -+ } -+ default: -+ return xla::Internal("Unsupported comparator: %s", -+ ToString(block)); -+ } -+ break; -+ } -+ } -+ case 1: -+ if (llvm::dyn_cast(op)) break; -+ default: -+ return xla::Internal("Unsupported comparator: %s", ToString(block)); -+ } -+ op_count++; -+ } -+ return std::make_pair(input_idx, comparator); -+} -+ -+mx::array getOperandArray( -+ const Value& operand, -+ const std::unordered_map>& block_ctx, -+ const std::vector& inputs) { -+ // check if operand is the result of a previous operation -+ if (auto defining_op = operand.getDefiningOp()) { -+ if (auto search = block_ctx.find(defining_op); search != block_ctx.end()) { -+ for (auto i = 0; i < defining_op->getNumResults(); i++) { -+ Value maybe_res = defining_op->getResult(i); -+ if (maybe_res == operand) { -+ return search->second[i]; -+ } -+ } -+ } -+ } else { -+ return inputs[llvm::cast(operand).getArgNumber()]; -+ } -+} -+ -+std::pair getValueInfo(Value v) { -+ auto val_type = mlir::cast(v.getType()); -+ return std::make_pair(*utils::dtype::fromMlirType(val_type.getElementType()), -+ std::vector(val_type.getShape().begin(), -+ val_type.getShape().end())); -+} -+ -+std::pair getTypeInfo(Type v) { -+ auto val_type = mlir::cast(v); -+ return std::make_pair(*utils::dtype::fromMlirType(val_type.getElementType()), -+ std::vector(val_type.getShape().begin(), -+ val_type.getShape().end())); -+} -+ -+CallOpInfo getCallOpInfo(ModuleOp mod, func::FuncOp o) { -+ auto func_type = o.getFunctionType(); -+ -+ std::vector> input_types; -+ for (auto arg : func_type.getInputs()) { -+ input_types.emplace_back(getTypeInfo(arg)); -+ } -+ std::vector> result_types; -+ for (auto result : func_type.getResults()) { -+ result_types.emplace_back(getTypeInfo(result)); -+ } -+ return CallOpInfo(input_types, result_types, o.getName().str()); -+} -+ -+CallOpInfo getCallOpInfo(ModuleOp mod, func::CallOp o) { -+ std::vector> input_types; -+ for (auto arg : o.getOperands()) { -+ input_types.emplace_back(getValueInfo(arg)); -+ } -+ std::vector> result_types; -+ for (auto result : o.getResults()) { -+ result_types.emplace_back(getValueInfo(result)); -+ } -+ auto func = mod.lookupSymbol(o.getCallee()); -+ return CallOpInfo(input_types, result_types, func.getName().str()); -+} -+ -+IotaOpInfo getIotaOpInfo(stablehlo::IotaOp o) { -+ std::pair result_info = getValueInfo(o.getResult()); -+ auto iota_dim = o.getIotaDimension(); -+ std::vector dimensions; -+ for (auto i = 0; i < dimensions.size(); i++) { -+ dimensions.push_back(i != iota_dim ? 1 : result_info.second[i]); -+ } -+ return IotaOpInfo(std::vector>{}, -+ {result_info}, dimensions, result_info.first, iota_dim, -+ result_info.second); -+} -+ -+CbrtOpInfo getCbrtOpInfo(stablehlo::CbrtOp o) { -+ return CbrtOpInfo({getValueInfo(o.getOperand())}, -+ {getValueInfo(o.getResult())}); -+} -+ -+BroadcastInDimOpInfo getBroadcastInDimOpInfo(stablehlo::BroadcastInDimOp o) { -+ auto result_info = getValueInfo(o.getResult()); -+ return BroadcastInDimOpInfo( -+ {getValueInfo(o.getOperand())}, {result_info}, -+ std::vector(o.getBroadcastDimensions().begin(), -+ o.getBroadcastDimensions().end()), -+ result_info.second); -+} -+ -+DotGeneralOpInfo getDotGeneralOpInfo(stablehlo::DotGeneralOp o) { -+ auto dot_dim_nums = o.getDotDimensionNumbers(); -+ std::vector lhs_batch_dims( -+ dot_dim_nums.getLhsBatchingDimensions().begin(), -+ dot_dim_nums.getLhsBatchingDimensions().end()); -+ std::vector rhs_batch_dims( -+ dot_dim_nums.getRhsBatchingDimensions().begin(), -+ dot_dim_nums.getRhsBatchingDimensions().end()); -+ std::vector lhs_contract_dims( -+ dot_dim_nums.getLhsContractingDimensions().begin(), -+ dot_dim_nums.getLhsContractingDimensions().end()); -+ std::vector rhs_contract_dims( -+ dot_dim_nums.getRhsContractingDimensions().begin(), -+ dot_dim_nums.getRhsContractingDimensions().end()); -+ return DotGeneralOpInfo({getValueInfo(o.getLhs()), getValueInfo(o.getRhs())}, -+ {getValueInfo(o.getResult())}, lhs_batch_dims, -+ rhs_batch_dims, lhs_contract_dims, rhs_contract_dims); -+} -+ -+GatherOpInfo getGatherOpInfo(stablehlo::GatherOp o) { -+ std::vector> operand_info; -+ for (auto operand : o.getOperands()) { -+ operand_info.emplace_back(getValueInfo(operand)); -+ } -+ operand_info.emplace_back(getValueInfo(o.getStartIndices())); -+ auto result_info = getValueInfo(o.getResult()); -+ auto dim_nums = o.getDimensionNumbers(); -+ std::vector offset_dims( -+ o.getDimensionNumbers().getOffsetDims().begin(), -+ o.getDimensionNumbers().getOffsetDims().end()); -+ std::vector collapsed_slice_dims( -+ o.getDimensionNumbers().getCollapsedSliceDims().begin(), -+ o.getDimensionNumbers().getCollapsedSliceDims().end()); -+ std::vector operand_batching_dims( -+ o.getDimensionNumbers().getOperandBatchingDims().begin(), -+ o.getDimensionNumbers().getOperandBatchingDims().end()); -+ std::vector start_indices_batching_dims( -+ o.getDimensionNumbers().getStartIndicesBatchingDims().begin(), -+ o.getDimensionNumbers().getStartIndicesBatchingDims().end()); -+ std::vector start_index_map( -+ o.getDimensionNumbers().getStartIndexMap().begin(), -+ o.getDimensionNumbers().getStartIndexMap().end()); -+ auto index_vector_dim = o.getDimensionNumbers().getIndexVectorDim(); -+ std::vector slice_sizes(o.getSliceSizes().begin(), -+ o.getSliceSizes().end()); -+ return GatherOpInfo(operand_info, {getValueInfo(o.getResult())}, -+ collapsed_slice_dims, index_vector_dim, offset_dims, -+ operand_batching_dims, result_info.second, slice_sizes, -+ start_index_map, start_indices_batching_dims); -+} -+ -+ScatterOpInfo getScatterOpInfo(stablehlo::ScatterOp o) { -+ std::vector> inputs_info; -+ for (auto input : o.getInputs()) { -+ inputs_info.emplace_back(getValueInfo(input)); -+ } -+ for (auto update : o.getUpdates()) { -+ inputs_info.emplace_back(getValueInfo(update)); -+ } -+ inputs_info.emplace_back(getValueInfo(o.getScatterIndices())); -+ std::vector>> results_info; -+ for (auto result : o.getResults()) { -+ results_info.emplace_back(getValueInfo(result)); -+ } -+ // Get scatter dimension numbers -+ auto scatter_dim_nums = o.getScatterDimensionNumbers(); -+ std::vector update_window_dims( -+ scatter_dim_nums.getUpdateWindowDims().begin(), -+ scatter_dim_nums.getUpdateWindowDims().end()); -+ std::vector inserted_window_dims( -+ scatter_dim_nums.getInsertedWindowDims().begin(), -+ scatter_dim_nums.getInsertedWindowDims().end()); -+ std::vector input_batching_dims( -+ scatter_dim_nums.getInputBatchingDims().begin(), -+ scatter_dim_nums.getInputBatchingDims().end()); -+ -+ std::vector scatter_indices_batching_dims( -+ scatter_dim_nums.getScatterIndicesBatchingDims().begin(), -+ scatter_dim_nums.getScatterIndicesBatchingDims().end()); -+ -+ std::vector scatter_dims_to_operand_dims( -+ scatter_dim_nums.getScatterDimsToOperandDims().begin(), -+ scatter_dim_nums.getScatterDimsToOperandDims().end()); -+ -+ auto index_vector_dim = scatter_dim_nums.getIndexVectorDim(); -+ // Get update computation -+ mlir::Block& update_computation = o.getUpdateComputation().front(); -+ -+ // TODO(@cryptodeal): need to add a check in `Compile` -+ utils::ScatterType scatter_type = *getScatterType(update_computation); -+ return ScatterOpInfo( -+ inputs_info, results_info, index_vector_dim, input_batching_dims, -+ inserted_window_dims, scatter_dims_to_operand_dims, -+ scatter_indices_batching_dims, scatter_type, update_window_dims); -+} -+ -+PadOpInfo getPadOpInfo(stablehlo::PadOp o) { -+ auto result_info = getValueInfo(o.getResult()); -+ std::vector edge_pad_low(o.getEdgePaddingLow().begin(), -+ o.getEdgePaddingLow().end()); -+ std::vector interior_pad(o.getInteriorPadding().begin(), -+ o.getInteriorPadding().end()); -+ return PadOpInfo( -+ {getValueInfo(o.getOperand()), getValueInfo(o.getPaddingValue())}, -+ {result_info}, result_info.second, edge_pad_low, interior_pad); -+} -+ -+void compileCallOp(OpLookup& op_lookup, ModuleOp mod, func::FuncOp o) { -+ CallOpInfo call_op_info = getCallOpInfo(mod, o); -+ if (auto search = op_lookup.call.find(call_op_info); -+ search == op_lookup.call.end()) { -+ // std::cout << "Compiling CallOp: " << o.getName().str().c_str() << -+ // std::endl; -+ auto& block = o.front(); -+ auto call_op_func = [&op_lookup, &block, mod, -+ call_op_info](const std::vector& inputs) { -+ std::unordered_map> block_ctx; -+ std::vector res; -+ for (Operation& op : block.getOperations()) { -+ res = -+ llvm::TypeSwitch>(&op) -+ .Case([&block_ctx, -+ &inputs](auto o) { -+ std::vector res; -+ for (Value val : o.getOperands()) { -+ res.emplace_back(getOperandArray(val, block_ctx, inputs)); -+ } -+ return res; -+ }) -+ .Case([&block_ctx, &inputs, mod, -+ &op_lookup](auto o) { -+ auto call_op_info = getCallOpInfo(mod, o); -+ std::vector operands; -+ for (Value val : o.getOperands()) { -+ operands.emplace_back( -+ getOperandArray(val, block_ctx, inputs)); -+ } -+ return op_lookup.call.find(call_op_info)->second(operands); -+ }) -+ // Handle StableHLO nullary ops -+ .Case([](auto o) { -+ return std::vector{ -+ *utils::array::fromDenseElementsAttr( -+ mlir::cast(o.getValue()))}; -+ }) -+ .Case([&op_lookup](auto o) { -+ return op_lookup.iota.find(getIotaOpInfo(o))->second({}); -+ }) -+ // .Case([](auto o) {}) -+ /* -+ .Case([](auto o) {}) -+ Deprecated see: -+ https://github.com/openxla/stablehlo/issues/2340 -+ https://github.com/openxla/stablehlo/pull/2283 -+ */ -+ // Handle StableHLO unary elementwise op -+ .Case([&block_ctx, &inputs](auto o) { -+ return std::vector{mx::abs( -+ getOperandArray(o.getOperand(), block_ctx, inputs))}; -+ }) -+ .Case([&block_ctx, &inputs, -+ &op_lookup](auto o) { -+ return op_lookup.cbrt.find(getCbrtOpInfo(o)) -+ ->second( -+ {getOperandArray(o.getOperand(), block_ctx, inputs)}); -+ }) -+ .Case([&block_ctx, &inputs](auto o) { -+ return std::vector{mx::ceil( -+ getOperandArray(o.getOperand(), block_ctx, inputs))}; -+ }) -+ .Case([&block_ctx, &inputs](auto o) { -+ return std::vector{mx::astype( -+ getOperandArray(o.getOperand(), block_ctx, inputs), -+ *utils::dtype::fromMlirType( -+ o.getResult().getType().getElementType()))}; -+ }) -+ // .Case([](auto o) {}) -+ .Case([&block_ctx, &inputs](auto o) { -+ return std::vector{mx::cos( -+ getOperandArray(o.getOperand(), block_ctx, inputs))}; -+ }) -+ .Case([&block_ctx, &inputs](auto o) { -+ return std::vector{mx::exp( -+ getOperandArray(o.getOperand(), block_ctx, inputs))}; -+ }) -+ .Case([&block_ctx, &inputs](auto o) { -+ return std::vector{mx::expm1( -+ getOperandArray(o.getOperand(), block_ctx, inputs))}; -+ }) -+ .Case([&block_ctx, &inputs](auto o) { -+ return std::vector{mx::floor( -+ getOperandArray(o.getOperand(), block_ctx, inputs))}; -+ }) -+ .Case([&block_ctx, &inputs](auto o) { -+ return std::vector{mx::imag( -+ getOperandArray(o.getOperand(), block_ctx, inputs))}; -+ }) -+ .Case([&block_ctx, &inputs](auto o) { -+ return std::vector{mx::isfinite( -+ getOperandArray(o.getOperand(), block_ctx, inputs))}; -+ }) -+ .Case([&block_ctx, &inputs](auto o) { -+ return std::vector{mx::log( -+ getOperandArray(o.getOperand(), block_ctx, inputs))}; -+ }) -+ .Case([&block_ctx, &inputs](auto o) { -+ return std::vector{mx::log1p( -+ getOperandArray(o.getOperand(), block_ctx, inputs))}; -+ }) -+ .Case([&block_ctx, &inputs](auto o) { -+ return std::vector{mx::sigmoid( -+ getOperandArray(o.getOperand(), block_ctx, inputs))}; -+ }) -+ .Case([&block_ctx, &inputs](auto o) { -+ return std::vector{mx::logical_not( -+ getOperandArray(o.getOperand(), block_ctx, inputs))}; -+ }) -+ .Case([&block_ctx, &inputs](auto o) { -+ return std::vector{mx::negative( -+ getOperandArray(o.getOperand(), block_ctx, inputs))}; -+ }) -+ // .Case([](auto o) {}) -+ .Case([&block_ctx, &inputs](auto o) { -+ return std::vector{mx::real( -+ getOperandArray(o.getOperand(), block_ctx, inputs))}; -+ }) -+ // TODO(@cryptodeal): `stablehlo::RoundOp` does not match with -+ // the mlx metal implementation -+ // .Case([](auto o) {}) -+ .Case( -+ [&block_ctx, &inputs](auto o) { -+ return std::vector{mx::round( -+ getOperandArray(o.getOperand(), block_ctx, inputs))}; -+ }) -+ .Case([&block_ctx, &inputs](auto o) { -+ return std::vector{mx::rsqrt( -+ getOperandArray(o.getOperand(), block_ctx, inputs))}; -+ }) -+ .Case([&block_ctx, &inputs](auto o) { -+ return std::vector{mx::sign( -+ getOperandArray(o.getOperand(), block_ctx, inputs))}; -+ }) -+ .Case([&block_ctx, &inputs](auto o) { -+ return std::vector{mx::sin( -+ getOperandArray(o.getOperand(), block_ctx, inputs))}; -+ }) -+ .Case([&block_ctx, &inputs](auto o) { -+ return std::vector{mx::sqrt( -+ getOperandArray(o.getOperand(), block_ctx, inputs))}; -+ }) -+ .Case([&block_ctx, &inputs](auto o) { -+ return std::vector{mx::tan( -+ getOperandArray(o.getOperand(), block_ctx, inputs))}; -+ }) -+ .Case([&block_ctx, &inputs](auto o) { -+ return std::vector{mx::tanh( -+ getOperandArray(o.getOperand(), block_ctx, inputs))}; -+ }) -+ // Handle StableHLO binary elementwise ops -+ .Case([&block_ctx, &inputs](auto o) { -+ return std::vector{ -+ mx::add(getOperandArray(o.getLhs(), block_ctx, inputs), -+ getOperandArray(o.getRhs(), block_ctx, inputs))}; -+ }) -+ .Case([&block_ctx, &inputs](auto o) { -+ return std::vector{mx::arctan2( -+ getOperandArray(o.getLhs(), block_ctx, inputs), -+ getOperandArray(o.getRhs(), block_ctx, inputs))}; -+ }) -+ // TODO(@cryptodeal): implement complex op -+ // .Case([&block_ctx](auto -+ // op) {}) -+ .Case([&block_ctx, &inputs](auto o) { -+ return std::vector{mx::divide( -+ getOperandArray(o.getLhs(), block_ctx, inputs), -+ getOperandArray(o.getRhs(), block_ctx, inputs))}; -+ }) -+ .Case([&block_ctx, &inputs](auto o) { -+ return std::vector{mx::maximum( -+ getOperandArray(o.getLhs(), block_ctx, inputs), -+ getOperandArray(o.getRhs(), block_ctx, inputs))}; -+ }) -+ .Case([&block_ctx, &inputs](auto o) { -+ return std::vector{mx::minimum( -+ getOperandArray(o.getLhs(), block_ctx, inputs), -+ getOperandArray(o.getRhs(), block_ctx, inputs))}; -+ }) -+ .Case([&block_ctx, &inputs](auto o) { -+ return std::vector{mx::multiply( -+ getOperandArray(o.getLhs(), block_ctx, inputs), -+ getOperandArray(o.getRhs(), block_ctx, inputs))}; -+ }) -+ .Case([&block_ctx, &inputs](auto o) { -+ return std::vector{mx::power( -+ getOperandArray(o.getLhs(), block_ctx, inputs), -+ getOperandArray(o.getRhs(), block_ctx, inputs))}; -+ }) -+ .Case([&block_ctx, &inputs](auto o) { -+ return std::vector{mx::remainder( -+ getOperandArray(o.getLhs(), block_ctx, inputs), -+ getOperandArray(o.getRhs(), block_ctx, inputs))}; -+ }) -+ .Case([&block_ctx, &inputs](auto o) { -+ return std::vector{mx::left_shift( -+ getOperandArray(o.getLhs(), block_ctx, inputs), -+ getOperandArray(o.getRhs(), block_ctx, inputs))}; -+ }) -+ .Case( -+ [&block_ctx, &inputs](auto o) { -+ /** -+ * Per Metal Spec: -+ * For the right-shift operator, if E1 has an unsigned -+ * type or if E1 has a signed type and a nonnegative -+ * value, the vacated bits are filled with zeros. If E1 -+ * has a signed type and a negative value, the vacated -+ * bits are filled with ones. -+ */ -+ auto lhs = getOperandArray(o.getLhs(), block_ctx, inputs); -+ auto rhs = getOperandArray(o.getRhs(), block_ctx, inputs); -+ auto target_dtype = rhs.dtype(); -+ switch (lhs.dtype().size()) { -+ case 1: -+ target_dtype = mx::int8; -+ break; -+ case 2: -+ target_dtype = mx::int16; -+ break; -+ case 4: -+ target_dtype = mx::int32; -+ break; -+ case 8: -+ target_dtype = mx::int64; -+ break; -+ } -+ return std::vector{mx::view( -+ mx::right_shift(mx::view(lhs, target_dtype), -+ mx::astype(rhs, target_dtype)), -+ lhs.dtype())}; -+ }) -+ .Case( -+ [&block_ctx, &inputs](auto o) { -+ // Ensures that we bitcast to `uint` type before -+ // performing the right shift. Should ensure that vacated -+ // bits are zero populated. -+ auto lhs = getOperandArray(o.getLhs(), block_ctx, inputs); -+ auto rhs = getOperandArray(o.getRhs(), block_ctx, inputs); -+ auto target_dtype = rhs.dtype(); -+ switch (lhs.dtype().size()) { -+ case 1: -+ target_dtype = mx::uint8; -+ break; -+ case 2: -+ target_dtype = mx::uint16; -+ break; -+ case 4: -+ target_dtype = mx::uint32; -+ break; -+ case 8: -+ target_dtype = mx::uint64; -+ break; -+ } -+ return std::vector{mx::view( -+ mx::right_shift(mx::view(lhs, target_dtype), -+ mx::astype(rhs, target_dtype)), -+ lhs.dtype())}; -+ }) -+ .Case([&block_ctx, &inputs](auto o) { -+ return std::vector{mx::subtract( -+ getOperandArray(o.getLhs(), block_ctx, inputs), -+ getOperandArray(o.getRhs(), block_ctx, inputs))}; -+ }) -+ // Handle StableHLO binary logical elementwise ops -+ .Case([&block_ctx, &inputs](auto o) { -+ auto lhs = getOperandArray(o.getLhs(), block_ctx, inputs); -+ auto rhs = getOperandArray(o.getRhs(), block_ctx, inputs); -+ switch (mx::kindof(lhs.dtype())) { -+ case mx::Dtype::Kind::b: -+ return std::vector{mx::logical_and(lhs, rhs)}; -+ default: -+ return std::vector{mx::bitwise_and(lhs, rhs)}; -+ } -+ }) -+ .Case([&block_ctx, &inputs](auto o) { -+ auto lhs = getOperandArray(o.getLhs(), block_ctx, inputs); -+ auto rhs = getOperandArray(o.getRhs(), block_ctx, inputs); -+ switch (mx::kindof(lhs.dtype())) { -+ case mx::Dtype::Kind::b: -+ return std::vector{mx::logical_or(lhs, rhs)}; -+ default: -+ return std::vector{mx::bitwise_or(lhs, rhs)}; -+ } -+ }) -+ .Case([&block_ctx, &inputs](auto o) { -+ return std::vector{mx::bitwise_xor( -+ getOperandArray(o.getLhs(), block_ctx, inputs), -+ getOperandArray(o.getRhs(), block_ctx, inputs))}; -+ }) -+ // Handle StableHLO communication ops -+ // .Case([](auto o) {}) -+ // .Case([](auto o) {}) -+ // .Case([](auto o) {}) -+ // .Case([](auto o) {}) -+ -+ // Handle StableHLO parallelism related ops -+ // .Case([](auto o) {}) -+ // .Case([](auto o) {}) -+ -+ // Handle StableHLO control flow ops -+ // .Case([](auto o) {}) -+ // TODO(@cryptodeal): need custom kernels in order to -+ // remove the need to call `mx::eval` for the following ops -+ // .Case([](auto o)) -+ // .Case([](auto o)) -+ // .Case([](auto o)) -+ // .Case([](auto o) {}) -+ // .Case([](auto o) {}) -+ // .Case([](auto o) {}) -+ // .Case([](auto o) {}) -+ .Case([&block_ctx, &inputs](auto o) { -+ auto operand = -+ getOperandArray(o.getOperand(0), block_ctx, inputs); -+ if (isArgMaxReduce(o)) { -+ auto axis = static_cast(o.getDimensions()[0]); -+ auto indices = mx::argmax(operand, axis); -+ -+ mx::Dtype result_type = *utils::dtype::fromMlirType( -+ mlir::cast(o.getResults()[1].getType()) -+ .getElementType()); -+ return std::vector{ -+ mx::take(operand, indices, axis), -+ mx::astype(indices, result_type)}; -+ } -+ if (isSumReduce(o)) { -+ return std::vector{mx::sum( -+ operand, static_cast(o.getDimensions()[0]))}; -+ } -+ if (isMaxReduce(o)) { -+ return std::vector{mx::max( -+ operand, static_cast(o.getDimensions()[0]))}; -+ } -+ if (isMinReduce(o)) { -+ return std::vector{mx::min( -+ operand, static_cast(o.getDimensions()[0]))}; -+ } -+ // TODO(@cryptodeal): further cleanup module validation -+ // done via call to `Compile` to check for valid reduce types. -+ -+ // For now, we assume `mx::any` if none of the above -+ -+ // if (isAnyReduce(o)) { -+ return std::vector{ -+ mx::any(operand, static_cast(o.getDimensions()[0]))}; -+ //} -+ }) -+ // Handle StableHLO tuple ops -+ // .Case([](auto o) {}) -+ // .Case([](auto o) {}) -+ .Case([&block_ctx, &inputs](auto o) { -+ auto lhs = getOperandArray(o.getLhs(), block_ctx, inputs); -+ auto rhs = getOperandArray(o.getRhs(), block_ctx, inputs); -+ switch (o.getComparisonDirection()) { -+ case ComparisonDirection::NE: -+ return std::vector{mx::not_equal(lhs, rhs)}; -+ case ComparisonDirection::GE: -+ return std::vector{ -+ mx::greater_equal(lhs, rhs)}; -+ case ComparisonDirection::GT: -+ return std::vector{mx::greater(lhs, rhs)}; -+ case ComparisonDirection::LE: -+ return std::vector{mx::less_equal(lhs, rhs)}; -+ case ComparisonDirection::LT: -+ return std::vector{mx::less(lhs, rhs)}; -+ case ComparisonDirection::EQ: -+ return std::vector{mx::equal(lhs, rhs)}; -+ } -+ }) -+ // Handle StableHLO Slice ops -+ .Case([&block_ctx, &inputs](auto o) { -+ std::vector start_indices( -+ o.getStartIndices().begin(), o.getStartIndices().end()); -+ std::vector limit_indices( -+ o.getLimitIndices().begin(), o.getLimitIndices().end()); -+ std::vector strides(o.getStrides().begin(), -+ o.getStrides().end()); -+ return std::vector{mx::slice( -+ getOperandArray(o.getOperand(), block_ctx, inputs), -+ start_indices, limit_indices, strides)}; -+ }) -+ .Case([&block_ctx, &inputs](auto o) { -+ auto operand = -+ getOperandArray(o.getOperand(), block_ctx, inputs); -+ std::vector indices; -+ for (auto val : o.getStartIndices()) { -+ indices.emplace_back( -+ getOperandArray(val, block_ctx, inputs)); -+ } -+ auto start_indices = mx::concatenate(indices); -+ std::vector axes(operand.ndim()); -+ std::iota(axes.begin(), axes.end(), 0); -+ std::vector slice_sizes(o.getSliceSizes().begin(), -+ o.getSliceSizes().end()); -+ return std::vector{ -+ mx::slice(operand, start_indices, axes, slice_sizes)}; -+ }) -+ .Case( -+ [&block_ctx, &inputs](auto o) { -+ auto operand = -+ getOperandArray(o.getOperand(), block_ctx, inputs); -+ auto update = -+ getOperandArray(o.getUpdate(), block_ctx, inputs); -+ std::vector indices; -+ for (auto val : o.getStartIndices()) { -+ indices.emplace_back( -+ getOperandArray(val, block_ctx, inputs)); -+ } -+ -+ auto start_indices = mx::concatenate(indices); -+ std::vector axes(operand.ndim()); -+ std::iota(axes.begin(), axes.end(), 0); -+ return std::vector{mx::slice_update( -+ operand, update, start_indices, axes)}; -+ }) -+ // Handle StableHLO Other ops -+ // .Case([](auto o) {}) -+ // .Case([](auto o) {}) -+ // .Case([](auto o) {}) -+ .Case( -+ [&block_ctx, &inputs](auto o) { -+ auto result_type = *utils::dtype::fromMlirType( -+ mlir::cast(o.getResult().getType()) -+ .getElementType()); -+ return std::vector{mx::view( -+ getOperandArray(o.getOperand(), block_ctx, inputs), -+ result_type)}; -+ }) -+ /* -+ .Case([](auto o) {}) -+ Deprecated see: -+ https://github.com/openxla/stablehlo/issues/2340 -+ https://github.com/openxla/stablehlo/pull/2283 -+ */ -+ .Case([&block_ctx, &inputs, -+ &op_lookup](auto o) { -+ return op_lookup.broadcast_in_dim -+ .find(getBroadcastInDimOpInfo(o)) -+ ->second( -+ {getOperandArray(o.getOperand(), block_ctx, inputs)}); -+ }) -+ // .Case([](auto o) {}) -+ .Case([&block_ctx, &inputs](auto o) { -+ return std::vector{mx::linalg::cholesky( -+ getOperandArray(o.getOperand(), block_ctx, inputs), -+ o.getLower())}; -+ }) -+ .Case([&block_ctx, &inputs](auto o) { -+ return std::vector{mx::clip( -+ getOperandArray(o.getOperand(), block_ctx, inputs), -+ getOperandArray(o.getMin(), block_ctx, inputs), -+ getOperandArray(o.getMax(), block_ctx, inputs))}; -+ }) -+ .Case([&block_ctx, &inputs](auto o) { -+ std::vector inputs; -+ for (auto val : o.getOperands()) { -+ inputs.emplace_back( -+ getOperandArray(val, block_ctx, inputs)); -+ } -+ return std::vector{ -+ mx::concatenate(inputs, o.getDimension())}; -+ }) -+ // .Case([](auto o) {}) -+ // .Case([](auto o) {}) -+ // .Case([](auto o) {}) -+ // .Case([](auto o) {}) -+ // .Case([](auto o) {}) -+ // .Case([](auto o) {}) -+ /* -+ .Case([](auto o) {}) -+ Deprecated see: -+ https://github.com/openxla/stablehlo/issues/2340 -+ https://github.com/openxla/stablehlo/pull/2283 -+ */ -+ .Case( -+ [&block_ctx, &inputs, &op_lookup](auto o) { -+ return op_lookup.dot_general.find(getDotGeneralOpInfo(o)) -+ ->second( -+ {getOperandArray(o.getLhs(), block_ctx, inputs), -+ getOperandArray(o.getRhs(), block_ctx, inputs)}); -+ }) -+ /* -+ .Case([](auto op) {}) -+ .Case([](auto op) {}) -+ Deprecated see: -+ https://github.com/openxla/stablehlo/issues/2340 -+ https://github.com/openxla/stablehlo/pull/2283 -+ */ -+ // .Case([](auto op) {}) -+ // TODO(@cryptodeal): definitely room for optimization here -+ .Case([&block_ctx, &inputs, -+ &op_lookup](auto o) { -+ return op_lookup.gather.find(getGatherOpInfo(o)) -+ ->second( -+ {getOperandArray(o.getOperand(), block_ctx, inputs), -+ getOperandArray(o.getStartIndices(), block_ctx, -+ inputs)}); -+ }) -+ .Case( -+ [&block_ctx, &inputs](auto o) { -+ return std::vector{mx::array( -+ (getOperandArray(o.getOperand(), block_ctx, inputs)) -+ .shape(o.getDimension()))}; -+ }) -+ // .Case([](auto o) {}) -+ .Case([&block_ctx, &inputs](auto o) { -+ auto result_type = -+ mlir::cast(o.getResult().getType()); -+ return std::vector{mx::reshape( -+ getOperandArray(o.getOperand(), block_ctx, inputs), -+ std::vector(result_type.getShape().begin(), -+ result_type.getShape().end()))}; -+ }) -+ // .Case([](auto o) {}) -+ .Case( -+ [&block_ctx, &inputs, &op_lookup](auto o) { -+ std::vector operands; -+ for (auto val : o.getInputs()) { -+ operands.emplace_back( -+ getOperandArray(val, block_ctx, inputs)); -+ } -+ // Add updates to input list -+ for (auto val : o.getUpdates()) { -+ operands.emplace_back( -+ getOperandArray(val, block_ctx, inputs)); -+ } -+ // Add scatter indices to input list -+ operands.emplace_back(getOperandArray( -+ o.getScatterIndices(), block_ctx, inputs)); -+ return op_lookup.scatter.find(getScatterOpInfo(o)) -+ ->second(operands); -+ }) -+ .Case([&block_ctx, &inputs](auto o) { -+ return std::vector{mx::where( -+ getOperandArray(o.getOperand(0), block_ctx, inputs), -+ getOperandArray(o.getOperand(1), block_ctx, inputs), -+ getOperandArray(o.getOperand(2), block_ctx, inputs))}; -+ }) -+ // .Case([](auto o) {}) -+ // .Case([](auto o) {}) -+ .Case([&block_ctx, &inputs](auto o) { -+ std::vector tmp_inputs; -+ for (auto val : o.getInputs()) { -+ tmp_inputs.emplace_back( -+ getOperandArray(val, block_ctx, inputs)); -+ } -+ int dim = static_cast(o.getDimension()); -+ // Get update computation -+ mlir::Block& comparator = o.getComparator().front(); -+ auto [index, comparator_type] = *getSortInfo(comparator); -+ auto indices = -+ mx::argsort(tmp_inputs[index], dim, comparator_type); -+ std::vector res; -+ mx::Shape slice_sizes(tmp_inputs[0].ndim(), 1); -+ for (auto in : tmp_inputs) { -+ res.push_back(mx::gather(in, indices, dim, slice_sizes)); -+ } -+ return res; -+ }) -+ // .Case([](auto o) {}) -+ // TODO (@cryptodeal): lots of room for optimization here -+ .Case([&block_ctx, &inputs, -+ &op_lookup](auto o) { -+ return op_lookup.pad.find(getPadOpInfo(o)) -+ ->second( -+ {getOperandArray(o.getOperand(), block_ctx, inputs), -+ getOperandArray(o.getPaddingValue(), block_ctx, -+ inputs)}); -+ }) -+ .Case([&block_ctx, &inputs](auto o) { -+ std::vector axes(o.getPermutation().begin(), -+ o.getPermutation().end()); -+ return std::vector{mx::transpose( -+ getOperandArray(o.getOperand(), block_ctx, inputs), -+ axes)}; -+ }) -+ // .Case([](auto o) {}) -+ // .Case([](auto o) {}) -+ // .Case([](auto o) {}) -+ // .Case([](auto o) {}) -+ // .Case([](auto o) {}) -+ // .Case([](auto o) {}) -+ // Need to modify such that `rng.state` is held outside of scope -+ // of `mx::compile` -+ // TODO(@cryptodeal): re-enable once we've handled ensuring -+ // state is handled correctly -+ // .Case([&block_ctx](auto o) { -+ // auto rng_func = [&o](const std::vector& inputs) -+ // { -+ // auto a = inputs[0]; -+ // auto b = inputs[1]; -+ // auto result_type = -+ // mlir::cast(o.getResult().getType()); -+ // std::vector -+ // shape(result_type.getShape().begin(), -+ // result_type.getShape().end()); -+ -+ // switch (o.getRngDistribution()) { -+ // case RngDistribution::UNIFORM: -+ // return std::vector{ -+ // mx::random::uniform(a, b, shape, a.dtype())}; -+ // default: -+ // return std::vector{ -+ // mx::random::normal(shape, a.dtype(), a, b)}; -+ // } -+ // }; -+ // return mx::compile(rng_func)( -+ // {getOperandArray(o.getA(), block_ctx), -+ // getOperandArray(o.getB(), block_ctx)}); -+ // }) -+ // .Case([&block_ctx](auto o) { -+ // auto rng_bit_gen_func = [&o](const std::vector& -+ // inputs) { -+ -+ // }; -+ // return mx::compile(rng_bit_gen_func)( -+ // {getOperandArray(o.getA(), block_ctx), -+ // getOperandArray(o.getB(), block_ctx)}); -+ // }) -+ // Handle StableHLO Quantize ops -+ // .Case([](auto op) {}) -+ // .Case([](auto op) {}) -+ // .Case([](auto op) {}) -+ /* -+ .Case([](auto o) {}) -+ Deprecated see: -+ https://github.com/openxla/stablehlo/issues/2340 -+ https://github.com/openxla/stablehlo/pull/2283 -+ */ -+ // .Case([](auto o) {}) -+ // .Case([](auto o) {}) -+ // .Case([](auto o) {}) -+ -+ .Default([&op](auto o) { -+ // Shouldn't be possible to reach this point, return -+ // empty vector. -+ return std::vector{}; -+ }); -+ block_ctx.emplace(&op, res); -+ } -+ return res; -+ }; -+ op_lookup.call.emplace(call_op_info, mx::compile(call_op_func)); -+ } -+} -+ -+void compileIotaOp(OpLookup& op_lookup, stablehlo::IotaOp o) { -+ IotaOpInfo iota_op_info = getIotaOpInfo(o); -+ auto dimensions = std::get<0>(iota_op_info.captures); -+ auto dtype = std::get<1>(iota_op_info.captures); -+ auto iota_dim = std::get<2>(iota_op_info.captures); -+ auto shape = std::get<3>(iota_op_info.captures); -+ if (auto search = op_lookup.iota.find(iota_op_info); -+ search == op_lookup.iota.end()) { -+ // std::cout << "Compiling IotaOp" << std::endl; -+ auto iota_func = [dimensions, dtype, iota_dim, -+ shape](const std::vector& inputs) { -+ auto init_arange = -+ mx::arange(static_cast(shape[iota_dim]), dtype); -+ if (dimensions.size()) { -+ init_arange = mx::reshape(init_arange, dimensions); -+ } -+ return std::vector{mx::broadcast_to(init_arange, shape)}; -+ }; -+ op_lookup.iota.emplace(iota_op_info, mx::compile(iota_func)); -+ } -+} -+ -+void compileCbrtOp(OpLookup& op_lookup, stablehlo::CbrtOp o) { -+ CbrtOpInfo cbrt_op_info = getCbrtOpInfo(o); -+ if (auto search = op_lookup.cbrt.find(cbrt_op_info); -+ search == op_lookup.cbrt.end()) { -+ // std::cout << "Compiling CbrtOp" << std::endl; -+ -+ auto cbrt_func = [](const std::vector& inputs) { -+ return std::vector{mx::power( -+ inputs[0], -+ mx::full(inputs[0].shape(), 1 / 3, inputs[0].dtype()))}; -+ }; -+ op_lookup.cbrt.emplace(cbrt_op_info, mx::compile(cbrt_func)); -+ } -+} -+ -+void compileBroadcastInDimOp(OpLookup& op_lookup, -+ stablehlo::BroadcastInDimOp o) { -+ BroadcastInDimOpInfo broadcast_in_dim_op_info = getBroadcastInDimOpInfo(o); -+ auto broadcast_dims = std::get<0>(broadcast_in_dim_op_info.captures); -+ auto target_shape = std::get<1>(broadcast_in_dim_op_info.captures); -+ if (auto search = op_lookup.broadcast_in_dim.find(broadcast_in_dim_op_info); -+ search == op_lookup.broadcast_in_dim.end()) { -+ // std::cout << "Compiling BroadcastInDimOp" << std::endl; -+ -+ auto broadcast_in_dim_func = [broadcast_dims, target_shape]( -+ const std::vector& inputs) { -+ auto operand = inputs[0]; -+ -+ // potentially reshape to pad with ones, which allows -+ // broadcasting -+ auto min_rank = std::min(operand.ndim(), target_shape.size()); -+ if (min_rank == operand.ndim()) { -+ std::vector padded_shape = operand.shape(); -+ for (auto i = 0; i < target_shape.size(); i++) { -+ if (auto search = -+ std::find(broadcast_dims.begin(), broadcast_dims.end(), i); -+ search == broadcast_dims.end()) { -+ padded_shape.insert(padded_shape.begin() + i, 1); -+ } -+ } -+ operand = mx::reshape(operand, padded_shape); -+ } -+ return std::vector{mx::broadcast_to(operand, target_shape)}; -+ }; -+ op_lookup.broadcast_in_dim.emplace(broadcast_in_dim_op_info, -+ mx::compile(broadcast_in_dim_func)); -+ } -+} -+ -+void compileDotGeneralOp(OpLookup& op_lookup, stablehlo::DotGeneralOp o) { -+ DotGeneralOpInfo dot_general_op_info = getDotGeneralOpInfo(o); -+ auto lhs_batch_dims = std::get<0>(dot_general_op_info.captures); -+ auto rhs_batch_dims = std::get<1>(dot_general_op_info.captures); -+ auto lhs_contract_dims = std::get<2>(dot_general_op_info.captures); -+ auto rhs_contract_dims = std::get<3>(dot_general_op_info.captures); -+ if (auto search = op_lookup.dot_general.find(dot_general_op_info); -+ search == op_lookup.dot_general.end()) { -+ // std::cout << "Compiling DotGeneralOp" << std::endl; -+ -+ auto dot_general_func = [lhs_batch_dims, rhs_batch_dims, lhs_contract_dims, -+ rhs_contract_dims]( -+ const std::vector& inputs) { -+ auto lhs = inputs[0]; -+ auto rhs = inputs[1]; -+ -+ int dim_count = 0; -+ auto getDimChar = [&dim_count]() -> char { return 'a' + dim_count++; }; -+ -+ std::unordered_map lhs_batch_map; -+ std::unordered_map rhs_batch_map; -+ std::unordered_map lhs_contract_map; -+ std::unordered_map rhs_contract_map; -+ -+ std::string res_subscript; -+ for (auto i = 0; i < lhs_batch_dims.size(); ++i) { -+ auto dim_char = getDimChar(); -+ res_subscript = res_subscript + dim_char; -+ lhs_batch_map.emplace(lhs_batch_dims[i], dim_char); -+ rhs_batch_map.emplace(rhs_batch_dims[i], dim_char); -+ } -+ -+ for (auto i = 0; i < lhs_contract_dims.size(); ++i) { -+ auto dim_char = getDimChar(); -+ lhs_contract_map.emplace(lhs_contract_dims[i], dim_char); -+ rhs_contract_map.emplace(rhs_contract_dims[i], dim_char); -+ } -+ -+ std::string lhs_subscript; -+ for (auto i = 0; i < lhs.ndim(); ++i) { -+ if (auto match = lhs_batch_map.find(i); match != lhs_batch_map.end()) { -+ lhs_subscript = lhs_subscript + match->second; -+ } else if (auto match = lhs_contract_map.find(i); -+ match != lhs_contract_map.end()) { -+ lhs_subscript = lhs_subscript + match->second; -+ } else { -+ auto dim_char = getDimChar(); -+ res_subscript = res_subscript + dim_char; -+ lhs_subscript = lhs_subscript + dim_char; -+ } -+ } -+ -+ std::string rhs_subscript; -+ for (auto i = 0; i < rhs.ndim(); ++i) { -+ if (auto match = rhs_batch_map.find(i); match != rhs_batch_map.end()) { -+ rhs_subscript = rhs_subscript + match->second; -+ } else if (auto match = rhs_contract_map.find(i); -+ match != rhs_contract_map.end()) { -+ rhs_subscript = rhs_subscript + match->second; -+ } else { -+ auto dim_char = getDimChar(); -+ res_subscript = res_subscript + dim_char; -+ rhs_subscript = rhs_subscript + dim_char; -+ } -+ } -+ -+ return std::vector{mx::einsum( -+ lhs_subscript + "," + rhs_subscript + "->" + res_subscript, inputs)}; -+ }; -+ op_lookup.dot_general.emplace(dot_general_op_info, -+ mx::compile(dot_general_func)); -+ } -+} -+ -+void compileGatherOp(OpLookup& op_lookup, stablehlo::GatherOp o) { -+ GatherOpInfo gather_op_info = getGatherOpInfo(o); -+ auto collapsed_slice_dims = std::get<0>(gather_op_info.captures); -+ auto index_vector_dim = std::get<1>(gather_op_info.captures); -+ auto offset_dims = std::get<2>(gather_op_info.captures); -+ auto operand_batching_dims = std::get<3>(gather_op_info.captures); -+ auto result_shape = std::get<4>(gather_op_info.captures); -+ auto slice_sizes = std::get<5>(gather_op_info.captures); -+ auto start_index_map = std::get<6>(gather_op_info.captures); -+ auto start_indices_batching_dims = std::get<7>(gather_op_info.captures); -+ -+ if (auto search = op_lookup.gather.find(gather_op_info); -+ search == op_lookup.gather.end()) { -+ // std::cout << "Compiling GatherOp" << std::endl; -+ auto gather_func = [collapsed_slice_dims, index_vector_dim, offset_dims, -+ operand_batching_dims, result_shape, slice_sizes, -+ start_index_map, start_indices_batching_dims]( -+ const std::vector& inputs) { -+ auto operand = inputs[0]; -+ auto start_indices = inputs[1]; -+ -+ // Calculate batch dims -+ std::vector batch_dims_vec; -+ for (int64_t i = 0; i < result_shape.size(); i++) { -+ if (std::find(offset_dims.begin(), offset_dims.end(), i) == -+ offset_dims.end()) { -+ batch_dims_vec.emplace_back(static_cast(i)); -+ } -+ } -+ mx::array batch_dims = mx::array( -+ batch_dims_vec.data(), {static_cast(batch_dims_vec.size())}); -+ mx::array start_idx_map = -+ mx::array(start_index_map.data(), -+ {static_cast(start_index_map.size())}); -+ mx::array op_batching_dims = -+ mx::array(operand_batching_dims.data(), -+ {static_cast(operand_batching_dims.size())}); -+ -+ // Calculate result indices -+ std::vector meshgrid_arrays; -+ for (const auto& dim : result_shape) { -+ meshgrid_arrays.push_back(mx::arange(dim)); -+ } -+ mx::array result_indices = -+ mx::reshape(mx::stack(mx::meshgrid(meshgrid_arrays, false, "ij"), -1), -+ {-1, static_cast(result_shape.size())}); -+ -+ // Initialize operand indices with zeros -+ mx::array operand_indices = mx::zeros( -+ {result_indices.shape(0), static_cast(operand.ndim())}, -+ mx::int32); -+ -+ // Slice batch indices -+ mx::array batch_indices = mx::take(result_indices, batch_dims, 1); -+ if (batch_indices.size()) { -+ // Calculate start indices -+ std::vector tmp_indices = -+ mx::split(batch_indices, batch_indices.shape(1), 1); -+ for (auto& idx : tmp_indices) { -+ idx = mx::flatten(idx); -+ } -+ -+ std::vector tmp_axes; -+ for (auto i = 0; i < start_indices.ndim(); ++i) { -+ if (i == index_vector_dim) continue; -+ tmp_axes.push_back(i); -+ } -+ std::vector tmp_slice_sizes(start_indices.ndim(), 1); -+ if (index_vector_dim < tmp_slice_sizes.size()) { -+ tmp_slice_sizes[index_vector_dim] = -+ start_indices.shape(index_vector_dim); -+ } -+ -+ mx::array batch_start_indices = mx::reshape( -+ mx::gather(start_indices, tmp_indices, tmp_axes, tmp_slice_sizes), -+ {result_indices.shape(0), -1}); -+ -+ // Seed with full start indices -+ std::vector max; -+ for (auto d_start = 0; d_start < start_index_map.size(); ++d_start) { -+ int d_operand = start_index_map[d_start]; -+ max.push_back(operand.shape(d_operand) - slice_sizes[d_operand]); -+ } -+ -+ operand_indices = mx::put_along_axis( -+ operand_indices, -+ mx::broadcast_to(start_idx_map, -+ {operand_indices.shape(0), -+ static_cast(start_index_map.size())}), -+ mx::clip( -+ mx::take(batch_start_indices, -+ mx::arange(static_cast(start_index_map.size())), -+ 1), -+ mx::zeros(batch_start_indices.shape(), mx::int32), -+ mx::array(max.data(), {static_cast(max.size())})), -+ 1); -+ } -+ -+ // Calculate full batching indices -+ mx::array i_batching = -+ mx::arange(static_cast(operand_batching_dims.size())); -+ mx::array d_operands = mx::take(op_batching_dims, i_batching); -+ mx::array d_starts = mx::take( -+ mx::array(start_indices_batching_dims.data(), -+ {static_cast(start_indices_batching_dims.size())}), -+ i_batching); -+ auto tmp_batch_start_indices = mx::subtract( -+ d_starts, mx::where(mx::less(d_starts, mx::full(d_starts.shape(), -+ index_vector_dim)), -+ mx::zeros(d_starts.shape(), mx::int32), -+ mx::ones(d_starts.shape(), mx::int32))); -+ -+ // Add batching indices -+ operand_indices = mx::scatter_add_axis( -+ operand_indices, -+ mx::broadcast_to(d_operands, -+ {operand_indices.shape(0), d_operands.shape(0)}), -+ mx::take(batch_indices, tmp_batch_start_indices, 1), 1); -+ -+ // Calculate offset indices -+ mx::array offset_indices = -+ mx::take(result_indices, -+ mx::array(offset_dims.data(), -+ {static_cast(offset_dims.size())}), -+ 1); -+ -+ std::vector result_offset_idx; -+ for (unsigned i = 0; i < operand_indices.shape(1); i++) { -+ if (std::find(operand_batching_dims.begin(), -+ operand_batching_dims.end(), -+ static_cast(i)) != operand_batching_dims.end() || -+ std::find(collapsed_slice_dims.begin(), collapsed_slice_dims.end(), -+ static_cast(i)) != collapsed_slice_dims.end()) { -+ continue; -+ } -+ result_offset_idx.push_back(i); -+ } -+ -+ // Add offset indices -+ operand_indices = mx::scatter_add_axis( -+ operand_indices, -+ mx::broadcast_to( -+ mx::array(result_offset_idx.data(), -+ {static_cast(result_offset_idx.size())}), -+ {operand_indices.shape(0), -+ static_cast(result_offset_idx.size())}), -+ offset_indices, 1); -+ -+ std::vector gather_axes(operand.ndim()); -+ std::iota(gather_axes.begin(), gather_axes.end(), 0); -+ return std::vector{mx::reshape( -+ mx::gather(operand, -+ mx::split(operand_indices, operand_indices.shape(1), 1), -+ gather_axes, std::vector(operand.ndim(), 1)), -+ result_shape)}; -+ }; -+ -+ op_lookup.gather.emplace(gather_op_info, mx::compile(gather_func)); -+ } -+} -+ -+void compileScatterOp(OpLookup& op_lookup, stablehlo::ScatterOp o) { -+ ScatterOpInfo scatter_op_info = getScatterOpInfo(o); -+ auto index_vector_dim = -+ static_cast(std::get<0>(scatter_op_info.captures)); -+ auto input_batching_dims = std::get<1>(scatter_op_info.captures); -+ auto inserted_window_dims = std::get<2>(scatter_op_info.captures); -+ auto scatter_dims_to_operand_dims = std::get<3>(scatter_op_info.captures); -+ auto scatter_indices_batching_dims = std::get<4>(scatter_op_info.captures); -+ auto scatter_type = std::get<5>(scatter_op_info.captures); -+ auto update_window_dims = std::get<6>(scatter_op_info.captures); -+ if (auto search = op_lookup.scatter.find(scatter_op_info); -+ search == op_lookup.scatter.end()) { -+ // std::cout << "Compiling ScatterOp" << std::endl; -+ auto scatter_func = [index_vector_dim, input_batching_dims, -+ inserted_window_dims, scatter_dims_to_operand_dims, -+ scatter_indices_batching_dims, scatter_type, -+ update_window_dims](const std::vector& in) { -+ auto input_count = (in.size() - 1) / 2; -+ std::vector inputs(in.begin(), in.begin() + input_count); -+ std::vector updates(in.begin() + input_count, in.end() - 1); -+ auto scatter_indices = in[in.size() - 1]; -+ // Calculate update scatter dims -+ std::vector update_scatter_dims; -+ for (auto i = 0; i < updates[0].ndim(); i++) { -+ if (std::find(update_window_dims.begin(), update_window_dims.end(), -+ static_cast(i)) == update_window_dims.end()) { -+ update_scatter_dims.emplace_back(static_cast(i)); -+ } -+ } -+ -+ // Calculate update indices -+ std::vector meshgrid_arrays; -+ for (const auto& dim : updates[0].shape()) { -+ meshgrid_arrays.push_back(mx::arange(dim)); -+ } -+ mx::array update_indices = -+ mx::reshape(mx::stack(mx::meshgrid(meshgrid_arrays, false, "ij"), -1), -+ {-1, static_cast(updates[0].ndim())}); -+ -+ // Calculate update scatter index -+ mx::array update_scatter_indices = mx::take( -+ update_indices, -+ mx::array(update_scatter_dims.data(), -+ {static_cast(update_scatter_dims.size())}), -+ 1); -+ -+ // Initialize result indices with zeros -+ auto result_indices = mx::zeros( -+ {update_indices.shape(0), static_cast(inputs[0].ndim())}, -+ mx::int32); -+ -+ // Gather start indices; populating the corresponding -+ // result indices. -+ auto gather_dim_count = index_vector_dim < scatter_indices.ndim() -+ ? update_scatter_dims.size() + 1 -+ : update_scatter_dims.size(); -+ std::vector tmp_gather_axes(update_scatter_dims.size()); -+ std::iota(tmp_gather_axes.begin(), tmp_gather_axes.end(), 0); -+ std::vector gather_slice_sizes(gather_dim_count, 1); -+ if (index_vector_dim < scatter_indices.ndim()) { -+ gather_slice_sizes[index_vector_dim] = -+ scatter_indices.shape(index_vector_dim); -+ } -+ mx::array start_indices = -+ mx::reshape(mx::gather(scatter_indices, -+ mx::split(update_scatter_indices, -+ update_scatter_indices.shape(1), 1), -+ tmp_gather_axes, gather_slice_sizes), -+ {update_indices.shape(0), -1}); -+ -+ // Seed with full start indices -+ result_indices = mx::put_along_axis( -+ result_indices, -+ mx::broadcast_to( -+ mx::array( -+ scatter_dims_to_operand_dims.data(), -+ {static_cast(scatter_dims_to_operand_dims.size())}), -+ {result_indices.shape(0), -+ static_cast(scatter_dims_to_operand_dims.size())}), -+ start_indices, 1); -+ -+ // Calculate full batching indices; accumulating to the -+ // corresponding result indices. -+ auto d_input = -+ mx::array(input_batching_dims.data(), -+ {static_cast(input_batching_dims.size())}); -+ auto d_start = mx::array( -+ scatter_indices_batching_dims.data(), -+ {static_cast(scatter_indices_batching_dims.size())}); -+ result_indices = mx::scatter_add_axis( -+ result_indices, -+ mx::broadcast_to(d_input, -+ {result_indices.shape(0), d_input.shape(0)}), -+ mx::take(update_scatter_indices, -+ mx::subtract( -+ d_start, -+ mx::where(mx::less(d_start, mx::full(d_start.shape(), -+ index_vector_dim)), -+ mx::zeros(d_start.shape(), mx::int32), -+ mx::ones(d_start.shape(), mx::int32))), -+ 1), -+ 1); -+ -+ // Calculate update window indices -+ auto update_window_indices = -+ mx::take(update_indices, -+ mx::array(update_window_dims.data(), -+ {static_cast(update_window_dims.size())}), -+ 1); -+ -+ // Calculate full window indices; accumulating to the -+ // corresponding result indices. -+ std::vector tmp_indices; -+ auto full_window_idx_count = static_cast( -+ update_window_indices.shape(1) + inserted_window_dims.size() + -+ input_batching_dims.size()); -+ for (auto i = 0; i < full_window_idx_count; ++i) { -+ if (std::find(inserted_window_dims.begin(), inserted_window_dims.end(), -+ i) != inserted_window_dims.end() || -+ std::find(input_batching_dims.begin(), input_batching_dims.end(), -+ i) != input_batching_dims.end()) { -+ continue; -+ } -+ tmp_indices.push_back(i); -+ } -+ result_indices = mx::scatter_add_axis( -+ result_indices, -+ mx::broadcast_to( -+ mx::array(tmp_indices.data(), -+ {static_cast(tmp_indices.size())}), -+ {result_indices.shape(0), -+ static_cast(tmp_indices.size())}), -+ update_window_indices, 1); -+ -+ std::vector scatter_axes(inputs[0].ndim()); -+ std::iota(scatter_axes.begin(), scatter_axes.end(), 0); -+ std::vector gather_axes(updates[0].ndim()); -+ std::iota(gather_axes.begin(), gather_axes.end(), 0); -+ std::vector res; -+ -+ auto tmp_gather_indices = -+ mx::split(update_indices, update_indices.shape(1), 1); -+ auto tmp_scatter_indices = -+ mx::split(result_indices, result_indices.shape(1), 1); -+ -+ std::vector update_shape( -+ inputs[0].ndim() + tmp_scatter_indices[0].ndim(), 1); -+ update_shape[0] = update_indices.shape(0); -+ std::vector tmp_gather_slice_sizes(updates[0].ndim(), 1); -+ for (auto i = 0; i < inputs.size(); ++i) { -+ auto update_vals = -+ mx::reshape(mx::gather(updates[i], tmp_gather_indices, gather_axes, -+ tmp_gather_slice_sizes), -+ update_shape); -+ switch (scatter_type) { -+ case utils::ScatterType::Replace: -+ res.push_back(mx::scatter(inputs[i], tmp_scatter_indices, -+ update_vals, scatter_axes)); -+ break; -+ case utils::ScatterType::Add: -+ res.push_back(mx::scatter_add(inputs[i], tmp_scatter_indices, -+ update_vals, scatter_axes)); -+ break; -+ case utils::ScatterType::Prod: -+ res.push_back(mx::scatter_prod(inputs[i], tmp_scatter_indices, -+ update_vals, scatter_axes)); -+ break; -+ case utils::ScatterType::Max: -+ res.push_back(mx::scatter_max(inputs[i], tmp_scatter_indices, -+ update_vals, scatter_axes)); -+ break; -+ case utils::ScatterType::Min: -+ res.push_back(mx::scatter_min(inputs[i], tmp_scatter_indices, -+ update_vals, scatter_axes)); -+ break; -+ } -+ } -+ return res; -+ }; -+ op_lookup.scatter.emplace(scatter_op_info, mx::compile(scatter_func)); -+ } -+} -+ -+void compilePadOp(OpLookup& op_lookup, stablehlo::PadOp o) { -+ PadOpInfo pad_op_info = getPadOpInfo(o); -+ auto result_shape = std::get<0>(pad_op_info.captures); -+ auto edge_pad_low = std::get<1>(pad_op_info.captures); -+ auto interior_pad = std::get<2>(pad_op_info.captures); -+ if (auto search = op_lookup.pad.find(pad_op_info); -+ search == op_lookup.pad.end()) { -+ // std::cout << "Compiling PadOp" << std::endl; -+ auto pad_func = [result_shape, edge_pad_low, -+ interior_pad](const std::vector& inputs) { -+ auto operand = inputs[0]; -+ auto padding_value = inputs[1]; -+ auto edge_padding_low = -+ mx::array(edge_pad_low.data(), -+ {static_cast(edge_pad_low.size())}, mx::int32); -+ auto interior_padding = -+ mx::array(interior_pad.data(), -+ {static_cast(edge_pad_low.size())}, mx::int32); -+ -+ // Calculate operand indices -+ std::vector meshgrid_arrays; -+ for (const auto& dim : operand.shape()) { -+ meshgrid_arrays.push_back(mx::arange(dim)); -+ } -+ mx::array operand_indices = -+ mx::reshape(mx::stack(mx::meshgrid(meshgrid_arrays, false, "ij"), -1), -+ {-1, static_cast(operand.ndim())}); -+ -+ mx::array result_indices = -+ edge_padding_low + -+ operand_indices * (interior_padding + mx::array(1, mx::int32)); -+ -+ // initialize array full of padding values -+ mx::array result = mx::full(result_shape, padding_value); -+ std::vector axes(operand.ndim()); -+ std::iota(axes.begin(), axes.end(), 0); -+ return std::vector{mx::scatter( -+ result, mx::split(result_indices, result_indices.shape(1), 1), -+ mx::gather(operand, -+ mx::split(operand_indices, operand_indices.shape(1), 1), -+ axes, std::vector(operand.ndim(), 1)), -+ axes)}; -+ }; -+ op_lookup.pad.emplace(pad_op_info, mx::compile(pad_func)); -+ } -+} -+ -+void compileModule(ModuleOp mod, OpLookup& op_lookup) { -+ // the top level block to compile -+ // auto main = mod.lookupSymbol("main"); -+ -+ Tree dep_tree; -+ resolveFuncDeps(mod, "main", dep_tree); -+ -+ // walk each function and pre-compile lookup table -+ // for all StableHLO op variants -+ for (auto op_name : dep_tree.getAllFnNames()) { -+ auto func = mod.lookupSymbol(op_name); -+ auto& block = func.front(); -+ for (auto& op : block.getOperations()) { -+ llvm::TypeSwitch(&op) -+ .Case( -+ [&op_lookup](auto o) { compileIotaOp(op_lookup, o); }) -+ .Case( -+ [&op_lookup](auto o) { compileCbrtOp(op_lookup, o); }) -+ .Case( -+ [&op_lookup](auto o) { compileBroadcastInDimOp(op_lookup, o); }) -+ .Case( -+ [&op_lookup](auto o) { compileDotGeneralOp(op_lookup, o); }) -+ .Case( -+ [&op_lookup](auto o) { compileGatherOp(op_lookup, o); }) -+ .Case( -+ [&op_lookup](auto o) { compileScatterOp(op_lookup, o); }) -+ .Case( -+ [&op_lookup](auto o) { compilePadOp(op_lookup, o); }) -+ .Default([](auto o) {}); -+ } -+ } -+ -+ while (true) { -+ auto compilable_fns = dep_tree.getCompilableFns(); -+ if (compilable_fns.empty()) { -+ break; -+ } -+ for (auto fn_name : compilable_fns) { -+ // std::cout << fn_name.c_str() << std::endl; -+ auto func = mod.lookupSymbol(fn_name); -+ compileCallOp(op_lookup, mod, func); -+ dep_tree.remove(fn_name); -+ } -+ } -+} -+ +class MlirLoadedExecutable : public PjRtLoadedExecutable { + public: + MlirLoadedExecutable(ModuleOp module, DeviceAssignment assignment, @@ -3373,17 +2972,15 @@ index 0000000000..02aaf61559 + context_(), + module_(cloneIntoContext(module, context_)) { + // TRACE_ME_MEMBER; ++ + auto main = module.lookupSymbol("main"); ++ // std::cout << "Compiling module: " << std::to_string(getHashValue(main)) ++ // << std::endl ++ // << std::endl; ++ // std::cout << ToString(module).c_str() << std::endl << std::endl; ++ compiled_module_ = compile::func(module_.get(), main); + -+ // std::cout << "Started call to `mlx::compile`" << std::endl << std::endl; -+ // module.dump(); -+ // std::cout << std::endl; -+ compileModule(module_.get(), op_lookup_); -+ // try { + auto& main_block = main.front(); -+ compiled_module_ = -+ op_lookup_.call.find(getCallOpInfo(module, main))->second; -+ + // force compilation by running w zeroed inputs + std::vector tmp_inputs; + for (auto arg : main_block.getArguments()) { @@ -3394,10 +2991,6 @@ index 0000000000..02aaf61559 + *utils::dtype::fromMlirType(shaped_type.getElementType()))); + } + compiled_module_(tmp_inputs); -+ // std::cout << "Finished call to `mlx::compile`" << std::endl << std::endl; -+ // } catch (std::exception& e) { -+ // std::cout << "Error in `mlx::compile`" << std::endl; -+ // } + } + + static absl::StatusOr> CompileAndLoad( @@ -3423,15 +3016,10 @@ index 0000000000..02aaf61559 + // auto device_info = mx::metal::device_info(); + // for (const auto& [key, value] : device_info) { + // std::cout << key << ": "; -+ + // std::visit([](const auto& v) { std::cout << v; }, value); -+ + // std::cout << std::endl; + // } + -+ // std::cout << ToString(module).c_str() << std::endl << std::endl << -+ // std::endl; -+ + auto executable = std::make_unique(module, assignment, + devices, client); + @@ -3466,72 +3054,22 @@ index 0000000000..02aaf61559 + return devices_[0]->default_memory_space().value_or(nullptr); + } + -+ // absl::StatusOr> evalModule( -+ // ModuleOp& module, const SmallVector& inputs) { -+ // // TRACE_ME_MEMBER; -+ // std::vector block_arguments; -+ // for (auto input : inputs) { -+ // TF_ASSIGN_OR_RETURN(auto input_arr, -+ // utils::array::fromDenseElementsAttr(input)); -+ // block_arguments.emplace_back(input_arr); -+ // } -+ -+ // // std::cout << std::endl; -+ // // module.dump(); -+ // // std::cout << std::endl; -+ // auto main = module.lookupSymbol("main"); -+ // std::unordered_map< -+ // mlir::Block*, -+ // std::tuple, -+ // std::unordered_map>>> -+ // block_ctx; -+ // TF_ASSIGN_OR_RETURN(auto mlx_res, evalBlock(module, main.front(), -+ // block_arguments, block_ctx)); -+ -+ // for (auto& a : mlx_res) { -+ // a = mx::contiguous(a); -+ // } -+ // mx::eval(mlx_res); -+ // return mlx_res; -+ // } -+ + absl::StatusOr ExecuteWithMlxInterpreter( + absl::Span argument_handles, ModuleOp module, + PjRtDevice* device, bool fill_future) { + // TRACE_ME_MEMBER; -+ // SmallVector input_attrs; -+ // for (auto* arg : argument_handles) { -+ // TF_ASSIGN_OR_RETURN(auto mlirArg, GetAttributeFromBuffer(arg)); -+ // auto mlirArgInModuleContext = -+ // CloneIntoContext(mlirArg, *module->getContext()); -+ // input_attrs.push_back(mlirArgInModuleContext); -+ // } -+ // LOG(INFO) << "EvalModule:\n" << ToString(module) << "\n"; -+ // LOG(INFO) << "Inputs: " << ToString(inputs) << "\n"; -+ // TF_ASSIGN_OR_RETURN(auto mlx_res, evalModule(module, inputs)); + + std::vector inputs; -+ // unsigned in_count = 0; + for (auto& buffer : argument_handles) { + TF_ASSIGN_OR_RETURN(mx::array input, GetArrayFromBuffer(buffer)); -+ // std::cout << "inputs[" << std::to_string(in_count++) << "]: " << input -+ // << std::endl; + inputs.push_back(input); + } + auto mlx_res = compiled_module_(inputs); -+ for (auto& a : mlx_res) { -+ a = mx::contiguous(a); ++ for (auto i = 0; i < mlx_res.size(); ++i) { ++ mlx_res[i] = mx::contiguous(mlx_res[i]); + } + mx::eval(mlx_res); + -+ // unsigned out_count = 0; -+ // for (auto& a : mlx_res) { -+ // std::cout << "outputs[" << std::to_string(out_count++) << "]: " << a -+ // << std::endl; -+ // } -+ -+ // LOG(INFO) << "Results: " << ToString(result.value()) << "\n"; -+ + // Naive memory space selection, only using CPU global memory. + PjRtMemorySpace* memory_space = + device->default_memory_space().value_or(nullptr); @@ -3677,9 +3215,8 @@ index 0000000000..02aaf61559 + DeviceAssignment assignment_; + absl::Span devices_; + PjRtClient* client_; -+ std::function(const std::vector&)> -+ compiled_module_; -+ OpLookup op_lookup_; ++ // MLX compilation context ++ MlxFunc compiled_module_; + + // MLIR + MLIRContext context_; @@ -3706,10 +3243,10 @@ index 0000000000..02aaf61559 +} // namespace mlir::stablehlo diff --git a/xla/pjrt/plugin/mlx/executable.h b/xla/pjrt/plugin/mlx/executable.h new file mode 100644 -index 0000000000..e579af4e1f +index 0000000000..d970cde59d --- /dev/null +++ b/xla/pjrt/plugin/mlx/executable.h -@@ -0,0 +1,244 @@ +@@ -0,0 +1,33 @@ +/* Copyright 2024 The OpenXLA Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. @@ -3724,193 +3261,15 @@ index 0000000000..e579af4e1f + +#ifndef XLA_PJRT_PLUGIN_STABLEHLO_MLX_EXECUTABLE_H_ +#define XLA_PJRT_PLUGIN_STABLEHLO_MLX_EXECUTABLE_H_ -+#include +#include -+#include -+#include -+#include -+#include -+#include + +#include "mlir/IR/BuiltinOps.h" -+#include "mlx/mlx.h" -+#include "xla/pjrt/plugin/mlx/utils.h" +#include "xla/hlo/builder/xla_computation.h" +#include "xla/pjrt/pjrt_client.h" +#include "xla/service/computation_placer.h" + +namespace mlir::stablehlo { + -+struct Node { -+ std::string name; -+ std::vector> children; -+ -+ Node(std::string name) : name(name) {} -+}; -+ -+class Tree { -+ public: -+ std::unique_ptr root; -+ -+ Tree() : root(nullptr) {} -+ -+ Node* insert(std::string name, Node* parent = nullptr) { -+ auto new_node = std::make_unique(name); -+ if (!root) { -+ root = std::move(new_node); -+ return root.get(); -+ } else if (parent) { -+ parent->children.push_back(std::move(new_node)); -+ return parent->children.back().get(); -+ } -+ } -+ -+ void remove(const std::string& name) { -+ if (!root) return; -+ -+ // Special case: root node matches -+ if (root->name == name) { -+ root.reset(); // deletes entire tree -+ return; -+ } -+ -+ remove_helper(root, name); -+ } -+ -+ std::set getAllFnNames() { -+ std::set fn_names; -+ if (!root) return fn_names; -+ -+ std::function traverse = [&](Node* node) { -+ fn_names.insert(node->name); -+ for (auto& child : node->children) { -+ traverse(child.get()); -+ } -+ }; -+ traverse(root.get()); -+ return fn_names; -+ } -+ -+ std::unordered_set getCompilableFns() { -+ std::unordered_set compilable_fns; -+ if (!root) return compilable_fns; -+ -+ std::function traverse = [&](Node* node) { -+ if (node->children.empty()) { -+ compilable_fns.insert(node->name); -+ } else { -+ for (auto& child : node->children) { -+ traverse(child.get()); -+ } -+ } -+ }; -+ traverse(root.get()); -+ return compilable_fns; -+ } -+ -+ private: -+ void remove_helper(std::unique_ptr& current, const std::string& name) { -+ if (!current) return; -+ -+ auto& children = current->children; -+ -+ // Remove matching children -+ children.erase(std::remove_if(children.begin(), children.end(), -+ [&](std::unique_ptr& child) { -+ return child->name == name; -+ }), -+ children.end()); -+ -+ // Recurse on remaining children -+ for (auto& child : children) { -+ remove_helper(child, name); -+ } -+ } -+}; -+ -+template -+class CompiledOpInfo { -+ public: -+ std::tuple captures; -+ std::vector> input_metadata; -+ std::vector> output_metadata; -+ -+ CompiledOpInfo(std::vector inputs, -+ std::vector> outputs, -+ T... capture_values); -+ -+ CompiledOpInfo(std::vector> inputs, -+ std::vector> outputs, -+ T... capture_values); -+ -+ bool operator==(const CompiledOpInfo& other) const { -+ return captures == other.captures && -+ input_metadata == other.input_metadata && -+ output_metadata == other.output_metadata; -+ } -+}; -+ -+template -+CompiledOpInfo::CompiledOpInfo( -+ std::vector inputs, -+ std::vector> outputs, T... capture_values) -+ : captures(std::make_tuple(capture_values...)), output_metadata(outputs) { -+ input_metadata.reserve(inputs.size()); -+ for (const auto& input : inputs) { -+ input_metadata.push_back(std::make_pair(input.dtype(), input.shape())); -+ } -+} -+ -+template -+CompiledOpInfo::CompiledOpInfo( -+ std::vector> inputs, -+ std::vector> outputs, T... capture_values) -+ : captures(std::make_tuple(capture_values...)), -+ input_metadata(inputs), -+ output_metadata(outputs) {} -+ -+using CompiledOpSign = -+ std::function(const std::vector&)>; -+ -+using CallOpInfo = CompiledOpInfo; -+ -+// stablehlo::IotaOp - Captures: (dimensions, dtype, iota_dim, result_shape) -+using IotaOpInfo = CompiledOpInfo, mx::Dtype, int64_t, -+ std::vector>; -+ -+// stablehlo::CbrtOp - Captures: () -+using CbrtOpInfo = CompiledOpInfo<>; -+ -+// stablehlo::BroadcastInDimOp - Captures: (broadcast_dims, target_shape) -+using BroadcastInDimOpInfo = -+ CompiledOpInfo, std::vector>; -+ -+// stablehlo::DotGeneralOp - Captures: (lhs_batch_dims, rhs_batch_dims, -+// lhs_contract_dims, rhs_contract_dims) -+using DotGeneralOpInfo = -+ CompiledOpInfo, std::vector, -+ std::vector, std::vector>; -+ -+// stablehlo::GatherOp - Captures: (collapsed_slice_dims, index_vector_dim, -+// offset_dims, operand_batching_dims, result_shape, slice_sizes, -+// start_index_map, start_indices_batching_dims) -+using GatherOpInfo = CompiledOpInfo, int64_t, -+ std::vector, std::vector, -+ std::vector, std::vector, -+ std::vector, std::vector>; -+ -+// stablehlo::ScatterOp - Captures: (index_vector_dim, input_batching_dims, -+// inserted_window_dims, scatter_dims_to_operand_dims, -+// scatter_indices_batching_dims, scatter_type, update_window_dims) -+using ScatterOpInfo = -+ CompiledOpInfo, std::vector, -+ std::vector, std::vector, -+ utils::ScatterType, std::vector>; -+ -+// stablehlo::PadOp - Captures: (result_shape, edge_pad_low, interior_pad) -+using PadOpInfo = CompiledOpInfo, std::vector, -+ std::vector>; -+ +absl::StatusOr> StablehloMlxCompile( + mlir::ModuleOp module, xla::DeviceAssignment assignment, + xla::PjRtClient* client); @@ -3920,39 +3279,6 @@ index 0000000000..e579af4e1f + xla::PjRtClient* client); + +} // namespace mlir::stablehlo -+ -+template -+struct std::hash> { -+ std::size_t operator()( -+ const mlir::stablehlo::CompiledOpInfo& info) const noexcept { -+ std::size_t seed = 0; -+ hash_tuple(seed, info.captures); -+ for (const auto& meta : info.input_metadata) { -+ hash_combine(seed, std::hash{}(meta.first)); -+ hash_combine(seed, std::hash>{}(meta.second)); -+ } -+ for (const auto& meta : info.output_metadata) { -+ hash_combine(seed, std::hash{}(meta.first)); -+ hash_combine(seed, std::hash>{}(meta.second)); -+ } -+ return seed; -+ } -+}; -+ -+namespace mlir::stablehlo { -+struct OpLookup { -+ std::unordered_map, CompiledOpSign> call; -+ std::unordered_map iota; -+ std::unordered_map cbrt; -+ std::unordered_map broadcast_in_dim; -+ std::unordered_map dot_general; -+ std::unordered_map gather; -+ std::unordered_map scatter; -+ std::unordered_map pad; -+}; -+ -+}; // namespace mlir::stablehlo -+ +#endif // XLA_PJRT_PLUGIN_STABLEHLO_MLX_EXECUTABLE_H_ \ No newline at end of file diff --git a/xla/pjrt/plugin/mlx/logging.cc b/xla/pjrt/plugin/mlx/logging.cc @@ -4142,12 +3468,10 @@ index 0000000000..d56a22565f \ No newline at end of file diff --git a/xla/pjrt/plugin/mlx/utils.cc b/xla/pjrt/plugin/mlx/utils.cc new file mode 100644 -index 0000000000..5fcfc318ab +index 0000000000..384c5f1a4d --- /dev/null +++ b/xla/pjrt/plugin/mlx/utils.cc -@@ -0,0 +1,292 @@ -+#include -+#include +@@ -0,0 +1,632 @@ +#include +#include +#include @@ -4155,23 +3479,15 @@ index 0000000000..5fcfc318ab +#include +#include + -+#include "xla/pjrt/plugin/mlx/utils.h" -+#include "absl/status/status.h" -+#include "absl/status/statusor.h" -+#include "absl/types/span.h" -+#include "llvm/ADT/TypeSwitch.h" -+#include "mlir/IR/BuiltinAttributes.h" -+#include "mlir/IR/Types.h" -+#include "mlx/mlx.h" -+#include "tsl/platform/statusor.h" ++#include "llvm/ADT/Hashing.h" ++#include "mlir/IR/OperationSupport.h" ++#include "mlir/Support/LLVM.h" +#include "xla/mlir/utils/type_util.h" ++#include "xla/pjrt/plugin/mlx/utils.h" ++#include "xla/pjrt/plugin/mlx/logging.h" +#include "xla/shape_util.h" -+ -+#include "xla/literal.h" +#include "xla/util.h" + -+namespace mx = mlx::core; -+ +std::tuple shapeAndBytes( + absl::Span dims, + std::optional> byte_strides, mx::Dtype dtype) { @@ -4276,11 +3592,6 @@ index 0000000000..5fcfc318ab + } + return fromXlaPrimitiveType(primitive_type); +} -+ -+int mlirTypeByteWidth(mlir::Type type) { -+ return xla::primitive_util::ByteWidth( -+ xla::ConvertMlirTypeToPrimitiveType(type)); -+} +} // namespace dtype + +namespace array { @@ -4291,17 +3602,16 @@ index 0000000000..5fcfc318ab + TF_ASSIGN_OR_RETURN(mx::Dtype dtype, dtype::fromXlaPrimitiveType(type)); + auto [shape, raw_buffer_shape, strides] = + shapeAndBytes(dims, byte_strides, dtype); -+ mx::array typed_view = mx::view(mx::array(reinterpret_cast(data), -+ raw_buffer_shape, mx::uint8), -+ dtype); ++ mx::array typed_view = ++ mx::view(mx::array(reinterpret_cast(data), ++ raw_buffer_shape, mx::uint8), ++ dtype); + if (byte_strides.has_value()) { + auto res = mx::contiguous(mx::as_strided(typed_view, shape, strides, 0)); + res.eval(); + return res; + } else { -+ auto res = mx::reshape(typed_view, shape); -+ res.eval(); -+ return res; ++ return mx::reshape(typed_view, shape); + } +} + @@ -4408,17 +3718,41 @@ index 0000000000..5fcfc318ab + } +} + -+absl::StatusOr fromOperand( -+ mlir::Value operand, absl::Span block_args, ++mx::array fromOperand( ++ const mlir::Value& operand, + const std::unordered_map>& -+ transient_buffers, -+ const std::unordered_map& init_values) {} -+ -+absl::StatusOr fromOperand( -+ mlir::Value operand, absl::Span block_args, -+ const std::unordered_map>& -+ transient_buffers) { -+ return fromOperand(operand, block_args, transient_buffers, {}); ++ block_ctx, ++ const std::vector& inputs, bool debug) { ++ if (auto defining_op = operand.getDefiningOp(); defining_op) { ++ if (auto search = block_ctx.find(defining_op); search != block_ctx.end()) { ++ for (auto i = 0; i < defining_op->getNumResults(); i++) { ++ mlir::Value maybe_res = defining_op->getResult(i); ++ if (maybe_res == operand) { ++ if (debug) ++ std::cout << "Operator from block context @ idx:" ++ << std::to_string(i) << std::endl ++ << mlir::stablehlo::ToString(defining_op).c_str() ++ << std::endl ++ << std::endl; ++ return search->second[i]; ++ } ++ } ++ if (debug) ++ std::cout << "Operator from block context not found:" << std::endl ++ << mlir::stablehlo::ToString(defining_op).c_str() << std::endl ++ << std::endl; ++ } else { ++ std::cout << "Defining op not found in block context: " ++ << mlir::stablehlo::ToString(defining_op).c_str() << std::endl; ++ return inputs[llvm::cast(operand).getArgNumber()]; ++ } ++ } ++ auto block_arg = llvm::cast(operand); ++ if (debug) ++ std::cout << "Operator from block argument @ idx:" ++ << std::to_string(block_arg.getArgNumber()) << std::endl ++ << std::endl; ++ return inputs[block_arg.getArgNumber()]; +} + +} // namespace array @@ -4437,13 +3771,345 @@ index 0000000000..5fcfc318ab + } + std::cout << " }" << std::endl; +} ++ ++absl::StatusOr getScatterType(mlir::Block& block) { ++ unsigned op_count = 0; ++ utils::ScatterType scatter_type = utils::ScatterType::Replace; ++ for (const auto& op : block.getOperations()) { ++ switch (op_count) { ++ case 0: ++ if (llvm::dyn_cast(op)) { ++ return scatter_type; ++ } ++ if (llvm::dyn_cast(op)) { ++ scatter_type = ScatterType::Add; ++ } else if (llvm::dyn_cast(op)) { ++ scatter_type = ScatterType::Prod; ++ } else if (llvm::dyn_cast(op)) { ++ scatter_type = ScatterType::Prod; ++ } else if (llvm::dyn_cast(op)) { ++ scatter_type = ScatterType::Max; ++ } else if (llvm::dyn_cast(op)) { ++ scatter_type = ScatterType::Min; ++ } else { ++ return xla::Internal("Unsupported comparator: %s", ++ mlir::stablehlo::ToString(block)); ++ } ++ break; ++ case 1: ++ if (llvm::dyn_cast(op)) { ++ return scatter_type; ++ } ++ default: ++ return xla::Internal("Unsupported comparator: %s", ++ mlir::stablehlo::ToString(block)); ++ } ++ } ++} ++ ++absl::StatusOr> getSortInfo( ++ mlir::Block& block) { ++ unsigned op_count = 0; ++ mx::ComparatorType comparator = mx::ComparatorType::LessThan; ++ unsigned input_idx = 0; ++ for (auto& op : block.getOperations()) { ++ switch (op_count) { ++ case 0: { ++ if (auto compare_op = llvm::dyn_cast(op)) { ++ switch (compare_op.getComparisonDirection()) { ++ case mlir::stablehlo::ComparisonDirection::GT: { ++ auto lhs = ++ llvm::dyn_cast(compare_op.getLhs()); ++ auto rhs = ++ llvm::dyn_cast(compare_op.getRhs()); ++ if (lhs && rhs) { ++ unsigned lhs_idx = static_cast(lhs.getArgNumber()); ++ unsigned rhs_idx = static_cast(rhs.getArgNumber()); ++ if (rhs_idx - 1 == lhs_idx) { ++ comparator = mx::ComparatorType::GreaterThan; ++ input_idx = lhs_idx == 0 ? 0 : lhs_idx / 2; ++ break; ++ } ++ } ++ return xla::Internal("Unsupported comparator: %s", ++ mlir::stablehlo::ToString(block)); ++ } ++ case mlir::stablehlo::ComparisonDirection::LT: { ++ auto lhs = ++ llvm::dyn_cast(compare_op.getLhs()); ++ auto rhs = ++ llvm::dyn_cast(compare_op.getRhs()); ++ if (lhs && rhs) { ++ unsigned lhs_idx = static_cast(lhs.getArgNumber()); ++ unsigned rhs_idx = static_cast(rhs.getArgNumber()); ++ if (rhs_idx - 1 == lhs_idx) { ++ comparator = mx::ComparatorType::LessThan; ++ input_idx = lhs_idx == 0 ? 0 : lhs_idx / 2; ++ break; ++ } ++ } ++ return xla::Internal("Unsupported comparator: %s", ++ mlir::stablehlo::ToString(block)); ++ } ++ default: ++ return xla::Internal("Unsupported comparator: %s", ++ mlir::stablehlo::ToString(block)); ++ } ++ break; ++ } ++ } ++ case 1: ++ if (llvm::dyn_cast(op)) break; ++ default: ++ return xla::Internal("Unsupported comparator: %s", ++ mlir::stablehlo::ToString(block)); ++ } ++ op_count++; ++ } ++ return std::make_pair(input_idx, comparator); ++} ++ ++// Returns true if the given `ReduceOp` matches ++// `zml.Tensor.argMax` op implementation. ++bool isArgMaxReduce(mlir::stablehlo::ReduceOp reduce_op) { ++ mlir::Block& body_block = reduce_op.getBody().front(); ++ unsigned op_count = 0; ++ // TODO (@cryptodeal): This is a lazy verification; maybe check operands ++ // for each operation in the block. ++ for (auto& op : body_block.getOperations()) { ++ switch (op_count) { ++ case 0: { ++ auto compare_op = llvm::dyn_cast(op); ++ if (compare_op && compare_op.getComparisonDirection() == ++ mlir::stablehlo::ComparisonDirection::GT) { ++ break; ++ } else ++ return false; ++ } ++ case 1: { ++ auto compare_op = llvm::dyn_cast(op); ++ if (compare_op && compare_op.getComparisonDirection() == ++ mlir::stablehlo::ComparisonDirection::NE) { ++ break; ++ } else ++ return false; ++ } ++ case 2: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ case 3: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ case 4: { ++ auto compare_op = llvm::dyn_cast(op); ++ if (compare_op && compare_op.getComparisonDirection() == ++ mlir::stablehlo::ComparisonDirection::EQ) { ++ break; ++ } else ++ return false; ++ } ++ case 5: { ++ auto compare_op = llvm::dyn_cast(op); ++ if (compare_op && compare_op.getComparisonDirection() == ++ mlir::stablehlo::ComparisonDirection::LT) { ++ break; ++ } else ++ return false; ++ } ++ case 6: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ case 7: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ case 8: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ case 9: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ default: ++ return false; ++ } ++ op_count++; ++ } ++ return true; ++} ++ ++// Returns true if the given `ReduceOp` matches ++// `zml.Tensor.sum` op implementation. ++bool isSumReduce(mlir::stablehlo::ReduceOp reduce_op) { ++ mlir::Block& body_block = reduce_op.getBody().front(); ++ unsigned op_count = 0; ++ // TODO (@cryptodeal): This is a lazy verification; maybe check operands ++ // for each operation in the block. ++ for (auto& op : body_block.getOperations()) { ++ switch (op_count) { ++ case 0: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ case 1: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ } ++ op_count++; ++ } ++ return true; ++} ++ ++// Returns true if the given `ReduceOp` matches ++// `zml.Tensor.max` op implementation. ++bool isMaxReduce(mlir::stablehlo::ReduceOp reduce_op) { ++ mlir::Block& body_block = reduce_op.getBody().front(); ++ unsigned op_count = 0; ++ // TODO (@cryptodeal): This is a lazy verification; maybe check operands ++ // for each operation in the block. ++ for (auto& op : body_block.getOperations()) { ++ switch (op_count) { ++ case 0: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ case 1: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ } ++ op_count++; ++ } ++ return true; ++} ++ ++// Returns true if the given `ReduceOp` matches ++// `zml.Tensor.max` op implementation. ++bool isMinReduce(mlir::stablehlo::ReduceOp reduce_op) { ++ mlir::Block& body_block = reduce_op.getBody().front(); ++ unsigned op_count = 0; ++ // TODO (@cryptodeal): This is a lazy verification; maybe check operands ++ // for each operation in the block. ++ for (auto& op : body_block.getOperations()) { ++ switch (op_count) { ++ case 0: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ case 1: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ } ++ op_count++; ++ } ++ return true; ++} ++ ++// Returns true if the given `ReduceOp` matches ++// `zml.Tensor.any` op implementation. ++bool isAnyReduce(mlir::stablehlo::ReduceOp reduce_op) { ++ mlir::Block& body_block = reduce_op.getBody().front(); ++ unsigned op_count = 0; ++ // TODO (@cryptodeal): This is a lazy verification; maybe check operands ++ // for each operation in the block. ++ for (auto& op : body_block.getOperations()) { ++ switch (op_count) { ++ case 0: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ case 1: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ } ++ op_count++; ++ } ++ return true; ++} ++ ++llvm::hash_code computeHash( ++ mlir::Operation* op, ++ llvm::function_ref hashOperands, ++ llvm::function_ref hashResults) { ++ // Hash operations based upon their: ++ // - Operation Name ++ // - Attributes ++ // - Properties ++ ++ // Potentially need further specialization for: ++ // - stablehlo::ScatterOp (requires determining the scatter type) ++ ++ llvm::hash_code hash = llvm::hash_combine( ++ op->getName(), op->getRawDictionaryAttrs(), op->hashProperties()); ++ ++ // - Operands ++ if (op->hasTrait() && ++ op->getNumOperands() > 0) { ++ size_t operandHash = hashOperands(op->getOperand(0)); ++ for (auto operand : op->getOperands().drop_front()) ++ operandHash += hashOperands(operand); ++ hash = llvm::hash_combine(hash, operandHash); ++ } else { ++ for (mlir::Value operand : op->getOperands()) ++ hash = llvm::hash_combine(hash, hashOperands(operand)); ++ } ++ ++ // - Results ++ for (mlir::Value result : op->getResults()) ++ hash = llvm::hash_combine(hash, hashResults(result)); ++ ++ return hash; ++} ++ ++uint64_t getHashValue(mlir::func::FuncOp func_op) { ++ llvm::hash_code hash = {}; ++ for (mlir::NamedAttribute attr : func_op->getAttrs()) { ++ if (attr.getName() == func_op.getSymNameAttrName()) { ++ continue; ++ } ++ hash = llvm::hash_combine(hash, attr); ++ } ++ func_op.getBody().walk([&](mlir::Operation* op) { ++ hash = llvm::hash_combine( ++ hash, computeHash(op, mlir::OperationEquivalence::ignoreHashValue, ++ mlir::OperationEquivalence::ignoreHashValue)); ++ }); ++ return hash; ++} ++ ++uint64_t getHashValue(mlir::Operation* op) { ++ return computeHash(op, mlir::OperationEquivalence::ignoreHashValue, ++ mlir::OperationEquivalence::ignoreHashValue); ++} ++ +} // namespace utils diff --git a/xla/pjrt/plugin/mlx/utils.h b/xla/pjrt/plugin/mlx/utils.h new file mode 100644 -index 0000000000..1a6d85c060 +index 0000000000..f19257b5a2 --- /dev/null +++ b/xla/pjrt/plugin/mlx/utils.h -@@ -0,0 +1,129 @@ +@@ -0,0 +1,94 @@ +/* Copyright 2024 The OpenXLA Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. @@ -4461,23 +4127,36 @@ index 0000000000..1a6d85c060 + +#include +#include -+#include +#include +#include + -+#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/types/span.h" -+#include "mlir/IR/BuiltinAttributes.h" ++#include "mlir/Dialect/Func/IR/FuncOps.h" ++#include "stablehlo/reference/Api.h" ++#include "mlir/IR/OwningOpRef.h" +#include "mlir/IR/Types.h" ++#include "mlir/IR/Operation.h" ++#include "mlir/IR/Value.h" +#include "mlx/mlx.h" +#include "xla/literal.h" -+#include "xla/util.h" + +namespace mx = mlx::core; + ++using MlxFunc = ++ std::function(const std::vector&)>; ++using OwnedMlxFunc = std::shared_ptr; ++ +namespace utils { + ++struct FuncInfo { ++ mlir::OwningOpRef mod_; ++ std::string name_; ++ ++ FuncInfo(mlir::ModuleOp mod, std::string name) ++ : mod_(mod.clone()), name_(name) {} ++}; ++ +enum ScatterType { + Replace, + Add, @@ -4492,7 +4171,6 @@ index 0000000000..1a6d85c060 + +absl::StatusOr fromMlirType(mlir::Type type); + -+int mlirTypeByteWidth(mlir::Type type); +} // namespace dtype + +namespace array { @@ -4504,73 +4182,26 @@ index 0000000000..1a6d85c060 +absl::StatusOr fromHostLiteral(const xla::LiteralSlice& literal); + +absl::StatusOr fromDenseElementsAttr(mlir::DenseElementsAttr attr); -+ -+absl::StatusOr fromOperand( -+ mlir::Value operand, absl::Span block_args, ++mx::array fromOperand( ++ const mlir::Value& operand, + const std::unordered_map>& -+ transient_buffers, -+ const std::unordered_map& init_values); -+ -+absl::StatusOr fromOperand( -+ mlir::Value operand, absl::Span block_args, -+ const std::unordered_map>& -+ transient_buffers); ++ block_ctx, ++ const std::vector& inputs, bool debug = false); +} // namespace array + +void printVector(const std::string& name, const std::vector& vec, + bool indent = false); ++absl::StatusOr getScatterType(mlir::Block& block); ++absl::StatusOr> getSortInfo( ++ mlir::Block& block); ++bool isArgMaxReduce(mlir::stablehlo::ReduceOp reduce_op); ++bool isSumReduce(mlir::stablehlo::ReduceOp reduce_op); ++bool isMaxReduce(mlir::stablehlo::ReduceOp reduce_op); ++bool isMinReduce(mlir::stablehlo::ReduceOp reduce_op); ++bool isAnyReduce(mlir::stablehlo::ReduceOp reduce_op); ++uint64_t getHashValue(mlir::func::FuncOp func_op); ++uint64_t getHashValue(mlir::Operation* op); +} // namespace utils + -+// Helper function to hash a tuple -+template -+inline void hash_combine(std::size_t& seed, const T& val) { -+ seed ^= std::hash{}(val) + 0x9e3779b9 + (seed << 6) + (seed >> 2); -+} -+ -+template -+inline -+ typename std::enable_if::value, void>::type -+ hash_tuple(std::size_t&, const Tuple&) {} -+ -+template -+ inline typename std::enable_if < -+ Index::value, void>::type hash_tuple( -+ std::size_t& seed, const Tuple& tuple) { -+ hash_combine(seed, std::get(tuple)); -+ hash_tuple(seed, tuple); -+} -+ -+template <> -+struct std::hash { -+ std::size_t operator()(const utils::ScatterType& value) const noexcept { -+ return std::hash::type>{}( -+ static_cast::type>(value)); -+ } -+}; -+ -+template -+struct std::hash> { -+ std::size_t operator()(const std::vector& vec) const { -+ std::size_t seed = 0; -+ for (const T& elem : vec) { -+ seed ^= std::hash{}(elem) + 0x9e3779b9 + (seed << 6) + (seed >> 2); -+ } -+ return seed; -+ } -+}; -+ -+template <> -+struct std::hash { -+ std::size_t operator()(const mx::Dtype& dtype) const noexcept { -+ std::size_t seed = 0; -+ hash_combine(seed, std::hash{}(dtype.size())); -+ hash_combine(seed, -+ std::hash::type>{}( -+ static_cast::type>( -+ dtype.val()))); -+ return seed; -+ } -+}; -+ +#endif // XLA_PJRT_PLUGIN_STABLEHLO_MLX_UTILS_H_ \ No newline at end of file From 53934139c90b0b9af6b4579dc4d8f683bdcd9026 Mon Sep 17 00:00:00 2001 From: James Deal Date: Wed, 23 Jul 2025 15:14:28 -0500 Subject: [PATCH 12/13] Working Llama on MLX Plugin --- MODULE.bazel.lock | 15 ++- examples/MODULE.bazel.lock | 10 +- .../20250605.0-c763fe1/overlay/BUILD.bazel | 2 +- .../patches/0004-Added-MLX-PjRt-Plugin.patch | 98 ++++++++++--------- 4 files changed, 68 insertions(+), 57 deletions(-) diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 98406392..0b70e72e 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -7,7 +7,6 @@ "https://bazel-registry.zml.ai/modules/abseil-cpp/20230802.0.bcr.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/abseil-cpp/20230802.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/abseil-cpp/20230802.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/abseil-cpp/20240116.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/abseil-cpp/20240116.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/abseil-cpp/20240116.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/abseil-cpp/20240722.0.bcr.2/MODULE.bazel": "not found", @@ -51,7 +50,7 @@ "https://bazel-registry.zml.ai/modules/buildifier_prebuilt/7.3.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/buildifier_prebuilt/8.0.3/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/buildozer/7.1.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/fmt/11.2.0/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/fmt/11.0.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/google_benchmark/1.8.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/googletest/1.11.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/googletest/1.14.0.bcr.1/MODULE.bazel": "not found", @@ -84,11 +83,11 @@ "https://bazel-registry.zml.ai/modules/protobuf/3.19.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/pybind11_bazel/2.11.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/pybind11_bazel/2.12.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/range-v3/0.12.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/re2/2023-09-01/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/re2/2024-07-02.bcr.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/re2/2024-07-02/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_android/0.1.1/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/rules_apple/3.13.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_apple/3.22.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.0.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.0.10/MODULE.bazel": "not found", @@ -161,6 +160,7 @@ "https://bazel-registry.zml.ai/modules/rules_shell/0.3.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_shell/0.4.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_swift/1.16.0/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/rules_swift/2.1.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_swift/2.4.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_uv/0.65.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.5.1/MODULE.bazel": "not found", @@ -186,7 +186,6 @@ "https://bcr.bazel.build/modules/abseil-cpp/20230802.0.bcr.1/MODULE.bazel": "1c8cec495288dccd14fdae6e3f95f772c1c91857047a098fad772034264cc8cb", "https://bcr.bazel.build/modules/abseil-cpp/20230802.0/MODULE.bazel": "d253ae36a8bd9ee3c5955384096ccb6baf16a1b1e93e858370da0a3b94f77c16", "https://bcr.bazel.build/modules/abseil-cpp/20230802.1/MODULE.bazel": "fa92e2eb41a04df73cdabeec37107316f7e5272650f81d6cc096418fe647b915", - "https://bcr.bazel.build/modules/abseil-cpp/20240116.0/MODULE.bazel": "98dc378d64c12a4e4741ad3362f87fb737ee6a0886b2d90c3cdbb4d93ea3e0bf", "https://bcr.bazel.build/modules/abseil-cpp/20240116.1/MODULE.bazel": "37bcdb4440fbb61df6a1c296ae01b327f19e9bb521f9b8e26ec854b6f97309ed", "https://bcr.bazel.build/modules/abseil-cpp/20240116.2/MODULE.bazel": "73939767a4686cd9a520d16af5ab440071ed75cec1a876bf2fcfaf1f71987a16", "https://bcr.bazel.build/modules/abseil-cpp/20240722.0.bcr.2/MODULE.bazel": "c3661b44c9d3f17f0b65ffb544896aaeb89127398ea867537babac18133a002a", @@ -238,8 +237,8 @@ "https://bcr.bazel.build/modules/buildifier_prebuilt/8.0.3/source.json": "68ee16210562f325ff961aabeb4d46ed10e6638b5e842cd48d8d83dea220dfbd", "https://bcr.bazel.build/modules/buildozer/7.1.2/MODULE.bazel": "2e8dd40ede9c454042645fd8d8d0cd1527966aa5c919de86661e62953cd73d84", "https://bcr.bazel.build/modules/buildozer/7.1.2/source.json": "c9028a501d2db85793a6996205c8de120944f50a0d570438fcae0457a5f9d1f8", - "https://bcr.bazel.build/modules/fmt/11.2.0/MODULE.bazel": "bb2cef47f22163e23aa39ff7812cfe8cec85adb6337493fe38d1fa039133ca5c", - "https://bcr.bazel.build/modules/fmt/11.2.0/source.json": "a2c3846c868dbb75e3793e14c7bb9afbb0e34d3967c9a22b335df71eb2ff1481", + "https://bcr.bazel.build/modules/fmt/11.0.2/MODULE.bazel": "d9e0ea6f68a762a84055172ce8f55cc01da950c94b5e27f66c1acc14e6fc59ce", + "https://bcr.bazel.build/modules/fmt/11.0.2/source.json": "36cde5a3cbbee5eefc1e7f8a5a66d8ac9f352f7742bdd0c9b2d869601169c881", "https://bcr.bazel.build/modules/google_benchmark/1.8.2/MODULE.bazel": "a70cf1bba851000ba93b58ae2f6d76490a9feb74192e57ab8e8ff13c34ec50cb", "https://bcr.bazel.build/modules/googletest/1.11.0/MODULE.bazel": "3a83f095183f66345ca86aa13c58b59f9f94a2f81999c093d4eeaa2d262d12f4", "https://bcr.bazel.build/modules/googletest/1.14.0.bcr.1/MODULE.bazel": "22c31a561553727960057361aa33bf20fb2e98584bc4fec007906e27053f80c6", @@ -281,14 +280,13 @@ "https://bcr.bazel.build/modules/pybind11_bazel/2.11.1/MODULE.bazel": "88af1c246226d87e65be78ed49ecd1e6f5e98648558c14ce99176da041dc378e", "https://bcr.bazel.build/modules/pybind11_bazel/2.12.0/MODULE.bazel": "e6f4c20442eaa7c90d7190d8dc539d0ab422f95c65a57cc59562170c58ae3d34", "https://bcr.bazel.build/modules/pybind11_bazel/2.12.0/source.json": "6900fdc8a9e95866b8c0d4ad4aba4d4236317b5c1cd04c502df3f0d33afed680", - "https://bcr.bazel.build/modules/range-v3/0.12.0/MODULE.bazel": "46ee6c1fcb121df6e6345b3898313fef3a173d74c87db2e6196276200daadcba", - "https://bcr.bazel.build/modules/range-v3/0.12.0/source.json": "09f0087034f1b93202d9d43b8b772a709329e45eab9c2ba92ec60525b19c28d2", "https://bcr.bazel.build/modules/re2/2023-09-01/MODULE.bazel": "cb3d511531b16cfc78a225a9e2136007a48cf8a677e4264baeab57fe78a80206", "https://bcr.bazel.build/modules/re2/2024-07-02.bcr.1/MODULE.bazel": "b4963dda9b31080be1905ef085ecd7dd6cd47c05c79b9cdf83ade83ab2ab271a", "https://bcr.bazel.build/modules/re2/2024-07-02.bcr.1/source.json": "2ff292be6ef3340325ce8a045ecc326e92cbfab47c7cbab4bd85d28971b97ac4", "https://bcr.bazel.build/modules/re2/2024-07-02/MODULE.bazel": "0eadc4395959969297cbcf31a249ff457f2f1d456228c67719480205aa306daa", "https://bcr.bazel.build/modules/rules_android/0.1.1/MODULE.bazel": "48809ab0091b07ad0182defb787c4c5328bd3a278938415c00a7b69b50c4d3a8", "https://bcr.bazel.build/modules/rules_android/0.1.1/source.json": "e6986b41626ee10bdc864937ffb6d6bf275bb5b9c65120e6137d56e6331f089e", + "https://bcr.bazel.build/modules/rules_apple/3.13.0/MODULE.bazel": "b4559a2c6281ca3165275bb36c1f0ac74666632adc5bdb680e366de7ce845f43", "https://bcr.bazel.build/modules/rules_apple/3.22.0/MODULE.bazel": "0fd8c630d6cecc18abe84114ee21c1a0fb759b45063d3d2600ec22669e93ea43", "https://bcr.bazel.build/modules/rules_apple/3.22.0/source.json": "ee41f63c292bd876fd264a830cf45500619a063e2fa3a18cca6fa12b153d68e3", "https://bcr.bazel.build/modules/rules_cc/0.0.1/MODULE.bazel": "cb2aa0747f84c6c3a78dad4e2049c154f08ab9d166b1273835a8174940365647", @@ -375,6 +373,7 @@ "https://bcr.bazel.build/modules/rules_shell/0.4.1/MODULE.bazel": "00e501db01bbf4e3e1dd1595959092c2fadf2087b2852d3f553b5370f5633592", "https://bcr.bazel.build/modules/rules_shell/0.4.1/source.json": "4757bd277fe1567763991c4425b483477bb82e35e777a56fd846eb5cceda324a", "https://bcr.bazel.build/modules/rules_swift/1.16.0/MODULE.bazel": "4a09f199545a60d09895e8281362b1ff3bb08bbde69c6fc87aff5b92fcc916ca", + "https://bcr.bazel.build/modules/rules_swift/2.1.1/MODULE.bazel": "494900a80f944fc7aa61500c2073d9729dff0b764f0e89b824eb746959bc1046", "https://bcr.bazel.build/modules/rules_swift/2.4.0/MODULE.bazel": "1639617eb1ede28d774d967a738b4a68b0accb40650beadb57c21846beab5efd", "https://bcr.bazel.build/modules/rules_swift/2.4.0/source.json": "a6577f57f9febbdc015a01f2a8f3487422032f134d6c61d18ed8e8ca3b9acc7c", "https://bcr.bazel.build/modules/rules_uv/0.65.0/MODULE.bazel": "a700505a80ad0ad65f062c9335458d137f504b98c5937b338807bb5cbe305e73", diff --git a/examples/MODULE.bazel.lock b/examples/MODULE.bazel.lock index 7b0a6eb4..3a6f4e9d 100644 --- a/examples/MODULE.bazel.lock +++ b/examples/MODULE.bazel.lock @@ -50,7 +50,7 @@ "https://bazel-registry.zml.ai/modules/buildifier_prebuilt/6.4.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/buildifier_prebuilt/7.3.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/buildozer/7.1.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/fmt/11.2.0/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/fmt/11.0.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/google_benchmark/1.8.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/googletest/1.11.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/googletest/1.14.0.bcr.1/MODULE.bazel": "not found", @@ -87,6 +87,7 @@ "https://bazel-registry.zml.ai/modules/re2/2024-07-02.bcr.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/re2/2024-07-02/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_android/0.1.1/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/rules_apple/3.13.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_apple/3.22.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.0.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.0.10/MODULE.bazel": "not found", @@ -160,6 +161,7 @@ "https://bazel-registry.zml.ai/modules/rules_shell/0.3.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_shell/0.4.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_swift/1.16.0/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/rules_swift/2.1.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_swift/2.4.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_uv/0.65.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.5.1/MODULE.bazel": "not found", @@ -237,8 +239,8 @@ "https://bcr.bazel.build/modules/buildifier_prebuilt/7.3.1/source.json": "55153a5e6ca9c8a7e266c4b46b951e8a010d25ec6062bc35d5d4f89925796bad", "https://bcr.bazel.build/modules/buildozer/7.1.2/MODULE.bazel": "2e8dd40ede9c454042645fd8d8d0cd1527966aa5c919de86661e62953cd73d84", "https://bcr.bazel.build/modules/buildozer/7.1.2/source.json": "c9028a501d2db85793a6996205c8de120944f50a0d570438fcae0457a5f9d1f8", - "https://bcr.bazel.build/modules/fmt/11.2.0/MODULE.bazel": "bb2cef47f22163e23aa39ff7812cfe8cec85adb6337493fe38d1fa039133ca5c", - "https://bcr.bazel.build/modules/fmt/11.2.0/source.json": "a2c3846c868dbb75e3793e14c7bb9afbb0e34d3967c9a22b335df71eb2ff1481", + "https://bcr.bazel.build/modules/fmt/11.0.2/MODULE.bazel": "d9e0ea6f68a762a84055172ce8f55cc01da950c94b5e27f66c1acc14e6fc59ce", + "https://bcr.bazel.build/modules/fmt/11.0.2/source.json": "36cde5a3cbbee5eefc1e7f8a5a66d8ac9f352f7742bdd0c9b2d869601169c881", "https://bcr.bazel.build/modules/google_benchmark/1.8.2/MODULE.bazel": "a70cf1bba851000ba93b58ae2f6d76490a9feb74192e57ab8e8ff13c34ec50cb", "https://bcr.bazel.build/modules/googletest/1.11.0/MODULE.bazel": "3a83f095183f66345ca86aa13c58b59f9f94a2f81999c093d4eeaa2d262d12f4", "https://bcr.bazel.build/modules/googletest/1.14.0.bcr.1/MODULE.bazel": "22c31a561553727960057361aa33bf20fb2e98584bc4fec007906e27053f80c6", @@ -286,6 +288,7 @@ "https://bcr.bazel.build/modules/re2/2024-07-02/MODULE.bazel": "0eadc4395959969297cbcf31a249ff457f2f1d456228c67719480205aa306daa", "https://bcr.bazel.build/modules/rules_android/0.1.1/MODULE.bazel": "48809ab0091b07ad0182defb787c4c5328bd3a278938415c00a7b69b50c4d3a8", "https://bcr.bazel.build/modules/rules_android/0.1.1/source.json": "e6986b41626ee10bdc864937ffb6d6bf275bb5b9c65120e6137d56e6331f089e", + "https://bcr.bazel.build/modules/rules_apple/3.13.0/MODULE.bazel": "b4559a2c6281ca3165275bb36c1f0ac74666632adc5bdb680e366de7ce845f43", "https://bcr.bazel.build/modules/rules_apple/3.22.0/MODULE.bazel": "0fd8c630d6cecc18abe84114ee21c1a0fb759b45063d3d2600ec22669e93ea43", "https://bcr.bazel.build/modules/rules_apple/3.22.0/source.json": "ee41f63c292bd876fd264a830cf45500619a063e2fa3a18cca6fa12b153d68e3", "https://bcr.bazel.build/modules/rules_cc/0.0.1/MODULE.bazel": "cb2aa0747f84c6c3a78dad4e2049c154f08ab9d166b1273835a8174940365647", @@ -374,6 +377,7 @@ "https://bcr.bazel.build/modules/rules_shell/0.4.1/MODULE.bazel": "00e501db01bbf4e3e1dd1595959092c2fadf2087b2852d3f553b5370f5633592", "https://bcr.bazel.build/modules/rules_shell/0.4.1/source.json": "4757bd277fe1567763991c4425b483477bb82e35e777a56fd846eb5cceda324a", "https://bcr.bazel.build/modules/rules_swift/1.16.0/MODULE.bazel": "4a09f199545a60d09895e8281362b1ff3bb08bbde69c6fc87aff5b92fcc916ca", + "https://bcr.bazel.build/modules/rules_swift/2.1.1/MODULE.bazel": "494900a80f944fc7aa61500c2073d9729dff0b764f0e89b824eb746959bc1046", "https://bcr.bazel.build/modules/rules_swift/2.4.0/MODULE.bazel": "1639617eb1ede28d774d967a738b4a68b0accb40650beadb57c21846beab5efd", "https://bcr.bazel.build/modules/rules_swift/2.4.0/source.json": "a6577f57f9febbdc015a01f2a8f3487422032f134d6c61d18ed8e8ca3b9acc7c", "https://bcr.bazel.build/modules/rules_uv/0.65.0/MODULE.bazel": "a700505a80ad0ad65f062c9335458d137f504b98c5937b338807bb5cbe305e73", diff --git a/third_party/modules/mlx/20250605.0-c763fe1/overlay/BUILD.bazel b/third_party/modules/mlx/20250605.0-c763fe1/overlay/BUILD.bazel index c53b4afa..6286dd3a 100644 --- a/third_party/modules/mlx/20250605.0-c763fe1/overlay/BUILD.bazel +++ b/third_party/modules/mlx/20250605.0-c763fe1/overlay/BUILD.bazel @@ -408,7 +408,7 @@ apple_metal_library( visibility = ["//visibility:public"], copts = [ "-target", - "air64-apple-macosx15.4.1", + "air64-apple-macosx15.5", "-Iexternal/mlx+", "-Iexternal/mlx+/mlx/backend/metal/kernels/metal_3_1", # "-I.", diff --git a/third_party/modules/xla/20250527.0-cb67f2f/patches/0004-Added-MLX-PjRt-Plugin.patch b/third_party/modules/xla/20250527.0-cb67f2f/patches/0004-Added-MLX-PjRt-Plugin.patch index 6eeb2ca1..466f2f64 100644 --- a/third_party/modules/xla/20250527.0-cb67f2f/patches/0004-Added-MLX-PjRt-Plugin.patch +++ b/third_party/modules/xla/20250527.0-cb67f2f/patches/0004-Added-MLX-PjRt-Plugin.patch @@ -1126,10 +1126,10 @@ index 0000000000..8c90942f0c \ No newline at end of file diff --git a/xla/pjrt/plugin/mlx/compile.cc b/xla/pjrt/plugin/mlx/compile.cc new file mode 100644 -index 0000000000..889ad0fffe +index 0000000000..70d9a620e1 --- /dev/null +++ b/xla/pjrt/plugin/mlx/compile.cc -@@ -0,0 +1,1297 @@ +@@ -0,0 +1,1305 @@ +#include "xla/pjrt/plugin/mlx/compile.h" + +#include @@ -1199,9 +1199,9 @@ index 0000000000..889ad0fffe + bool recursive_compile) { + auto& block = op.front(); + std::unordered_map> block_ctx; -+ std::vector res; ++ mlir::Operation* last_op = nullptr; + for (mlir::Operation& op : block.getOperations()) { -+ res = ++ std::vector res = + llvm::TypeSwitch>(&op) + .Case( + [&block_ctx, &inputs](auto o) { @@ -1885,9 +1885,10 @@ index 0000000000..889ad0fffe + // empty vector. + return std::vector{}; + }); -+ block_ctx.emplace(&op, res); ++ block_ctx.emplace(&op, std::move(res)); ++ last_op = &op; + } -+ return res; ++ return block_ctx.find(last_op)->second; +} +} // namespace detail + @@ -2056,49 +2057,56 @@ index 0000000000..889ad0fffe + // Slice batch indices + mx::array batch_indices = mx::take(result_indices, batch_dims, 1); + ++ // Calculate start indices ++ std::vector tmp_indices; + if (batch_indices.size()) { -+ // Calculate start indices -+ std::vector tmp_indices = -+ mx::split(batch_indices, batch_indices.shape(1), 1); -+ for (auto& idx : tmp_indices) { -+ idx = mx::flatten(idx); -+ } ++ tmp_indices = mx::split(batch_indices, batch_indices.shape(1), 1); ++ } ++ for (auto& idx : tmp_indices) { ++ idx = mx::flatten(idx); ++ } + -+ std::vector tmp_axes; -+ for (auto i = 0; i < start_indices.ndim(); ++i) { -+ if (i == index_vector_dim) continue; -+ tmp_axes.push_back(i); -+ } -+ std::vector tmp_slice_sizes(start_indices.ndim(), 1); -+ if (index_vector_dim < tmp_slice_sizes.size()) { -+ tmp_slice_sizes[index_vector_dim] = -+ start_indices.shape(index_vector_dim); -+ } ++ std::vector tmp_axes; ++ for (auto i = 0; i < start_indices.ndim(); ++i) { ++ if (i == index_vector_dim) continue; ++ tmp_axes.push_back(i); ++ } ++ std::vector tmp_slice_sizes(start_indices.ndim(), 1); ++ if (index_vector_dim < tmp_slice_sizes.size()) { ++ tmp_slice_sizes[index_vector_dim] = start_indices.shape(index_vector_dim); ++ } + -+ mx::array batch_start_indices = mx::reshape( ++ mx::array batch_start_indices = ++ mx::gather(start_indices, tmp_indices, tmp_axes, tmp_slice_sizes); ++ if (tmp_axes.size()) { ++ batch_start_indices = mx::reshape( + mx::gather(start_indices, tmp_indices, tmp_axes, tmp_slice_sizes), + {result_indices.shape(0), -1}); -+ -+ // Seed with full start indices -+ std::vector max; -+ for (auto d_start = 0; d_start < start_index_map.size(); ++d_start) { -+ int d_operand = start_index_map[d_start]; -+ max.push_back(operand.shape(d_operand) - slice_sizes[d_operand]); -+ } -+ -+ operand_indices = mx::put_along_axis( -+ operand_indices, -+ mx::broadcast_to(start_idx_map, -+ {operand_indices.shape(0), -+ static_cast(start_index_map.size())}), -+ mx::clip( -+ mx::take(batch_start_indices, -+ mx::arange(static_cast(start_index_map.size())), 1), -+ mx::zeros(batch_start_indices.shape(), mx::int32), -+ mx::array(max.data(), {static_cast(max.size())})), -+ 1); ++ } else if (batch_start_indices.ndim() == 1) { ++ batch_start_indices = mx::broadcast_to( ++ mx::reshape(batch_start_indices, {1, batch_start_indices.shape(0)}), ++ {result_indices.shape(0), batch_start_indices.shape(0)}); + } + ++ // Seed with full start indices ++ std::vector max; ++ for (auto d_start = 0; d_start < start_index_map.size(); ++d_start) { ++ int d_operand = start_index_map[d_start]; ++ max.push_back(operand.shape(d_operand) - slice_sizes[d_operand]); ++ } ++ ++ operand_indices = mx::put_along_axis( ++ operand_indices, ++ mx::broadcast_to(start_idx_map, ++ {operand_indices.shape(0), ++ static_cast(start_index_map.size())}), ++ mx::clip( ++ mx::take(batch_start_indices, ++ mx::arange(static_cast(start_index_map.size())), 1), ++ mx::zeros(batch_start_indices.shape(), mx::int32), ++ mx::array(max.data(), {static_cast(max.size())})), ++ 1); ++ + // Calculate full batching indices + mx::array i_batching = + mx::arange(static_cast(operand_batching_dims.size())); @@ -2730,7 +2738,7 @@ index 0000000000..ef42968a3a \ No newline at end of file diff --git a/xla/pjrt/plugin/mlx/executable.cc b/xla/pjrt/plugin/mlx/executable.cc new file mode 100644 -index 0000000000..dfdfb33f87 +index 0000000000..ec612eca89 --- /dev/null +++ b/xla/pjrt/plugin/mlx/executable.cc @@ -0,0 +1,507 @@ @@ -2977,7 +2985,7 @@ index 0000000000..dfdfb33f87 + // std::cout << "Compiling module: " << std::to_string(getHashValue(main)) + // << std::endl + // << std::endl; -+ // std::cout << ToString(module).c_str() << std::endl << std::endl; ++ // std::cout << ToString(module_.get()).c_str() << std::endl << std::endl; + compiled_module_ = compile::func(module_.get(), main); + + auto& main_block = main.front(); @@ -3062,7 +3070,7 @@ index 0000000000..dfdfb33f87 + std::vector inputs; + for (auto& buffer : argument_handles) { + TF_ASSIGN_OR_RETURN(mx::array input, GetArrayFromBuffer(buffer)); -+ inputs.push_back(input); ++ inputs.push_back(std::move(input)); + } + auto mlx_res = compiled_module_(inputs); + for (auto i = 0; i < mlx_res.size(); ++i) { From 55c4a9a6a73a2a9a1be6e2d2d712baa3aac04911 Mon Sep 17 00:00:00 2001 From: James Deal Date: Fri, 29 Aug 2025 00:23:11 -0500 Subject: [PATCH 13/13] Executable level compiled Operation/function caching --- .../20250605.0-c763fe1/overlay/BUILD.bazel | 1 + ...1-Modify-Sort-And-Comparison-Kernels.patch | 158 ++- .../patches/0004-Added-MLX-PjRt-Plugin.patch | 1190 ++++++++++------- 3 files changed, 886 insertions(+), 463 deletions(-) diff --git a/third_party/modules/mlx/20250605.0-c763fe1/overlay/BUILD.bazel b/third_party/modules/mlx/20250605.0-c763fe1/overlay/BUILD.bazel index 6286dd3a..62cd72c5 100644 --- a/third_party/modules/mlx/20250605.0-c763fe1/overlay/BUILD.bazel +++ b/third_party/modules/mlx/20250605.0-c763fe1/overlay/BUILD.bazel @@ -583,6 +583,7 @@ cc_library( ], hdrs = [ "mlx/mlx.h", + "mlx/compile_impl.h", ], deps = [ ":io", diff --git a/third_party/modules/mlx/20250605.0-c763fe1/patches/0001-Modify-Sort-And-Comparison-Kernels.patch b/third_party/modules/mlx/20250605.0-c763fe1/patches/0001-Modify-Sort-And-Comparison-Kernels.patch index f7c27b51..e0c2644e 100644 --- a/third_party/modules/mlx/20250605.0-c763fe1/patches/0001-Modify-Sort-And-Comparison-Kernels.patch +++ b/third_party/modules/mlx/20250605.0-c763fe1/patches/0001-Modify-Sort-And-Comparison-Kernels.patch @@ -59,6 +59,24 @@ index 000000000..28ed3894b + ], + deps = ["@mlx//:mlx"] +) +diff --git a/mlx/array.h b/mlx/array.h +index 98eef2e33..54ee53402 100644 +--- a/mlx/array.h ++++ b/mlx/array.h +@@ -517,9 +517,10 @@ template + array::array( + std::initializer_list data, + Dtype dtype /* = TypeToDtype() */) +- : array_desc_(std::make_shared( +- Shape{static_cast(data.size())}, +- dtype)) { ++ : array_desc_( ++ std::make_shared( ++ Shape{static_cast(data.size())}, ++ dtype)) { + init(data.begin()); + } + diff --git a/mlx/backend/cpu/make_compiled_preamble.sh b/mlx/backend/cpu/make_compiled_preamble.sh index 88b4c4615..ca567d83e 100644 --- a/mlx/backend/cpu/make_compiled_preamble.sh @@ -846,6 +864,114 @@ index 409aa2c89..6d49f1c60 100644 NO_GPU(Real) NO_GPU(Reduce) NO_GPU(Reshape) +diff --git a/mlx/compile.cpp b/mlx/compile.cpp +index 79a55ba8f..ae073facc 100644 +--- a/mlx/compile.cpp ++++ b/mlx/compile.cpp +@@ -194,6 +194,7 @@ std::vector Compiled::output_shapes(const std::vector& inputs) { + } + + namespace detail { ++std::mutex compilation_mutex; + + CompileMode& compile_mode() { + auto get_val = []() { +@@ -244,17 +245,6 @@ void merge(array& dst, array& src, ParentsMap& parents_map) { + merge_one(dests[i], sources[i], parents_map); + } + } +- +-template +-std::uintptr_t get_function_address(const std::function& fun) { +- using FunType = T (*)(U...); +- const FunType* fun_ptr = fun.template target(); +- if (fun_ptr == nullptr) { +- return 0; +- } +- return reinterpret_cast(*fun_ptr); +-} +- + class CompilerCache { + public: + struct CacheEntry { +diff --git a/mlx/compile.h b/mlx/compile.h +index a076cfbca..ecd7b27ff 100644 +--- a/mlx/compile.h ++++ b/mlx/compile.h +@@ -8,6 +8,9 @@ namespace mlx::core { + + enum class CompileMode { disabled, no_simplify, no_fuse, enabled }; + ++template ++std::uintptr_t get_function_address(const std::function& fun); ++ + /** Compile takes a function and returns a compiled function. */ + std::function(const std::vector&)> compile( + std::function(const std::vector&)> fun, +diff --git a/mlx/compile_impl.h b/mlx/compile_impl.h +index bc3f44b4e..d601cb181 100644 +--- a/mlx/compile_impl.h ++++ b/mlx/compile_impl.h +@@ -1,12 +1,14 @@ + // Copyright © 2023-2024 Apple Inc. + + #pragma once +- ++#include ++#include + #include + + #include "mlx/array.h" + + namespace mlx::core::detail { ++extern std::mutex compilation_mutex; + + // This is not part of the general C++ API as calling with a bad id is a bad + // idea. +@@ -19,6 +21,19 @@ std::function(const std::vector&)> compile( + // Erase cached compile functions + void compile_erase(std::uintptr_t fun_id); + ++template ++std::uintptr_t get_function_address(const std::function& fun) { ++ using FunType = T (*)(U...); ++ const FunType* fun_ptr = fun.template target(); ++ if (fun_ptr == nullptr) { ++ return 0; ++ } ++ return reinterpret_cast(*fun_ptr); ++} ++ ++// Returns true if `skip_compile` is set ++bool skip_compile(); ++ + // Clear the compiler cache causing a recompilation of all compiled functions + // when called again. + void compile_clear_cache(); +diff --git a/mlx/dtype_utils.h b/mlx/dtype_utils.h +index 55de890f2..9208285b2 100644 +--- a/mlx/dtype_utils.h ++++ b/mlx/dtype_utils.h +@@ -92,12 +92,13 @@ MLX_FORALL_DTYPES(SPECIALIZE_CppTypeToDtype) + break; \ + } + +-#define MLX_INTERNAL_SWITCH_CHECKED(TYPE, NAME, ...) \ +- switch (TYPE) { \ +- __VA_ARGS__ \ +- default: \ +- throw std::invalid_argument(fmt::format( \ +- "Unhandled dtype %s for %s", dtype_to_string(TYPE), NAME)); \ ++#define MLX_INTERNAL_SWITCH_CHECKED(TYPE, NAME, ...) \ ++ switch (TYPE) { \ ++ __VA_ARGS__ \ ++ default: \ ++ throw std::invalid_argument( \ ++ fmt::format( \ ++ "Unhandled dtype %s for %s", dtype_to_string(TYPE), NAME)); \ + } + + #define MLX_INTERNAL_SWITCH_CASE_INT_TYPES(CTYPE_ALIAS, ...) \ diff --git a/mlx/mlx.h b/mlx/mlx.h index cef8d806d..f8932873c 100644 --- a/mlx/mlx.h @@ -1247,10 +1373,18 @@ index cc60bcfb9..2b79c0d67 100644 class Split : public Primitive { diff --git a/mlx/random.cpp b/mlx/random.cpp -index 6c6d1eb95..5a4745a52 100644 +index 6c6d1eb95..15b1cff9e 100644 --- a/mlx/random.cpp +++ b/mlx/random.cpp -@@ -225,6 +225,23 @@ array normal( +@@ -14,6 +14,7 @@ namespace mlx::core::random { + KeySequence::KeySequence(uint64_t seed) : key_(key(seed)) {} + + void KeySequence::seed(uint64_t seed) { ++ std::lock_guard lock(mutex_); + key_ = key((seed)); + } + +@@ -225,6 +226,23 @@ array normal( return samples; } @@ -1275,10 +1409,26 @@ index 6c6d1eb95..5a4745a52 100644 const array& mean, const array& cov, diff --git a/mlx/random.h b/mlx/random.h -index 0dfdab7a1..ebee726c8 100644 +index 0dfdab7a1..9358dd900 100644 --- a/mlx/random.h +++ b/mlx/random.h -@@ -134,6 +134,14 @@ inline array normal( +@@ -3,6 +3,7 @@ + #pragma once + + #include ++#include + #include + + #include "mlx/array.h" +@@ -26,6 +27,7 @@ class KeySequence { + + private: + array key_; ++ std::mutex mutex_; + static uint64_t get_current_time_seed() { + auto now = std::chrono::system_clock::now(); + return std::chrono::duration_cast( +@@ -134,6 +136,14 @@ inline array normal( return normal(shape, float32, std::nullopt, std::nullopt, key, s); } diff --git a/third_party/modules/xla/20250527.0-cb67f2f/patches/0004-Added-MLX-PjRt-Plugin.patch b/third_party/modules/xla/20250527.0-cb67f2f/patches/0004-Added-MLX-PjRt-Plugin.patch index 466f2f64..ea86579f 100644 --- a/third_party/modules/xla/20250527.0-cb67f2f/patches/0004-Added-MLX-PjRt-Plugin.patch +++ b/third_party/modules/xla/20250527.0-cb67f2f/patches/0004-Added-MLX-PjRt-Plugin.patch @@ -783,10 +783,10 @@ index 0000000000..cc56626953 \ No newline at end of file diff --git a/xla/pjrt/plugin/mlx/client_cpp_pjrt.cc b/xla/pjrt/plugin/mlx/client_cpp_pjrt.cc new file mode 100644 -index 0000000000..4c64c20b7b +index 0000000000..ca5caa92ea --- /dev/null +++ b/xla/pjrt/plugin/mlx/client_cpp_pjrt.cc -@@ -0,0 +1,299 @@ +@@ -0,0 +1,303 @@ +/* Copyright 2024 The OpenXLA Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. @@ -802,7 +802,6 @@ index 0000000000..4c64c20b7b +#include "xla/pjrt/plugin/mlx/client_cpp_pjrt.h" + +#include -+#include +#include +#include +#include @@ -819,6 +818,7 @@ index 0000000000..4c64c20b7b +#include "xla/pjrt/pjrt_common.h" +#include "xla/pjrt/pjrt_compiler.h" +#include "xla/pjrt/plugin/mlx/buffer.h" ++#include "xla/pjrt/plugin/mlx/compile.h" +#include "xla/pjrt/plugin/mlx/device.h" +#include "xla/pjrt/plugin/mlx/executable.h" +#include "xla/pjrt/plugin/mlx/logging.h" @@ -1077,6 +1077,10 @@ index 0000000000..4c64c20b7b + std::vector devices_; + std::unique_ptr owned_memory_space_; + xla::PjRtMemorySpace* memory_space_; ++ // TODO(@cryptodeal): shared cache for optimized ops ++ // with the same hash value between executables. ++ // std::mutex cache_mutex_; ++ // std::unordered_map mlx_cache_; +}; // end class + +std::unique_ptr CreateStablehloMlxPjrtClient() { @@ -1126,10 +1130,10 @@ index 0000000000..8c90942f0c \ No newline at end of file diff --git a/xla/pjrt/plugin/mlx/compile.cc b/xla/pjrt/plugin/mlx/compile.cc new file mode 100644 -index 0000000000..70d9a620e1 +index 0000000000..eec6b037b2 --- /dev/null +++ b/xla/pjrt/plugin/mlx/compile.cc -@@ -0,0 +1,1305 @@ +@@ -0,0 +1,1563 @@ +#include "xla/pjrt/plugin/mlx/compile.h" + +#include @@ -1138,9 +1142,6 @@ index 0000000000..70d9a620e1 +#include +#include +#include -+#include -+#include -+#include +#include +#include +#include @@ -1149,10 +1150,8 @@ index 0000000000..70d9a620e1 +// TODO(@cryptodeal): might need to update `BUILD` +#include "mlir/IR/Visitors.h" + -+#include "mlx/mlx.h" +#include "mlx/compile_impl.h" +#include "absl/log/log.h" -+#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "llvm/ADT/TypeSwitch.h" +#include "mlir/Bytecode/BytecodeWriter.h" @@ -1178,25 +1177,47 @@ index 0000000000..70d9a620e1 +#include "xla/pjrt/pjrt_future.h" +#include "xla/pjrt/plugin/mlx/buffer.h" +#include "xla/pjrt/plugin/mlx/logging.h" -+#include "xla/pjrt/plugin/mlx/utils.h" +#include "xla/service/computation_placer.h" +#include "tsl/platform/status.h" +#include "tsl/platform/statusor.h" + -+namespace mx = mlx::core; -+ +namespace compile { + +namespace detail { -+// cbrt implementation -+std::vector cbrt(const std::vector& inputs) { -+ return std::vector{mx::power( -+ inputs[0], mx::full(inputs[0].shape(), 1 / 3, inputs[0].dtype()))}; ++std::pair compile( ++ std::function(const std::vector&)> fun, ++ bool shapeless /* false */) { ++ if (mx::detail::skip_compile()) { ++ return {0, fun}; ++ } ++ auto fun_id = mx::detail::get_function_address(fun); ++ if (fun_id) { ++ // If the function has an addressable target then no need to manage it's ++ // lifetime ++ return {fun_id, mx::detail::compile(std::move(fun), fun_id, shapeless)}; ++ } else { ++ auto pfun = std::shared_ptr< ++ std::function(const std::vector&)>>( ++ new std::function( ++ const std::vector&)>{fun}, ++ [](auto* p) { ++ mx::detail::compile_erase(reinterpret_cast(p)); ++ delete p; ++ }); ++ fun_id = reinterpret_cast(pfun.get()); ++ return {fun_id, mx::detail::compile( ++ [pfun = std::move(pfun)](const auto& inputs) { ++ return (*pfun)(inputs); ++ }, ++ fun_id, shapeless)}; ++ } +} + -+std::vector func(mlir::ModuleOp mod, mlir::func::FuncOp op, -+ const std::vector& inputs, -+ bool recursive_compile) { ++std::vector func( ++ mlir::ModuleOp& mod, std::string func_name, ++ std::unordered_map& mlx_cache, ++ const std::vector& inputs) { ++ auto op = mod.lookupSymbol(func_name); + auto& block = op.front(); + std::unordered_map> block_ctx; + mlir::Operation* last_op = nullptr; @@ -1214,7 +1235,7 @@ index 0000000000..70d9a620e1 + return res; + }) + .Case( -+ [&block_ctx, &inputs, &mod, recursive_compile](auto o) { ++ [&block_ctx, &inputs, &mlx_cache, &mod](auto o) { + std::vector operands; + for (mlir::Value val : o.getOperands()) { + operands.emplace_back(utils::array::fromOperand( @@ -1222,14 +1243,7 @@ index 0000000000..70d9a620e1 + } + auto func_op = + mod.lookupSymbol(o.getCallee()); -+ // std::cout << "Calling function: " << -+ // func_op.getName().str() -+ // << std::endl; -+ if (recursive_compile) { -+ return compile::func(mod, func_op)(operands); -+ } else { -+ return func(mod, func_op, operands, false); -+ } ++ return mlx_cache[utils::getHashValue(func_op)].call(operands); + }) + // Handle StableHLO nullary ops + .Case([](auto o) { @@ -1268,8 +1282,9 @@ index 0000000000..70d9a620e1 + o.getOperand(), block_ctx, inputs))}; + }) + .Case([&block_ctx, &inputs](auto o) { -+ return mx::compile(cbrt)({utils::array::fromOperand( -+ o.getOperand(), block_ctx, inputs)}); ++ return std::vector{ ++ mx::power(inputs[0], mx::full(inputs[0].shape(), 1 / 3, ++ inputs[0].dtype()))}; + }) + .Case([&block_ctx, &inputs](auto o) { + return std::vector{mx::ceil(utils::array::fromOperand( @@ -1535,45 +1550,40 @@ index 0000000000..70d9a620e1 + // .Case([](auto o) {}) + // .Case([](auto o) {}) + // .Case([](auto o) {}) -+ .Case([&block_ctx, &inputs](auto o) { ++ .Case([&block_ctx, &inputs, ++ &op](auto o) { + auto operand = + utils::array::fromOperand(o.getOperand(0), block_ctx, inputs); -+ if (utils::isArgMaxReduce(o)) { -+ // std::cout << "ArgMaxReduce op: " << std::endl; -+ auto axis = static_cast(o.getDimensions()[0]); -+ auto indices = mx::argmax(operand, axis); ++ mlir::Block& body_block = o.getBody().front(); + -+ mx::Dtype result_type = *utils::dtype::fromMlirType( -+ mlir::cast(o.getResults()[1].getType()) -+ .getElementType()); -+ return std::vector{mx::take(operand, indices, axis), -+ mx::astype(indices, result_type)}; -+ } -+ if (utils::isSumReduce(o)) { -+ // std::cout << "SumReduce op: " << std::endl; -+ return std::vector{ -+ mx::sum(operand, static_cast(o.getDimensions()[0]))}; -+ } -+ if (utils::isMaxReduce(o)) { -+ // std::cout << "MaxReduce op: " << std::endl; -+ return std::vector{ -+ mx::max(operand, static_cast(o.getDimensions()[0]))}; -+ } -+ if (utils::isMinReduce(o)) { -+ // std::cout << "MinReduce op: " << std::endl; -+ return std::vector{ -+ mx::min(operand, static_cast(o.getDimensions()[0]))}; -+ } -+ // TODO(@cryptodeal): further cleanup module validation -+ // done via call to `Compile` to check for valid reduce types. ++ switch (*utils::getReduceType(op, body_block)) { ++ case utils::ReduceType::ArgMax: { ++ auto axis = static_cast(o.getDimensions()[0]); ++ auto indices = mx::argmax(operand, axis); + -+ // For now, we assume `mx::any` if none of the above ++ mx::Dtype result_type = *utils::dtype::fromMlirType( ++ mlir::cast(o.getResults()[1].getType()) ++ .getElementType()); ++ return std::vector{ ++ mx::take(operand, indices, axis), ++ mx::astype(indices, result_type)}; ++ } ++ case utils::ReduceType::Sum: ++ return std::vector{ ++ mx::sum(operand, static_cast(o.getDimensions()[0]))}; + -+ // if (isAnyReduce(o)) { -+ // std::cout << "Unhandled reduce op: " << std::endl; -+ return std::vector{ -+ mx::any(operand, static_cast(o.getDimensions()[0]))}; -+ //} ++ case utils::ReduceType::Max: ++ return std::vector{ ++ mx::max(operand, static_cast(o.getDimensions()[0]))}; ++ case utils::ReduceType::Min: ++ return std::vector{ ++ mx::min(operand, static_cast(o.getDimensions()[0]))}; ++ case utils::ReduceType::Any: ++ return std::vector{ ++ mx::any(operand, static_cast(o.getDimensions()[0]))}; ++ default: ++ __builtin_unreachable(); ++ } + }) + // Handle StableHLO tuple ops + // .Case([](auto o) {}) @@ -1727,8 +1737,9 @@ index 0000000000..70d9a620e1 + https://github.com/openxla/stablehlo/issues/2340 + https://github.com/openxla/stablehlo/pull/2283 + */ -+ .Case([&block_ctx, &inputs](auto o) { -+ return compile::dotGeneral(o)( ++ .Case([&block_ctx, &inputs, ++ &mlx_cache, &op](auto o) { ++ return mlx_cache[utils::getHashValue(&op)].call( + {utils::array::fromOperand(o.getLhs(), block_ctx, inputs), + utils::array::fromOperand(o.getRhs(), block_ctx, inputs)}); + }) @@ -1741,8 +1752,9 @@ index 0000000000..70d9a620e1 + */ + // .Case([](auto op) {}) + // TODO(@cryptodeal): definitely room for optimization here -+ .Case([&block_ctx, &inputs](auto o) { -+ return compile::gather(o)( ++ .Case([&block_ctx, &inputs, &mlx_cache, ++ &op](auto o) { ++ return mlx_cache[utils::getHashValue(&op)].call( + {utils::array::fromOperand(o.getOperand(), block_ctx, inputs), + utils::array::fromOperand(o.getStartIndices(), block_ctx, + inputs)}); @@ -1763,22 +1775,23 @@ index 0000000000..70d9a620e1 + result_type.getShape().end()))}; + }) + // .Case([](auto o) {}) -+ .Case([&block_ctx, &inputs](auto o) { -+ std::vector operands; -+ for (auto val : o.getInputs()) { -+ operands.emplace_back( -+ utils::array::fromOperand(val, block_ctx, inputs)); -+ } -+ // Add updates to input list -+ for (auto val : o.getUpdates()) { -+ operands.emplace_back( -+ utils::array::fromOperand(val, block_ctx, inputs)); -+ } -+ // Add scatter indices to input list -+ operands.emplace_back(utils::array::fromOperand( -+ o.getScatterIndices(), block_ctx, inputs)); -+ return compile::scatter(o)(operands); -+ }) ++ .Case( ++ [&block_ctx, &inputs, &mlx_cache, &op](auto o) { ++ std::vector operands; ++ for (auto val : o.getInputs()) { ++ operands.emplace_back( ++ utils::array::fromOperand(val, block_ctx, inputs)); ++ } ++ // Add updates to input list ++ for (auto val : o.getUpdates()) { ++ operands.emplace_back( ++ utils::array::fromOperand(val, block_ctx, inputs)); ++ } ++ // Add scatter indices to input list ++ operands.emplace_back(utils::array::fromOperand( ++ o.getScatterIndices(), block_ctx, inputs)); ++ return mlx_cache[utils::getHashValue(&op)].call(operands); ++ }) + .Case([&block_ctx, &inputs](auto o) { + return std::vector{mx::where( + utils::array::fromOperand(o.getOperand(0), block_ctx, inputs), @@ -1810,8 +1823,9 @@ index 0000000000..70d9a620e1 + }) + // .Case([](auto o) {}) + // TODO (@cryptodeal): lots of room for optimization here -+ .Case([&block_ctx, &inputs](auto o) { -+ return compile::pad(o)( ++ .Case([&block_ctx, &inputs, &mlx_cache, ++ &op](auto o) { ++ return mlx_cache[utils::getHashValue(&op)].call( + {utils::array::fromOperand(o.getOperand(), block_ctx, inputs), + utils::array::fromOperand(o.getPaddingValue(), block_ctx, + inputs)}); @@ -1890,9 +1904,38 @@ index 0000000000..70d9a620e1 + } + return block_ctx.find(last_op)->second; +} -+} // namespace detail + -+MlxFunc dotGeneral(mlir::stablehlo::DotGeneralOp op) { ++bool compileFunc(mlir::ModuleOp& mod, std::string func_name, ++ std::unordered_map& mlx_cache) { ++ auto func = mod.lookupSymbol(func_name); ++ auto& cache_entry = mlx_cache[utils::getHashValue(func)]; ++ if (cache_entry.isInitialized()) { ++ return true; ++ } else { ++ bool all_compiled = true; ++ auto& block = func.front(); ++ auto call_ops = block.getOps(); ++ for (mlir::func::CallOp call_op : call_ops) { ++ auto callee = mod.lookupSymbol(call_op.getCallee()); ++ if (!compileFunc(mod, callee.getName().str(), mlx_cache)) { ++ all_compiled = false; ++ } ++ } ++ if (all_compiled) { ++ MlxFunc fun = [&mod, &mlx_cache, ++ func_name](const std::vector& inputs) { ++ return detail::func(mod, func_name, mlx_cache, inputs); ++ }; ++ auto compiled_fun = detail::compile(std::move(fun)); ++ cache_entry.initialize(std::move(compiled_fun.second), ++ compiled_fun.first); ++ } ++ return all_compiled; ++ } ++} ++ ++std::pair dotGeneral( ++ mlir::stablehlo::DotGeneralOp op) { + auto dot_dim_nums = op.getDotDimensionNumbers(); + std::vector lhs_batch_dims( + dot_dim_nums.getLhsBatchingDimensions().begin(), @@ -1968,10 +2011,10 @@ index 0000000000..70d9a620e1 + return std::vector{mx::einsum( + lhs_subscript + "," + rhs_subscript + "->" + res_subscript, inputs)}; + }; -+ return mx::compile(std::move(fun)); ++ return detail::compile(std::move(fun)); +} + -+MlxFunc gather(mlir::stablehlo::GatherOp op) { ++std::pair gather(mlir::stablehlo::GatherOp op) { + auto dim_nums = op.getDimensionNumbers(); + std::vector offset_dims(dim_nums.getOffsetDims().begin(), + dim_nums.getOffsetDims().end()); @@ -2164,10 +2207,10 @@ index 0000000000..70d9a620e1 + gather_axes, std::vector(operand.ndim(), 1)), + result_shape)}; + }; -+ return mx::compile(fun); ++ return detail::compile(fun); +} + -+MlxFunc pad(mlir::stablehlo::PadOp op) { ++std::pair pad(mlir::stablehlo::PadOp op) { + std::vector edge_pad_low(op.getEdgePaddingLow().begin(), + op.getEdgePaddingLow().end()); + std::vector edge_pad_high(op.getEdgePaddingHigh().begin(), @@ -2220,10 +2263,11 @@ index 0000000000..70d9a620e1 + axes, std::vector(operand.ndim(), 1)), + axes)}; + }; -+ return mx::compile(fun); ++ return detail::compile(std::move(fun)); +} + -+MlxFunc scatter(mlir::stablehlo::ScatterOp op) { ++std::pair scatter(mlir::stablehlo::ScatterOp op, ++ utils::ScatterType scatter_type) { + // Get scatter dimension numbers + auto scatter_dim_nums = op.getScatterDimensionNumbers(); + auto index_vector_dim = scatter_dim_nums.getIndexVectorDim(); @@ -2242,14 +2286,8 @@ index 0000000000..70d9a620e1 + std::vector update_window_dims( + scatter_dim_nums.getUpdateWindowDims().begin(), + scatter_dim_nums.getUpdateWindowDims().end()); -+ mlir::Block& update_computation = op.getUpdateComputation().front(); -+ // TODO(@cryptodeal): need to experiment to see how `computeHash` ++ // TODO(@cryptodeal): need to experiment to see how `getHashValue` + // handles hashing `update_computation` property. -+ utils::ScatterType scatter_type = *utils::getScatterType(update_computation); -+ -+ // mlir::Block& update_computation = o.getUpdateComputation().front(); -+ // utils::ScatterType scatter_type = *getScatterType(update_computation); -+ + MlxFunc fun = [index_vector_dim, + input_batching_dims = std::move(input_batching_dims), + inserted_window_dims = std::move(inserted_window_dims), @@ -2420,55 +2458,282 @@ index 0000000000..70d9a620e1 + } + return res; + }; -+ return mx::compile(fun); ++ return detail::compile(std::move(fun)); ++} ++} // namespace detail ++ ++absl::StatusOr> ++validateAndPrecompile(mlir::ModuleOp& mod) { ++ std::unordered_map mlx_cache; ++ absl::Status status; ++ // TODO: some ops should have additional checks (e.g. `stablehlo::ReduceOp`, ++ // `stablehlo::SortOp`, `stablehlo::ScatterOp`, etc.) where we need additional ++ // checks at compile time to ensure that we're correctly mapping to the ++ // corresponding MLX op. ++ mod.walk([&mlx_cache, &status](mlir::Operation* op) { ++ // Series of checks to ensure we support the module. ++ // For some ops, we have to validate that the specific ++ // inner function (e.g. `comparator` for `stablehlo::SortOp`) ++ // is supported. Pre-compile complex ops (e.g. `stablehlo::DotGeneralOp`), ++ // which require chaining multiple MLX ops together in order ++ // to implement the op per StableHLO spec. ++ return llvm::TypeSwitch(op) ++ // Verify that we support the requested `sort` op ++ .Case([&status, &op](auto o) { ++ if (!utils::validateTypeRange(o->getOperands()) || ++ !utils::validateTypeRange(o->getResults())) { ++ status = absl::UnimplementedError(absl::StrCat( ++ "Unsupported type: ", mlir::stablehlo::ToString(op))); ++ return mlir::WalkResult::interrupt(); ++ } ++ mlir::Block& comparator = o.getComparator().front(); ++ auto maybe_sort_info = utils::getSortInfo(comparator); ++ if (!maybe_sort_info.ok()) { ++ status = maybe_sort_info.status(); ++ return mlir::WalkResult::interrupt(); ++ } else ++ return mlir::WalkResult::advance(); ++ }) ++ // Verify that we support the requested `reduce` op ++ .Case([&status, &op](auto o) { ++ if (!utils::validateTypeRange(o->getOperands()) || ++ !utils::validateTypeRange(o->getResults())) { ++ status = absl::UnimplementedError(absl::StrCat( ++ "Unsupported type: ", mlir::stablehlo::ToString(op))); ++ return mlir::WalkResult::interrupt(); ++ } ++ mlir::Block& body_block = o.getBody().front(); ++ auto maybe_reduce_type = utils::getReduceType(*op, body_block); ++ if (!maybe_reduce_type.ok()) { ++ status = maybe_reduce_type.status(); ++ return mlir::WalkResult::interrupt(); ++ } else ++ return mlir::WalkResult::advance(); ++ }) ++ // Validate, precompile, and cache `dot_general` op ++ .Case( ++ [&status, &mlx_cache, &op](auto o) { ++ if (!utils::validateTypeRange(o->getOperands()) || ++ !utils::validateTypeRange(o->getResults())) { ++ status = absl::UnimplementedError(absl::StrCat( ++ "Unsupported type: ", mlir::stablehlo::ToString(op))); ++ return mlir::WalkResult::interrupt(); ++ } ++ auto hash = utils::getHashValue(op); ++ auto& cache_entry = mlx_cache[hash]; ++ if (!cache_entry.isInitialized()) { ++ // std::cout << "Compiling DotGeneralOp: " << ++ // std::to_string(hash) ++ // << std::endl ++ // << mlir::stablehlo::ToString(op).c_str() << ++ // std::endl; ++ auto compiled_dot = detail::dotGeneral(o); ++ cache_entry.initialize(std::move(compiled_dot.second), ++ compiled_dot.first); ++ } ++ return mlir::WalkResult::advance(); ++ }) ++ // Verify that we support the requested `scatter` op; ++ // hash this instance of the op, precompile and cache ++ // accordingly. ++ .Case([&mlx_cache, &op, &status](auto o) { ++ if (!utils::validateTypeRange(o->getOperands()) || ++ !utils::validateTypeRange(o->getResults())) { ++ status = absl::UnimplementedError(absl::StrCat( ++ "Unsupported type: ", mlir::stablehlo::ToString(op))); ++ return mlir::WalkResult::interrupt(); ++ } ++ mlir::Block& update_computation = o.getUpdateComputation().front(); ++ auto hash = utils::getHashValue(op); ++ auto& cache_entry = mlx_cache[hash]; ++ if (!cache_entry.isInitialized()) { ++ // only need to validate if we've not already encountered ++ auto maybe_scatter_type = utils::getScatterType(update_computation); ++ if (!maybe_scatter_type.ok()) { ++ status = maybe_scatter_type.status(); ++ return mlir::WalkResult::interrupt(); ++ } ++ auto compiled_scatter = detail::scatter(o, *maybe_scatter_type); ++ cache_entry.initialize(std::move(compiled_scatter.second), ++ compiled_scatter.first); ++ } ++ return mlir::WalkResult::advance(); ++ }) ++ // Validate, precompile, and cache `gather` op ++ .Case([&status, &mlx_cache, &op](auto o) { ++ if (!utils::validateTypeRange(o->getOperands()) || ++ !utils::validateTypeRange(o->getResults())) { ++ status = absl::UnimplementedError(absl::StrCat( ++ "Unsupported type: ", mlir::stablehlo::ToString(op))); ++ return mlir::WalkResult::interrupt(); ++ } ++ auto hash = utils::getHashValue(op); ++ auto& cache_entry = mlx_cache[hash]; ++ if (!cache_entry.isInitialized()) { ++ auto compiled_gather = detail::gather(o); ++ cache_entry.initialize(std::move(compiled_gather.second), ++ compiled_gather.first); ++ } ++ return mlir::WalkResult::advance(); ++ }) ++ // Validate, precompile, and cache `pad` op ++ .Case([&status, &mlx_cache, &op](auto o) { ++ if (!utils::validateTypeRange(o->getOperands()) || ++ !utils::validateTypeRange(o->getResults())) { ++ status = absl::UnimplementedError(absl::StrCat( ++ "Unsupported type: ", mlir::stablehlo::ToString(op))); ++ return mlir::WalkResult::interrupt(); ++ } ++ auto hash = utils::getHashValue(op); ++ auto& cache_entry = mlx_cache[hash]; ++ if (!cache_entry.isInitialized()) { ++ auto compiled_pad = detail::pad(o); ++ cache_entry.initialize(std::move(compiled_pad.second), ++ compiled_pad.first); ++ } ++ return mlir::WalkResult::advance(); ++ }) ++ .Case( ++ [&status, &op](auto o) { ++ // Validate all operand and result types are compatible with mlx ++ if (!utils::validateTypeRange(o->getOperands()) || ++ !utils::validateTypeRange(o->getResults())) { ++ status = absl::UnimplementedError(absl::StrCat( ++ "Unsupported type: ", mlir::stablehlo::ToString(op))); ++ return mlir::WalkResult::interrupt(); ++ } ++ return mlir::WalkResult::advance(); ++ }) ++ // TODO(@cryptodeal): These ops were supported prior to `mlx::compile` ++ // need to modify the implementation to ensure no use of `mlx::eval` and ++ // ensure that `mlx::random::key` is being propogated correctly. ++ .Default([&status, &op](auto o) { ++ status = absl::UnimplementedError( ++ absl::StrCat("Unsupported op: ", mlir::stablehlo::ToString(op))); ++ return mlir::WalkResult::interrupt(); ++ }); ++ }); ++ ++ // If we encounter an unsupported op, type, etc., clear ++ // the compilation cache and return the corresponding status ++ if (!status.ok()) { ++ mlx_cache.clear(); ++ return status; ++ } ++ ++ return mlx_cache; +} + -+MlxFunc func(mlir::ModuleOp mod, mlir::func::FuncOp op) { -+ MlxFunc fun = [func_info = std::shared_ptr( -+ new utils::FuncInfo(mod, op.getName().str()))]( -+ const std::vector& inputs) mutable { -+ auto mod = func_info->mod_.get(); -+ auto op = mod.lookupSymbol(func_info->name_); -+ return detail::func(mod, op, inputs); ++utils::MlxFuncRef* compileModule( ++ mlir::ModuleOp& mod, ++ std::unordered_map& mlx_cache) { ++ auto op = mod.lookupSymbol("main"); ++ ++ // compile inner function calls ++ auto& main_block = op.getBody(); ++ while (true) { ++ auto call_ops = main_block.getOps(); ++ bool all_compiled = true; ++ for (mlir::func::CallOp call_op : call_ops) { ++ auto callee = mod.lookupSymbol(call_op.getCallee()); ++ ++ if (!detail::compileFunc(mod, callee.getName().str(), mlx_cache)) { ++ all_compiled = false; ++ } ++ } ++ if (all_compiled) break; ++ } ++ ++ MlxFunc fun = [&mod, &mlx_cache](const std::vector& inputs) { ++ return detail::func(mod, "main", mlx_cache, inputs); + }; -+ return mx::compile(std::move(fun)); ++ auto hash = utils::getHashValue(op); ++ auto compiled_mod = detail::compile(fun); ++ auto& cache_entry = mlx_cache[hash]; ++ cache_entry.initialize(std::move(compiled_mod.second), compiled_mod.first); ++ ++ // force compilation by running w zeroed inputs ++ // input sizes remain constant, so we never modify ++ // mlx compiler state after initial compilation ++ std::vector tmp_inputs; ++ for (auto arg : main_block.getArguments()) { ++ auto shaped_type = mlir::cast(arg.getType()); ++ tmp_inputs.push_back( ++ mx::zeros(std::vector(shaped_type.getShape().begin(), ++ shaped_type.getShape().end()), ++ *utils::dtype::fromMlirType(shaped_type.getElementType()))); ++ } ++ ++ cache_entry.call(tmp_inputs); ++ return &cache_entry; +} + +} // namespace compile \ No newline at end of file diff --git a/xla/pjrt/plugin/mlx/compile.h b/xla/pjrt/plugin/mlx/compile.h new file mode 100644 -index 0000000000..af5cc7e342 +index 0000000000..d8ce0c1256 --- /dev/null +++ b/xla/pjrt/plugin/mlx/compile.h -@@ -0,0 +1,30 @@ +@@ -0,0 +1,33 @@ +#ifndef XLA_PJRT_PLUGIN_STABLEHLO_MLX_COMPILE_H_ +#define XLA_PJRT_PLUGIN_STABLEHLO_MLX_COMPILE_H_ + ++#include ++#include ++#include +#include + ++#include "absl/status/statusor.h" +#include "mlx/mlx.h" +#include "mlir/IR/Operation.h" + +#include "stablehlo/reference/Api.h" +#include "mlir/IR/Operation.h" -+#include "mlir/Dialect/Func/IR/FuncOps.h" +#include "xla/pjrt/plugin/mlx/utils.h" + +namespace mx = mlx::core; + +namespace compile { +namespace detail { -+std::vector cbrt(const std::vector& inputs); -+std::vector func(mlir::ModuleOp mod, mlir::func::FuncOp op, -+ const std::vector& inputs, -+ bool recursive_compile = true); ++std::pair compile( ++ std::function(const std::vector&)> fun, ++ bool shapeless = false); +} // namespace detail -+MlxFunc dotGeneral(mlir::stablehlo::DotGeneralOp op); -+MlxFunc gather(mlir::stablehlo::GatherOp op); -+MlxFunc pad(mlir::stablehlo::PadOp op); -+MlxFunc scatter(mlir::stablehlo::ScatterOp op); -+MlxFunc func(mlir::ModuleOp mod, mlir::func::FuncOp op); ++ ++absl::StatusOr> ++validateAndPrecompile(mlir::ModuleOp& mod); ++utils::MlxFuncRef* compileModule( ++ mlir::ModuleOp& mod, ++ std::unordered_map& mlx_cache); +} // namespace compile + +#endif // XLA_PJRT_PLUGIN_STABLEHLO_MLX_COMPILE_H_ @@ -2738,10 +3003,10 @@ index 0000000000..ef42968a3a \ No newline at end of file diff --git a/xla/pjrt/plugin/mlx/executable.cc b/xla/pjrt/plugin/mlx/executable.cc new file mode 100644 -index 0000000000..ec612eca89 +index 0000000000..b3dd63d442 --- /dev/null +++ b/xla/pjrt/plugin/mlx/executable.cc -@@ -0,0 +1,507 @@ +@@ -0,0 +1,409 @@ +/* Copyright 2024 The OpenXLA Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. @@ -2771,7 +3036,6 @@ index 0000000000..ec612eca89 + +// TODO(@cryptodeal): might need to update `BUILD` +#include "mlir/IR/Visitors.h" -+#include "mlx/mlx.h" +#include "absl/log/log.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" @@ -2787,6 +3051,8 @@ index 0000000000..ec612eca89 +#include "mlir/Interfaces/DataLayoutInterfaces.h" +#include "mlir/Pass/PassManager.h" +#include "mlir/Support/LLVM.h" ++#include "mlx/compile_impl.h" ++#include "mlx/mlx.h" +#include "stablehlo/reference/Api.h" +#include "stablehlo/transforms/optimization/Passes.h" +#include "xla/hlo/translate/stablehlo.h" @@ -2808,6 +3074,7 @@ index 0000000000..ec612eca89 +#define DEBUG_TYPE "stablehlo-pjrt" + +namespace mx = mlx::core; ++ +namespace mlir::stablehlo { + +#define UNIMPLEMENTED(name) \ @@ -2862,143 +3129,23 @@ index 0000000000..ec612eca89 + return success(); +} + -+absl::Status validateModule(ModuleOp mod) { -+ absl::Status status; -+ // TODO: some ops should have additional checks (e.g. `stablehlo::ReduceOp`, -+ // `stablehlo::SortOp`, `stablehlo::ScatterOp`, etc.) where we need additional -+ // checks at compile time to ensure that we're correctly mapping to the -+ // corresponding MLX op. -+ mod.walk([&status](Operation* op) { -+ // Series of checks to ensure we support the module -+ return llvm::TypeSwitch(op) -+ .Case([&status, &op](auto o) { -+ // Validate all input types are compatible with mlx -+ for (auto operand : o->getOperands()) { -+ auto primitive_type = xla::ConvertMlirTypeToPrimitiveType( -+ mlir::cast(operand.getType()).getElementType()); -+ switch (primitive_type) { -+ case xla::PrimitiveType::PRED: -+ case xla::PrimitiveType::U8: -+ case xla::PrimitiveType::S8: -+ case xla::PrimitiveType::U16: -+ case xla::PrimitiveType::S16: -+ case xla::PrimitiveType::U32: -+ case xla::PrimitiveType::S32: -+ case xla::PrimitiveType::U64: -+ case xla::PrimitiveType::S64: -+ case xla::PrimitiveType::F16: -+ case xla::PrimitiveType::BF16: -+ case xla::PrimitiveType::F32: -+ case xla::PrimitiveType::C64: -+ break; -+ default: { -+ std::cout << "Unsupported type: " -+ << xla::PrimitiveType_Name(primitive_type).c_str() -+ << std::endl; -+ status = absl::UnimplementedError( -+ absl::StrCat("Unsupported type: ", ToString(op))); -+ return WalkResult::interrupt(); -+ } -+ } -+ } -+ -+ // Validate all result types are compatible with mlx -+ for (auto result : o->getResults()) { -+ auto primitive_type = xla::ConvertMlirTypeToPrimitiveType( -+ mlir::cast(result.getType()).getElementType()); -+ switch (primitive_type) { -+ case xla::PrimitiveType::PRED: -+ case xla::PrimitiveType::U8: -+ case xla::PrimitiveType::S8: -+ case xla::PrimitiveType::U16: -+ case xla::PrimitiveType::S16: -+ case xla::PrimitiveType::U32: -+ case xla::PrimitiveType::S32: -+ case xla::PrimitiveType::U64: -+ case xla::PrimitiveType::S64: -+ case xla::PrimitiveType::F16: -+ case xla::PrimitiveType::BF16: -+ case xla::PrimitiveType::F32: -+ case xla::PrimitiveType::C64: -+ break; -+ default: { -+ std::cout << "Unsupported type: " -+ << xla::PrimitiveType_Name(primitive_type).c_str() -+ << std::endl; -+ status = absl::UnimplementedError( -+ absl::StrCat("Unsupported type: ", ToString(op))); -+ return WalkResult::interrupt(); -+ } -+ } -+ } -+ return mlir::WalkResult::advance(); -+ }) -+ // TODO(@cryptodeal): These ops were supported prior to `mlx::compile` -+ // need to modify the implementation to ensure no use of `mlx::eval` and -+ // ensure that `mlx::random::key` is being propogated correctly. -+ .Default([&status, &op](auto o) { -+ status = absl::UnimplementedError( -+ absl::StrCat("Unsupported op: ", ToString(op))); -+ return WalkResult::interrupt(); -+ }); -+ }); -+ return status; -+} -+ +class MlirLoadedExecutable : public PjRtLoadedExecutable { + public: -+ MlirLoadedExecutable(ModuleOp module, DeviceAssignment assignment, -+ absl::Span devices, -+ PjRtClient* client) ++ MlirLoadedExecutable( ++ ModuleOp module, ++ std::unordered_map mlx_cache, ++ DeviceAssignment assignment, absl::Span devices, ++ PjRtClient* client) + : PjRtLoadedExecutable(), + name_("MlirLoadedExecutable"), + assignment_(assignment), + devices_(client->devices()), + client_(client), + context_(), ++ mlx_cache_(std::move(mlx_cache)), + module_(cloneIntoContext(module, context_)) { + // TRACE_ME_MEMBER; -+ -+ auto main = module.lookupSymbol("main"); -+ // std::cout << "Compiling module: " << std::to_string(getHashValue(main)) -+ // << std::endl -+ // << std::endl; -+ // std::cout << ToString(module_.get()).c_str() << std::endl << std::endl; -+ compiled_module_ = compile::func(module_.get(), main); -+ -+ auto& main_block = main.front(); -+ // force compilation by running w zeroed inputs -+ std::vector tmp_inputs; -+ for (auto arg : main_block.getArguments()) { -+ auto shaped_type = mlir::cast(arg.getType()); -+ tmp_inputs.push_back( -+ mx::zeros(std::vector(shaped_type.getShape().begin(), -+ shaped_type.getShape().end()), -+ *utils::dtype::fromMlirType(shaped_type.getElementType()))); -+ } -+ compiled_module_(tmp_inputs); ++ compiled_module_ = compile::compileModule(module, mlx_cache_); + } + + static absl::StatusOr> CompileAndLoad( @@ -3016,9 +3163,15 @@ index 0000000000..ec612eca89 + return diagnostic_handler.ConsumeStatus(); + } + -+ auto module_status = validateModule(module); -+ if (!module_status.ok()) { -+ return module_status; ++ // Lock mlx mutex whenever a thread modifies the ++ // mlx compilation cache state; since we force compilation ++ // by evaluating with zeroed inputs, the compilation cache ++ // will not be modified by this `Executable` until it is destroyed. ++ std::lock_guard lock(mx::detail::compilation_mutex); ++ ++ auto maybe_mlx_cache = compile::validateAndPrecompile(module); ++ if (!maybe_mlx_cache.ok()) { ++ return maybe_mlx_cache.status(); + } + + // auto device_info = mx::metal::device_info(); @@ -3028,10 +3181,9 @@ index 0000000000..ec612eca89 + // std::cout << std::endl; + // } + -+ auto executable = std::make_unique(module, assignment, -+ devices, client); -+ -+ return executable; ++ return std::make_unique( ++ module, std::move(maybe_mlx_cache).value(), assignment, devices, ++ client); + } + + PjRtClient* client() const override { @@ -3072,7 +3224,15 @@ index 0000000000..ec612eca89 + TF_ASSIGN_OR_RETURN(mx::array input, GetArrayFromBuffer(buffer)); + inputs.push_back(std::move(input)); + } -+ auto mlx_res = compiled_module_(inputs); ++ std::vector mlx_res; ++ { ++ // Lock mlx compilation mutex while calling compiled ++ // functions, since it performs a lookup on MLX compilation ++ // cache, which might be modified by other threads. ++ std::lock_guard lock(mx::detail::compilation_mutex); ++ mlx_res = compiled_module_->call(inputs); ++ } ++ // TODO (@cryptodeal): figure out best practice for returning strided data + for (auto i = 0; i < mlx_res.size(); ++i) { + mlx_res[i] = mx::contiguous(mlx_res[i]); + } @@ -3164,6 +3324,12 @@ index 0000000000..ec612eca89 + + void Delete() override { + // TRACE_ME_MEMBER; ++ { ++ // lock mutex and clear compiled functions owned by this executable ++ std::lock_guard lock(mx::detail::compilation_mutex); ++ mlx_cache_.clear(); ++ compiled_module_ = nullptr; ++ } + module_.release(); + module_ = nullptr; + } @@ -3224,7 +3390,8 @@ index 0000000000..ec612eca89 + absl::Span devices_; + PjRtClient* client_; + // MLX compilation context -+ MlxFunc compiled_module_; ++ std::unordered_map mlx_cache_; ++ utils::MlxFuncRef* compiled_module_ = nullptr; + + // MLIR + MLIRContext context_; @@ -3251,10 +3418,10 @@ index 0000000000..ec612eca89 +} // namespace mlir::stablehlo diff --git a/xla/pjrt/plugin/mlx/executable.h b/xla/pjrt/plugin/mlx/executable.h new file mode 100644 -index 0000000000..d970cde59d +index 0000000000..cc11793e50 --- /dev/null +++ b/xla/pjrt/plugin/mlx/executable.h -@@ -0,0 +1,33 @@ +@@ -0,0 +1,36 @@ +/* Copyright 2024 The OpenXLA Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. @@ -3272,10 +3439,13 @@ index 0000000000..d970cde59d +#include + +#include "mlir/IR/BuiltinOps.h" ++#include "mlx/mlx.h" +#include "xla/hlo/builder/xla_computation.h" +#include "xla/pjrt/pjrt_client.h" ++#include "xla/pjrt/plugin/mlx/utils.h" +#include "xla/service/computation_placer.h" + ++namespace mx = mlx::core; +namespace mlir::stablehlo { + +absl::StatusOr> StablehloMlxCompile( @@ -3476,10 +3646,10 @@ index 0000000000..d56a22565f \ No newline at end of file diff --git a/xla/pjrt/plugin/mlx/utils.cc b/xla/pjrt/plugin/mlx/utils.cc new file mode 100644 -index 0000000000..384c5f1a4d +index 0000000000..643a842d46 --- /dev/null +++ b/xla/pjrt/plugin/mlx/utils.cc -@@ -0,0 +1,632 @@ +@@ -0,0 +1,692 @@ +#include +#include +#include @@ -3526,6 +3696,182 @@ index 0000000000..384c5f1a4d +} + +namespace utils { ++namespace detail { ++// Returns true if the given `ReduceOp` matches ++// `zml.Tensor.argMax` op implementation. ++bool isArgMaxReduce(mlir::Block& body_block) { ++ unsigned op_count = 0; ++ // TODO (@cryptodeal): This is a lazy verification; maybe check operands ++ // for each operation in the block. ++ for (auto& op : body_block.getOperations()) { ++ switch (op_count) { ++ case 0: { ++ auto compare_op = llvm::dyn_cast(op); ++ if (compare_op && compare_op.getComparisonDirection() == ++ mlir::stablehlo::ComparisonDirection::GT) { ++ break; ++ } else ++ return false; ++ } ++ case 1: { ++ auto compare_op = llvm::dyn_cast(op); ++ if (compare_op && compare_op.getComparisonDirection() == ++ mlir::stablehlo::ComparisonDirection::NE) { ++ break; ++ } else ++ return false; ++ } ++ case 2: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ case 3: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ case 4: { ++ auto compare_op = llvm::dyn_cast(op); ++ if (compare_op && compare_op.getComparisonDirection() == ++ mlir::stablehlo::ComparisonDirection::EQ) { ++ break; ++ } else ++ return false; ++ } ++ case 5: { ++ auto compare_op = llvm::dyn_cast(op); ++ if (compare_op && compare_op.getComparisonDirection() == ++ mlir::stablehlo::ComparisonDirection::LT) { ++ break; ++ } else ++ return false; ++ } ++ case 6: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ case 7: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ case 8: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ case 9: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ default: ++ return false; ++ } ++ op_count++; ++ } ++ return true; ++} ++ ++// Returns true if the given `ReduceOp` matches ++// `zml.Tensor.sum` op implementation. ++bool isSumReduce(mlir::Block& body_block) { ++ unsigned op_count = 0; ++ // TODO (@cryptodeal): This is a lazy verification; maybe check operands ++ // for each operation in the block. ++ for (auto& op : body_block.getOperations()) { ++ switch (op_count) { ++ case 0: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ case 1: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ } ++ op_count++; ++ } ++ return true; ++} ++ ++// Returns true if the given `ReduceOp` matches ++// `zml.Tensor.max` op implementation. ++bool isMaxReduce(mlir::Block& body_block) { ++ unsigned op_count = 0; ++ // TODO (@cryptodeal): This is a lazy verification; maybe check operands ++ // for each operation in the block. ++ for (auto& op : body_block.getOperations()) { ++ switch (op_count) { ++ case 0: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ case 1: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ } ++ op_count++; ++ } ++ return true; ++} ++ ++// Returns true if the given `ReduceOp` matches ++// `zml.Tensor.max` op implementation. ++bool isMinReduce(mlir::Block& body_block) { ++ unsigned op_count = 0; ++ // TODO (@cryptodeal): This is a lazy verification; maybe check operands ++ // for each operation in the block. ++ for (auto& op : body_block.getOperations()) { ++ switch (op_count) { ++ case 0: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ case 1: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ } ++ op_count++; ++ } ++ return true; ++} ++ ++// Returns true if the given `ReduceOp` matches ++// `zml.Tensor.any` op implementation. ++bool isAnyReduce(mlir::Block& body_block) { ++ unsigned op_count = 0; ++ // TODO (@cryptodeal): This is a lazy verification; maybe check operands ++ // for each operation in the block. ++ for (auto& op : body_block.getOperations()) { ++ switch (op_count) { ++ case 0: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ case 1: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ } ++ op_count++; ++ } ++ return true; ++} ++ ++} // namespace detail + +namespace dtype { +xla::PrimitiveType asXlaPrimitiveType(mx::Dtype dtype) { @@ -3877,185 +4223,6 @@ index 0000000000..384c5f1a4d + return std::make_pair(input_idx, comparator); +} + -+// Returns true if the given `ReduceOp` matches -+// `zml.Tensor.argMax` op implementation. -+bool isArgMaxReduce(mlir::stablehlo::ReduceOp reduce_op) { -+ mlir::Block& body_block = reduce_op.getBody().front(); -+ unsigned op_count = 0; -+ // TODO (@cryptodeal): This is a lazy verification; maybe check operands -+ // for each operation in the block. -+ for (auto& op : body_block.getOperations()) { -+ switch (op_count) { -+ case 0: { -+ auto compare_op = llvm::dyn_cast(op); -+ if (compare_op && compare_op.getComparisonDirection() == -+ mlir::stablehlo::ComparisonDirection::GT) { -+ break; -+ } else -+ return false; -+ } -+ case 1: { -+ auto compare_op = llvm::dyn_cast(op); -+ if (compare_op && compare_op.getComparisonDirection() == -+ mlir::stablehlo::ComparisonDirection::NE) { -+ break; -+ } else -+ return false; -+ } -+ case 2: -+ if (llvm::dyn_cast(op)) { -+ break; -+ } else -+ return false; -+ case 3: -+ if (llvm::dyn_cast(op)) { -+ break; -+ } else -+ return false; -+ case 4: { -+ auto compare_op = llvm::dyn_cast(op); -+ if (compare_op && compare_op.getComparisonDirection() == -+ mlir::stablehlo::ComparisonDirection::EQ) { -+ break; -+ } else -+ return false; -+ } -+ case 5: { -+ auto compare_op = llvm::dyn_cast(op); -+ if (compare_op && compare_op.getComparisonDirection() == -+ mlir::stablehlo::ComparisonDirection::LT) { -+ break; -+ } else -+ return false; -+ } -+ case 6: -+ if (llvm::dyn_cast(op)) { -+ break; -+ } else -+ return false; -+ case 7: -+ if (llvm::dyn_cast(op)) { -+ break; -+ } else -+ return false; -+ case 8: -+ if (llvm::dyn_cast(op)) { -+ break; -+ } else -+ return false; -+ case 9: -+ if (llvm::dyn_cast(op)) { -+ break; -+ } else -+ return false; -+ default: -+ return false; -+ } -+ op_count++; -+ } -+ return true; -+} -+ -+// Returns true if the given `ReduceOp` matches -+// `zml.Tensor.sum` op implementation. -+bool isSumReduce(mlir::stablehlo::ReduceOp reduce_op) { -+ mlir::Block& body_block = reduce_op.getBody().front(); -+ unsigned op_count = 0; -+ // TODO (@cryptodeal): This is a lazy verification; maybe check operands -+ // for each operation in the block. -+ for (auto& op : body_block.getOperations()) { -+ switch (op_count) { -+ case 0: -+ if (llvm::dyn_cast(op)) { -+ break; -+ } else -+ return false; -+ case 1: -+ if (llvm::dyn_cast(op)) { -+ break; -+ } else -+ return false; -+ } -+ op_count++; -+ } -+ return true; -+} -+ -+// Returns true if the given `ReduceOp` matches -+// `zml.Tensor.max` op implementation. -+bool isMaxReduce(mlir::stablehlo::ReduceOp reduce_op) { -+ mlir::Block& body_block = reduce_op.getBody().front(); -+ unsigned op_count = 0; -+ // TODO (@cryptodeal): This is a lazy verification; maybe check operands -+ // for each operation in the block. -+ for (auto& op : body_block.getOperations()) { -+ switch (op_count) { -+ case 0: -+ if (llvm::dyn_cast(op)) { -+ break; -+ } else -+ return false; -+ case 1: -+ if (llvm::dyn_cast(op)) { -+ break; -+ } else -+ return false; -+ } -+ op_count++; -+ } -+ return true; -+} -+ -+// Returns true if the given `ReduceOp` matches -+// `zml.Tensor.max` op implementation. -+bool isMinReduce(mlir::stablehlo::ReduceOp reduce_op) { -+ mlir::Block& body_block = reduce_op.getBody().front(); -+ unsigned op_count = 0; -+ // TODO (@cryptodeal): This is a lazy verification; maybe check operands -+ // for each operation in the block. -+ for (auto& op : body_block.getOperations()) { -+ switch (op_count) { -+ case 0: -+ if (llvm::dyn_cast(op)) { -+ break; -+ } else -+ return false; -+ case 1: -+ if (llvm::dyn_cast(op)) { -+ break; -+ } else -+ return false; -+ } -+ op_count++; -+ } -+ return true; -+} -+ -+// Returns true if the given `ReduceOp` matches -+// `zml.Tensor.any` op implementation. -+bool isAnyReduce(mlir::stablehlo::ReduceOp reduce_op) { -+ mlir::Block& body_block = reduce_op.getBody().front(); -+ unsigned op_count = 0; -+ // TODO (@cryptodeal): This is a lazy verification; maybe check operands -+ // for each operation in the block. -+ for (auto& op : body_block.getOperations()) { -+ switch (op_count) { -+ case 0: -+ if (llvm::dyn_cast(op)) { -+ break; -+ } else -+ return false; -+ case 1: -+ if (llvm::dyn_cast(op)) { -+ break; -+ } else -+ return false; -+ } -+ op_count++; -+ } -+ return true; -+} -+ +llvm::hash_code computeHash( + mlir::Operation* op, + llvm::function_ref hashOperands, @@ -4090,6 +4257,24 @@ index 0000000000..384c5f1a4d + return hash; +} + ++absl::StatusOr getReduceType(mlir::Operation& op, ++ mlir::Block& body_block) { ++ if (detail::isArgMaxReduce(body_block)) { ++ return ReduceType::ArgMax; ++ } else if (detail::isSumReduce(body_block)) { ++ return ReduceType::Sum; ++ } else if (detail::isMaxReduce(body_block)) { ++ return ReduceType::Max; ++ } else if (detail::isMinReduce(body_block)) { ++ return ReduceType::Min; ++ } else if (detail::isAnyReduce(body_block)) { ++ return ReduceType::Any; ++ } else { ++ return xla::Internal("Unsupported reduce op: %s", ++ mlir::stablehlo::ToString(op)); ++ } ++} ++ +uint64_t getHashValue(mlir::func::FuncOp func_op) { + llvm::hash_code hash = {}; + for (mlir::NamedAttribute attr : func_op->getAttrs()) { @@ -4111,13 +4296,58 @@ index 0000000000..384c5f1a4d + mlir::OperationEquivalence::ignoreHashValue); +} + ++bool isValidType(mlir::Type type) { ++ auto primitive_type = xla::ConvertMlirTypeToPrimitiveType( ++ mlir::cast(type).getElementType()); ++ switch (primitive_type) { ++ case xla::PrimitiveType::PRED: ++ case xla::PrimitiveType::U8: ++ case xla::PrimitiveType::S8: ++ case xla::PrimitiveType::U16: ++ case xla::PrimitiveType::S16: ++ case xla::PrimitiveType::U32: ++ case xla::PrimitiveType::S32: ++ case xla::PrimitiveType::U64: ++ case xla::PrimitiveType::S64: ++ case xla::PrimitiveType::F16: ++ case xla::PrimitiveType::BF16: ++ case xla::PrimitiveType::F32: ++ case xla::PrimitiveType::C64: ++ return true; ++ default: { ++ // std::cout << "Unsupported type: " ++ // << xla::PrimitiveType_Name(primitive_type).c_str() ++ // << std::endl; ++ return false; ++ } ++ } ++} ++ ++bool validateTypeRange(mlir::Operation::operand_range range) { ++ for (auto operand : range) { ++ if (!isValidType(operand.getType())) { ++ return false; ++ } ++ } ++ return true; ++} ++ ++bool validateTypeRange(mlir::Operation::result_range range) { ++ for (auto result : range) { ++ if (!isValidType(result.getType())) { ++ return false; ++ } ++ } ++ return true; ++} ++ +} // namespace utils diff --git a/xla/pjrt/plugin/mlx/utils.h b/xla/pjrt/plugin/mlx/utils.h new file mode 100644 -index 0000000000..f19257b5a2 +index 0000000000..6e38b50502 --- /dev/null +++ b/xla/pjrt/plugin/mlx/utils.h -@@ -0,0 +1,94 @@ +@@ -0,0 +1,136 @@ +/* Copyright 2024 The OpenXLA Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. @@ -4132,7 +4362,8 @@ index 0000000000..f19257b5a2 + +#ifndef XLA_PJRT_PLUGIN_STABLEHLO_MLX_UTILS_H_ +#define XLA_PJRT_PLUGIN_STABLEHLO_MLX_UTILS_H_ -+ ++#include ++#include +#include +#include +#include @@ -4146,6 +4377,7 @@ index 0000000000..f19257b5a2 +#include "mlir/IR/Types.h" +#include "mlir/IR/Operation.h" +#include "mlir/IR/Value.h" ++#include "mlx/compile_impl.h" +#include "mlx/mlx.h" +#include "xla/literal.h" + @@ -4156,22 +4388,63 @@ index 0000000000..f19257b5a2 +using OwnedMlxFunc = std::shared_ptr; + +namespace utils { ++class MlxFuncRef { ++ public: ++ MlxFuncRef() : fun_id_(0) {} ++ MlxFuncRef(MlxFunc ref, std::uintptr_t fun_id) ++ : ref_(std::move(ref)), fun_id_(fun_id) {} + -+struct FuncInfo { -+ mlir::OwningOpRef mod_; -+ std::string name_; ++ ~MlxFuncRef() { ++ if (fun_id_) { ++ std::cout << "Destroying MlxFuncRef: " << std::to_string(fun_id_) ++ << std::endl; ++ mx::detail::compile_erase(fun_id_); ++ } ++ } + -+ FuncInfo(mlir::ModuleOp mod, std::string name) -+ : mod_(mod.clone()), name_(name) {} ++ MlxFuncRef(MlxFuncRef&& other) noexcept { ++ ref_ = std::move(other.ref_); ++ fun_id_ = other.fun_id_; ++ } ++ ++ MlxFuncRef& operator=(MlxFuncRef&& other) noexcept { ++ ref_ = std::move(other.ref_); ++ fun_id_ = other.fun_id_; ++ return *this; ++ } ++ ++ std::vector call(const std::vector& inputs) { ++ return ref_(inputs); ++ } ++ ++ bool isInitialized() { return fun_id_ != 0; } ++ ++ void initialize(MlxFunc ref, std::uintptr_t fun_id) { ++ // if (fun_id_ != 0) mx::detail::compile_erase(fun_id_); ++ ref_ = std::move(ref); ++ fun_id_ = fun_id; ++ } ++ ++ private: ++ MlxFunc ref_; ++ std::uintptr_t fun_id_; +}; + -+enum ScatterType { ++enum class ScatterType { + Replace, + Add, + Prod, + Max, + Min, +}; ++ ++enum class ReduceType { ++ ArgMax, ++ Sum, ++ Max, ++ Min, ++ Any, ++}; +namespace dtype { +xla::PrimitiveType asXlaPrimitiveType(mx::Dtype dtype); + @@ -4202,13 +4475,12 @@ index 0000000000..f19257b5a2 +absl::StatusOr getScatterType(mlir::Block& block); +absl::StatusOr> getSortInfo( + mlir::Block& block); -+bool isArgMaxReduce(mlir::stablehlo::ReduceOp reduce_op); -+bool isSumReduce(mlir::stablehlo::ReduceOp reduce_op); -+bool isMaxReduce(mlir::stablehlo::ReduceOp reduce_op); -+bool isMinReduce(mlir::stablehlo::ReduceOp reduce_op); -+bool isAnyReduce(mlir::stablehlo::ReduceOp reduce_op); ++absl::StatusOr getReduceType(mlir::Operation& op, ++ mlir::Block& body_block); +uint64_t getHashValue(mlir::func::FuncOp func_op); +uint64_t getHashValue(mlir::Operation* op); ++bool validateTypeRange(mlir::Operation::operand_range range); ++bool validateTypeRange(mlir::Operation::result_range range); +} // namespace utils + +#endif // XLA_PJRT_PLUGIN_STABLEHLO_MLX_UTILS_H_