diff --git a/MODULE.bazel b/MODULE.bazel index 61572c88..8e2faf42 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -1,6 +1,15 @@ module( name = "zml", ) +# local_path_override( +# module_name = "xla", +# path = "../xla", +# ) + +# local_path_override( +# module_name = "mlx", +# path = "../mlx_master", +# ) bazel_dep(name = "abseil-cpp", version = "20240722.0.bcr.2") bazel_dep(name = "aspect_bazel_lib", version = "2.14.0") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 3132fc11..0b70e72e 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -7,7 +7,6 @@ "https://bazel-registry.zml.ai/modules/abseil-cpp/20230802.0.bcr.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/abseil-cpp/20230802.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/abseil-cpp/20230802.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/abseil-cpp/20240116.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/abseil-cpp/20240116.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/abseil-cpp/20240116.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/abseil-cpp/20240722.0.bcr.2/MODULE.bazel": "not found", @@ -51,6 +50,7 @@ "https://bazel-registry.zml.ai/modules/buildifier_prebuilt/7.3.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/buildifier_prebuilt/8.0.3/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/buildozer/7.1.2/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/fmt/11.0.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/google_benchmark/1.8.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/googletest/1.11.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/googletest/1.14.0.bcr.1/MODULE.bazel": "not found", @@ -87,6 +87,7 @@ "https://bazel-registry.zml.ai/modules/re2/2024-07-02.bcr.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/re2/2024-07-02/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_android/0.1.1/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/rules_apple/3.13.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_apple/3.22.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.0.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.0.10/MODULE.bazel": "not found", @@ -159,6 +160,7 @@ "https://bazel-registry.zml.ai/modules/rules_shell/0.3.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_shell/0.4.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_swift/1.16.0/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/rules_swift/2.1.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_swift/2.4.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_uv/0.65.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.5.1/MODULE.bazel": "not found", @@ -184,7 +186,6 @@ "https://bcr.bazel.build/modules/abseil-cpp/20230802.0.bcr.1/MODULE.bazel": "1c8cec495288dccd14fdae6e3f95f772c1c91857047a098fad772034264cc8cb", "https://bcr.bazel.build/modules/abseil-cpp/20230802.0/MODULE.bazel": "d253ae36a8bd9ee3c5955384096ccb6baf16a1b1e93e858370da0a3b94f77c16", "https://bcr.bazel.build/modules/abseil-cpp/20230802.1/MODULE.bazel": "fa92e2eb41a04df73cdabeec37107316f7e5272650f81d6cc096418fe647b915", - "https://bcr.bazel.build/modules/abseil-cpp/20240116.0/MODULE.bazel": "98dc378d64c12a4e4741ad3362f87fb737ee6a0886b2d90c3cdbb4d93ea3e0bf", "https://bcr.bazel.build/modules/abseil-cpp/20240116.1/MODULE.bazel": "37bcdb4440fbb61df6a1c296ae01b327f19e9bb521f9b8e26ec854b6f97309ed", "https://bcr.bazel.build/modules/abseil-cpp/20240116.2/MODULE.bazel": "73939767a4686cd9a520d16af5ab440071ed75cec1a876bf2fcfaf1f71987a16", "https://bcr.bazel.build/modules/abseil-cpp/20240722.0.bcr.2/MODULE.bazel": "c3661b44c9d3f17f0b65ffb544896aaeb89127398ea867537babac18133a002a", @@ -236,6 +237,8 @@ "https://bcr.bazel.build/modules/buildifier_prebuilt/8.0.3/source.json": "68ee16210562f325ff961aabeb4d46ed10e6638b5e842cd48d8d83dea220dfbd", "https://bcr.bazel.build/modules/buildozer/7.1.2/MODULE.bazel": "2e8dd40ede9c454042645fd8d8d0cd1527966aa5c919de86661e62953cd73d84", "https://bcr.bazel.build/modules/buildozer/7.1.2/source.json": "c9028a501d2db85793a6996205c8de120944f50a0d570438fcae0457a5f9d1f8", + "https://bcr.bazel.build/modules/fmt/11.0.2/MODULE.bazel": "d9e0ea6f68a762a84055172ce8f55cc01da950c94b5e27f66c1acc14e6fc59ce", + "https://bcr.bazel.build/modules/fmt/11.0.2/source.json": "36cde5a3cbbee5eefc1e7f8a5a66d8ac9f352f7742bdd0c9b2d869601169c881", "https://bcr.bazel.build/modules/google_benchmark/1.8.2/MODULE.bazel": "a70cf1bba851000ba93b58ae2f6d76490a9feb74192e57ab8e8ff13c34ec50cb", "https://bcr.bazel.build/modules/googletest/1.11.0/MODULE.bazel": "3a83f095183f66345ca86aa13c58b59f9f94a2f81999c093d4eeaa2d262d12f4", "https://bcr.bazel.build/modules/googletest/1.14.0.bcr.1/MODULE.bazel": "22c31a561553727960057361aa33bf20fb2e98584bc4fec007906e27053f80c6", @@ -283,6 +286,7 @@ "https://bcr.bazel.build/modules/re2/2024-07-02/MODULE.bazel": "0eadc4395959969297cbcf31a249ff457f2f1d456228c67719480205aa306daa", "https://bcr.bazel.build/modules/rules_android/0.1.1/MODULE.bazel": "48809ab0091b07ad0182defb787c4c5328bd3a278938415c00a7b69b50c4d3a8", "https://bcr.bazel.build/modules/rules_android/0.1.1/source.json": "e6986b41626ee10bdc864937ffb6d6bf275bb5b9c65120e6137d56e6331f089e", + "https://bcr.bazel.build/modules/rules_apple/3.13.0/MODULE.bazel": "b4559a2c6281ca3165275bb36c1f0ac74666632adc5bdb680e366de7ce845f43", "https://bcr.bazel.build/modules/rules_apple/3.22.0/MODULE.bazel": "0fd8c630d6cecc18abe84114ee21c1a0fb759b45063d3d2600ec22669e93ea43", "https://bcr.bazel.build/modules/rules_apple/3.22.0/source.json": "ee41f63c292bd876fd264a830cf45500619a063e2fa3a18cca6fa12b153d68e3", "https://bcr.bazel.build/modules/rules_cc/0.0.1/MODULE.bazel": "cb2aa0747f84c6c3a78dad4e2049c154f08ab9d166b1273835a8174940365647", @@ -369,6 +373,7 @@ "https://bcr.bazel.build/modules/rules_shell/0.4.1/MODULE.bazel": "00e501db01bbf4e3e1dd1595959092c2fadf2087b2852d3f553b5370f5633592", "https://bcr.bazel.build/modules/rules_shell/0.4.1/source.json": "4757bd277fe1567763991c4425b483477bb82e35e777a56fd846eb5cceda324a", "https://bcr.bazel.build/modules/rules_swift/1.16.0/MODULE.bazel": "4a09f199545a60d09895e8281362b1ff3bb08bbde69c6fc87aff5b92fcc916ca", + "https://bcr.bazel.build/modules/rules_swift/2.1.1/MODULE.bazel": "494900a80f944fc7aa61500c2073d9729dff0b764f0e89b824eb746959bc1046", "https://bcr.bazel.build/modules/rules_swift/2.4.0/MODULE.bazel": "1639617eb1ede28d774d967a738b4a68b0accb40650beadb57c21846beab5efd", "https://bcr.bazel.build/modules/rules_swift/2.4.0/source.json": "a6577f57f9febbdc015a01f2a8f3487422032f134d6c61d18ed8e8ca3b9acc7c", "https://bcr.bazel.build/modules/rules_uv/0.65.0/MODULE.bazel": "a700505a80ad0ad65f062c9335458d137f504b98c5937b338807bb5cbe305e73", diff --git a/examples/MODULE.bazel b/examples/MODULE.bazel index e207c9fb..4ed306cc 100644 --- a/examples/MODULE.bazel +++ b/examples/MODULE.bazel @@ -1,5 +1,15 @@ module(name = "examples") +# local_path_override( +# module_name = "xla", +# path = "../../xla", +# ) + +# local_path_override( +# module_name = "mlx", +# path = "../../mlx_master", +# ) + bazel_dep(name = "aspect_bazel_lib", version = "2.14.0") bazel_dep(name = "bazel_skylib", version = "1.7.1") bazel_dep(name = "platforms", version = "0.0.11") diff --git a/examples/MODULE.bazel.lock b/examples/MODULE.bazel.lock index 854c8dbf..3a6f4e9d 100644 --- a/examples/MODULE.bazel.lock +++ b/examples/MODULE.bazel.lock @@ -7,7 +7,6 @@ "https://bazel-registry.zml.ai/modules/abseil-cpp/20230802.0.bcr.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/abseil-cpp/20230802.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/abseil-cpp/20230802.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/abseil-cpp/20240116.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/abseil-cpp/20240116.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/abseil-cpp/20240116.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/abseil-cpp/20240722.0.bcr.2/MODULE.bazel": "not found", @@ -51,6 +50,7 @@ "https://bazel-registry.zml.ai/modules/buildifier_prebuilt/6.4.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/buildifier_prebuilt/7.3.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/buildozer/7.1.2/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/fmt/11.0.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/google_benchmark/1.8.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/googletest/1.11.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/googletest/1.14.0.bcr.1/MODULE.bazel": "not found", @@ -87,6 +87,7 @@ "https://bazel-registry.zml.ai/modules/re2/2024-07-02.bcr.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/re2/2024-07-02/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_android/0.1.1/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/rules_apple/3.13.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_apple/3.22.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.0.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.0.10/MODULE.bazel": "not found", @@ -160,6 +161,7 @@ "https://bazel-registry.zml.ai/modules/rules_shell/0.3.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_shell/0.4.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_swift/1.16.0/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/rules_swift/2.1.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_swift/2.4.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_uv/0.65.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.5.1/MODULE.bazel": "not found", @@ -186,7 +188,6 @@ "https://bcr.bazel.build/modules/abseil-cpp/20230802.0.bcr.1/MODULE.bazel": "1c8cec495288dccd14fdae6e3f95f772c1c91857047a098fad772034264cc8cb", "https://bcr.bazel.build/modules/abseil-cpp/20230802.0/MODULE.bazel": "d253ae36a8bd9ee3c5955384096ccb6baf16a1b1e93e858370da0a3b94f77c16", "https://bcr.bazel.build/modules/abseil-cpp/20230802.1/MODULE.bazel": "fa92e2eb41a04df73cdabeec37107316f7e5272650f81d6cc096418fe647b915", - "https://bcr.bazel.build/modules/abseil-cpp/20240116.0/MODULE.bazel": "98dc378d64c12a4e4741ad3362f87fb737ee6a0886b2d90c3cdbb4d93ea3e0bf", "https://bcr.bazel.build/modules/abseil-cpp/20240116.1/MODULE.bazel": "37bcdb4440fbb61df6a1c296ae01b327f19e9bb521f9b8e26ec854b6f97309ed", "https://bcr.bazel.build/modules/abseil-cpp/20240116.2/MODULE.bazel": "73939767a4686cd9a520d16af5ab440071ed75cec1a876bf2fcfaf1f71987a16", "https://bcr.bazel.build/modules/abseil-cpp/20240722.0.bcr.2/MODULE.bazel": "c3661b44c9d3f17f0b65ffb544896aaeb89127398ea867537babac18133a002a", @@ -238,6 +239,8 @@ "https://bcr.bazel.build/modules/buildifier_prebuilt/7.3.1/source.json": "55153a5e6ca9c8a7e266c4b46b951e8a010d25ec6062bc35d5d4f89925796bad", "https://bcr.bazel.build/modules/buildozer/7.1.2/MODULE.bazel": "2e8dd40ede9c454042645fd8d8d0cd1527966aa5c919de86661e62953cd73d84", "https://bcr.bazel.build/modules/buildozer/7.1.2/source.json": "c9028a501d2db85793a6996205c8de120944f50a0d570438fcae0457a5f9d1f8", + "https://bcr.bazel.build/modules/fmt/11.0.2/MODULE.bazel": "d9e0ea6f68a762a84055172ce8f55cc01da950c94b5e27f66c1acc14e6fc59ce", + "https://bcr.bazel.build/modules/fmt/11.0.2/source.json": "36cde5a3cbbee5eefc1e7f8a5a66d8ac9f352f7742bdd0c9b2d869601169c881", "https://bcr.bazel.build/modules/google_benchmark/1.8.2/MODULE.bazel": "a70cf1bba851000ba93b58ae2f6d76490a9feb74192e57ab8e8ff13c34ec50cb", "https://bcr.bazel.build/modules/googletest/1.11.0/MODULE.bazel": "3a83f095183f66345ca86aa13c58b59f9f94a2f81999c093d4eeaa2d262d12f4", "https://bcr.bazel.build/modules/googletest/1.14.0.bcr.1/MODULE.bazel": "22c31a561553727960057361aa33bf20fb2e98584bc4fec007906e27053f80c6", @@ -285,6 +288,7 @@ "https://bcr.bazel.build/modules/re2/2024-07-02/MODULE.bazel": "0eadc4395959969297cbcf31a249ff457f2f1d456228c67719480205aa306daa", "https://bcr.bazel.build/modules/rules_android/0.1.1/MODULE.bazel": "48809ab0091b07ad0182defb787c4c5328bd3a278938415c00a7b69b50c4d3a8", "https://bcr.bazel.build/modules/rules_android/0.1.1/source.json": "e6986b41626ee10bdc864937ffb6d6bf275bb5b9c65120e6137d56e6331f089e", + "https://bcr.bazel.build/modules/rules_apple/3.13.0/MODULE.bazel": "b4559a2c6281ca3165275bb36c1f0ac74666632adc5bdb680e366de7ce845f43", "https://bcr.bazel.build/modules/rules_apple/3.22.0/MODULE.bazel": "0fd8c630d6cecc18abe84114ee21c1a0fb759b45063d3d2600ec22669e93ea43", "https://bcr.bazel.build/modules/rules_apple/3.22.0/source.json": "ee41f63c292bd876fd264a830cf45500619a063e2fa3a18cca6fa12b153d68e3", "https://bcr.bazel.build/modules/rules_cc/0.0.1/MODULE.bazel": "cb2aa0747f84c6c3a78dad4e2049c154f08ab9d166b1273835a8174940365647", @@ -373,6 +377,7 @@ "https://bcr.bazel.build/modules/rules_shell/0.4.1/MODULE.bazel": "00e501db01bbf4e3e1dd1595959092c2fadf2087b2852d3f553b5370f5633592", "https://bcr.bazel.build/modules/rules_shell/0.4.1/source.json": "4757bd277fe1567763991c4425b483477bb82e35e777a56fd846eb5cceda324a", "https://bcr.bazel.build/modules/rules_swift/1.16.0/MODULE.bazel": "4a09f199545a60d09895e8281362b1ff3bb08bbde69c6fc87aff5b92fcc916ca", + "https://bcr.bazel.build/modules/rules_swift/2.1.1/MODULE.bazel": "494900a80f944fc7aa61500c2073d9729dff0b764f0e89b824eb746959bc1046", "https://bcr.bazel.build/modules/rules_swift/2.4.0/MODULE.bazel": "1639617eb1ede28d774d967a738b4a68b0accb40650beadb57c21846beab5efd", "https://bcr.bazel.build/modules/rules_swift/2.4.0/source.json": "a6577f57f9febbdc015a01f2a8f3487422032f134d6c61d18ed8e8ca3b9acc7c", "https://bcr.bazel.build/modules/rules_uv/0.65.0/MODULE.bazel": "a700505a80ad0ad65f062c9335458d137f504b98c5937b338807bb5cbe305e73", diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index 8314a307..ee9c852f 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -51,6 +51,7 @@ pub const LlamaLM = struct { .freq_base = config.rope_theta, .scaling = config.rope_scaling, }; + self.model.norm.eps = config.rms_norm_eps; for (self.model.layers) |*layer| { layer.self_attn.num_heads = self.model.num_heads; layer.self_attn.num_kv_heads = self.model.num_kv_heads; diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 0b121e65..d8148e85 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -216,16 +216,19 @@ pub fn asyncMain() !void { var context = try zml.Context.init(); defer context.deinit(); - const compilation_options = zml.CompilationOptions{ - .xla_dump_to = "/tmp/zml/llama", - .sharding_enabled = res.args.sharding orelse true, - }; - // initialize ZML platform with optional create options // eg: --create-options='{"cuda":{"allocator":{"bfc":{"memory_fraction": 0.99}}}}' const create_opts_json = res.args.@"create-options" orelse "{}"; const create_opts = try std.json.parseFromSlice(zml.Platform.CreateOptions, allocator, create_opts_json, .{}); - const platform = context.autoPlatform(create_opts.value).withCompilationOptions(compilation_options); + + const platform = blk: { + var auto_platform = context.autoPlatform(create_opts.value); + const compilation_options: zml.CompilationOptions = if (auto_platform.target == .mlx) .{} else .{ + .xla_dump_to = "/tmp/zml/llama", + .sharding_enabled = res.args.sharding orelse true, + }; + break :blk auto_platform.withCompilationOptions(compilation_options); + }; create_opts.deinit(); context.printAvailablePlatforms(platform); diff --git a/runtimes/BUILD.bazel b/runtimes/BUILD.bazel index d0a1bcdb..ab0792ce 100644 --- a/runtimes/BUILD.bazel +++ b/runtimes/BUILD.bazel @@ -6,6 +6,7 @@ load("@rules_zig//zig:defs.bzl", "zig_library") RUNTIMES = { "cpu": True, "cuda": False, + "mlx": False, "rocm": False, "tpu": False, "neuron": False, @@ -48,6 +49,9 @@ filegroup( }) + select({ ":cuda.enabled": ["//runtimes/cuda:layers"], "//conditions:default": [], + }) + select({ + ":mlx.enabled": ["//runtimes/mlx:layers"], + "//conditions:default": [], }) + select({ ":neuron.enabled": ["//runtimes/neuron:layers"], "//conditions:default": [], diff --git a/runtimes/mlx/BUILD.bazel b/runtimes/mlx/BUILD.bazel new file mode 100644 index 00000000..f54f21c4 --- /dev/null +++ b/runtimes/mlx/BUILD.bazel @@ -0,0 +1,36 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +cc_import( + name = "libpjrt_mlx_bindings", + shared_library = "@xla//xla/pjrt/plugin/mlx:stablehlo_mlx_plugin.so", + visibility = ["//visibility:public"], +) + +cc_library( + name = "empty", +) + +cc_library( + name = "libpjrt_mlx", + defines = ["ZML_RUNTIME_MLX"], + deps = [":libpjrt_mlx_bindings"], +) + +zig_library( + name = "mlx", + import_name = "runtimes/mlx", + main = "mlx.zig", + visibility = ["//visibility:public"], + deps = [ + "//pjrt", + ] + select({ + "//runtimes:mlx.enabled": [":libpjrt_mlx"], + "//conditions:default": [":empty"], + }), +) + +filegroup( + name = "layers", + srcs = [], + visibility = ["//visibility:public"], +) diff --git a/runtimes/mlx/mlx.zig b/runtimes/mlx/mlx.zig new file mode 100644 index 00000000..1af0cffa --- /dev/null +++ b/runtimes/mlx/mlx.zig @@ -0,0 +1,18 @@ +const builtin = @import("builtin"); +const asynk = @import("async"); +const pjrt = @import("pjrt"); +const c = @import("c"); + +pub fn isEnabled() bool { + return @hasDecl(c, "ZML_RUNTIME_MLX"); +} + +pub fn load() !*const pjrt.Api { + if (comptime !isEnabled()) { + return error.Unavailable; + } + if (comptime builtin.os.tag != .macos) { + return error.Unavailable; + } + return pjrt.Api.loadFrom("stablehlo_mlx_plugin.so"); +} diff --git a/runtimes/runtimes.zig b/runtimes/runtimes.zig index 7319da76..d9496f0d 100644 --- a/runtimes/runtimes.zig +++ b/runtimes/runtimes.zig @@ -1,6 +1,7 @@ const pjrt = @import("pjrt"); const cpu = @import("runtimes/cpu"); const cuda = @import("runtimes/cuda"); +const mlx = @import("runtimes/mlx"); const rocm = @import("runtimes/rocm"); const tpu = @import("runtimes/tpu"); const neuron = @import("runtimes/neuron"); @@ -8,6 +9,7 @@ const neuron = @import("runtimes/neuron"); pub const Platform = enum { cpu, cuda, + mlx, rocm, tpu, neuron, @@ -17,6 +19,7 @@ pub fn load(tag: Platform) !*const pjrt.Api { return switch (tag) { .cpu => try cpu.load(), .cuda => try cuda.load(), + .mlx => try mlx.load(), .rocm => try rocm.load(), .tpu => try tpu.load(), .neuron => try neuron.load(), @@ -27,6 +30,7 @@ pub fn isEnabled(tag: Platform) bool { return switch (tag) { .cpu => cpu.isEnabled(), .cuda => cuda.isEnabled(), + .mlx => mlx.isEnabled(), .rocm => rocm.isEnabled(), .tpu => tpu.isEnabled(), .neuron => neuron.isEnabled(), diff --git a/third_party/modules/mlx/20250518.0-eebe730/MODULE.bazel b/third_party/modules/mlx/20250518.0-eebe730/MODULE.bazel new file mode 100644 index 00000000..c24f5489 --- /dev/null +++ b/third_party/modules/mlx/20250518.0-eebe730/MODULE.bazel @@ -0,0 +1,10 @@ +module( + name = "mlx", + version = "20250518.0-eebe730", + compatibility_level = 1, +) + +bazel_dep(name = "rules_apple", version = "3.22.0", repo_name = "build_bazel_rules_apple") +bazel_dep(name = "fmt", version = "11.2.0") +metal_cpp = use_extension("//:metal_cpp.bzl", "metal_cpp") +use_repo(metal_cpp, "metal_cpp") diff --git a/third_party/modules/mlx/20250518.0-eebe730/overlay/BUILD.bazel b/third_party/modules/mlx/20250518.0-eebe730/overlay/BUILD.bazel new file mode 100644 index 00000000..5a664afe --- /dev/null +++ b/third_party/modules/mlx/20250518.0-eebe730/overlay/BUILD.bazel @@ -0,0 +1,592 @@ +load("@build_bazel_rules_apple//apple:resources.bzl", "apple_metal_library") +load("//:jit.bzl", "make_jit_source") + +filegroup( + name = "mlx_h", + srcs = glob(["mlx/*.h"], ["mlx/mlx.h"]), +) + +# Contains `mlx/types/*.h` headers +cc_library( + name = "types", + hdrs = glob(["mlx/types/*.h"]), +) + +cc_library( + name = "pocketfft", + hdrs = [ + "mlx/3rdparty/pocketfft.h", + ], +) + +# mlx/distributed/ring/CMakeLists.txt +cc_library( + name = "ring", + srcs = [ + "mlx/distributed/ring/no_ring.cpp", + + # Other sources (maybe reorganize dependencies?) + ":mlx_h", + "mlx/distributed/distributed.h", + ], + hdrs = [ + "mlx/distributed/ring/ring.h", + ], + deps = [ + ":types" + ], +) + +# mlx/distributed/mpi/CMakeLists.txt +cc_library( + name = "mpi", + srcs = [ + "mlx/distributed/mpi/no_mpi.cpp", + # Other sources (maybe reorganize dependencies?) + "mlx/distributed/distributed.h", + ":mlx_h", + ], + hdrs = [ + "mlx/distributed/mpi/mpi.h", + ], + deps = [ + ":types", + ] +) + +# mlx/distributed/CMakeLists.txt +cc_library( + name = "distributed", + srcs = [ + "mlx/distributed/primitives.cpp", + "mlx/distributed/ops.cpp", + "mlx/distributed/distributed.cpp", + + # Other sources (maybe reorganize dependencies?) + ":mlx_h", + ], + hdrs = [ + "mlx/distributed/distributed_impl.h", + "mlx/distributed/distributed.h", + "mlx/distributed/ops.h", + "mlx/distributed/primitives.h", + ], + deps = [ + ":io", + ":mpi", + ":ring", + ":types", + ] +) + +# mlx/io/CMakeLists.txt +cc_library( + name = "io", + srcs = [ + "mlx/io/load.cpp", + "mlx/io/no_safetensors.cpp", + "mlx/io/no_gguf.cpp", + + # Other sources (maybe reorganize dependencies?) + ":mlx_h", + ], + hdrs = [ + "mlx/io/load.h", + ], + deps = [ + ":types", + ], +) + +# mlx/backend/common/CMakeLists.txt +cc_library( + name = "backend_common", + srcs = [ + "mlx/backend/common/broadcasting.cpp", + "mlx/backend/common/compiled.cpp", + "mlx/backend/common/common.cpp", + "mlx/backend/common/load.cpp", + "mlx/backend/common/reduce.cpp", + "mlx/backend/common/slicing.cpp", + "mlx/backend/common/utils.cpp", + # Other sources (maybe reorganize dependencies?) + "mlx/backend/gpu/eval.h", + ":mlx_h", + ], + hdrs = [ + "mlx/backend/common/binary.h", + "mlx/backend/common/broadcasting.h", + "mlx/backend/common/compiled.h", + "mlx/backend/common/copy.h", + "mlx/backend/common/hadamard.h", + "mlx/backend/common/reduce.h", + "mlx/backend/common/slicing.h", + "mlx/backend/common/ternary.h", + "mlx/backend/common/utils.h", + ], + deps = [ + + ":io", + ":types", + ], +) + +# mlx/backend/cpu/CMakeLists.txt +genrule( + name = "compiled_preamble", + visibility = ["//visibility:public"], + srcs = ["mlx/backend/cpu/compiled_preamble.h"], + outs = ["mlx/backend/cpu/compiled_preamable.cpp"], + cmd = "bash $(location :mlx/backend/cpu/make_compiled_preamble.sh) $@ $(location mlx/backend/cpu/compiled_preamble.h)", + tools = [":mlx/backend/cpu/make_compiled_preamble.sh"], +) + +# TODO (@cryptodeal): finish bazel config for cpu backend +cc_library( + name = "backend_cpu", + linkopts = [ + "-framework", + "Accelerate", + ], + defines = [ + "MLX_USE_ACCELERATE", + "ACCELERATE_NEW_LAPACK" + ], + srcs = [ + "mlx/backend/cpu/arg_reduce.cpp", + "mlx/backend/cpu/binary.cpp", + "mlx/backend/cpu/conv.cpp", + "mlx/backend/cpu/copy.cpp", + "mlx/backend/cpu/eigh.cpp", + "mlx/backend/cpu/fft.cpp", + "mlx/backend/cpu/hadamard.cpp", + "mlx/backend/cpu/matmul.cpp", + "mlx/backend/cpu/gemms/cblas.cpp", + "mlx/backend/cpu/masked_mm.cpp", + "mlx/backend/cpu/primitives.cpp", + "mlx/backend/cpu/quantized.cpp", + "mlx/backend/cpu/reduce.cpp", + "mlx/backend/cpu/scan.cpp", + "mlx/backend/cpu/select.cpp", + "mlx/backend/cpu/softmax.cpp", + "mlx/backend/cpu/sort.cpp", + "mlx/backend/cpu/threefry.cpp", + "mlx/backend/cpu/indexing.cpp", + "mlx/backend/cpu/luf.cpp", + "mlx/backend/cpu/qrf.cpp", + "mlx/backend/cpu/svd.cpp", + "mlx/backend/cpu/inverse.cpp", + "mlx/backend/cpu/cholesky.cpp", + "mlx/backend/cpu/unary.cpp", + # Build with Accelerate + "mlx/backend/cpu/gemms/bnns.cpp", + # Not Building for IOS + "mlx/backend/cpu/compiled.cpp", + "mlx/backend/cpu/jit_compiler.cpp", + # Additional Header Sources + "mlx/backend/cpu/simd/simd.h", + "mlx/backend/cpu/simd/math.h", + "mlx/backend/cpu/simd/type.h", + "mlx/backend/cpu/simd/base_simd.h", + "mlx/backend/cpu/simd/accelerate_simd.h", + "mlx/backend/cpu/simd/accelerate_fp16_simd.h", + # generated `compiled_preamble.cpp` + ":compiled_preamble", + + # Other sources (maybe reorganize dependencies?) + ":mlx_h", + "mlx/io/load.h", + ], + hdrs = [ + "mlx/backend/cpu/arange.h", + "mlx/backend/cpu/binary_ops.h", + "mlx/backend/cpu/binary_two.h", + "mlx/backend/cpu/binary.h", + "mlx/backend/cpu/compiled_preamble.h", + "mlx/backend/cpu/copy.h", + "mlx/backend/cpu/gemm.h", + "mlx/backend/cpu/jit_compiler.h", + "mlx/backend/cpu/lapack.h", + "mlx/backend/cpu/slicing.h", + "mlx/backend/cpu/ternary.h", + "mlx/backend/cpu/threefry.h", + "mlx/backend/cpu/unary_ops.h", + "mlx/backend/cpu/unary.h", + ], + deps = [ + "@fmt", + ":backend_common", + ":pocketfft", + ":types", + ], +) + +# mlx/backend/no_cpu/CMakeLists.txt +cc_library( + name = "backend_no_cpu", + srcs = [ + "mlx/backend/no_cpu/available.cpp", + "mlx/backend/no_cpu/primitives.cpp", + "mlx/backend/cpu/eval.cpp", + "mlx/backend/cpu/encoder.cpp", + "mlx/backend/no_cpu/compiled.cpp", + # Other sources (maybe reorganize dependencies?) + "mlx/backend/cpu/available.h", + "mlx/backend/cpu/encoder.h", + "mlx/backend/cpu/eval.h", + "mlx/backend/gpu/eval.h", + ":mlx_h", + ], + deps = [ + ":distributed", + ":io", + # ":backend_common", + ":types", + ], +) + +# mlx/backend/metal/CMakeLists.txt +make_jit_source( + name = "utils", + deps = [ + "mlx/backend/metal/kernels/jit/bf16.h", + "mlx/backend/metal/kernels/metal_3_0/bf16.h", + "mlx/backend/metal/kernels/metal_3_1/bf16.h", + "mlx/backend/metal/kernels/bf16_math.h", + "mlx/backend/metal/kernels/complex.h", + "mlx/backend/metal/kernels/defines.h", + ], +) + +make_jit_source( + name = "unary_ops", + deps = [ + "mlx/backend/metal/kernels/erf.h", + "mlx/backend/metal/kernels/expm1f.h", + ], +) + +make_jit_source(name = "binary_ops") + +make_jit_source(name = "ternary_ops") + +make_jit_source( + name = "reduce_utils", + deps = [ + "mlx/backend/metal/kernels/atomic.h", + "mlx/backend/metal/kernels/reduction/ops.h" + ], +) + +make_jit_source( + name = "scatter", + deps = [ + "mlx/backend/metal/kernels/indexing.h", + ], +) + +make_jit_source( + name = "gather", + deps = [ + "mlx/backend/metal/kernels/indexing.h", + ], +) + +make_jit_source(name = "gather_axis") + +make_jit_source(name = "scatter_axis") + +make_jit_source(name = "hadamard") + +cc_library( + name = "backend_metal", + linkopts = [ + "-framework Metal", + "-framework Foundation", + "-framework QuartzCore", + ], + # defines = ["MLX_METAL_DEBUG"], + srcs = [ + "mlx/backend/metal/allocator.cpp", + "mlx/backend/metal/binary.cpp", + "mlx/backend/metal/compiled.cpp", + "mlx/backend/metal/conv.cpp", + "mlx/backend/metal/copy.cpp", + "mlx/backend/metal/custom_kernel.cpp", + "mlx/backend/metal/distributed.cpp", + "mlx/backend/metal/device.cpp", + "mlx/backend/metal/event.cpp", + "mlx/backend/metal/eval.cpp", + "mlx/backend/metal/fence.cpp", + "mlx/backend/metal/fft.cpp", + "mlx/backend/metal/hadamard.cpp", + "mlx/backend/metal/indexing.cpp", + "mlx/backend/metal/logsumexp.cpp", + "mlx/backend/metal/matmul.cpp", + "mlx/backend/metal/scaled_dot_product_attention.cpp", + "mlx/backend/metal/metal.cpp", + "mlx/backend/metal/primitives.cpp", + "mlx/backend/metal/quantized.cpp", + "mlx/backend/metal/normalization.cpp", + "mlx/backend/metal/rope.cpp", + "mlx/backend/metal/scan.cpp", + "mlx/backend/metal/slicing.cpp", + "mlx/backend/metal/softmax.cpp", + "mlx/backend/metal/sort.cpp", + "mlx/backend/metal/reduce.cpp", + "mlx/backend/metal/ternary.cpp", + "mlx/backend/metal/unary.cpp", + "mlx/backend/metal/resident.cpp", + "mlx/backend/metal/utils.cpp", + "mlx/backend/metal/nojit_kernels.cpp", + # Additional Header Sources + "mlx/backend/metal/jit/indexing.h", + "mlx/backend/metal/kernels/defines.h", + "mlx/backend/metal/kernels/steel/conv/params.h", + "mlx/backend/metal/kernels/steel/gemm/params.h", + "mlx/backend/metal/kernels/steel/attn/params.h", + "mlx/backend/metal/jit/includes.h", + # Generate METAL_PATH define + ":generate_metal_path", + # Generated sources + ":utils", + ":unary_ops", + ":binary_ops", + ":ternary_ops", + ":reduce_utils", + ":scatter", + ":gather", + ":gather_axis", + ":scatter_axis", + ":hadamard", + + # Other sources (maybe reorganize dependencies?) + ":mlx_h", + ], + hdrs = [ + "mlx/backend/metal/allocator.h", + "mlx/backend/metal/binary.h", + "mlx/backend/metal/device.h", + "mlx/backend/metal/kernels.h", + "mlx/backend/metal/matmul.h", + "mlx/backend/metal/metal.h", + "mlx/backend/metal/reduce.h", + "mlx/backend/metal/resident.h", + "mlx/backend/metal/ternary.h", + "mlx/backend/metal/unary.h", + "mlx/backend/metal/utils.h", + ], + deps = [ + "@metal_cpp//:metal_cpp", + "@fmt", + ":backend_common", + ":backend_gpu", + ":distributed", + ":pocketfft", + ], +) + +genrule( + name = "generate_metal_path", + outs = [ + "mlx/backend/metal/metal_path.h", + ], + cmd = "echo \"#define METAL_PATH \\\"`realpath $(location //:mlx_metallib)`\\\"\" > $@", + tools = [ + "//:mlx_metallib", + ], +) + +# TODO (@cryptodeal): find a better solution than `copts` includes +apple_metal_library( + name = "mlx_metallib", + visibility = ["//visibility:public"], + copts = [ + "-target", + "air64-apple-macosx15.4", + "-Iexternal/mlx+", + "-Iexternal/mlx+/mlx/backend/metal/kernels/metal_3_1", + # "-I.", + # "-Imlx/backend/metal/kernels/metal_3_1", + "-Wall", + "-Wextra", + "-fno-fast-math", + "-Wno-c++17-extensions", + # "-gline-tables-only", + # "-frecord-sources", + ], + srcs = [ + "mlx/backend/metal/kernels/arg_reduce.metal", + "mlx/backend/metal/kernels/conv.metal", + "mlx/backend/metal/kernels/gemv.metal", + "mlx/backend/metal/kernels/layer_norm.metal", + "mlx/backend/metal/kernels/random.metal", + "mlx/backend/metal/kernels/rms_norm.metal", + "mlx/backend/metal/kernels/rope.metal", + "mlx/backend/metal/kernels/scaled_dot_product_attention.metal", + "mlx/backend/metal/kernels/fence.metal", + "mlx/backend/metal/kernels/steel/attn/kernels/steel_attention.metal", + # Disable JIT kernels (for default ops) + "mlx/backend/metal/kernels/arange.metal", + "mlx/backend/metal/kernels/binary.metal", + "mlx/backend/metal/kernels/binary_two.metal", + "mlx/backend/metal/kernels/copy.metal", + "mlx/backend/metal/kernels/fft.metal", + "mlx/backend/metal/kernels/reduce.metal", + "mlx/backend/metal/kernels/quantized.metal", + "mlx/backend/metal/kernels/scan.metal", + "mlx/backend/metal/kernels/softmax.metal", + "mlx/backend/metal/kernels/sort.metal", + "mlx/backend/metal/kernels/ternary.metal", + "mlx/backend/metal/kernels/unary.metal", + "mlx/backend/metal/kernels/steel/conv/kernels/steel_conv.metal", + "mlx/backend/metal/kernels/steel/conv/kernels/steel_conv_general.metal", + "mlx/backend/metal/kernels/steel/gemm/kernels/steel_gemm_fused.metal", + "mlx/backend/metal/kernels/steel/gemm/kernels/steel_gemm_masked.metal", + "mlx/backend/metal/kernels/steel/gemm/kernels/steel_gemm_splitk.metal", + "mlx/backend/metal/kernels/gemv_masked.metal", + ], + hdrs = [ + "mlx/backend/metal/kernels/metal_3_1/bf16.h", + "mlx/backend/metal/kernels/metal_3_0/bf16.h", + "mlx/backend/metal/kernels/bf16_math.h", + "mlx/backend/metal/kernels/complex.h", + "mlx/backend/metal/kernels/defines.h", + "mlx/backend/metal/kernels/erf.h", + "mlx/backend/metal/kernels/expm1f.h", + "mlx/backend/metal/kernels/utils.h", + # `conv` headers + "mlx/backend/metal/kernels/steel/conv/params.h", + # `scaled_dot_product_attention` headers + "mlx/backend/metal/kernels/sdpa_vector.h", + # Steel Headers + "mlx/backend/metal/kernels/steel/defines.h", + "mlx/backend/metal/kernels/steel/utils.h", + "mlx/backend/metal/kernels/steel/conv/conv.h", + "mlx/backend/metal/kernels/steel/conv/loader.h", + "mlx/backend/metal/kernels/steel/conv/loaders/loader_channel_l.h", + "mlx/backend/metal/kernels/steel/conv/loaders/loader_channel_n.h", + "mlx/backend/metal/kernels/steel/conv/loaders/loader_general.h", + "mlx/backend/metal/kernels/steel/conv/kernels/steel_conv.h", + "mlx/backend/metal/kernels/steel/conv/kernels/steel_conv_general.h", + "mlx/backend/metal/kernels/steel/gemm/gemm.h", + "mlx/backend/metal/kernels/steel/gemm/mma.h", + "mlx/backend/metal/kernels/steel/gemm/loader.h", + "mlx/backend/metal/kernels/steel/gemm/params.h", + "mlx/backend/metal/kernels/steel/gemm/transforms.h", + "mlx/backend/metal/kernels/steel/gemm/kernels/steel_gemm_fused.h", + "mlx/backend/metal/kernels/steel/gemm/kernels/steel_gemm_masked.h", + "mlx/backend/metal/kernels/steel/gemm/kernels/steel_gemm_splitk.h", + "mlx/backend/metal/kernels/steel/utils/type_traits.h", + "mlx/backend/metal/kernels/steel/utils/integral_constant.h", + # Steel Attn Headers (not repeating previously listed) + "mlx/backend/metal/kernels/steel/attn/attn.h", + "mlx/backend/metal/kernels/steel/attn/loader.h", + "mlx/backend/metal/kernels/steel/attn/mma.h", + "mlx/backend/metal/kernels/steel/attn/params.h", + "mlx/backend/metal/kernels/steel/attn/transforms.h", + "mlx/backend/metal/kernels/steel/attn/kernels/steel_attention.h", + # `arange` headers + "mlx/backend/metal/kernels/arange.h", + # `binary` headers + "mlx/backend/metal/kernels/binary.h", + "mlx/backend/metal/kernels/binary_ops.h", + # `binary_two` headers + "mlx/backend/metal/kernels/binary_two.h", + # `copy` headers + "mlx/backend/metal/kernels/copy.h", + # `fft` headers + "mlx/backend/metal/kernels/fft.h", + "mlx/backend/metal/kernels/fft/radix.h", + "mlx/backend/metal/kernels/fft/readwrite.h", + # `reduce` headers + "mlx/backend/metal/kernels/reduce.h", + "mlx/backend/metal/kernels/atomic.h", + "mlx/backend/metal/kernels/reduction/ops.h", + "mlx/backend/metal/kernels/reduction/reduce_init.h", + "mlx/backend/metal/kernels/reduction/reduce_all.h", + "mlx/backend/metal/kernels/reduction/reduce_col.h", + "mlx/backend/metal/kernels/reduction/reduce_row.h", + # `quantized` headers + "mlx/backend/metal/kernels/quantized.h", + # `scan` headers + "mlx/backend/metal/kernels/scan.h", + # `softmax` headers + "mlx/backend/metal/kernels/softmax.h", + # `sort` headers + "mlx/backend/metal/kernels/sort.h", + # `ternary` headers + "mlx/backend/metal/kernels/ternary.h", + "mlx/backend/metal/kernels/ternary_ops.h", + # `unary` headers + "mlx/backend/metal/kernels/unary.h", + "mlx/backend/metal/kernels/unary_ops.h", + # `gemv` headers + "mlx/backend/metal/kernels/gemv_masked.h", + ], + out = "mlx.metallib", +) + +cc_library( + name = "backend_gpu", + srcs = [ + "mlx/backend/gpu/copy.cpp", + "mlx/backend/gpu/primitives.cpp", + "mlx/backend/gpu/slicing.cpp", + # Other sources (maybe reorganize dependencies?) + ":mlx_h", + ], + hdrs = [ + "mlx/backend/gpu/available.h", + "mlx/backend/gpu/copy.h", + "mlx/backend/gpu/eval.h", + "mlx/backend/gpu/slicing.h", + ], + deps = [ + ":backend_common", + ], +) + +cc_library( + name = "mlx", + visibility = ["//visibility:public"], + data = [ + ":mlx_metallib", + ], + srcs = [ + "mlx/allocator.cpp", + "mlx/array.cpp", + "mlx/compile.cpp", + "mlx/device.cpp", + "mlx/dtype.cpp", + "mlx/dtype_utils.cpp", + "mlx/export.cpp", + "mlx/einsum.cpp", + "mlx/fast.cpp", + "mlx/fft.cpp", + "mlx/ops.cpp", + "mlx/graph_utils.cpp", + "mlx/primitives.cpp", + "mlx/random.cpp", + "mlx/scheduler.cpp", + "mlx/transforms.cpp", + "mlx/utils.cpp", + "mlx/linalg.cpp", + "mlx/backend/metal/metal.h", + ":mlx_h", + ], + hdrs = [ + "mlx/mlx.h", + ], + deps = [ + ":io", + ":distributed", + ":backend_common", + ":backend_metal", + ":backend_gpu", + ":backend_no_cpu", + # TODO (@cryptodeal): enable cpu backend once changes to `sort`, and + # comparison ops have been made for parity with metal backend changes. + # ":backend_cpu", + ], +) diff --git a/third_party/modules/mlx/20250518.0-eebe730/overlay/MODULE.bazel b/third_party/modules/mlx/20250518.0-eebe730/overlay/MODULE.bazel new file mode 120000 index 00000000..9b599e3a --- /dev/null +++ b/third_party/modules/mlx/20250518.0-eebe730/overlay/MODULE.bazel @@ -0,0 +1 @@ +../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/mlx/20250518.0-eebe730/overlay/jit.bzl b/third_party/modules/mlx/20250518.0-eebe730/overlay/jit.bzl new file mode 100644 index 00000000..b8ba8aa0 --- /dev/null +++ b/third_party/modules/mlx/20250518.0-eebe730/overlay/jit.bzl @@ -0,0 +1,23 @@ +def make_jit_source(name, deps = [], **kwargs): + """ + Bazel equivalent of the CMake make_jit_source function. + Generates a C++ source file from a Metal header file by running a script. + + Args: + name: The Metal header file to process. + deps: Additional dependencies required for processing. + kwargs: Additional arguments to pass to the native.genrule function. + """ + src_name = name.split("/")[-1] # Extract the filename + output_file = "mlx/backend/metal/jit/{}.cpp".format(src_name) + + native.genrule( + name = name, + srcs = ["mlx/backend/metal/kernels/{}.h".format(src_name)] + deps, + outs = [output_file], + cmd = ( + "bash $(location :mlx/backend/metal/make_compiled_preamble.sh) $@ $(location mlx/backend/metal/kernels/{}.h)" + ).format(name), + tools = [":mlx/backend/metal/make_compiled_preamble.sh"], + **kwargs, + ) diff --git a/third_party/modules/mlx/20250518.0-eebe730/overlay/metal_cpp.bzl b/third_party/modules/mlx/20250518.0-eebe730/overlay/metal_cpp.bzl new file mode 100644 index 00000000..64874afe --- /dev/null +++ b/third_party/modules/mlx/20250518.0-eebe730/overlay/metal_cpp.bzl @@ -0,0 +1,23 @@ +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +def _metal_cpp_impl(mctx): + http_archive( + name = "metal_cpp", + build_file_content = """\ +cc_library( + name = "metal_cpp", + hdrs = glob(["metal-cpp/**/*.hpp"]), + strip_include_prefix = "metal-cpp", + visibility = ["//visibility:public"], +) +""", + sha256 = "0433df1e0ab13c2b0becbd78665071e3fa28381e9714a3fce28a497892b8a184", + url = "https://developer.apple.com/metal/cpp/files/metal-cpp_macOS15_iOS18.zip", + ) + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = "all", + root_module_direct_dev_deps = [], + ) +metal_cpp = module_extension( + implementation = _metal_cpp_impl, +) diff --git a/third_party/modules/mlx/20250518.0-eebe730/patches/0001-Modify-Sort-And-Comparison-Kernels.patch b/third_party/modules/mlx/20250518.0-eebe730/patches/0001-Modify-Sort-And-Comparison-Kernels.patch new file mode 100644 index 00000000..dae2293b --- /dev/null +++ b/third_party/modules/mlx/20250518.0-eebe730/patches/0001-Modify-Sort-And-Comparison-Kernels.patch @@ -0,0 +1,1132 @@ +diff --git a/.gitignore b/.gitignore +index 43629548..0b74c60d 100644 +--- a/.gitignore ++++ b/.gitignore +@@ -1,3 +1,8 @@ ++# Bazel symlinks ++bazel-* ++ ++# macOS ++.DS_Store + # Byte-compiled / optimized / DLL files + __pycache__/ + *.py[cod] +diff --git a/mlx/backend/metal/device.cpp b/mlx/backend/metal/device.cpp +index ebc3cc77..7bdca47d 100644 +--- a/mlx/backend/metal/device.cpp ++++ b/mlx/backend/metal/device.cpp +@@ -10,6 +10,7 @@ + + #include "mlx/backend/metal/device.h" + #include "mlx/backend/metal/metal.h" ++#include "mlx/backend/metal/metal_path.h" + #include "mlx/backend/metal/utils.h" + #include "mlx/utils.h" + +diff --git a/mlx/backend/metal/jit_kernels.cpp b/mlx/backend/metal/jit_kernels.cpp +index 5206c9b5..6678e7ac 100644 +--- a/mlx/backend/metal/jit_kernels.cpp ++++ b/mlx/backend/metal/jit_kernels.cpp +@@ -328,7 +328,8 @@ MTL::ComputePipelineState* get_sort_kernel( + const array& in, + const array& out, + int bn, +- int tn) { ++ int tn, ++ const std::string& comparator) { + std::string lib_name = kernel_name.substr(kernel_name.find("_") + 1); + auto lib = d.get_library(lib_name, [&]() { + std::ostringstream kernel_source; +@@ -345,7 +346,8 @@ MTL::ComputePipelineState* get_sort_kernel( + out_type, + bool_string, + bn, +- tn); ++ tn, ++ comparator); + kernel_source << get_template_definition( + "n" + func_string + lib_name, + "block_sort_nc", +@@ -353,7 +355,8 @@ MTL::ComputePipelineState* get_sort_kernel( + out_type, + bool_string, + bn, +- tn); ++ tn, ++ comparator); + } + return kernel_source.str(); + }); +@@ -366,7 +369,8 @@ MTL::ComputePipelineState* get_mb_sort_kernel( + const array& in, + const array& idx, + int bn, +- int tn) { ++ int tn, ++ const std::string& comparator) { + std::string lib_name = kernel_name.substr(kernel_name.find("_") + 1); + auto lib = d.get_library(lib_name, [&]() { + std::ostringstream kernel_source; +@@ -383,7 +387,8 @@ MTL::ComputePipelineState* get_mb_sort_kernel( + get_type_string(idx.dtype()), + "true", + bn, +- tn); ++ tn, ++ comparator); + } + return kernel_source.str(); + }); +diff --git a/mlx/backend/metal/kernels.h b/mlx/backend/metal/kernels.h +index 6d886438..ff2dd14b 100644 +--- a/mlx/backend/metal/kernels.h ++++ b/mlx/backend/metal/kernels.h +@@ -79,7 +79,8 @@ MTL::ComputePipelineState* get_sort_kernel( + const array& in, + const array& out, + int bn, +- int tn); ++ int tn, ++ const std::string& comparator); + + MTL::ComputePipelineState* get_mb_sort_kernel( + metal::Device& d, +@@ -87,7 +88,8 @@ MTL::ComputePipelineState* get_mb_sort_kernel( + const array& in, + const array& idx, + int bn, +- int tn); ++ int tn, ++ const std::string& comparator); + + MTL::ComputePipelineState* get_reduce_init_kernel( + metal::Device& d, +diff --git a/mlx/backend/metal/kernels/binary_ops.h b/mlx/backend/metal/kernels/binary_ops.h +index 4aaf2b4d..2c6e27b1 100644 +--- a/mlx/backend/metal/kernels/binary_ops.h ++++ b/mlx/backend/metal/kernels/binary_ops.h +@@ -37,29 +37,14 @@ struct Divide { + return x / y; + } + }; +- + struct Remainder { + template +- metal::enable_if_t & !metal::is_signed_v, T> +- operator()(T x, T y) { ++ metal::enable_if_t, T> operator()(T x, T y) { + return x % y; + } + template +- metal::enable_if_t & metal::is_signed_v, T> +- operator()(T x, T y) { +- auto r = x % y; +- if (r != 0 && (r < 0 != y < 0)) { +- r += y; +- } +- return r; +- } +- template + metal::enable_if_t, T> operator()(T x, T y) { +- T r = fmod(x, y); +- if (r != 0 && (r < 0 != y < 0)) { +- r += y; +- } +- return r; ++ return fmod(x, y); + } + template <> + complex64_t operator()(complex64_t x, complex64_t y) { +diff --git a/mlx/backend/metal/kernels/complex.h b/mlx/backend/metal/kernels/complex.h +index c88002cb..54fc8d52 100644 +--- a/mlx/backend/metal/kernels/complex.h ++++ b/mlx/backend/metal/kernels/complex.h +@@ -142,11 +142,12 @@ constexpr complex64_t operator/(float a, complex64_t b) { + constexpr complex64_t operator%(complex64_t a, complex64_t b) { + auto real = a.real - (b.real * static_cast(a.real / b.real)); + auto imag = a.imag - (b.imag * static_cast(a.imag / b.imag)); +- if (real != 0 && (real < 0 != b.real < 0)) { +- real += b.real; +- } +- if (imag != 0 && (imag < 0 != b.imag < 0)) { +- imag += b.imag; +- } ++ /* Disable to match `stablehlo` implementation semantics */ ++ // if (real != 0 && (real < 0 != b.real < 0)) { ++ // real += b.real; ++ // } ++ // if (imag != 0 && (imag < 0 != b.imag < 0)) { ++ // imag += b.imag; ++ // } + return {real, imag}; + } +diff --git a/mlx/backend/metal/kernels/sort.h b/mlx/backend/metal/kernels/sort.h +index b067150d..a24e429f 100644 +--- a/mlx/backend/metal/kernels/sort.h ++++ b/mlx/backend/metal/kernels/sort.h +@@ -28,6 +28,15 @@ struct LessThan { + } + }; + ++template ++struct GreaterThan { ++ static constexpr constant T init = Limits::min; ++ ++ METAL_FUNC bool operator()(T a, T b) { ++ return a > b; ++ } ++}; ++ + template < + typename ValT, + typename IdxT, +@@ -216,7 +225,7 @@ template < + bool ARG_SORT, + short BLOCK_THREADS, + short N_PER_THREAD, +- typename CompareOp = LessThan> ++ typename CompareOp> + struct KernelMergeSort { + using ValT = T; + using IdxT = uint; +@@ -278,7 +287,8 @@ template < + typename U, + bool ARG_SORT, + short BLOCK_THREADS, +- short N_PER_THREAD> ++ short N_PER_THREAD, ++ typename CompareOp = LessThan> + [[kernel, max_total_threads_per_threadgroup(BLOCK_THREADS)]] void block_sort( + const device T* inp [[buffer(0)]], + device U* out [[buffer(1)]], +@@ -290,7 +300,7 @@ template < + uint3 tid [[threadgroup_position_in_grid]], + uint3 lid [[thread_position_in_threadgroup]]) { + using sort_kernel = +- KernelMergeSort; ++ KernelMergeSort; + using ValT = typename sort_kernel::ValT; + using IdxT = typename sort_kernel::IdxT; + +@@ -333,7 +343,8 @@ template < + typename U, + bool ARG_SORT, + short BLOCK_THREADS, +- short N_PER_THREAD> ++ short N_PER_THREAD, ++ typename CompareOp> + [[kernel, max_total_threads_per_threadgroup(BLOCK_THREADS)]] void block_sort_nc( + const device T* inp [[buffer(0)]], + device U* out [[buffer(1)]], +@@ -347,7 +358,7 @@ template < + uint3 tid [[threadgroup_position_in_grid]], + uint3 lid [[thread_position_in_threadgroup]]) { + using sort_kernel = +- KernelMergeSort; ++ KernelMergeSort; + using ValT = typename sort_kernel::ValT; + using IdxT = typename sort_kernel::IdxT; + +@@ -394,7 +405,7 @@ template < + bool ARG_SORT, + short BLOCK_THREADS, + short N_PER_THREAD, +- typename CompareOp = LessThan> ++ typename CompareOp> + struct KernelMultiBlockMergeSort { + using block_merge_sort_t = BlockMergeSort< + ValT, +@@ -476,7 +487,8 @@ template < + typename IdxT, + bool ARG_SORT, + short BLOCK_THREADS, +- short N_PER_THREAD> ++ short N_PER_THREAD, ++ typename CompareOp> + [[kernel, max_total_threads_per_threadgroup(BLOCK_THREADS)]] void mb_block_sort( + const device ValT* inp [[buffer(0)]], + device ValT* out_vals [[buffer(1)]], +@@ -493,7 +505,8 @@ template < + IdxT, + ARG_SORT, + BLOCK_THREADS, +- N_PER_THREAD>; ++ N_PER_THREAD, ++ CompareOp>; + + auto block_idx = elem_to_loc(tid.y, nc_shape, nc_strides, nc_dim); + inp += block_idx; +@@ -520,7 +533,8 @@ template < + typename IdxT, + bool ARG_SORT, + short BLOCK_THREADS, +- short N_PER_THREAD> ++ short N_PER_THREAD, ++ typename CompareOp> + [[kernel]] void mb_block_partition( + device IdxT* block_partitions [[buffer(0)]], + const device ValT* dev_vals [[buffer(1)]], +@@ -536,7 +550,8 @@ template < + IdxT, + ARG_SORT, + BLOCK_THREADS, +- N_PER_THREAD>; ++ N_PER_THREAD, ++ CompareOp>; + + block_partitions += tid.y * tgp_dims.x; + dev_vals += tid.y * size_sorted_axis; +@@ -573,7 +588,7 @@ template < + bool ARG_SORT, + short BLOCK_THREADS, + short N_PER_THREAD, +- typename CompareOp = LessThan> ++ typename CompareOp> + [[kernel, max_total_threads_per_threadgroup(BLOCK_THREADS)]] void + mb_block_merge( + const device IdxT* block_partitions [[buffer(0)]], +diff --git a/mlx/backend/metal/kernels/sort.metal b/mlx/backend/metal/kernels/sort.metal +index 7f198d55..f1ec796c 100644 +--- a/mlx/backend/metal/kernels/sort.metal ++++ b/mlx/backend/metal/kernels/sort.metal +@@ -6,24 +6,29 @@ + #include "mlx/backend/metal/kernels/utils.h" + #include "mlx/backend/metal/kernels/sort.h" + ++#define instantiate_comparator(name, type) \ ++ name ++ + #define instantiate_block_sort( \ +- name, itname, itype, otname, otype, arg_sort, bn, tn) \ +- instantiate_kernel("c" #name "_" #itname "_" #otname "_bn" #bn "_tn" #tn, \ +- block_sort, itype, otype, arg_sort, bn, tn) \ +- instantiate_kernel("nc" #name "_" #itname "_" #otname "_bn" #bn "_tn" #tn, \ +- block_sort_nc, itype, otype, arg_sort, bn, tn) ++ name, itname, itype, otname, otype, arg_sort, bn, tn, vcomparator) \ ++ instantiate_kernel("c" #name "_" #itname "_" #otname "_bn" #bn "_tn" #tn "_comp" #vcomparator, \ ++ block_sort, itype, otype, arg_sort, bn, tn, instantiate_comparator(vcomparator, itype)) \ ++ instantiate_kernel("nc" #name "_" #itname "_" #otname "_bn" #bn "_tn" #tn "_comp" #vcomparator, \ ++ block_sort_nc, itype, otype, arg_sort, bn, tn, instantiate_comparator(vcomparator, itype)) + +-#define instantiate_arg_block_sort_base(itname, itype, bn, tn) \ ++#define instantiate_arg_block_sort_base(itname, itype, bn, tn, comparator) \ + instantiate_block_sort( \ +- arg_block_sort, itname, itype, uint32, uint32_t, true, bn, tn) ++ arg_block_sort, itname, itype, uint32, uint32_t, true, bn, tn, comparator) + +-#define instantiate_block_sort_base(itname, itype, bn, tn) \ ++#define instantiate_block_sort_base(itname, itype, bn, tn, comparator) \ + instantiate_block_sort( \ +- _block_sort, itname, itype, itname, itype, false, bn, tn) ++ _block_sort, itname, itype, itname, itype, false, bn, tn, comparator) + + #define instantiate_block_sort_tn(itname, itype, bn) \ +- instantiate_block_sort_base(itname, itype, bn, 4) \ +- instantiate_arg_block_sort_base(itname, itype, bn, 4) ++ instantiate_block_sort_base(itname, itype, bn, 4, LessThan) \ ++ instantiate_block_sort_base(itname, itype, bn, 4, GreaterThan) \ ++ instantiate_arg_block_sort_base(itname, itype, bn, 4, LessThan) \ ++ instantiate_arg_block_sort_base(itname, itype, bn, 4, GreaterThan) + + #define instantiate_block_sort_bn(itname, itype) \ + instantiate_block_sort_tn(itname, itype, 32) \ +@@ -52,16 +57,17 @@ instantiate_block_sort_long(uint64, uint64_t) + instantiate_block_sort_long(int64, int64_t) + + #define instantiate_multi_block_sort( \ +- vtname, vtype, itname, itype, arg_sort, bn, tn) \ +- instantiate_kernel("sort_mbsort_" #vtname "_" #itname "_bn" #bn "_tn" #tn, \ +- mb_block_sort, vtype, itype, arg_sort, bn, tn) \ +- instantiate_kernel("partition_mbsort_" #vtname "_" #itname "_bn" #bn "_tn" #tn, \ +- mb_block_partition, vtype, itype, arg_sort, bn, tn) \ +- instantiate_kernel("merge_mbsort_" #vtname "_" #itname "_bn" #bn "_tn" #tn, \ +- mb_block_merge, vtype, itype, arg_sort, bn, tn) ++ vtname, vtype, itname, itype, arg_sort, bn, tn, vcomparator) \ ++ instantiate_kernel("sort_mbsort_" #vtname "_" #itname "_bn" #bn "_tn" #tn "_comp" #vcomparator, \ ++ mb_block_sort, vtype, itype, arg_sort, bn, tn, instantiate_comparator(vcomparator, vtype)) \ ++ instantiate_kernel("partition_mbsort_" #vtname "_" #itname "_bn" #bn "_tn" #tn "_comp" #vcomparator, \ ++ mb_block_partition, vtype, itype, arg_sort, bn, tn, instantiate_comparator(vcomparator, vtype)) \ ++ instantiate_kernel("merge_mbsort_" #vtname "_" #itname "_bn" #bn "_tn" #tn "_comp" #vcomparator, \ ++ mb_block_merge, vtype, itype, arg_sort, bn, tn, instantiate_comparator(vcomparator, vtype)) + + #define instantiate_multi_block_sort_base(vtname, vtype) \ +- instantiate_multi_block_sort(vtname, vtype, uint32, uint32_t, true, 512, 4) ++ instantiate_multi_block_sort(vtname, vtype, uint32, uint32_t, true, 512, 4, LessThan) \ ++ instantiate_multi_block_sort(vtname, vtype, uint32, uint32_t, true, 512, 4, GreaterThan) + + instantiate_multi_block_sort_base(uint8, uint8_t) + instantiate_multi_block_sort_base(uint16, uint16_t) +@@ -74,7 +80,8 @@ instantiate_multi_block_sort_base(float32, float) + instantiate_multi_block_sort_base(bfloat16, bfloat16_t) + + #define instantiate_multi_block_sort_long(vtname, vtype) \ +- instantiate_multi_block_sort(vtname, vtype, uint32, uint32_t, true, 256, 4) ++ instantiate_multi_block_sort(vtname, vtype, uint32, uint32_t, true, 256, 4, LessThan) \ ++ instantiate_multi_block_sort(vtname, vtype, uint32, uint32_t, true, 256, 4, GreaterThan) + + instantiate_multi_block_sort_long(uint64, uint64_t) + instantiate_multi_block_sort_long(int64, int64_t) // clang-format on +diff --git a/mlx/backend/metal/make_compiled_preamble.sh b/mlx/backend/metal/make_compiled_preamble.sh +index 425cd8d7..a48d0080 100644 +--- a/mlx/backend/metal/make_compiled_preamble.sh ++++ b/mlx/backend/metal/make_compiled_preamble.sh +@@ -5,18 +5,21 @@ + # + # Copyright © 2023-24 Apple Inc. + +-OUTPUT_DIR=$1 +-CC=$2 +-SRC_DIR=$3 +-SRC_FILE=$4 +-CFLAGS=$5 +-SRC_NAME=$(basename -- "${SRC_FILE}") +-JIT_INCLUDES=${SRC_DIR}/mlx/backend/metal/kernels/jit +-INPUT_FILE=${SRC_DIR}/mlx/backend/metal/kernels/${SRC_FILE}.h +-OUTPUT_FILE=${OUTPUT_DIR}/${SRC_NAME}.cpp ++OUTPUT_FILE=$1 ++INPUT_FILE=$2 ++CFLAGS=$3 + +-mkdir -p "$OUTPUT_DIR" +-CONTENT=$($CC -I"$SRC_DIR" -I"$JIT_INCLUDES" -DMLX_METAL_JIT -E -P "$INPUT_FILE" $CFLAGS 2>/dev/null) ++KERNEL_DIR=$(dirname $(realpath $INPUT_FILE)) ++# echo "KERNEL_DIR: $KERNEL_DIR" ++SRC_DIR=$(echo "$KERNEL_DIR" | cut -d'/' -f-$(($(echo "$KERNEL_DIR" | awk -F'/' '{print NF}')-4))) ++# echo "SRC_DIR: $SRC_DIR" ++JIT_INCLUDES="$KERNEL_DIR/jit" ++# echo "JIT_INCLUDES: $JIT_INCLUDES" ++ ++SRC_BASENAME=$(basename -- "${INPUT_FILE}") ++SRC_NAME="${SRC_BASENAME%.*}" ++ ++CONTENT=$(clang -I"$SRC_DIR" -I"$JIT_INCLUDES" -DMLX_METAL_JIT -E -P "$INPUT_FILE" $CFLAGS 2>/dev/null) + + cat << EOF > "$OUTPUT_FILE" + namespace mlx::core::metal { +diff --git a/mlx/backend/metal/nojit_kernels.cpp b/mlx/backend/metal/nojit_kernels.cpp +index 8da14797..1098f918 100644 +--- a/mlx/backend/metal/nojit_kernels.cpp ++++ b/mlx/backend/metal/nojit_kernels.cpp +@@ -96,7 +96,8 @@ MTL::ComputePipelineState* get_sort_kernel( + const array&, + const array&, + int, +- int) { ++ int, ++ const std::string&) { + return d.get_kernel(kernel_name); + } + +@@ -106,7 +107,8 @@ MTL::ComputePipelineState* get_mb_sort_kernel( + const array&, + const array&, + int, +- int) { ++ int, ++ const std::string&) { + return d.get_kernel(kernel_name); + } + +diff --git a/mlx/backend/metal/sort.cpp b/mlx/backend/metal/sort.cpp +index 3c84022f..bca888f6 100644 +--- a/mlx/backend/metal/sort.cpp ++++ b/mlx/backend/metal/sort.cpp +@@ -6,7 +6,9 @@ + #include "mlx/backend/metal/device.h" + #include "mlx/backend/metal/kernels.h" + #include "mlx/backend/metal/utils.h" ++#include "mlx/dtype_utils.h", + #include "mlx/primitives.h" ++#include "mlx/sort.h" + + namespace mlx::core { + +@@ -20,7 +22,8 @@ void single_block_sort( + int axis, + int bn, + int tn, +- bool argsort) { ++ bool argsort, ++ ComparatorType comparator_) { + // Prepare shapes + int n_rows = in.size() / in.shape(axis); + +@@ -58,9 +61,26 @@ void single_block_sort( + kname << "arg"; + } + ++ std::string comparator; ++ switch (comparator_) { ++ case ComparatorType::LessThan: ++ comparator = "LessThan"; ++ break; ++ case ComparatorType::GreaterThan: ++ comparator = "GreaterThan"; ++ break; ++ } ++ + kname << "_block_sort_" << type_to_name(in) << "_" << type_to_name(out) +- << "_bn" << bn << "_tn" << tn; +- auto kernel = get_sort_kernel(d, kname.str(), in, out, bn, tn); ++ << "_bn" << bn << "_tn" << tn << "_comp" << comparator; ++ auto kernel = get_sort_kernel( ++ d, ++ kname.str(), ++ in, ++ out, ++ bn, ++ tn, ++ comparator + "<" + dtype_to_string(in.dtype()) + ">"); + + // Prepare command encoder + auto& compute_encoder = d.get_command_encoder(s.index); +@@ -120,7 +140,8 @@ void multi_block_sort( + int bn, + int tn, + int n_blocks, +- bool argsort) { ++ bool argsort, ++ ComparatorType comparator_) { + // Prepare shapes + int n_rows = in.size() / in.shape(axis); + +@@ -159,6 +180,16 @@ void multi_block_sort( + std::vector copies = { + dev_vals_0, dev_vals_1, dev_idxs_0, dev_idxs_1, block_partitions}; + ++ std::string comparator; ++ switch (comparator_) { ++ case ComparatorType::LessThan: ++ comparator = "LessThan"; ++ break; ++ case ComparatorType::GreaterThan: ++ comparator = "GreaterThan"; ++ break; ++ } ++ + // Prepare command encoder + auto& compute_encoder = d.get_command_encoder(s.index); + +@@ -166,9 +197,16 @@ void multi_block_sort( + { + std::ostringstream kname; + kname << "sort_mbsort_" << type_to_name(dev_vals_0) << "_" +- << type_to_name(dev_idxs_0) << "_bn" << bn << "_tn" << tn; +- auto kernel = +- get_mb_sort_kernel(d, kname.str(), dev_vals_0, dev_idxs_0, bn, tn); ++ << type_to_name(dev_idxs_0) << "_bn" << bn << "_tn" << tn << "_comp" ++ << comparator; ++ auto kernel = get_mb_sort_kernel( ++ d, ++ kname.str(), ++ dev_vals_0, ++ dev_idxs_0, ++ bn, ++ tn, ++ comparator + "<" + dtype_to_string(in.dtype()) + ">"); + compute_encoder.set_compute_pipeline_state(kernel); + + compute_encoder.set_input_array(in, 0); +@@ -206,10 +244,17 @@ void multi_block_sort( + { + std::ostringstream kname; + kname << "partition_mbsort_" << type_to_name(dev_vals_in) << "_" +- << type_to_name(dev_idxs_in) << "_bn" << bn << "_tn" << tn; +- +- auto kernel = +- get_mb_sort_kernel(d, kname.str(), dev_vals_0, dev_idxs_0, bn, tn); ++ << type_to_name(dev_idxs_in) << "_bn" << bn << "_tn" << tn ++ << "_comp" << comparator; ++ ++ auto kernel = get_mb_sort_kernel( ++ d, ++ kname.str(), ++ dev_vals_0, ++ dev_idxs_0, ++ bn, ++ tn, ++ comparator + "<" + dtype_to_string(in.dtype()) + ">"); + compute_encoder.set_compute_pipeline_state(kernel); + + compute_encoder.set_output_array(block_partitions, 0); +@@ -229,10 +274,17 @@ void multi_block_sort( + { + std::ostringstream kname; + kname << "merge_mbsort_" << type_to_name(dev_vals_in) << "_" +- << type_to_name(dev_idxs_in) << "_bn" << bn << "_tn" << tn; +- +- auto kernel = +- get_mb_sort_kernel(d, kname.str(), dev_vals_0, dev_idxs_0, bn, tn); ++ << type_to_name(dev_idxs_in) << "_bn" << bn << "_tn" << tn ++ << "_comp" << comparator; ++ ++ auto kernel = get_mb_sort_kernel( ++ d, ++ kname.str(), ++ dev_vals_0, ++ dev_idxs_0, ++ bn, ++ tn, ++ comparator + "<" + dtype_to_string(in.dtype()) + ">"); + compute_encoder.set_compute_pipeline_state(kernel); + + compute_encoder.set_input_array(block_partitions, 0); +@@ -277,7 +329,8 @@ void gpu_merge_sort( + const array& in, + array& out, + int axis_, +- bool argsort) { ++ bool argsort, ++ ComparatorType comparator) { + // Get size info + int axis = axis_ < 0 ? axis_ + in.ndim() : axis_; + int size_sorted_axis = in.shape(axis); +@@ -307,9 +360,10 @@ void gpu_merge_sort( + int n_blocks = (size_sorted_axis + n_per_block - 1) / n_per_block; + + if (n_blocks > 1) { +- return multi_block_sort(s, d, in, out, axis, bn, tn, n_blocks, argsort); ++ return multi_block_sort( ++ s, d, in, out, axis, bn, tn, n_blocks, argsort, comparator); + } else { +- return single_block_sort(s, d, in, out, axis, bn, tn, argsort); ++ return single_block_sort(s, d, in, out, axis, bn, tn, argsort, comparator); + } + } + +@@ -324,7 +378,7 @@ void ArgSort::eval_gpu(const std::vector& inputs, array& out) { + auto& d = metal::device(s.device); + auto& in = inputs[0]; + +- gpu_merge_sort(s, d, in, out, axis_, true); ++ gpu_merge_sort(s, d, in, out, axis_, true, comparator_); + } + + void Sort::eval_gpu(const std::vector& inputs, array& out) { +@@ -336,7 +390,7 @@ void Sort::eval_gpu(const std::vector& inputs, array& out) { + auto& d = metal::device(s.device); + auto& in = inputs[0]; + +- gpu_merge_sort(s, d, in, out, axis_, false); ++ gpu_merge_sort(s, d, in, out, axis_, false, comparator_); + } + + void ArgPartition::eval_gpu(const std::vector& inputs, array& out) { +@@ -349,7 +403,7 @@ void ArgPartition::eval_gpu(const std::vector& inputs, array& out) { + auto& d = metal::device(s.device); + auto& in = inputs[0]; + +- gpu_merge_sort(s, d, in, out, axis_, true); ++ gpu_merge_sort(s, d, in, out, axis_, true, comparator_); + } + + void Partition::eval_gpu(const std::vector& inputs, array& out) { +@@ -362,7 +416,7 @@ void Partition::eval_gpu(const std::vector& inputs, array& out) { + auto& d = metal::device(s.device); + auto& in = inputs[0]; + +- gpu_merge_sort(s, d, in, out, axis_, false); ++ gpu_merge_sort(s, d, in, out, axis_, false, comparator_); + } + + } // namespace mlx::core +diff --git a/mlx/backend/no_gpu/primitives.cpp b/mlx/backend/no_gpu/primitives.cpp +index 676a6e55..178eb23f 100644 +--- a/mlx/backend/no_gpu/primitives.cpp ++++ b/mlx/backend/no_gpu/primitives.cpp +@@ -95,7 +95,7 @@ NO_GPU(Partition) + NO_GPU(Power) + NO_GPU_MULTI(QRF) + NO_GPU(QuantizedMatmul) +-NO_GPU(RandomBits) ++NO_GPU() + NO_GPU(Real) + NO_GPU(Reduce) + NO_GPU(Reshape) +diff --git a/mlx/mlx.h b/mlx/mlx.h +index cef8d806..f8932873 100644 +--- a/mlx/mlx.h ++++ b/mlx/mlx.h +@@ -17,6 +17,7 @@ + #include "mlx/memory.h" + #include "mlx/ops.h" + #include "mlx/random.h" ++#include "mlx/sort.h" + #include "mlx/stream.h" + #include "mlx/transforms.h" + #include "mlx/utils.h" +diff --git a/mlx/ops.cpp b/mlx/ops.cpp +index a72c2bc8..04ac921f 100644 +--- a/mlx/ops.cpp ++++ b/mlx/ops.cpp +@@ -2205,13 +2205,20 @@ array argmax( + } + + /** Returns a sorted copy of the flattened array. */ +-array sort(const array& a, StreamOrDevice s /* = {} */) { ++array sort( ++ const array& a, ++ ComparatorType comparator, ++ StreamOrDevice s /* = {} */) { + int size = a.size(); +- return sort(reshape(a, {size}, s), 0, s); ++ return sort(reshape(a, {size}, s), 0, comparator, s); + } + + /** Returns a sorted copy of the array along a given axis. */ +-array sort(const array& a, int axis, StreamOrDevice s /* = {} */) { ++array sort( ++ const array& a, ++ int axis, ++ ComparatorType comparator, ++ StreamOrDevice s /* = {} */) { + // Check for valid axis + if (axis + static_cast(a.ndim()) < 0 || + axis >= static_cast(a.ndim())) { +@@ -2222,17 +2229,27 @@ array sort(const array& a, int axis, StreamOrDevice s /* = {} */) { + } + + return array( +- a.shape(), a.dtype(), std::make_shared(to_stream(s), axis), {a}); ++ a.shape(), ++ a.dtype(), ++ std::make_shared(to_stream(s), axis, comparator), ++ {a}); + } + + /** Returns indices that sort the flattened array. */ +-array argsort(const array& a, StreamOrDevice s /* = {} */) { ++array argsort( ++ const array& a, ++ ComparatorType comparator, ++ StreamOrDevice s /* = {} */) { + int size = a.size(); +- return argsort(reshape(a, {size}, s), 0, s); ++ return argsort(reshape(a, {size}, s), 0, comparator, s); + } + + /** Returns indices that sort the array along a given axis. */ +-array argsort(const array& a, int axis, StreamOrDevice s /* = {} */) { ++array argsort( ++ const array& a, ++ int axis, ++ ComparatorType comparator, ++ StreamOrDevice s /* = {} */) { + // Check for valid axis + if (axis + static_cast(a.ndim()) < 0 || + axis >= static_cast(a.ndim())) { +@@ -2243,16 +2260,23 @@ array argsort(const array& a, int axis, StreamOrDevice s /* = {} */) { + } + + return array( +- a.shape(), uint32, std::make_shared(to_stream(s), axis), {a}); ++ a.shape(), ++ uint32, ++ std::make_shared(to_stream(s), axis, comparator), ++ {a}); + } + + /** + * Returns a partitioned copy of the flattened array + * such that the smaller kth elements are first. + **/ +-array partition(const array& a, int kth, StreamOrDevice s /* = {} */) { ++array partition( ++ const array& a, ++ int kth, ++ ComparatorType comparator, ++ StreamOrDevice s /* = {} */) { + int size = a.size(); +- return partition(reshape(a, {size}, s), kth, 0, s); ++ return partition(reshape(a, {size}, s), kth, 0, comparator, s); + } + + /** +@@ -2263,6 +2287,7 @@ array partition( + const array& a, + int kth, + int axis, ++ ComparatorType comparator, + StreamOrDevice s /* = {} */) { + // Check for valid axis + if (axis + static_cast(a.ndim()) < 0 || +@@ -2283,7 +2308,7 @@ array partition( + return array( + a.shape(), + a.dtype(), +- std::make_shared(to_stream(s), kth_, axis_), ++ std::make_shared(to_stream(s), kth_, axis_, comparator), + {a}); + } + +@@ -2291,9 +2316,13 @@ array partition( + * Returns indices that partition the flattened array + * such that the smaller kth elements are first. + **/ +-array argpartition(const array& a, int kth, StreamOrDevice s /* = {} */) { ++array argpartition( ++ const array& a, ++ int kth, ++ ComparatorType comparator, ++ StreamOrDevice s /* = {} */) { + int size = a.size(); +- return argpartition(reshape(a, {size}, s), kth, 0, s); ++ return argpartition(reshape(a, {size}, s), kth, 0, comparator, s); + } + + /** +@@ -2304,6 +2333,7 @@ array argpartition( + const array& a, + int kth, + int axis, ++ ComparatorType comparator, + StreamOrDevice s /* = {} */) { + // Check for valid axis + if (axis + static_cast(a.ndim()) < 0 || +@@ -2324,7 +2354,7 @@ array argpartition( + return array( + a.shape(), + uint32, +- std::make_shared(to_stream(s), kth_, axis_), ++ std::make_shared(to_stream(s), kth_, axis_, comparator), + {a}); + } + +diff --git a/mlx/ops.h b/mlx/ops.h +index af3cdb5b..be762b8a 100644 +--- a/mlx/ops.h ++++ b/mlx/ops.h +@@ -6,6 +6,7 @@ + + #include "mlx/array.h" + #include "mlx/device.h" ++#include "mlx/sort.h" + #include "mlx/stream.h" + #include "mlx/utils.h" + +@@ -674,40 +675,98 @@ array argmax( + StreamOrDevice s = {}); + + /** Returns a sorted copy of the flattened array. */ +-array sort(const array& a, StreamOrDevice s = {}); ++array sort( ++ const array& a, ++ ComparatorType comparator = ComparatorType::LessThan, ++ StreamOrDevice s = {}); ++inline array sort(const array& a, StreamOrDevice s = {}) { ++ return sort(a, ComparatorType::LessThan, s); ++} + + /** Returns a sorted copy of the array along a given axis. */ +-array sort(const array& a, int axis, StreamOrDevice s = {}); ++array sort( ++ const array& a, ++ int axis, ++ ComparatorType comparator = ComparatorType::LessThan, ++ StreamOrDevice s = {}); ++inline array sort(const array& a, int axis, StreamOrDevice s = {}) { ++ return sort(a, axis, ComparatorType::LessThan, s); ++} + + /** Returns indices that sort the flattened array. */ +-array argsort(const array& a, StreamOrDevice s = {}); ++array argsort( ++ const array& a, ++ ComparatorType comparator = ComparatorType::LessThan, ++ StreamOrDevice s = {}); ++inline array argsort(const array& a, StreamOrDevice s = {}) { ++ return argsort(a, ComparatorType::LessThan, s); ++} + + /** Returns indices that sort the array along a given axis. */ +-array argsort(const array& a, int axis, StreamOrDevice s = {}); ++array argsort( ++ const array& a, ++ int axis, ++ ComparatorType comparator = ComparatorType::LessThan, ++ StreamOrDevice s = {}); ++inline array argsort(const array& a, int axis, StreamOrDevice s = {}) { ++ return argsort(a, axis, ComparatorType::LessThan, s); ++} + + /** + * Returns a partitioned copy of the flattened array + * such that the smaller kth elements are first. + **/ +-array partition(const array& a, int kth, StreamOrDevice s = {}); ++array partition( ++ const array& a, ++ int kth, ++ ComparatorType comparator = ComparatorType::LessThan, ++ StreamOrDevice s = {}); ++inline array partition(const array& a, int kth, StreamOrDevice s = {}) { ++ return partition(a, kth, ComparatorType::LessThan, s); ++} + + /** + * Returns a partitioned copy of the array along a given axis + * such that the smaller kth elements are first. + **/ +-array partition(const array& a, int kth, int axis, StreamOrDevice s = {}); ++array partition( ++ const array& a, ++ int kth, ++ int axis, ++ ComparatorType comparator = ComparatorType::LessThan, ++ StreamOrDevice s = {}); ++inline array ++partition(const array& a, int kth, int axis, StreamOrDevice s = {}) { ++ return partition(a, kth, axis, ComparatorType::LessThan, s); ++} + + /** + * Returns indices that partition the flattened array + * such that the smaller kth elements are first. + **/ +-array argpartition(const array& a, int kth, StreamOrDevice s = {}); ++array argpartition( ++ const array& a, ++ int kth, ++ ComparatorType comparator = ComparatorType::LessThan, ++ StreamOrDevice s = {}); ++inline array argpartition(const array& a, int kth, StreamOrDevice s = {}) { ++ return argpartition(a, kth, ComparatorType::LessThan, s); ++} + + /** + * Returns indices that partition the array along a given axis + * such that the smaller kth elements are first. + **/ +-array argpartition(const array& a, int kth, int axis, StreamOrDevice s = {}); ++array argpartition( ++ const array& a, ++ int kth, ++ int axis, ++ ComparatorType comparator = ComparatorType::LessThan, ++ StreamOrDevice s = {}); ++inline array ++argpartition(const array& a, int kth, int axis, StreamOrDevice s = {}) { ++ return argpartition(a, kth, axis, ComparatorType::LessThan, s); ++} + + /** Returns topk elements of the flattened array. */ + array topk(const array& a, int k, StreamOrDevice s = {}); +diff --git a/mlx/primitives.h b/mlx/primitives.h +index c0fbfc84..13d67201 100644 +--- a/mlx/primitives.h ++++ b/mlx/primitives.h +@@ -7,6 +7,7 @@ + #include "mlx/array.h" + #include "mlx/device.h" + #include "mlx/io/load.h" ++#include "mlx/sort.h" + #include "mlx/stream.h" + + #define DEFINE_VMAP() \ +@@ -322,8 +323,15 @@ class ArcTanh : public UnaryPrimitive { + + class ArgPartition : public UnaryPrimitive { + public: +- explicit ArgPartition(Stream stream, int kth, int axis) +- : UnaryPrimitive(stream), kth_(kth), axis_(axis) {} ++ explicit ArgPartition( ++ Stream stream, ++ int kth, ++ int axis, ++ ComparatorType comparator) ++ : UnaryPrimitive(stream), ++ kth_(kth), ++ axis_(axis), ++ comparator_(comparator) {} + + void eval_cpu(const std::vector& inputs, array& out) override; + void eval_gpu(const std::vector& inputs, array& out) override; +@@ -333,13 +341,14 @@ class ArgPartition : public UnaryPrimitive { + DEFINE_PRINT(ArgPartition) + DEFINE_INPUT_OUTPUT_SHAPE() + bool is_equivalent(const Primitive& other) const override; +- std::pair state() const { +- return {kth_, axis_}; ++ auto state() const { ++ return std::make_tuple(kth_, axis_, comparator_); + }; + + private: + int kth_; + int axis_; ++ ComparatorType comparator_; + }; + + class ArgReduce : public UnaryPrimitive { +@@ -371,8 +380,8 @@ class ArgReduce : public UnaryPrimitive { + + class ArgSort : public UnaryPrimitive { + public: +- explicit ArgSort(Stream stream, int axis) +- : UnaryPrimitive(stream), axis_(axis) {} ++ explicit ArgSort(Stream stream, int axis, ComparatorType comparator) ++ : UnaryPrimitive(stream), axis_(axis), comparator_(comparator) {} + + void eval_cpu(const std::vector& inputs, array& out) override; + void eval_gpu(const std::vector& inputs, array& out) override; +@@ -381,12 +390,13 @@ class ArgSort : public UnaryPrimitive { + DEFINE_PRINT(ArgSort) + DEFINE_INPUT_OUTPUT_SHAPE() + bool is_equivalent(const Primitive& other) const override; +- int state() const { +- return axis_; ++ auto state() const { ++ return std::make_pair(axis_, comparator_); + }; + + private: + int axis_; ++ ComparatorType comparator_; + }; + + class AsType : public UnaryPrimitive { +@@ -1530,8 +1540,15 @@ class Pad : public UnaryPrimitive { + + class Partition : public UnaryPrimitive { + public: +- explicit Partition(Stream stream, int kth, int axis) +- : UnaryPrimitive(stream), kth_(kth), axis_(axis) {} ++ explicit Partition( ++ Stream stream, ++ int kth, ++ int axis, ++ ComparatorType comparator) ++ : UnaryPrimitive(stream), ++ kth_(kth), ++ axis_(axis), ++ comparator_(comparator) {} + + void eval_cpu(const std::vector& inputs, array& out) override; + void eval_gpu(const std::vector& inputs, array& out) override; +@@ -1542,12 +1559,13 @@ class Partition : public UnaryPrimitive { + DEFINE_INPUT_OUTPUT_SHAPE() + bool is_equivalent(const Primitive& other) const override; + auto state() const { +- return std::make_pair(kth_, axis_); ++ return std::make_tuple(kth_, axis_, comparator_); + }; + + private: + int kth_; + int axis_; ++ ComparatorType comparator_; + }; + + class Power : public UnaryPrimitive { +@@ -2073,8 +2091,8 @@ class Softmax : public UnaryPrimitive { + + class Sort : public UnaryPrimitive { + public: +- explicit Sort(Stream stream, int axis) +- : UnaryPrimitive(stream), axis_(axis) {} ++ explicit Sort(Stream stream, int axis, ComparatorType comparator) ++ : UnaryPrimitive(stream), axis_(axis), comparator_(comparator) {} + + void eval_cpu(const std::vector& inputs, array& out) override; + void eval_gpu(const std::vector& inputs, array& out) override; +@@ -2085,11 +2103,12 @@ class Sort : public UnaryPrimitive { + DEFINE_INPUT_OUTPUT_SHAPE() + bool is_equivalent(const Primitive& other) const override; + auto state() const { +- return axis_; ++ return std::make_pair(axis_, comparator_); + } + + private: + int axis_; ++ ComparatorType comparator_; + }; + + class Split : public Primitive { +diff --git a/mlx/random.cpp b/mlx/random.cpp +index 6c6d1eb9..5a4745a5 100644 +--- a/mlx/random.cpp ++++ b/mlx/random.cpp +@@ -225,6 +225,23 @@ array normal( + return samples; + } + ++array normal( ++ const Shape& shape, ++ Dtype dtype, ++ const array& loc, ++ const array& scale, ++ const std::optional& key /*= nullopt */, ++ StreamOrDevice s /* = {} */) { ++ auto stream = to_stream(s); ++ auto low = above_minus_one_with_default(dtype); ++ auto high = array(1.0f, dtype); ++ auto samples = uniform(low, high, shape, dtype, key, stream); ++ samples = multiply(array(std::sqrt(2.0), dtype), erfinv(samples, stream), stream); ++ samples = multiply(astype(scale, dtype), samples, stream); ++ samples = add(astype(loc, dtype), samples, stream); ++ return samples; ++} ++ + array multivariate_normal( + const array& mean, + const array& cov, +diff --git a/mlx/random.h b/mlx/random.h +index 0dfdab7a..ebee726c 100644 +--- a/mlx/random.h ++++ b/mlx/random.h +@@ -134,6 +134,14 @@ inline array normal( + return normal(shape, float32, std::nullopt, std::nullopt, key, s); + } + ++array normal( ++ const Shape& shape, ++ Dtype dtype, ++ const array& loc, ++ const array& scale, ++ const std::optional& key = std::nullopt, ++ StreamOrDevice s = {}); ++ + /** Generate samples from a multivariate normal distribution. **/ + array multivariate_normal( + const array& mean, +diff --git a/mlx/sort.h b/mlx/sort.h +new file mode 100644 +index 00000000..0afc6b62 +--- /dev/null ++++ b/mlx/sort.h +@@ -0,0 +1,14 @@ ++// Copyright © 2023-2024 Apple Inc. ++ ++#pragma once ++ ++#include "mlx/array.h" ++ ++namespace mlx::core { ++ ++enum class ComparatorType { ++ LessThan, ++ GreaterThan, ++}; ++ ++} // namespace mlx::core +diff --git a/mlx/types/complex.h b/mlx/types/complex.h +index 51101cc9..344d80f8 100644 +--- a/mlx/types/complex.h ++++ b/mlx/types/complex.h +@@ -59,10 +59,11 @@ inline bool operator>(const complex64_t& a, const complex64_t& b) { + inline complex64_t operator%(complex64_t a, complex64_t b) { + auto real = a.real() - (b.real() * static_cast(a.real() / b.real())); + auto imag = a.imag() - (b.imag() * static_cast(a.imag() / b.imag())); +- if (real != 0 && ((real < 0) != (b.real() < 0))) +- real += b.real(); +- if (imag != 0 && ((imag < 0) != (b.imag() < 0))) +- imag += b.imag(); ++ /* Disable to match `stablehlo` implementation semantics */ ++ // if (real != 0 && ((real < 0) != (b.real() < 0))) ++ // real += b.real(); ++ // if (imag != 0 && ((imag < 0) != (b.imag() < 0))) ++ // imag += b.imag(); + return {real, imag}; + } + diff --git a/third_party/modules/mlx/20250518.0-eebe730/source.json b/third_party/modules/mlx/20250518.0-eebe730/source.json new file mode 100644 index 00000000..ad6540c6 --- /dev/null +++ b/third_party/modules/mlx/20250518.0-eebe730/source.json @@ -0,0 +1,15 @@ +{ + "strip_prefix": "mlx-eebe73001affcb424171e9d49657e508f70a9201", + "url": "https://github.com/ml-explore/mlx/archive/eebe73001affcb424171e9d49657e508f70a9201.tar.gz", + "integrity": "sha256-bRpLtmR9R/bSlzEQ403T8/K2t8r3PLISd5JWmB22cgQ=", + "overlay": { + "metal_cpp.bzl": "", + "jit.bzl": "", + "MODULE.bazel": "", + "BUILD.bazel": "" + }, + "patch_strip": 1, + "patches": { + "0001-Modify-Sort-And-Comparison-Kernels.patch": "" + } +} diff --git a/third_party/modules/mlx/20250605.0-c763fe1/MODULE.bazel b/third_party/modules/mlx/20250605.0-c763fe1/MODULE.bazel new file mode 100644 index 00000000..c81ba37f --- /dev/null +++ b/third_party/modules/mlx/20250605.0-c763fe1/MODULE.bazel @@ -0,0 +1,10 @@ +module( + name = "mlx", + version = "20250605.0-c763fe1", + compatibility_level = 1, +) + +bazel_dep(name = "rules_apple", version = "3.13.0", repo_name = "build_bazel_rules_apple") +bazel_dep(name = "fmt", version = "11.0.2") +metal_cpp = use_extension("metal_cpp.bzl", "metal_cpp") +use_repo(metal_cpp, "metal_cpp") diff --git a/third_party/modules/mlx/20250605.0-c763fe1/overlay/BUILD.bazel b/third_party/modules/mlx/20250605.0-c763fe1/overlay/BUILD.bazel new file mode 100644 index 00000000..62cd72c5 --- /dev/null +++ b/third_party/modules/mlx/20250605.0-c763fe1/overlay/BUILD.bazel @@ -0,0 +1,599 @@ +load("@build_bazel_rules_apple//apple:resources.bzl", "apple_metal_library") +load("//:jit.bzl", "make_jit_source") + +filegroup( + name = "mlx_h", + srcs = glob(["mlx/*.h"], ["mlx/mlx.h"]), +) + +# Contains `mlx/types/*.h` headers +cc_library( + name = "types", + hdrs = glob(["mlx/types/*.h"]), +) + +cc_library( + name = "pocketfft", + hdrs = [ + "mlx/3rdparty/pocketfft.h", + ], +) + +# mlx/distributed/ring/CMakeLists.txt +cc_library( + name = "ring", + srcs = [ + "mlx/distributed/ring/no_ring.cpp", + + # Other sources (maybe reorganize dependencies?) + ":mlx_h", + "mlx/distributed/distributed.h", + ], + hdrs = [ + "mlx/distributed/ring/ring.h", + ], + deps = [ + ":types" + ], +) + +# mlx/distributed/mpi/CMakeLists.txt +cc_library( + name = "mpi", + srcs = [ + "mlx/distributed/mpi/no_mpi.cpp", + # Other sources (maybe reorganize dependencies?) + "mlx/distributed/distributed.h", + ":mlx_h", + ], + hdrs = [ + "mlx/distributed/mpi/mpi.h", + ], + deps = [ + ":types", + ] +) + +# mlx/distributed/CMakeLists.txt +cc_library( + name = "distributed", + srcs = [ + "mlx/distributed/primitives.cpp", + "mlx/distributed/ops.cpp", + "mlx/distributed/distributed.cpp", + + # Other sources (maybe reorganize dependencies?) + ":mlx_h", + ], + hdrs = [ + "mlx/distributed/distributed_impl.h", + "mlx/distributed/distributed.h", + "mlx/distributed/ops.h", + "mlx/distributed/primitives.h", + ], + deps = [ + ":io", + ":mpi", + ":ring", + ":types", + ] +) + +# mlx/io/CMakeLists.txt +cc_library( + name = "io", + srcs = [ + "mlx/io/load.cpp", + "mlx/io/no_safetensors.cpp", + "mlx/io/no_gguf.cpp", + + # Other sources (maybe reorganize dependencies?) + ":mlx_h", + ], + hdrs = [ + "mlx/io/load.h", + ], + deps = [ + ":types", + ], +) + +# mlx/backend/common/CMakeLists.txt +cc_library( + name = "backend_common", + srcs = [ + "mlx/backend/common/broadcasting.cpp", + "mlx/backend/common/compiled.cpp", + "mlx/backend/common/common.cpp", + "mlx/backend/common/load.cpp", + "mlx/backend/common/reduce.cpp", + "mlx/backend/common/slicing.cpp", + "mlx/backend/common/utils.cpp", + # Other sources (maybe reorganize dependencies?) + "mlx/backend/gpu/eval.h", + ":mlx_h", + ], + hdrs = [ + "mlx/backend/common/binary.h", + "mlx/backend/common/broadcasting.h", + "mlx/backend/common/buffer_cache.h", + "mlx/backend/common/compiled.h", + "mlx/backend/common/copy.h", + "mlx/backend/common/hadamard.h", + "mlx/backend/common/reduce.h", + "mlx/backend/common/slicing.h", + "mlx/backend/common/ternary.h", + "mlx/backend/common/utils.h", + ], + deps = [ + + ":io", + ":types", + ], +) + +# mlx/backend/cpu/CMakeLists.txt +genrule( + name = "compiled_preamble", + visibility = ["//visibility:public"], + srcs = ["mlx/backend/cpu/compiled_preamble.h"], + outs = ["mlx/backend/cpu/compiled_preamable.cpp"], + cmd = "bash $(location :mlx/backend/cpu/make_compiled_preamble.sh) $@ $(location mlx/backend/cpu/compiled_preamble.h)", + tools = [":mlx/backend/cpu/make_compiled_preamble.sh"], +) + +# TODO (@cryptodeal): finish bazel config for cpu backend +cc_library( + name = "backend_cpu", + linkopts = [ + "-framework", + "Accelerate", + ], + defines = [ + "MLX_USE_ACCELERATE", + "ACCELERATE_NEW_LAPACK" + ], + srcs = [ + "mlx/backend/cpu/arg_reduce.cpp", + "mlx/backend/cpu/binary.cpp", + "mlx/backend/cpu/conv.cpp", + "mlx/backend/cpu/copy.cpp", + "mlx/backend/cpu/eigh.cpp", + "mlx/backend/cpu/fft.cpp", + "mlx/backend/cpu/hadamard.cpp", + "mlx/backend/cpu/matmul.cpp", + "mlx/backend/cpu/gemms/cblas.cpp", + "mlx/backend/cpu/masked_mm.cpp", + "mlx/backend/cpu/primitives.cpp", + "mlx/backend/cpu/quantized.cpp", + "mlx/backend/cpu/reduce.cpp", + "mlx/backend/cpu/scan.cpp", + "mlx/backend/cpu/select.cpp", + "mlx/backend/cpu/softmax.cpp", + "mlx/backend/cpu/sort.cpp", + "mlx/backend/cpu/threefry.cpp", + "mlx/backend/cpu/indexing.cpp", + "mlx/backend/cpu/luf.cpp", + "mlx/backend/cpu/qrf.cpp", + "mlx/backend/cpu/svd.cpp", + "mlx/backend/cpu/inverse.cpp", + "mlx/backend/cpu/cholesky.cpp", + "mlx/backend/cpu/unary.cpp", + # Build with Accelerate + "mlx/backend/cpu/gemms/bnns.cpp", + # Not Building for IOS + "mlx/backend/cpu/compiled.cpp", + "mlx/backend/cpu/jit_compiler.cpp", + # Additional Header Sources + "mlx/backend/cpu/simd/simd.h", + "mlx/backend/cpu/simd/math.h", + "mlx/backend/cpu/simd/type.h", + "mlx/backend/cpu/simd/base_simd.h", + "mlx/backend/cpu/simd/accelerate_simd.h", + "mlx/backend/cpu/simd/accelerate_fp16_simd.h", + # generated `compiled_preamble.cpp` + ":compiled_preamble", + + # Other sources (maybe reorganize dependencies?) + ":mlx_h", + "mlx/io/load.h", + ], + hdrs = [ + "mlx/backend/cpu/arange.h", + "mlx/backend/cpu/binary_ops.h", + "mlx/backend/cpu/binary_two.h", + "mlx/backend/cpu/binary.h", + "mlx/backend/cpu/compiled_preamble.h", + "mlx/backend/cpu/copy.h", + "mlx/backend/cpu/gemm.h", + "mlx/backend/cpu/jit_compiler.h", + "mlx/backend/cpu/lapack.h", + "mlx/backend/cpu/slicing.h", + "mlx/backend/cpu/ternary.h", + "mlx/backend/cpu/threefry.h", + "mlx/backend/cpu/unary_ops.h", + "mlx/backend/cpu/unary.h", + ], + deps = [ + "@fmt", + ":backend_common", + ":pocketfft", + ":types", + ], +) + +# mlx/backend/no_cpu/CMakeLists.txt +cc_library( + name = "backend_no_cpu", + srcs = [ + "mlx/backend/no_cpu/available.cpp", + "mlx/backend/no_cpu/primitives.cpp", + "mlx/backend/cpu/eval.cpp", + "mlx/backend/cpu/encoder.cpp", + "mlx/backend/no_cpu/compiled.cpp", + # Other sources (maybe reorganize dependencies?) + "mlx/backend/cpu/available.h", + "mlx/backend/cpu/encoder.h", + "mlx/backend/cpu/eval.h", + "mlx/backend/gpu/eval.h", + ":mlx_h", + ], + deps = [ + ":distributed", + ":io", + # ":backend_common", + ":types", + ], +) + +# mlx/backend/metal/CMakeLists.txt +make_jit_source( + name = "utils", + deps = [ + "mlx/backend/metal/kernels/jit/bf16.h", + "mlx/backend/metal/kernels/metal_3_0/bf16.h", + "mlx/backend/metal/kernels/metal_3_1/bf16.h", + "mlx/backend/metal/kernels/bf16_math.h", + "mlx/backend/metal/kernels/complex.h", + "mlx/backend/metal/kernels/defines.h", + ], +) + +make_jit_source( + name = "unary_ops", + deps = [ + "mlx/backend/metal/kernels/erf.h", + "mlx/backend/metal/kernels/expm1f.h", + ], +) + +make_jit_source(name = "binary_ops") + +make_jit_source(name = "ternary_ops") + +make_jit_source( + name = "reduce_utils", + deps = [ + "mlx/backend/metal/kernels/atomic.h", + "mlx/backend/metal/kernels/reduction/ops.h" + ], +) + +make_jit_source( + name = "scatter", + deps = [ + "mlx/backend/metal/kernels/indexing.h", + ], +) + +make_jit_source( + name = "gather", + deps = [ + "mlx/backend/metal/kernels/indexing.h", + ], +) + +make_jit_source(name = "gather_axis") + +make_jit_source(name = "scatter_axis") + +make_jit_source(name = "hadamard") + +cc_library( + name = "backend_metal", + linkopts = [ + "-framework Metal", + "-framework Foundation", + "-framework QuartzCore", + ], + # Use for metal debugging/traces + # local_defines = [ + # "MLX_METAL_DEBUG", + # ], + srcs = [ + "mlx/backend/metal/allocator.cpp", + "mlx/backend/metal/binary.cpp", + "mlx/backend/metal/compiled.cpp", + "mlx/backend/metal/conv.cpp", + "mlx/backend/metal/copy.cpp", + "mlx/backend/metal/custom_kernel.cpp", + "mlx/backend/metal/distributed.cpp", + "mlx/backend/metal/device.cpp", + "mlx/backend/metal/event.cpp", + "mlx/backend/metal/eval.cpp", + "mlx/backend/metal/fence.cpp", + "mlx/backend/metal/fft.cpp", + "mlx/backend/metal/hadamard.cpp", + "mlx/backend/metal/indexing.cpp", + "mlx/backend/metal/logsumexp.cpp", + "mlx/backend/metal/matmul.cpp", + "mlx/backend/metal/scaled_dot_product_attention.cpp", + "mlx/backend/metal/metal.cpp", + "mlx/backend/metal/primitives.cpp", + "mlx/backend/metal/quantized.cpp", + "mlx/backend/metal/normalization.cpp", + "mlx/backend/metal/rope.cpp", + "mlx/backend/metal/scan.cpp", + "mlx/backend/metal/slicing.cpp", + "mlx/backend/metal/softmax.cpp", + "mlx/backend/metal/sort.cpp", + "mlx/backend/metal/reduce.cpp", + "mlx/backend/metal/ternary.cpp", + "mlx/backend/metal/unary.cpp", + "mlx/backend/metal/resident.cpp", + "mlx/backend/metal/utils.cpp", + "mlx/backend/metal/nojit_kernels.cpp", + # Additional Header Sources + "mlx/backend/metal/jit/indexing.h", + "mlx/backend/metal/kernels/defines.h", + "mlx/backend/metal/kernels/steel/conv/params.h", + "mlx/backend/metal/kernels/steel/gemm/params.h", + "mlx/backend/metal/kernels/steel/attn/params.h", + "mlx/backend/metal/jit/includes.h", + # Generate METAL_PATH define + ":generate_metal_path", + # Generated sources + ":utils", + ":unary_ops", + ":binary_ops", + ":ternary_ops", + ":reduce_utils", + ":scatter", + ":gather", + ":gather_axis", + ":scatter_axis", + ":hadamard", + + # Other sources (maybe reorganize dependencies?) + ":mlx_h", + ], + hdrs = [ + "mlx/backend/metal/allocator.h", + "mlx/backend/metal/binary.h", + "mlx/backend/metal/device.h", + "mlx/backend/metal/thread_safety.h", + "mlx/backend/metal/kernels.h", + "mlx/backend/metal/matmul.h", + "mlx/backend/metal/metal.h", + "mlx/backend/metal/reduce.h", + "mlx/backend/metal/resident.h", + "mlx/backend/metal/ternary.h", + "mlx/backend/metal/unary.h", + "mlx/backend/metal/utils.h", + ], + deps = [ + "@metal_cpp//:metal_cpp", + "@fmt", + ":backend_common", + ":backend_gpu", + ":distributed", + ":pocketfft", + ], +) + +genrule( + name = "generate_metal_path", + outs = [ + "mlx/backend/metal/metal_path.h", + ], + cmd = "echo \"#define METAL_PATH \\\"`realpath $(location //:mlx_metallib)`\\\"\" > $@", + tools = [ + "//:mlx_metallib", + ], +) + +# TODO (@cryptodeal): find a better solution than `copts` includes +apple_metal_library( + name = "mlx_metallib", + visibility = ["//visibility:public"], + copts = [ + "-target", + "air64-apple-macosx15.5", + "-Iexternal/mlx+", + "-Iexternal/mlx+/mlx/backend/metal/kernels/metal_3_1", + # "-I.", + # "-Imlx/backend/metal/kernels/metal_3_1", + "-Wall", + "-Wextra", + "-fno-fast-math", + "-Wno-c++17-extensions", + # Need for Metal GPU trace + # "-gline-tables-only", + # "-frecord-sources", + ], + srcs = [ + "mlx/backend/metal/kernels/arg_reduce.metal", + "mlx/backend/metal/kernels/conv.metal", + "mlx/backend/metal/kernels/gemv.metal", + "mlx/backend/metal/kernels/layer_norm.metal", + "mlx/backend/metal/kernels/random.metal", + "mlx/backend/metal/kernels/rms_norm.metal", + "mlx/backend/metal/kernels/rope.metal", + "mlx/backend/metal/kernels/scaled_dot_product_attention.metal", + "mlx/backend/metal/kernels/fence.metal", + "mlx/backend/metal/kernels/steel/attn/kernels/steel_attention.metal", + # Disable JIT kernels (for default ops) + "mlx/backend/metal/kernels/arange.metal", + "mlx/backend/metal/kernels/binary.metal", + "mlx/backend/metal/kernels/binary_two.metal", + "mlx/backend/metal/kernels/copy.metal", + "mlx/backend/metal/kernels/fft.metal", + "mlx/backend/metal/kernels/reduce.metal", + "mlx/backend/metal/kernels/quantized.metal", + "mlx/backend/metal/kernels/scan.metal", + "mlx/backend/metal/kernels/softmax.metal", + "mlx/backend/metal/kernels/sort.metal", + "mlx/backend/metal/kernels/ternary.metal", + "mlx/backend/metal/kernels/unary.metal", + "mlx/backend/metal/kernels/steel/conv/kernels/steel_conv.metal", + "mlx/backend/metal/kernels/steel/conv/kernels/steel_conv_general.metal", + "mlx/backend/metal/kernels/steel/gemm/kernels/steel_gemm_fused.metal", + "mlx/backend/metal/kernels/steel/gemm/kernels/steel_gemm_masked.metal", + "mlx/backend/metal/kernels/steel/gemm/kernels/steel_gemm_splitk.metal", + "mlx/backend/metal/kernels/gemv_masked.metal", + ], + hdrs = [ + "mlx/backend/metal/kernels/metal_3_1/bf16.h", + "mlx/backend/metal/kernels/metal_3_0/bf16.h", + "mlx/backend/metal/kernels/bf16_math.h", + "mlx/backend/metal/kernels/complex.h", + "mlx/backend/metal/kernels/defines.h", + "mlx/backend/metal/kernels/erf.h", + "mlx/backend/metal/kernels/expm1f.h", + "mlx/backend/metal/kernels/utils.h", + # `conv` headers + "mlx/backend/metal/kernels/steel/conv/params.h", + # `scaled_dot_product_attention` headers + "mlx/backend/metal/kernels/sdpa_vector.h", + # Steel Headers + "mlx/backend/metal/kernels/steel/defines.h", + "mlx/backend/metal/kernels/steel/utils.h", + "mlx/backend/metal/kernels/steel/conv/conv.h", + "mlx/backend/metal/kernels/steel/conv/loader.h", + "mlx/backend/metal/kernels/steel/conv/loaders/loader_channel_l.h", + "mlx/backend/metal/kernels/steel/conv/loaders/loader_channel_n.h", + "mlx/backend/metal/kernels/steel/conv/loaders/loader_general.h", + "mlx/backend/metal/kernels/steel/conv/kernels/steel_conv.h", + "mlx/backend/metal/kernels/steel/conv/kernels/steel_conv_general.h", + "mlx/backend/metal/kernels/steel/gemm/gemm.h", + "mlx/backend/metal/kernels/steel/gemm/mma.h", + "mlx/backend/metal/kernels/steel/gemm/loader.h", + "mlx/backend/metal/kernels/steel/gemm/params.h", + "mlx/backend/metal/kernels/steel/gemm/transforms.h", + "mlx/backend/metal/kernels/steel/gemm/kernels/steel_gemm_fused.h", + "mlx/backend/metal/kernels/steel/gemm/kernels/steel_gemm_masked.h", + "mlx/backend/metal/kernels/steel/gemm/kernels/steel_gemm_splitk.h", + "mlx/backend/metal/kernels/steel/utils/type_traits.h", + "mlx/backend/metal/kernels/steel/utils/integral_constant.h", + # Steel Attn Headers (not repeating previously listed) + "mlx/backend/metal/kernels/steel/attn/attn.h", + "mlx/backend/metal/kernels/steel/attn/loader.h", + "mlx/backend/metal/kernels/steel/attn/mma.h", + "mlx/backend/metal/kernels/steel/attn/params.h", + "mlx/backend/metal/kernels/steel/attn/transforms.h", + "mlx/backend/metal/kernels/steel/attn/kernels/steel_attention.h", + # `arange` headers + "mlx/backend/metal/kernels/arange.h", + # `binary` headers + "mlx/backend/metal/kernels/binary.h", + "mlx/backend/metal/kernels/binary_ops.h", + # `binary_two` headers + "mlx/backend/metal/kernels/binary_two.h", + # `copy` headers + "mlx/backend/metal/kernels/copy.h", + # `fft` headers + "mlx/backend/metal/kernels/fft.h", + "mlx/backend/metal/kernels/fft/radix.h", + "mlx/backend/metal/kernels/fft/readwrite.h", + # `reduce` headers + "mlx/backend/metal/kernels/reduce.h", + "mlx/backend/metal/kernels/atomic.h", + "mlx/backend/metal/kernels/reduction/ops.h", + "mlx/backend/metal/kernels/reduction/reduce_init.h", + "mlx/backend/metal/kernels/reduction/reduce_all.h", + "mlx/backend/metal/kernels/reduction/reduce_col.h", + "mlx/backend/metal/kernels/reduction/reduce_row.h", + # `quantized` headers + "mlx/backend/metal/kernels/quantized.h", + # `scan` headers + "mlx/backend/metal/kernels/scan.h", + # `softmax` headers + "mlx/backend/metal/kernels/softmax.h", + # `sort` headers + "mlx/backend/metal/kernels/sort.h", + # `ternary` headers + "mlx/backend/metal/kernels/ternary.h", + "mlx/backend/metal/kernels/ternary_ops.h", + # `unary` headers + "mlx/backend/metal/kernels/unary.h", + "mlx/backend/metal/kernels/unary_ops.h", + # `gemv` headers + "mlx/backend/metal/kernels/gemv_masked.h", + ], + out = "mlx.metallib", +) + +cc_library( + name = "backend_gpu", + srcs = [ + "mlx/backend/gpu/copy.cpp", + "mlx/backend/gpu/primitives.cpp", + "mlx/backend/gpu/slicing.cpp", + # Other sources (maybe reorganize dependencies?) + ":mlx_h", + ], + hdrs = [ + "mlx/backend/gpu/available.h", + "mlx/backend/gpu/copy.h", + "mlx/backend/gpu/eval.h", + "mlx/backend/gpu/slicing.h", + ], + deps = [ + ":backend_common", + ], +) + +cc_library( + name = "mlx", + visibility = ["//visibility:public"], + data = [ + ":mlx_metallib", + ], + srcs = [ + "mlx/allocator.cpp", + "mlx/array.cpp", + "mlx/compile.cpp", + "mlx/device.cpp", + "mlx/dtype.cpp", + "mlx/dtype_utils.cpp", + "mlx/export.cpp", + "mlx/einsum.cpp", + "mlx/fast.cpp", + "mlx/fft.cpp", + "mlx/ops.cpp", + "mlx/graph_utils.cpp", + "mlx/primitives.cpp", + "mlx/random.cpp", + "mlx/scheduler.cpp", + "mlx/transforms.cpp", + "mlx/utils.cpp", + "mlx/linalg.cpp", + "mlx/backend/metal/metal.h", + ":mlx_h", + ], + hdrs = [ + "mlx/mlx.h", + "mlx/compile_impl.h", + ], + deps = [ + ":io", + ":distributed", + ":backend_common", + ":backend_metal", + ":backend_gpu", + ":backend_no_cpu", + # TODO (@cryptodeal): enable cpu backend once changes to `sort`, and + # comparison ops have been made for parity with metal backend changes. + # ":backend_cpu", + ], +) diff --git a/third_party/modules/mlx/20250605.0-c763fe1/overlay/MODULE.bazel b/third_party/modules/mlx/20250605.0-c763fe1/overlay/MODULE.bazel new file mode 120000 index 00000000..9b599e3a --- /dev/null +++ b/third_party/modules/mlx/20250605.0-c763fe1/overlay/MODULE.bazel @@ -0,0 +1 @@ +../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/mlx/20250605.0-c763fe1/overlay/jit.bzl b/third_party/modules/mlx/20250605.0-c763fe1/overlay/jit.bzl new file mode 100644 index 00000000..ca877e9b --- /dev/null +++ b/third_party/modules/mlx/20250605.0-c763fe1/overlay/jit.bzl @@ -0,0 +1,28 @@ +def make_jit_source(name, deps = [], **kwargs): + """ + Bazel equivalent of the CMake make_jit_source function. + Generates a C++ source file from a Metal header file by running a script. + + Args: + name: The Metal header file to process. + deps: Additional dependencies required for processing. + kwargs: Additional arguments to pass to the native.genrule function. + """ + src_name = name.split("/")[-1] # Extract the filename + output_file = "mlx/backend/metal/jit/{}.cpp".format(src_name) + + native.genrule( + name = name, + srcs = ["mlx/backend/metal/kernels/{}.h".format(src_name)] + deps, + outs = [output_file], + cmd = ( + "bash $(location :mlx/backend/metal/make_compiled_preamble.sh) $@ $(location mlx/backend/metal/kernels/{}.h)" + ).format(name), + tools = [":mlx/backend/metal/make_compiled_preamble.sh"], + **kwargs, + ) + + # native.cc_library( + # name = name + "_jit_lib", + # srcs = [output_file], + # ) diff --git a/third_party/modules/mlx/20250605.0-c763fe1/overlay/metal_cpp.bzl b/third_party/modules/mlx/20250605.0-c763fe1/overlay/metal_cpp.bzl new file mode 100644 index 00000000..64874afe --- /dev/null +++ b/third_party/modules/mlx/20250605.0-c763fe1/overlay/metal_cpp.bzl @@ -0,0 +1,23 @@ +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +def _metal_cpp_impl(mctx): + http_archive( + name = "metal_cpp", + build_file_content = """\ +cc_library( + name = "metal_cpp", + hdrs = glob(["metal-cpp/**/*.hpp"]), + strip_include_prefix = "metal-cpp", + visibility = ["//visibility:public"], +) +""", + sha256 = "0433df1e0ab13c2b0becbd78665071e3fa28381e9714a3fce28a497892b8a184", + url = "https://developer.apple.com/metal/cpp/files/metal-cpp_macOS15_iOS18.zip", + ) + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = "all", + root_module_direct_dev_deps = [], + ) +metal_cpp = module_extension( + implementation = _metal_cpp_impl, +) diff --git a/third_party/modules/mlx/20250605.0-c763fe1/patches/0001-Modify-Sort-And-Comparison-Kernels.patch b/third_party/modules/mlx/20250605.0-c763fe1/patches/0001-Modify-Sort-And-Comparison-Kernels.patch new file mode 100644 index 00000000..e0c2644e --- /dev/null +++ b/third_party/modules/mlx/20250605.0-c763fe1/patches/0001-Modify-Sort-And-Comparison-Kernels.patch @@ -0,0 +1,2623 @@ +diff --git a/.bazelrc b/.bazelrc +new file mode 100644 +index 000000000..562cc0ce7 +--- /dev/null ++++ b/.bazelrc +@@ -0,0 +1,6 @@ ++build --cxxopt='-std=c++17' ++ ++# Sets the MLX debug flag (for capturing Metal traces) ++# build --define=MLX_METAL_DEBUG=ON ++# run --define=MLX_METAL_DEBUG=ON ++ +diff --git a/.bazelversion b/.bazelversion +new file mode 100644 +index 000000000..2b0aa2121 +--- /dev/null ++++ b/.bazelversion +@@ -0,0 +1 @@ ++8.2.1 +diff --git a/.gitignore b/.gitignore +index 43629548d..0b74c60d8 100644 +--- a/.gitignore ++++ b/.gitignore +@@ -1,3 +1,8 @@ ++# Bazel symlinks ++bazel-* ++ ++# macOS ++.DS_Store + # Byte-compiled / optimized / DLL files + __pycache__/ + *.py[cod] +diff --git a/examples/cpp/BUILD.bazel b/examples/cpp/BUILD.bazel +new file mode 100644 +index 000000000..28ed3894b +--- /dev/null ++++ b/examples/cpp/BUILD.bazel +@@ -0,0 +1,23 @@ ++cc_binary( ++ name = "tutorial", ++ srcs = ["tutorial.cpp"], ++ deps = ["@mlx//:mlx"] ++) ++ ++cc_binary( ++ name = "linear_regression", ++ srcs = [ ++ "linear_regression.cpp", ++ "timer.h", ++ ], ++ deps = ["@mlx//:mlx"] ++) ++ ++cc_binary( ++ name = "logistic_regression", ++ srcs = [ ++ "linear_regression.cpp", ++ "timer.h", ++ ], ++ deps = ["@mlx//:mlx"] ++) +diff --git a/mlx/array.h b/mlx/array.h +index 98eef2e33..54ee53402 100644 +--- a/mlx/array.h ++++ b/mlx/array.h +@@ -517,9 +517,10 @@ template + array::array( + std::initializer_list data, + Dtype dtype /* = TypeToDtype() */) +- : array_desc_(std::make_shared( +- Shape{static_cast(data.size())}, +- dtype)) { ++ : array_desc_( ++ std::make_shared( ++ Shape{static_cast(data.size())}, ++ dtype)) { + init(data.begin()); + } + +diff --git a/mlx/backend/cpu/make_compiled_preamble.sh b/mlx/backend/cpu/make_compiled_preamble.sh +index 88b4c4615..ca567d83e 100644 +--- a/mlx/backend/cpu/make_compiled_preamble.sh ++++ b/mlx/backend/cpu/make_compiled_preamble.sh +@@ -7,13 +7,17 @@ + + + OUTPUT_FILE=$1 +-GCC=$2 +-SRCDIR=$3 +-CLANG=$4 +-ARCH=$5 ++HDR_FILE=$(realpath $2) + +-if [ "$CLANG" = "TRUE" ]; then +- read -r -d '' INCLUDES <<- EOM ++ARCH=$(uname -m) ++ ++CPUDIR=$(dirname $HDR_FILE) ++# echo "CPUDIR: $CPUDIR" ++SRCDIR=$(echo "$CPUDIR" | cut -d'/' -f-$(($(echo "$CPUDIR" | awk -F'/' '{print NF}')-3))) ++# echo "SRCDIR: $SRCDIR" ++ ++ ++read -r -d '' INCLUDES <<- EOM + #include + #include + #include +@@ -23,11 +27,8 @@ if [ "$CLANG" = "TRUE" ]; then + #endif + EOM + CC_FLAGS="-arch ${ARCH} -nobuiltininc -nostdinc" +-else +-CC_FLAGS="-std=c++17" +-fi + +-CONTENT=$($GCC $CC_FLAGS -I "$SRCDIR" -E -P "$SRCDIR/mlx/backend/cpu/compiled_preamble.h" 2>/dev/null) ++CONTENT=$(clang $CC_FLAGS -I "$SRCDIR" -E -P "$HDR_FILE" 2>/dev/null) + + cat << EOF > "$OUTPUT_FILE" + const char* get_kernel_preamble() { +diff --git a/mlx/backend/metal/device.cpp b/mlx/backend/metal/device.cpp +index ebc3cc77f..7bdca47d7 100644 +--- a/mlx/backend/metal/device.cpp ++++ b/mlx/backend/metal/device.cpp +@@ -10,6 +10,7 @@ + + #include "mlx/backend/metal/device.h" + #include "mlx/backend/metal/metal.h" ++#include "mlx/backend/metal/metal_path.h" + #include "mlx/backend/metal/utils.h" + #include "mlx/utils.h" + +diff --git a/mlx/backend/metal/eval.cpp b/mlx/backend/metal/eval.cpp +index 49783200a..fd9caf147 100644 +--- a/mlx/backend/metal/eval.cpp ++++ b/mlx/backend/metal/eval.cpp +@@ -4,12 +4,15 @@ + #include "mlx/backend/gpu/available.h" + #include "mlx/backend/gpu/eval.h" + #include "mlx/backend/metal/device.h" ++#include "mlx/backend/metal/thread_safety.h" + #include "mlx/backend/metal/utils.h" + #include "mlx/primitives.h" + #include "mlx/scheduler.h" + + namespace mlx::core::gpu { + ++std::mutex metal_operation_mutex; ++ + bool is_available() { + return true; + } +@@ -30,6 +33,7 @@ inline void check_error(MTL::CommandBuffer* cbuf) { + } + + void eval(array& arr) { ++ std::lock_guard lock(metal_operation_mutex); + auto pool = metal::new_scoped_memory_pool(); + auto s = arr.primitive().stream(); + auto& d = metal::device(s.device); +@@ -78,6 +82,7 @@ void eval(array& arr) { + } + + void finalize(Stream s) { ++ std::lock_guard lock(metal_operation_mutex); + auto pool = metal::new_scoped_memory_pool(); + auto& d = metal::device(s.device); + auto cb = d.get_command_buffer(s.index); +@@ -88,6 +93,7 @@ void finalize(Stream s) { + } + + void synchronize(Stream s) { ++ std::lock_guard lock(metal_operation_mutex); + auto pool = metal::new_scoped_memory_pool(); + auto& d = metal::device(s.device); + auto cb = d.get_command_buffer(s.index); +diff --git a/mlx/backend/metal/event.cpp b/mlx/backend/metal/event.cpp +index eb7f1b58a..5c9cc3808 100644 +--- a/mlx/backend/metal/event.cpp ++++ b/mlx/backend/metal/event.cpp +@@ -2,6 +2,7 @@ + + #include "mlx/event.h" + #include "mlx/backend/metal/device.h" ++#include "mlx/backend/metal/thread_safety.h" + #include "mlx/scheduler.h" + + namespace mlx::core { +@@ -27,6 +28,7 @@ void Event::wait(Stream stream) { + if (stream.device == Device::cpu) { + scheduler::enqueue(stream, [*this]() mutable { wait(); }); + } else { ++ std::lock_guard lock(gpu::metal_operation_mutex); + auto& d = metal::device(stream.device); + d.end_encoding(stream.index); + auto command_buffer = d.get_command_buffer(stream.index); +@@ -41,6 +43,7 @@ void Event::signal(Stream stream) { + static_cast(event_.get())->setSignaledValue(value()); + }); + } else { ++ std::lock_guard lock(gpu::metal_operation_mutex); + auto& d = metal::device(stream.device); + d.end_encoding(stream.index); + auto command_buffer = d.get_command_buffer(stream.index); +diff --git a/mlx/backend/metal/fence.cpp b/mlx/backend/metal/fence.cpp +index 5abdf7309..e1bfdaa9a 100644 +--- a/mlx/backend/metal/fence.cpp ++++ b/mlx/backend/metal/fence.cpp +@@ -1,6 +1,7 @@ + // Copyright © 2024 Apple Inc. + #include "mlx/fence.h" + #include "mlx/backend/metal/device.h" ++#include "mlx/backend/metal/thread_safety.h" + #include "mlx/scheduler.h" + #include "mlx/utils.h" + +@@ -68,6 +69,7 @@ void Fence::wait(Stream stream, const array& x) { + return; + } + ++ std::lock_guard lock(gpu::metal_operation_mutex); + auto& d = metal::device(stream.device); + auto idx = stream.index; + +@@ -116,6 +118,7 @@ void Fence::update(Stream stream, const array& x) { + return; + } + ++ std::lock_guard lock(gpu::metal_operation_mutex); + auto& d = metal::device(stream.device); + auto idx = stream.index; + if (!f.use_fast) { +diff --git a/mlx/backend/metal/jit_kernels.cpp b/mlx/backend/metal/jit_kernels.cpp +index 5206c9b54..6678e7acb 100644 +--- a/mlx/backend/metal/jit_kernels.cpp ++++ b/mlx/backend/metal/jit_kernels.cpp +@@ -328,7 +328,8 @@ MTL::ComputePipelineState* get_sort_kernel( + const array& in, + const array& out, + int bn, +- int tn) { ++ int tn, ++ const std::string& comparator) { + std::string lib_name = kernel_name.substr(kernel_name.find("_") + 1); + auto lib = d.get_library(lib_name, [&]() { + std::ostringstream kernel_source; +@@ -345,7 +346,8 @@ MTL::ComputePipelineState* get_sort_kernel( + out_type, + bool_string, + bn, +- tn); ++ tn, ++ comparator); + kernel_source << get_template_definition( + "n" + func_string + lib_name, + "block_sort_nc", +@@ -353,7 +355,8 @@ MTL::ComputePipelineState* get_sort_kernel( + out_type, + bool_string, + bn, +- tn); ++ tn, ++ comparator); + } + return kernel_source.str(); + }); +@@ -366,7 +369,8 @@ MTL::ComputePipelineState* get_mb_sort_kernel( + const array& in, + const array& idx, + int bn, +- int tn) { ++ int tn, ++ const std::string& comparator) { + std::string lib_name = kernel_name.substr(kernel_name.find("_") + 1); + auto lib = d.get_library(lib_name, [&]() { + std::ostringstream kernel_source; +@@ -383,7 +387,8 @@ MTL::ComputePipelineState* get_mb_sort_kernel( + get_type_string(idx.dtype()), + "true", + bn, +- tn); ++ tn, ++ comparator); + } + return kernel_source.str(); + }); +diff --git a/mlx/backend/metal/kernels.h b/mlx/backend/metal/kernels.h +index 6d8864385..ff2dd14be 100644 +--- a/mlx/backend/metal/kernels.h ++++ b/mlx/backend/metal/kernels.h +@@ -79,7 +79,8 @@ MTL::ComputePipelineState* get_sort_kernel( + const array& in, + const array& out, + int bn, +- int tn); ++ int tn, ++ const std::string& comparator); + + MTL::ComputePipelineState* get_mb_sort_kernel( + metal::Device& d, +@@ -87,7 +88,8 @@ MTL::ComputePipelineState* get_mb_sort_kernel( + const array& in, + const array& idx, + int bn, +- int tn); ++ int tn, ++ const std::string& comparator); + + MTL::ComputePipelineState* get_reduce_init_kernel( + metal::Device& d, +diff --git a/mlx/backend/metal/kernels/binary_ops.h b/mlx/backend/metal/kernels/binary_ops.h +index 4aaf2b4da..2c6e27b18 100644 +--- a/mlx/backend/metal/kernels/binary_ops.h ++++ b/mlx/backend/metal/kernels/binary_ops.h +@@ -37,29 +37,14 @@ struct Divide { + return x / y; + } + }; +- + struct Remainder { + template +- metal::enable_if_t & !metal::is_signed_v, T> +- operator()(T x, T y) { ++ metal::enable_if_t, T> operator()(T x, T y) { + return x % y; + } + template +- metal::enable_if_t & metal::is_signed_v, T> +- operator()(T x, T y) { +- auto r = x % y; +- if (r != 0 && (r < 0 != y < 0)) { +- r += y; +- } +- return r; +- } +- template + metal::enable_if_t, T> operator()(T x, T y) { +- T r = fmod(x, y); +- if (r != 0 && (r < 0 != y < 0)) { +- r += y; +- } +- return r; ++ return fmod(x, y); + } + template <> + complex64_t operator()(complex64_t x, complex64_t y) { +diff --git a/mlx/backend/metal/kernels/complex.h b/mlx/backend/metal/kernels/complex.h +index c88002cb3..54fc8d520 100644 +--- a/mlx/backend/metal/kernels/complex.h ++++ b/mlx/backend/metal/kernels/complex.h +@@ -142,11 +142,12 @@ constexpr complex64_t operator/(float a, complex64_t b) { + constexpr complex64_t operator%(complex64_t a, complex64_t b) { + auto real = a.real - (b.real * static_cast(a.real / b.real)); + auto imag = a.imag - (b.imag * static_cast(a.imag / b.imag)); +- if (real != 0 && (real < 0 != b.real < 0)) { +- real += b.real; +- } +- if (imag != 0 && (imag < 0 != b.imag < 0)) { +- imag += b.imag; +- } ++ /* Disable to match `stablehlo` implementation semantics */ ++ // if (real != 0 && (real < 0 != b.real < 0)) { ++ // real += b.real; ++ // } ++ // if (imag != 0 && (imag < 0 != b.imag < 0)) { ++ // imag += b.imag; ++ // } + return {real, imag}; + } +diff --git a/mlx/backend/metal/kernels/sort.h b/mlx/backend/metal/kernels/sort.h +index b067150d8..a24e429fd 100644 +--- a/mlx/backend/metal/kernels/sort.h ++++ b/mlx/backend/metal/kernels/sort.h +@@ -28,6 +28,15 @@ struct LessThan { + } + }; + ++template ++struct GreaterThan { ++ static constexpr constant T init = Limits::min; ++ ++ METAL_FUNC bool operator()(T a, T b) { ++ return a > b; ++ } ++}; ++ + template < + typename ValT, + typename IdxT, +@@ -216,7 +225,7 @@ template < + bool ARG_SORT, + short BLOCK_THREADS, + short N_PER_THREAD, +- typename CompareOp = LessThan> ++ typename CompareOp> + struct KernelMergeSort { + using ValT = T; + using IdxT = uint; +@@ -278,7 +287,8 @@ template < + typename U, + bool ARG_SORT, + short BLOCK_THREADS, +- short N_PER_THREAD> ++ short N_PER_THREAD, ++ typename CompareOp = LessThan> + [[kernel, max_total_threads_per_threadgroup(BLOCK_THREADS)]] void block_sort( + const device T* inp [[buffer(0)]], + device U* out [[buffer(1)]], +@@ -290,7 +300,7 @@ template < + uint3 tid [[threadgroup_position_in_grid]], + uint3 lid [[thread_position_in_threadgroup]]) { + using sort_kernel = +- KernelMergeSort; ++ KernelMergeSort; + using ValT = typename sort_kernel::ValT; + using IdxT = typename sort_kernel::IdxT; + +@@ -333,7 +343,8 @@ template < + typename U, + bool ARG_SORT, + short BLOCK_THREADS, +- short N_PER_THREAD> ++ short N_PER_THREAD, ++ typename CompareOp> + [[kernel, max_total_threads_per_threadgroup(BLOCK_THREADS)]] void block_sort_nc( + const device T* inp [[buffer(0)]], + device U* out [[buffer(1)]], +@@ -347,7 +358,7 @@ template < + uint3 tid [[threadgroup_position_in_grid]], + uint3 lid [[thread_position_in_threadgroup]]) { + using sort_kernel = +- KernelMergeSort; ++ KernelMergeSort; + using ValT = typename sort_kernel::ValT; + using IdxT = typename sort_kernel::IdxT; + +@@ -394,7 +405,7 @@ template < + bool ARG_SORT, + short BLOCK_THREADS, + short N_PER_THREAD, +- typename CompareOp = LessThan> ++ typename CompareOp> + struct KernelMultiBlockMergeSort { + using block_merge_sort_t = BlockMergeSort< + ValT, +@@ -476,7 +487,8 @@ template < + typename IdxT, + bool ARG_SORT, + short BLOCK_THREADS, +- short N_PER_THREAD> ++ short N_PER_THREAD, ++ typename CompareOp> + [[kernel, max_total_threads_per_threadgroup(BLOCK_THREADS)]] void mb_block_sort( + const device ValT* inp [[buffer(0)]], + device ValT* out_vals [[buffer(1)]], +@@ -493,7 +505,8 @@ template < + IdxT, + ARG_SORT, + BLOCK_THREADS, +- N_PER_THREAD>; ++ N_PER_THREAD, ++ CompareOp>; + + auto block_idx = elem_to_loc(tid.y, nc_shape, nc_strides, nc_dim); + inp += block_idx; +@@ -520,7 +533,8 @@ template < + typename IdxT, + bool ARG_SORT, + short BLOCK_THREADS, +- short N_PER_THREAD> ++ short N_PER_THREAD, ++ typename CompareOp> + [[kernel]] void mb_block_partition( + device IdxT* block_partitions [[buffer(0)]], + const device ValT* dev_vals [[buffer(1)]], +@@ -536,7 +550,8 @@ template < + IdxT, + ARG_SORT, + BLOCK_THREADS, +- N_PER_THREAD>; ++ N_PER_THREAD, ++ CompareOp>; + + block_partitions += tid.y * tgp_dims.x; + dev_vals += tid.y * size_sorted_axis; +@@ -573,7 +588,7 @@ template < + bool ARG_SORT, + short BLOCK_THREADS, + short N_PER_THREAD, +- typename CompareOp = LessThan> ++ typename CompareOp> + [[kernel, max_total_threads_per_threadgroup(BLOCK_THREADS)]] void + mb_block_merge( + const device IdxT* block_partitions [[buffer(0)]], +diff --git a/mlx/backend/metal/kernels/sort.metal b/mlx/backend/metal/kernels/sort.metal +index 7f198d55d..f1ec796c9 100644 +--- a/mlx/backend/metal/kernels/sort.metal ++++ b/mlx/backend/metal/kernels/sort.metal +@@ -6,24 +6,29 @@ + #include "mlx/backend/metal/kernels/utils.h" + #include "mlx/backend/metal/kernels/sort.h" + ++#define instantiate_comparator(name, type) \ ++ name ++ + #define instantiate_block_sort( \ +- name, itname, itype, otname, otype, arg_sort, bn, tn) \ +- instantiate_kernel("c" #name "_" #itname "_" #otname "_bn" #bn "_tn" #tn, \ +- block_sort, itype, otype, arg_sort, bn, tn) \ +- instantiate_kernel("nc" #name "_" #itname "_" #otname "_bn" #bn "_tn" #tn, \ +- block_sort_nc, itype, otype, arg_sort, bn, tn) ++ name, itname, itype, otname, otype, arg_sort, bn, tn, vcomparator) \ ++ instantiate_kernel("c" #name "_" #itname "_" #otname "_bn" #bn "_tn" #tn "_comp" #vcomparator, \ ++ block_sort, itype, otype, arg_sort, bn, tn, instantiate_comparator(vcomparator, itype)) \ ++ instantiate_kernel("nc" #name "_" #itname "_" #otname "_bn" #bn "_tn" #tn "_comp" #vcomparator, \ ++ block_sort_nc, itype, otype, arg_sort, bn, tn, instantiate_comparator(vcomparator, itype)) + +-#define instantiate_arg_block_sort_base(itname, itype, bn, tn) \ ++#define instantiate_arg_block_sort_base(itname, itype, bn, tn, comparator) \ + instantiate_block_sort( \ +- arg_block_sort, itname, itype, uint32, uint32_t, true, bn, tn) ++ arg_block_sort, itname, itype, uint32, uint32_t, true, bn, tn, comparator) + +-#define instantiate_block_sort_base(itname, itype, bn, tn) \ ++#define instantiate_block_sort_base(itname, itype, bn, tn, comparator) \ + instantiate_block_sort( \ +- _block_sort, itname, itype, itname, itype, false, bn, tn) ++ _block_sort, itname, itype, itname, itype, false, bn, tn, comparator) + + #define instantiate_block_sort_tn(itname, itype, bn) \ +- instantiate_block_sort_base(itname, itype, bn, 4) \ +- instantiate_arg_block_sort_base(itname, itype, bn, 4) ++ instantiate_block_sort_base(itname, itype, bn, 4, LessThan) \ ++ instantiate_block_sort_base(itname, itype, bn, 4, GreaterThan) \ ++ instantiate_arg_block_sort_base(itname, itype, bn, 4, LessThan) \ ++ instantiate_arg_block_sort_base(itname, itype, bn, 4, GreaterThan) + + #define instantiate_block_sort_bn(itname, itype) \ + instantiate_block_sort_tn(itname, itype, 32) \ +@@ -52,16 +57,17 @@ instantiate_block_sort_long(uint64, uint64_t) + instantiate_block_sort_long(int64, int64_t) + + #define instantiate_multi_block_sort( \ +- vtname, vtype, itname, itype, arg_sort, bn, tn) \ +- instantiate_kernel("sort_mbsort_" #vtname "_" #itname "_bn" #bn "_tn" #tn, \ +- mb_block_sort, vtype, itype, arg_sort, bn, tn) \ +- instantiate_kernel("partition_mbsort_" #vtname "_" #itname "_bn" #bn "_tn" #tn, \ +- mb_block_partition, vtype, itype, arg_sort, bn, tn) \ +- instantiate_kernel("merge_mbsort_" #vtname "_" #itname "_bn" #bn "_tn" #tn, \ +- mb_block_merge, vtype, itype, arg_sort, bn, tn) ++ vtname, vtype, itname, itype, arg_sort, bn, tn, vcomparator) \ ++ instantiate_kernel("sort_mbsort_" #vtname "_" #itname "_bn" #bn "_tn" #tn "_comp" #vcomparator, \ ++ mb_block_sort, vtype, itype, arg_sort, bn, tn, instantiate_comparator(vcomparator, vtype)) \ ++ instantiate_kernel("partition_mbsort_" #vtname "_" #itname "_bn" #bn "_tn" #tn "_comp" #vcomparator, \ ++ mb_block_partition, vtype, itype, arg_sort, bn, tn, instantiate_comparator(vcomparator, vtype)) \ ++ instantiate_kernel("merge_mbsort_" #vtname "_" #itname "_bn" #bn "_tn" #tn "_comp" #vcomparator, \ ++ mb_block_merge, vtype, itype, arg_sort, bn, tn, instantiate_comparator(vcomparator, vtype)) + + #define instantiate_multi_block_sort_base(vtname, vtype) \ +- instantiate_multi_block_sort(vtname, vtype, uint32, uint32_t, true, 512, 4) ++ instantiate_multi_block_sort(vtname, vtype, uint32, uint32_t, true, 512, 4, LessThan) \ ++ instantiate_multi_block_sort(vtname, vtype, uint32, uint32_t, true, 512, 4, GreaterThan) + + instantiate_multi_block_sort_base(uint8, uint8_t) + instantiate_multi_block_sort_base(uint16, uint16_t) +@@ -74,7 +80,8 @@ instantiate_multi_block_sort_base(float32, float) + instantiate_multi_block_sort_base(bfloat16, bfloat16_t) + + #define instantiate_multi_block_sort_long(vtname, vtype) \ +- instantiate_multi_block_sort(vtname, vtype, uint32, uint32_t, true, 256, 4) ++ instantiate_multi_block_sort(vtname, vtype, uint32, uint32_t, true, 256, 4, LessThan) \ ++ instantiate_multi_block_sort(vtname, vtype, uint32, uint32_t, true, 256, 4, GreaterThan) + + instantiate_multi_block_sort_long(uint64, uint64_t) + instantiate_multi_block_sort_long(int64, int64_t) // clang-format on +diff --git a/mlx/backend/metal/make_compiled_preamble.sh b/mlx/backend/metal/make_compiled_preamble.sh +index 425cd8d70..a48d00808 100644 +--- a/mlx/backend/metal/make_compiled_preamble.sh ++++ b/mlx/backend/metal/make_compiled_preamble.sh +@@ -5,18 +5,21 @@ + # + # Copyright © 2023-24 Apple Inc. + +-OUTPUT_DIR=$1 +-CC=$2 +-SRC_DIR=$3 +-SRC_FILE=$4 +-CFLAGS=$5 +-SRC_NAME=$(basename -- "${SRC_FILE}") +-JIT_INCLUDES=${SRC_DIR}/mlx/backend/metal/kernels/jit +-INPUT_FILE=${SRC_DIR}/mlx/backend/metal/kernels/${SRC_FILE}.h +-OUTPUT_FILE=${OUTPUT_DIR}/${SRC_NAME}.cpp ++OUTPUT_FILE=$1 ++INPUT_FILE=$2 ++CFLAGS=$3 + +-mkdir -p "$OUTPUT_DIR" +-CONTENT=$($CC -I"$SRC_DIR" -I"$JIT_INCLUDES" -DMLX_METAL_JIT -E -P "$INPUT_FILE" $CFLAGS 2>/dev/null) ++KERNEL_DIR=$(dirname $(realpath $INPUT_FILE)) ++# echo "KERNEL_DIR: $KERNEL_DIR" ++SRC_DIR=$(echo "$KERNEL_DIR" | cut -d'/' -f-$(($(echo "$KERNEL_DIR" | awk -F'/' '{print NF}')-4))) ++# echo "SRC_DIR: $SRC_DIR" ++JIT_INCLUDES="$KERNEL_DIR/jit" ++# echo "JIT_INCLUDES: $JIT_INCLUDES" ++ ++SRC_BASENAME=$(basename -- "${INPUT_FILE}") ++SRC_NAME="${SRC_BASENAME%.*}" ++ ++CONTENT=$(clang -I"$SRC_DIR" -I"$JIT_INCLUDES" -DMLX_METAL_JIT -E -P "$INPUT_FILE" $CFLAGS 2>/dev/null) + + cat << EOF > "$OUTPUT_FILE" + namespace mlx::core::metal { +diff --git a/mlx/backend/metal/nojit_kernels.cpp b/mlx/backend/metal/nojit_kernels.cpp +index 8da147971..1098f9188 100644 +--- a/mlx/backend/metal/nojit_kernels.cpp ++++ b/mlx/backend/metal/nojit_kernels.cpp +@@ -96,7 +96,8 @@ MTL::ComputePipelineState* get_sort_kernel( + const array&, + const array&, + int, +- int) { ++ int, ++ const std::string&) { + return d.get_kernel(kernel_name); + } + +@@ -106,7 +107,8 @@ MTL::ComputePipelineState* get_mb_sort_kernel( + const array&, + const array&, + int, +- int) { ++ int, ++ const std::string&) { + return d.get_kernel(kernel_name); + } + +diff --git a/mlx/backend/metal/sort.cpp b/mlx/backend/metal/sort.cpp +index 3c84022f2..bca888f66 100644 +--- a/mlx/backend/metal/sort.cpp ++++ b/mlx/backend/metal/sort.cpp +@@ -6,7 +6,9 @@ + #include "mlx/backend/metal/device.h" + #include "mlx/backend/metal/kernels.h" + #include "mlx/backend/metal/utils.h" ++#include "mlx/dtype_utils.h", + #include "mlx/primitives.h" ++#include "mlx/sort.h" + + namespace mlx::core { + +@@ -20,7 +22,8 @@ void single_block_sort( + int axis, + int bn, + int tn, +- bool argsort) { ++ bool argsort, ++ ComparatorType comparator_) { + // Prepare shapes + int n_rows = in.size() / in.shape(axis); + +@@ -58,9 +61,26 @@ void single_block_sort( + kname << "arg"; + } + ++ std::string comparator; ++ switch (comparator_) { ++ case ComparatorType::LessThan: ++ comparator = "LessThan"; ++ break; ++ case ComparatorType::GreaterThan: ++ comparator = "GreaterThan"; ++ break; ++ } ++ + kname << "_block_sort_" << type_to_name(in) << "_" << type_to_name(out) +- << "_bn" << bn << "_tn" << tn; +- auto kernel = get_sort_kernel(d, kname.str(), in, out, bn, tn); ++ << "_bn" << bn << "_tn" << tn << "_comp" << comparator; ++ auto kernel = get_sort_kernel( ++ d, ++ kname.str(), ++ in, ++ out, ++ bn, ++ tn, ++ comparator + "<" + dtype_to_string(in.dtype()) + ">"); + + // Prepare command encoder + auto& compute_encoder = d.get_command_encoder(s.index); +@@ -120,7 +140,8 @@ void multi_block_sort( + int bn, + int tn, + int n_blocks, +- bool argsort) { ++ bool argsort, ++ ComparatorType comparator_) { + // Prepare shapes + int n_rows = in.size() / in.shape(axis); + +@@ -159,6 +180,16 @@ void multi_block_sort( + std::vector copies = { + dev_vals_0, dev_vals_1, dev_idxs_0, dev_idxs_1, block_partitions}; + ++ std::string comparator; ++ switch (comparator_) { ++ case ComparatorType::LessThan: ++ comparator = "LessThan"; ++ break; ++ case ComparatorType::GreaterThan: ++ comparator = "GreaterThan"; ++ break; ++ } ++ + // Prepare command encoder + auto& compute_encoder = d.get_command_encoder(s.index); + +@@ -166,9 +197,16 @@ void multi_block_sort( + { + std::ostringstream kname; + kname << "sort_mbsort_" << type_to_name(dev_vals_0) << "_" +- << type_to_name(dev_idxs_0) << "_bn" << bn << "_tn" << tn; +- auto kernel = +- get_mb_sort_kernel(d, kname.str(), dev_vals_0, dev_idxs_0, bn, tn); ++ << type_to_name(dev_idxs_0) << "_bn" << bn << "_tn" << tn << "_comp" ++ << comparator; ++ auto kernel = get_mb_sort_kernel( ++ d, ++ kname.str(), ++ dev_vals_0, ++ dev_idxs_0, ++ bn, ++ tn, ++ comparator + "<" + dtype_to_string(in.dtype()) + ">"); + compute_encoder.set_compute_pipeline_state(kernel); + + compute_encoder.set_input_array(in, 0); +@@ -206,10 +244,17 @@ void multi_block_sort( + { + std::ostringstream kname; + kname << "partition_mbsort_" << type_to_name(dev_vals_in) << "_" +- << type_to_name(dev_idxs_in) << "_bn" << bn << "_tn" << tn; +- +- auto kernel = +- get_mb_sort_kernel(d, kname.str(), dev_vals_0, dev_idxs_0, bn, tn); ++ << type_to_name(dev_idxs_in) << "_bn" << bn << "_tn" << tn ++ << "_comp" << comparator; ++ ++ auto kernel = get_mb_sort_kernel( ++ d, ++ kname.str(), ++ dev_vals_0, ++ dev_idxs_0, ++ bn, ++ tn, ++ comparator + "<" + dtype_to_string(in.dtype()) + ">"); + compute_encoder.set_compute_pipeline_state(kernel); + + compute_encoder.set_output_array(block_partitions, 0); +@@ -229,10 +274,17 @@ void multi_block_sort( + { + std::ostringstream kname; + kname << "merge_mbsort_" << type_to_name(dev_vals_in) << "_" +- << type_to_name(dev_idxs_in) << "_bn" << bn << "_tn" << tn; +- +- auto kernel = +- get_mb_sort_kernel(d, kname.str(), dev_vals_0, dev_idxs_0, bn, tn); ++ << type_to_name(dev_idxs_in) << "_bn" << bn << "_tn" << tn ++ << "_comp" << comparator; ++ ++ auto kernel = get_mb_sort_kernel( ++ d, ++ kname.str(), ++ dev_vals_0, ++ dev_idxs_0, ++ bn, ++ tn, ++ comparator + "<" + dtype_to_string(in.dtype()) + ">"); + compute_encoder.set_compute_pipeline_state(kernel); + + compute_encoder.set_input_array(block_partitions, 0); +@@ -277,7 +329,8 @@ void gpu_merge_sort( + const array& in, + array& out, + int axis_, +- bool argsort) { ++ bool argsort, ++ ComparatorType comparator) { + // Get size info + int axis = axis_ < 0 ? axis_ + in.ndim() : axis_; + int size_sorted_axis = in.shape(axis); +@@ -307,9 +360,10 @@ void gpu_merge_sort( + int n_blocks = (size_sorted_axis + n_per_block - 1) / n_per_block; + + if (n_blocks > 1) { +- return multi_block_sort(s, d, in, out, axis, bn, tn, n_blocks, argsort); ++ return multi_block_sort( ++ s, d, in, out, axis, bn, tn, n_blocks, argsort, comparator); + } else { +- return single_block_sort(s, d, in, out, axis, bn, tn, argsort); ++ return single_block_sort(s, d, in, out, axis, bn, tn, argsort, comparator); + } + } + +@@ -324,7 +378,7 @@ void ArgSort::eval_gpu(const std::vector& inputs, array& out) { + auto& d = metal::device(s.device); + auto& in = inputs[0]; + +- gpu_merge_sort(s, d, in, out, axis_, true); ++ gpu_merge_sort(s, d, in, out, axis_, true, comparator_); + } + + void Sort::eval_gpu(const std::vector& inputs, array& out) { +@@ -336,7 +390,7 @@ void Sort::eval_gpu(const std::vector& inputs, array& out) { + auto& d = metal::device(s.device); + auto& in = inputs[0]; + +- gpu_merge_sort(s, d, in, out, axis_, false); ++ gpu_merge_sort(s, d, in, out, axis_, false, comparator_); + } + + void ArgPartition::eval_gpu(const std::vector& inputs, array& out) { +@@ -349,7 +403,7 @@ void ArgPartition::eval_gpu(const std::vector& inputs, array& out) { + auto& d = metal::device(s.device); + auto& in = inputs[0]; + +- gpu_merge_sort(s, d, in, out, axis_, true); ++ gpu_merge_sort(s, d, in, out, axis_, true, comparator_); + } + + void Partition::eval_gpu(const std::vector& inputs, array& out) { +@@ -362,7 +416,7 @@ void Partition::eval_gpu(const std::vector& inputs, array& out) { + auto& d = metal::device(s.device); + auto& in = inputs[0]; + +- gpu_merge_sort(s, d, in, out, axis_, false); ++ gpu_merge_sort(s, d, in, out, axis_, false, comparator_); + } + + } // namespace mlx::core +diff --git a/mlx/backend/metal/thread_safety.h b/mlx/backend/metal/thread_safety.h +new file mode 100644 +index 000000000..d4724a654 +--- /dev/null ++++ b/mlx/backend/metal/thread_safety.h +@@ -0,0 +1,7 @@ ++#pragma once ++ ++#include ++ ++namespace mlx::core::gpu { ++extern std::mutex metal_operation_mutex; ++} +\ No newline at end of file +diff --git a/mlx/backend/no_gpu/primitives.cpp b/mlx/backend/no_gpu/primitives.cpp +index 409aa2c89..6d49f1c60 100644 +--- a/mlx/backend/no_gpu/primitives.cpp ++++ b/mlx/backend/no_gpu/primitives.cpp +@@ -112,7 +112,7 @@ NO_GPU(Partition) + NO_GPU(Power) + NO_GPU_MULTI(QRF) + NO_GPU(QuantizedMatmul) +-NO_GPU(RandomBits) ++NO_GPU() + NO_GPU(Real) + NO_GPU(Reduce) + NO_GPU(Reshape) +diff --git a/mlx/compile.cpp b/mlx/compile.cpp +index 79a55ba8f..ae073facc 100644 +--- a/mlx/compile.cpp ++++ b/mlx/compile.cpp +@@ -194,6 +194,7 @@ std::vector Compiled::output_shapes(const std::vector& inputs) { + } + + namespace detail { ++std::mutex compilation_mutex; + + CompileMode& compile_mode() { + auto get_val = []() { +@@ -244,17 +245,6 @@ void merge(array& dst, array& src, ParentsMap& parents_map) { + merge_one(dests[i], sources[i], parents_map); + } + } +- +-template +-std::uintptr_t get_function_address(const std::function& fun) { +- using FunType = T (*)(U...); +- const FunType* fun_ptr = fun.template target(); +- if (fun_ptr == nullptr) { +- return 0; +- } +- return reinterpret_cast(*fun_ptr); +-} +- + class CompilerCache { + public: + struct CacheEntry { +diff --git a/mlx/compile.h b/mlx/compile.h +index a076cfbca..ecd7b27ff 100644 +--- a/mlx/compile.h ++++ b/mlx/compile.h +@@ -8,6 +8,9 @@ namespace mlx::core { + + enum class CompileMode { disabled, no_simplify, no_fuse, enabled }; + ++template ++std::uintptr_t get_function_address(const std::function& fun); ++ + /** Compile takes a function and returns a compiled function. */ + std::function(const std::vector&)> compile( + std::function(const std::vector&)> fun, +diff --git a/mlx/compile_impl.h b/mlx/compile_impl.h +index bc3f44b4e..d601cb181 100644 +--- a/mlx/compile_impl.h ++++ b/mlx/compile_impl.h +@@ -1,12 +1,14 @@ + // Copyright © 2023-2024 Apple Inc. + + #pragma once +- ++#include ++#include + #include + + #include "mlx/array.h" + + namespace mlx::core::detail { ++extern std::mutex compilation_mutex; + + // This is not part of the general C++ API as calling with a bad id is a bad + // idea. +@@ -19,6 +21,19 @@ std::function(const std::vector&)> compile( + // Erase cached compile functions + void compile_erase(std::uintptr_t fun_id); + ++template ++std::uintptr_t get_function_address(const std::function& fun) { ++ using FunType = T (*)(U...); ++ const FunType* fun_ptr = fun.template target(); ++ if (fun_ptr == nullptr) { ++ return 0; ++ } ++ return reinterpret_cast(*fun_ptr); ++} ++ ++// Returns true if `skip_compile` is set ++bool skip_compile(); ++ + // Clear the compiler cache causing a recompilation of all compiled functions + // when called again. + void compile_clear_cache(); +diff --git a/mlx/dtype_utils.h b/mlx/dtype_utils.h +index 55de890f2..9208285b2 100644 +--- a/mlx/dtype_utils.h ++++ b/mlx/dtype_utils.h +@@ -92,12 +92,13 @@ MLX_FORALL_DTYPES(SPECIALIZE_CppTypeToDtype) + break; \ + } + +-#define MLX_INTERNAL_SWITCH_CHECKED(TYPE, NAME, ...) \ +- switch (TYPE) { \ +- __VA_ARGS__ \ +- default: \ +- throw std::invalid_argument(fmt::format( \ +- "Unhandled dtype %s for %s", dtype_to_string(TYPE), NAME)); \ ++#define MLX_INTERNAL_SWITCH_CHECKED(TYPE, NAME, ...) \ ++ switch (TYPE) { \ ++ __VA_ARGS__ \ ++ default: \ ++ throw std::invalid_argument( \ ++ fmt::format( \ ++ "Unhandled dtype %s for %s", dtype_to_string(TYPE), NAME)); \ + } + + #define MLX_INTERNAL_SWITCH_CASE_INT_TYPES(CTYPE_ALIAS, ...) \ +diff --git a/mlx/mlx.h b/mlx/mlx.h +index cef8d806d..f8932873c 100644 +--- a/mlx/mlx.h ++++ b/mlx/mlx.h +@@ -17,6 +17,7 @@ + #include "mlx/memory.h" + #include "mlx/ops.h" + #include "mlx/random.h" ++#include "mlx/sort.h" + #include "mlx/stream.h" + #include "mlx/transforms.h" + #include "mlx/utils.h" +diff --git a/mlx/ops.cpp b/mlx/ops.cpp +index 9602f667a..82bc8883c 100644 +--- a/mlx/ops.cpp ++++ b/mlx/ops.cpp +@@ -2205,13 +2205,20 @@ array argmax( + } + + /** Returns a sorted copy of the flattened array. */ +-array sort(const array& a, StreamOrDevice s /* = {} */) { ++array sort( ++ const array& a, ++ ComparatorType comparator, ++ StreamOrDevice s /* = {} */) { + int size = a.size(); +- return sort(reshape(a, {size}, s), 0, s); ++ return sort(reshape(a, {size}, s), 0, comparator, s); + } + + /** Returns a sorted copy of the array along a given axis. */ +-array sort(const array& a, int axis, StreamOrDevice s /* = {} */) { ++array sort( ++ const array& a, ++ int axis, ++ ComparatorType comparator, ++ StreamOrDevice s /* = {} */) { + // Check for valid axis + if (axis + static_cast(a.ndim()) < 0 || + axis >= static_cast(a.ndim())) { +@@ -2222,17 +2229,27 @@ array sort(const array& a, int axis, StreamOrDevice s /* = {} */) { + } + + return array( +- a.shape(), a.dtype(), std::make_shared(to_stream(s), axis), {a}); ++ a.shape(), ++ a.dtype(), ++ std::make_shared(to_stream(s), axis, comparator), ++ {a}); + } + + /** Returns indices that sort the flattened array. */ +-array argsort(const array& a, StreamOrDevice s /* = {} */) { ++array argsort( ++ const array& a, ++ ComparatorType comparator, ++ StreamOrDevice s /* = {} */) { + int size = a.size(); +- return argsort(reshape(a, {size}, s), 0, s); ++ return argsort(reshape(a, {size}, s), 0, comparator, s); + } + + /** Returns indices that sort the array along a given axis. */ +-array argsort(const array& a, int axis, StreamOrDevice s /* = {} */) { ++array argsort( ++ const array& a, ++ int axis, ++ ComparatorType comparator, ++ StreamOrDevice s /* = {} */) { + // Check for valid axis + if (axis + static_cast(a.ndim()) < 0 || + axis >= static_cast(a.ndim())) { +@@ -2243,16 +2260,23 @@ array argsort(const array& a, int axis, StreamOrDevice s /* = {} */) { + } + + return array( +- a.shape(), uint32, std::make_shared(to_stream(s), axis), {a}); ++ a.shape(), ++ uint32, ++ std::make_shared(to_stream(s), axis, comparator), ++ {a}); + } + + /** + * Returns a partitioned copy of the flattened array + * such that the smaller kth elements are first. + **/ +-array partition(const array& a, int kth, StreamOrDevice s /* = {} */) { ++array partition( ++ const array& a, ++ int kth, ++ ComparatorType comparator, ++ StreamOrDevice s /* = {} */) { + int size = a.size(); +- return partition(reshape(a, {size}, s), kth, 0, s); ++ return partition(reshape(a, {size}, s), kth, 0, comparator, s); + } + + /** +@@ -2263,6 +2287,7 @@ array partition( + const array& a, + int kth, + int axis, ++ ComparatorType comparator, + StreamOrDevice s /* = {} */) { + // Check for valid axis + if (axis + static_cast(a.ndim()) < 0 || +@@ -2283,7 +2308,7 @@ array partition( + return array( + a.shape(), + a.dtype(), +- std::make_shared(to_stream(s), kth_, axis_), ++ std::make_shared(to_stream(s), kth_, axis_, comparator), + {a}); + } + +@@ -2291,9 +2316,13 @@ array partition( + * Returns indices that partition the flattened array + * such that the smaller kth elements are first. + **/ +-array argpartition(const array& a, int kth, StreamOrDevice s /* = {} */) { ++array argpartition( ++ const array& a, ++ int kth, ++ ComparatorType comparator, ++ StreamOrDevice s /* = {} */) { + int size = a.size(); +- return argpartition(reshape(a, {size}, s), kth, 0, s); ++ return argpartition(reshape(a, {size}, s), kth, 0, comparator, s); + } + + /** +@@ -2304,6 +2333,7 @@ array argpartition( + const array& a, + int kth, + int axis, ++ ComparatorType comparator, + StreamOrDevice s /* = {} */) { + // Check for valid axis + if (axis + static_cast(a.ndim()) < 0 || +@@ -2324,7 +2354,7 @@ array argpartition( + return array( + a.shape(), + uint32, +- std::make_shared(to_stream(s), kth_, axis_), ++ std::make_shared(to_stream(s), kth_, axis_, comparator), + {a}); + } + +diff --git a/mlx/ops.h b/mlx/ops.h +index af3cdb5bd..be762b8a6 100644 +--- a/mlx/ops.h ++++ b/mlx/ops.h +@@ -6,6 +6,7 @@ + + #include "mlx/array.h" + #include "mlx/device.h" ++#include "mlx/sort.h" + #include "mlx/stream.h" + #include "mlx/utils.h" + +@@ -674,40 +675,98 @@ array argmax( + StreamOrDevice s = {}); + + /** Returns a sorted copy of the flattened array. */ +-array sort(const array& a, StreamOrDevice s = {}); ++array sort( ++ const array& a, ++ ComparatorType comparator = ComparatorType::LessThan, ++ StreamOrDevice s = {}); ++inline array sort(const array& a, StreamOrDevice s = {}) { ++ return sort(a, ComparatorType::LessThan, s); ++} + + /** Returns a sorted copy of the array along a given axis. */ +-array sort(const array& a, int axis, StreamOrDevice s = {}); ++array sort( ++ const array& a, ++ int axis, ++ ComparatorType comparator = ComparatorType::LessThan, ++ StreamOrDevice s = {}); ++inline array sort(const array& a, int axis, StreamOrDevice s = {}) { ++ return sort(a, axis, ComparatorType::LessThan, s); ++} + + /** Returns indices that sort the flattened array. */ +-array argsort(const array& a, StreamOrDevice s = {}); ++array argsort( ++ const array& a, ++ ComparatorType comparator = ComparatorType::LessThan, ++ StreamOrDevice s = {}); ++inline array argsort(const array& a, StreamOrDevice s = {}) { ++ return argsort(a, ComparatorType::LessThan, s); ++} + + /** Returns indices that sort the array along a given axis. */ +-array argsort(const array& a, int axis, StreamOrDevice s = {}); ++array argsort( ++ const array& a, ++ int axis, ++ ComparatorType comparator = ComparatorType::LessThan, ++ StreamOrDevice s = {}); ++inline array argsort(const array& a, int axis, StreamOrDevice s = {}) { ++ return argsort(a, axis, ComparatorType::LessThan, s); ++} + + /** + * Returns a partitioned copy of the flattened array + * such that the smaller kth elements are first. + **/ +-array partition(const array& a, int kth, StreamOrDevice s = {}); ++array partition( ++ const array& a, ++ int kth, ++ ComparatorType comparator = ComparatorType::LessThan, ++ StreamOrDevice s = {}); ++inline array partition(const array& a, int kth, StreamOrDevice s = {}) { ++ return partition(a, kth, ComparatorType::LessThan, s); ++} + + /** + * Returns a partitioned copy of the array along a given axis + * such that the smaller kth elements are first. + **/ +-array partition(const array& a, int kth, int axis, StreamOrDevice s = {}); ++array partition( ++ const array& a, ++ int kth, ++ int axis, ++ ComparatorType comparator = ComparatorType::LessThan, ++ StreamOrDevice s = {}); ++inline array ++partition(const array& a, int kth, int axis, StreamOrDevice s = {}) { ++ return partition(a, kth, axis, ComparatorType::LessThan, s); ++} + + /** + * Returns indices that partition the flattened array + * such that the smaller kth elements are first. + **/ +-array argpartition(const array& a, int kth, StreamOrDevice s = {}); ++array argpartition( ++ const array& a, ++ int kth, ++ ComparatorType comparator = ComparatorType::LessThan, ++ StreamOrDevice s = {}); ++inline array argpartition(const array& a, int kth, StreamOrDevice s = {}) { ++ return argpartition(a, kth, ComparatorType::LessThan, s); ++} + + /** + * Returns indices that partition the array along a given axis + * such that the smaller kth elements are first. + **/ +-array argpartition(const array& a, int kth, int axis, StreamOrDevice s = {}); ++array argpartition( ++ const array& a, ++ int kth, ++ int axis, ++ ComparatorType comparator = ComparatorType::LessThan, ++ StreamOrDevice s = {}); ++inline array ++argpartition(const array& a, int kth, int axis, StreamOrDevice s = {}) { ++ return argpartition(a, kth, axis, ComparatorType::LessThan, s); ++} + + /** Returns topk elements of the flattened array. */ + array topk(const array& a, int k, StreamOrDevice s = {}); +diff --git a/mlx/primitives.h b/mlx/primitives.h +index cc60bcfb9..2b79c0d67 100644 +--- a/mlx/primitives.h ++++ b/mlx/primitives.h +@@ -7,6 +7,7 @@ + #include "mlx/array.h" + #include "mlx/device.h" + #include "mlx/io/load.h" ++#include "mlx/sort.h" + #include "mlx/stream.h" + + #define DEFINE_VMAP() \ +@@ -322,8 +323,15 @@ class ArcTanh : public UnaryPrimitive { + + class ArgPartition : public UnaryPrimitive { + public: +- explicit ArgPartition(Stream stream, int kth, int axis) +- : UnaryPrimitive(stream), kth_(kth), axis_(axis) {} ++ explicit ArgPartition( ++ Stream stream, ++ int kth, ++ int axis, ++ ComparatorType comparator) ++ : UnaryPrimitive(stream), ++ kth_(kth), ++ axis_(axis), ++ comparator_(comparator) {} + + void eval_cpu(const std::vector& inputs, array& out) override; + void eval_gpu(const std::vector& inputs, array& out) override; +@@ -333,13 +341,14 @@ class ArgPartition : public UnaryPrimitive { + DEFINE_PRINT(ArgPartition) + DEFINE_INPUT_OUTPUT_SHAPE() + bool is_equivalent(const Primitive& other) const override; +- std::pair state() const { +- return {kth_, axis_}; ++ auto state() const { ++ return std::make_tuple(kth_, axis_, comparator_); + }; + + private: + int kth_; + int axis_; ++ ComparatorType comparator_; + }; + + class ArgReduce : public UnaryPrimitive { +@@ -371,8 +380,8 @@ class ArgReduce : public UnaryPrimitive { + + class ArgSort : public UnaryPrimitive { + public: +- explicit ArgSort(Stream stream, int axis) +- : UnaryPrimitive(stream), axis_(axis) {} ++ explicit ArgSort(Stream stream, int axis, ComparatorType comparator) ++ : UnaryPrimitive(stream), axis_(axis), comparator_(comparator) {} + + void eval_cpu(const std::vector& inputs, array& out) override; + void eval_gpu(const std::vector& inputs, array& out) override; +@@ -381,12 +390,13 @@ class ArgSort : public UnaryPrimitive { + DEFINE_PRINT(ArgSort) + DEFINE_INPUT_OUTPUT_SHAPE() + bool is_equivalent(const Primitive& other) const override; +- int state() const { +- return axis_; ++ auto state() const { ++ return std::make_pair(axis_, comparator_); + }; + + private: + int axis_; ++ ComparatorType comparator_; + }; + + class AsType : public UnaryPrimitive { +@@ -1531,8 +1541,15 @@ class Pad : public UnaryPrimitive { + + class Partition : public UnaryPrimitive { + public: +- explicit Partition(Stream stream, int kth, int axis) +- : UnaryPrimitive(stream), kth_(kth), axis_(axis) {} ++ explicit Partition( ++ Stream stream, ++ int kth, ++ int axis, ++ ComparatorType comparator) ++ : UnaryPrimitive(stream), ++ kth_(kth), ++ axis_(axis), ++ comparator_(comparator) {} + + void eval_cpu(const std::vector& inputs, array& out) override; + void eval_gpu(const std::vector& inputs, array& out) override; +@@ -1543,12 +1560,13 @@ class Partition : public UnaryPrimitive { + DEFINE_INPUT_OUTPUT_SHAPE() + bool is_equivalent(const Primitive& other) const override; + auto state() const { +- return std::make_pair(kth_, axis_); ++ return std::make_tuple(kth_, axis_, comparator_); + }; + + private: + int kth_; + int axis_; ++ ComparatorType comparator_; + }; + + class Power : public UnaryPrimitive { +@@ -2074,8 +2092,8 @@ class Softmax : public UnaryPrimitive { + + class Sort : public UnaryPrimitive { + public: +- explicit Sort(Stream stream, int axis) +- : UnaryPrimitive(stream), axis_(axis) {} ++ explicit Sort(Stream stream, int axis, ComparatorType comparator) ++ : UnaryPrimitive(stream), axis_(axis), comparator_(comparator) {} + + void eval_cpu(const std::vector& inputs, array& out) override; + void eval_gpu(const std::vector& inputs, array& out) override; +@@ -2086,11 +2104,12 @@ class Sort : public UnaryPrimitive { + DEFINE_INPUT_OUTPUT_SHAPE() + bool is_equivalent(const Primitive& other) const override; + auto state() const { +- return axis_; ++ return std::make_pair(axis_, comparator_); + } + + private: + int axis_; ++ ComparatorType comparator_; + }; + + class Split : public Primitive { +diff --git a/mlx/random.cpp b/mlx/random.cpp +index 6c6d1eb95..15b1cff9e 100644 +--- a/mlx/random.cpp ++++ b/mlx/random.cpp +@@ -14,6 +14,7 @@ namespace mlx::core::random { + KeySequence::KeySequence(uint64_t seed) : key_(key(seed)) {} + + void KeySequence::seed(uint64_t seed) { ++ std::lock_guard lock(mutex_); + key_ = key((seed)); + } + +@@ -225,6 +226,23 @@ array normal( + return samples; + } + ++array normal( ++ const Shape& shape, ++ Dtype dtype, ++ const array& loc, ++ const array& scale, ++ const std::optional& key /*= nullopt */, ++ StreamOrDevice s /* = {} */) { ++ auto stream = to_stream(s); ++ auto low = above_minus_one_with_default(dtype); ++ auto high = array(1.0f, dtype); ++ auto samples = uniform(low, high, shape, dtype, key, stream); ++ samples = multiply(array(std::sqrt(2.0), dtype), erfinv(samples, stream), stream); ++ samples = multiply(astype(scale, dtype), samples, stream); ++ samples = add(astype(loc, dtype), samples, stream); ++ return samples; ++} ++ + array multivariate_normal( + const array& mean, + const array& cov, +diff --git a/mlx/random.h b/mlx/random.h +index 0dfdab7a1..9358dd900 100644 +--- a/mlx/random.h ++++ b/mlx/random.h +@@ -3,6 +3,7 @@ + #pragma once + + #include ++#include + #include + + #include "mlx/array.h" +@@ -26,6 +27,7 @@ class KeySequence { + + private: + array key_; ++ std::mutex mutex_; + static uint64_t get_current_time_seed() { + auto now = std::chrono::system_clock::now(); + return std::chrono::duration_cast( +@@ -134,6 +136,14 @@ inline array normal( + return normal(shape, float32, std::nullopt, std::nullopt, key, s); + } + ++array normal( ++ const Shape& shape, ++ Dtype dtype, ++ const array& loc, ++ const array& scale, ++ const std::optional& key = std::nullopt, ++ StreamOrDevice s = {}); ++ + /** Generate samples from a multivariate normal distribution. **/ + array multivariate_normal( + const array& mean, +diff --git a/mlx/sort.h b/mlx/sort.h +new file mode 100644 +index 000000000..0afc6b62b +--- /dev/null ++++ b/mlx/sort.h +@@ -0,0 +1,14 @@ ++// Copyright © 2023-2024 Apple Inc. ++ ++#pragma once ++ ++#include "mlx/array.h" ++ ++namespace mlx::core { ++ ++enum class ComparatorType { ++ LessThan, ++ GreaterThan, ++}; ++ ++} // namespace mlx::core +diff --git a/mlx/types/complex.h b/mlx/types/complex.h +index 51101cc97..344d80f87 100644 +--- a/mlx/types/complex.h ++++ b/mlx/types/complex.h +@@ -59,10 +59,11 @@ inline bool operator>(const complex64_t& a, const complex64_t& b) { + inline complex64_t operator%(complex64_t a, complex64_t b) { + auto real = a.real() - (b.real() * static_cast(a.real() / b.real())); + auto imag = a.imag() - (b.imag() * static_cast(a.imag() / b.imag())); +- if (real != 0 && ((real < 0) != (b.real() < 0))) +- real += b.real(); +- if (imag != 0 && ((imag < 0) != (b.imag() < 0))) +- imag += b.imag(); ++ /* Disable to match `stablehlo` implementation semantics */ ++ // if (real != 0 && ((real < 0) != (b.real() < 0))) ++ // real += b.real(); ++ // if (imag != 0 && ((imag < 0) != (b.imag() < 0))) ++ // imag += b.imag(); + return {real, imag}; + } + +diff --git a/test.patch b/test.patch +new file mode 100644 +index 000000000..dae2293ba +--- /dev/null ++++ b/test.patch +@@ -0,0 +1,1132 @@ ++diff --git a/.gitignore b/.gitignore ++index 43629548..0b74c60d 100644 ++--- a/.gitignore +++++ b/.gitignore ++@@ -1,3 +1,8 @@ +++# Bazel symlinks +++bazel-* +++ +++# macOS +++.DS_Store ++ # Byte-compiled / optimized / DLL files ++ __pycache__/ ++ *.py[cod] ++diff --git a/mlx/backend/metal/device.cpp b/mlx/backend/metal/device.cpp ++index ebc3cc77..7bdca47d 100644 ++--- a/mlx/backend/metal/device.cpp +++++ b/mlx/backend/metal/device.cpp ++@@ -10,6 +10,7 @@ ++ ++ #include "mlx/backend/metal/device.h" ++ #include "mlx/backend/metal/metal.h" +++#include "mlx/backend/metal/metal_path.h" ++ #include "mlx/backend/metal/utils.h" ++ #include "mlx/utils.h" ++ ++diff --git a/mlx/backend/metal/jit_kernels.cpp b/mlx/backend/metal/jit_kernels.cpp ++index 5206c9b5..6678e7ac 100644 ++--- a/mlx/backend/metal/jit_kernels.cpp +++++ b/mlx/backend/metal/jit_kernels.cpp ++@@ -328,7 +328,8 @@ MTL::ComputePipelineState* get_sort_kernel( ++ const array& in, ++ const array& out, ++ int bn, ++- int tn) { +++ int tn, +++ const std::string& comparator) { ++ std::string lib_name = kernel_name.substr(kernel_name.find("_") + 1); ++ auto lib = d.get_library(lib_name, [&]() { ++ std::ostringstream kernel_source; ++@@ -345,7 +346,8 @@ MTL::ComputePipelineState* get_sort_kernel( ++ out_type, ++ bool_string, ++ bn, ++- tn); +++ tn, +++ comparator); ++ kernel_source << get_template_definition( ++ "n" + func_string + lib_name, ++ "block_sort_nc", ++@@ -353,7 +355,8 @@ MTL::ComputePipelineState* get_sort_kernel( ++ out_type, ++ bool_string, ++ bn, ++- tn); +++ tn, +++ comparator); ++ } ++ return kernel_source.str(); ++ }); ++@@ -366,7 +369,8 @@ MTL::ComputePipelineState* get_mb_sort_kernel( ++ const array& in, ++ const array& idx, ++ int bn, ++- int tn) { +++ int tn, +++ const std::string& comparator) { ++ std::string lib_name = kernel_name.substr(kernel_name.find("_") + 1); ++ auto lib = d.get_library(lib_name, [&]() { ++ std::ostringstream kernel_source; ++@@ -383,7 +387,8 @@ MTL::ComputePipelineState* get_mb_sort_kernel( ++ get_type_string(idx.dtype()), ++ "true", ++ bn, ++- tn); +++ tn, +++ comparator); ++ } ++ return kernel_source.str(); ++ }); ++diff --git a/mlx/backend/metal/kernels.h b/mlx/backend/metal/kernels.h ++index 6d886438..ff2dd14b 100644 ++--- a/mlx/backend/metal/kernels.h +++++ b/mlx/backend/metal/kernels.h ++@@ -79,7 +79,8 @@ MTL::ComputePipelineState* get_sort_kernel( ++ const array& in, ++ const array& out, ++ int bn, ++- int tn); +++ int tn, +++ const std::string& comparator); ++ ++ MTL::ComputePipelineState* get_mb_sort_kernel( ++ metal::Device& d, ++@@ -87,7 +88,8 @@ MTL::ComputePipelineState* get_mb_sort_kernel( ++ const array& in, ++ const array& idx, ++ int bn, ++- int tn); +++ int tn, +++ const std::string& comparator); ++ ++ MTL::ComputePipelineState* get_reduce_init_kernel( ++ metal::Device& d, ++diff --git a/mlx/backend/metal/kernels/binary_ops.h b/mlx/backend/metal/kernels/binary_ops.h ++index 4aaf2b4d..2c6e27b1 100644 ++--- a/mlx/backend/metal/kernels/binary_ops.h +++++ b/mlx/backend/metal/kernels/binary_ops.h ++@@ -37,29 +37,14 @@ struct Divide { ++ return x / y; ++ } ++ }; ++- ++ struct Remainder { ++ template ++- metal::enable_if_t & !metal::is_signed_v, T> ++- operator()(T x, T y) { +++ metal::enable_if_t, T> operator()(T x, T y) { ++ return x % y; ++ } ++ template ++- metal::enable_if_t & metal::is_signed_v, T> ++- operator()(T x, T y) { ++- auto r = x % y; ++- if (r != 0 && (r < 0 != y < 0)) { ++- r += y; ++- } ++- return r; ++- } ++- template ++ metal::enable_if_t, T> operator()(T x, T y) { ++- T r = fmod(x, y); ++- if (r != 0 && (r < 0 != y < 0)) { ++- r += y; ++- } ++- return r; +++ return fmod(x, y); ++ } ++ template <> ++ complex64_t operator()(complex64_t x, complex64_t y) { ++diff --git a/mlx/backend/metal/kernels/complex.h b/mlx/backend/metal/kernels/complex.h ++index c88002cb..54fc8d52 100644 ++--- a/mlx/backend/metal/kernels/complex.h +++++ b/mlx/backend/metal/kernels/complex.h ++@@ -142,11 +142,12 @@ constexpr complex64_t operator/(float a, complex64_t b) { ++ constexpr complex64_t operator%(complex64_t a, complex64_t b) { ++ auto real = a.real - (b.real * static_cast(a.real / b.real)); ++ auto imag = a.imag - (b.imag * static_cast(a.imag / b.imag)); ++- if (real != 0 && (real < 0 != b.real < 0)) { ++- real += b.real; ++- } ++- if (imag != 0 && (imag < 0 != b.imag < 0)) { ++- imag += b.imag; ++- } +++ /* Disable to match `stablehlo` implementation semantics */ +++ // if (real != 0 && (real < 0 != b.real < 0)) { +++ // real += b.real; +++ // } +++ // if (imag != 0 && (imag < 0 != b.imag < 0)) { +++ // imag += b.imag; +++ // } ++ return {real, imag}; ++ } ++diff --git a/mlx/backend/metal/kernels/sort.h b/mlx/backend/metal/kernels/sort.h ++index b067150d..a24e429f 100644 ++--- a/mlx/backend/metal/kernels/sort.h +++++ b/mlx/backend/metal/kernels/sort.h ++@@ -28,6 +28,15 @@ struct LessThan { ++ } ++ }; ++ +++template +++struct GreaterThan { +++ static constexpr constant T init = Limits::min; +++ +++ METAL_FUNC bool operator()(T a, T b) { +++ return a > b; +++ } +++}; +++ ++ template < ++ typename ValT, ++ typename IdxT, ++@@ -216,7 +225,7 @@ template < ++ bool ARG_SORT, ++ short BLOCK_THREADS, ++ short N_PER_THREAD, ++- typename CompareOp = LessThan> +++ typename CompareOp> ++ struct KernelMergeSort { ++ using ValT = T; ++ using IdxT = uint; ++@@ -278,7 +287,8 @@ template < ++ typename U, ++ bool ARG_SORT, ++ short BLOCK_THREADS, ++- short N_PER_THREAD> +++ short N_PER_THREAD, +++ typename CompareOp = LessThan> ++ [[kernel, max_total_threads_per_threadgroup(BLOCK_THREADS)]] void block_sort( ++ const device T* inp [[buffer(0)]], ++ device U* out [[buffer(1)]], ++@@ -290,7 +300,7 @@ template < ++ uint3 tid [[threadgroup_position_in_grid]], ++ uint3 lid [[thread_position_in_threadgroup]]) { ++ using sort_kernel = ++- KernelMergeSort; +++ KernelMergeSort; ++ using ValT = typename sort_kernel::ValT; ++ using IdxT = typename sort_kernel::IdxT; ++ ++@@ -333,7 +343,8 @@ template < ++ typename U, ++ bool ARG_SORT, ++ short BLOCK_THREADS, ++- short N_PER_THREAD> +++ short N_PER_THREAD, +++ typename CompareOp> ++ [[kernel, max_total_threads_per_threadgroup(BLOCK_THREADS)]] void block_sort_nc( ++ const device T* inp [[buffer(0)]], ++ device U* out [[buffer(1)]], ++@@ -347,7 +358,7 @@ template < ++ uint3 tid [[threadgroup_position_in_grid]], ++ uint3 lid [[thread_position_in_threadgroup]]) { ++ using sort_kernel = ++- KernelMergeSort; +++ KernelMergeSort; ++ using ValT = typename sort_kernel::ValT; ++ using IdxT = typename sort_kernel::IdxT; ++ ++@@ -394,7 +405,7 @@ template < ++ bool ARG_SORT, ++ short BLOCK_THREADS, ++ short N_PER_THREAD, ++- typename CompareOp = LessThan> +++ typename CompareOp> ++ struct KernelMultiBlockMergeSort { ++ using block_merge_sort_t = BlockMergeSort< ++ ValT, ++@@ -476,7 +487,8 @@ template < ++ typename IdxT, ++ bool ARG_SORT, ++ short BLOCK_THREADS, ++- short N_PER_THREAD> +++ short N_PER_THREAD, +++ typename CompareOp> ++ [[kernel, max_total_threads_per_threadgroup(BLOCK_THREADS)]] void mb_block_sort( ++ const device ValT* inp [[buffer(0)]], ++ device ValT* out_vals [[buffer(1)]], ++@@ -493,7 +505,8 @@ template < ++ IdxT, ++ ARG_SORT, ++ BLOCK_THREADS, ++- N_PER_THREAD>; +++ N_PER_THREAD, +++ CompareOp>; ++ ++ auto block_idx = elem_to_loc(tid.y, nc_shape, nc_strides, nc_dim); ++ inp += block_idx; ++@@ -520,7 +533,8 @@ template < ++ typename IdxT, ++ bool ARG_SORT, ++ short BLOCK_THREADS, ++- short N_PER_THREAD> +++ short N_PER_THREAD, +++ typename CompareOp> ++ [[kernel]] void mb_block_partition( ++ device IdxT* block_partitions [[buffer(0)]], ++ const device ValT* dev_vals [[buffer(1)]], ++@@ -536,7 +550,8 @@ template < ++ IdxT, ++ ARG_SORT, ++ BLOCK_THREADS, ++- N_PER_THREAD>; +++ N_PER_THREAD, +++ CompareOp>; ++ ++ block_partitions += tid.y * tgp_dims.x; ++ dev_vals += tid.y * size_sorted_axis; ++@@ -573,7 +588,7 @@ template < ++ bool ARG_SORT, ++ short BLOCK_THREADS, ++ short N_PER_THREAD, ++- typename CompareOp = LessThan> +++ typename CompareOp> ++ [[kernel, max_total_threads_per_threadgroup(BLOCK_THREADS)]] void ++ mb_block_merge( ++ const device IdxT* block_partitions [[buffer(0)]], ++diff --git a/mlx/backend/metal/kernels/sort.metal b/mlx/backend/metal/kernels/sort.metal ++index 7f198d55..f1ec796c 100644 ++--- a/mlx/backend/metal/kernels/sort.metal +++++ b/mlx/backend/metal/kernels/sort.metal ++@@ -6,24 +6,29 @@ ++ #include "mlx/backend/metal/kernels/utils.h" ++ #include "mlx/backend/metal/kernels/sort.h" ++ +++#define instantiate_comparator(name, type) \ +++ name +++ ++ #define instantiate_block_sort( \ ++- name, itname, itype, otname, otype, arg_sort, bn, tn) \ ++- instantiate_kernel("c" #name "_" #itname "_" #otname "_bn" #bn "_tn" #tn, \ ++- block_sort, itype, otype, arg_sort, bn, tn) \ ++- instantiate_kernel("nc" #name "_" #itname "_" #otname "_bn" #bn "_tn" #tn, \ ++- block_sort_nc, itype, otype, arg_sort, bn, tn) +++ name, itname, itype, otname, otype, arg_sort, bn, tn, vcomparator) \ +++ instantiate_kernel("c" #name "_" #itname "_" #otname "_bn" #bn "_tn" #tn "_comp" #vcomparator, \ +++ block_sort, itype, otype, arg_sort, bn, tn, instantiate_comparator(vcomparator, itype)) \ +++ instantiate_kernel("nc" #name "_" #itname "_" #otname "_bn" #bn "_tn" #tn "_comp" #vcomparator, \ +++ block_sort_nc, itype, otype, arg_sort, bn, tn, instantiate_comparator(vcomparator, itype)) ++ ++-#define instantiate_arg_block_sort_base(itname, itype, bn, tn) \ +++#define instantiate_arg_block_sort_base(itname, itype, bn, tn, comparator) \ ++ instantiate_block_sort( \ ++- arg_block_sort, itname, itype, uint32, uint32_t, true, bn, tn) +++ arg_block_sort, itname, itype, uint32, uint32_t, true, bn, tn, comparator) ++ ++-#define instantiate_block_sort_base(itname, itype, bn, tn) \ +++#define instantiate_block_sort_base(itname, itype, bn, tn, comparator) \ ++ instantiate_block_sort( \ ++- _block_sort, itname, itype, itname, itype, false, bn, tn) +++ _block_sort, itname, itype, itname, itype, false, bn, tn, comparator) ++ ++ #define instantiate_block_sort_tn(itname, itype, bn) \ ++- instantiate_block_sort_base(itname, itype, bn, 4) \ ++- instantiate_arg_block_sort_base(itname, itype, bn, 4) +++ instantiate_block_sort_base(itname, itype, bn, 4, LessThan) \ +++ instantiate_block_sort_base(itname, itype, bn, 4, GreaterThan) \ +++ instantiate_arg_block_sort_base(itname, itype, bn, 4, LessThan) \ +++ instantiate_arg_block_sort_base(itname, itype, bn, 4, GreaterThan) ++ ++ #define instantiate_block_sort_bn(itname, itype) \ ++ instantiate_block_sort_tn(itname, itype, 32) \ ++@@ -52,16 +57,17 @@ instantiate_block_sort_long(uint64, uint64_t) ++ instantiate_block_sort_long(int64, int64_t) ++ ++ #define instantiate_multi_block_sort( \ ++- vtname, vtype, itname, itype, arg_sort, bn, tn) \ ++- instantiate_kernel("sort_mbsort_" #vtname "_" #itname "_bn" #bn "_tn" #tn, \ ++- mb_block_sort, vtype, itype, arg_sort, bn, tn) \ ++- instantiate_kernel("partition_mbsort_" #vtname "_" #itname "_bn" #bn "_tn" #tn, \ ++- mb_block_partition, vtype, itype, arg_sort, bn, tn) \ ++- instantiate_kernel("merge_mbsort_" #vtname "_" #itname "_bn" #bn "_tn" #tn, \ ++- mb_block_merge, vtype, itype, arg_sort, bn, tn) +++ vtname, vtype, itname, itype, arg_sort, bn, tn, vcomparator) \ +++ instantiate_kernel("sort_mbsort_" #vtname "_" #itname "_bn" #bn "_tn" #tn "_comp" #vcomparator, \ +++ mb_block_sort, vtype, itype, arg_sort, bn, tn, instantiate_comparator(vcomparator, vtype)) \ +++ instantiate_kernel("partition_mbsort_" #vtname "_" #itname "_bn" #bn "_tn" #tn "_comp" #vcomparator, \ +++ mb_block_partition, vtype, itype, arg_sort, bn, tn, instantiate_comparator(vcomparator, vtype)) \ +++ instantiate_kernel("merge_mbsort_" #vtname "_" #itname "_bn" #bn "_tn" #tn "_comp" #vcomparator, \ +++ mb_block_merge, vtype, itype, arg_sort, bn, tn, instantiate_comparator(vcomparator, vtype)) ++ ++ #define instantiate_multi_block_sort_base(vtname, vtype) \ ++- instantiate_multi_block_sort(vtname, vtype, uint32, uint32_t, true, 512, 4) +++ instantiate_multi_block_sort(vtname, vtype, uint32, uint32_t, true, 512, 4, LessThan) \ +++ instantiate_multi_block_sort(vtname, vtype, uint32, uint32_t, true, 512, 4, GreaterThan) ++ ++ instantiate_multi_block_sort_base(uint8, uint8_t) ++ instantiate_multi_block_sort_base(uint16, uint16_t) ++@@ -74,7 +80,8 @@ instantiate_multi_block_sort_base(float32, float) ++ instantiate_multi_block_sort_base(bfloat16, bfloat16_t) ++ ++ #define instantiate_multi_block_sort_long(vtname, vtype) \ ++- instantiate_multi_block_sort(vtname, vtype, uint32, uint32_t, true, 256, 4) +++ instantiate_multi_block_sort(vtname, vtype, uint32, uint32_t, true, 256, 4, LessThan) \ +++ instantiate_multi_block_sort(vtname, vtype, uint32, uint32_t, true, 256, 4, GreaterThan) ++ ++ instantiate_multi_block_sort_long(uint64, uint64_t) ++ instantiate_multi_block_sort_long(int64, int64_t) // clang-format on ++diff --git a/mlx/backend/metal/make_compiled_preamble.sh b/mlx/backend/metal/make_compiled_preamble.sh ++index 425cd8d7..a48d0080 100644 ++--- a/mlx/backend/metal/make_compiled_preamble.sh +++++ b/mlx/backend/metal/make_compiled_preamble.sh ++@@ -5,18 +5,21 @@ ++ # ++ # Copyright © 2023-24 Apple Inc. ++ ++-OUTPUT_DIR=$1 ++-CC=$2 ++-SRC_DIR=$3 ++-SRC_FILE=$4 ++-CFLAGS=$5 ++-SRC_NAME=$(basename -- "${SRC_FILE}") ++-JIT_INCLUDES=${SRC_DIR}/mlx/backend/metal/kernels/jit ++-INPUT_FILE=${SRC_DIR}/mlx/backend/metal/kernels/${SRC_FILE}.h ++-OUTPUT_FILE=${OUTPUT_DIR}/${SRC_NAME}.cpp +++OUTPUT_FILE=$1 +++INPUT_FILE=$2 +++CFLAGS=$3 ++ ++-mkdir -p "$OUTPUT_DIR" ++-CONTENT=$($CC -I"$SRC_DIR" -I"$JIT_INCLUDES" -DMLX_METAL_JIT -E -P "$INPUT_FILE" $CFLAGS 2>/dev/null) +++KERNEL_DIR=$(dirname $(realpath $INPUT_FILE)) +++# echo "KERNEL_DIR: $KERNEL_DIR" +++SRC_DIR=$(echo "$KERNEL_DIR" | cut -d'/' -f-$(($(echo "$KERNEL_DIR" | awk -F'/' '{print NF}')-4))) +++# echo "SRC_DIR: $SRC_DIR" +++JIT_INCLUDES="$KERNEL_DIR/jit" +++# echo "JIT_INCLUDES: $JIT_INCLUDES" +++ +++SRC_BASENAME=$(basename -- "${INPUT_FILE}") +++SRC_NAME="${SRC_BASENAME%.*}" +++ +++CONTENT=$(clang -I"$SRC_DIR" -I"$JIT_INCLUDES" -DMLX_METAL_JIT -E -P "$INPUT_FILE" $CFLAGS 2>/dev/null) ++ ++ cat << EOF > "$OUTPUT_FILE" ++ namespace mlx::core::metal { ++diff --git a/mlx/backend/metal/nojit_kernels.cpp b/mlx/backend/metal/nojit_kernels.cpp ++index 8da14797..1098f918 100644 ++--- a/mlx/backend/metal/nojit_kernels.cpp +++++ b/mlx/backend/metal/nojit_kernels.cpp ++@@ -96,7 +96,8 @@ MTL::ComputePipelineState* get_sort_kernel( ++ const array&, ++ const array&, ++ int, ++- int) { +++ int, +++ const std::string&) { ++ return d.get_kernel(kernel_name); ++ } ++ ++@@ -106,7 +107,8 @@ MTL::ComputePipelineState* get_mb_sort_kernel( ++ const array&, ++ const array&, ++ int, ++- int) { +++ int, +++ const std::string&) { ++ return d.get_kernel(kernel_name); ++ } ++ ++diff --git a/mlx/backend/metal/sort.cpp b/mlx/backend/metal/sort.cpp ++index 3c84022f..bca888f6 100644 ++--- a/mlx/backend/metal/sort.cpp +++++ b/mlx/backend/metal/sort.cpp ++@@ -6,7 +6,9 @@ ++ #include "mlx/backend/metal/device.h" ++ #include "mlx/backend/metal/kernels.h" ++ #include "mlx/backend/metal/utils.h" +++#include "mlx/dtype_utils.h", ++ #include "mlx/primitives.h" +++#include "mlx/sort.h" ++ ++ namespace mlx::core { ++ ++@@ -20,7 +22,8 @@ void single_block_sort( ++ int axis, ++ int bn, ++ int tn, ++- bool argsort) { +++ bool argsort, +++ ComparatorType comparator_) { ++ // Prepare shapes ++ int n_rows = in.size() / in.shape(axis); ++ ++@@ -58,9 +61,26 @@ void single_block_sort( ++ kname << "arg"; ++ } ++ +++ std::string comparator; +++ switch (comparator_) { +++ case ComparatorType::LessThan: +++ comparator = "LessThan"; +++ break; +++ case ComparatorType::GreaterThan: +++ comparator = "GreaterThan"; +++ break; +++ } +++ ++ kname << "_block_sort_" << type_to_name(in) << "_" << type_to_name(out) ++- << "_bn" << bn << "_tn" << tn; ++- auto kernel = get_sort_kernel(d, kname.str(), in, out, bn, tn); +++ << "_bn" << bn << "_tn" << tn << "_comp" << comparator; +++ auto kernel = get_sort_kernel( +++ d, +++ kname.str(), +++ in, +++ out, +++ bn, +++ tn, +++ comparator + "<" + dtype_to_string(in.dtype()) + ">"); ++ ++ // Prepare command encoder ++ auto& compute_encoder = d.get_command_encoder(s.index); ++@@ -120,7 +140,8 @@ void multi_block_sort( ++ int bn, ++ int tn, ++ int n_blocks, ++- bool argsort) { +++ bool argsort, +++ ComparatorType comparator_) { ++ // Prepare shapes ++ int n_rows = in.size() / in.shape(axis); ++ ++@@ -159,6 +180,16 @@ void multi_block_sort( ++ std::vector copies = { ++ dev_vals_0, dev_vals_1, dev_idxs_0, dev_idxs_1, block_partitions}; ++ +++ std::string comparator; +++ switch (comparator_) { +++ case ComparatorType::LessThan: +++ comparator = "LessThan"; +++ break; +++ case ComparatorType::GreaterThan: +++ comparator = "GreaterThan"; +++ break; +++ } +++ ++ // Prepare command encoder ++ auto& compute_encoder = d.get_command_encoder(s.index); ++ ++@@ -166,9 +197,16 @@ void multi_block_sort( ++ { ++ std::ostringstream kname; ++ kname << "sort_mbsort_" << type_to_name(dev_vals_0) << "_" ++- << type_to_name(dev_idxs_0) << "_bn" << bn << "_tn" << tn; ++- auto kernel = ++- get_mb_sort_kernel(d, kname.str(), dev_vals_0, dev_idxs_0, bn, tn); +++ << type_to_name(dev_idxs_0) << "_bn" << bn << "_tn" << tn << "_comp" +++ << comparator; +++ auto kernel = get_mb_sort_kernel( +++ d, +++ kname.str(), +++ dev_vals_0, +++ dev_idxs_0, +++ bn, +++ tn, +++ comparator + "<" + dtype_to_string(in.dtype()) + ">"); ++ compute_encoder.set_compute_pipeline_state(kernel); ++ ++ compute_encoder.set_input_array(in, 0); ++@@ -206,10 +244,17 @@ void multi_block_sort( ++ { ++ std::ostringstream kname; ++ kname << "partition_mbsort_" << type_to_name(dev_vals_in) << "_" ++- << type_to_name(dev_idxs_in) << "_bn" << bn << "_tn" << tn; ++- ++- auto kernel = ++- get_mb_sort_kernel(d, kname.str(), dev_vals_0, dev_idxs_0, bn, tn); +++ << type_to_name(dev_idxs_in) << "_bn" << bn << "_tn" << tn +++ << "_comp" << comparator; +++ +++ auto kernel = get_mb_sort_kernel( +++ d, +++ kname.str(), +++ dev_vals_0, +++ dev_idxs_0, +++ bn, +++ tn, +++ comparator + "<" + dtype_to_string(in.dtype()) + ">"); ++ compute_encoder.set_compute_pipeline_state(kernel); ++ ++ compute_encoder.set_output_array(block_partitions, 0); ++@@ -229,10 +274,17 @@ void multi_block_sort( ++ { ++ std::ostringstream kname; ++ kname << "merge_mbsort_" << type_to_name(dev_vals_in) << "_" ++- << type_to_name(dev_idxs_in) << "_bn" << bn << "_tn" << tn; ++- ++- auto kernel = ++- get_mb_sort_kernel(d, kname.str(), dev_vals_0, dev_idxs_0, bn, tn); +++ << type_to_name(dev_idxs_in) << "_bn" << bn << "_tn" << tn +++ << "_comp" << comparator; +++ +++ auto kernel = get_mb_sort_kernel( +++ d, +++ kname.str(), +++ dev_vals_0, +++ dev_idxs_0, +++ bn, +++ tn, +++ comparator + "<" + dtype_to_string(in.dtype()) + ">"); ++ compute_encoder.set_compute_pipeline_state(kernel); ++ ++ compute_encoder.set_input_array(block_partitions, 0); ++@@ -277,7 +329,8 @@ void gpu_merge_sort( ++ const array& in, ++ array& out, ++ int axis_, ++- bool argsort) { +++ bool argsort, +++ ComparatorType comparator) { ++ // Get size info ++ int axis = axis_ < 0 ? axis_ + in.ndim() : axis_; ++ int size_sorted_axis = in.shape(axis); ++@@ -307,9 +360,10 @@ void gpu_merge_sort( ++ int n_blocks = (size_sorted_axis + n_per_block - 1) / n_per_block; ++ ++ if (n_blocks > 1) { ++- return multi_block_sort(s, d, in, out, axis, bn, tn, n_blocks, argsort); +++ return multi_block_sort( +++ s, d, in, out, axis, bn, tn, n_blocks, argsort, comparator); ++ } else { ++- return single_block_sort(s, d, in, out, axis, bn, tn, argsort); +++ return single_block_sort(s, d, in, out, axis, bn, tn, argsort, comparator); ++ } ++ } ++ ++@@ -324,7 +378,7 @@ void ArgSort::eval_gpu(const std::vector& inputs, array& out) { ++ auto& d = metal::device(s.device); ++ auto& in = inputs[0]; ++ ++- gpu_merge_sort(s, d, in, out, axis_, true); +++ gpu_merge_sort(s, d, in, out, axis_, true, comparator_); ++ } ++ ++ void Sort::eval_gpu(const std::vector& inputs, array& out) { ++@@ -336,7 +390,7 @@ void Sort::eval_gpu(const std::vector& inputs, array& out) { ++ auto& d = metal::device(s.device); ++ auto& in = inputs[0]; ++ ++- gpu_merge_sort(s, d, in, out, axis_, false); +++ gpu_merge_sort(s, d, in, out, axis_, false, comparator_); ++ } ++ ++ void ArgPartition::eval_gpu(const std::vector& inputs, array& out) { ++@@ -349,7 +403,7 @@ void ArgPartition::eval_gpu(const std::vector& inputs, array& out) { ++ auto& d = metal::device(s.device); ++ auto& in = inputs[0]; ++ ++- gpu_merge_sort(s, d, in, out, axis_, true); +++ gpu_merge_sort(s, d, in, out, axis_, true, comparator_); ++ } ++ ++ void Partition::eval_gpu(const std::vector& inputs, array& out) { ++@@ -362,7 +416,7 @@ void Partition::eval_gpu(const std::vector& inputs, array& out) { ++ auto& d = metal::device(s.device); ++ auto& in = inputs[0]; ++ ++- gpu_merge_sort(s, d, in, out, axis_, false); +++ gpu_merge_sort(s, d, in, out, axis_, false, comparator_); ++ } ++ ++ } // namespace mlx::core ++diff --git a/mlx/backend/no_gpu/primitives.cpp b/mlx/backend/no_gpu/primitives.cpp ++index 676a6e55..178eb23f 100644 ++--- a/mlx/backend/no_gpu/primitives.cpp +++++ b/mlx/backend/no_gpu/primitives.cpp ++@@ -95,7 +95,7 @@ NO_GPU(Partition) ++ NO_GPU(Power) ++ NO_GPU_MULTI(QRF) ++ NO_GPU(QuantizedMatmul) ++-NO_GPU(RandomBits) +++NO_GPU() ++ NO_GPU(Real) ++ NO_GPU(Reduce) ++ NO_GPU(Reshape) ++diff --git a/mlx/mlx.h b/mlx/mlx.h ++index cef8d806..f8932873 100644 ++--- a/mlx/mlx.h +++++ b/mlx/mlx.h ++@@ -17,6 +17,7 @@ ++ #include "mlx/memory.h" ++ #include "mlx/ops.h" ++ #include "mlx/random.h" +++#include "mlx/sort.h" ++ #include "mlx/stream.h" ++ #include "mlx/transforms.h" ++ #include "mlx/utils.h" ++diff --git a/mlx/ops.cpp b/mlx/ops.cpp ++index a72c2bc8..04ac921f 100644 ++--- a/mlx/ops.cpp +++++ b/mlx/ops.cpp ++@@ -2205,13 +2205,20 @@ array argmax( ++ } ++ ++ /** Returns a sorted copy of the flattened array. */ ++-array sort(const array& a, StreamOrDevice s /* = {} */) { +++array sort( +++ const array& a, +++ ComparatorType comparator, +++ StreamOrDevice s /* = {} */) { ++ int size = a.size(); ++- return sort(reshape(a, {size}, s), 0, s); +++ return sort(reshape(a, {size}, s), 0, comparator, s); ++ } ++ ++ /** Returns a sorted copy of the array along a given axis. */ ++-array sort(const array& a, int axis, StreamOrDevice s /* = {} */) { +++array sort( +++ const array& a, +++ int axis, +++ ComparatorType comparator, +++ StreamOrDevice s /* = {} */) { ++ // Check for valid axis ++ if (axis + static_cast(a.ndim()) < 0 || ++ axis >= static_cast(a.ndim())) { ++@@ -2222,17 +2229,27 @@ array sort(const array& a, int axis, StreamOrDevice s /* = {} */) { ++ } ++ ++ return array( ++- a.shape(), a.dtype(), std::make_shared(to_stream(s), axis), {a}); +++ a.shape(), +++ a.dtype(), +++ std::make_shared(to_stream(s), axis, comparator), +++ {a}); ++ } ++ ++ /** Returns indices that sort the flattened array. */ ++-array argsort(const array& a, StreamOrDevice s /* = {} */) { +++array argsort( +++ const array& a, +++ ComparatorType comparator, +++ StreamOrDevice s /* = {} */) { ++ int size = a.size(); ++- return argsort(reshape(a, {size}, s), 0, s); +++ return argsort(reshape(a, {size}, s), 0, comparator, s); ++ } ++ ++ /** Returns indices that sort the array along a given axis. */ ++-array argsort(const array& a, int axis, StreamOrDevice s /* = {} */) { +++array argsort( +++ const array& a, +++ int axis, +++ ComparatorType comparator, +++ StreamOrDevice s /* = {} */) { ++ // Check for valid axis ++ if (axis + static_cast(a.ndim()) < 0 || ++ axis >= static_cast(a.ndim())) { ++@@ -2243,16 +2260,23 @@ array argsort(const array& a, int axis, StreamOrDevice s /* = {} */) { ++ } ++ ++ return array( ++- a.shape(), uint32, std::make_shared(to_stream(s), axis), {a}); +++ a.shape(), +++ uint32, +++ std::make_shared(to_stream(s), axis, comparator), +++ {a}); ++ } ++ ++ /** ++ * Returns a partitioned copy of the flattened array ++ * such that the smaller kth elements are first. ++ **/ ++-array partition(const array& a, int kth, StreamOrDevice s /* = {} */) { +++array partition( +++ const array& a, +++ int kth, +++ ComparatorType comparator, +++ StreamOrDevice s /* = {} */) { ++ int size = a.size(); ++- return partition(reshape(a, {size}, s), kth, 0, s); +++ return partition(reshape(a, {size}, s), kth, 0, comparator, s); ++ } ++ ++ /** ++@@ -2263,6 +2287,7 @@ array partition( ++ const array& a, ++ int kth, ++ int axis, +++ ComparatorType comparator, ++ StreamOrDevice s /* = {} */) { ++ // Check for valid axis ++ if (axis + static_cast(a.ndim()) < 0 || ++@@ -2283,7 +2308,7 @@ array partition( ++ return array( ++ a.shape(), ++ a.dtype(), ++- std::make_shared(to_stream(s), kth_, axis_), +++ std::make_shared(to_stream(s), kth_, axis_, comparator), ++ {a}); ++ } ++ ++@@ -2291,9 +2316,13 @@ array partition( ++ * Returns indices that partition the flattened array ++ * such that the smaller kth elements are first. ++ **/ ++-array argpartition(const array& a, int kth, StreamOrDevice s /* = {} */) { +++array argpartition( +++ const array& a, +++ int kth, +++ ComparatorType comparator, +++ StreamOrDevice s /* = {} */) { ++ int size = a.size(); ++- return argpartition(reshape(a, {size}, s), kth, 0, s); +++ return argpartition(reshape(a, {size}, s), kth, 0, comparator, s); ++ } ++ ++ /** ++@@ -2304,6 +2333,7 @@ array argpartition( ++ const array& a, ++ int kth, ++ int axis, +++ ComparatorType comparator, ++ StreamOrDevice s /* = {} */) { ++ // Check for valid axis ++ if (axis + static_cast(a.ndim()) < 0 || ++@@ -2324,7 +2354,7 @@ array argpartition( ++ return array( ++ a.shape(), ++ uint32, ++- std::make_shared(to_stream(s), kth_, axis_), +++ std::make_shared(to_stream(s), kth_, axis_, comparator), ++ {a}); ++ } ++ ++diff --git a/mlx/ops.h b/mlx/ops.h ++index af3cdb5b..be762b8a 100644 ++--- a/mlx/ops.h +++++ b/mlx/ops.h ++@@ -6,6 +6,7 @@ ++ ++ #include "mlx/array.h" ++ #include "mlx/device.h" +++#include "mlx/sort.h" ++ #include "mlx/stream.h" ++ #include "mlx/utils.h" ++ ++@@ -674,40 +675,98 @@ array argmax( ++ StreamOrDevice s = {}); ++ ++ /** Returns a sorted copy of the flattened array. */ ++-array sort(const array& a, StreamOrDevice s = {}); +++array sort( +++ const array& a, +++ ComparatorType comparator = ComparatorType::LessThan, +++ StreamOrDevice s = {}); +++inline array sort(const array& a, StreamOrDevice s = {}) { +++ return sort(a, ComparatorType::LessThan, s); +++} ++ ++ /** Returns a sorted copy of the array along a given axis. */ ++-array sort(const array& a, int axis, StreamOrDevice s = {}); +++array sort( +++ const array& a, +++ int axis, +++ ComparatorType comparator = ComparatorType::LessThan, +++ StreamOrDevice s = {}); +++inline array sort(const array& a, int axis, StreamOrDevice s = {}) { +++ return sort(a, axis, ComparatorType::LessThan, s); +++} ++ ++ /** Returns indices that sort the flattened array. */ ++-array argsort(const array& a, StreamOrDevice s = {}); +++array argsort( +++ const array& a, +++ ComparatorType comparator = ComparatorType::LessThan, +++ StreamOrDevice s = {}); +++inline array argsort(const array& a, StreamOrDevice s = {}) { +++ return argsort(a, ComparatorType::LessThan, s); +++} ++ ++ /** Returns indices that sort the array along a given axis. */ ++-array argsort(const array& a, int axis, StreamOrDevice s = {}); +++array argsort( +++ const array& a, +++ int axis, +++ ComparatorType comparator = ComparatorType::LessThan, +++ StreamOrDevice s = {}); +++inline array argsort(const array& a, int axis, StreamOrDevice s = {}) { +++ return argsort(a, axis, ComparatorType::LessThan, s); +++} ++ ++ /** ++ * Returns a partitioned copy of the flattened array ++ * such that the smaller kth elements are first. ++ **/ ++-array partition(const array& a, int kth, StreamOrDevice s = {}); +++array partition( +++ const array& a, +++ int kth, +++ ComparatorType comparator = ComparatorType::LessThan, +++ StreamOrDevice s = {}); +++inline array partition(const array& a, int kth, StreamOrDevice s = {}) { +++ return partition(a, kth, ComparatorType::LessThan, s); +++} ++ ++ /** ++ * Returns a partitioned copy of the array along a given axis ++ * such that the smaller kth elements are first. ++ **/ ++-array partition(const array& a, int kth, int axis, StreamOrDevice s = {}); +++array partition( +++ const array& a, +++ int kth, +++ int axis, +++ ComparatorType comparator = ComparatorType::LessThan, +++ StreamOrDevice s = {}); +++inline array +++partition(const array& a, int kth, int axis, StreamOrDevice s = {}) { +++ return partition(a, kth, axis, ComparatorType::LessThan, s); +++} ++ ++ /** ++ * Returns indices that partition the flattened array ++ * such that the smaller kth elements are first. ++ **/ ++-array argpartition(const array& a, int kth, StreamOrDevice s = {}); +++array argpartition( +++ const array& a, +++ int kth, +++ ComparatorType comparator = ComparatorType::LessThan, +++ StreamOrDevice s = {}); +++inline array argpartition(const array& a, int kth, StreamOrDevice s = {}) { +++ return argpartition(a, kth, ComparatorType::LessThan, s); +++} ++ ++ /** ++ * Returns indices that partition the array along a given axis ++ * such that the smaller kth elements are first. ++ **/ ++-array argpartition(const array& a, int kth, int axis, StreamOrDevice s = {}); +++array argpartition( +++ const array& a, +++ int kth, +++ int axis, +++ ComparatorType comparator = ComparatorType::LessThan, +++ StreamOrDevice s = {}); +++inline array +++argpartition(const array& a, int kth, int axis, StreamOrDevice s = {}) { +++ return argpartition(a, kth, axis, ComparatorType::LessThan, s); +++} ++ ++ /** Returns topk elements of the flattened array. */ ++ array topk(const array& a, int k, StreamOrDevice s = {}); ++diff --git a/mlx/primitives.h b/mlx/primitives.h ++index c0fbfc84..13d67201 100644 ++--- a/mlx/primitives.h +++++ b/mlx/primitives.h ++@@ -7,6 +7,7 @@ ++ #include "mlx/array.h" ++ #include "mlx/device.h" ++ #include "mlx/io/load.h" +++#include "mlx/sort.h" ++ #include "mlx/stream.h" ++ ++ #define DEFINE_VMAP() \ ++@@ -322,8 +323,15 @@ class ArcTanh : public UnaryPrimitive { ++ ++ class ArgPartition : public UnaryPrimitive { ++ public: ++- explicit ArgPartition(Stream stream, int kth, int axis) ++- : UnaryPrimitive(stream), kth_(kth), axis_(axis) {} +++ explicit ArgPartition( +++ Stream stream, +++ int kth, +++ int axis, +++ ComparatorType comparator) +++ : UnaryPrimitive(stream), +++ kth_(kth), +++ axis_(axis), +++ comparator_(comparator) {} ++ ++ void eval_cpu(const std::vector& inputs, array& out) override; ++ void eval_gpu(const std::vector& inputs, array& out) override; ++@@ -333,13 +341,14 @@ class ArgPartition : public UnaryPrimitive { ++ DEFINE_PRINT(ArgPartition) ++ DEFINE_INPUT_OUTPUT_SHAPE() ++ bool is_equivalent(const Primitive& other) const override; ++- std::pair state() const { ++- return {kth_, axis_}; +++ auto state() const { +++ return std::make_tuple(kth_, axis_, comparator_); ++ }; ++ ++ private: ++ int kth_; ++ int axis_; +++ ComparatorType comparator_; ++ }; ++ ++ class ArgReduce : public UnaryPrimitive { ++@@ -371,8 +380,8 @@ class ArgReduce : public UnaryPrimitive { ++ ++ class ArgSort : public UnaryPrimitive { ++ public: ++- explicit ArgSort(Stream stream, int axis) ++- : UnaryPrimitive(stream), axis_(axis) {} +++ explicit ArgSort(Stream stream, int axis, ComparatorType comparator) +++ : UnaryPrimitive(stream), axis_(axis), comparator_(comparator) {} ++ ++ void eval_cpu(const std::vector& inputs, array& out) override; ++ void eval_gpu(const std::vector& inputs, array& out) override; ++@@ -381,12 +390,13 @@ class ArgSort : public UnaryPrimitive { ++ DEFINE_PRINT(ArgSort) ++ DEFINE_INPUT_OUTPUT_SHAPE() ++ bool is_equivalent(const Primitive& other) const override; ++- int state() const { ++- return axis_; +++ auto state() const { +++ return std::make_pair(axis_, comparator_); ++ }; ++ ++ private: ++ int axis_; +++ ComparatorType comparator_; ++ }; ++ ++ class AsType : public UnaryPrimitive { ++@@ -1530,8 +1540,15 @@ class Pad : public UnaryPrimitive { ++ ++ class Partition : public UnaryPrimitive { ++ public: ++- explicit Partition(Stream stream, int kth, int axis) ++- : UnaryPrimitive(stream), kth_(kth), axis_(axis) {} +++ explicit Partition( +++ Stream stream, +++ int kth, +++ int axis, +++ ComparatorType comparator) +++ : UnaryPrimitive(stream), +++ kth_(kth), +++ axis_(axis), +++ comparator_(comparator) {} ++ ++ void eval_cpu(const std::vector& inputs, array& out) override; ++ void eval_gpu(const std::vector& inputs, array& out) override; ++@@ -1542,12 +1559,13 @@ class Partition : public UnaryPrimitive { ++ DEFINE_INPUT_OUTPUT_SHAPE() ++ bool is_equivalent(const Primitive& other) const override; ++ auto state() const { ++- return std::make_pair(kth_, axis_); +++ return std::make_tuple(kth_, axis_, comparator_); ++ }; ++ ++ private: ++ int kth_; ++ int axis_; +++ ComparatorType comparator_; ++ }; ++ ++ class Power : public UnaryPrimitive { ++@@ -2073,8 +2091,8 @@ class Softmax : public UnaryPrimitive { ++ ++ class Sort : public UnaryPrimitive { ++ public: ++- explicit Sort(Stream stream, int axis) ++- : UnaryPrimitive(stream), axis_(axis) {} +++ explicit Sort(Stream stream, int axis, ComparatorType comparator) +++ : UnaryPrimitive(stream), axis_(axis), comparator_(comparator) {} ++ ++ void eval_cpu(const std::vector& inputs, array& out) override; ++ void eval_gpu(const std::vector& inputs, array& out) override; ++@@ -2085,11 +2103,12 @@ class Sort : public UnaryPrimitive { ++ DEFINE_INPUT_OUTPUT_SHAPE() ++ bool is_equivalent(const Primitive& other) const override; ++ auto state() const { ++- return axis_; +++ return std::make_pair(axis_, comparator_); ++ } ++ ++ private: ++ int axis_; +++ ComparatorType comparator_; ++ }; ++ ++ class Split : public Primitive { ++diff --git a/mlx/random.cpp b/mlx/random.cpp ++index 6c6d1eb9..5a4745a5 100644 ++--- a/mlx/random.cpp +++++ b/mlx/random.cpp ++@@ -225,6 +225,23 @@ array normal( ++ return samples; ++ } ++ +++array normal( +++ const Shape& shape, +++ Dtype dtype, +++ const array& loc, +++ const array& scale, +++ const std::optional& key /*= nullopt */, +++ StreamOrDevice s /* = {} */) { +++ auto stream = to_stream(s); +++ auto low = above_minus_one_with_default(dtype); +++ auto high = array(1.0f, dtype); +++ auto samples = uniform(low, high, shape, dtype, key, stream); +++ samples = multiply(array(std::sqrt(2.0), dtype), erfinv(samples, stream), stream); +++ samples = multiply(astype(scale, dtype), samples, stream); +++ samples = add(astype(loc, dtype), samples, stream); +++ return samples; +++} +++ ++ array multivariate_normal( ++ const array& mean, ++ const array& cov, ++diff --git a/mlx/random.h b/mlx/random.h ++index 0dfdab7a..ebee726c 100644 ++--- a/mlx/random.h +++++ b/mlx/random.h ++@@ -134,6 +134,14 @@ inline array normal( ++ return normal(shape, float32, std::nullopt, std::nullopt, key, s); ++ } ++ +++array normal( +++ const Shape& shape, +++ Dtype dtype, +++ const array& loc, +++ const array& scale, +++ const std::optional& key = std::nullopt, +++ StreamOrDevice s = {}); +++ ++ /** Generate samples from a multivariate normal distribution. **/ ++ array multivariate_normal( ++ const array& mean, ++diff --git a/mlx/sort.h b/mlx/sort.h ++new file mode 100644 ++index 00000000..0afc6b62 ++--- /dev/null +++++ b/mlx/sort.h ++@@ -0,0 +1,14 @@ +++// Copyright © 2023-2024 Apple Inc. +++ +++#pragma once +++ +++#include "mlx/array.h" +++ +++namespace mlx::core { +++ +++enum class ComparatorType { +++ LessThan, +++ GreaterThan, +++}; +++ +++} // namespace mlx::core ++diff --git a/mlx/types/complex.h b/mlx/types/complex.h ++index 51101cc9..344d80f8 100644 ++--- a/mlx/types/complex.h +++++ b/mlx/types/complex.h ++@@ -59,10 +59,11 @@ inline bool operator>(const complex64_t& a, const complex64_t& b) { ++ inline complex64_t operator%(complex64_t a, complex64_t b) { ++ auto real = a.real() - (b.real() * static_cast(a.real() / b.real())); ++ auto imag = a.imag() - (b.imag() * static_cast(a.imag() / b.imag())); ++- if (real != 0 && ((real < 0) != (b.real() < 0))) ++- real += b.real(); ++- if (imag != 0 && ((imag < 0) != (b.imag() < 0))) ++- imag += b.imag(); +++ /* Disable to match `stablehlo` implementation semantics */ +++ // if (real != 0 && ((real < 0) != (b.real() < 0))) +++ // real += b.real(); +++ // if (imag != 0 && ((imag < 0) != (b.imag() < 0))) +++ // imag += b.imag(); ++ return {real, imag}; ++ } ++ diff --git a/third_party/modules/mlx/20250605.0-c763fe1/source.json b/third_party/modules/mlx/20250605.0-c763fe1/source.json new file mode 100644 index 00000000..d15eec9a --- /dev/null +++ b/third_party/modules/mlx/20250605.0-c763fe1/source.json @@ -0,0 +1,15 @@ +{ + "strip_prefix": "mlx-c763fe1be0f1158e0f53f6f6a28f56d69b1c7fe8", + "url": "https://github.com/ml-explore/mlx/archive/c763fe1be0f1158e0f53f6f6a28f56d69b1c7fe8.tar.gz", + "integrity": "sha256-vE091y6Qc9r0cw/dU2OCCDosZzCXZfTv0tp7Zi2YqYE=", + "overlay": { + "metal_cpp.bzl": "", + "jit.bzl": "", + "MODULE.bazel": "", + "BUILD.bazel": "" + }, + "patch_strip": 1, + "patches": { + "0001-Modify-Sort-And-Comparison-Kernels.patch": "" + } +} diff --git a/third_party/modules/mlx/metadata.json b/third_party/modules/mlx/metadata.json new file mode 100644 index 00000000..805d7f05 --- /dev/null +++ b/third_party/modules/mlx/metadata.json @@ -0,0 +1,18 @@ +{ + "homepage": "https://github.com/ml-explore/mlx", + "maintainers": [ + { + "email": "bzlmod@zml.ai", + "github": "zml", + "name": "ZML Engineering Team" + } + ], + "repository": [ + "github:ml-explore/mlx" + ], + "versions": [ + "20250518.0-eebe730", + "20250605.0-c763fe1" + ], + "yanked_versions": {} +} diff --git a/third_party/modules/xla/20250317.2-71c67e2/MODULE.bazel b/third_party/modules/xla/20250317.2-71c67e2/MODULE.bazel index 042d899c..5b6b999f 100644 --- a/third_party/modules/xla/20250317.2-71c67e2/MODULE.bazel +++ b/third_party/modules/xla/20250317.2-71c67e2/MODULE.bazel @@ -17,6 +17,10 @@ bazel_dep(name = "zlib", version = "1.2.13") bazel_dep(name = "re2", version = "2024-07-02.bcr.1", repo_name = "com_googlesource_code_re2") bazel_dep(name = "rules_license", version = "0.0.8") +# MLX PjRt Plugin Dependencies +bazel_dep(name = "mlx", version = "20250518.0-eebe730") +bazel_dep(name = "range-v3", version = "0.12.0") + tsl = use_extension("//:tsl.bzl", "tsl") use_repo(tsl, "tsl", "python_version_repo") @@ -28,9 +32,15 @@ use_repo( "local_config_cuda", "local_config_remote_execution", "local_config_rocm", + "local_config_sycl", "local_config_tensorrt", + "eigen_archive", + "farmhash_archive", + "ml_dtypes_py", "llvm-raw", "stablehlo", + "shardy", + "snappy", ) llvm_configure = use_extension("//:llvm.bzl", "llvm_configure") diff --git a/third_party/modules/xla/20250317.2-71c67e2/overlay/workspace.bzl b/third_party/modules/xla/20250317.2-71c67e2/overlay/workspace.bzl index 192e06f8..fa890885 100644 --- a/third_party/modules/xla/20250317.2-71c67e2/overlay/workspace.bzl +++ b/third_party/modules/xla/20250317.2-71c67e2/overlay/workspace.bzl @@ -1,9 +1,11 @@ load("//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls") load("//third_party/gpus:cuda_configure.bzl", "cuda_configure") load("//third_party/gpus:rocm_configure.bzl", "rocm_configure") +load("//third_party/gpus:sycl_configure.bzl", "sycl_configure") load("//third_party/llvm:workspace.bzl", llvm = "repo") load("//third_party/pybind11_bazel:workspace.bzl", pybind11_bazel = "repo") load("//third_party/stablehlo:workspace.bzl", stablehlo = "repo") +load("//third_party/shardy:workspace.bzl", shardy = "repo") load("//third_party/tensorrt:tensorrt_configure.bzl", "tensorrt_configure") load("//third_party/triton:workspace.bzl", triton = "repo") load("//tools/toolchains/remote:configure.bzl", "remote_execution_configure") @@ -12,11 +14,45 @@ def _xla_workspace_impl(mctx): cuda_configure(name = "local_config_cuda") remote_execution_configure(name = "local_config_remote_execution") rocm_configure(name = "local_config_rocm") + sycl_configure(name = "local_config_sycl") tensorrt_configure(name = "local_config_tensorrt") pybind11_bazel() triton() llvm("llvm-raw") stablehlo() + shardy() + tf_http_archive( + name = "eigen_archive", + build_file = "//third_party/eigen3:eigen_archive.BUILD", + sha256 = "1a432ccbd597ea7b9faa1557b1752328d6adc1a3db8969f6fe793ff704be3bf0", + strip_prefix = "eigen-4c38131a16803130b66266a912029504f2cf23cd", + urls = tf_mirror_urls("https://gitlab.com/libeigen/eigen/-/archive/4c38131a16803130b66266a912029504f2cf23cd/eigen-4c38131a16803130b66266a912029504f2cf23cd.tar.gz"), + ) + tf_http_archive( + name = "farmhash_archive", + build_file = "//third_party/farmhash:farmhash.BUILD", + sha256 = "18392cf0736e1d62ecbb8d695c31496b6507859e8c75541d7ad0ba092dc52115", + strip_prefix = "farmhash-0d859a811870d10f53a594927d0d0b97573ad06d", + urls = tf_mirror_urls("https://github.com/google/farmhash/archive/0d859a811870d10f53a594927d0d0b97573ad06d.tar.gz"), + ) + tf_http_archive( + name = "ml_dtypes_py", + build_file = "//third_party/py/ml_dtypes:ml_dtypes_py.BUILD", + link_files = { + "//third_party/py/ml_dtypes:ml_dtypes.BUILD": "ml_dtypes/BUILD.bazel", + }, + sha256 = "f6e5880666661351e6cd084ac4178ddc4dabcde7e9a73722981c0d1500cf5937", + strip_prefix = "ml_dtypes-00d98cd92ade342fef589c0470379abb27baebe9", + urls = tf_mirror_urls("https://github.com/jax-ml/ml_dtypes/archive/00d98cd92ade342fef589c0470379abb27baebe9/ml_dtypes-00d98cd92ade342fef589c0470379abb27baebe9.tar.gz"), + ) + tf_http_archive( + name = "snappy", + build_file = "//third_party:snappy.BUILD", + sha256 = "2e458b7017cd58dcf1469ab315389e85e7f445bd035188f2983f81fb19ecfb29", + strip_prefix = "snappy-984b191f0fefdeb17050b42a90b7625999c13b8d", + system_build_file = "//third_party/systemlibs:snappy.BUILD", + urls = tf_mirror_urls("https://github.com/google/snappy/archive/984b191f0fefdeb17050b42a90b7625999c13b8d.tar.gz"), + ) tf_http_archive( name = "com_github_grpc_grpc", sha256 = "b956598d8cbe168b5ee717b5dafa56563eb5201a947856a6688bbeac9cac4e1f", diff --git a/third_party/modules/xla/20250317.2-71c67e2/patches/0003-Added-MLX-PjRt-Plugin.patch b/third_party/modules/xla/20250317.2-71c67e2/patches/0003-Added-MLX-PjRt-Plugin.patch new file mode 100644 index 00000000..410ea7e1 --- /dev/null +++ b/third_party/modules/xla/20250317.2-71c67e2/patches/0003-Added-MLX-PjRt-Plugin.patch @@ -0,0 +1,4597 @@ +diff --git a/xla/pjrt/plugin/mlx/BUILD b/xla/pjrt/plugin/mlx/BUILD +new file mode 100644 +index 0000000000..ca86b466bf +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/BUILD +@@ -0,0 +1,201 @@ ++load("//xla:xla.bzl", "xla_cc_binary", "xla_cc_test") ++ ++package( ++ # copybara:uncomment default_applicable_licenses = ["//tensorflow:license"], ++ default_visibility = ["//visibility:public"], ++ licenses = ["notice"], ++) ++ ++cc_library( ++ name = "buffer", ++ srcs = ["buffer.cc"], ++ hdrs = ["buffer.h"], ++ deps = [ ++ ":logging", ++ ":utils", ++ "//xla:literal", ++ "//xla:shape_util", ++ "//xla:util", ++ "//xla/hlo/translate/hlo_to_mhlo:hlo_utils", ++ "//xla/hlo/translate/mhlo_to_hlo:literal_exporter", ++ "//xla/hlo/translate/mhlo_to_hlo:type_to_shape", ++ "//xla/pjrt:pjrt_client", ++ "//xla/pjrt:pjrt_compiler", ++ "//xla/pjrt:pjrt_future", ++ "@com_google_absl//absl/functional:any_invocable", ++ "@com_google_absl//absl/log", ++ "@com_google_absl//absl/status:statusor", ++ "@com_google_absl//absl/types:span", ++ "@llvm-project//llvm:Support", ++ "@llvm-project//mlir:AsmParser", ++ "@llvm-project//mlir:IR", ++ "@llvm-project//mlir:Support", ++ ], ++) ++ ++ ++cc_library( ++ name = "executable", ++ srcs = ["executable.cc"], ++ hdrs = ["executable.h"], ++ deps = [ ++ "@range-v3", ++ "@mlx//:mlx", ++ ":buffer", ++ ":logging", ++ ":utils", ++ "//xla/hlo/builder:xla_computation", ++ "//xla/hlo/translate:stablehlo", ++ "//xla/mlir/utils:error_util", ++ "//xla/mlir_hlo:mhlo_passes", ++ "//xla/mlir_hlo:stablehlo_extension_passes", ++ # "//xla/mlir/utils:type_util", ++ "//xla/pjrt:mlir_to_hlo", ++ "//xla/pjrt:pjrt_client", ++ "//xla/pjrt:pjrt_executable", ++ "//xla/pjrt:pjrt_future", ++ "//xla/service:computation_placer_hdr", ++ "@com_google_absl//absl/log", ++ "@com_google_absl//absl/status", ++ "@com_google_absl//absl/status:statusor", ++ "@llvm-project//llvm:Support", ++ "@llvm-project//mlir:AsmParser", ++ "@llvm-project//mlir:BytecodeWriter", ++ "@llvm-project//mlir:DataLayoutInterfaces", ++ "@llvm-project//mlir:FuncDialect", ++ "@llvm-project//mlir:IR", ++ "@llvm-project//mlir:Parser", ++ "@llvm-project//mlir:Pass", ++ "@llvm-project//mlir:Support", ++ "@llvm-project//mlir:Transforms", ++ "@stablehlo//:reference_api", ++ "@stablehlo//:stablehlo_passes_optimization", ++ "@tsl//tsl/platform:statusor", ++ ], ++) ++ ++cc_library( ++ name = "device", ++ srcs = ["device.cc"], ++ hdrs = ["device.h"], ++ deps = [ ++ ":logging", ++ "//xla:util", ++ "//xla/pjrt:pjrt_client", ++ "//xla/pjrt:pjrt_common", ++ "//xla/pjrt:pjrt_compiler", ++ "//xla/pjrt:pjrt_device_description", ++ "@com_google_absl//absl/status:statusor", ++ "@com_google_absl//absl/strings:str_format", ++ "@com_google_absl//absl/strings:string_view", ++ "@com_google_absl//absl/types:span", ++ "@mlx//:mlx", ++ ], ++) ++ ++cc_library( ++ name = "client_cpp_pjrt", ++ srcs = [ ++ "client_cpp_pjrt.cc", ++ ], ++ hdrs = [ ++ "client_cpp_pjrt.h", ++ ], ++ deps = [ ++ ":buffer", ++ ":device", ++ ":executable", ++ ":logging", ++ ":utils", ++ "//xla:literal", ++ "//xla:shape_util", ++ "//xla:util", ++ "//xla/pjrt:host_memory_spaces", ++ "//xla/pjrt:pjrt_client", ++ "//xla/pjrt:pjrt_common", ++ "//xla/pjrt:pjrt_compiler", ++ "//xla/tsl/framework:allocator", ++ "@com_google_absl//absl/status:statusor", ++ "@com_google_absl//absl/strings:str_format", ++ "@com_google_absl//absl/strings:string_view", ++ "@com_google_absl//absl/types:span", ++ "@mlx//:mlx", ++ "@tsl//tsl/platform:fingerprint", ++ "@tsl//tsl/platform:statusor" ++ ], ++) ++ ++cc_library( ++ name = "client_c_pjrt", ++ srcs = [ ++ "client_c_pjrt.cc", ++ ], ++ hdrs = ["client_c_pjrt.h"], ++ deps = [ ++ ":client_cpp_pjrt", ++ "//xla/pjrt:pjrt_client", ++ "//xla/pjrt/c:pjrt_c_api_hdrs", ++ "//xla/pjrt/c:pjrt_c_api_layouts_extension_hdrs", ++ "//xla/pjrt/c:pjrt_c_api_wrapper_impl", ++ "@com_google_absl//absl/status", ++ "@com_google_absl//absl/strings:string_view", ++ ], ++ alwayslink = 1, ++) ++ ++cc_library( ++ name = "logging", ++ srcs = ["logging.cc"], ++ hdrs = ["logging.h"], ++ deps = [ ++ "@com_google_absl//absl/base:log_severity", ++ "@com_google_absl//absl/log", ++ "@com_google_absl//absl/log:globals", ++ "@llvm-project//llvm:Support", ++ "@llvm-project//mlir:AsmParser", ++ "@llvm-project//mlir:DataLayoutInterfaces", ++ "@llvm-project//mlir:IR", ++ ], ++) ++ ++cc_library( ++ name = "utils", ++ srcs = ["utils.cc"], ++ hdrs = ["utils.h"], ++ deps = [ ++ "@com_google_absl//absl/status", ++ "@com_google_absl//absl/status:statusor", ++ "@com_google_absl//absl/types:span", ++ "@llvm-project//llvm:Support", ++ "@llvm-project//mlir:IR", ++ "@llvm-project//mlir:Support", ++ "@mlx//:mlx", ++ "@tsl//tsl/platform:statusor", ++ "//xla/mlir/utils:type_util", ++ "//xla:literal", ++ "//xla:shape_util", ++ "//xla:util", ++ ++ ], ++) ++ ++xla_cc_test( ++ name = "plugin_pjrt_test", ++ srcs = ["plugin_pjrt_test.cc"], ++ deps = [ ++ ":client_c_pjrt", ++ ":client_cpp_pjrt", ++ "//xla/pjrt:pjrt_client_test_common", ++ "//xla/pjrt/c:pjrt_c_api_test_common", ++ "//xla/pjrt/c:pjrt_c_api_wrapper_impl", ++ "@com_google_googletest//:gtest_main", ++ ], ++) ++ ++xla_cc_binary( ++ name = "stablehlo_mlx_plugin.so", ++ linkshared = True, ++ deps = [ ++ ":client_c_pjrt", ++ ], ++) +\ No newline at end of file +diff --git a/xla/pjrt/plugin/mlx/buffer.cc b/xla/pjrt/plugin/mlx/buffer.cc +new file mode 100644 +index 0000000000..c4e7d410ac +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/buffer.cc +@@ -0,0 +1,371 @@ ++/* Copyright 2024 The OpenXLA Authors. ++Licensed under the Apache License, Version 2.0 (the "License"); ++you may not use this file except in compliance with the License. ++You may obtain a copy of the License at ++ http://www.apache.org/licenses/LICENSE-2.0 ++Unless required by applicable law or agreed to in writing, software ++distributed under the License is distributed on an "AS IS" BASIS, ++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++See the License for the specific language governing permissions and ++limitations under the License. ++==============================================================================*/ ++ ++#include "xla/pjrt/plugin/mlx/buffer.h" ++ ++#include ++#include ++#include ++#include ++#include ++ ++#include "absl/functional/any_invocable.h" ++#include "absl/log/log.h" ++#include "absl/status/statusor.h" ++#include "absl/types/span.h" ++#include "llvm/Support/Casting.h" ++#include "llvm/Support/raw_ostream.h" ++#include "mlir/AsmParser/AsmParser.h" ++#include "mlir/IR/Attributes.h" ++#include "mlir/IR/Builders.h" ++#include "mlir/IR/BuiltinAttributes.h" ++#include "mlir/IR/BuiltinTypeInterfaces.h" ++#include "mlir/IR/MLIRContext.h" ++#include "mlx/mlx.h" ++#include "mlir/Support/DebugStringHelper.h" ++#include "xla/hlo/translate/hlo_to_mhlo/hlo_utils.h" ++#include "xla/hlo/translate/mhlo_to_hlo/literal_exporter.h" ++#include "xla/hlo/translate/mhlo_to_hlo/type_to_shape.h" ++#include "xla/literal.h" ++#include "xla/pjrt/pjrt_client.h" ++#include "xla/pjrt/pjrt_compiler.h" ++#include "xla/pjrt/pjrt_future.h" ++#include "xla/pjrt/plugin/mlx/logging.h" ++#include "xla/pjrt/plugin/mlx/utils.h" ++#include "xla/shape.h" ++#include "xla/util.h" ++ ++namespace mx = mlx::core; ++namespace mlir::stablehlo { ++ ++using xla::MutableLiteralBase; ++using xla::PjRtBuffer; ++using xla::PjRtClient; ++using xla::PjRtDevice; ++using xla::PjRtFuture; ++using xla::PjRtMemorySpace; ++using xla::PjRtPlatformId; ++using xla::Shape; ++ ++#define UNIMPLEMENTED(name) \ ++ xla::Unimplemented("MlxPjrtBuffer::" #name " is not implemented") ++ ++class MlxPjrtBuffer : public PjRtBuffer { ++ public: ++ MlxPjrtBuffer(mx::array array, const Shape& shape, ++ PjRtMemorySpace* memory_space) ++ : xla::PjRtBuffer(), ++ context_(), ++ buffer_(), ++ array_(array), ++ shape_(shape), ++ memory_space_(memory_space) { ++ // TRACE_ME_MEMBER; ++ } ++ ++ class MlirClonedExternalReference : public ExternalReference { ++ public: ++ explicit MlirClonedExternalReference(PjRtBuffer* buffer, ++ PjRtMemorySpace* memory_space) ++ : buffer_() { ++ // TRACE_ME_MEMBER; ++ auto mlir_buffer = GetAttributeFromBuffer(buffer); ++ if (!mlir_buffer.ok()) { ++ LOG(ERROR) << "Could not get attribute from buffer: " ++ << mlir_buffer.status(); ++ } ++ ++ auto array = GetArrayFromBuffer(buffer); ++ if (!array.ok()) { ++ LOG(ERROR) << "Could not get attribute from buffer: " << array.status(); ++ } ++ buffer_ = CreateMlirBufferFromAttribute( ++ array.value(), mlir_buffer.value(), memory_space); ++ data_ptr_ = (void*)mlir_buffer.value().getRawData().data(); ++ } ++ ++ private: ++ std::unique_ptr buffer_; ++ }; ++ ++ // All buffers are managed by the MLIR Context ++ ~MlxPjrtBuffer() override = default; ++ ++ MlxPjrtBuffer(const MlxPjrtBuffer&) = delete; ++ MlxPjrtBuffer(MlxPjrtBuffer&&) = delete; ++ MlxPjrtBuffer& operator=(const MlxPjrtBuffer&) = delete; ++ MlxPjrtBuffer& operator=(MlxPjrtBuffer&&) = delete; ++ ++ static std::unique_ptr CreateFromLiteral( ++ mx::array array, const xla::LiteralSlice& literal, ++ xla::PjRtMemorySpace* memory_space) { ++ // TRACE_ME; ++ LOG(INFO) << "CreateFromLiteral: " << literal.ToString() << "\n"; ++ auto buffer = ++ std::make_unique(array, literal.shape(), memory_space); ++ LOG(INFO) << "CreateFromLiteral -> " << (void*)buffer.get() << "\n"; ++ mlir::Builder builder(&buffer->context_); ++ auto attr = xla::CreateDenseElementsAttrFromLiteral(literal, builder); ++ if (!attr.ok()) { ++ LOG(ERROR) << "Could not create dense elements attr from literal: " ++ << attr.status(); ++ return nullptr; ++ } ++ buffer->buffer_ = attr.value(); ++ return buffer; ++ } ++ ++ static std::unique_ptr CreateFromAttribute( ++ mx::array array, DenseElementsAttr attr, ++ xla::PjRtMemorySpace* memory_space) { ++ // TRACE_ME; ++ ++ // MLIR type to xla shape: ++ Shape shape = xla::TypeToShape(attr.getType()); ++ auto buffer = std::make_unique(array, shape, memory_space); ++ buffer->buffer_ = CloneIntoContext(attr, buffer->context_); ++ LOG(INFO) << "CreateFromAttribute(" << ToString(attr) << ") -> " ++ << (void*)buffer.get() << "\n"; ++ return buffer; ++ } ++ ++ const Shape& on_device_shape() const override { ++ // TRACE_ME_MEMBER; ++ return shape_; ++ } ++ absl::StatusOr logical_on_device_shape() override { ++ // TRACE_ME_MEMBER; ++ return shape_; ++ } ++ ++ PjRtPlatformId platform_id() const { ++ // TRACE_ME_MEMBER; ++ return client()->platform_id(); ++ } ++ absl::string_view platform_name() const { ++ // TRACE_ME_MEMBER; ++ return client()->platform_name(); ++ } ++ ++ bool IsEmptyTuple() const { ++ // TRACE_ME_MEMBER; ++ return shape_.IsTuple() && shape_.tuple_shapes().empty(); ++ } ++ ++ // Buffer knows device + client per older design, should only need ++ // memory_space. ++ PjRtMemorySpace* memory_space() const override { ++ // TRACE_ME_MEMBER; ++ return memory_space_; ++ } ++ PjRtDevice* device() const override { ++ // TRACE_ME_MEMBER; ++ return memory_space_->devices().front(); ++ } ++ PjRtClient* client() const override { ++ // TRACE_ME_MEMBER; ++ return memory_space_->client(); ++ } ++ ++ absl::StatusOr> ++ AcquireExternalReference() override { ++ // TRACE_ME_MEMBER; ++ return std::make_unique(this, memory_space_); ++ } ++ ++ xla::PjRtFuture<> ToLiteral(xla::MutableLiteralBase* literal) override { ++ // TRACE_ME_MEMBER; ++ if (IsEmptyTuple()) { ++ return PjRtFuture<>( ++ xla::InvalidArgument("ToLiteral called on empty tuple")); ++ } ++ ++ absl::StatusOr to_copy = ++ mhlo::CreateLiteralFromAttribute(buffer_, {}); ++ if (!to_copy.ok()) return PjRtFuture<>(to_copy.status()); ++ ++ // Synchronous! To make async, make the buffer, start the copy, and return a ++ // future that is ready when the copy is done. ++ auto status = literal->CopyFrom(to_copy.value()); ++ if (!status.ok()) return PjRtFuture<>(status); ++ return PjRtFuture<>(absl::OkStatus()); ++ } ++ ++ PjRtFuture<> LazyToLiteral( ++ absl::AnyInvocable() &&> generator) ++ override { ++ // TRACE_ME_MEMBER; ++ auto buffer = std::move(generator)(); ++ if (!buffer.ok()) return PjRtFuture<>(buffer.status()); ++ return ToLiteral(buffer.value()); ++ } ++ ++ absl::StatusOr GetOnDeviceSizeInBytes() const override { ++ // This is needed by AcquireExternalReference, for framework figuring out ++ // how to read the underlying buffer data. ++ // TRACE_ME_MEMBER; ++ if (!buffer_) return 0; ++ return buffer_.getRawData().size(); ++ } ++ ++ PjRtFuture<> CopyRawToHost(void* dst, int64_t offset, ++ int64_t transfer_size) override { ++ // TRACE_ME_MEMBER; ++ return PjRtFuture<>(UNIMPLEMENTED(CopyRawToHost)); ++ } ++ ++ absl::StatusOr> ++ ReleaseDeviceMemoryOwnership(bool wait_for_operations_to_complete) override { ++ // TRACE_ME_MEMBER; ++ auto external_ref = AcquireExternalReference(); ++ Delete(); ++ return external_ref; ++ } ++ ++ // Remove the buffer if deleted. ++ // Note: deleted and uninitialized appear the same in this scenario. ++ // Consider changing to mlir::NoneType when deleted. ++ void Delete() override { ++ // TRACE_ME_MEMBER; ++ buffer_ = {}; ++ } ++ ++ bool IsDeleted() override { ++ // TRACE_ME_MEMBER; ++ return !buffer_; ++ } ++ ++ absl::StatusOr> CopyToDevice( ++ xla::PjRtDevice* dst_device) { ++ // TRACE_ME_MEMBER; ++ return CopyToMemorySpace( ++ dst_device->default_memory_space().value_or(nullptr)); ++ } ++ ++ absl::StatusOr> CopyToMemorySpace( ++ xla::PjRtMemorySpace* dst_memory_space) override { ++ // TRACE_ME_MEMBER; ++ return CreateMlirBufferFromAttribute(array_, buffer_, dst_memory_space); ++ } ++ ++ void CopyToRemoteDevice( ++ xla::PjRtFuture serialized_descriptor, ++ xla::PjRtBuffer::RemoteSendCallback on_done) override { ++ // TRACE_ME_MEMBER; ++ on_done(UNIMPLEMENTED(CopyToRemoteDevice), false); ++ } ++ ++ xla::PjRtFuture<> GetReadyFuture() override { ++ // TRACE_ME_MEMBER; ++ LOG(INFO) << "GetReadyFuture(" << (void*)this << ")\n"; ++ // Synchronous! To make async, have the device make a buffer with a ready ++ // future that is ready when the computation is done / buffer is ready. ++ return PjRtFuture<>(absl::OkStatus()); ++ } ++ ++ bool IsOnCpu() const override { ++ // If buffer is on CPU, it will be shared with framework via ++ // GetExternalReference, lse it is copied back to host. ++ // Since we are using reference interpreter, we are running on CPU in a ++ // shared memory space. ++ // TRACE_ME_MEMBER; ++ return false; ++ } ++ ++ mlir::DenseElementsAttr GetBufferAttribute() const { return buffer_; } ++ ++ mx::array GetArray() const { return array_; } ++ ++ private: ++ MLIRContext context_; ++ mlir::DenseElementsAttr buffer_; ++ mx::array array_; ++ ++ xla::Shape shape_; ++ PjRtMemorySpace* memory_space_; ++}; ++ ++std::unique_ptr CreateMlirBufferFromMlxArray( ++ mx::array array, xla::PjRtMemorySpace* memory_space) { ++ // TRACE_ME; ++ std::vector span_shape(array.shape().begin(), array.shape().end()); ++ auto shape = xla::ShapeUtil::MakeShape( ++ utils::dtype::asXlaPrimitiveType(array.dtype()), ++ absl::Span(span_shape.data(), span_shape.size())); ++ auto literal = xla::BorrowingLiteral( ++ reinterpret_cast(array.data()), shape); ++ return MlxPjrtBuffer::CreateFromLiteral(array, literal, memory_space); ++} ++ ++std::unique_ptr CreateMlirBufferFromLiteral( ++ const xla::LiteralSlice& literal, xla::PjRtMemorySpace* memory_space) { ++ // TRACE_ME; ++ auto maybe_array = utils::array::fromHostLiteral(literal); ++ return MlxPjrtBuffer::CreateFromLiteral(maybe_array.value(), literal, ++ memory_space); ++} ++ ++std::unique_ptr CreateMlirBufferFromAttribute( ++ mx::array array, DenseElementsAttr attr, ++ xla::PjRtMemorySpace* memory_space) { ++ // TRACE_ME; ++ return MlxPjrtBuffer::CreateFromAttribute(array, attr, memory_space); ++} ++ ++std::unique_ptr CreateMlirBufferUninitialized( ++ const xla::Shape& shape, PjRtMemorySpace* memory_space) { ++ // TRACE_ME; ++ // TODO (@cryptodeal): C API doesn't implement this, but ++ // we'll want to ensure when the Buffer is initialized, ++ // the resulting array is correct shape/dtype. ++ ++ // Init empty array of dtype ++ auto array = mx::array({}); ++ return std::make_unique(array, shape, memory_space); ++} ++ ++absl::StatusOr GetAttributeFromBuffer( ++ xla::PjRtBuffer* buffer) { ++ // TRACE_ME; ++ if (buffer == nullptr || buffer->IsDeleted()) { ++ return xla::InvalidArgument("Buffer is null or deleted"); ++ } ++ auto mlir_buffer = dynamic_cast(buffer); ++ if (mlir_buffer == nullptr) { ++ return xla::InvalidArgument("Buffer is not a MlxPjrtBuffer"); ++ } ++ LOG(INFO) << "GetAttributeFromBuffer(" << (void*)buffer << ") -> " ++ << ToString(mlir_buffer->GetBufferAttribute()) << "\n"; ++ return mlir_buffer->GetBufferAttribute(); ++} ++ ++absl::StatusOr GetArrayFromBuffer(xla::PjRtBuffer* buffer) { ++ // TRACE_ME; ++ if (buffer == nullptr || buffer->IsDeleted()) { ++ return xla::InvalidArgument("Buffer is null or deleted"); ++ } ++ auto mlir_buffer = dynamic_cast(buffer); ++ if (mlir_buffer == nullptr) { ++ return xla::InvalidArgument("Buffer is not a MlxPjrtBuffer"); ++ } ++ ++ return mlir_buffer->GetArray(); ++} ++ ++DenseElementsAttr CloneIntoContext(DenseElementsAttr attr, ++ MLIRContext& context) { ++ Type type = mlir::parseType(mlir::debugString(attr.getType()), &context); ++ return DenseElementsAttr::getFromRawBuffer(llvm::cast(type), ++ attr.getRawData()); ++} ++ ++} // namespace mlir::stablehlo +\ No newline at end of file +diff --git a/xla/pjrt/plugin/mlx/buffer.h b/xla/pjrt/plugin/mlx/buffer.h +new file mode 100644 +index 0000000000..1740f3c779 +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/buffer.h +@@ -0,0 +1,43 @@ ++/* Copyright 2024 The OpenXLA Authors. ++Licensed under the Apache License, Version 2.0 (the "License"); ++you may not use this file except in compliance with the License. ++You may obtain a copy of the License at ++ http://www.apache.org/licenses/LICENSE-2.0 ++Unless required by applicable law or agreed to in writing, software ++distributed under the License is distributed on an "AS IS" BASIS, ++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++See the License for the specific language governing permissions and ++limitations under the License. ++==============================================================================*/ ++ ++#ifndef XLA_PJRT_PLUGIN_STABLEHLO_MLX_BUFFER_H_ ++#define XLA_PJRT_PLUGIN_STABLEHLO_MLX_BUFFER_H_ ++ ++#include ++ ++#include "mlir/IR/BuiltinAttributes.h" ++#include "mlx/mlx.h" ++#include "xla/literal.h" ++#include "xla/pjrt/pjrt_client.h" ++ ++namespace mx = mlx::core; ++ ++namespace mlir::stablehlo { ++std::unique_ptr CreateMlirBufferFromMlxArray( ++ mx::array array, xla::PjRtMemorySpace* memory_space); ++std::unique_ptr CreateMlirBufferFromLiteral( ++ const xla::LiteralSlice& literal, xla::PjRtMemorySpace* memory_space); ++std::unique_ptr CreateMlirBufferFromAttribute( ++ mx::array array, DenseElementsAttr attribute, ++ xla::PjRtMemorySpace* memory_space); ++std::unique_ptr CreateMlirBufferUninitialized( ++ const xla::Shape& shape, xla::PjRtMemorySpace* memory_space); ++absl::StatusOr GetAttributeFromBuffer( ++ xla::PjRtBuffer* buffer); ++absl::StatusOr GetArrayFromBuffer(xla::PjRtBuffer* buffer); ++DenseElementsAttr CloneIntoContext(DenseElementsAttr attr, ++ MLIRContext& context); ++ ++} // namespace mlir::stablehlo ++ ++#endif // XLA_PJRT_PLUGIN_STABLEHLO_MLX_BUFFER_H_ +\ No newline at end of file +diff --git a/xla/pjrt/plugin/mlx/client_c_pjrt.cc b/xla/pjrt/plugin/mlx/client_c_pjrt.cc +new file mode 100644 +index 0000000000..4ce81dcc46 +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/client_c_pjrt.cc +@@ -0,0 +1,72 @@ ++/* Copyright 2024 The OpenXLA Authors. ++Licensed under the Apache License, Version 2.0 (the "License"); ++you may not use this file except in compliance with the License. ++You may obtain a copy of the License at ++ http://www.apache.org/licenses/LICENSE-2.0 ++Unless required by applicable law or agreed to in writing, software ++distributed under the License is distributed on an "AS IS" BASIS, ++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++See the License for the specific language governing permissions and ++limitations under the License. ++==============================================================================*/ ++ ++#include "xla/pjrt/plugin/mlx/client_c_pjrt.h" ++ ++#include ++#include ++#include ++ ++#include "absl/status/status.h" ++#include "absl/strings/string_view.h" ++#include "xla/pjrt/c/pjrt_c_api.h" ++#include "xla/pjrt/c/pjrt_c_api_layouts_extension.h" ++#include "xla/pjrt/c/pjrt_c_api_wrapper_impl.h" ++#include "xla/pjrt/pjrt_client.h" ++#include "xla/pjrt/plugin/mlx/client_cpp_pjrt.h" ++ ++namespace mlir::stablehlo { ++ ++using xla::PjRtClient; ++ ++std::unique_ptr GetPluginPjRtClient() { ++ return CreateStablehloMlxPjrtClient(); ++} ++ ++// Create my client ++PJRT_Error* PJRT_StablehloMlxClient_Create(PJRT_Client_Create_Args* args) { ++ std::unique_ptr client = GetPluginPjRtClient(); ++ args->client = pjrt::CreateWrapperClient(std::move(client)); ++ // printf("Creating PJRT Client from client\n"); ++ return nullptr; ++} ++ ++PJRT_Error* PJRT_StablehloMlxExecuteContext_Create( ++ PJRT_ExecuteContext_Create_Args* args) { ++ return new PJRT_Error{absl::UnimplementedError( ++ "ExecuteContext not supported for client execution.")}; ++} ++ ++PJRT_Error* PJRT_StablehloMlxDeviceTopology_Create( ++ PJRT_TopologyDescription_Create_Args* args) { ++ return new PJRT_Error{absl::UnimplementedError( ++ "Topology not supported for client compilation.")}; ++} ++ ++} // namespace mlir::stablehlo ++ ++const PJRT_Api* GetPjrtApi() { ++ // printf("C++ Calling GetPjrtApi"); ++ static PJRT_Layouts_Extension layouts_extension = ++ pjrt::CreateLayoutsExtension(nullptr); ++ ++ static const PJRT_Api pjrt_api = pjrt::CreatePjrtApi( ++ mlir::stablehlo::PJRT_StablehloMlxClient_Create, ++ mlir::stablehlo::PJRT_StablehloMlxExecuteContext_Create, ++ mlir::stablehlo::PJRT_StablehloMlxDeviceTopology_Create, ++ pjrt::PJRT_Plugin_Initialize_NoOp, ++ reinterpret_cast(&layouts_extension), ++ pjrt::PJRT_Plugin_Attributes_Xla); ++ ++ // printf("stablehlo_mlx client called GetPjrtApi\n"); ++ return &pjrt_api; ++} +\ No newline at end of file +diff --git a/xla/pjrt/plugin/mlx/client_c_pjrt.h b/xla/pjrt/plugin/mlx/client_c_pjrt.h +new file mode 100644 +index 0000000000..cc56626953 +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/client_c_pjrt.h +@@ -0,0 +1,24 @@ ++/* Copyright 2024 The OpenXLA Authors. ++Licensed under the Apache License, Version 2.0 (the "License"); ++you may not use this file except in compliance with the License. ++You may obtain a copy of the License at ++ http://www.apache.org/licenses/LICENSE-2.0 ++Unless required by applicable law or agreed to in writing, software ++distributed under the License is distributed on an "AS IS" BASIS, ++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++See the License for the specific language governing permissions and ++limitations under the License. ++==============================================================================*/ ++ ++#ifndef XLA_PJRT_PLUGIN_STABLEHLO_MLX_CLIENT_C_PJRT_H_ ++#define XLA_PJRT_PLUGIN_STABLEHLO_MLX_CLIENT_C_PJRT_H_ ++ ++#include "xla/pjrt/c/pjrt_c_api.h" ++ ++extern "C" { ++ ++// Does not pass ownership of returned PJRT_Api* to caller. ++const PJRT_Api* GetPjrtApi(); ++} ++ ++#endif // XLA_PJRT_PLUGIN_STABLEHLO_MLX_CLIENT_C_PJRT_H_ +\ No newline at end of file +diff --git a/xla/pjrt/plugin/mlx/client_cpp_pjrt.cc b/xla/pjrt/plugin/mlx/client_cpp_pjrt.cc +new file mode 100644 +index 0000000000..9bdf4095b5 +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/client_cpp_pjrt.cc +@@ -0,0 +1,298 @@ ++/* Copyright 2024 The OpenXLA Authors. ++Licensed under the Apache License, Version 2.0 (the "License"); ++you may not use this file except in compliance with the License. ++You may obtain a copy of the License at ++ http://www.apache.org/licenses/LICENSE-2.0 ++Unless required by applicable law or agreed to in writing, software ++distributed under the License is distributed on an "AS IS" BASIS, ++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++See the License for the specific language governing permissions and ++limitations under the License. ++==============================================================================*/ ++ ++#include "xla/pjrt/plugin/mlx/client_cpp_pjrt.h" ++ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "absl/status/statusor.h" ++#include "absl/strings/str_format.h" ++#include "absl/strings/string_view.h" ++#include "absl/types/span.h" ++#include "mlx/mlx.h" ++#include "xla/literal.h" ++#include "xla/pjrt/host_memory_spaces.h" ++#include "xla/pjrt/pjrt_client.h" ++#include "xla/pjrt/pjrt_common.h" ++#include "xla/pjrt/pjrt_compiler.h" ++#include "xla/pjrt/plugin/mlx/buffer.h" ++#include "xla/pjrt/plugin/mlx/device.h" ++#include "xla/pjrt/plugin/mlx/executable.h" ++#include "xla/pjrt/plugin/mlx/logging.h" ++#include "xla/pjrt/plugin/mlx/utils.h" ++#include "xla/shape_util.h" ++#include "xla/util.h" ++#include "tsl/platform/fingerprint.h" ++#include "tsl/platform/statusor.h" ++ ++namespace mx = mlx::core; ++ ++#define UNIMPLEMENTED(name) \ ++ xla::Unimplemented("MlxPjrtBuffer::" #name " is not implemented") ++ ++namespace mlir::stablehlo { ++ ++const xla::PjRtPlatformId kStablehloMlxBackendId = ++ tsl::Fingerprint64(kStablehloMlxBackendName); ++ ++class StablehloMlxPjrtClient : public xla::PjRtClient { ++ public: ++ StablehloMlxPjrtClient() ++ : xla::PjRtClient(), ++ owned_devices_(), ++ devices_(), ++ owned_memory_space_(), ++ memory_space_(nullptr) { ++ // Init device and memory space. ++ // TRACE_ME_MEMBER; ++ owned_devices_.push_back(GetStablehloMlxDevice(this)); ++ devices_.push_back(owned_devices_.back().get()); ++ owned_memory_space_ = std::make_unique( ++ /*id=*/0, devices_.front()); ++ memory_space_ = owned_memory_space_.get(); ++ AttachStablehloMlxMemorySpace(devices_.front(), memory_space_); ++ } ++ ++ ~StablehloMlxPjrtClient() override { ++ // mx::metal::stop_capture(); ++ }; ++ ++ absl::string_view platform_name() const override { ++ // TRACE_ME_MEMBER; ++ return kStablehloMlxBackendName; ++ } ++ int process_index() const override { ++ // TRACE_ME_MEMBER; ++ return 0; ++ } ++ ++ int device_count() const override { ++ // TRACE_ME_MEMBER; ++ return devices_.size(); ++ } ++ ++ int addressable_device_count() const override { ++ // TRACE_ME_MEMBER; ++ return devices_.size(); ++ } ++ ++ absl::Span devices() const override { ++ // TRACE_ME_MEMBER; ++ return devices_; ++ } ++ absl::Span addressable_devices() const override { ++ // TRACE_ME_MEMBER; ++ return devices_; ++ } ++ ++ absl::Span memory_spaces() const override { ++ // TRACE_ME_MEMBER; ++ return absl::MakeSpan(&memory_space_, 1); ++ } ++ ++ // Return an ID that identifies the platform via tsl fingerprint. ++ xla::PjRtPlatformId platform_id() const override { ++ // TRACE_ME_MEMBER; ++ return kStablehloMlxBackendId; ++ } ++ ++ // Returns a string containing human-readable, platform-specific version ++ // info (e.g. the CUDA version on GPU or libtpu version on Cloud TPU). ++ absl::string_view platform_version() const override { ++ // TRACE_ME_MEMBER; ++ return "StableHLO MLX v0.1"; ++ } ++ ++ ///////////// ++ // Device ++ ///////////// ++ ++ // Lookup any PjRtDevice for a given PjRtDevice::id(). ++ // TODO: Should this be a base class? I.e. why doesn't the base client have ++ // a vector a device pointers? ++ absl::StatusOr LookupDevice( ++ xla::PjRtGlobalDeviceId global_device_id) const override { ++ // TRACE_ME_MEMBER; ++ for (auto device : devices_) { ++ if (device->global_device_id() == global_device_id) { ++ return device; ++ } ++ } ++ // TODO: This error should be a base class method since its used in tests. ++ return xla::InvalidArgument("No matching device found for device_id %d", ++ global_device_id.value()); ++ } ++ ++ absl::StatusOr LookupAddressableDevice( ++ xla::PjRtLocalDeviceId local_device_id) const override { ++ // TRACE_ME_MEMBER; ++ ++ for (auto* device : addressable_devices()) { ++ if (local_device_id == device->local_device_id()) { ++ return device; ++ } ++ } ++ return xla::InvalidArgument( ++ "No matching device found for local_device_id %d", ++ local_device_id.value()); ++ } ++ ++ absl::StatusOr GetDefaultDeviceAssignment( ++ int num_replicas, int num_partitions) const override { ++ // TRACE_ME_MEMBER; ++ xla::DeviceAssignment assignment(num_replicas, num_partitions); ++ for (int64_t i = 0; i < num_replicas; ++i) { ++ for (int64_t j = 0; j < num_partitions; ++j) { ++ auto idx = (i + (j * num_replicas)) % devices_.size(); ++ assignment(i, j) = devices_[idx]->global_device_id().value(); ++ } ++ } ++ return assignment; ++ } ++ ++ ///////////////// ++ // Buffer methods ++ ///////////////// ++ ++ absl::StatusOr> CreateErrorBuffer( ++ absl::Status error, const xla::Shape& shape, xla::PjRtDevice* device) { ++ // Prefer memory space implementation, device holding buffer is ++ // deprecated. ++ return CreateErrorBuffer(error, shape, ++ device->default_memory_space().value_or(nullptr)); ++ } ++ ++ absl::StatusOr> CreateErrorBuffer( ++ absl::Status error, const xla::Shape& shape, ++ xla::PjRtMemorySpace* memory) { ++ return UNIMPLEMENTED(CreateErrorBuffer); ++ } ++ ++ absl::StatusOr> CreateUninitializedBuffer( ++ const xla::Shape& shape, xla::PjRtDevice* device) { ++ // TRACE_ME_MEMBER; ++ return CreateMlirBufferUninitialized( ++ shape, device->default_memory_space().value_or(nullptr)); ++ } ++ ++ absl::StatusOr> ++ CreateBuffersForAsyncHostToDevice(absl::Span shapes, ++ xla::PjRtDevice* device) { ++ // TRACE_ME_MEMBER; ++ return CreateBuffersForAsyncHostToDevice( ++ shapes, device->default_memory_space().value_or(nullptr)); ++ } ++ ++ absl::StatusOr> ++ CreateBuffersForAsyncHostToDevice(absl::Span shapes, ++ xla::PjRtMemorySpace* memory_space) { ++ // TRACE_ME_MEMBER; ++ return UNIMPLEMENTED(CreateBuffersForAsyncHostToDevice); ++ } ++ ++ absl::StatusOr> BufferFromHostBuffer( ++ const void* data, xla::PrimitiveType type, absl::Span dims, ++ std::optional> byte_strides, ++ HostBufferSemantics host_buffer_semantics, ++ absl::AnyInvocable on_done_with_host_buffer, ++ xla::PjRtDevice* device) { ++ // TRACE_ME_MEMBER; ++ return BufferFromHostBuffer( ++ data, type, dims, byte_strides, host_buffer_semantics, ++ std::move(on_done_with_host_buffer), ++ device->default_memory_space().value_or(nullptr), nullptr); ++ } ++ ++ absl::StatusOr> BufferFromHostBuffer( ++ const void* data, xla::PrimitiveType type, absl::Span dims, ++ std::optional> byte_strides, ++ HostBufferSemantics host_buffer_semantics, ++ absl::AnyInvocable on_done_with_host_buffer, ++ xla::PjRtDevice* device, const xla::Layout* device_layout) { ++ // TRACE_ME_MEMBER; ++ return BufferFromHostBuffer( ++ data, type, dims, byte_strides, host_buffer_semantics, ++ std::move(on_done_with_host_buffer), ++ device->default_memory_space().value_or(nullptr), device_layout); ++ } ++ ++ absl::StatusOr> BufferFromHostBuffer( ++ const void* data, xla::PrimitiveType type, absl::Span dims, ++ std::optional> byte_strides, ++ HostBufferSemantics host_buffer_semantics, ++ absl::AnyInvocable on_done_with_host_buffer, ++ xla::PjRtMemorySpace* memory_space, const xla::Layout* device_layout) { ++ // TRACE_ME_MEMBER; ++ TF_ASSIGN_OR_RETURN( ++ mx::array array_buffer, ++ utils::array::fromHostBuffer(data, dims, byte_strides, type)); ++ auto buffer = CreateMlirBufferFromMlxArray(array_buffer, memory_space); ++ if (on_done_with_host_buffer) { ++ // If host is awaiting the result, must call this function. ++ std::move(on_done_with_host_buffer)(); ++ on_done_with_host_buffer = nullptr; ++ } ++ return buffer; ++ } ++ ++ absl::StatusOr> BufferFromHostLiteral( ++ const xla::LiteralSlice& literal, xla::PjRtDevice* device) { ++ // TRACE_ME_MEMBER; ++ return CreateMlirBufferFromLiteral( ++ literal, device->default_memory_space().value_or(nullptr)); ++ } ++ ++ absl::StatusOr> BufferFromHostLiteral( ++ const xla::LiteralSlice& literal, xla::PjRtMemorySpace* memory_space) { ++ // TRACE_ME_MEMBER; ++ return CreateMlirBufferFromLiteral(literal, memory_space); ++ } ++ ++ /////////// ++ // Compile ++ /////////// ++ absl::StatusOr> Compile( ++ mlir::ModuleOp module, xla::CompileOptions options) override { ++ // TRACE_ME_MEMBER; ++ return mlir::stablehlo::StablehloMlxCompile( ++ module, GetDefaultDeviceAssignment(1, devices_.size()).value(), this); ++ } ++ ++ // Compile `computation` with given `options`. ++ absl::StatusOr> Compile( ++ const xla::XlaComputation& computation, ++ xla::CompileOptions options) override { ++ // TRACE_ME_MEMBER; ++ return mlir::stablehlo::StablehloMlxCompile( ++ computation.proto(), ++ GetDefaultDeviceAssignment(1, devices_.size()).value(), this); ++ } ++ ++ private: ++ std::vector> owned_devices_; ++ std::vector devices_; ++ std::unique_ptr owned_memory_space_; ++ xla::PjRtMemorySpace* memory_space_; ++}; // end class ++ ++std::unique_ptr CreateStablehloMlxPjrtClient() { ++ // mx::metal::start_capture("mlx_trace.gputrace"); ++ SetupLogLevelFromEnv(); ++ return std::make_unique(); ++} ++ ++} // namespace mlir::stablehlo +\ No newline at end of file +diff --git a/xla/pjrt/plugin/mlx/client_cpp_pjrt.h b/xla/pjrt/plugin/mlx/client_cpp_pjrt.h +new file mode 100644 +index 0000000000..8c90942f0c +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/client_cpp_pjrt.h +@@ -0,0 +1,30 @@ ++/* Copyright 2024 The OpenXLA Authors. ++Licensed under the Apache License, Version 2.0 (the "License"); ++you may not use this file except in compliance with the License. ++You may obtain a copy of the License at ++ http://www.apache.org/licenses/LICENSE-2.0 ++Unless required by applicable law or agreed to in writing, software ++distributed under the License is distributed on an "AS IS" BASIS, ++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++See the License for the specific language governing permissions and ++limitations under the License. ++==============================================================================*/ ++ ++#ifndef XLA_PJRT_PLUGIN_STABLEHLO_MLX_CLIENT_CPP_PJRT_H_ ++#define XLA_PJRT_PLUGIN_STABLEHLO_MLX_CLIENT_CPP_PJRT_H_ ++ ++#include ++ ++#include "xla/pjrt/pjrt_client.h" ++#include "xla/pjrt/pjrt_compiler.h" ++ ++namespace mlir::stablehlo { ++ ++inline constexpr char kStablehloMlxBackendName[] = "stablehlo_mlx"; ++extern const xla::PjRtPlatformId kStablehloMlxBackendId; ++ ++std::unique_ptr CreateStablehloMlxPjrtClient(); ++ ++} // namespace mlir::stablehlo ++ ++#endif // XLA_PJRT_PLUGIN_STABLEHLO_MLX_CLIENT_CPP_PJRT_H_ +\ No newline at end of file +diff --git a/xla/pjrt/plugin/mlx/device.cc b/xla/pjrt/plugin/mlx/device.cc +new file mode 100644 +index 0000000000..6984da5b7f +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/device.cc +@@ -0,0 +1,213 @@ ++/* Copyright 2024 The OpenXLA Authors. ++ ++ ++Licensed under the Apache License, Version 2.0 (the "License"); ++you may not use this file except in compliance with the License. ++You may obtain a copy of the License at ++ http://www.apache.org/licenses/LICENSE-2.0 ++Unless required by applicable law or agreed to in writing, software ++distributed under the License is distributed on an "AS IS" BASIS, ++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++See the License for the specific language governing permissions and ++limitations under the License. ++==============================================================================*/ ++ ++#include "mlx/mlx.h" ++ ++#include "xla/pjrt/plugin/mlx/device.h" ++ ++#include "absl/status/statusor.h" ++#include "absl/strings/str_format.h" ++#include "absl/strings/string_view.h" ++#include "absl/types/span.h" ++#include "xla/pjrt/pjrt_client.h" ++#include "xla/pjrt/pjrt_common.h" ++#include "xla/pjrt/pjrt_compiler.h" ++#include "xla/pjrt/pjrt_device_description.h" ++#include "xla/pjrt/plugin/mlx/logging.h" ++#include "xla/util.h" ++ ++#define UNIMPLEMENTED(name) \ ++ xla::Unimplemented("StablehloMlxDevice::" #name " is not implemented") ++ ++namespace mlir::stablehlo { ++ ++using xla::LiteralSlice; ++using xla::MutableBorrowingLiteral; ++using xla::PjRtClient; ++using xla::PjRtDevice; ++using xla::PjRtDeviceAttribute; ++using xla::PjRtDeviceDescription; ++using xla::PjRtGlobalDeviceId; ++using xla::PjRtLocalDeviceId; ++using xla::PjRtLocalHardwareId; ++using xla::PjRtMemorySpace; ++using xla::ScopedAsyncTrackingEvent; ++using xla::Unimplemented; ++ ++namespace mx = mlx::core; ++ ++inline std::string mlxDeviceToString(const mx::Device& device) { ++ std::ostringstream os; ++ os << device; ++ return os.str(); ++} ++// Devices need a device description. ++class StablehloMlxDeviceDescription final : public PjRtDeviceDescription { ++ public: ++ explicit StablehloMlxDeviceDescription(int process_id, int local_device_id) ++ : id_(local_device_id), ++ process_index_(process_id), ++ local_hardware_id_(local_device_id), ++ debug_string_(mlxDeviceToString(mx::default_device())), ++ to_string_(absl::StrFormat("%s(id=%d,pid=%d)", debug_string_, ++ id_.value(), process_index_)) { ++ // TRACE_ME_MEMBER; ++ } ++ ++ int id() const override { ++ // TRACE_ME_MEMBER; ++ return id_.value(); ++ } ++ int process_index() const override { ++ // TRACE_ME_MEMBER; ++ return process_index_; ++ } ++ int local_hardware_id() const { ++ // TRACE_ME_MEMBER; ++ return local_hardware_id_; ++ } ++ ++ absl::string_view device_kind() const override { ++ // TRACE_ME_MEMBER; ++ return DebugString(); ++ } ++ ++ absl::string_view DebugString() const override { ++ // TRACE_ME_MEMBER; ++ return debug_string_; ++ } ++ ++ absl::string_view ToString() const override { ++ // TRACE_ME_MEMBER; ++ return to_string_; ++ } ++ ++ const absl::flat_hash_map& Attributes() ++ const override { ++ // TRACE_ME_MEMBER; ++ return attributes_; ++ } ++ ++ private: ++ PjRtGlobalDeviceId id_; ++ int process_index_; ++ int local_hardware_id_; ++ std::string debug_string_; ++ std::string to_string_; ++ absl::flat_hash_map attributes_ = {}; ++}; ++ ++// Clients need devices, and clients own the devices. ++class StablehloMlxDevice : public PjRtDevice { ++ public: ++ explicit StablehloMlxDevice(PjRtClient* client) ++ : PjRtDevice(), client_(client), description_(0, 0) { ++ // TRACE_ME_MEMBER; ++ } ++ ++ absl::string_view DebugString() const override { ++ // TRACE_ME_MEMBER; ++ return "StablehloMlxDevice"; ++ } ++ ++ PjRtLocalDeviceId local_device_id() const override { ++ // TRACE_ME_MEMBER; ++ return PjRtLocalDeviceId(local_hardware_id().value()); ++ } ++ ++ PjRtLocalHardwareId local_hardware_id() const override { ++ // TRACE_ME_MEMBER; ++ return PjRtLocalHardwareId(description_.local_hardware_id()); ++ } ++ ++ PjRtClient* client() const override { ++ // TRACE_ME_MEMBER; ++ return client_; ++ } ++ ++ bool IsAddressable() const override { ++ // TRACE_ME_MEMBER; ++ return process_index() == client()->process_index(); ++ } ++ ++ absl::Status TransferToInfeed(const LiteralSlice& literal) override { ++ // TRACE_ME_MEMBER; ++ return UNIMPLEMENTED(TransferToInfeed); ++ } ++ ++ absl::Status TransferFromOutfeed(MutableBorrowingLiteral literal) override { ++ // TRACE_ME_MEMBER; ++ return UNIMPLEMENTED(TransferFromOutfeed); ++ } ++ ++ void AttachDefaultMemorySpace(PjRtMemorySpace* memory_space) { ++ // TRACE_ME_MEMBER; ++ memory_space_ = memory_space; ++ } ++ ++ absl::Span memory_spaces() const override { ++ // TRACE_ME_MEMBER; ++ return absl::MakeSpan(&memory_space_, 1); ++ } ++ ++ absl::StatusOr default_memory_space() const override { ++ // TRACE_ME_MEMBER; ++ if (!memory_space_) ++ return absl::InternalError("Plugin memory space unset."); ++ ++ return memory_space_; ++ } ++ ++ std::unique_ptr CreateAsyncTrackingEvent( ++ absl::string_view description) const override { ++ // TRACE_ME_MEMBER; ++ LOG(FATAL) << "Plugin does not implement CreateAsyncTrackingEvent."; ++ return nullptr; ++ } ++ ++ const PjRtDeviceDescription& description() const override { ++ // TRACE_ME_MEMBER; ++ return description_; ++ } ++ ++ private: ++ PjRtClient* client_; ++ PjRtMemorySpace* memory_space_; // unpinned memory owned by client ++ StablehloMlxDeviceDescription description_; ++}; ++ ++// Device Description ++std::unique_ptr GetStablehloMlxDeviceDescription( ++ int process_id, int local_device_id) { ++ return std::make_unique(process_id, ++ local_device_id); ++} ++ ++// Mlx Device ++std::unique_ptr GetStablehloMlxDevice(PjRtClient* client) { ++ return std::make_unique(client); ++} ++ ++void AttachStablehloMlxMemorySpace(PjRtDevice* device, ++ PjRtMemorySpace* memory_space) { ++ auto stablehlo_device = dynamic_cast(device); ++ if (stablehlo_device == nullptr) { ++ LOG(FATAL) << "Plugin cannot attach memory space to device of kind " ++ << device->device_kind(); ++ return; ++ } ++ stablehlo_device->AttachDefaultMemorySpace(memory_space); ++} ++ ++} // namespace mlir::stablehlo +\ No newline at end of file +diff --git a/xla/pjrt/plugin/mlx/device.h b/xla/pjrt/plugin/mlx/device.h +new file mode 100644 +index 0000000000..ef42968a3a +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/device.h +@@ -0,0 +1,36 @@ ++/* Copyright 2024 The OpenXLA Authors. ++Licensed under the Apache License, Version 2.0 (the "License"); ++you may not use this file except in compliance with the License. ++You may obtain a copy of the License at ++ http://www.apache.org/licenses/LICENSE-2.0 ++Unless required by applicable law or agreed to in writing, software ++distributed under the License is distributed on an "AS IS" BASIS, ++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++See the License for the specific language governing permissions and ++limitations under the License. ++==============================================================================*/ ++ ++#ifndef XLA_PJRT_PLUGIN_STABLEHLO_MLX_DEVICE_H_ ++#define XLA_PJRT_PLUGIN_STABLEHLO_MLX_DEVICE_H_ ++ ++#include ++ ++#include "xla/pjrt/pjrt_client.h" ++#include "xla/pjrt/pjrt_compiler.h" ++#include "xla/pjrt/pjrt_device_description.h" ++ ++namespace mlir::stablehlo { ++ ++// Device Description ++std::unique_ptr GetStablehloMlxDeviceDescription( ++ int process_id, int local_device_id); ++ ++// MLX Device ++std::unique_ptr GetStablehloMlxDevice(xla::PjRtClient* client); ++ ++void AttachStablehloMlxMemorySpace(xla::PjRtDevice* device, ++ xla::PjRtMemorySpace* memory_space); ++ ++} // namespace mlir::stablehlo ++ ++#endif // XLA_PJRT_PLUGIN_STABLEHLO_MLX_DEVICE_H_ +\ No newline at end of file +diff --git a/xla/pjrt/plugin/mlx/executable.cc b/xla/pjrt/plugin/mlx/executable.cc +new file mode 100644 +index 0000000000..cf551b2dc7 +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/executable.cc +@@ -0,0 +1,2370 @@ ++/* Copyright 2024 The OpenXLA Authors. ++Licensed under the Apache License, Version 2.0 (the "License"); ++you may not use this file except in compliance with the License. ++You may obtain a copy of the License at ++ http://www.apache.org/licenses/LICENSE-2.0 ++Unless required by applicable law or agreed to in writing, software ++distributed under the License is distributed on an "AS IS" BASIS, ++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++See the License for the specific language governing permissions and ++limitations under the License. ++==============================================================================*/ ++ ++#include "xla/pjrt/plugin/mlx/executable.h" ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++// TODO(@cryptodeal): might need to update `BUILD` ++#include "mlir/IR/Visitors.h" ++#include "llvm/ADT/TypeSwitch.h" ++#include "llvm/ADT/Hashing.h" ++ ++#include "mlx/mlx.h" ++#include "absl/log/log.h" ++#include "absl/status/status.h" ++#include "absl/status/statusor.h" ++#include "mlir/Bytecode/BytecodeWriter.h" ++#include "mlir/Dialect/Func/IR/FuncOps.h" ++#include "mlir/IR/AsmState.h" ++#include "mlir/IR/BuiltinAttributes.h" ++#include "mlir/IR/MLIRContext.h" ++#include "mlir/IR/Operation.h" ++#include "mlir/IR/OwningOpRef.h" ++#include "mlir/Interfaces/DataLayoutInterfaces.h" ++#include "mlir/Pass/PassManager.h" ++#include "mlir/Support/LLVM.h" ++#include "stablehlo/reference/Api.h" ++#include "stablehlo/transforms/optimization/Passes.h" ++#include "xla/hlo/translate/stablehlo.h" ++#include "xla/mlir/utils/error_util.h" ++#include "xla/mlir/utils/type_util.h" ++#include "xla/mlir_hlo/mhlo/transforms/passes.h" ++#include "xla/mlir_hlo/stablehlo_ext/transforms/passes.h" ++#include "xla/pjrt/pjrt_client.h" ++#include "xla/pjrt/pjrt_executable.h" ++#include "xla/pjrt/pjrt_future.h" ++#include "xla/pjrt/plugin/mlx/buffer.h" ++#include "xla/pjrt/plugin/mlx/logging.h" ++#include "xla/pjrt/plugin/mlx/utils.h" ++#include "xla/service/computation_placer.h" ++#include "tsl/platform/status.h" ++#include "tsl/platform/statusor.h" ++ ++#define DEBUG_TYPE "stablehlo-pjrt" ++ ++namespace mx = mlx::core; ++ ++typedef absl::StatusOr StatusOrArray; ++typedef absl::StatusOr> StatusOrArrays; ++typedef absl::StatusOr StatusOrInt; ++ ++// ZML supports up to 8 dimensions ++auto index_space(const std::vector& dims) { ++ using namespace ranges; ++ // `MAX_RANK` for zml is 8 ++ std::vector used_dims(8, 1); ++ std::copy(dims.begin(), dims.end(), used_dims.begin()); ++ return views::cartesian_product( ++ views::indices(used_dims[0]), views::indices(used_dims[1]), ++ views::indices(used_dims[2]), views::indices(used_dims[3]), ++ views::indices(used_dims[4]), views::indices(used_dims[5]), ++ views::indices(used_dims[6]), views::indices(used_dims[7])); ++} ++ ++template >>> ++std::vector to_vector(Tuple&& tuple) { ++ return std::apply( ++ [](auto&&... elems) { ++ return std::vector{std::forward(elems)...}; ++ }, ++ std::forward(tuple)); ++} ++ ++template >>> ++std::vector to_vector(Tuple&& tuple, size_t size) { ++ std::vector result = to_vector(tuple); ++ result.resize(size); ++ return result; ++} ++ ++namespace mlir::stablehlo { ++ ++#define UNIMPLEMENTED(name) \ ++ xla::Unimplemented("MlxPjRtBuffer::" #name " is not implemented") ++ ++using xla::DeviceAssignment; ++using xla::PjRtBuffer; ++using xla::PjRtClient; ++using xla::PjRtDevice; ++using xla::PjRtFuture; ++using xla::PjRtLoadedExecutable; ++using xla::PjRtMemorySpace; ++ ++void resolveFuncDeps(ModuleOp mod, std::string func_name, Tree& dep_tree, ++ Node* parent = nullptr) { ++ auto new_node = dep_tree.insert(func_name, parent); ++ auto func = mod.lookupSymbol(func_name); ++ auto& block = func.front(); ++ std::set func_names; ++ for (auto op : block.getOps()) { ++ std::string callee_name = op.getCallee().str(); ++ func_names.insert(callee_name); ++ resolveFuncDeps(mod, callee_name, dep_tree, new_node); ++ } ++} ++ ++absl::Status validateModule(ModuleOp mod) { ++ absl::Status status; ++ // TODO: some ops should have additional checks (e.g. `stablehlo::ReduceOp`, ++ // `stablehlo::SortOp`, etc.) where we need additional checks at compile time ++ // to ensure that we're correctly mapping to the corresponding MLX op. ++ mod.walk([&status](Operation* op) { ++ // Series of checks to ensure we support the module ++ return llvm::TypeSwitch(op) ++ .Case([&status, &op](auto o) { ++ // Validate all input types are compatible with mlx ++ for (auto operand : o->getOperands()) { ++ auto primitive_type = xla::ConvertMlirTypeToPrimitiveType( ++ mlir::cast(operand.getType()).getElementType()); ++ switch (primitive_type) { ++ case xla::PrimitiveType::PRED: ++ case xla::PrimitiveType::U8: ++ case xla::PrimitiveType::S8: ++ case xla::PrimitiveType::U16: ++ case xla::PrimitiveType::S16: ++ case xla::PrimitiveType::U32: ++ case xla::PrimitiveType::S32: ++ case xla::PrimitiveType::U64: ++ case xla::PrimitiveType::S64: ++ case xla::PrimitiveType::F16: ++ case xla::PrimitiveType::BF16: ++ case xla::PrimitiveType::F32: ++ case xla::PrimitiveType::C64: ++ break; ++ default: { ++ std::cout << "Unsupported type: " ++ << xla::PrimitiveType_Name(primitive_type).c_str() ++ << std::endl; ++ status = absl::UnimplementedError( ++ absl::StrCat("Unsupported type: ", ToString(op))); ++ return WalkResult::interrupt(); ++ } ++ } ++ } ++ ++ // Validate all result types are compatible with mlx ++ for (auto result : o->getResults()) { ++ auto primitive_type = xla::ConvertMlirTypeToPrimitiveType( ++ mlir::cast(result.getType()).getElementType()); ++ switch (primitive_type) { ++ case xla::PrimitiveType::PRED: ++ case xla::PrimitiveType::U8: ++ case xla::PrimitiveType::S8: ++ case xla::PrimitiveType::U16: ++ case xla::PrimitiveType::S16: ++ case xla::PrimitiveType::U32: ++ case xla::PrimitiveType::S32: ++ case xla::PrimitiveType::U64: ++ case xla::PrimitiveType::S64: ++ case xla::PrimitiveType::F16: ++ case xla::PrimitiveType::BF16: ++ case xla::PrimitiveType::F32: ++ case xla::PrimitiveType::C64: ++ break; ++ default: { ++ std::cout << "Unsupported type: " ++ << xla::PrimitiveType_Name(primitive_type).c_str() ++ << std::endl; ++ status = absl::UnimplementedError( ++ absl::StrCat("Unsupported type: ", ToString(op))); ++ return WalkResult::interrupt(); ++ } ++ } ++ } ++ return mlir::WalkResult::advance(); ++ }) ++ // TODO(@cryptodeal): These ops were supported prior to `mlx::compile` ++ // need to modify the implementation to ensure no use of `mlx::eval` and ++ // ensure that `mlx::random::key` is being propogated correctly. ++ .Case( ++ [&status, &op](auto o) { ++ status = absl::UnimplementedError( ++ absl::StrCat("Unsupported op: ", ToString(op))); ++ return WalkResult::interrupt(); ++ }) ++ .Default([&status, &op](auto o) { ++ status = absl::UnimplementedError( ++ absl::StrCat("Unsupported op: ", ToString(op))); ++ return WalkResult::interrupt(); ++ }); ++ }); ++ return status; ++} ++ ++mlir::OwningOpRef cloneIntoContext(ModuleOp module, ++ MLIRContext& context) { ++ // Clone the module into the context. MHLO->StableHLO just in case. ++ PassManager pm(module->getContext()); ++ pm.addPass(mhlo::createHloLegalizeToStablehloPass()); ++ if (failed(pm.run(module))) { ++ LOG(ERROR) << "Failed to convert MHLO to StableHLO"; ++ return nullptr; ++ } ++ ++ std::string bytecode; ++ llvm::raw_string_ostream os(bytecode); ++ mlir::BytecodeWriterConfig config; ++ mlir::OwningOpRef cloned = module.clone(); ++ if (mlir::failed(mlir::writeBytecodeToFile(*cloned, os, config))) { ++ LOG(ERROR) << "Failed to write bytecode to string\n"; ++ return nullptr; ++ } ++ return *parseStablehloModule(bytecode, context); ++} ++ ++LogicalResult decomposeChloToStablehlo(ModuleOp module) { ++ PassManager pm(module->getContext()); ++ stablehlo_ext::createChloLegalizeToStablehloPipeline(pm); ++ if (failed(pm.run(module))) { ++ return module->emitError() << "Failed to recompose CHLO"; ++ } ++ return success(); ++} ++ ++std::optional getUnsupportedOp(ModuleOp module) { ++ std::optional unsupported_op(std::nullopt); ++ module.walk([&unsupported_op](Operation* op) { ++ auto cc = llvm::dyn_cast(op); ++ if (cc) { ++ unsupported_op = op; ++ return WalkResult::interrupt(); ++ } ++ return WalkResult::advance(); ++ }); ++ return unsupported_op; ++} ++ ++LogicalResult runHardwareIndependentOptimizations(ModuleOp module) { ++ PassManager pm(module->getContext()); ++ pm.addNestedPass( ++ stablehlo::createStablehloAggressiveFolderPass()); ++ pm.addNestedPass( ++ stablehlo::createStablehloAggressiveSimplificationPass()); ++ if (failed(pm.run(module))) { ++ return module->emitError() ++ << "Failed to run hardware independent optimizations"; ++ } ++ return success(); ++} ++ ++mlir::ShapedType getResultType(ModuleOp module, mx::Dtype dtype, ++ SmallVector shape) { ++ switch (dtype) { ++ case mx::bfloat16: ++ return RankedTensorType::get(ArrayRef(shape), ++ BFloat16Type::get(module.getContext())); ++ case mx::float16: ++ return RankedTensorType::get(ArrayRef(shape), ++ Float16Type::get(module.getContext())); ++ case mx::int8: ++ return RankedTensorType::get( ++ ArrayRef(shape), ++ IntegerType::get(module.getContext(), 8, ++ mlir::IntegerType::Signless)); ++ case mx::uint8: ++ return RankedTensorType::get( ++ ArrayRef(shape), ++ IntegerType::get(module.getContext(), 8, ++ mlir::IntegerType::Unsigned)); ++ case mx::int16: ++ return RankedTensorType::get( ++ ArrayRef(shape), ++ IntegerType::get(module.getContext(), 16, ++ mlir::IntegerType::Signless)); ++ case mx::uint16: ++ return RankedTensorType::get( ++ ArrayRef(shape), ++ IntegerType::get(module.getContext(), 16, ++ mlir::IntegerType::Unsigned)); ++ case mx::int32: ++ return RankedTensorType::get( ++ ArrayRef(shape), ++ IntegerType::get(module.getContext(), 32, ++ mlir::IntegerType::Signless)); ++ case mx::uint32: ++ return RankedTensorType::get( ++ ArrayRef(shape), ++ IntegerType::get(module.getContext(), 32, ++ mlir::IntegerType::Unsigned)); ++ case mx::int64: ++ return RankedTensorType::get( ++ ArrayRef(shape), ++ IntegerType::get(module.getContext(), 64, ++ mlir::IntegerType::Signless)); ++ case mx::uint64: ++ return RankedTensorType::get( ++ ArrayRef(shape), ++ IntegerType::get(module.getContext(), 64, ++ mlir::IntegerType::Unsigned)); ++ case mx::complex64: ++ return RankedTensorType::get( ++ ArrayRef(shape), ++ ComplexType::get(Float32Type::get(module.getContext()))); ++ default: ++ return RankedTensorType::get(ArrayRef(shape), ++ Float32Type::get(module.getContext())); ++ } ++} ++ ++// Returns true if the given `ReduceOp` matches ++// `zml.Tensor.argMax` op implementation. ++bool isArgMaxReduce(stablehlo::ReduceOp& reduce_op) { ++ Block& body_block = reduce_op.getBody().front(); ++ unsigned op_count = 0; ++ // TODO (@cryptodeal): This is a lazy verification; maybe check operands ++ // for each operation in the block. ++ for (auto& op : body_block.getOperations()) { ++ switch (op_count) { ++ case 0: { ++ auto compare_op = llvm::dyn_cast(op); ++ if (compare_op && ++ compare_op.getComparisonDirection() == ComparisonDirection::GT) { ++ break; ++ } else ++ return false; ++ } ++ case 1: { ++ auto compare_op = llvm::dyn_cast(op); ++ if (compare_op && ++ compare_op.getComparisonDirection() == ComparisonDirection::NE) { ++ break; ++ } else ++ return false; ++ } ++ case 2: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ case 3: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ case 4: { ++ auto compare_op = llvm::dyn_cast(op); ++ if (compare_op && ++ compare_op.getComparisonDirection() == ComparisonDirection::EQ) { ++ break; ++ } else ++ return false; ++ } ++ case 5: { ++ auto compare_op = llvm::dyn_cast(op); ++ if (compare_op && ++ compare_op.getComparisonDirection() == ComparisonDirection::LT) { ++ break; ++ } else ++ return false; ++ } ++ case 6: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ case 7: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ case 8: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ case 9: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ default: ++ return false; ++ } ++ op_count++; ++ } ++ return true; ++} ++ ++// Returns true if the given `ReduceOp` matches ++// `zml.Tensor.sum` op implementation. ++bool isSumReduce(stablehlo::ReduceOp& reduce_op) { ++ Block& body_block = reduce_op.getBody().front(); ++ unsigned op_count = 0; ++ // TODO (@cryptodeal): This is a lazy verification; maybe check operands ++ // for each operation in the block. ++ for (auto& op : body_block.getOperations()) { ++ switch (op_count) { ++ case 0: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ case 1: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ } ++ op_count++; ++ } ++ return true; ++} ++ ++// Returns true if the given `ReduceOp` matches ++// `zml.Tensor.max` op implementation. ++bool isMaxReduce(stablehlo::ReduceOp& reduce_op) { ++ Block& body_block = reduce_op.getBody().front(); ++ unsigned op_count = 0; ++ // TODO (@cryptodeal): This is a lazy verification; maybe check operands ++ // for each operation in the block. ++ for (auto& op : body_block.getOperations()) { ++ switch (op_count) { ++ case 0: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ case 1: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ } ++ op_count++; ++ } ++ return true; ++} ++ ++// Returns true if the given `ReduceOp` matches ++// `zml.Tensor.max` op implementation. ++bool isMinReduce(stablehlo::ReduceOp& reduce_op) { ++ Block& body_block = reduce_op.getBody().front(); ++ unsigned op_count = 0; ++ // TODO (@cryptodeal): This is a lazy verification; maybe check operands ++ // for each operation in the block. ++ for (auto& op : body_block.getOperations()) { ++ switch (op_count) { ++ case 0: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ case 1: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ } ++ op_count++; ++ } ++ return true; ++} ++ ++// Returns true if the given `ReduceOp` matches ++// `zml.Tensor.any` op implementation. ++bool isAnyReduce(stablehlo::ReduceOp& reduce_op) { ++ Block& body_block = reduce_op.getBody().front(); ++ unsigned op_count = 0; ++ // TODO (@cryptodeal): This is a lazy verification; maybe check operands ++ // for each operation in the block. ++ for (auto& op : body_block.getOperations()) { ++ switch (op_count) { ++ case 0: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ case 1: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ } ++ op_count++; ++ } ++ return true; ++} ++ ++absl::StatusOr getScatterType(mlir::Block& block) { ++ unsigned op_count = 0; ++ utils::ScatterType scatter_type = utils::ScatterType::Replace; ++ for (const auto& op : block.getOperations()) { ++ switch (op_count) { ++ case 0: ++ if (llvm::dyn_cast(op)) { ++ return scatter_type; ++ } ++ if (llvm::dyn_cast(op)) { ++ scatter_type = utils::ScatterType::Add; ++ } else if (llvm::dyn_cast(op)) { ++ scatter_type = utils::ScatterType::Prod; ++ } else if (llvm::dyn_cast(op)) { ++ scatter_type = utils::ScatterType::Prod; ++ } else if (llvm::dyn_cast(op)) { ++ scatter_type = utils::ScatterType::Max; ++ } else if (llvm::dyn_cast(op)) { ++ scatter_type = utils::ScatterType::Min; ++ } else { ++ return xla::Internal("Unsupported comparator: %s", ToString(block)); ++ } ++ break; ++ case 1: ++ if (llvm::dyn_cast(op)) { ++ return scatter_type; ++ } ++ default: ++ return xla::Internal("Unsupported comparator: %s", ToString(block)); ++ } ++ } ++} ++ ++absl::StatusOr> getSortInfo( ++ mlir::Block& block) { ++ unsigned op_count = 0; ++ mx::ComparatorType comparator = mx::ComparatorType::LessThan; ++ unsigned input_idx = 0; ++ for (const auto& op : block.getOperations()) { ++ switch (op_count) { ++ case 0: { ++ if (auto compare_op = llvm::dyn_cast(op)) { ++ switch (compare_op.getComparisonDirection()) { ++ case ComparisonDirection::GT: { ++ auto lhs = llvm::dyn_cast(compare_op.getLhs()); ++ auto rhs = llvm::dyn_cast(compare_op.getRhs()); ++ if (lhs && rhs) { ++ unsigned lhs_idx = static_cast(lhs.getArgNumber()); ++ unsigned rhs_idx = static_cast(rhs.getArgNumber()); ++ if (rhs_idx - 1 == lhs_idx) { ++ comparator = mx::ComparatorType::GreaterThan; ++ input_idx = lhs_idx == 0 ? 0 : lhs_idx / 2; ++ break; ++ } ++ } ++ return xla::Internal("Unsupported comparator: %s", ++ ToString(block)); ++ } ++ case ComparisonDirection::LT: { ++ auto lhs = llvm::dyn_cast(compare_op.getLhs()); ++ auto rhs = llvm::dyn_cast(compare_op.getRhs()); ++ if (lhs && rhs) { ++ unsigned lhs_idx = static_cast(lhs.getArgNumber()); ++ unsigned rhs_idx = static_cast(rhs.getArgNumber()); ++ if (rhs_idx - 1 == lhs_idx) { ++ comparator = mx::ComparatorType::LessThan; ++ input_idx = lhs_idx == 0 ? 0 : lhs_idx / 2; ++ break; ++ } ++ } ++ return xla::Internal("Unsupported comparator: %s", ++ ToString(block)); ++ } ++ default: ++ return xla::Internal("Unsupported comparator: %s", ++ ToString(block)); ++ } ++ break; ++ } ++ } ++ case 1: ++ if (llvm::dyn_cast(op)) break; ++ default: ++ return xla::Internal("Unsupported comparator: %s", ToString(block)); ++ } ++ op_count++; ++ } ++ return std::make_pair(input_idx, comparator); ++} ++ ++mx::array getOperandArray( ++ const Value& operand, ++ const std::unordered_map>& block_ctx, ++ const std::vector& inputs) { ++ // check if operand is the result of a previous operation ++ if (auto defining_op = operand.getDefiningOp()) { ++ if (auto search = block_ctx.find(defining_op); search != block_ctx.end()) { ++ for (auto i = 0; i < defining_op->getNumResults(); i++) { ++ Value maybe_res = defining_op->getResult(i); ++ if (maybe_res == operand) { ++ return search->second[i]; ++ } ++ } ++ } ++ } else { ++ return inputs[operand.cast().getArgNumber()]; ++ } ++} ++ ++std::pair getValueInfo(Value v) { ++ auto val_type = mlir::cast(v.getType()); ++ return std::make_pair(*utils::dtype::fromMlirType(val_type.getElementType()), ++ std::vector(val_type.getShape().begin(), ++ val_type.getShape().end())); ++} ++ ++std::pair getTypeInfo(Type v) { ++ auto val_type = mlir::cast(v); ++ return std::make_pair(*utils::dtype::fromMlirType(val_type.getElementType()), ++ std::vector(val_type.getShape().begin(), ++ val_type.getShape().end())); ++} ++ ++CallOpInfo getCallOpInfo(ModuleOp mod, func::FuncOp o) { ++ auto func_type = o.getFunctionType(); ++ ++ std::vector> input_types; ++ for (auto arg : func_type.getInputs()) { ++ input_types.emplace_back(getTypeInfo(arg)); ++ } ++ std::vector> result_types; ++ for (auto result : func_type.getResults()) { ++ result_types.emplace_back(getTypeInfo(result)); ++ } ++ return CallOpInfo(input_types, result_types, o.getName().str()); ++} ++ ++CallOpInfo getCallOpInfo(ModuleOp mod, func::CallOp o) { ++ std::vector> input_types; ++ for (auto arg : o.getOperands()) { ++ input_types.emplace_back(getValueInfo(arg)); ++ } ++ std::vector> result_types; ++ for (auto result : o.getResults()) { ++ result_types.emplace_back(getValueInfo(result)); ++ } ++ auto func = mod.lookupSymbol(o.getCallee()); ++ return CallOpInfo(input_types, result_types, func.getName().str()); ++} ++ ++IotaOpInfo getIotaOpInfo(stablehlo::IotaOp o) { ++ std::pair result_info = getValueInfo(o.getResult()); ++ auto iota_dim = o.getIotaDimension(); ++ std::vector dimensions; ++ for (auto i = 0; i < dimensions.size(); i++) { ++ dimensions.push_back(i != iota_dim ? 1 : result_info.second[i]); ++ } ++ return IotaOpInfo(std::vector>{}, ++ {result_info}, dimensions, result_info.first, iota_dim, ++ result_info.second); ++} ++ ++CbrtOpInfo getCbrtOpInfo(stablehlo::CbrtOp o) { ++ return CbrtOpInfo({getValueInfo(o.getOperand())}, ++ {getValueInfo(o.getResult())}); ++} ++ ++BroadcastInDimOpInfo getBroadcastInDimOpInfo(stablehlo::BroadcastInDimOp o) { ++ auto result_info = getValueInfo(o.getResult()); ++ return BroadcastInDimOpInfo( ++ {getValueInfo(o.getOperand())}, {result_info}, ++ std::vector(o.getBroadcastDimensions().begin(), ++ o.getBroadcastDimensions().end()), ++ result_info.second); ++} ++ ++DotGeneralOpInfo getDotGeneralOpInfo(stablehlo::DotGeneralOp o) { ++ auto dot_dim_nums = o.getDotDimensionNumbers(); ++ std::vector lhs_batch_dims( ++ dot_dim_nums.getLhsBatchingDimensions().begin(), ++ dot_dim_nums.getLhsBatchingDimensions().end()); ++ std::vector rhs_batch_dims( ++ dot_dim_nums.getRhsBatchingDimensions().begin(), ++ dot_dim_nums.getRhsBatchingDimensions().end()); ++ std::vector lhs_contract_dims( ++ dot_dim_nums.getLhsContractingDimensions().begin(), ++ dot_dim_nums.getLhsContractingDimensions().end()); ++ std::vector rhs_contract_dims( ++ dot_dim_nums.getRhsContractingDimensions().begin(), ++ dot_dim_nums.getRhsContractingDimensions().end()); ++ return DotGeneralOpInfo({getValueInfo(o.getLhs()), getValueInfo(o.getRhs())}, ++ {getValueInfo(o.getResult())}, lhs_batch_dims, ++ rhs_batch_dims, lhs_contract_dims, rhs_contract_dims); ++} ++ ++GatherOpInfo getGatherOpInfo(stablehlo::GatherOp o) { ++ std::vector> operand_info; ++ for (auto operand : o.getOperands()) { ++ operand_info.emplace_back(getValueInfo(operand)); ++ } ++ operand_info.emplace_back(getValueInfo(o.getStartIndices())); ++ auto result_info = getValueInfo(o.getResult()); ++ auto dim_nums = o.getDimensionNumbers(); ++ std::vector offset_dims( ++ o.getDimensionNumbers().getOffsetDims().begin(), ++ o.getDimensionNumbers().getOffsetDims().end()); ++ std::vector collapsed_slice_dims( ++ o.getDimensionNumbers().getCollapsedSliceDims().begin(), ++ o.getDimensionNumbers().getCollapsedSliceDims().end()); ++ std::vector operand_batching_dims( ++ o.getDimensionNumbers().getOperandBatchingDims().begin(), ++ o.getDimensionNumbers().getOperandBatchingDims().end()); ++ std::vector start_indices_batching_dims( ++ o.getDimensionNumbers().getStartIndicesBatchingDims().begin(), ++ o.getDimensionNumbers().getStartIndicesBatchingDims().end()); ++ std::vector start_index_map( ++ o.getDimensionNumbers().getStartIndexMap().begin(), ++ o.getDimensionNumbers().getStartIndexMap().end()); ++ auto index_vector_dim = o.getDimensionNumbers().getIndexVectorDim(); ++ std::vector slice_sizes(o.getSliceSizes().begin(), ++ o.getSliceSizes().end()); ++ return GatherOpInfo(operand_info, {getValueInfo(o.getResult())}, ++ collapsed_slice_dims, index_vector_dim, offset_dims, ++ operand_batching_dims, result_info.second, slice_sizes, ++ start_index_map, start_indices_batching_dims); ++} ++ ++ScatterOpInfo getScatterOpInfo(stablehlo::ScatterOp o) { ++ std::vector> inputs_info; ++ for (auto input : o.getInputs()) { ++ inputs_info.emplace_back(getValueInfo(input)); ++ } ++ for (auto update : o.getUpdates()) { ++ inputs_info.emplace_back(getValueInfo(update)); ++ } ++ inputs_info.emplace_back(getValueInfo(o.getScatterIndices())); ++ std::vector>> results_info; ++ for (auto result : o.getResults()) { ++ results_info.emplace_back(getValueInfo(result)); ++ } ++ // Get scatter dimension numbers ++ auto scatter_dim_nums = o.getScatterDimensionNumbers(); ++ std::vector update_window_dims( ++ scatter_dim_nums.getUpdateWindowDims().begin(), ++ scatter_dim_nums.getUpdateWindowDims().end()); ++ std::vector inserted_window_dims( ++ scatter_dim_nums.getInsertedWindowDims().begin(), ++ scatter_dim_nums.getInsertedWindowDims().end()); ++ std::vector input_batching_dims( ++ scatter_dim_nums.getInputBatchingDims().begin(), ++ scatter_dim_nums.getInputBatchingDims().end()); ++ ++ std::vector scatter_indices_batching_dims( ++ scatter_dim_nums.getScatterIndicesBatchingDims().begin(), ++ scatter_dim_nums.getScatterIndicesBatchingDims().end()); ++ ++ std::vector scatter_dims_to_operand_dims( ++ scatter_dim_nums.getScatterDimsToOperandDims().begin(), ++ scatter_dim_nums.getScatterDimsToOperandDims().end()); ++ ++ auto index_vector_dim = scatter_dim_nums.getIndexVectorDim(); ++ // Get update computation ++ mlir::Block& update_computation = o.getUpdateComputation().front(); ++ ++ // TODO(@cryptodeal): need to add a check in `Compile` ++ utils::ScatterType scatter_type = *getScatterType(update_computation); ++ return ScatterOpInfo( ++ inputs_info, results_info, index_vector_dim, input_batching_dims, ++ inserted_window_dims, scatter_dims_to_operand_dims, ++ scatter_indices_batching_dims, scatter_type, update_window_dims); ++} ++ ++PadOpInfo getPadOpInfo(stablehlo::PadOp o) { ++ auto result_info = getValueInfo(o.getResult()); ++ std::vector edge_pad_low(o.getEdgePaddingLow().begin(), ++ o.getEdgePaddingLow().end()); ++ std::vector interior_pad(o.getInteriorPadding().begin(), ++ o.getInteriorPadding().end()); ++ return PadOpInfo( ++ {getValueInfo(o.getOperand()), getValueInfo(o.getPaddingValue())}, ++ {result_info}, result_info.second, edge_pad_low, interior_pad); ++} ++ ++void compileCallOp(OpLookup& op_lookup, ModuleOp mod, func::FuncOp o) { ++ CallOpInfo call_op_info = getCallOpInfo(mod, o); ++ if (auto search = op_lookup.call.find(call_op_info); ++ search == op_lookup.call.end()) { ++ // std::cout << "Compiling CallOp: " << o.getName().str().c_str() << ++ // std::endl; ++ auto& block = o.front(); ++ auto call_op_func = [&op_lookup, &block, mod, ++ call_op_info](const std::vector& inputs) { ++ std::unordered_map> block_ctx; ++ std::vector res; ++ for (Operation& op : block.getOperations()) { ++ res = ++ llvm::TypeSwitch>(&op) ++ .Case([&block_ctx, ++ &inputs](auto o) { ++ std::vector res; ++ for (Value val : o.getOperands()) { ++ res.emplace_back(getOperandArray(val, block_ctx, inputs)); ++ } ++ return res; ++ }) ++ .Case([&block_ctx, &inputs, mod, ++ &op_lookup](auto o) { ++ auto call_op_info = getCallOpInfo(mod, o); ++ std::vector operands; ++ for (Value val : o.getOperands()) { ++ operands.emplace_back( ++ getOperandArray(val, block_ctx, inputs)); ++ } ++ return op_lookup.call.find(call_op_info)->second(operands); ++ }) ++ // Handle StableHLO nullary ops ++ .Case([](auto o) { ++ return std::vector{ ++ *utils::array::fromDenseElementsAttr( ++ mlir::cast(o.getValue()))}; ++ }) ++ .Case([&op_lookup](auto o) { ++ return op_lookup.iota.find(getIotaOpInfo(o))->second({}); ++ }) ++ // .Case([](auto o) {}) ++ /* ++ .Case([](auto o) {}) ++ Deprecated see: ++ https://github.com/openxla/stablehlo/issues/2340 ++ https://github.com/openxla/stablehlo/pull/2283 ++ */ ++ // Handle StableHLO unary elementwise op ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::abs( ++ getOperandArray(o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs, ++ &op_lookup](auto o) { ++ return op_lookup.cbrt.find(getCbrtOpInfo(o)) ++ ->second( ++ {getOperandArray(o.getOperand(), block_ctx, inputs)}); ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::ceil( ++ getOperandArray(o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::astype( ++ getOperandArray(o.getOperand(), block_ctx, inputs), ++ *utils::dtype::fromMlirType( ++ o.getResult().getType().getElementType()))}; ++ }) ++ // .Case([](auto o) {}) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::cos( ++ getOperandArray(o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::exp( ++ getOperandArray(o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::expm1( ++ getOperandArray(o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::floor( ++ getOperandArray(o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::imag( ++ getOperandArray(o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::isfinite( ++ getOperandArray(o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::log( ++ getOperandArray(o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::log1p( ++ getOperandArray(o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::sigmoid( ++ getOperandArray(o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::logical_not( ++ getOperandArray(o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::negative( ++ getOperandArray(o.getOperand(), block_ctx, inputs))}; ++ }) ++ // .Case([](auto o) {}) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::real( ++ getOperandArray(o.getOperand(), block_ctx, inputs))}; ++ }) ++ // TODO(@cryptodeal): `stablehlo::RoundOp` does not match with ++ // the mlx metal implementation ++ // .Case([](auto o) {}) ++ .Case( ++ [&block_ctx, &inputs](auto o) { ++ return std::vector{mx::round( ++ getOperandArray(o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::rsqrt( ++ getOperandArray(o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::sign( ++ getOperandArray(o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::sin( ++ getOperandArray(o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::sqrt( ++ getOperandArray(o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::tan( ++ getOperandArray(o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::tanh( ++ getOperandArray(o.getOperand(), block_ctx, inputs))}; ++ }) ++ // Handle StableHLO binary elementwise ops ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{ ++ mx::add(getOperandArray(o.getLhs(), block_ctx, inputs), ++ getOperandArray(o.getRhs(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::arctan2( ++ getOperandArray(o.getLhs(), block_ctx, inputs), ++ getOperandArray(o.getRhs(), block_ctx, inputs))}; ++ }) ++ // TODO(@cryptodeal): implement complex op ++ // .Case([&block_ctx](auto ++ // op) {}) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::divide( ++ getOperandArray(o.getLhs(), block_ctx, inputs), ++ getOperandArray(o.getRhs(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::maximum( ++ getOperandArray(o.getLhs(), block_ctx, inputs), ++ getOperandArray(o.getRhs(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::minimum( ++ getOperandArray(o.getLhs(), block_ctx, inputs), ++ getOperandArray(o.getRhs(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::multiply( ++ getOperandArray(o.getLhs(), block_ctx, inputs), ++ getOperandArray(o.getRhs(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::power( ++ getOperandArray(o.getLhs(), block_ctx, inputs), ++ getOperandArray(o.getRhs(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::remainder( ++ getOperandArray(o.getLhs(), block_ctx, inputs), ++ getOperandArray(o.getRhs(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::left_shift( ++ getOperandArray(o.getLhs(), block_ctx, inputs), ++ getOperandArray(o.getRhs(), block_ctx, inputs))}; ++ }) ++ .Case( ++ [&block_ctx, &inputs](auto o) { ++ /** ++ * Per Metal Spec: ++ * For the right-shift operator, if E1 has an unsigned ++ * type or if E1 has a signed type and a nonnegative ++ * value, the vacated bits are filled with zeros. If E1 ++ * has a signed type and a negative value, the vacated ++ * bits are filled with ones. ++ */ ++ auto lhs = getOperandArray(o.getLhs(), block_ctx, inputs); ++ auto rhs = getOperandArray(o.getRhs(), block_ctx, inputs); ++ auto target_dtype = rhs.dtype(); ++ switch (lhs.dtype().size()) { ++ case 1: ++ target_dtype = mx::int8; ++ break; ++ case 2: ++ target_dtype = mx::int16; ++ break; ++ case 4: ++ target_dtype = mx::int32; ++ break; ++ case 8: ++ target_dtype = mx::int64; ++ break; ++ } ++ return std::vector{mx::view( ++ mx::right_shift(mx::view(lhs, target_dtype), ++ mx::astype(rhs, target_dtype)), ++ lhs.dtype())}; ++ }) ++ .Case( ++ [&block_ctx, &inputs](auto o) { ++ // Ensures that we bitcast to `uint` type before ++ // performing the right shift. Should ensure that vacated ++ // bits are zero populated. ++ auto lhs = getOperandArray(o.getLhs(), block_ctx, inputs); ++ auto rhs = getOperandArray(o.getRhs(), block_ctx, inputs); ++ auto target_dtype = rhs.dtype(); ++ switch (lhs.dtype().size()) { ++ case 1: ++ target_dtype = mx::uint8; ++ break; ++ case 2: ++ target_dtype = mx::uint16; ++ break; ++ case 4: ++ target_dtype = mx::uint32; ++ break; ++ case 8: ++ target_dtype = mx::uint64; ++ break; ++ } ++ return std::vector{mx::view( ++ mx::right_shift(mx::view(lhs, target_dtype), ++ mx::astype(rhs, target_dtype)), ++ lhs.dtype())}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::subtract( ++ getOperandArray(o.getLhs(), block_ctx, inputs), ++ getOperandArray(o.getRhs(), block_ctx, inputs))}; ++ }) ++ // Handle StableHLO binary logical elementwise ops ++ .Case([&block_ctx, &inputs](auto o) { ++ auto lhs = getOperandArray(o.getLhs(), block_ctx, inputs); ++ auto rhs = getOperandArray(o.getRhs(), block_ctx, inputs); ++ switch (mx::kindof(lhs.dtype())) { ++ case mx::Dtype::Kind::b: ++ return std::vector{mx::logical_and(lhs, rhs)}; ++ default: ++ return std::vector{mx::bitwise_and(lhs, rhs)}; ++ } ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ auto lhs = getOperandArray(o.getLhs(), block_ctx, inputs); ++ auto rhs = getOperandArray(o.getRhs(), block_ctx, inputs); ++ switch (mx::kindof(lhs.dtype())) { ++ case mx::Dtype::Kind::b: ++ return std::vector{mx::logical_or(lhs, rhs)}; ++ default: ++ return std::vector{mx::bitwise_or(lhs, rhs)}; ++ } ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::bitwise_xor( ++ getOperandArray(o.getLhs(), block_ctx, inputs), ++ getOperandArray(o.getRhs(), block_ctx, inputs))}; ++ }) ++ // Handle StableHLO communication ops ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ ++ // Handle StableHLO parallelism related ops ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ ++ // Handle StableHLO control flow ops ++ // .Case([](auto o) {}) ++ // TODO(@cryptodeal): need custom kernels in order to ++ // remove the need to call `mx::eval` for the following ops ++ // .Case([](auto o)) ++ // .Case([](auto o)) ++ // .Case([](auto o)) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ .Case([&block_ctx, &inputs](auto o) { ++ auto operand = ++ getOperandArray(o.getOperand(0), block_ctx, inputs); ++ if (isArgMaxReduce(o)) { ++ auto axis = static_cast(o.getDimensions()[0]); ++ auto indices = mx::argmax(operand, axis); ++ ++ mx::Dtype result_type = *utils::dtype::fromMlirType( ++ mlir::cast(o.getResults()[1].getType()) ++ .getElementType()); ++ return std::vector{ ++ mx::take(operand, indices, axis), ++ mx::astype(indices, result_type)}; ++ } ++ if (isSumReduce(o)) { ++ return std::vector{mx::sum( ++ operand, static_cast(o.getDimensions()[0]))}; ++ } ++ if (isMaxReduce(o)) { ++ return std::vector{mx::max( ++ operand, static_cast(o.getDimensions()[0]))}; ++ } ++ if (isMinReduce(o)) { ++ return std::vector{mx::min( ++ operand, static_cast(o.getDimensions()[0]))}; ++ } ++ // TODO(@cryptodeal): further cleanup module validation ++ // done via call to `Compile` to check for valid reduce types. ++ ++ // For now, we assume `mx::any` if none of the above ++ ++ // if (isAnyReduce(o)) { ++ return std::vector{ ++ mx::any(operand, static_cast(o.getDimensions()[0]))}; ++ //} ++ }) ++ // Handle StableHLO tuple ops ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ .Case([&block_ctx, &inputs](auto o) { ++ auto lhs = getOperandArray(o.getLhs(), block_ctx, inputs); ++ auto rhs = getOperandArray(o.getRhs(), block_ctx, inputs); ++ switch (o.getComparisonDirection()) { ++ case ComparisonDirection::NE: ++ return std::vector{mx::not_equal(lhs, rhs)}; ++ case ComparisonDirection::GE: ++ return std::vector{ ++ mx::greater_equal(lhs, rhs)}; ++ case ComparisonDirection::GT: ++ return std::vector{mx::greater(lhs, rhs)}; ++ case ComparisonDirection::LE: ++ return std::vector{mx::less_equal(lhs, rhs)}; ++ case ComparisonDirection::LT: ++ return std::vector{mx::less(lhs, rhs)}; ++ case ComparisonDirection::EQ: ++ return std::vector{mx::equal(lhs, rhs)}; ++ } ++ }) ++ // Handle StableHLO Slice ops ++ .Case([&block_ctx, &inputs](auto o) { ++ std::vector start_indices( ++ o.getStartIndices().begin(), o.getStartIndices().end()); ++ std::vector limit_indices( ++ o.getLimitIndices().begin(), o.getLimitIndices().end()); ++ std::vector strides(o.getStrides().begin(), ++ o.getStrides().end()); ++ return std::vector{mx::slice( ++ getOperandArray(o.getOperand(), block_ctx, inputs), ++ start_indices, limit_indices, strides)}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ auto operand = ++ getOperandArray(o.getOperand(), block_ctx, inputs); ++ std::vector indices; ++ for (auto val : o.getStartIndices()) { ++ indices.emplace_back( ++ getOperandArray(val, block_ctx, inputs)); ++ } ++ auto start_indices = mx::concatenate(indices); ++ std::vector axes(operand.ndim()); ++ std::iota(axes.begin(), axes.end(), 0); ++ std::vector slice_sizes(o.getSliceSizes().begin(), ++ o.getSliceSizes().end()); ++ return std::vector{ ++ mx::slice(operand, start_indices, axes, slice_sizes)}; ++ }) ++ .Case( ++ [&block_ctx, &inputs](auto o) { ++ auto operand = ++ getOperandArray(o.getOperand(), block_ctx, inputs); ++ auto update = ++ getOperandArray(o.getUpdate(), block_ctx, inputs); ++ std::vector indices; ++ for (auto val : o.getStartIndices()) { ++ indices.emplace_back( ++ getOperandArray(val, block_ctx, inputs)); ++ } ++ ++ auto start_indices = mx::concatenate(indices); ++ std::vector axes(operand.ndim()); ++ std::iota(axes.begin(), axes.end(), 0); ++ return std::vector{mx::slice_update( ++ operand, update, start_indices, axes)}; ++ }) ++ // Handle StableHLO Other ops ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ .Case( ++ [&block_ctx, &inputs](auto o) { ++ auto result_type = *utils::dtype::fromMlirType( ++ mlir::cast(o.getResult().getType()) ++ .getElementType()); ++ return std::vector{mx::view( ++ getOperandArray(o.getOperand(), block_ctx, inputs), ++ result_type)}; ++ }) ++ /* ++ .Case([](auto o) {}) ++ Deprecated see: ++ https://github.com/openxla/stablehlo/issues/2340 ++ https://github.com/openxla/stablehlo/pull/2283 ++ */ ++ .Case([&block_ctx, &inputs, ++ &op_lookup](auto o) { ++ return op_lookup.broadcast_in_dim ++ .find(getBroadcastInDimOpInfo(o)) ++ ->second( ++ {getOperandArray(o.getOperand(), block_ctx, inputs)}); ++ }) ++ // .Case([](auto o) {}) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::linalg::cholesky( ++ getOperandArray(o.getOperand(), block_ctx, inputs), ++ o.getLower())}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::clip( ++ getOperandArray(o.getOperand(), block_ctx, inputs), ++ getOperandArray(o.getMin(), block_ctx, inputs), ++ getOperandArray(o.getMax(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ std::vector inputs; ++ for (auto val : o.getOperands()) { ++ inputs.emplace_back( ++ getOperandArray(val, block_ctx, inputs)); ++ } ++ return std::vector{ ++ mx::concatenate(inputs, o.getDimension())}; ++ }) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ /* ++ .Case([](auto o) {}) ++ Deprecated see: ++ https://github.com/openxla/stablehlo/issues/2340 ++ https://github.com/openxla/stablehlo/pull/2283 ++ */ ++ .Case( ++ [&block_ctx, &inputs, &op_lookup](auto o) { ++ return op_lookup.dot_general.find(getDotGeneralOpInfo(o)) ++ ->second( ++ {getOperandArray(o.getLhs(), block_ctx, inputs), ++ getOperandArray(o.getRhs(), block_ctx, inputs)}); ++ }) ++ /* ++ .Case([](auto op) {}) ++ .Case([](auto op) {}) ++ Deprecated see: ++ https://github.com/openxla/stablehlo/issues/2340 ++ https://github.com/openxla/stablehlo/pull/2283 ++ */ ++ // .Case([](auto op) {}) ++ // TODO(@cryptodeal): definitely room for optimization here ++ .Case([&block_ctx, &inputs, ++ &op_lookup](auto o) { ++ return op_lookup.gather.find(getGatherOpInfo(o)) ++ ->second( ++ {getOperandArray(o.getOperand(), block_ctx, inputs), ++ getOperandArray(o.getStartIndices(), block_ctx, ++ inputs)}); ++ }) ++ .Case( ++ [&block_ctx, &inputs](auto o) { ++ return std::vector{mx::array( ++ (getOperandArray(o.getOperand(), block_ctx, inputs)) ++ .shape(o.getDimension()))}; ++ }) ++ // .Case([](auto o) {}) ++ .Case([&block_ctx, &inputs](auto o) { ++ auto result_type = ++ mlir::cast(o.getResult().getType()); ++ return std::vector{mx::reshape( ++ getOperandArray(o.getOperand(), block_ctx, inputs), ++ std::vector(result_type.getShape().begin(), ++ result_type.getShape().end()))}; ++ }) ++ // .Case([](auto o) {}) ++ .Case( ++ [&block_ctx, &inputs, &op_lookup](auto o) { ++ std::vector operands; ++ for (auto val : o.getInputs()) { ++ operands.emplace_back( ++ getOperandArray(val, block_ctx, inputs)); ++ } ++ // Add updates to input list ++ for (auto val : o.getUpdates()) { ++ operands.emplace_back( ++ getOperandArray(val, block_ctx, inputs)); ++ } ++ // Add scatter indices to input list ++ operands.emplace_back(getOperandArray( ++ o.getScatterIndices(), block_ctx, inputs)); ++ return op_lookup.scatter.find(getScatterOpInfo(o)) ++ ->second(operands); ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::where( ++ getOperandArray(o.getOperand(0), block_ctx, inputs), ++ getOperandArray(o.getOperand(1), block_ctx, inputs), ++ getOperandArray(o.getOperand(2), block_ctx, inputs))}; ++ }) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ .Case([&block_ctx, &inputs](auto o) { ++ std::vector tmp_inputs; ++ for (auto val : o.getInputs()) { ++ tmp_inputs.emplace_back( ++ getOperandArray(val, block_ctx, inputs)); ++ } ++ int dim = static_cast(o.getDimension()); ++ // Get update computation ++ mlir::Block& comparator = o.getComparator().front(); ++ auto [index, comparator_type] = *getSortInfo(comparator); ++ auto indices = ++ mx::argsort(tmp_inputs[index], dim, comparator_type); ++ std::vector res; ++ mx::Shape slice_sizes(tmp_inputs[0].ndim(), 1); ++ for (auto in : tmp_inputs) { ++ res.push_back(mx::gather(in, indices, dim, slice_sizes)); ++ } ++ return res; ++ }) ++ // .Case([](auto o) {}) ++ // TODO (@cryptodeal): lots of room for optimization here ++ .Case([&block_ctx, &inputs, ++ &op_lookup](auto o) { ++ return op_lookup.pad.find(getPadOpInfo(o)) ++ ->second( ++ {getOperandArray(o.getOperand(), block_ctx, inputs), ++ getOperandArray(o.getPaddingValue(), block_ctx, ++ inputs)}); ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ std::vector axes(o.getPermutation().begin(), ++ o.getPermutation().end()); ++ return std::vector{mx::transpose( ++ getOperandArray(o.getOperand(), block_ctx, inputs), ++ axes)}; ++ }) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // Need to modify such that `rng.state` is held outside of scope ++ // of `mx::compile` ++ // TODO(@cryptodeal): re-enable once we've handled ensuring ++ // state is handled correctly ++ // .Case([&block_ctx](auto o) { ++ // auto rng_func = [&o](const std::vector& inputs) ++ // { ++ // auto a = inputs[0]; ++ // auto b = inputs[1]; ++ // auto result_type = ++ // mlir::cast(o.getResult().getType()); ++ // std::vector ++ // shape(result_type.getShape().begin(), ++ // result_type.getShape().end()); ++ ++ // switch (o.getRngDistribution()) { ++ // case RngDistribution::UNIFORM: ++ // return std::vector{ ++ // mx::random::uniform(a, b, shape, a.dtype())}; ++ // default: ++ // return std::vector{ ++ // mx::random::normal(shape, a.dtype(), a, b)}; ++ // } ++ // }; ++ // return mx::compile(rng_func)( ++ // {getOperandArray(o.getA(), block_ctx), ++ // getOperandArray(o.getB(), block_ctx)}); ++ // }) ++ // .Case([&block_ctx](auto o) { ++ // auto rng_bit_gen_func = [&o](const std::vector& ++ // inputs) { ++ ++ // }; ++ // return mx::compile(rng_bit_gen_func)( ++ // {getOperandArray(o.getA(), block_ctx), ++ // getOperandArray(o.getB(), block_ctx)}); ++ // }) ++ // Handle StableHLO Quantize ops ++ // .Case([](auto op) {}) ++ // .Case([](auto op) {}) ++ // .Case([](auto op) {}) ++ /* ++ .Case([](auto o) {}) ++ Deprecated see: ++ https://github.com/openxla/stablehlo/issues/2340 ++ https://github.com/openxla/stablehlo/pull/2283 ++ */ ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ ++ .Default([&op](auto o) { ++ // Shouldn't be possible to reach this point, return ++ // empty vector. ++ return std::vector{}; ++ }); ++ block_ctx.emplace(&op, res); ++ } ++ return res; ++ }; ++ op_lookup.call.emplace(call_op_info, mx::compile(call_op_func)); ++ } ++} ++ ++void compileIotaOp(OpLookup& op_lookup, stablehlo::IotaOp o) { ++ IotaOpInfo iota_op_info = getIotaOpInfo(o); ++ auto dimensions = std::get<0>(iota_op_info.captures); ++ auto dtype = std::get<1>(iota_op_info.captures); ++ auto iota_dim = std::get<2>(iota_op_info.captures); ++ auto shape = std::get<3>(iota_op_info.captures); ++ if (auto search = op_lookup.iota.find(iota_op_info); ++ search == op_lookup.iota.end()) { ++ // std::cout << "Compiling IotaOp" << std::endl; ++ auto iota_func = [dimensions, dtype, iota_dim, ++ shape](const std::vector& inputs) { ++ auto init_arange = ++ mx::arange(static_cast(shape[iota_dim]), dtype); ++ if (dimensions.size()) { ++ init_arange = mx::reshape(init_arange, dimensions); ++ } ++ return std::vector{mx::broadcast_to(init_arange, shape)}; ++ }; ++ op_lookup.iota.emplace(iota_op_info, mx::compile(iota_func)); ++ } ++} ++ ++void compileCbrtOp(OpLookup& op_lookup, stablehlo::CbrtOp o) { ++ CbrtOpInfo cbrt_op_info = getCbrtOpInfo(o); ++ if (auto search = op_lookup.cbrt.find(cbrt_op_info); ++ search == op_lookup.cbrt.end()) { ++ // std::cout << "Compiling CbrtOp" << std::endl; ++ ++ auto cbrt_func = [](const std::vector& inputs) { ++ return std::vector{mx::power( ++ inputs[0], ++ mx::full(inputs[0].shape(), 1 / 3, inputs[0].dtype()))}; ++ }; ++ op_lookup.cbrt.emplace(cbrt_op_info, mx::compile(cbrt_func)); ++ } ++} ++ ++void compileBroadcastInDimOp(OpLookup& op_lookup, ++ stablehlo::BroadcastInDimOp o) { ++ BroadcastInDimOpInfo broadcast_in_dim_op_info = getBroadcastInDimOpInfo(o); ++ auto broadcast_dims = std::get<0>(broadcast_in_dim_op_info.captures); ++ auto target_shape = std::get<1>(broadcast_in_dim_op_info.captures); ++ if (auto search = op_lookup.broadcast_in_dim.find(broadcast_in_dim_op_info); ++ search == op_lookup.broadcast_in_dim.end()) { ++ // std::cout << "Compiling BroadcastInDimOp" << std::endl; ++ ++ auto broadcast_in_dim_func = [broadcast_dims, target_shape]( ++ const std::vector& inputs) { ++ auto operand = inputs[0]; ++ ++ // potentially reshape to pad with ones, which allows ++ // broadcasting ++ auto min_rank = std::min(operand.ndim(), target_shape.size()); ++ if (min_rank == operand.ndim()) { ++ std::vector padded_shape = operand.shape(); ++ for (auto i = 0; i < target_shape.size(); i++) { ++ if (auto search = ++ std::find(broadcast_dims.begin(), broadcast_dims.end(), i); ++ search == broadcast_dims.end()) { ++ padded_shape.insert(padded_shape.begin() + i, 1); ++ } ++ } ++ operand = mx::reshape(operand, padded_shape); ++ } ++ return std::vector{mx::broadcast_to(operand, target_shape)}; ++ }; ++ op_lookup.broadcast_in_dim.emplace(broadcast_in_dim_op_info, ++ mx::compile(broadcast_in_dim_func)); ++ } ++} ++ ++void compileDotGeneralOp(OpLookup& op_lookup, stablehlo::DotGeneralOp o) { ++ DotGeneralOpInfo dot_general_op_info = getDotGeneralOpInfo(o); ++ auto lhs_batch_dims = std::get<0>(dot_general_op_info.captures); ++ auto rhs_batch_dims = std::get<1>(dot_general_op_info.captures); ++ auto lhs_contract_dims = std::get<2>(dot_general_op_info.captures); ++ auto rhs_contract_dims = std::get<3>(dot_general_op_info.captures); ++ if (auto search = op_lookup.dot_general.find(dot_general_op_info); ++ search == op_lookup.dot_general.end()) { ++ // std::cout << "Compiling DotGeneralOp" << std::endl; ++ ++ auto dot_general_func = [lhs_batch_dims, rhs_batch_dims, lhs_contract_dims, ++ rhs_contract_dims]( ++ const std::vector& inputs) { ++ auto lhs = inputs[0]; ++ auto rhs = inputs[1]; ++ ++ int dim_count = 0; ++ auto getDimChar = [&dim_count]() -> char { return 'a' + dim_count++; }; ++ ++ std::unordered_map lhs_batch_map; ++ std::unordered_map rhs_batch_map; ++ std::unordered_map lhs_contract_map; ++ std::unordered_map rhs_contract_map; ++ ++ std::string res_subscript; ++ for (auto i = 0; i < lhs_batch_dims.size(); ++i) { ++ auto dim_char = getDimChar(); ++ res_subscript = res_subscript + dim_char; ++ lhs_batch_map.emplace(lhs_batch_dims[i], dim_char); ++ rhs_batch_map.emplace(rhs_batch_dims[i], dim_char); ++ } ++ ++ for (auto i = 0; i < lhs_contract_dims.size(); ++i) { ++ auto dim_char = getDimChar(); ++ lhs_contract_map.emplace(lhs_contract_dims[i], dim_char); ++ rhs_contract_map.emplace(rhs_contract_dims[i], dim_char); ++ } ++ ++ std::string lhs_subscript; ++ for (auto i = 0; i < lhs.ndim(); ++i) { ++ if (auto match = lhs_batch_map.find(i); match != lhs_batch_map.end()) { ++ lhs_subscript = lhs_subscript + match->second; ++ } else if (auto match = lhs_contract_map.find(i); ++ match != lhs_contract_map.end()) { ++ lhs_subscript = lhs_subscript + match->second; ++ } else { ++ auto dim_char = getDimChar(); ++ res_subscript = res_subscript + dim_char; ++ lhs_subscript = lhs_subscript + dim_char; ++ } ++ } ++ ++ std::string rhs_subscript; ++ for (auto i = 0; i < rhs.ndim(); ++i) { ++ if (auto match = rhs_batch_map.find(i); match != rhs_batch_map.end()) { ++ rhs_subscript = rhs_subscript + match->second; ++ } else if (auto match = rhs_contract_map.find(i); ++ match != rhs_contract_map.end()) { ++ rhs_subscript = rhs_subscript + match->second; ++ } else { ++ auto dim_char = getDimChar(); ++ res_subscript = res_subscript + dim_char; ++ rhs_subscript = rhs_subscript + dim_char; ++ } ++ } ++ ++ return std::vector{mx::einsum( ++ lhs_subscript + "," + rhs_subscript + "->" + res_subscript, inputs)}; ++ }; ++ op_lookup.dot_general.emplace(dot_general_op_info, ++ mx::compile(dot_general_func)); ++ } ++} ++ ++void compileGatherOp(OpLookup& op_lookup, stablehlo::GatherOp o) { ++ GatherOpInfo gather_op_info = getGatherOpInfo(o); ++ auto collapsed_slice_dims = std::get<0>(gather_op_info.captures); ++ auto index_vector_dim = std::get<1>(gather_op_info.captures); ++ auto offset_dims = std::get<2>(gather_op_info.captures); ++ auto operand_batching_dims = std::get<3>(gather_op_info.captures); ++ auto result_shape = std::get<4>(gather_op_info.captures); ++ auto slice_sizes = std::get<5>(gather_op_info.captures); ++ auto start_index_map = std::get<6>(gather_op_info.captures); ++ auto start_indices_batching_dims = std::get<7>(gather_op_info.captures); ++ ++ if (auto search = op_lookup.gather.find(gather_op_info); ++ search == op_lookup.gather.end()) { ++ // std::cout << "Compiling GatherOp" << std::endl; ++ auto gather_func = [collapsed_slice_dims, index_vector_dim, offset_dims, ++ operand_batching_dims, result_shape, slice_sizes, ++ start_index_map, start_indices_batching_dims]( ++ const std::vector& inputs) { ++ auto operand = inputs[0]; ++ auto start_indices = inputs[1]; ++ ++ // Calculate batch dims ++ std::vector batch_dims_vec; ++ for (int64_t i = 0; i < result_shape.size(); i++) { ++ if (std::find(offset_dims.begin(), offset_dims.end(), i) == ++ offset_dims.end()) { ++ batch_dims_vec.emplace_back(static_cast(i)); ++ } ++ } ++ mx::array batch_dims = mx::array( ++ batch_dims_vec.data(), {static_cast(batch_dims_vec.size())}); ++ mx::array start_idx_map = ++ mx::array(start_index_map.data(), ++ {static_cast(start_index_map.size())}); ++ mx::array op_batching_dims = ++ mx::array(operand_batching_dims.data(), ++ {static_cast(operand_batching_dims.size())}); ++ ++ // Calculate result indices ++ std::vector meshgrid_arrays; ++ for (const auto& dim : result_shape) { ++ meshgrid_arrays.push_back(mx::arange(dim)); ++ } ++ mx::array result_indices = ++ mx::reshape(mx::stack(mx::meshgrid(meshgrid_arrays, false, "ij"), -1), ++ {-1, static_cast(result_shape.size())}); ++ ++ // Initialize operand indices with zeros ++ mx::array operand_indices = mx::zeros( ++ {result_indices.shape(0), static_cast(operand.ndim())}, ++ mx::int32); ++ ++ // Slice batch indices ++ mx::array batch_indices = mx::take(result_indices, batch_dims, 1); ++ if (batch_indices.size()) { ++ // Calculate start indices ++ std::vector tmp_indices = ++ mx::split(batch_indices, batch_indices.shape(1), 1); ++ for (auto& idx : tmp_indices) { ++ idx = mx::flatten(idx); ++ } ++ ++ std::vector tmp_axes; ++ for (auto i = 0; i < start_indices.ndim(); ++i) { ++ if (i == index_vector_dim) continue; ++ tmp_axes.push_back(i); ++ } ++ std::vector tmp_slice_sizes(start_indices.ndim(), 1); ++ if (index_vector_dim < tmp_slice_sizes.size()) { ++ tmp_slice_sizes[index_vector_dim] = ++ start_indices.shape(index_vector_dim); ++ } ++ ++ mx::array batch_start_indices = mx::reshape( ++ mx::gather(start_indices, tmp_indices, tmp_axes, tmp_slice_sizes), ++ {result_indices.shape(0), -1}); ++ ++ // Seed with full start indices ++ std::vector max; ++ for (auto d_start = 0; d_start < start_index_map.size(); ++d_start) { ++ int d_operand = start_index_map[d_start]; ++ max.push_back(operand.shape(d_operand) - slice_sizes[d_operand]); ++ } ++ ++ operand_indices = mx::put_along_axis( ++ operand_indices, ++ mx::broadcast_to(start_idx_map, ++ {operand_indices.shape(0), ++ static_cast(start_index_map.size())}), ++ mx::clip( ++ mx::take(batch_start_indices, ++ mx::arange(static_cast(start_index_map.size())), ++ 1), ++ mx::zeros(batch_start_indices.shape(), mx::int32), ++ mx::array(max.data(), {static_cast(max.size())})), ++ 1); ++ } ++ ++ // Calculate full batching indices ++ mx::array i_batching = ++ mx::arange(static_cast(operand_batching_dims.size())); ++ mx::array d_operands = mx::take(op_batching_dims, i_batching); ++ mx::array d_starts = mx::take( ++ mx::array(start_indices_batching_dims.data(), ++ {static_cast(start_indices_batching_dims.size())}), ++ i_batching); ++ auto tmp_batch_start_indices = mx::subtract( ++ d_starts, mx::where(mx::less(d_starts, mx::full(d_starts.shape(), ++ index_vector_dim)), ++ mx::zeros(d_starts.shape(), mx::int32), ++ mx::ones(d_starts.shape(), mx::int32))); ++ ++ // Add batching indices ++ operand_indices = mx::scatter_add_axis( ++ operand_indices, ++ mx::broadcast_to(d_operands, ++ {operand_indices.shape(0), d_operands.shape(0)}), ++ mx::take(batch_indices, tmp_batch_start_indices, 1), 1); ++ ++ // Calculate offset indices ++ mx::array offset_indices = ++ mx::take(result_indices, ++ mx::array(offset_dims.data(), ++ {static_cast(offset_dims.size())}), ++ 1); ++ ++ std::vector result_offset_idx; ++ for (unsigned i = 0; i < operand_indices.shape(1); i++) { ++ if (std::find(operand_batching_dims.begin(), ++ operand_batching_dims.end(), ++ static_cast(i)) != operand_batching_dims.end() || ++ std::find(collapsed_slice_dims.begin(), collapsed_slice_dims.end(), ++ static_cast(i)) != collapsed_slice_dims.end()) { ++ continue; ++ } ++ result_offset_idx.push_back(i); ++ } ++ ++ // Add offset indices ++ operand_indices = mx::scatter_add_axis( ++ operand_indices, ++ mx::broadcast_to( ++ mx::array(result_offset_idx.data(), ++ {static_cast(result_offset_idx.size())}), ++ {operand_indices.shape(0), ++ static_cast(result_offset_idx.size())}), ++ offset_indices, 1); ++ ++ std::vector gather_axes(operand.ndim()); ++ std::iota(gather_axes.begin(), gather_axes.end(), 0); ++ return std::vector{mx::reshape( ++ mx::gather(operand, ++ mx::split(operand_indices, operand_indices.shape(1), 1), ++ gather_axes, std::vector(operand.ndim(), 1)), ++ result_shape)}; ++ }; ++ ++ op_lookup.gather.emplace(gather_op_info, mx::compile(gather_func)); ++ } ++} ++ ++void compileScatterOp(OpLookup& op_lookup, stablehlo::ScatterOp o) { ++ ScatterOpInfo scatter_op_info = getScatterOpInfo(o); ++ auto index_vector_dim = ++ static_cast(std::get<0>(scatter_op_info.captures)); ++ auto input_batching_dims = std::get<1>(scatter_op_info.captures); ++ auto inserted_window_dims = std::get<2>(scatter_op_info.captures); ++ auto scatter_dims_to_operand_dims = std::get<3>(scatter_op_info.captures); ++ auto scatter_indices_batching_dims = std::get<4>(scatter_op_info.captures); ++ auto scatter_type = std::get<5>(scatter_op_info.captures); ++ auto update_window_dims = std::get<6>(scatter_op_info.captures); ++ if (auto search = op_lookup.scatter.find(scatter_op_info); ++ search == op_lookup.scatter.end()) { ++ // std::cout << "Compiling ScatterOp" << std::endl; ++ auto scatter_func = [index_vector_dim, input_batching_dims, ++ inserted_window_dims, scatter_dims_to_operand_dims, ++ scatter_indices_batching_dims, scatter_type, ++ update_window_dims](const std::vector& in) { ++ auto input_count = (in.size() - 1) / 2; ++ std::vector inputs(in.begin(), in.begin() + input_count); ++ std::vector updates(in.begin() + input_count, in.end() - 1); ++ auto scatter_indices = in[in.size() - 1]; ++ // Calculate update scatter dims ++ std::vector update_scatter_dims; ++ for (auto i = 0; i < updates[0].ndim(); i++) { ++ if (std::find(update_window_dims.begin(), update_window_dims.end(), ++ static_cast(i)) == update_window_dims.end()) { ++ update_scatter_dims.emplace_back(static_cast(i)); ++ } ++ } ++ ++ // Calculate update indices ++ std::vector meshgrid_arrays; ++ for (const auto& dim : updates[0].shape()) { ++ meshgrid_arrays.push_back(mx::arange(dim)); ++ } ++ mx::array update_indices = ++ mx::reshape(mx::stack(mx::meshgrid(meshgrid_arrays, false, "ij"), -1), ++ {-1, static_cast(updates[0].ndim())}); ++ ++ // Calculate update scatter index ++ mx::array update_scatter_indices = mx::take( ++ update_indices, ++ mx::array(update_scatter_dims.data(), ++ {static_cast(update_scatter_dims.size())}), ++ 1); ++ ++ // Initialize result indices with zeros ++ auto result_indices = mx::zeros( ++ {update_indices.shape(0), static_cast(inputs[0].ndim())}, ++ mx::int32); ++ ++ // Gather start indices; populating the corresponding ++ // result indices. ++ auto gather_dim_count = index_vector_dim < scatter_indices.ndim() ++ ? update_scatter_dims.size() + 1 ++ : update_scatter_dims.size(); ++ std::vector tmp_gather_axes(update_scatter_dims.size()); ++ std::iota(tmp_gather_axes.begin(), tmp_gather_axes.end(), 0); ++ std::vector gather_slice_sizes(gather_dim_count, 1); ++ if (index_vector_dim < scatter_indices.ndim()) { ++ gather_slice_sizes[index_vector_dim] = ++ scatter_indices.shape(index_vector_dim); ++ } ++ mx::array start_indices = ++ mx::reshape(mx::gather(scatter_indices, ++ mx::split(update_scatter_indices, ++ update_scatter_indices.shape(1), 1), ++ tmp_gather_axes, gather_slice_sizes), ++ {update_indices.shape(0), -1}); ++ ++ // Seed with full start indices ++ result_indices = mx::put_along_axis( ++ result_indices, ++ mx::broadcast_to( ++ mx::array( ++ scatter_dims_to_operand_dims.data(), ++ {static_cast(scatter_dims_to_operand_dims.size())}), ++ {result_indices.shape(0), ++ static_cast(scatter_dims_to_operand_dims.size())}), ++ start_indices, 1); ++ ++ // Calculate full batching indices; accumulating to the ++ // corresponding result indices. ++ auto d_input = ++ mx::array(input_batching_dims.data(), ++ {static_cast(input_batching_dims.size())}); ++ auto d_start = mx::array( ++ scatter_indices_batching_dims.data(), ++ {static_cast(scatter_indices_batching_dims.size())}); ++ result_indices = mx::scatter_add_axis( ++ result_indices, ++ mx::broadcast_to(d_input, ++ {result_indices.shape(0), d_input.shape(0)}), ++ mx::take(update_scatter_indices, ++ mx::subtract( ++ d_start, ++ mx::where(mx::less(d_start, mx::full(d_start.shape(), ++ index_vector_dim)), ++ mx::zeros(d_start.shape(), mx::int32), ++ mx::ones(d_start.shape(), mx::int32))), ++ 1), ++ 1); ++ ++ // Calculate update window indices ++ auto update_window_indices = ++ mx::take(update_indices, ++ mx::array(update_window_dims.data(), ++ {static_cast(update_window_dims.size())}), ++ 1); ++ ++ // Calculate full window indices; accumulating to the ++ // corresponding result indices. ++ std::vector tmp_indices; ++ auto full_window_idx_count = static_cast( ++ update_window_indices.shape(1) + inserted_window_dims.size() + ++ input_batching_dims.size()); ++ for (auto i = 0; i < full_window_idx_count; ++i) { ++ if (std::find(inserted_window_dims.begin(), inserted_window_dims.end(), ++ i) != inserted_window_dims.end() || ++ std::find(input_batching_dims.begin(), input_batching_dims.end(), ++ i) != input_batching_dims.end()) { ++ continue; ++ } ++ tmp_indices.push_back(i); ++ } ++ result_indices = mx::scatter_add_axis( ++ result_indices, ++ mx::broadcast_to( ++ mx::array(tmp_indices.data(), ++ {static_cast(tmp_indices.size())}), ++ {result_indices.shape(0), ++ static_cast(tmp_indices.size())}), ++ update_window_indices, 1); ++ ++ std::vector scatter_axes(inputs[0].ndim()); ++ std::iota(scatter_axes.begin(), scatter_axes.end(), 0); ++ std::vector gather_axes(updates[0].ndim()); ++ std::iota(gather_axes.begin(), gather_axes.end(), 0); ++ std::vector res; ++ ++ auto tmp_gather_indices = ++ mx::split(update_indices, update_indices.shape(1), 1); ++ auto tmp_scatter_indices = ++ mx::split(result_indices, result_indices.shape(1), 1); ++ ++ std::vector update_shape( ++ inputs[0].ndim() + tmp_scatter_indices[0].ndim(), 1); ++ update_shape[0] = update_indices.shape(0); ++ std::vector tmp_gather_slice_sizes(updates[0].ndim(), 1); ++ for (auto i = 0; i < inputs.size(); ++i) { ++ auto update_vals = ++ mx::reshape(mx::gather(updates[i], tmp_gather_indices, gather_axes, ++ tmp_gather_slice_sizes), ++ update_shape); ++ switch (scatter_type) { ++ case utils::ScatterType::Replace: ++ res.push_back(mx::scatter(inputs[i], tmp_scatter_indices, ++ update_vals, scatter_axes)); ++ break; ++ case utils::ScatterType::Add: ++ res.push_back(mx::scatter_add(inputs[i], tmp_scatter_indices, ++ update_vals, scatter_axes)); ++ break; ++ case utils::ScatterType::Prod: ++ res.push_back(mx::scatter_prod(inputs[i], tmp_scatter_indices, ++ update_vals, scatter_axes)); ++ break; ++ case utils::ScatterType::Max: ++ res.push_back(mx::scatter_max(inputs[i], tmp_scatter_indices, ++ update_vals, scatter_axes)); ++ break; ++ case utils::ScatterType::Min: ++ res.push_back(mx::scatter_min(inputs[i], tmp_scatter_indices, ++ update_vals, scatter_axes)); ++ break; ++ } ++ } ++ return res; ++ }; ++ op_lookup.scatter.emplace(scatter_op_info, mx::compile(scatter_func)); ++ } ++} ++ ++void compilePadOp(OpLookup& op_lookup, stablehlo::PadOp o) { ++ PadOpInfo pad_op_info = getPadOpInfo(o); ++ auto result_shape = std::get<0>(pad_op_info.captures); ++ auto edge_pad_low = std::get<1>(pad_op_info.captures); ++ auto interior_pad = std::get<2>(pad_op_info.captures); ++ if (auto search = op_lookup.pad.find(pad_op_info); ++ search == op_lookup.pad.end()) { ++ // std::cout << "Compiling PadOp" << std::endl; ++ auto pad_func = [result_shape, edge_pad_low, ++ interior_pad](const std::vector& inputs) { ++ auto operand = inputs[0]; ++ auto padding_value = inputs[1]; ++ auto edge_padding_low = ++ mx::array(edge_pad_low.data(), ++ {static_cast(edge_pad_low.size())}, mx::int32); ++ auto interior_padding = ++ mx::array(interior_pad.data(), ++ {static_cast(edge_pad_low.size())}, mx::int32); ++ ++ // Calculate operand indices ++ std::vector meshgrid_arrays; ++ for (const auto& dim : operand.shape()) { ++ meshgrid_arrays.push_back(mx::arange(dim)); ++ } ++ mx::array operand_indices = ++ mx::reshape(mx::stack(mx::meshgrid(meshgrid_arrays, false, "ij"), -1), ++ {-1, static_cast(operand.ndim())}); ++ ++ mx::array result_indices = ++ edge_padding_low + ++ operand_indices * (interior_padding + mx::array(1, mx::int32)); ++ ++ // initialize array full of padding values ++ mx::array result = mx::full(result_shape, padding_value); ++ std::vector axes(operand.ndim()); ++ std::iota(axes.begin(), axes.end(), 0); ++ return std::vector{mx::scatter( ++ result, mx::split(result_indices, result_indices.shape(1), 1), ++ mx::gather(operand, ++ mx::split(operand_indices, operand_indices.shape(1), 1), ++ axes, std::vector(operand.ndim(), 1)), ++ axes)}; ++ }; ++ op_lookup.pad.emplace(pad_op_info, mx::compile(pad_func)); ++ } ++} ++ ++void compileModule(ModuleOp mod, OpLookup& op_lookup) { ++ // the top level block to compile ++ // auto main = mod.lookupSymbol("main"); ++ ++ Tree dep_tree; ++ resolveFuncDeps(mod, "main", dep_tree); ++ ++ // walk each function and pre-compile lookup table ++ // for all StableHLO op variants ++ for (auto op_name : dep_tree.getAllFnNames()) { ++ auto func = mod.lookupSymbol(op_name); ++ auto& block = func.front(); ++ for (auto& op : block.getOperations()) { ++ llvm::TypeSwitch(&op) ++ .Case( ++ [&op_lookup](auto o) { compileIotaOp(op_lookup, o); }) ++ .Case( ++ [&op_lookup](auto o) { compileCbrtOp(op_lookup, o); }) ++ .Case( ++ [&op_lookup](auto o) { compileBroadcastInDimOp(op_lookup, o); }) ++ .Case( ++ [&op_lookup](auto o) { compileDotGeneralOp(op_lookup, o); }) ++ .Case( ++ [&op_lookup](auto o) { compileGatherOp(op_lookup, o); }) ++ .Case( ++ [&op_lookup](auto o) { compileScatterOp(op_lookup, o); }) ++ .Case( ++ [&op_lookup](auto o) { compilePadOp(op_lookup, o); }) ++ .Default([](auto o) {}); ++ } ++ } ++ ++ while (true) { ++ auto compilable_fns = dep_tree.getCompilableFns(); ++ if (compilable_fns.empty()) { ++ break; ++ } ++ for (auto fn_name : compilable_fns) { ++ // std::cout << fn_name.c_str() << std::endl; ++ auto func = mod.lookupSymbol(fn_name); ++ compileCallOp(op_lookup, mod, func); ++ dep_tree.remove(fn_name); ++ } ++ } ++} ++ ++class MlirLoadedExecutable : public PjRtLoadedExecutable { ++ public: ++ MlirLoadedExecutable(ModuleOp module, DeviceAssignment assignment, ++ absl::Span devices, ++ PjRtClient* client) ++ : PjRtLoadedExecutable(), ++ name_("MlirLoadedExecutable"), ++ assignment_(assignment), ++ devices_(client->devices()), ++ client_(client), ++ context_(), ++ module_(cloneIntoContext(module, context_)) { ++ // TRACE_ME_MEMBER; ++ auto main = module.lookupSymbol("main"); ++ // std::cout << "Started call to `mlx::compile`" << std::endl << std::endl; ++ // module.dump(); ++ // std::cout << std::endl; ++ compileModule(module, op_lookup_); ++ // try { ++ auto& main_block = main.front(); ++ compiled_module_ = ++ op_lookup_.call.find(getCallOpInfo(module, main))->second; ++ ++ // force compilation by running w zeroed inputs ++ std::vector tmp_inputs; ++ for (auto arg : main_block.getArguments()) { ++ auto shaped_type = mlir::cast(arg.getType()); ++ tmp_inputs.push_back( ++ mx::zeros(std::vector(shaped_type.getShape().begin(), ++ shaped_type.getShape().end()), ++ *utils::dtype::fromMlirType(shaped_type.getElementType()))); ++ } ++ compiled_module_(tmp_inputs); ++ // std::cout << "Finished call to `mlx::compile`" << std::endl << std::endl; ++ // } catch (std::exception& e) { ++ // std::cout << "Error in `mlx::compile`" << std::endl; ++ // } ++ } ++ ++ static absl::StatusOr> Compile( ++ ModuleOp module, DeviceAssignment assignment, ++ absl::Span devices, PjRtClient* client) { ++ // TRACE_ME; ++ ++ mlir::BaseScopedDiagnosticHandler diagnostic_handler(module->getContext()); ++ if (failed(decomposeChloToStablehlo(module))) { ++ return diagnostic_handler.ConsumeStatus(); ++ } ++ ++ // Simplify the graph using available HWI passes. ++ if (failed(runHardwareIndependentOptimizations(module))) { ++ return diagnostic_handler.ConsumeStatus(); ++ } ++ ++ auto module_status = validateModule(module); ++ if (!module_status.ok()) { ++ return module_status; ++ } ++ ++ // std::cout << ToString(module).c_str() << std::endl << std::endl; ++ ++ auto executable = std::make_unique(module, assignment, ++ devices, client); ++ ++ return executable; ++ } ++ ++ PjRtClient* client() const override { ++ // TRACE_ME_MEMBER; ++ return client_; ++ } ++ ++ const DeviceAssignment& device_assignment() const override { ++ // TRACE_ME_MEMBER; ++ return assignment_; ++ } ++ ++ absl::Span ++ addressable_device_logical_ids() const override { ++ // TRACE_ME_MEMBER; ++ LOG_UNIMPLEMENTED(addressable_device_logical_ids); ++ return {}; ++ } ++ ++ absl::Span addressable_devices() const override { ++ // TRACE_ME_MEMBER; ++ return devices_; ++ } ++ ++ // Helper function to get default mem from device. ++ PjRtMemorySpace* get_default_memory_space() const { ++ // TRACE_ME_MEMBER; ++ return devices_[0]->default_memory_space().value_or(nullptr); ++ } ++ ++ // absl::StatusOr> evalModule( ++ // ModuleOp& module, const SmallVector& inputs) { ++ // TRACE_ME_MEMBER; ++ // std::vector block_arguments; ++ // for (auto input : inputs) { ++ // TF_ASSIGN_OR_RETURN(auto input_arr, ++ // utils::array::fromDenseElementsAttr(input)); ++ // block_arguments.emplace_back(input_arr); ++ // } ++ ++ // // std::cout << std::endl; ++ // // module.dump(); ++ // // std::cout << std::endl; ++ // auto main = module.lookupSymbol("main"); ++ // std::unordered_map< ++ // mlir::Block*, ++ // std::tuple, ++ // std::unordered_map>>> ++ // block_ctx; ++ // TF_ASSIGN_OR_RETURN(auto mlx_res, evalBlock(module, main.front(), ++ // block_arguments, block_ctx)); ++ ++ // for (auto& a : mlx_res) { ++ // a = mx::contiguous(a); ++ // } ++ // mx::eval(mlx_res); ++ // return mlx_res; ++ // } ++ ++ absl::StatusOr ExecuteWithMlxInterpreter( ++ absl::Span argument_handles, ModuleOp module, ++ PjRtDevice* device, bool fill_future) { ++ // TRACE_ME_MEMBER; ++ SmallVector input_attrs; ++ for (auto* arg : argument_handles) { ++ TF_ASSIGN_OR_RETURN(auto mlirArg, GetAttributeFromBuffer(arg)); ++ auto mlirArgInModuleContext = ++ CloneIntoContext(mlirArg, *module->getContext()); ++ input_attrs.push_back(mlirArgInModuleContext); ++ } ++ // LOG(INFO) << "EvalModule:\n" << ToString(module) << "\n"; ++ // LOG(INFO) << "Inputs: " << ToString(inputs) << "\n"; ++ // TF_ASSIGN_OR_RETURN(auto mlx_res, evalModule(module, inputs)); ++ ++ std::vector inputs; ++ unsigned in_count = 0; ++ for (auto& attr : input_attrs) { ++ TF_ASSIGN_OR_RETURN(mx::array input, ++ utils::array::fromDenseElementsAttr(attr)); ++ // std::cout << "inputs[" << std::to_string(in_count++) << "]: " << input ++ // << std::endl; ++ inputs.push_back(input); ++ } ++ ++ auto mlx_res = compiled_module_(inputs); ++ for (auto& a : mlx_res) { ++ a = mx::contiguous(a); ++ } ++ mx::eval(mlx_res); ++ // unsigned out_count = 0; ++ // for (auto& a : mlx_res) { ++ // std::cout << "outputs[" << std::to_string(out_count++) << "]: " << a ++ // << std::endl; ++ // } ++ ++ // LOG(INFO) << "Results: " << ToString(result.value()) << "\n"; ++ ++ // Naive memory space selection, only using CPU global memory. ++ PjRtMemorySpace* memory_space = ++ device->default_memory_space().value_or(nullptr); ++ std::vector> buffer_results; ++ for (auto i = 0; i < mlx_res.size(); i++) { ++ buffer_results.push_back( ++ CreateMlirBufferFromMlxArray(mlx_res[i], memory_space)); ++ } ++ ++ std::optional> future; ++ if (fill_future) { ++ // Synchronous! To make async, this would need to return a future that ++ // is ready when the computation is done. ++ future = PjRtFuture<>(absl::OkStatus()); ++ } ++ return PjRtLoadedExecutable::Result{future, std::move(buffer_results)}; ++ } ++ ++ absl::StatusOr>>> Execute( ++ absl::Span> argument_handles, ++ const xla::ExecuteOptions& options, ++ std::optional>>& returned_futures) override { ++ // TRACE_ME_MEMBER; ++ if (argument_handles.size() != 1) { ++ // One arg handle per device. ++ return absl::InvalidArgumentError( ++ "MlirLoadedExecutable::Execute only supports a single argument " ++ "vector"); ++ } ++ ++ // Single device, synchronous, can always use 0. ++ PjRtDevice* device = devices_[0]; ++ bool fill_future = returned_futures.has_value(); ++ TF_ASSIGN_OR_RETURN( ++ PjRtLoadedExecutable::Result result, ++ ExecuteWithMlxInterpreter(argument_handles[0], module_.get(), device, ++ fill_future)); ++ std::vector>> results; ++ results.push_back(std::move(result.buffers)); ++ if (returned_futures.has_value()) { ++ returned_futures->push_back(std::move(result.future.value())); ++ } ++ return results; ++ } ++ ++ absl::StatusOr>> ExecuteSharded( ++ absl::Span argument_handles, PjRtDevice* device, ++ const xla::ExecuteOptions& options, ++ std::optional>& returned_future, bool fill_future) override { ++ // TRACE_ME_MEMBER; ++ // Synchronous! To make async, have the device make a buffer with a ready ++ // future that is ready when the computation is done / buffer is ready. ++ TF_ASSIGN_OR_RETURN( ++ PjRtLoadedExecutable::Result result, ++ ExecuteWithMlxInterpreter(argument_handles, module_.get(), device, ++ fill_future)); ++ if (returned_future.has_value() && fill_future) { ++ returned_future = std::move(result.future); ++ } ++ return std::move(result.buffers); ++ } ++ ++ absl::StatusOr>> ExecutePortable( ++ absl::Span argument_handles, PjRtDevice* device, ++ const xla::ExecuteOptions& options, ++ std::optional>& returned_future, bool fill_future) override { ++ // TRACE_ME_MEMBER; ++ // Synchronous! To make async, have the device make a buffer with a ready ++ // future that is ready when the computation is done / buffer is ready. ++ TF_ASSIGN_OR_RETURN( ++ PjRtLoadedExecutable::Result result, ++ ExecuteWithMlxInterpreter(argument_handles, module_.get(), device, ++ fill_future)); ++ if (returned_future.has_value() && fill_future) { ++ returned_future = std::move(result.future); ++ } ++ return std::move(result.buffers); ++ } ++ ++ void Delete() override { ++ // TRACE_ME_MEMBER; ++ module_.release(); ++ module_ = nullptr; ++ } ++ bool IsDeleted() override { ++ // TRACE_ME_MEMBER; ++ return !module_; ++ } ++ ++ // PjRtExecutable API. ++ int num_replicas() const override { ++ // TRACE_ME_MEMBER; ++ return assignment_.replica_count(); ++ } ++ int num_partitions() const override { ++ // TRACE_ME_MEMBER; ++ return assignment_.computation_count(); ++ } ++ int64_t SizeOfGeneratedCodeInBytes() const override { ++ // No generated code.. so just return 1. ++ // TRACE_ME_MEMBER; ++ return 1; ++ } ++ absl::string_view name() const override { ++ // TRACE_ME_MEMBER; ++ return name_; ++ } ++ ++ absl::StatusOr>> GetHloModules() ++ const override { ++ // TODO: This shouldn't be needed for an MLIR plugin, its only used in the ++ // JAX layer for determining output sharding, which exists on the mlir ++ // module. ++ // TRACE_ME_MEMBER; ++ auto moduleClone = llvm::cast(module_.get()->clone()); ++ TF_ASSIGN_OR_RETURN(auto hlo_module, ++ xla::ConvertStablehloToHlo(moduleClone)); ++ return std::vector>{std::move(hlo_module)}; ++ } ++ ++ absl::StatusOr>> ++ GetOutputMemoryKinds() const override { ++ // TRACE_ME_MEMBER; ++ return UNIMPLEMENTED(GetOutputMemoryKinds); ++ } ++ ++ absl::StatusOr FingerprintExecutable() const override { ++ // TRACE_ME_MEMBER; ++ return UNIMPLEMENTED(FingerprintExecutable); ++ } ++ ++ private: ++ std::string name_; ++ DeviceAssignment assignment_; ++ absl::Span devices_; ++ PjRtClient* client_; ++ std::function(const std::vector&)> ++ compiled_module_; ++ OpLookup op_lookup_; ++ ++ // MLIR ++ MLIRContext context_; ++ mlir::OwningOpRef module_; ++}; ++ ++absl::StatusOr> StablehloMlxCompile( ++ mlir::ModuleOp module, DeviceAssignment assignment, PjRtClient* client) { ++ // TRACE_ME; ++ return MlirLoadedExecutable::Compile(module, assignment, client->devices(), ++ client); ++} ++ ++absl::StatusOr> StablehloMlxCompile( ++ xla::XlaComputation const& computation, xla::DeviceAssignment assignment, ++ xla::PjRtClient* client) { ++ // TRACE_ME; ++ MLIRContext context; ++ TF_ASSIGN_OR_RETURN(auto module, ++ ConvertHloToStablehlo(context, &computation.proto())); ++ return StablehloMlxCompile(module.get(), assignment, client); ++} ++ ++} // namespace mlir::stablehlo +diff --git a/xla/pjrt/plugin/mlx/executable.h b/xla/pjrt/plugin/mlx/executable.h +new file mode 100644 +index 0000000000..e579af4e1f +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/executable.h +@@ -0,0 +1,244 @@ ++/* Copyright 2024 The OpenXLA Authors. ++Licensed under the Apache License, Version 2.0 (the "License"); ++you may not use this file except in compliance with the License. ++You may obtain a copy of the License at ++ http://www.apache.org/licenses/LICENSE-2.0 ++Unless required by applicable law or agreed to in writing, software ++distributed under the License is distributed on an "AS IS" BASIS, ++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++See the License for the specific language governing permissions and ++limitations under the License. ++==============================================================================*/ ++ ++#ifndef XLA_PJRT_PLUGIN_STABLEHLO_MLX_EXECUTABLE_H_ ++#define XLA_PJRT_PLUGIN_STABLEHLO_MLX_EXECUTABLE_H_ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "mlir/IR/BuiltinOps.h" ++#include "mlx/mlx.h" ++#include "xla/pjrt/plugin/mlx/utils.h" ++#include "xla/hlo/builder/xla_computation.h" ++#include "xla/pjrt/pjrt_client.h" ++#include "xla/service/computation_placer.h" ++ ++namespace mlir::stablehlo { ++ ++struct Node { ++ std::string name; ++ std::vector> children; ++ ++ Node(std::string name) : name(name) {} ++}; ++ ++class Tree { ++ public: ++ std::unique_ptr root; ++ ++ Tree() : root(nullptr) {} ++ ++ Node* insert(std::string name, Node* parent = nullptr) { ++ auto new_node = std::make_unique(name); ++ if (!root) { ++ root = std::move(new_node); ++ return root.get(); ++ } else if (parent) { ++ parent->children.push_back(std::move(new_node)); ++ return parent->children.back().get(); ++ } ++ } ++ ++ void remove(const std::string& name) { ++ if (!root) return; ++ ++ // Special case: root node matches ++ if (root->name == name) { ++ root.reset(); // deletes entire tree ++ return; ++ } ++ ++ remove_helper(root, name); ++ } ++ ++ std::set getAllFnNames() { ++ std::set fn_names; ++ if (!root) return fn_names; ++ ++ std::function traverse = [&](Node* node) { ++ fn_names.insert(node->name); ++ for (auto& child : node->children) { ++ traverse(child.get()); ++ } ++ }; ++ traverse(root.get()); ++ return fn_names; ++ } ++ ++ std::unordered_set getCompilableFns() { ++ std::unordered_set compilable_fns; ++ if (!root) return compilable_fns; ++ ++ std::function traverse = [&](Node* node) { ++ if (node->children.empty()) { ++ compilable_fns.insert(node->name); ++ } else { ++ for (auto& child : node->children) { ++ traverse(child.get()); ++ } ++ } ++ }; ++ traverse(root.get()); ++ return compilable_fns; ++ } ++ ++ private: ++ void remove_helper(std::unique_ptr& current, const std::string& name) { ++ if (!current) return; ++ ++ auto& children = current->children; ++ ++ // Remove matching children ++ children.erase(std::remove_if(children.begin(), children.end(), ++ [&](std::unique_ptr& child) { ++ return child->name == name; ++ }), ++ children.end()); ++ ++ // Recurse on remaining children ++ for (auto& child : children) { ++ remove_helper(child, name); ++ } ++ } ++}; ++ ++template ++class CompiledOpInfo { ++ public: ++ std::tuple captures; ++ std::vector> input_metadata; ++ std::vector> output_metadata; ++ ++ CompiledOpInfo(std::vector inputs, ++ std::vector> outputs, ++ T... capture_values); ++ ++ CompiledOpInfo(std::vector> inputs, ++ std::vector> outputs, ++ T... capture_values); ++ ++ bool operator==(const CompiledOpInfo& other) const { ++ return captures == other.captures && ++ input_metadata == other.input_metadata && ++ output_metadata == other.output_metadata; ++ } ++}; ++ ++template ++CompiledOpInfo::CompiledOpInfo( ++ std::vector inputs, ++ std::vector> outputs, T... capture_values) ++ : captures(std::make_tuple(capture_values...)), output_metadata(outputs) { ++ input_metadata.reserve(inputs.size()); ++ for (const auto& input : inputs) { ++ input_metadata.push_back(std::make_pair(input.dtype(), input.shape())); ++ } ++} ++ ++template ++CompiledOpInfo::CompiledOpInfo( ++ std::vector> inputs, ++ std::vector> outputs, T... capture_values) ++ : captures(std::make_tuple(capture_values...)), ++ input_metadata(inputs), ++ output_metadata(outputs) {} ++ ++using CompiledOpSign = ++ std::function(const std::vector&)>; ++ ++using CallOpInfo = CompiledOpInfo; ++ ++// stablehlo::IotaOp - Captures: (dimensions, dtype, iota_dim, result_shape) ++using IotaOpInfo = CompiledOpInfo, mx::Dtype, int64_t, ++ std::vector>; ++ ++// stablehlo::CbrtOp - Captures: () ++using CbrtOpInfo = CompiledOpInfo<>; ++ ++// stablehlo::BroadcastInDimOp - Captures: (broadcast_dims, target_shape) ++using BroadcastInDimOpInfo = ++ CompiledOpInfo, std::vector>; ++ ++// stablehlo::DotGeneralOp - Captures: (lhs_batch_dims, rhs_batch_dims, ++// lhs_contract_dims, rhs_contract_dims) ++using DotGeneralOpInfo = ++ CompiledOpInfo, std::vector, ++ std::vector, std::vector>; ++ ++// stablehlo::GatherOp - Captures: (collapsed_slice_dims, index_vector_dim, ++// offset_dims, operand_batching_dims, result_shape, slice_sizes, ++// start_index_map, start_indices_batching_dims) ++using GatherOpInfo = CompiledOpInfo, int64_t, ++ std::vector, std::vector, ++ std::vector, std::vector, ++ std::vector, std::vector>; ++ ++// stablehlo::ScatterOp - Captures: (index_vector_dim, input_batching_dims, ++// inserted_window_dims, scatter_dims_to_operand_dims, ++// scatter_indices_batching_dims, scatter_type, update_window_dims) ++using ScatterOpInfo = ++ CompiledOpInfo, std::vector, ++ std::vector, std::vector, ++ utils::ScatterType, std::vector>; ++ ++// stablehlo::PadOp - Captures: (result_shape, edge_pad_low, interior_pad) ++using PadOpInfo = CompiledOpInfo, std::vector, ++ std::vector>; ++ ++absl::StatusOr> StablehloMlxCompile( ++ mlir::ModuleOp module, xla::DeviceAssignment assignment, ++ xla::PjRtClient* client); ++ ++absl::StatusOr> StablehloMlxCompile( ++ xla::XlaComputation const& computation, xla::DeviceAssignment assignment, ++ xla::PjRtClient* client); ++ ++} // namespace mlir::stablehlo ++ ++template ++struct std::hash> { ++ std::size_t operator()( ++ const mlir::stablehlo::CompiledOpInfo& info) const noexcept { ++ std::size_t seed = 0; ++ hash_tuple(seed, info.captures); ++ for (const auto& meta : info.input_metadata) { ++ hash_combine(seed, std::hash{}(meta.first)); ++ hash_combine(seed, std::hash>{}(meta.second)); ++ } ++ for (const auto& meta : info.output_metadata) { ++ hash_combine(seed, std::hash{}(meta.first)); ++ hash_combine(seed, std::hash>{}(meta.second)); ++ } ++ return seed; ++ } ++}; ++ ++namespace mlir::stablehlo { ++struct OpLookup { ++ std::unordered_map, CompiledOpSign> call; ++ std::unordered_map iota; ++ std::unordered_map cbrt; ++ std::unordered_map broadcast_in_dim; ++ std::unordered_map dot_general; ++ std::unordered_map gather; ++ std::unordered_map scatter; ++ std::unordered_map pad; ++}; ++ ++}; // namespace mlir::stablehlo ++ ++#endif // XLA_PJRT_PLUGIN_STABLEHLO_MLX_EXECUTABLE_H_ +\ No newline at end of file +diff --git a/xla/pjrt/plugin/mlx/logging.cc b/xla/pjrt/plugin/mlx/logging.cc +new file mode 100644 +index 0000000000..3f21f67baf +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/logging.cc +@@ -0,0 +1,89 @@ ++/* Copyright 2024 The OpenXLA Authors. ++Licensed under the Apache License, Version 2.0 (the "License"); ++you may not use this file except in compliance with the License. ++You may obtain a copy of the License at ++ http://www.apache.org/licenses/LICENSE-2.0 ++Unless required by applicable law or agreed to in writing, software ++distributed under the License is distributed on an "AS IS" BASIS, ++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++See the License for the specific language governing permissions and ++limitations under the License. ++==============================================================================*/ ++ ++#include "xla/pjrt/plugin/mlx/logging.h" ++ ++#include ++ ++#include "absl/base/log_severity.h" ++#include "absl/log/globals.h" ++#include "llvm/Support/raw_ostream.h" ++#include "mlir/IR/Attributes.h" ++#include "mlir/IR/BuiltinAttributes.h" ++#include "mlir/IR/Operation.h" ++ ++namespace mlir::stablehlo { ++ ++std::string ToString(mlir::Attribute attr) { ++ std::string out; ++ llvm::raw_string_ostream os(out); ++ attr.print(os); ++ return out; ++} ++ ++std::string ToString(SmallVector attrs) { ++ std::string out; ++ llvm::raw_string_ostream os(out); ++ os << "["; ++ bool first = true; ++ for (auto attr : attrs) { ++ if (!first) os << ", "; ++ first = false; ++ attr.print(os); ++ } ++ os << "]"; ++ return out; ++} ++std::string ToString(Operation& op) { ++ std::string out; ++ llvm::raw_string_ostream os(out); ++ op.print(os); ++ return out; ++} ++ ++std::string ToString(Operation* op) { ++ std::string out; ++ llvm::raw_string_ostream os(out); ++ os << *op; ++ return out; ++} ++ ++std::string ToString(mlir::Value value) { ++ std::string out; ++ llvm::raw_string_ostream os(out); ++ value.print(os); ++ return out; ++} ++ ++std::string ToString(mlir::Block& block) { ++ std::string out; ++ llvm::raw_string_ostream os(out); ++ block.print(os); ++ return out; ++} ++ ++void SetupLogLevelFromEnv() { ++ absl::SetMinLogLevel(absl::LogSeverityAtLeast::kError); ++ const char* log_env = std::getenv("PJRT_LOG_LEVEL"); ++ if (!log_env) return; ++ if (strcmp(log_env, "INFO") == 0) { ++ absl::SetMinLogLevel(absl::LogSeverityAtLeast::kInfo); ++ } else if (strcmp(log_env, "WARNING") == 0) { ++ absl::SetMinLogLevel(absl::LogSeverityAtLeast::kWarning); ++ } else if (strcmp(log_env, "ERROR") == 0) { ++ absl::SetMinLogLevel(absl::LogSeverityAtLeast::kError); ++ } else { ++ LOG(ERROR) << "Invalid PJRT_LOG_LEVEL: " << log_env; ++ } ++} ++ ++} // namespace mlir::stablehlo +\ No newline at end of file +diff --git a/xla/pjrt/plugin/mlx/logging.h b/xla/pjrt/plugin/mlx/logging.h +new file mode 100644 +index 0000000000..762dfabbcc +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/logging.h +@@ -0,0 +1,42 @@ ++/* Copyright 2024 The OpenXLA Authors. ++Licensed under the Apache License, Version 2.0 (the "License"); ++you may not use this file except in compliance with the License. ++You may obtain a copy of the License at ++ http://www.apache.org/licenses/LICENSE-2.0 ++Unless required by applicable law or agreed to in writing, software ++distributed under the License is distributed on an "AS IS" BASIS, ++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++See the License for the specific language governing permissions and ++limitations under the License. ++==============================================================================*/ ++ ++#ifndef XLA_PJRT_PLUGIN_STABLEHLO_MLX_LOGGING_H_ ++#define XLA_PJRT_PLUGIN_STABLEHLO_MLX_LOGGING_H_ ++ ++// This file has some joint logging to allow LOG and VLOG to play well with ++// MLIR data structures ++ ++#include "absl/log/log.h" ++#include "mlir/IR/Attributes.h" ++#include "mlir/IR/BuiltinAttributes.h" ++ ++#define LOG_UNIMPLEMENTED(name) \ ++ LOG(ERROR) << "MlxPjRtBuffer::" #name " is not implemented" ++ ++#define TRACE_ME LOG(INFO) << __func__; ++ ++#define TRACE_ME_MEMBER LOG(INFO) << __func__ << "(" << (void*)this << ")\n"; ++ ++namespace mlir::stablehlo { ++std::string ToString(mlir::Attribute attr); ++std::string ToString(llvm::SmallVector attrs); ++std::string ToString(Operation* op); ++std::string ToString(Operation& op); ++std::string ToString(mlir::Value op); ++std::string ToString(mlir::Block& block); ++ ++// Looks for `PJRT_LOG_LEVEL = INFO|WARNING|ERROR` in env variables. ++void SetupLogLevelFromEnv(); ++} // namespace mlir::stablehlo ++ ++#endif // XLA_PJRT_PLUGIN_STABLEHLO_MLX_LOGGING_H_ +\ No newline at end of file +diff --git a/xla/pjrt/plugin/mlx/plugin_pjrt_test.cc b/xla/pjrt/plugin/mlx/plugin_pjrt_test.cc +new file mode 100644 +index 0000000000..d56a22565f +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/plugin_pjrt_test.cc +@@ -0,0 +1,33 @@ ++/* Copyright 2022 The OpenXLA Authors. ++Licensed under the Apache License, Version 2.0 (the "License"); ++you may not use this file except in compliance with the License. ++You may obtain a copy of the License at ++ http://www.apache.org/licenses/LICENSE-2.0 ++Unless required by applicable law or agreed to in writing, software ++distributed under the License is distributed on an "AS IS" BASIS, ++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++See the License for the specific language governing permissions and ++limitations under the License. ++==============================================================================*/ ++ ++#include "xla/pjrt/c/pjrt_c_api_test.h" ++#include "xla/pjrt/c/pjrt_c_api_wrapper_impl.h" ++#include "xla/pjrt/pjrt_client_test.h" ++#include "xla/pjrt/plugin/mlx/client_c_pjrt.h" ++#include "xla/pjrt/plugin/mlx/client_cpp_pjrt.h" ++ ++namespace pjrt { ++namespace { ++ ++const bool kUnused = ++ (RegisterPjRtCApiTestFactory([]() { return GetPjrtApi(); }, ++ /*platform_name=*/"stablehlo_mlx"), ++ true); ++ ++const bool kUnused2 = (xla::RegisterTestClientFactory([]() { ++ return mlir::stablehlo::CreateStablehloMlxPjrtClient(); ++ }), ++ true); ++ ++} // namespace ++} // namespace pjrt +\ No newline at end of file +diff --git a/xla/pjrt/plugin/mlx/utils.cc b/xla/pjrt/plugin/mlx/utils.cc +new file mode 100644 +index 0000000000..5fcfc318ab +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/utils.cc +@@ -0,0 +1,292 @@ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "xla/pjrt/plugin/mlx/utils.h" ++#include "absl/status/status.h" ++#include "absl/status/statusor.h" ++#include "absl/types/span.h" ++#include "llvm/ADT/TypeSwitch.h" ++#include "mlir/IR/BuiltinAttributes.h" ++#include "mlir/IR/Types.h" ++#include "mlx/mlx.h" ++#include "tsl/platform/statusor.h" ++#include "xla/mlir/utils/type_util.h" ++#include "xla/shape_util.h" ++ ++#include "xla/literal.h" ++#include "xla/util.h" ++ ++namespace mx = mlx::core; ++ ++std::tuple shapeAndBytes( ++ absl::Span dims, ++ std::optional> byte_strides, mx::Dtype dtype) { ++ mx::Shape shape(dims.size()); ++ mx::Strides strides; ++ mx::Shape raw_buffer_shape = {shape.size() && !byte_strides.has_value() ? 1 ++ : 0}; ++ for (auto i = 0; i < dims.size(); ++i) { ++ shape[i] = static_cast(dims[i]); ++ if (byte_strides.has_value()) { ++ auto stride_value = byte_strides.value()[i]; ++ strides.push_back(stride_value / dtype.size()); ++ if (strides[i] != 0) { ++ raw_buffer_shape[0] = std::max( ++ raw_buffer_shape[0], static_cast(stride_value) * shape[i]); ++ } ++ } else { ++ raw_buffer_shape[0] *= static_cast(dims[i]); ++ } ++ } ++ if (!byte_strides.has_value()) { ++ raw_buffer_shape[0] *= dtype.size(); ++ } else if (raw_buffer_shape[0] == 0 and ++ std::find(dims.begin(), dims.end(), 0) == dims.end()) { ++ raw_buffer_shape[0] += dtype.size(); ++ } ++ return std::make_tuple(shape, raw_buffer_shape, strides); ++} ++ ++namespace utils { ++ ++namespace dtype { ++xla::PrimitiveType asXlaPrimitiveType(mx::Dtype dtype) { ++ switch (dtype) { ++ case mx::bool_: ++ return xla::PrimitiveType::PRED; ++ case mx::uint8: ++ return xla::PrimitiveType::U8; ++ case mx::uint16: ++ return xla::PrimitiveType::U16; ++ case mx::uint32: ++ return xla::PrimitiveType::U32; ++ case mx::uint64: ++ return xla::PrimitiveType::U64; ++ case mx::int8: ++ return xla::PrimitiveType::S8; ++ case mx::int16: ++ return xla::PrimitiveType::S16; ++ case mx::int32: ++ return xla::PrimitiveType::S32; ++ case mx::int64: ++ return xla::PrimitiveType::S64; ++ case mx::float16: ++ return xla::PrimitiveType::F16; ++ case mx::float32: ++ return xla::PrimitiveType::F32; ++ case mx::bfloat16: ++ return xla::PrimitiveType::BF16; ++ case mx::complex64: ++ return xla::PrimitiveType::C64; ++ } ++} ++ ++absl::StatusOr fromXlaPrimitiveType(xla::PrimitiveType dtype) { ++ switch (dtype) { ++ case xla::PrimitiveType::PRED: ++ return mx::bool_; ++ case xla::PrimitiveType::BF16: ++ return mx::bfloat16; ++ case xla::PrimitiveType::F16: ++ return mx::float16; ++ case xla::PrimitiveType::F32: ++ return mx::float32; ++ case xla::PrimitiveType::S8: ++ return mx::int8; ++ case xla::PrimitiveType::S16: ++ return mx::int16; ++ case xla::PrimitiveType::S32: ++ return mx::int32; ++ case xla::PrimitiveType::S64: ++ return mx::int64; ++ case xla::PrimitiveType::U8: ++ return mx::uint8; ++ case xla::PrimitiveType::U16: ++ return mx::uint16; ++ case xla::PrimitiveType::U32: ++ return mx::uint32; ++ case xla::PrimitiveType::U64: ++ return mx::uint64; ++ case xla::PrimitiveType::C64: ++ return mx::complex64; ++ default: ++ return absl::InvalidArgumentError("Unsupported type"); ++ } ++} ++ ++absl::StatusOr fromMlirType(mlir::Type type) { ++ auto primitive_type = xla::ConvertMlirTypeToPrimitiveType(type); ++ if (primitive_type == xla::PrimitiveType::PRIMITIVE_TYPE_INVALID) { ++ return xla::Internal("Unsupported type: %s", ++ xla::PrimitiveType_Name(primitive_type)); ++ } ++ return fromXlaPrimitiveType(primitive_type); ++} ++ ++int mlirTypeByteWidth(mlir::Type type) { ++ return xla::primitive_util::ByteWidth( ++ xla::ConvertMlirTypeToPrimitiveType(type)); ++} ++} // namespace dtype ++ ++namespace array { ++absl::StatusOr fromHostBuffer( ++ const void* data, absl::Span dims, ++ std::optional> byte_strides, ++ xla::PrimitiveType type) { ++ TF_ASSIGN_OR_RETURN(mx::Dtype dtype, dtype::fromXlaPrimitiveType(type)); ++ auto [shape, raw_buffer_shape, strides] = ++ shapeAndBytes(dims, byte_strides, dtype); ++ mx::array typed_view = mx::view(mx::array(reinterpret_cast(data), ++ raw_buffer_shape, mx::uint8), ++ dtype); ++ if (byte_strides.has_value()) { ++ auto res = mx::contiguous(mx::as_strided(typed_view, shape, strides, 0)); ++ res.eval(); ++ return res; ++ } else { ++ auto res = mx::reshape(typed_view, shape); ++ res.eval(); ++ return res; ++ } ++} ++ ++absl::StatusOr fromHostLiteral(const xla::LiteralSlice& literal) { ++ return fromHostBuffer(literal.untyped_data(), literal.shape().dimensions(), ++ std::nullopt, literal.shape().element_type()); ++} ++ ++template ++absl::StatusOr fromDenseElementsAttr(mlir::DenseElementsAttr attr) { ++ auto attr_type = attr.getType(); ++ // convert to mlx shape ++ std::vector shape(attr_type.getShape().begin(), ++ attr_type.getShape().end()); ++ ++ // handle splat values ++ if (attr.isSplat()) { ++ if constexpr (std::is_same::value) { ++ return mx::full( ++ shape, static_cast(attr.getSplatValue())); ++ } else if constexpr (std::is_same::value) { ++ return mx::full( ++ shape, static_cast(attr.getSplatValue())); ++ } else if constexpr (std::is_same>::value) { ++ return mx::full( ++ shape, static_cast(attr.getSplatValue())); ++ } else { ++ return mx::full(shape, attr.getSplatValue()); ++ } ++ } ++ ++ // handle non-splat values that are expanded to fit shape ++ auto it = attr.getValues(); ++ auto buffer = llvm::to_vector(it); ++ if (attr.size() != attr.getNumElements()) { ++ if constexpr (std::is_same::value) { ++ return mx::full( ++ shape, ++ mx::array(reinterpret_cast(buffer.data()), ++ {static_cast(attr.size())})); ++ } else if constexpr (std::is_same::value) { ++ return mx::full( ++ shape, ++ mx::array(reinterpret_cast(buffer.data()), ++ {static_cast(attr.size())})); ++ } else if constexpr (std::is_same>::value) { ++ return mx::full( ++ shape, ++ mx::array(reinterpret_cast(buffer.data()), ++ {static_cast(attr.size())})); ++ } else { ++ return mx::full(shape, mx::array(buffer.begin(), ++ {static_cast(attr.size())})); ++ } ++ } ++ ++ if constexpr (std::is_same::value) { ++ return mx::array(reinterpret_cast(buffer.data()), ++ shape); ++ } else if constexpr (std::is_same::value) { ++ return mx::array(reinterpret_cast(buffer.data()), ++ shape); ++ } else if constexpr (std::is_same>::value) { ++ return mx::array(reinterpret_cast(buffer.data()), ++ shape); ++ } else { ++ return mx::array(buffer.begin(), shape); ++ } ++} ++ ++absl::StatusOr fromDenseElementsAttr(mlir::DenseElementsAttr attr) { ++ auto element_type = ++ xla::ConvertMlirTypeToPrimitiveType(attr.getType().getElementType()); ++ switch (element_type) { ++ case xla::PrimitiveType::PRED: ++ return fromDenseElementsAttr(attr); ++ case xla::PrimitiveType::U8: ++ return fromDenseElementsAttr(attr); ++ case xla::PrimitiveType::S8: ++ return fromDenseElementsAttr(attr); ++ case xla::PrimitiveType::U16: ++ return fromDenseElementsAttr(attr); ++ case xla::PrimitiveType::S16: ++ return fromDenseElementsAttr(attr); ++ case xla::PrimitiveType::U32: ++ return fromDenseElementsAttr(attr); ++ case xla::PrimitiveType::S32: ++ return fromDenseElementsAttr(attr); ++ case xla::PrimitiveType::U64: ++ return fromDenseElementsAttr(attr); ++ case xla::PrimitiveType::S64: ++ return fromDenseElementsAttr(attr); ++ case xla::PrimitiveType::F16: ++ return fromDenseElementsAttr(attr); ++ case xla::PrimitiveType::BF16: ++ return fromDenseElementsAttr(attr); ++ case xla::PrimitiveType::F32: ++ return fromDenseElementsAttr(attr); ++ case xla::PrimitiveType::C64: ++ return fromDenseElementsAttr>(attr); ++ default: ++ return xla::Internal("Unsupported type: %s", ++ xla::PrimitiveType_Name(element_type)); ++ } ++} ++ ++absl::StatusOr fromOperand( ++ mlir::Value operand, absl::Span block_args, ++ const std::unordered_map>& ++ transient_buffers, ++ const std::unordered_map& init_values) {} ++ ++absl::StatusOr fromOperand( ++ mlir::Value operand, absl::Span block_args, ++ const std::unordered_map>& ++ transient_buffers) { ++ return fromOperand(operand, block_args, transient_buffers, {}); ++} ++ ++} // namespace array ++ ++void printVector(const std::string& name, const std::vector& vec, ++ bool indent) { ++ if (indent) { ++ std::cout << "\t"; ++ } ++ std::cout << name.c_str() << ": { "; ++ for (auto i = 0; i < vec.size(); i++) { ++ std::cout << std::to_string(vec[i]); ++ if (i < vec.size() - 1) { ++ std::cout << ", "; ++ } ++ } ++ std::cout << " }" << std::endl; ++} ++} // namespace utils +diff --git a/xla/pjrt/plugin/mlx/utils.h b/xla/pjrt/plugin/mlx/utils.h +new file mode 100644 +index 0000000000..1a6d85c060 +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/utils.h +@@ -0,0 +1,129 @@ ++/* Copyright 2024 The OpenXLA Authors. ++Licensed under the Apache License, Version 2.0 (the "License"); ++you may not use this file except in compliance with the License. ++You may obtain a copy of the License at ++ http://www.apache.org/licenses/LICENSE-2.0 ++Unless required by applicable law or agreed to in writing, software ++distributed under the License is distributed on an "AS IS" BASIS, ++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++See the License for the specific language governing permissions and ++limitations under the License. ++==============================================================================*/ ++ ++#ifndef XLA_PJRT_PLUGIN_STABLEHLO_MLX_UTILS_H_ ++#define XLA_PJRT_PLUGIN_STABLEHLO_MLX_UTILS_H_ ++ ++#include ++#include ++#include ++#include ++#include ++ ++#include "absl/status/status.h" ++#include "absl/status/statusor.h" ++#include "absl/types/span.h" ++#include "mlir/IR/BuiltinAttributes.h" ++#include "mlir/IR/Types.h" ++#include "mlx/mlx.h" ++#include "xla/literal.h" ++#include "xla/util.h" ++ ++namespace mx = mlx::core; ++ ++namespace utils { ++ ++enum ScatterType { ++ Replace, ++ Add, ++ Prod, ++ Max, ++ Min, ++}; ++namespace dtype { ++xla::PrimitiveType asXlaPrimitiveType(mx::Dtype dtype); ++ ++absl::StatusOr fromXlaPrimitiveType(xla::PrimitiveType dtype); ++ ++absl::StatusOr fromMlirType(mlir::Type type); ++ ++int mlirTypeByteWidth(mlir::Type type); ++} // namespace dtype ++ ++namespace array { ++absl::StatusOr fromHostBuffer( ++ const void* data, absl::Span dims, ++ std::optional> byte_strides, ++ xla::PrimitiveType type); ++ ++absl::StatusOr fromHostLiteral(const xla::LiteralSlice& literal); ++ ++absl::StatusOr fromDenseElementsAttr(mlir::DenseElementsAttr attr); ++ ++absl::StatusOr fromOperand( ++ mlir::Value operand, absl::Span block_args, ++ const std::unordered_map>& ++ transient_buffers, ++ const std::unordered_map& init_values); ++ ++absl::StatusOr fromOperand( ++ mlir::Value operand, absl::Span block_args, ++ const std::unordered_map>& ++ transient_buffers); ++} // namespace array ++ ++void printVector(const std::string& name, const std::vector& vec, ++ bool indent = false); ++} // namespace utils ++ ++// Helper function to hash a tuple ++template ++inline void hash_combine(std::size_t& seed, const T& val) { ++ seed ^= std::hash{}(val) + 0x9e3779b9 + (seed << 6) + (seed >> 2); ++} ++ ++template ++inline ++ typename std::enable_if::value, void>::type ++ hash_tuple(std::size_t&, const Tuple&) {} ++ ++template ++ inline typename std::enable_if < ++ Index::value, void>::type hash_tuple( ++ std::size_t& seed, const Tuple& tuple) { ++ hash_combine(seed, std::get(tuple)); ++ hash_tuple(seed, tuple); ++} ++ ++template <> ++struct std::hash { ++ std::size_t operator()(const utils::ScatterType& value) const noexcept { ++ return std::hash::type>{}( ++ static_cast::type>(value)); ++ } ++}; ++ ++template ++struct std::hash> { ++ std::size_t operator()(const std::vector& vec) const { ++ std::size_t seed = 0; ++ for (const T& elem : vec) { ++ seed ^= std::hash{}(elem) + 0x9e3779b9 + (seed << 6) + (seed >> 2); ++ } ++ return seed; ++ } ++}; ++ ++template <> ++struct std::hash { ++ std::size_t operator()(const mx::Dtype& dtype) const noexcept { ++ std::size_t seed = 0; ++ hash_combine(seed, std::hash{}(dtype.size())); ++ hash_combine(seed, ++ std::hash::type>{}( ++ static_cast::type>( ++ dtype.val()))); ++ return seed; ++ } ++}; ++ ++#endif // XLA_PJRT_PLUGIN_STABLEHLO_MLX_UTILS_H_ +\ No newline at end of file diff --git a/third_party/modules/xla/20250317.2-71c67e2/source.json b/third_party/modules/xla/20250317.2-71c67e2/source.json index 9d579abc..2a9d52ba 100644 --- a/third_party/modules/xla/20250317.2-71c67e2/source.json +++ b/third_party/modules/xla/20250317.2-71c67e2/source.json @@ -1,16 +1,17 @@ { - "strip_prefix": "xla-71c67e2a4f40267115a0d4ea7c36748bbe7e750e", - "url": "https://github.com/openxla/xla/archive/71c67e2a4f40267115a0d4ea7c36748bbe7e750e.tar.gz", - "integrity": "sha256-j6D1MC7+WsbZ+Ve3hPmDlCzZX1yV6RIP2BWDgQcbcYc=", - "overlay": { - "llvm.bzl": "", - "MODULE.bazel": "", - "tsl.bzl": "", - "workspace.bzl": "" - }, - "patch_strip": 1, - "patches": { - "0001-bazel-migration-to-bazel-8.1.1.patch": "", - "0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch": "" - } + "strip_prefix": "xla-71c67e2a4f40267115a0d4ea7c36748bbe7e750e", + "url": "https://github.com/openxla/xla/archive/71c67e2a4f40267115a0d4ea7c36748bbe7e750e.tar.gz", + "integrity": "sha256-j6D1MC7+WsbZ+Ve3hPmDlCzZX1yV6RIP2BWDgQcbcYc=", + "overlay": { + "llvm.bzl": "", + "MODULE.bazel": "", + "tsl.bzl": "", + "workspace.bzl": "" + }, + "patch_strip": 1, + "patches": { + "0001-bazel-migration-to-bazel-8.1.1.patch": "", + "0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch": "", + "0003-Added-MLX-PjRt-Plugin.patch": "" + } } diff --git a/third_party/modules/xla/20250527.0-cb67f2f/MODULE.bazel b/third_party/modules/xla/20250527.0-cb67f2f/MODULE.bazel index 74c9eaad..2cd155e8 100644 --- a/third_party/modules/xla/20250527.0-cb67f2f/MODULE.bazel +++ b/third_party/modules/xla/20250527.0-cb67f2f/MODULE.bazel @@ -8,7 +8,6 @@ bazel_dep(name = "platforms", version = "0.0.8") bazel_dep(name = "bazel_skylib", version = "1.5.0") bazel_dep(name = "rules_cc", version = "0.0.17") bazel_dep(name = "rules_apple", version = "3.22.0", repo_name = "build_bazel_rules_apple") -bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "com_google_absl") bazel_dep(name = "rules_python", version = "0.39.0") bazel_dep(name = "rules_proto", version = "6.0.0-rc1") bazel_dep(name = "rules_java", version = "7.3.2") @@ -19,17 +18,28 @@ bazel_dep(name = "rules_license", version = "0.0.8") bazel_dep(name = "rules_shell", version = "0.4.1") bazel_dep(name = "bazel_features", version = "1.25.0", repo_name = "proto_bazel_features") +# MLX PjRt Plugin Dependencies +bazel_dep(name = "mlx", version = "20250605.0-c763fe1") + workspace_private = use_extension("//:workspace_private.bzl", "workspace_private") use_repo( workspace_private, "com_github_grpc_grpc", "com_google_protobuf", + "com_google_absl", "local_config_cuda", "local_config_remote_execution", "local_config_rocm", + "local_config_sycl", "local_config_tensorrt", "python_version_repo", "tsl", + "eigen_archive", + "farmhash_archive", + "highwayhash", + "ml_dtypes_py", + "shardy", + "snappy", ) workspace_public = use_extension("//:xla.bzl", "xla") diff --git a/third_party/modules/xla/20250527.0-cb67f2f/overlay/workspace_private.bzl b/third_party/modules/xla/20250527.0-cb67f2f/overlay/workspace_private.bzl index b234d122..4d229862 100644 --- a/third_party/modules/xla/20250527.0-cb67f2f/overlay/workspace_private.bzl +++ b/third_party/modules/xla/20250527.0-cb67f2f/overlay/workspace_private.bzl @@ -1,22 +1,41 @@ +load("//third_party/absl:workspace.bzl", absl = "repo") load("//third_party/gpus:cuda_configure.bzl", "cuda_configure") load("//third_party/gpus:rocm_configure.bzl", "rocm_configure") -load("//third_party/llvm:workspace.bzl", llvm = "repo") +load("//third_party/gpus:sycl_configure.bzl", "sycl_configure") +load("//third_party/eigen3:workspace.bzl", eigen3 = "repo") +load("//third_party/farmhash:workspace.bzl", farmhash = "repo") +load("//third_party/highwayhash:workspace.bzl", highwayhash = "repo") +load("//third_party/py/ml_dtypes:workspace.bzl", ml_dtypes = "repo") load("//third_party/py:python_repo.bzl", "python_repository") load("//third_party/pybind11_bazel:workspace.bzl", pybind11_bazel = "repo") -load("//third_party/stablehlo:workspace.bzl", stablehlo = "repo") +load("//third_party/shardy:workspace.bzl", shardy = "repo") load("//third_party/tensorrt:tensorrt_configure.bzl", "tensorrt_configure") -load("//third_party/triton:workspace.bzl", triton = "repo") load("//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls") load("//third_party:repo.bzl", "tf_vendored") load("//tools/toolchains/remote:configure.bzl", "remote_execution_configure") def _workspace_private_impl(mctx): + absl() cuda_configure(name = "local_config_cuda") remote_execution_configure(name = "local_config_remote_execution") rocm_configure(name = "local_config_rocm") + sycl_configure(name = "local_config_sycl") + shardy() tensorrt_configure(name = "local_config_tensorrt") tf_vendored(name = "tsl", relpath = "third_party/tsl") pybind11_bazel() + eigen3() + farmhash() + highwayhash() + ml_dtypes() + tf_http_archive( + name = "snappy", + build_file = "//third_party:snappy.BUILD", + sha256 = "2e458b7017cd58dcf1469ab315389e85e7f445bd035188f2983f81fb19ecfb29", + strip_prefix = "snappy-984b191f0fefdeb17050b42a90b7625999c13b8d", + system_build_file = "//third_party/systemlibs:snappy.BUILD", + urls = tf_mirror_urls("https://github.com/google/snappy/archive/984b191f0fefdeb17050b42a90b7625999c13b8d.tar.gz"), + ) tf_http_archive( name = "com_github_grpc_grpc", sha256 = "b956598d8cbe168b5ee717b5dafa56563eb5201a947856a6688bbeac9cac4e1f", diff --git a/third_party/modules/xla/20250527.0-cb67f2f/patches/0004-Added-MLX-PjRt-Plugin.patch b/third_party/modules/xla/20250527.0-cb67f2f/patches/0004-Added-MLX-PjRt-Plugin.patch new file mode 100644 index 00000000..ea86579f --- /dev/null +++ b/third_party/modules/xla/20250527.0-cb67f2f/patches/0004-Added-MLX-PjRt-Plugin.patch @@ -0,0 +1,4487 @@ +diff --git a/xla/pjrt/plugin/mlx/BUILD b/xla/pjrt/plugin/mlx/BUILD +new file mode 100644 +index 0000000000..8255851b7f +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/BUILD +@@ -0,0 +1,243 @@ ++load("//xla:xla.default.bzl", "xla_cc_binary", "xla_cc_test") ++ ++package( ++ # copybara:uncomment default_applicable_licenses = ["//tensorflow:license"], ++ default_visibility = ["//visibility:public"], ++ licenses = ["notice"], ++) ++ ++cc_library( ++ name = "buffer", ++ srcs = ["buffer.cc"], ++ hdrs = ["buffer.h"], ++ deps = [ ++ ":logging", ++ ":utils", ++ "//xla:literal", ++ "//xla:shape_util", ++ "//xla:util", ++ "//xla/hlo/translate/hlo_to_mhlo:hlo_utils", ++ "//xla/hlo/translate/mhlo_to_hlo:literal_exporter", ++ "//xla/hlo/translate/mhlo_to_hlo:type_to_shape", ++ "//xla/pjrt:pjrt_client", ++ "//xla/pjrt:pjrt_compiler", ++ "//xla/pjrt:pjrt_future", ++ "@com_google_absl//absl/functional:any_invocable", ++ "@com_google_absl//absl/log", ++ "@com_google_absl//absl/status:statusor", ++ "@com_google_absl//absl/types:span", ++ "@llvm-project//llvm:Support", ++ "@llvm-project//mlir:AsmParser", ++ "@llvm-project//mlir:IR", ++ "@llvm-project//mlir:Support", ++ ], ++) ++ ++cc_library( ++ name = "compile", ++ srcs = ["compile.cc"], ++ hdrs = ["compile.h"], ++ deps = [ ++ "@mlx//:mlx", ++ ":buffer", ++ ":logging", ++ ":utils", ++ "//xla/hlo/builder:xla_computation", ++ "//xla/hlo/translate:stablehlo", ++ "//xla/mlir/utils:error_util", ++ "//xla/mlir_hlo:mhlo_passes", ++ "//xla/mlir_hlo:stablehlo_extension_passes", ++ "//xla/mlir/utils:type_util", ++ # "//xla/pjrt:mlir_to_hlo", ++ "//xla/pjrt:pjrt_client", ++ "//xla/pjrt:pjrt_executable", ++ "//xla/pjrt:pjrt_future", ++ "//xla/service:computation_placer_hdr", ++ "@com_google_absl//absl/log", ++ "@com_google_absl//absl/status", ++ "@com_google_absl//absl/status:statusor", ++ "@llvm-project//llvm:Support", ++ "@llvm-project//mlir:AsmParser", ++ "@llvm-project//mlir:BytecodeWriter", ++ "@llvm-project//mlir:DataLayoutInterfaces", ++ "@llvm-project//mlir:FuncDialect", ++ "@llvm-project//mlir:IR", ++ "@llvm-project//mlir:Parser", ++ "@llvm-project//mlir:Pass", ++ "@llvm-project//mlir:Support", ++ "@llvm-project//mlir:Transforms", ++ "@stablehlo//:reference_api", ++ "@stablehlo//:stablehlo_passes_optimization", ++ "@tsl//tsl/platform:statusor", ++ ], ++) ++ ++ ++cc_library( ++ name = "executable", ++ srcs = ["executable.cc"], ++ hdrs = ["executable.h"], ++ deps = [ ++ "@mlx//:mlx", ++ ":buffer", ++ ":compile", ++ ":logging", ++ ":utils", ++ "//xla/hlo/builder:xla_computation", ++ "//xla/hlo/translate:stablehlo", ++ "//xla/mlir/utils:error_util", ++ "//xla/mlir_hlo:mhlo_passes", ++ "//xla/mlir_hlo:stablehlo_extension_passes", ++ "//xla/mlir/utils:type_util", ++ # "//xla/pjrt:mlir_to_hlo", ++ "//xla/pjrt:pjrt_client", ++ "//xla/pjrt:pjrt_executable", ++ "//xla/pjrt:pjrt_future", ++ "//xla/service:computation_placer_hdr", ++ "@com_google_absl//absl/log", ++ "@com_google_absl//absl/status", ++ "@com_google_absl//absl/status:statusor", ++ "@llvm-project//llvm:Support", ++ "@llvm-project//mlir:AsmParser", ++ "@llvm-project//mlir:BytecodeWriter", ++ "@llvm-project//mlir:DataLayoutInterfaces", ++ "@llvm-project//mlir:FuncDialect", ++ "@llvm-project//mlir:IR", ++ "@llvm-project//mlir:Parser", ++ "@llvm-project//mlir:Pass", ++ "@llvm-project//mlir:Support", ++ "@llvm-project//mlir:Transforms", ++ "@stablehlo//:reference_api", ++ "@stablehlo//:stablehlo_passes_optimization", ++ "@tsl//tsl/platform:statusor", ++ ], ++) ++ ++cc_library( ++ name = "device", ++ srcs = ["device.cc"], ++ hdrs = ["device.h"], ++ deps = [ ++ ":logging", ++ "//xla:util", ++ "//xla/pjrt:pjrt_client", ++ "//xla/pjrt:pjrt_common", ++ "//xla/pjrt:pjrt_compiler", ++ "//xla/pjrt:pjrt_device_description", ++ "@com_google_absl//absl/status:statusor", ++ "@com_google_absl//absl/strings:str_format", ++ "@com_google_absl//absl/strings:string_view", ++ "@com_google_absl//absl/types:span", ++ "@mlx//:mlx", ++ ], ++) ++ ++cc_library( ++ name = "client_cpp_pjrt", ++ srcs = [ ++ "client_cpp_pjrt.cc", ++ ], ++ hdrs = [ ++ "client_cpp_pjrt.h", ++ ], ++ deps = [ ++ # Deps used in example plugin ++ "//xla/pjrt:pjrt_client", ++ "//xla/pjrt:pjrt_compiler", ++ "@com_google_absl//absl/strings:string_view", ++ "@com_google_absl//absl/types:span", ++ "@tsl//tsl/platform:fingerprint", ++ # Deps used for mlx plugin ++ ":buffer", ++ ":device", ++ ":executable", ++ ":logging", ++ ":utils", ++ "//xla:literal", ++ "//xla:shape_util", ++ "//xla:util", ++ "//xla/pjrt:host_memory_spaces", ++ "//xla/pjrt:pjrt_common", ++ "//xla/tsl/framework:allocator", ++ "@com_google_absl//absl/status:statusor", ++ "@com_google_absl//absl/strings:str_format", ++ "@mlx//:mlx", ++ "@tsl//tsl/platform:statusor" ++ ], ++) ++ ++cc_library( ++ name = "client_c_pjrt", ++ srcs = [ ++ "client_c_pjrt.cc", ++ ], ++ hdrs = ["client_c_pjrt.h"], ++ deps = [ ++ ":client_cpp_pjrt", ++ "//xla/pjrt:pjrt_client", ++ "//xla/pjrt/c:pjrt_c_api_hdrs", ++ "//xla/pjrt/c:pjrt_c_api_layouts_extension_hdrs", ++ "//xla/pjrt/c:pjrt_c_api_wrapper_impl", ++ "@com_google_absl//absl/status", ++ ], ++ alwayslink = 1, ++) ++ ++cc_library( ++ name = "logging", ++ srcs = ["logging.cc"], ++ hdrs = ["logging.h"], ++ deps = [ ++ "@com_google_absl//absl/base:log_severity", ++ "@com_google_absl//absl/log", ++ "@com_google_absl//absl/log:globals", ++ "@llvm-project//llvm:Support", ++ "@llvm-project//mlir:AsmParser", ++ "@llvm-project//mlir:DataLayoutInterfaces", ++ "@llvm-project//mlir:IR", ++ ], ++) ++ ++cc_library( ++ name = "utils", ++ srcs = ["utils.cc"], ++ hdrs = ["utils.h"], ++ deps = [ ++ "@com_google_absl//absl/status:statusor", ++ "@com_google_absl//absl/types:span", ++ "@llvm-project//mlir:IR", ++ "@llvm-project//mlir:Support", ++ "@mlx//:mlx", ++ "@stablehlo//:reference_api", ++ "//xla/mlir/utils:type_util", ++ "//xla:literal", ++ "//xla:shape_util", ++ "//xla:util", ++ ":logging", ++ ], ++) ++ ++xla_cc_test( ++ name = "plugin_pjrt_test", ++ srcs = ["plugin_pjrt_test.cc"], ++ deps = [ ++ ":client_c_pjrt", ++ ":client_cpp_pjrt", ++ "//xla/pjrt:pjrt_client_test_common", ++ "//xla/pjrt/c:pjrt_c_api_test_common", ++ "//xla/pjrt/c:pjrt_c_api_wrapper_impl", ++ "@com_google_googletest//:gtest_main", ++ ], ++) ++ ++xla_cc_binary( ++ name = "stablehlo_mlx_plugin.so", ++ linkshared = True, ++ linkopts = [ ++ "-Wl,-exported_symbol,_GetPjrtApi", ++ "-Wl,-undefined,error", ++ ], ++ deps = [ ++ ":client_c_pjrt", ++ ], ++) +\ No newline at end of file +diff --git a/xla/pjrt/plugin/mlx/buffer.cc b/xla/pjrt/plugin/mlx/buffer.cc +new file mode 100644 +index 0000000000..f87671b3d5 +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/buffer.cc +@@ -0,0 +1,372 @@ ++/* Copyright 2024 The OpenXLA Authors. ++Licensed under the Apache License, Version 2.0 (the "License"); ++you may not use this file except in compliance with the License. ++You may obtain a copy of the License at ++ http://www.apache.org/licenses/LICENSE-2.0 ++Unless required by applicable law or agreed to in writing, software ++distributed under the License is distributed on an "AS IS" BASIS, ++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++See the License for the specific language governing permissions and ++limitations under the License. ++==============================================================================*/ ++ ++#include "xla/pjrt/plugin/mlx/buffer.h" ++ ++#include ++#include ++#include ++#include ++#include ++ ++#include "absl/functional/any_invocable.h" ++#include "absl/log/log.h" ++#include "absl/status/statusor.h" ++#include "absl/types/span.h" ++#include "llvm/Support/Casting.h" ++#include "llvm/Support/raw_ostream.h" ++#include "mlir/AsmParser/AsmParser.h" ++#include "mlir/IR/Attributes.h" ++#include "mlir/IR/Builders.h" ++#include "mlir/IR/BuiltinAttributes.h" ++#include "mlir/IR/BuiltinTypeInterfaces.h" ++#include "mlir/IR/MLIRContext.h" ++#include "mlx/mlx.h" ++#include "mlir/Support/DebugStringHelper.h" ++#include "xla/hlo/translate/hlo_to_mhlo/hlo_utils.h" ++#include "xla/hlo/translate/mhlo_to_hlo/literal_exporter.h" ++#include "xla/hlo/translate/mhlo_to_hlo/type_to_shape.h" ++#include "xla/literal.h" ++#include "xla/pjrt/pjrt_client.h" ++#include "xla/pjrt/pjrt_compiler.h" ++#include "xla/pjrt/pjrt_future.h" ++#include "xla/pjrt/plugin/mlx/logging.h" ++#include "xla/pjrt/plugin/mlx/utils.h" ++#include "xla/shape.h" ++#include "xla/util.h" ++ ++namespace mx = mlx::core; ++namespace mlir::stablehlo { ++ ++using xla::MutableLiteralBase; ++using xla::PjRtBuffer; ++using xla::PjRtClient; ++using xla::PjRtDevice; ++using xla::PjRtFuture; ++using xla::PjRtMemorySpace; ++using xla::PjRtPlatformId; ++using xla::Shape; ++ ++#define UNIMPLEMENTED(name) \ ++ xla::Unimplemented("MlxPjrtBuffer::" #name " is not implemented") ++ ++class MlxPjrtBuffer : public PjRtBuffer { ++ public: ++ MlxPjrtBuffer(mx::array array, const Shape& shape, ++ PjRtMemorySpace* memory_space) ++ : xla::PjRtBuffer(), ++ context_(), ++ buffer_(), ++ array_(array), ++ shape_(shape), ++ memory_space_(memory_space) { ++ // TRACE_ME_MEMBER; ++ } ++ ++ class MlirClonedExternalReference : public ExternalReference { ++ public: ++ explicit MlirClonedExternalReference(PjRtBuffer* buffer, ++ PjRtMemorySpace* memory_space) ++ : buffer_() { ++ // TRACE_ME_MEMBER; ++ auto mlir_buffer = GetAttributeFromBuffer(buffer); ++ if (!mlir_buffer.ok()) { ++ LOG(ERROR) << "Could not get attribute from buffer: " ++ << mlir_buffer.status(); ++ } ++ ++ auto array = GetArrayFromBuffer(buffer); ++ if (!array.ok()) { ++ LOG(ERROR) << "Could not get attribute from buffer: " << array.status(); ++ } ++ buffer_ = CreateMlirBufferFromAttribute( ++ array.value(), mlir_buffer.value(), memory_space); ++ data_ptr_ = (void*)mlir_buffer.value().getRawData().data(); ++ } ++ ++ private: ++ std::unique_ptr buffer_; ++ }; ++ ++ // All buffers are managed by the MLIR Context ++ ~MlxPjrtBuffer() override = default; ++ ++ MlxPjrtBuffer(const MlxPjrtBuffer&) = delete; ++ MlxPjrtBuffer(MlxPjrtBuffer&&) = delete; ++ MlxPjrtBuffer& operator=(const MlxPjrtBuffer&) = delete; ++ MlxPjrtBuffer& operator=(MlxPjrtBuffer&&) = delete; ++ ++ static std::unique_ptr CreateFromLiteral( ++ mx::array array, const xla::LiteralSlice& literal, ++ xla::PjRtMemorySpace* memory_space) { ++ TRACE_ME; ++ auto buffer = ++ std::make_unique(array, literal.shape(), memory_space); ++ mlir::Builder builder(&buffer->context_); ++ auto attr = xla::CreateDenseElementsAttrFromLiteral(literal, builder); ++ if (!attr.ok()) { ++ LOG(ERROR) << "Could not create dense elements attr from literal: " ++ << attr.status(); ++ return nullptr; ++ } ++ buffer->buffer_ = attr.value(); ++ return buffer; ++ } ++ ++ static std::unique_ptr CreateFromAttribute( ++ mx::array array, DenseElementsAttr attr, ++ xla::PjRtMemorySpace* memory_space) { ++ TRACE_ME; ++ ++ // MLIR type to xla shape: ++ Shape shape = xla::TypeToShape(attr.getType()); ++ auto buffer = std::make_unique(array, shape, memory_space); ++ buffer->buffer_ = CloneIntoContext(attr, buffer->context_); ++ return buffer; ++ } ++ ++ const Shape& on_device_shape() const override { ++ // TRACE_ME_MEMBER; ++ return shape_; ++ } ++ absl::StatusOr logical_on_device_shape() override { ++ // TRACE_ME_MEMBER; ++ return shape_; ++ } ++ ++ PjRtPlatformId platform_id() const { ++ // TRACE_ME_MEMBER; ++ return client()->platform_id(); ++ } ++ ++ absl::string_view platform_name() const { ++ // TRACE_ME_MEMBER; ++ return client()->platform_name(); ++ } ++ ++ bool IsEmptyTuple() const { ++ // TRACE_ME_MEMBER; ++ return shape_.IsTuple() && shape_.tuple_shapes().empty(); ++ } ++ ++ // Buffer knows device + client per older design, should only need ++ // memory_space. ++ PjRtMemorySpace* memory_space() const override { ++ // TRACE_ME_MEMBER; ++ return memory_space_; ++ } ++ PjRtDevice* device() const override { ++ // TRACE_ME_MEMBER; ++ return memory_space_->devices().front(); ++ } ++ PjRtClient* client() const override { ++ // TRACE_ME_MEMBER; ++ return memory_space_->client(); ++ } ++ ++ absl::StatusOr> ++ AcquireExternalReference() override { ++ // TRACE_ME_MEMBER; ++ return std::make_unique(this, memory_space_); ++ } ++ ++ xla::PjRtFuture<> ToLiteral(xla::MutableLiteralBase* literal) override { ++ // TRACE_ME_MEMBER; ++ if (IsEmptyTuple()) { ++ return PjRtFuture<>( ++ xla::InvalidArgument("ToLiteral called on empty tuple")); ++ } ++ ++ absl::StatusOr to_copy = ++ mhlo::CreateLiteralFromAttribute(buffer_, {}); ++ if (!to_copy.ok()) return PjRtFuture<>(to_copy.status()); ++ ++ // Synchronous! To make async, make the buffer, start the copy, and return a ++ // future that is ready when the copy is done. ++ auto status = literal->CopyFrom(to_copy.value()); ++ if (!status.ok()) return PjRtFuture<>(status); ++ return PjRtFuture<>(absl::OkStatus()); ++ } ++ ++ PjRtFuture<> LazyToLiteral( ++ absl::AnyInvocable() &&> generator) ++ override { ++ // TRACE_ME_MEMBER; ++ auto buffer = std::move(generator)(); ++ if (!buffer.ok()) return PjRtFuture<>(buffer.status()); ++ return ToLiteral(buffer.value()); ++ } ++ ++ absl::StatusOr GetOnDeviceSizeInBytes() const override { ++ // This is needed by AcquireExternalReference, for framework figuring out ++ // how to read the underlying buffer data. ++ // TRACE_ME_MEMBER; ++ if (!buffer_) return 0; ++ return buffer_.getRawData().size(); ++ } ++ ++ PjRtFuture<> CopyRawToHost(void* dst, int64_t offset, ++ int64_t transfer_size) override { ++ // TRACE_ME_MEMBER; ++ return PjRtFuture<>(UNIMPLEMENTED(CopyRawToHost)); ++ } ++ ++ absl::StatusOr> ++ ReleaseDeviceMemoryOwnership(bool wait_for_operations_to_complete) override { ++ // TRACE_ME_MEMBER; ++ auto external_ref = AcquireExternalReference(); ++ Delete(); ++ return external_ref; ++ } ++ ++ // Remove the buffer if deleted. ++ // Note: deleted and uninitialized appear the same in this scenario. ++ // Consider changing to mlir::NoneType when deleted. ++ void Delete() override { ++ // TRACE_ME_MEMBER; ++ buffer_ = {}; ++ is_deleted_ = true; ++ } ++ ++ bool IsDeleted() override { ++ // TRACE_ME_MEMBER; ++ return !buffer_ && is_deleted_; ++ } ++ ++ absl::StatusOr> CopyToDevice( ++ xla::PjRtDevice* dst_device) { ++ // TRACE_ME_MEMBER; ++ return CopyToMemorySpace( ++ dst_device->default_memory_space().value_or(nullptr)); ++ } ++ ++ absl::StatusOr> CopyToMemorySpace( ++ xla::PjRtMemorySpace* dst_memory_space) override { ++ // TRACE_ME_MEMBER; ++ return CreateMlirBufferFromAttribute(array_, buffer_, dst_memory_space); ++ } ++ ++ void CopyToRemoteDevice( ++ xla::PjRtFuture serialized_descriptor, ++ xla::PjRtBuffer::RemoteSendCallback on_done) override { ++ // TRACE_ME_MEMBER; ++ on_done(UNIMPLEMENTED(CopyToRemoteDevice), false); ++ } ++ ++ xla::PjRtFuture<> GetReadyFuture() override { ++ // TRACE_ME_MEMBER; ++ // Synchronous! To make async, have the device make a buffer with a ready ++ // future that is ready when the computation is done / buffer is ready. ++ return PjRtFuture<>(absl::OkStatus()); ++ } ++ ++ bool IsOnCpu() const override { ++ // If buffer is on CPU, it will be shared with framework via ++ // GetExternalReference, lse it is copied back to host. ++ // Since we are using reference interpreter, we are running on CPU in a ++ // shared memory space. ++ // TRACE_ME_MEMBER; ++ return false; ++ } ++ ++ mlir::DenseElementsAttr GetBufferAttribute() const { return buffer_; } ++ ++ mx::array GetArray() { return array_; } ++ ++ private: ++ MLIRContext context_; ++ mlir::DenseElementsAttr buffer_; ++ bool is_deleted_ = false; ++ mx::array array_; ++ ++ xla::Shape shape_; ++ PjRtMemorySpace* memory_space_; ++}; ++ ++std::unique_ptr CreateMlirBufferFromMlxArray( ++ mx::array array, xla::PjRtMemorySpace* memory_space) { ++ TRACE_ME; ++ std::vector span_shape(array.shape().begin(), array.shape().end()); ++ auto shape = xla::ShapeUtil::MakeShape( ++ utils::dtype::asXlaPrimitiveType(array.dtype()), ++ absl::Span(span_shape.data(), span_shape.size())); ++ auto literal = xla::BorrowingLiteral(array.data(), shape); ++ return MlxPjrtBuffer::CreateFromLiteral(array, literal, memory_space); ++} ++ ++std::unique_ptr CreateMlirBufferFromLiteral( ++ const xla::LiteralSlice& literal, xla::PjRtMemorySpace* memory_space) { ++ TRACE_ME; ++ auto maybe_array = utils::array::fromHostLiteral(literal); ++ return MlxPjrtBuffer::CreateFromLiteral(maybe_array.value(), literal, ++ memory_space); ++} ++ ++std::unique_ptr CreateMlirBufferFromAttribute( ++ mx::array array, DenseElementsAttr attr, ++ xla::PjRtMemorySpace* memory_space) { ++ TRACE_ME; ++ return MlxPjrtBuffer::CreateFromAttribute(array, attr, memory_space); ++} ++ ++std::unique_ptr CreateMlirBufferUninitialized( ++ const xla::Shape& shape, PjRtMemorySpace* memory_space) { ++ TRACE_ME; ++ // TODO (@cryptodeal): C API doesn't implement this, but ++ // we'll want to ensure when the Buffer is initialized, ++ // the resulting array is correct shape/dtype. ++ ++ // Init empty array of dtype ++ auto array = mx::array({}); ++ return std::make_unique(array, shape, memory_space); ++} ++ ++absl::StatusOr GetAttributeFromBuffer( ++ xla::PjRtBuffer* buffer) { ++ TRACE_ME; ++ if (buffer == nullptr || buffer->IsDeleted()) { ++ return xla::InvalidArgument("Buffer is null or deleted"); ++ } ++ auto mlir_buffer = dynamic_cast(buffer); ++ if (mlir_buffer->IsDeleted()) { ++ return xla::InvalidArgument("Buffer is deleted"); ++ } ++ if (mlir_buffer == nullptr) { ++ return xla::InvalidArgument("Buffer is not a MlxPjrtBuffer"); ++ } ++ return mlir_buffer->GetBufferAttribute(); ++} ++ ++absl::StatusOr GetArrayFromBuffer(xla::PjRtBuffer* buffer) { ++ TRACE_ME; ++ if (buffer == nullptr || buffer->IsDeleted()) { ++ return xla::InvalidArgument("Buffer is null or deleted"); ++ } ++ auto mlir_buffer = dynamic_cast(buffer); ++ if (mlir_buffer->IsDeleted()) { ++ return xla::InvalidArgument("Buffer is deleted"); ++ } ++ if (mlir_buffer == nullptr) { ++ return xla::InvalidArgument("Buffer is not a MlxPjrtBuffer"); ++ } ++ ++ return mlir_buffer->GetArray(); ++} ++ ++DenseElementsAttr CloneIntoContext(DenseElementsAttr attr, ++ MLIRContext& context) { ++ Type type = mlir::parseType(mlir::debugString(attr.getType()), &context); ++ return DenseElementsAttr::getFromRawBuffer(llvm::cast(type), ++ attr.getRawData()); ++} ++ ++} // namespace mlir::stablehlo +\ No newline at end of file +diff --git a/xla/pjrt/plugin/mlx/buffer.h b/xla/pjrt/plugin/mlx/buffer.h +new file mode 100644 +index 0000000000..1740f3c779 +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/buffer.h +@@ -0,0 +1,43 @@ ++/* Copyright 2024 The OpenXLA Authors. ++Licensed under the Apache License, Version 2.0 (the "License"); ++you may not use this file except in compliance with the License. ++You may obtain a copy of the License at ++ http://www.apache.org/licenses/LICENSE-2.0 ++Unless required by applicable law or agreed to in writing, software ++distributed under the License is distributed on an "AS IS" BASIS, ++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++See the License for the specific language governing permissions and ++limitations under the License. ++==============================================================================*/ ++ ++#ifndef XLA_PJRT_PLUGIN_STABLEHLO_MLX_BUFFER_H_ ++#define XLA_PJRT_PLUGIN_STABLEHLO_MLX_BUFFER_H_ ++ ++#include ++ ++#include "mlir/IR/BuiltinAttributes.h" ++#include "mlx/mlx.h" ++#include "xla/literal.h" ++#include "xla/pjrt/pjrt_client.h" ++ ++namespace mx = mlx::core; ++ ++namespace mlir::stablehlo { ++std::unique_ptr CreateMlirBufferFromMlxArray( ++ mx::array array, xla::PjRtMemorySpace* memory_space); ++std::unique_ptr CreateMlirBufferFromLiteral( ++ const xla::LiteralSlice& literal, xla::PjRtMemorySpace* memory_space); ++std::unique_ptr CreateMlirBufferFromAttribute( ++ mx::array array, DenseElementsAttr attribute, ++ xla::PjRtMemorySpace* memory_space); ++std::unique_ptr CreateMlirBufferUninitialized( ++ const xla::Shape& shape, xla::PjRtMemorySpace* memory_space); ++absl::StatusOr GetAttributeFromBuffer( ++ xla::PjRtBuffer* buffer); ++absl::StatusOr GetArrayFromBuffer(xla::PjRtBuffer* buffer); ++DenseElementsAttr CloneIntoContext(DenseElementsAttr attr, ++ MLIRContext& context); ++ ++} // namespace mlir::stablehlo ++ ++#endif // XLA_PJRT_PLUGIN_STABLEHLO_MLX_BUFFER_H_ +\ No newline at end of file +diff --git a/xla/pjrt/plugin/mlx/client_c_pjrt.cc b/xla/pjrt/plugin/mlx/client_c_pjrt.cc +new file mode 100644 +index 0000000000..6a0fcbcbcf +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/client_c_pjrt.cc +@@ -0,0 +1,66 @@ ++/* Copyright 2024 The OpenXLA Authors. ++Licensed under the Apache License, Version 2.0 (the "License"); ++you may not use this file except in compliance with the License. ++You may obtain a copy of the License at ++ http://www.apache.org/licenses/LICENSE-2.0 ++Unless required by applicable law or agreed to in writing, software ++distributed under the License is distributed on an "AS IS" BASIS, ++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++See the License for the specific language governing permissions and ++limitations under the License. ++==============================================================================*/ ++ ++#include "xla/pjrt/plugin/mlx/client_c_pjrt.h" ++ ++#include ++#include ++#include ++ ++#include "absl/status/status.h" ++#include "xla/pjrt/c/pjrt_c_api.h" ++#include "xla/pjrt/c/pjrt_c_api_layouts_extension.h" ++#include "xla/pjrt/c/pjrt_c_api_wrapper_impl.h" ++#include "xla/pjrt/pjrt_client.h" ++#include "xla/pjrt/plugin/mlx/client_cpp_pjrt.h" ++ ++namespace mlir::stablehlo { ++ ++// Create my C++ client. Called by the C API and is the glue between the C API ++// and the C++ API. ++PJRT_Error* PJRT_StablehloMlxClient_Create(PJRT_Client_Create_Args* args) { ++ std::unique_ptr client = CreateStablehloMlxPjrtClient(); ++ args->client = pjrt::CreateWrapperClient(std::move(client)); ++ // printf("Creating PJRT Client from client\n"); ++ return nullptr; ++} ++ ++PJRT_Error* PJRT_StablehloMlxExecuteContext_Create( ++ PJRT_ExecuteContext_Create_Args* args) { ++ return new PJRT_Error{absl::UnimplementedError( ++ "ExecuteContext not supported for StablehloMlx execution.")}; ++} ++ ++PJRT_Error* PJRT_StablehloMlxDeviceTopology_Create( ++ PJRT_TopologyDescription_Create_Args* args) { ++ return new PJRT_Error{absl::UnimplementedError( ++ "Topology not supported for StablehloMlx compilation.")}; ++} ++ ++} // namespace mlir::stablehlo ++ ++const PJRT_Api* GetPjrtApi() { ++ // printf("C++ Calling GetPjrtApi"); ++ static PJRT_Layouts_Extension layouts_extension = ++ pjrt::CreateLayoutsExtension(nullptr); ++ ++ static const PJRT_Api pjrt_api = pjrt::CreatePjrtApi( ++ mlir::stablehlo::PJRT_StablehloMlxClient_Create, ++ mlir::stablehlo::PJRT_StablehloMlxExecuteContext_Create, ++ mlir::stablehlo::PJRT_StablehloMlxDeviceTopology_Create, ++ pjrt::PJRT_Plugin_Initialize_NoOp, ++ reinterpret_cast(&layouts_extension), ++ pjrt::PJRT_Plugin_Attributes_Xla); ++ ++ // printf("stablehlo_mlx client called GetPjrtApi\n"); ++ return &pjrt_api; ++} +\ No newline at end of file +diff --git a/xla/pjrt/plugin/mlx/client_c_pjrt.h b/xla/pjrt/plugin/mlx/client_c_pjrt.h +new file mode 100644 +index 0000000000..cc56626953 +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/client_c_pjrt.h +@@ -0,0 +1,24 @@ ++/* Copyright 2024 The OpenXLA Authors. ++Licensed under the Apache License, Version 2.0 (the "License"); ++you may not use this file except in compliance with the License. ++You may obtain a copy of the License at ++ http://www.apache.org/licenses/LICENSE-2.0 ++Unless required by applicable law or agreed to in writing, software ++distributed under the License is distributed on an "AS IS" BASIS, ++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++See the License for the specific language governing permissions and ++limitations under the License. ++==============================================================================*/ ++ ++#ifndef XLA_PJRT_PLUGIN_STABLEHLO_MLX_CLIENT_C_PJRT_H_ ++#define XLA_PJRT_PLUGIN_STABLEHLO_MLX_CLIENT_C_PJRT_H_ ++ ++#include "xla/pjrt/c/pjrt_c_api.h" ++ ++extern "C" { ++ ++// Does not pass ownership of returned PJRT_Api* to caller. ++const PJRT_Api* GetPjrtApi(); ++} ++ ++#endif // XLA_PJRT_PLUGIN_STABLEHLO_MLX_CLIENT_C_PJRT_H_ +\ No newline at end of file +diff --git a/xla/pjrt/plugin/mlx/client_cpp_pjrt.cc b/xla/pjrt/plugin/mlx/client_cpp_pjrt.cc +new file mode 100644 +index 0000000000..ca5caa92ea +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/client_cpp_pjrt.cc +@@ -0,0 +1,303 @@ ++/* Copyright 2024 The OpenXLA Authors. ++Licensed under the Apache License, Version 2.0 (the "License"); ++you may not use this file except in compliance with the License. ++You may obtain a copy of the License at ++ http://www.apache.org/licenses/LICENSE-2.0 ++Unless required by applicable law or agreed to in writing, software ++distributed under the License is distributed on an "AS IS" BASIS, ++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++See the License for the specific language governing permissions and ++limitations under the License. ++==============================================================================*/ ++ ++#include "xla/pjrt/plugin/mlx/client_cpp_pjrt.h" ++ ++#include ++#include ++#include ++#include ++#include ++ ++#include "absl/status/statusor.h" ++#include "absl/strings/str_format.h" ++#include "absl/strings/string_view.h" ++#include "absl/types/span.h" ++#include "mlx/mlx.h" ++#include "xla/literal.h" ++#include "xla/pjrt/host_memory_spaces.h" ++#include "xla/pjrt/pjrt_client.h" ++#include "xla/pjrt/pjrt_common.h" ++#include "xla/pjrt/pjrt_compiler.h" ++#include "xla/pjrt/plugin/mlx/buffer.h" ++#include "xla/pjrt/plugin/mlx/compile.h" ++#include "xla/pjrt/plugin/mlx/device.h" ++#include "xla/pjrt/plugin/mlx/executable.h" ++#include "xla/pjrt/plugin/mlx/logging.h" ++#include "xla/pjrt/plugin/mlx/utils.h" ++#include "xla/shape_util.h" ++#include "xla/util.h" ++#include "tsl/platform/fingerprint.h" ++#include "tsl/platform/statusor.h" ++ ++namespace mx = mlx::core; ++ ++#define UNIMPLEMENTED(name) \ ++ xla::Unimplemented("MlxPjrtBuffer::" #name " is not implemented") ++ ++namespace mlir::stablehlo { ++ ++const xla::PjRtPlatformId kStablehloMlxBackendId = ++ tsl::Fingerprint64(kStablehloMlxBackendName); ++ ++class StablehloMlxPjrtClient : public xla::PjRtClient { ++ public: ++ StablehloMlxPjrtClient() ++ : xla::PjRtClient(), ++ owned_devices_(), ++ devices_(), ++ owned_memory_space_(), ++ memory_space_(nullptr) { ++ // Init device and memory space. ++ // TRACE_ME_MEMBER; ++ owned_devices_.push_back(GetStablehloMlxDevice(this)); ++ devices_.push_back(owned_devices_.back().get()); ++ owned_memory_space_ = std::make_unique( ++ /*id=*/0, devices_.front()); ++ memory_space_ = owned_memory_space_.get(); ++ AttachStablehloMlxMemorySpace(devices_.front(), memory_space_); ++ // mx::disable_compile(); ++ } ++ ++ ~StablehloMlxPjrtClient() override { ++ // mx::metal::stop_capture(); ++ }; ++ ++ absl::string_view platform_name() const override { ++ // TRACE_ME_MEMBER; ++ return kStablehloMlxBackendName; ++ } ++ int process_index() const override { ++ // TRACE_ME_MEMBER; ++ return 0; ++ } ++ ++ int device_count() const override { ++ // TRACE_ME_MEMBER; ++ return devices_.size(); ++ } ++ ++ int addressable_device_count() const override { ++ // TRACE_ME_MEMBER; ++ return devices_.size(); ++ } ++ ++ absl::Span devices() const override { ++ // TRACE_ME_MEMBER; ++ return devices_; ++ } ++ absl::Span addressable_devices() const override { ++ // TRACE_ME_MEMBER; ++ return devices_; ++ } ++ ++ absl::Span memory_spaces() const override { ++ // TRACE_ME_MEMBER; ++ return absl::MakeSpan(&memory_space_, 1); ++ } ++ ++ // Return an ID that identifies the platform via tsl fingerprint. ++ xla::PjRtPlatformId platform_id() const override { ++ // TRACE_ME_MEMBER; ++ return kStablehloMlxBackendId; ++ } ++ ++ // Returns a string containing human-readable, platform-specific version ++ // info (e.g. the CUDA version on GPU or libtpu version on Cloud TPU). ++ absl::string_view platform_version() const override { ++ // TRACE_ME_MEMBER; ++ return "StableHLO MLX v0.1"; ++ } ++ ++ ///////////// ++ // Device ++ ///////////// ++ ++ // Lookup any PjRtDevice for a given PjRtDevice::id(). ++ // TODO: Should this be a base class? I.e. why doesn't the base client have ++ // a vector a device pointers? ++ absl::StatusOr LookupDevice( ++ xla::PjRtGlobalDeviceId global_device_id) const override { ++ // TRACE_ME_MEMBER; ++ for (auto device : devices_) { ++ if (device->global_device_id() == global_device_id) { ++ return device; ++ } ++ } ++ // TODO: This error should be a base class method since its used in tests. ++ return xla::InvalidArgument("No matching device found for device_id %d", ++ global_device_id.value()); ++ } ++ ++ absl::StatusOr LookupAddressableDevice( ++ xla::PjRtLocalDeviceId local_device_id) const override { ++ // TRACE_ME_MEMBER; ++ ++ for (auto* device : addressable_devices()) { ++ if (local_device_id == device->local_device_id()) { ++ return device; ++ } ++ } ++ return xla::InvalidArgument( ++ "No matching device found for local_device_id %d", ++ local_device_id.value()); ++ } ++ ++ absl::StatusOr GetDefaultDeviceAssignment( ++ int num_replicas, int num_partitions) const override { ++ // TRACE_ME_MEMBER; ++ xla::DeviceAssignment assignment(num_replicas, num_partitions); ++ for (int64_t i = 0; i < num_replicas; ++i) { ++ for (int64_t j = 0; j < num_partitions; ++j) { ++ auto idx = (i + (j * num_replicas)) % devices_.size(); ++ assignment(i, j) = devices_[idx]->global_device_id().value(); ++ } ++ } ++ return assignment; ++ } ++ ++ ///////////////// ++ // Buffer methods ++ ///////////////// ++ ++ absl::StatusOr> CreateErrorBuffer( ++ absl::Status error, const xla::Shape& shape, xla::PjRtDevice* device) { ++ // Prefer memory space implementation, device holding buffer is ++ // deprecated. ++ return CreateErrorBuffer(error, shape, ++ device->default_memory_space().value_or(nullptr)); ++ } ++ ++ absl::StatusOr> CreateErrorBuffer( ++ absl::Status error, const xla::Shape& shape, ++ xla::PjRtMemorySpace* memory) { ++ return UNIMPLEMENTED(CreateErrorBuffer); ++ } ++ ++ absl::StatusOr> CreateUninitializedBuffer( ++ const xla::Shape& shape, xla::PjRtDevice* device) { ++ // TRACE_ME_MEMBER; ++ return CreateMlirBufferUninitialized( ++ shape, device->default_memory_space().value_or(nullptr)); ++ } ++ ++ absl::StatusOr> ++ CreateBuffersForAsyncHostToDevice(absl::Span shapes, ++ xla::PjRtDevice* device) { ++ // TRACE_ME_MEMBER; ++ return CreateBuffersForAsyncHostToDevice( ++ shapes, device->default_memory_space().value_or(nullptr)); ++ } ++ ++ absl::StatusOr> ++ CreateBuffersForAsyncHostToDevice(absl::Span shapes, ++ xla::PjRtMemorySpace* memory_space) { ++ // TRACE_ME_MEMBER; ++ return UNIMPLEMENTED(CreateBuffersForAsyncHostToDevice); ++ } ++ ++ absl::StatusOr> BufferFromHostBuffer( ++ const void* data, xla::PrimitiveType type, absl::Span dims, ++ std::optional> byte_strides, ++ HostBufferSemantics host_buffer_semantics, ++ absl::AnyInvocable on_done_with_host_buffer, ++ xla::PjRtDevice* device) { ++ // TRACE_ME_MEMBER; ++ return BufferFromHostBuffer( ++ data, type, dims, byte_strides, host_buffer_semantics, ++ std::move(on_done_with_host_buffer), ++ device->default_memory_space().value_or(nullptr), nullptr); ++ } ++ ++ absl::StatusOr> BufferFromHostBuffer( ++ const void* data, xla::PrimitiveType type, absl::Span dims, ++ std::optional> byte_strides, ++ HostBufferSemantics host_buffer_semantics, ++ absl::AnyInvocable on_done_with_host_buffer, ++ xla::PjRtDevice* device, const xla::Layout* device_layout) { ++ // TRACE_ME_MEMBER; ++ return BufferFromHostBuffer( ++ data, type, dims, byte_strides, host_buffer_semantics, ++ std::move(on_done_with_host_buffer), ++ device->default_memory_space().value_or(nullptr), device_layout); ++ } ++ ++ absl::StatusOr> BufferFromHostBuffer( ++ const void* data, xla::PrimitiveType type, absl::Span dims, ++ std::optional> byte_strides, ++ HostBufferSemantics host_buffer_semantics, ++ absl::AnyInvocable on_done_with_host_buffer, ++ xla::PjRtMemorySpace* memory_space, const xla::Layout* device_layout) { ++ // TRACE_ME_MEMBER; ++ TF_ASSIGN_OR_RETURN( ++ mx::array array_buffer, ++ utils::array::fromHostBuffer(data, dims, byte_strides, type)); ++ auto buffer = CreateMlirBufferFromMlxArray(array_buffer, memory_space); ++ if (on_done_with_host_buffer) { ++ // If host is awaiting the result, must call this function. ++ std::move(on_done_with_host_buffer)(); ++ on_done_with_host_buffer = nullptr; ++ } ++ return buffer; ++ } ++ ++ absl::StatusOr> BufferFromHostLiteral( ++ const xla::LiteralSlice& literal, xla::PjRtDevice* device) { ++ // TRACE_ME_MEMBER; ++ return CreateMlirBufferFromLiteral( ++ literal, device->default_memory_space().value_or(nullptr)); ++ } ++ ++ absl::StatusOr> BufferFromHostLiteral( ++ const xla::LiteralSlice& literal, xla::PjRtMemorySpace* memory_space) { ++ // TRACE_ME_MEMBER; ++ return CreateMlirBufferFromLiteral(literal, memory_space); ++ } ++ ++ /////////// ++ // Compile ++ /////////// ++ absl::StatusOr> CompileAndLoad( ++ mlir::ModuleOp module, xla::CompileOptions options) override { ++ // TRACE_ME_MEMBER; ++ return mlir::stablehlo::StablehloMlxCompile( ++ module, GetDefaultDeviceAssignment(1, devices_.size()).value(), this); ++ } ++ ++ // Compile `computation` with given `options`. ++ absl::StatusOr> CompileAndLoad( ++ const xla::XlaComputation& computation, ++ xla::CompileOptions options) override { ++ // TRACE_ME_MEMBER; ++ return mlir::stablehlo::StablehloMlxCompile( ++ computation.proto(), ++ GetDefaultDeviceAssignment(1, devices_.size()).value(), this); ++ } ++ ++ private: ++ std::vector> owned_devices_; ++ std::vector devices_; ++ std::unique_ptr owned_memory_space_; ++ xla::PjRtMemorySpace* memory_space_; ++ // TODO(@cryptodeal): shared cache for optimized ops ++ // with the same hash value between executables. ++ // std::mutex cache_mutex_; ++ // std::unordered_map mlx_cache_; ++}; // end class ++ ++std::unique_ptr CreateStablehloMlxPjrtClient() { ++ // mx::metal::start_capture("mlx_trace.gputrace"); ++ SetupLogLevelFromEnv(); ++ return std::make_unique(); ++} ++ ++} // namespace mlir::stablehlo +\ No newline at end of file +diff --git a/xla/pjrt/plugin/mlx/client_cpp_pjrt.h b/xla/pjrt/plugin/mlx/client_cpp_pjrt.h +new file mode 100644 +index 0000000000..8c90942f0c +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/client_cpp_pjrt.h +@@ -0,0 +1,30 @@ ++/* Copyright 2024 The OpenXLA Authors. ++Licensed under the Apache License, Version 2.0 (the "License"); ++you may not use this file except in compliance with the License. ++You may obtain a copy of the License at ++ http://www.apache.org/licenses/LICENSE-2.0 ++Unless required by applicable law or agreed to in writing, software ++distributed under the License is distributed on an "AS IS" BASIS, ++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++See the License for the specific language governing permissions and ++limitations under the License. ++==============================================================================*/ ++ ++#ifndef XLA_PJRT_PLUGIN_STABLEHLO_MLX_CLIENT_CPP_PJRT_H_ ++#define XLA_PJRT_PLUGIN_STABLEHLO_MLX_CLIENT_CPP_PJRT_H_ ++ ++#include ++ ++#include "xla/pjrt/pjrt_client.h" ++#include "xla/pjrt/pjrt_compiler.h" ++ ++namespace mlir::stablehlo { ++ ++inline constexpr char kStablehloMlxBackendName[] = "stablehlo_mlx"; ++extern const xla::PjRtPlatformId kStablehloMlxBackendId; ++ ++std::unique_ptr CreateStablehloMlxPjrtClient(); ++ ++} // namespace mlir::stablehlo ++ ++#endif // XLA_PJRT_PLUGIN_STABLEHLO_MLX_CLIENT_CPP_PJRT_H_ +\ No newline at end of file +diff --git a/xla/pjrt/plugin/mlx/compile.cc b/xla/pjrt/plugin/mlx/compile.cc +new file mode 100644 +index 0000000000..eec6b037b2 +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/compile.cc +@@ -0,0 +1,1563 @@ ++#include "xla/pjrt/plugin/mlx/compile.h" ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++// TODO(@cryptodeal): might need to update `BUILD` ++#include "mlir/IR/Visitors.h" ++ ++#include "mlx/compile_impl.h" ++#include "absl/log/log.h" ++#include "absl/status/statusor.h" ++#include "llvm/ADT/TypeSwitch.h" ++#include "mlir/Bytecode/BytecodeWriter.h" ++#include "mlir/Dialect/Func/IR/FuncOps.h" ++#include "mlir/IR/AsmState.h" ++#include "mlir/IR/BuiltinAttributes.h" ++#include "mlir/IR/MLIRContext.h" ++#include "mlir/IR/Operation.h" ++#include "mlir/IR/OperationSupport.h" ++#include "mlir/IR/OwningOpRef.h" ++#include "mlir/Interfaces/DataLayoutInterfaces.h" ++#include "mlir/Pass/PassManager.h" ++#include "mlir/Support/LLVM.h" ++#include "stablehlo/reference/Api.h" ++#include "stablehlo/transforms/optimization/Passes.h" ++#include "xla/hlo/translate/stablehlo.h" ++#include "xla/mlir/utils/error_util.h" ++#include "xla/mlir/utils/type_util.h" ++#include "xla/mlir_hlo/mhlo/transforms/passes.h" ++#include "xla/mlir_hlo/stablehlo_ext/transforms/passes.h" ++#include "xla/pjrt/pjrt_client.h" ++#include "xla/pjrt/pjrt_executable.h" ++#include "xla/pjrt/pjrt_future.h" ++#include "xla/pjrt/plugin/mlx/buffer.h" ++#include "xla/pjrt/plugin/mlx/logging.h" ++#include "xla/service/computation_placer.h" ++#include "tsl/platform/status.h" ++#include "tsl/platform/statusor.h" ++ ++namespace compile { ++ ++namespace detail { ++std::pair compile( ++ std::function(const std::vector&)> fun, ++ bool shapeless /* false */) { ++ if (mx::detail::skip_compile()) { ++ return {0, fun}; ++ } ++ auto fun_id = mx::detail::get_function_address(fun); ++ if (fun_id) { ++ // If the function has an addressable target then no need to manage it's ++ // lifetime ++ return {fun_id, mx::detail::compile(std::move(fun), fun_id, shapeless)}; ++ } else { ++ auto pfun = std::shared_ptr< ++ std::function(const std::vector&)>>( ++ new std::function( ++ const std::vector&)>{fun}, ++ [](auto* p) { ++ mx::detail::compile_erase(reinterpret_cast(p)); ++ delete p; ++ }); ++ fun_id = reinterpret_cast(pfun.get()); ++ return {fun_id, mx::detail::compile( ++ [pfun = std::move(pfun)](const auto& inputs) { ++ return (*pfun)(inputs); ++ }, ++ fun_id, shapeless)}; ++ } ++} ++ ++std::vector func( ++ mlir::ModuleOp& mod, std::string func_name, ++ std::unordered_map& mlx_cache, ++ const std::vector& inputs) { ++ auto op = mod.lookupSymbol(func_name); ++ auto& block = op.front(); ++ std::unordered_map> block_ctx; ++ mlir::Operation* last_op = nullptr; ++ for (mlir::Operation& op : block.getOperations()) { ++ std::vector res = ++ llvm::TypeSwitch>(&op) ++ .Case( ++ [&block_ctx, &inputs](auto o) { ++ // std::cout << "return op:" << std::endl; ++ std::vector res; ++ for (mlir::Value val : o.getOperands()) { ++ res.emplace_back(utils::array::fromOperand(val, block_ctx, ++ inputs, false)); ++ } ++ return res; ++ }) ++ .Case( ++ [&block_ctx, &inputs, &mlx_cache, &mod](auto o) { ++ std::vector operands; ++ for (mlir::Value val : o.getOperands()) { ++ operands.emplace_back(utils::array::fromOperand( ++ val, block_ctx, inputs, false)); ++ } ++ auto func_op = ++ mod.lookupSymbol(o.getCallee()); ++ return mlx_cache[utils::getHashValue(func_op)].call(operands); ++ }) ++ // Handle StableHLO nullary ops ++ .Case([](auto o) { ++ return std::vector{ ++ *utils::array::fromDenseElementsAttr( ++ mlir::cast(o.getValue()))}; ++ }) ++ .Case([](auto o) { ++ auto result_type = ++ mlir::cast(o.getResult().getType()); ++ mx::Shape result_shape(result_type.getShape().begin(), ++ result_type.getShape().end()); ++ auto iota_dim = o.getIotaDimension(); ++ mx::Dtype dtype = ++ *utils::dtype::fromMlirType(result_type.getElementType()); ++ mx::Shape broadcast_dims(result_shape.size(), 1); ++ broadcast_dims[iota_dim] = result_shape[iota_dim]; ++ auto init_arange = mx::arange( ++ static_cast(result_shape[iota_dim]), dtype); ++ if (broadcast_dims.size()) { ++ init_arange = mx::reshape(init_arange, broadcast_dims); ++ } ++ return std::vector{ ++ mx::broadcast_to(init_arange, result_shape)}; ++ }) ++ // .Case([](auto o) {}) ++ /* ++ .Case([](auto o) {}) ++ Deprecated see: ++ https://github.com/openxla/stablehlo/issues/2340 ++ https://github.com/openxla/stablehlo/pull/2283 ++ */ ++ // Handle StableHLO unary elementwise op ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::abs(utils::array::fromOperand( ++ o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{ ++ mx::power(inputs[0], mx::full(inputs[0].shape(), 1 / 3, ++ inputs[0].dtype()))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::ceil(utils::array::fromOperand( ++ o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::astype( ++ utils::array::fromOperand(o.getOperand(), block_ctx, inputs), ++ *utils::dtype::fromMlirType( ++ o.getResult().getType().getElementType()))}; ++ }) ++ // .Case([](auto o) {}) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::cos(utils::array::fromOperand( ++ o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::exp(utils::array::fromOperand( ++ o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::expm1(utils::array::fromOperand( ++ o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::floor(utils::array::fromOperand( ++ o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::imag(utils::array::fromOperand( ++ o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{ ++ mx::isfinite(utils::array::fromOperand(o.getOperand(), ++ block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::log(utils::array::fromOperand( ++ o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::log1p(utils::array::fromOperand( ++ o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{ ++ mx::sigmoid(utils::array::fromOperand(o.getOperand(), ++ block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{ ++ mx::logical_not(utils::array::fromOperand( ++ o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{ ++ mx::negative(utils::array::fromOperand(o.getOperand(), ++ block_ctx, inputs))}; ++ }) ++ // .Case([](auto o) {}) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::real(utils::array::fromOperand( ++ o.getOperand(), block_ctx, inputs))}; ++ }) ++ // TODO(@cryptodeal): `mlir::stablehlo::RoundOp` does not match with ++ // the mlx metal implementation ++ // .Case([](auto o) {}) ++ .Case([&block_ctx, ++ &inputs](auto o) { ++ return std::vector{mx::round(utils::array::fromOperand( ++ o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::rsqrt(utils::array::fromOperand( ++ o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::sign(utils::array::fromOperand( ++ o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::sin(utils::array::fromOperand( ++ o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::sqrt(utils::array::fromOperand( ++ o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::tan(utils::array::fromOperand( ++ o.getOperand(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::tanh(utils::array::fromOperand( ++ o.getOperand(), block_ctx, inputs))}; ++ }) ++ // Handle StableHLO binary elementwise ops ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::add( ++ utils::array::fromOperand(o.getLhs(), block_ctx, inputs), ++ utils::array::fromOperand(o.getRhs(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::arctan2( ++ utils::array::fromOperand(o.getLhs(), block_ctx, inputs), ++ utils::array::fromOperand(o.getRhs(), block_ctx, inputs))}; ++ }) ++ // TODO(@cryptodeal): implement complex op ++ // .Case([&block_ctx](auto ++ // op) {}) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::divide( ++ utils::array::fromOperand(o.getLhs(), block_ctx, inputs), ++ utils::array::fromOperand(o.getRhs(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::maximum( ++ utils::array::fromOperand(o.getLhs(), block_ctx, inputs), ++ utils::array::fromOperand(o.getRhs(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::minimum( ++ utils::array::fromOperand(o.getLhs(), block_ctx, inputs), ++ utils::array::fromOperand(o.getRhs(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::multiply( ++ utils::array::fromOperand(o.getLhs(), block_ctx, inputs), ++ utils::array::fromOperand(o.getRhs(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::power( ++ utils::array::fromOperand(o.getLhs(), block_ctx, inputs), ++ utils::array::fromOperand(o.getRhs(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::remainder( ++ utils::array::fromOperand(o.getLhs(), block_ctx, inputs), ++ utils::array::fromOperand(o.getRhs(), block_ctx, inputs))}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::left_shift( ++ utils::array::fromOperand(o.getLhs(), block_ctx, inputs), ++ utils::array::fromOperand(o.getRhs(), block_ctx, inputs))}; ++ }) ++ .Case( ++ [&block_ctx, &inputs](auto o) { ++ /** ++ * Per Metal Spec: ++ * For the right-shift operator, if E1 has an unsigned ++ * type or if E1 has a signed type and a nonnegative ++ * value, the vacated bits are filled with zeros. If E1 ++ * has a signed type and a negative value, the vacated ++ * bits are filled with ones. ++ */ ++ auto lhs = ++ utils::array::fromOperand(o.getLhs(), block_ctx, inputs); ++ auto rhs = ++ utils::array::fromOperand(o.getRhs(), block_ctx, inputs); ++ auto target_dtype = rhs.dtype(); ++ switch (lhs.dtype().size()) { ++ case 1: ++ target_dtype = mx::int8; ++ break; ++ case 2: ++ target_dtype = mx::int16; ++ break; ++ case 4: ++ target_dtype = mx::int32; ++ break; ++ case 8: ++ target_dtype = mx::int64; ++ break; ++ } ++ return std::vector{ ++ mx::view(mx::right_shift(mx::view(lhs, target_dtype), ++ mx::astype(rhs, target_dtype)), ++ lhs.dtype())}; ++ }) ++ .Case( ++ [&block_ctx, &inputs](auto o) { ++ // Ensures that we bitcast to `uint` type before ++ // performing the right shift. Should ensure that vacated ++ // bits are zero populated. ++ auto lhs = ++ utils::array::fromOperand(o.getLhs(), block_ctx, inputs); ++ auto rhs = ++ utils::array::fromOperand(o.getRhs(), block_ctx, inputs); ++ auto target_dtype = rhs.dtype(); ++ switch (lhs.dtype().size()) { ++ case 1: ++ target_dtype = mx::uint8; ++ break; ++ case 2: ++ target_dtype = mx::uint16; ++ break; ++ case 4: ++ target_dtype = mx::uint32; ++ break; ++ case 8: ++ target_dtype = mx::uint64; ++ break; ++ } ++ return std::vector{ ++ mx::view(mx::right_shift(mx::view(lhs, target_dtype), ++ mx::astype(rhs, target_dtype)), ++ lhs.dtype())}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::subtract( ++ utils::array::fromOperand(o.getLhs(), block_ctx, inputs), ++ utils::array::fromOperand(o.getRhs(), block_ctx, inputs))}; ++ }) ++ // Handle StableHLO binary logical elementwise ops ++ .Case([&block_ctx, &inputs](auto o) { ++ auto lhs = ++ utils::array::fromOperand(o.getLhs(), block_ctx, inputs); ++ auto rhs = ++ utils::array::fromOperand(o.getRhs(), block_ctx, inputs); ++ switch (mx::kindof(lhs.dtype())) { ++ case mx::Dtype::Kind::b: ++ return std::vector{mx::logical_and(lhs, rhs)}; ++ default: ++ return std::vector{mx::bitwise_and(lhs, rhs)}; ++ } ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ auto lhs = ++ utils::array::fromOperand(o.getLhs(), block_ctx, inputs); ++ auto rhs = ++ utils::array::fromOperand(o.getRhs(), block_ctx, inputs); ++ switch (mx::kindof(lhs.dtype())) { ++ case mx::Dtype::Kind::b: ++ return std::vector{mx::logical_or(lhs, rhs)}; ++ default: ++ return std::vector{mx::bitwise_or(lhs, rhs)}; ++ } ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::bitwise_xor( ++ utils::array::fromOperand(o.getLhs(), block_ctx, inputs), ++ utils::array::fromOperand(o.getRhs(), block_ctx, inputs))}; ++ }) ++ // Handle StableHLO communication ops ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ ++ // Handle StableHLO parallelism related ops ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ ++ // Handle StableHLO control flow ops ++ // .Case([](auto o) {}) ++ // TODO(@cryptodeal): need custom kernels in order to ++ // remove the need to call `mx::eval` for the following ops ++ // .Case([](auto o)) ++ // .Case([](auto o)) ++ // .Case([](auto o)) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ .Case([&block_ctx, &inputs, ++ &op](auto o) { ++ auto operand = ++ utils::array::fromOperand(o.getOperand(0), block_ctx, inputs); ++ mlir::Block& body_block = o.getBody().front(); ++ ++ switch (*utils::getReduceType(op, body_block)) { ++ case utils::ReduceType::ArgMax: { ++ auto axis = static_cast(o.getDimensions()[0]); ++ auto indices = mx::argmax(operand, axis); ++ ++ mx::Dtype result_type = *utils::dtype::fromMlirType( ++ mlir::cast(o.getResults()[1].getType()) ++ .getElementType()); ++ return std::vector{ ++ mx::take(operand, indices, axis), ++ mx::astype(indices, result_type)}; ++ } ++ case utils::ReduceType::Sum: ++ return std::vector{ ++ mx::sum(operand, static_cast(o.getDimensions()[0]))}; ++ ++ case utils::ReduceType::Max: ++ return std::vector{ ++ mx::max(operand, static_cast(o.getDimensions()[0]))}; ++ case utils::ReduceType::Min: ++ return std::vector{ ++ mx::min(operand, static_cast(o.getDimensions()[0]))}; ++ case utils::ReduceType::Any: ++ return std::vector{ ++ mx::any(operand, static_cast(o.getDimensions()[0]))}; ++ default: ++ __builtin_unreachable(); ++ } ++ }) ++ // Handle StableHLO tuple ops ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ .Case([&block_ctx, &inputs](auto o) { ++ auto lhs = ++ utils::array::fromOperand(o.getLhs(), block_ctx, inputs); ++ auto rhs = ++ utils::array::fromOperand(o.getRhs(), block_ctx, inputs); ++ switch (o.getComparisonDirection()) { ++ case mlir::stablehlo::ComparisonDirection::NE: ++ return std::vector{mx::not_equal(lhs, rhs)}; ++ case mlir::stablehlo::ComparisonDirection::GE: ++ return std::vector{mx::greater_equal(lhs, rhs)}; ++ case mlir::stablehlo::ComparisonDirection::GT: ++ return std::vector{mx::greater(lhs, rhs)}; ++ case mlir::stablehlo::ComparisonDirection::LE: ++ return std::vector{mx::less_equal(lhs, rhs)}; ++ case mlir::stablehlo::ComparisonDirection::LT: ++ return std::vector{mx::less(lhs, rhs)}; ++ case mlir::stablehlo::ComparisonDirection::EQ: ++ return std::vector{mx::equal(lhs, rhs)}; ++ } ++ }) ++ // Handle StableHLO Slice ops ++ .Case([&block_ctx, &inputs](auto o) { ++ std::vector start_indices(o.getStartIndices().begin(), ++ o.getStartIndices().end()); ++ std::vector limit_indices(o.getLimitIndices().begin(), ++ o.getLimitIndices().end()); ++ std::vector strides(o.getStrides().begin(), ++ o.getStrides().end()); ++ return std::vector{mx::slice( ++ utils::array::fromOperand(o.getOperand(), block_ctx, inputs), ++ start_indices, limit_indices, strides)}; ++ }) ++ .Case([&block_ctx, ++ &inputs](auto o) { ++ auto operand = ++ utils::array::fromOperand(o.getOperand(), block_ctx, inputs); ++ std::vector indices; ++ for (auto val : o.getStartIndices()) { ++ indices.emplace_back( ++ utils::array::fromOperand(val, block_ctx, inputs)); ++ } ++ auto start_indices = mx::concatenate(indices); ++ std::vector axes(operand.ndim()); ++ std::iota(axes.begin(), axes.end(), 0); ++ std::vector slice_sizes(o.getSliceSizes().begin(), ++ o.getSliceSizes().end()); ++ return std::vector{ ++ mx::slice(operand, start_indices, axes, slice_sizes)}; ++ }) ++ .Case([&block_ctx, ++ &inputs](auto o) { ++ auto operand = ++ utils::array::fromOperand(o.getOperand(), block_ctx, inputs); ++ auto update = ++ utils::array::fromOperand(o.getUpdate(), block_ctx, inputs); ++ std::vector indices; ++ for (auto val : o.getStartIndices()) { ++ indices.emplace_back( ++ utils::array::fromOperand(val, block_ctx, inputs)); ++ } ++ ++ auto start_indices = mx::concatenate(indices); ++ std::vector axes(operand.ndim()); ++ std::iota(axes.begin(), axes.end(), 0); ++ return std::vector{ ++ mx::slice_update(operand, update, start_indices, axes)}; ++ }) ++ // Handle StableHLO Other ops ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ .Case([&block_ctx, ++ &inputs](auto o) { ++ auto result_type = *utils::dtype::fromMlirType( ++ mlir::cast(o.getResult().getType()) ++ .getElementType()); ++ return std::vector{mx::view( ++ utils::array::fromOperand(o.getOperand(), block_ctx, inputs), ++ result_type)}; ++ }) ++ /* ++ .Case([](auto o) {}) ++ Deprecated see: ++ https://github.com/openxla/stablehlo/issues/2340 ++ https://github.com/openxla/stablehlo/pull/2283 ++ */ ++ .Case([&block_ctx, ++ &inputs](auto o) { ++ auto result_type = ++ mlir::cast(o.getResult().getType()); ++ mx::Shape result_shape(result_type.getShape().begin(), ++ result_type.getShape().end()); ++ std::vector broadcast_dims( ++ o.getBroadcastDimensions().begin(), ++ o.getBroadcastDimensions().end()); ++ auto operand = ++ utils::array::fromOperand(o.getOperand(), block_ctx, inputs); ++ ++ // potentially reshape to pad with ones, which allows ++ // broadcasting ++ auto min_rank = std::min(operand.ndim(), result_shape.size()); ++ if (min_rank == operand.ndim()) { ++ mx::Shape padded_shape = operand.shape(); ++ for (auto i = 0; i < result_shape.size(); i++) { ++ if (auto search = std::find(broadcast_dims.begin(), ++ broadcast_dims.end(), i); ++ search == broadcast_dims.end()) { ++ padded_shape.insert(padded_shape.begin() + i, 1); ++ } ++ } ++ operand = mx::reshape(operand, padded_shape); ++ } ++ return std::vector{ ++ mx::broadcast_to(operand, result_shape)}; ++ }) ++ // .Case([](auto o) {}) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::linalg::cholesky( ++ utils::array::fromOperand(o.getOperand(), block_ctx, inputs), ++ o.getLower())}; ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::clip( ++ utils::array::fromOperand(o.getOperand(), block_ctx, inputs), ++ utils::array::fromOperand(o.getMin(), block_ctx, inputs), ++ utils::array::fromOperand(o.getMax(), block_ctx, inputs))}; ++ }) ++ .Case( ++ [&block_ctx, &inputs](auto o) { ++ std::vector inputs; ++ for (auto val : o.getOperands()) { ++ inputs.emplace_back( ++ utils::array::fromOperand(val, block_ctx, inputs)); ++ } ++ return std::vector{ ++ mx::concatenate(inputs, o.getDimension())}; ++ }) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ /* ++ .Case([](auto o) {}) ++ Deprecated see: ++ https://github.com/openxla/stablehlo/issues/2340 ++ https://github.com/openxla/stablehlo/pull/2283 ++ */ ++ .Case([&block_ctx, &inputs, ++ &mlx_cache, &op](auto o) { ++ return mlx_cache[utils::getHashValue(&op)].call( ++ {utils::array::fromOperand(o.getLhs(), block_ctx, inputs), ++ utils::array::fromOperand(o.getRhs(), block_ctx, inputs)}); ++ }) ++ /* ++ .Case([](auto op) {}) ++ .Case([](auto op) {}) ++ Deprecated see: ++ https://github.com/openxla/stablehlo/issues/2340 ++ https://github.com/openxla/stablehlo/pull/2283 ++ */ ++ // .Case([](auto op) {}) ++ // TODO(@cryptodeal): definitely room for optimization here ++ .Case([&block_ctx, &inputs, &mlx_cache, ++ &op](auto o) { ++ return mlx_cache[utils::getHashValue(&op)].call( ++ {utils::array::fromOperand(o.getOperand(), block_ctx, inputs), ++ utils::array::fromOperand(o.getStartIndices(), block_ctx, ++ inputs)}); ++ }) ++ .Case([&block_ctx, ++ &inputs](auto o) { ++ return std::vector{mx::array( ++ (utils::array::fromOperand(o.getOperand(), block_ctx, inputs)) ++ .shape(o.getDimension()))}; ++ }) ++ // .Case([](auto o) {}) ++ .Case([&block_ctx, &inputs](auto o) { ++ auto result_type = ++ mlir::cast(o.getResult().getType()); ++ return std::vector{mx::reshape( ++ utils::array::fromOperand(o.getOperand(), block_ctx, inputs), ++ std::vector(result_type.getShape().begin(), ++ result_type.getShape().end()))}; ++ }) ++ // .Case([](auto o) {}) ++ .Case( ++ [&block_ctx, &inputs, &mlx_cache, &op](auto o) { ++ std::vector operands; ++ for (auto val : o.getInputs()) { ++ operands.emplace_back( ++ utils::array::fromOperand(val, block_ctx, inputs)); ++ } ++ // Add updates to input list ++ for (auto val : o.getUpdates()) { ++ operands.emplace_back( ++ utils::array::fromOperand(val, block_ctx, inputs)); ++ } ++ // Add scatter indices to input list ++ operands.emplace_back(utils::array::fromOperand( ++ o.getScatterIndices(), block_ctx, inputs)); ++ return mlx_cache[utils::getHashValue(&op)].call(operands); ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ return std::vector{mx::where( ++ utils::array::fromOperand(o.getOperand(0), block_ctx, inputs), ++ utils::array::fromOperand(o.getOperand(1), block_ctx, inputs), ++ utils::array::fromOperand(o.getOperand(2), block_ctx, ++ inputs))}; ++ }) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ .Case([&block_ctx, &inputs](auto o) { ++ std::vector tmp_inputs; ++ for (auto val : o.getInputs()) { ++ tmp_inputs.emplace_back( ++ utils::array::fromOperand(val, block_ctx, inputs)); ++ } ++ int dim = static_cast(o.getDimension()); ++ // Get update computation ++ mlir::Block& comparator = o.getComparator().front(); ++ auto [index, comparator_type] = *utils::getSortInfo(comparator); ++ auto indices = ++ mx::argsort(tmp_inputs[index], dim, comparator_type); ++ std::vector res; ++ mx::Shape slice_sizes(tmp_inputs[0].ndim(), 1); ++ for (auto in : tmp_inputs) { ++ res.emplace_back(mx::reshape( ++ mx::gather(in, indices, dim, slice_sizes), in.shape())); ++ } ++ return res; ++ }) ++ // .Case([](auto o) {}) ++ // TODO (@cryptodeal): lots of room for optimization here ++ .Case([&block_ctx, &inputs, &mlx_cache, ++ &op](auto o) { ++ return mlx_cache[utils::getHashValue(&op)].call( ++ {utils::array::fromOperand(o.getOperand(), block_ctx, inputs), ++ utils::array::fromOperand(o.getPaddingValue(), block_ctx, ++ inputs)}); ++ }) ++ .Case([&block_ctx, &inputs](auto o) { ++ std::vector axes(o.getPermutation().begin(), ++ o.getPermutation().end()); ++ return std::vector{mx::transpose( ++ utils::array::fromOperand(o.getOperand(), block_ctx, inputs), ++ axes)}; ++ }) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // Need to modify such that `rng.state` is held outside of scope ++ // of `mx::compile` ++ // TODO(@cryptodeal): re-enable once we've handled ensuring ++ // state is handled correctly ++ // .Case([&block_ctx](auto o) { ++ // auto rng_func = [&o](const std::vector& inputs) ++ // { ++ // auto a = inputs[0]; ++ // auto b = inputs[1]; ++ // auto result_type = ++ // mlir::cast(o.getResult().getType()); ++ // std::vector ++ // shape(result_type.getShape().begin(), ++ // result_type.getShape().end()); ++ ++ // switch (o.getRngDistribution()) { ++ // case RngDistribution::UNIFORM: ++ // return std::vector{ ++ // mx::random::uniform(a, b, shape, a.dtype())}; ++ // default: ++ // return std::vector{ ++ // mx::random::normal(shape, a.dtype(), a, b)}; ++ // } ++ // }; ++ // return mx::compile(rng_func)( ++ // {utils::array::fromOperand(o.getA(), block_ctx), ++ // utils::array::fromOperand(o.getB(), block_ctx)}); ++ // }) ++ // .Case([&block_ctx](auto o) { ++ // auto rng_bit_gen_func = [&o](const std::vector& ++ // inputs) { ++ ++ // }; ++ // return mx::compile(rng_bit_gen_func)( ++ // {utils::array::fromOperand(o.getA(), block_ctx), ++ // utils::array::fromOperand(o.getB(), block_ctx)}); ++ // }) ++ // Handle StableHLO Quantize ops ++ // .Case([](auto op) {}) ++ // .Case([](auto op) {}) ++ // .Case([](auto op) {}) ++ /* ++ .Case([](auto o) {}) ++ Deprecated see: ++ https://github.com/openxla/stablehlo/issues/2340 ++ https://github.com/openxla/stablehlo/pull/2283 ++ */ ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ // .Case([](auto o) {}) ++ ++ .Default([&op](auto o) { ++ // Shouldn't be possible to reach this point, return ++ // empty vector. ++ return std::vector{}; ++ }); ++ block_ctx.emplace(&op, std::move(res)); ++ last_op = &op; ++ } ++ return block_ctx.find(last_op)->second; ++} ++ ++bool compileFunc(mlir::ModuleOp& mod, std::string func_name, ++ std::unordered_map& mlx_cache) { ++ auto func = mod.lookupSymbol(func_name); ++ auto& cache_entry = mlx_cache[utils::getHashValue(func)]; ++ if (cache_entry.isInitialized()) { ++ return true; ++ } else { ++ bool all_compiled = true; ++ auto& block = func.front(); ++ auto call_ops = block.getOps(); ++ for (mlir::func::CallOp call_op : call_ops) { ++ auto callee = mod.lookupSymbol(call_op.getCallee()); ++ if (!compileFunc(mod, callee.getName().str(), mlx_cache)) { ++ all_compiled = false; ++ } ++ } ++ if (all_compiled) { ++ MlxFunc fun = [&mod, &mlx_cache, ++ func_name](const std::vector& inputs) { ++ return detail::func(mod, func_name, mlx_cache, inputs); ++ }; ++ auto compiled_fun = detail::compile(std::move(fun)); ++ cache_entry.initialize(std::move(compiled_fun.second), ++ compiled_fun.first); ++ } ++ return all_compiled; ++ } ++} ++ ++std::pair dotGeneral( ++ mlir::stablehlo::DotGeneralOp op) { ++ auto dot_dim_nums = op.getDotDimensionNumbers(); ++ std::vector lhs_batch_dims( ++ dot_dim_nums.getLhsBatchingDimensions().begin(), ++ dot_dim_nums.getLhsBatchingDimensions().end()); ++ std::vector rhs_batch_dims( ++ dot_dim_nums.getRhsBatchingDimensions().begin(), ++ dot_dim_nums.getRhsBatchingDimensions().end()); ++ std::vector lhs_contract_dims( ++ dot_dim_nums.getLhsContractingDimensions().begin(), ++ dot_dim_nums.getLhsContractingDimensions().end()); ++ std::vector rhs_contract_dims( ++ dot_dim_nums.getRhsContractingDimensions().begin(), ++ dot_dim_nums.getRhsContractingDimensions().end()); ++ ++ MlxFunc fun = [lhs_batch_dims = std::move(lhs_batch_dims), ++ rhs_batch_dims = std::move(rhs_batch_dims), ++ lhs_contract_dims = std::move(lhs_contract_dims), ++ rhs_contract_dims = std::move(rhs_contract_dims)]( ++ const std::vector& inputs) { ++ auto lhs = inputs[0]; ++ auto rhs = inputs[1]; ++ ++ int dim_count = 0; ++ auto getDimChar = [&dim_count]() -> char { return 'a' + dim_count++; }; ++ ++ std::unordered_map lhs_batch_map; ++ std::unordered_map rhs_batch_map; ++ std::unordered_map lhs_contract_map; ++ std::unordered_map rhs_contract_map; ++ ++ std::string res_subscript; ++ for (auto i = 0; i < lhs_batch_dims.size(); ++i) { ++ auto dim_char = getDimChar(); ++ res_subscript = res_subscript + dim_char; ++ lhs_batch_map.emplace(lhs_batch_dims[i], dim_char); ++ rhs_batch_map.emplace(rhs_batch_dims[i], dim_char); ++ } ++ ++ for (auto i = 0; i < lhs_contract_dims.size(); ++i) { ++ auto dim_char = getDimChar(); ++ lhs_contract_map.emplace(lhs_contract_dims[i], dim_char); ++ rhs_contract_map.emplace(rhs_contract_dims[i], dim_char); ++ } ++ ++ std::string lhs_subscript; ++ for (auto i = 0; i < lhs.ndim(); ++i) { ++ if (auto match = lhs_batch_map.find(i); match != lhs_batch_map.end()) { ++ lhs_subscript = lhs_subscript + match->second; ++ } else if (auto match = lhs_contract_map.find(i); ++ match != lhs_contract_map.end()) { ++ lhs_subscript = lhs_subscript + match->second; ++ } else { ++ auto dim_char = getDimChar(); ++ res_subscript = res_subscript + dim_char; ++ lhs_subscript = lhs_subscript + dim_char; ++ } ++ } ++ ++ std::string rhs_subscript; ++ for (auto i = 0; i < rhs.ndim(); ++i) { ++ if (auto match = rhs_batch_map.find(i); match != rhs_batch_map.end()) { ++ rhs_subscript = rhs_subscript + match->second; ++ } else if (auto match = rhs_contract_map.find(i); ++ match != rhs_contract_map.end()) { ++ rhs_subscript = rhs_subscript + match->second; ++ } else { ++ auto dim_char = getDimChar(); ++ res_subscript = res_subscript + dim_char; ++ rhs_subscript = rhs_subscript + dim_char; ++ } ++ } ++ ++ return std::vector{mx::einsum( ++ lhs_subscript + "," + rhs_subscript + "->" + res_subscript, inputs)}; ++ }; ++ return detail::compile(std::move(fun)); ++} ++ ++std::pair gather(mlir::stablehlo::GatherOp op) { ++ auto dim_nums = op.getDimensionNumbers(); ++ std::vector offset_dims(dim_nums.getOffsetDims().begin(), ++ dim_nums.getOffsetDims().end()); ++ std::vector collapsed_slice_dims( ++ dim_nums.getCollapsedSliceDims().begin(), ++ dim_nums.getCollapsedSliceDims().end()); ++ std::vector operand_batching_dims( ++ dim_nums.getOperandBatchingDims().begin(), ++ dim_nums.getOperandBatchingDims().end()); ++ std::vector start_indices_batching_dims( ++ dim_nums.getStartIndicesBatchingDims().begin(), ++ dim_nums.getStartIndicesBatchingDims().end()); ++ std::vector start_index_map(dim_nums.getStartIndexMap().begin(), ++ dim_nums.getStartIndexMap().end()); ++ auto index_vector_dim = dim_nums.getIndexVectorDim(); ++ std::vector slice_sizes(op.getSliceSizes().begin(), ++ op.getSliceSizes().end()); ++ MlxFunc fun = [collapsed_slice_dims = std::move(collapsed_slice_dims), ++ index_vector_dim, offset_dims = std::move(offset_dims), ++ operand_batching_dims = std::move(operand_batching_dims), ++ slice_sizes = std::move(slice_sizes), ++ start_index_map = std::move(start_index_map), ++ start_indices_batching_dims = ++ std::move(start_indices_batching_dims)]( ++ const std::vector& inputs) { ++ auto operand = inputs[0]; ++ auto start_indices = inputs[1]; ++ ++ std::vector batch_dim_sizes = start_indices.shape(); ++ if (index_vector_dim < batch_dim_sizes.size()) { ++ batch_dim_sizes.erase(batch_dim_sizes.begin() + index_vector_dim); ++ } ++ ++ std::vector offset_dim_sizes; ++ for (auto i = 0; i < slice_sizes.size(); ++i) { ++ if (std::find(collapsed_slice_dims.begin(), collapsed_slice_dims.end(), ++ i) != collapsed_slice_dims.end() || ++ std::find(operand_batching_dims.begin(), operand_batching_dims.end(), ++ i) != operand_batching_dims.end()) { ++ continue; ++ } ++ offset_dim_sizes.push_back(slice_sizes[i]); ++ } ++ ++ // Calculate result shape and batch dims ++ std::vector result_shape(batch_dim_sizes.size() + ++ offset_dim_sizes.size()); ++ std::vector batch_dims_vec(batch_dim_sizes.size()); ++ auto batch_dims_count = 0; ++ auto offset_dims_count = 0; ++ for (auto i = 0; i < result_shape.size(); ++i) { ++ if (auto search = std::find(offset_dims.begin(), offset_dims.end(), i); ++ search == offset_dims.end()) { ++ batch_dims_vec[batch_dims_count] = i; ++ result_shape[i] = batch_dim_sizes[batch_dims_count++]; ++ } else { ++ result_shape[i] = offset_dim_sizes[offset_dims_count++]; ++ } ++ } ++ ++ mx::array batch_dims = mx::array( ++ batch_dims_vec.data(), {static_cast(batch_dims_vec.size())}); ++ mx::array start_idx_map = mx::array( ++ start_index_map.data(), {static_cast(start_index_map.size())}); ++ mx::array op_batching_dims = ++ mx::array(operand_batching_dims.data(), ++ {static_cast(operand_batching_dims.size())}); ++ ++ // Calculate result indices ++ std::vector meshgrid_arrays; ++ for (const auto& dim : result_shape) { ++ meshgrid_arrays.push_back(mx::arange(dim)); ++ } ++ mx::array result_indices = ++ mx::reshape(mx::stack(mx::meshgrid(meshgrid_arrays, false, "ij"), -1), ++ {-1, static_cast(result_shape.size())}); ++ ++ // Initialize operand indices with zeros ++ mx::array operand_indices = mx::zeros( ++ {result_indices.shape(0), static_cast(operand.ndim())}, ++ mx::int32); ++ ++ // Slice batch indices ++ mx::array batch_indices = mx::take(result_indices, batch_dims, 1); ++ ++ // Calculate start indices ++ std::vector tmp_indices; ++ if (batch_indices.size()) { ++ tmp_indices = mx::split(batch_indices, batch_indices.shape(1), 1); ++ } ++ for (auto& idx : tmp_indices) { ++ idx = mx::flatten(idx); ++ } ++ ++ std::vector tmp_axes; ++ for (auto i = 0; i < start_indices.ndim(); ++i) { ++ if (i == index_vector_dim) continue; ++ tmp_axes.push_back(i); ++ } ++ std::vector tmp_slice_sizes(start_indices.ndim(), 1); ++ if (index_vector_dim < tmp_slice_sizes.size()) { ++ tmp_slice_sizes[index_vector_dim] = start_indices.shape(index_vector_dim); ++ } ++ ++ mx::array batch_start_indices = ++ mx::gather(start_indices, tmp_indices, tmp_axes, tmp_slice_sizes); ++ if (tmp_axes.size()) { ++ batch_start_indices = mx::reshape( ++ mx::gather(start_indices, tmp_indices, tmp_axes, tmp_slice_sizes), ++ {result_indices.shape(0), -1}); ++ } else if (batch_start_indices.ndim() == 1) { ++ batch_start_indices = mx::broadcast_to( ++ mx::reshape(batch_start_indices, {1, batch_start_indices.shape(0)}), ++ {result_indices.shape(0), batch_start_indices.shape(0)}); ++ } ++ ++ // Seed with full start indices ++ std::vector max; ++ for (auto d_start = 0; d_start < start_index_map.size(); ++d_start) { ++ int d_operand = start_index_map[d_start]; ++ max.push_back(operand.shape(d_operand) - slice_sizes[d_operand]); ++ } ++ ++ operand_indices = mx::put_along_axis( ++ operand_indices, ++ mx::broadcast_to(start_idx_map, ++ {operand_indices.shape(0), ++ static_cast(start_index_map.size())}), ++ mx::clip( ++ mx::take(batch_start_indices, ++ mx::arange(static_cast(start_index_map.size())), 1), ++ mx::zeros(batch_start_indices.shape(), mx::int32), ++ mx::array(max.data(), {static_cast(max.size())})), ++ 1); ++ ++ // Calculate full batching indices ++ mx::array i_batching = ++ mx::arange(static_cast(operand_batching_dims.size())); ++ mx::array d_operands = mx::take(op_batching_dims, i_batching); ++ mx::array d_starts = mx::take( ++ mx::array(start_indices_batching_dims.data(), ++ {static_cast(start_indices_batching_dims.size())}), ++ i_batching); ++ auto tmp_batch_start_indices = mx::subtract( ++ d_starts, mx::where(mx::less(d_starts, mx::full(d_starts.shape(), ++ index_vector_dim)), ++ mx::zeros(d_starts.shape(), mx::int32), ++ mx::ones(d_starts.shape(), mx::int32))); ++ ++ // Add batching indices ++ operand_indices = mx::scatter_add_axis( ++ operand_indices, ++ mx::broadcast_to(d_operands, ++ {operand_indices.shape(0), d_operands.shape(0)}), ++ mx::take(batch_indices, tmp_batch_start_indices, 1), 1); ++ ++ // Calculate offset indices ++ mx::array offset_indices = ++ mx::take(result_indices, ++ mx::array(offset_dims.data(), ++ {static_cast(offset_dims.size())}), ++ 1); ++ ++ std::vector result_offset_idx; ++ for (unsigned i = 0; i < operand_indices.shape(1); i++) { ++ if (std::find(operand_batching_dims.begin(), operand_batching_dims.end(), ++ static_cast(i)) != operand_batching_dims.end() || ++ std::find(collapsed_slice_dims.begin(), collapsed_slice_dims.end(), ++ static_cast(i)) != collapsed_slice_dims.end()) { ++ continue; ++ } ++ result_offset_idx.push_back(i); ++ } ++ ++ // Add offset indices ++ operand_indices = mx::scatter_add_axis( ++ operand_indices, ++ mx::broadcast_to( ++ mx::array(result_offset_idx.data(), ++ {static_cast(result_offset_idx.size())}), ++ {operand_indices.shape(0), ++ static_cast(result_offset_idx.size())}), ++ offset_indices, 1); ++ ++ std::vector gather_axes(operand.ndim()); ++ std::iota(gather_axes.begin(), gather_axes.end(), 0); ++ return std::vector{mx::reshape( ++ mx::gather(operand, ++ mx::split(operand_indices, operand_indices.shape(1), 1), ++ gather_axes, std::vector(operand.ndim(), 1)), ++ result_shape)}; ++ }; ++ return detail::compile(fun); ++} ++ ++std::pair pad(mlir::stablehlo::PadOp op) { ++ std::vector edge_pad_low(op.getEdgePaddingLow().begin(), ++ op.getEdgePaddingLow().end()); ++ std::vector edge_pad_high(op.getEdgePaddingHigh().begin(), ++ op.getEdgePaddingHigh().end()); ++ std::vector interior_pad(op.getInteriorPadding().begin(), ++ op.getInteriorPadding().end()); ++ ++ MlxFunc fun = [edge_pad_low = std::move(edge_pad_low), ++ edge_pad_high = std::move(edge_pad_high), ++ interior_pad = std::move(interior_pad)]( ++ const std::vector& inputs) { ++ auto operand = inputs[0]; ++ auto padding_value = inputs[1]; ++ ++ std::vector result_shape(operand.ndim()); ++ for (auto i = 0; i < result_shape.size(); ++i) { ++ result_shape[i] = operand.shape(i) + edge_pad_low[i] + ++ std::max(operand.shape(i) - 1, 0) * interior_pad[i] + ++ edge_pad_high[i]; ++ } ++ ++ auto edge_padding_low = ++ mx::array(edge_pad_low.data(), ++ {static_cast(edge_pad_low.size())}, mx::int32); ++ auto interior_padding = ++ mx::array(interior_pad.data(), ++ {static_cast(edge_pad_low.size())}, mx::int32); ++ ++ // Calculate operand indices ++ std::vector meshgrid_arrays; ++ for (const auto& dim : operand.shape()) { ++ meshgrid_arrays.push_back(mx::arange(dim)); ++ } ++ mx::array operand_indices = ++ mx::reshape(mx::stack(mx::meshgrid(meshgrid_arrays, false, "ij"), -1), ++ {-1, static_cast(operand.ndim())}); ++ ++ mx::array result_indices = ++ edge_padding_low + ++ operand_indices * (interior_padding + mx::array(1, mx::int32)); ++ ++ // initialize array full of padding values ++ mx::array result = mx::full(result_shape, padding_value); ++ std::vector axes(operand.ndim()); ++ std::iota(axes.begin(), axes.end(), 0); ++ return std::vector{mx::scatter( ++ result, mx::split(result_indices, result_indices.shape(1), 1), ++ mx::gather(operand, ++ mx::split(operand_indices, operand_indices.shape(1), 1), ++ axes, std::vector(operand.ndim(), 1)), ++ axes)}; ++ }; ++ return detail::compile(std::move(fun)); ++} ++ ++std::pair scatter(mlir::stablehlo::ScatterOp op, ++ utils::ScatterType scatter_type) { ++ // Get scatter dimension numbers ++ auto scatter_dim_nums = op.getScatterDimensionNumbers(); ++ auto index_vector_dim = scatter_dim_nums.getIndexVectorDim(); ++ std::vector input_batching_dims( ++ scatter_dim_nums.getInputBatchingDims().begin(), ++ scatter_dim_nums.getInputBatchingDims().end()); ++ std::vector inserted_window_dims( ++ scatter_dim_nums.getInsertedWindowDims().begin(), ++ scatter_dim_nums.getInsertedWindowDims().end()); ++ std::vector scatter_dims_to_operand_dims( ++ scatter_dim_nums.getScatterDimsToOperandDims().begin(), ++ scatter_dim_nums.getScatterDimsToOperandDims().end()); ++ std::vector scatter_indices_batching_dims( ++ scatter_dim_nums.getScatterIndicesBatchingDims().begin(), ++ scatter_dim_nums.getScatterIndicesBatchingDims().end()); ++ std::vector update_window_dims( ++ scatter_dim_nums.getUpdateWindowDims().begin(), ++ scatter_dim_nums.getUpdateWindowDims().end()); ++ // TODO(@cryptodeal): need to experiment to see how `getHashValue` ++ // handles hashing `update_computation` property. ++ MlxFunc fun = [index_vector_dim, ++ input_batching_dims = std::move(input_batching_dims), ++ inserted_window_dims = std::move(inserted_window_dims), ++ scatter_dims_to_operand_dims = ++ std::move(scatter_dims_to_operand_dims), ++ scatter_indices_batching_dims = ++ std::move(scatter_indices_batching_dims), ++ scatter_type, ++ update_window_dims = std::move(update_window_dims)]( ++ const std::vector& in) { ++ auto input_count = (in.size() - 1) / 2; ++ std::vector inputs(in.begin(), in.begin() + input_count); ++ std::vector updates(in.begin() + input_count, in.end() - 1); ++ auto scatter_indices = in[in.size() - 1]; ++ // Calculate update scatter dims ++ std::vector update_scatter_dims; ++ for (auto i = 0; i < updates[0].ndim(); i++) { ++ if (std::find(update_window_dims.begin(), update_window_dims.end(), ++ static_cast(i)) == update_window_dims.end()) { ++ update_scatter_dims.emplace_back(static_cast(i)); ++ } ++ } ++ ++ // Calculate update indices ++ std::vector meshgrid_arrays; ++ for (const auto& dim : updates[0].shape()) { ++ meshgrid_arrays.push_back(mx::arange(dim)); ++ } ++ mx::array update_indices = ++ mx::reshape(mx::stack(mx::meshgrid(meshgrid_arrays, false, "ij"), -1), ++ {-1, static_cast(updates[0].ndim())}); ++ ++ // Calculate update scatter index ++ mx::array update_scatter_indices = ++ mx::take(update_indices, ++ mx::array(update_scatter_dims.data(), ++ {static_cast(update_scatter_dims.size())}), ++ 1); ++ ++ // Initialize result indices with zeros ++ auto result_indices = mx::zeros( ++ {update_indices.shape(0), static_cast(inputs[0].ndim())}, ++ mx::int32); ++ ++ // Gather start indices; populating the corresponding ++ // result indices. ++ auto gather_dim_count = index_vector_dim < scatter_indices.ndim() ++ ? update_scatter_dims.size() + 1 ++ : update_scatter_dims.size(); ++ std::vector tmp_gather_axes(update_scatter_dims.size()); ++ std::iota(tmp_gather_axes.begin(), tmp_gather_axes.end(), 0); ++ std::vector gather_slice_sizes(gather_dim_count, 1); ++ if (index_vector_dim < scatter_indices.ndim()) { ++ gather_slice_sizes[index_vector_dim] = ++ scatter_indices.shape(index_vector_dim); ++ } ++ mx::array start_indices = ++ mx::reshape(mx::gather(scatter_indices, ++ mx::split(update_scatter_indices, ++ update_scatter_indices.shape(1), 1), ++ tmp_gather_axes, gather_slice_sizes), ++ {update_indices.shape(0), -1}); ++ ++ // Seed with full start indices ++ result_indices = mx::put_along_axis( ++ result_indices, ++ mx::broadcast_to( ++ mx::array( ++ scatter_dims_to_operand_dims.data(), ++ {static_cast(scatter_dims_to_operand_dims.size())}), ++ {result_indices.shape(0), ++ static_cast(scatter_dims_to_operand_dims.size())}), ++ start_indices, 1); ++ ++ // Calculate full batching indices; accumulating to the ++ // corresponding result indices. ++ auto d_input = ++ mx::array(input_batching_dims.data(), ++ {static_cast(input_batching_dims.size())}); ++ auto d_start = ++ mx::array(scatter_indices_batching_dims.data(), ++ {static_cast(scatter_indices_batching_dims.size())}); ++ result_indices = mx::scatter_add_axis( ++ result_indices, ++ mx::broadcast_to(d_input, {result_indices.shape(0), d_input.shape(0)}), ++ mx::take(update_scatter_indices, ++ mx::subtract( ++ d_start, ++ mx::where(mx::less(d_start, mx::full(d_start.shape(), ++ index_vector_dim)), ++ mx::zeros(d_start.shape(), mx::int32), ++ mx::ones(d_start.shape(), mx::int32))), ++ 1), ++ 1); ++ ++ // Calculate update window indices ++ auto update_window_indices = ++ mx::take(update_indices, ++ mx::array(update_window_dims.data(), ++ {static_cast(update_window_dims.size())}), ++ 1); ++ ++ // Calculate full window indices; accumulating to the ++ // corresponding result indices. ++ std::vector tmp_indices; ++ auto full_window_idx_count = static_cast( ++ update_window_indices.shape(1) + inserted_window_dims.size() + ++ input_batching_dims.size()); ++ for (auto i = 0; i < full_window_idx_count; ++i) { ++ if (std::find(inserted_window_dims.begin(), inserted_window_dims.end(), ++ i) != inserted_window_dims.end() || ++ std::find(input_batching_dims.begin(), input_batching_dims.end(), ++ i) != input_batching_dims.end()) { ++ continue; ++ } ++ tmp_indices.push_back(i); ++ } ++ result_indices = mx::scatter_add_axis( ++ result_indices, ++ mx::broadcast_to(mx::array(tmp_indices.data(), ++ {static_cast(tmp_indices.size())}), ++ {result_indices.shape(0), ++ static_cast(tmp_indices.size())}), ++ update_window_indices, 1); ++ ++ std::vector scatter_axes(inputs[0].ndim()); ++ std::iota(scatter_axes.begin(), scatter_axes.end(), 0); ++ std::vector gather_axes(updates[0].ndim()); ++ std::iota(gather_axes.begin(), gather_axes.end(), 0); ++ std::vector res; ++ ++ auto tmp_gather_indices = ++ mx::split(update_indices, update_indices.shape(1), 1); ++ auto tmp_scatter_indices = ++ mx::split(result_indices, result_indices.shape(1), 1); ++ ++ std::vector update_shape( ++ inputs[0].ndim() + tmp_scatter_indices[0].ndim(), 1); ++ update_shape[0] = update_indices.shape(0); ++ std::vector tmp_gather_slice_sizes(updates[0].ndim(), 1); ++ for (auto i = 0; i < inputs.size(); ++i) { ++ auto update_vals = ++ mx::reshape(mx::gather(updates[i], tmp_gather_indices, gather_axes, ++ tmp_gather_slice_sizes), ++ update_shape); ++ switch (scatter_type) { ++ case utils::ScatterType::Replace: ++ res.push_back(mx::scatter(inputs[i], tmp_scatter_indices, update_vals, ++ scatter_axes)); ++ break; ++ case utils::ScatterType::Add: ++ res.push_back(mx::scatter_add(inputs[i], tmp_scatter_indices, ++ update_vals, scatter_axes)); ++ break; ++ case utils::ScatterType::Prod: ++ res.push_back(mx::scatter_prod(inputs[i], tmp_scatter_indices, ++ update_vals, scatter_axes)); ++ break; ++ case utils::ScatterType::Max: ++ res.push_back(mx::scatter_max(inputs[i], tmp_scatter_indices, ++ update_vals, scatter_axes)); ++ break; ++ case utils::ScatterType::Min: ++ res.push_back(mx::scatter_min(inputs[i], tmp_scatter_indices, ++ update_vals, scatter_axes)); ++ break; ++ } ++ } ++ return res; ++ }; ++ return detail::compile(std::move(fun)); ++} ++} // namespace detail ++ ++absl::StatusOr> ++validateAndPrecompile(mlir::ModuleOp& mod) { ++ std::unordered_map mlx_cache; ++ absl::Status status; ++ // TODO: some ops should have additional checks (e.g. `stablehlo::ReduceOp`, ++ // `stablehlo::SortOp`, `stablehlo::ScatterOp`, etc.) where we need additional ++ // checks at compile time to ensure that we're correctly mapping to the ++ // corresponding MLX op. ++ mod.walk([&mlx_cache, &status](mlir::Operation* op) { ++ // Series of checks to ensure we support the module. ++ // For some ops, we have to validate that the specific ++ // inner function (e.g. `comparator` for `stablehlo::SortOp`) ++ // is supported. Pre-compile complex ops (e.g. `stablehlo::DotGeneralOp`), ++ // which require chaining multiple MLX ops together in order ++ // to implement the op per StableHLO spec. ++ return llvm::TypeSwitch(op) ++ // Verify that we support the requested `sort` op ++ .Case([&status, &op](auto o) { ++ if (!utils::validateTypeRange(o->getOperands()) || ++ !utils::validateTypeRange(o->getResults())) { ++ status = absl::UnimplementedError(absl::StrCat( ++ "Unsupported type: ", mlir::stablehlo::ToString(op))); ++ return mlir::WalkResult::interrupt(); ++ } ++ mlir::Block& comparator = o.getComparator().front(); ++ auto maybe_sort_info = utils::getSortInfo(comparator); ++ if (!maybe_sort_info.ok()) { ++ status = maybe_sort_info.status(); ++ return mlir::WalkResult::interrupt(); ++ } else ++ return mlir::WalkResult::advance(); ++ }) ++ // Verify that we support the requested `reduce` op ++ .Case([&status, &op](auto o) { ++ if (!utils::validateTypeRange(o->getOperands()) || ++ !utils::validateTypeRange(o->getResults())) { ++ status = absl::UnimplementedError(absl::StrCat( ++ "Unsupported type: ", mlir::stablehlo::ToString(op))); ++ return mlir::WalkResult::interrupt(); ++ } ++ mlir::Block& body_block = o.getBody().front(); ++ auto maybe_reduce_type = utils::getReduceType(*op, body_block); ++ if (!maybe_reduce_type.ok()) { ++ status = maybe_reduce_type.status(); ++ return mlir::WalkResult::interrupt(); ++ } else ++ return mlir::WalkResult::advance(); ++ }) ++ // Validate, precompile, and cache `dot_general` op ++ .Case( ++ [&status, &mlx_cache, &op](auto o) { ++ if (!utils::validateTypeRange(o->getOperands()) || ++ !utils::validateTypeRange(o->getResults())) { ++ status = absl::UnimplementedError(absl::StrCat( ++ "Unsupported type: ", mlir::stablehlo::ToString(op))); ++ return mlir::WalkResult::interrupt(); ++ } ++ auto hash = utils::getHashValue(op); ++ auto& cache_entry = mlx_cache[hash]; ++ if (!cache_entry.isInitialized()) { ++ // std::cout << "Compiling DotGeneralOp: " << ++ // std::to_string(hash) ++ // << std::endl ++ // << mlir::stablehlo::ToString(op).c_str() << ++ // std::endl; ++ auto compiled_dot = detail::dotGeneral(o); ++ cache_entry.initialize(std::move(compiled_dot.second), ++ compiled_dot.first); ++ } ++ return mlir::WalkResult::advance(); ++ }) ++ // Verify that we support the requested `scatter` op; ++ // hash this instance of the op, precompile and cache ++ // accordingly. ++ .Case([&mlx_cache, &op, &status](auto o) { ++ if (!utils::validateTypeRange(o->getOperands()) || ++ !utils::validateTypeRange(o->getResults())) { ++ status = absl::UnimplementedError(absl::StrCat( ++ "Unsupported type: ", mlir::stablehlo::ToString(op))); ++ return mlir::WalkResult::interrupt(); ++ } ++ mlir::Block& update_computation = o.getUpdateComputation().front(); ++ auto hash = utils::getHashValue(op); ++ auto& cache_entry = mlx_cache[hash]; ++ if (!cache_entry.isInitialized()) { ++ // only need to validate if we've not already encountered ++ auto maybe_scatter_type = utils::getScatterType(update_computation); ++ if (!maybe_scatter_type.ok()) { ++ status = maybe_scatter_type.status(); ++ return mlir::WalkResult::interrupt(); ++ } ++ auto compiled_scatter = detail::scatter(o, *maybe_scatter_type); ++ cache_entry.initialize(std::move(compiled_scatter.second), ++ compiled_scatter.first); ++ } ++ return mlir::WalkResult::advance(); ++ }) ++ // Validate, precompile, and cache `gather` op ++ .Case([&status, &mlx_cache, &op](auto o) { ++ if (!utils::validateTypeRange(o->getOperands()) || ++ !utils::validateTypeRange(o->getResults())) { ++ status = absl::UnimplementedError(absl::StrCat( ++ "Unsupported type: ", mlir::stablehlo::ToString(op))); ++ return mlir::WalkResult::interrupt(); ++ } ++ auto hash = utils::getHashValue(op); ++ auto& cache_entry = mlx_cache[hash]; ++ if (!cache_entry.isInitialized()) { ++ auto compiled_gather = detail::gather(o); ++ cache_entry.initialize(std::move(compiled_gather.second), ++ compiled_gather.first); ++ } ++ return mlir::WalkResult::advance(); ++ }) ++ // Validate, precompile, and cache `pad` op ++ .Case([&status, &mlx_cache, &op](auto o) { ++ if (!utils::validateTypeRange(o->getOperands()) || ++ !utils::validateTypeRange(o->getResults())) { ++ status = absl::UnimplementedError(absl::StrCat( ++ "Unsupported type: ", mlir::stablehlo::ToString(op))); ++ return mlir::WalkResult::interrupt(); ++ } ++ auto hash = utils::getHashValue(op); ++ auto& cache_entry = mlx_cache[hash]; ++ if (!cache_entry.isInitialized()) { ++ auto compiled_pad = detail::pad(o); ++ cache_entry.initialize(std::move(compiled_pad.second), ++ compiled_pad.first); ++ } ++ return mlir::WalkResult::advance(); ++ }) ++ .Case( ++ [&status, &op](auto o) { ++ // Validate all operand and result types are compatible with mlx ++ if (!utils::validateTypeRange(o->getOperands()) || ++ !utils::validateTypeRange(o->getResults())) { ++ status = absl::UnimplementedError(absl::StrCat( ++ "Unsupported type: ", mlir::stablehlo::ToString(op))); ++ return mlir::WalkResult::interrupt(); ++ } ++ return mlir::WalkResult::advance(); ++ }) ++ // TODO(@cryptodeal): These ops were supported prior to `mlx::compile` ++ // need to modify the implementation to ensure no use of `mlx::eval` and ++ // ensure that `mlx::random::key` is being propogated correctly. ++ .Default([&status, &op](auto o) { ++ status = absl::UnimplementedError( ++ absl::StrCat("Unsupported op: ", mlir::stablehlo::ToString(op))); ++ return mlir::WalkResult::interrupt(); ++ }); ++ }); ++ ++ // If we encounter an unsupported op, type, etc., clear ++ // the compilation cache and return the corresponding status ++ if (!status.ok()) { ++ mlx_cache.clear(); ++ return status; ++ } ++ ++ return mlx_cache; ++} ++ ++utils::MlxFuncRef* compileModule( ++ mlir::ModuleOp& mod, ++ std::unordered_map& mlx_cache) { ++ auto op = mod.lookupSymbol("main"); ++ ++ // compile inner function calls ++ auto& main_block = op.getBody(); ++ while (true) { ++ auto call_ops = main_block.getOps(); ++ bool all_compiled = true; ++ for (mlir::func::CallOp call_op : call_ops) { ++ auto callee = mod.lookupSymbol(call_op.getCallee()); ++ ++ if (!detail::compileFunc(mod, callee.getName().str(), mlx_cache)) { ++ all_compiled = false; ++ } ++ } ++ if (all_compiled) break; ++ } ++ ++ MlxFunc fun = [&mod, &mlx_cache](const std::vector& inputs) { ++ return detail::func(mod, "main", mlx_cache, inputs); ++ }; ++ auto hash = utils::getHashValue(op); ++ auto compiled_mod = detail::compile(fun); ++ auto& cache_entry = mlx_cache[hash]; ++ cache_entry.initialize(std::move(compiled_mod.second), compiled_mod.first); ++ ++ // force compilation by running w zeroed inputs ++ // input sizes remain constant, so we never modify ++ // mlx compiler state after initial compilation ++ std::vector tmp_inputs; ++ for (auto arg : main_block.getArguments()) { ++ auto shaped_type = mlir::cast(arg.getType()); ++ tmp_inputs.push_back( ++ mx::zeros(std::vector(shaped_type.getShape().begin(), ++ shaped_type.getShape().end()), ++ *utils::dtype::fromMlirType(shaped_type.getElementType()))); ++ } ++ ++ cache_entry.call(tmp_inputs); ++ return &cache_entry; ++} ++ ++} // namespace compile +\ No newline at end of file +diff --git a/xla/pjrt/plugin/mlx/compile.h b/xla/pjrt/plugin/mlx/compile.h +new file mode 100644 +index 0000000000..d8ce0c1256 +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/compile.h +@@ -0,0 +1,33 @@ ++#ifndef XLA_PJRT_PLUGIN_STABLEHLO_MLX_COMPILE_H_ ++#define XLA_PJRT_PLUGIN_STABLEHLO_MLX_COMPILE_H_ ++ ++#include ++#include ++#include ++#include ++ ++#include "absl/status/statusor.h" ++#include "mlx/mlx.h" ++#include "mlir/IR/Operation.h" ++ ++#include "stablehlo/reference/Api.h" ++#include "mlir/IR/Operation.h" ++#include "xla/pjrt/plugin/mlx/utils.h" ++ ++namespace mx = mlx::core; ++ ++namespace compile { ++namespace detail { ++std::pair compile( ++ std::function(const std::vector&)> fun, ++ bool shapeless = false); ++} // namespace detail ++ ++absl::StatusOr> ++validateAndPrecompile(mlir::ModuleOp& mod); ++utils::MlxFuncRef* compileModule( ++ mlir::ModuleOp& mod, ++ std::unordered_map& mlx_cache); ++} // namespace compile ++ ++#endif // XLA_PJRT_PLUGIN_STABLEHLO_MLX_COMPILE_H_ +\ No newline at end of file +diff --git a/xla/pjrt/plugin/mlx/device.cc b/xla/pjrt/plugin/mlx/device.cc +new file mode 100644 +index 0000000000..6984da5b7f +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/device.cc +@@ -0,0 +1,213 @@ ++/* Copyright 2024 The OpenXLA Authors. ++ ++ ++Licensed under the Apache License, Version 2.0 (the "License"); ++you may not use this file except in compliance with the License. ++You may obtain a copy of the License at ++ http://www.apache.org/licenses/LICENSE-2.0 ++Unless required by applicable law or agreed to in writing, software ++distributed under the License is distributed on an "AS IS" BASIS, ++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++See the License for the specific language governing permissions and ++limitations under the License. ++==============================================================================*/ ++ ++#include "mlx/mlx.h" ++ ++#include "xla/pjrt/plugin/mlx/device.h" ++ ++#include "absl/status/statusor.h" ++#include "absl/strings/str_format.h" ++#include "absl/strings/string_view.h" ++#include "absl/types/span.h" ++#include "xla/pjrt/pjrt_client.h" ++#include "xla/pjrt/pjrt_common.h" ++#include "xla/pjrt/pjrt_compiler.h" ++#include "xla/pjrt/pjrt_device_description.h" ++#include "xla/pjrt/plugin/mlx/logging.h" ++#include "xla/util.h" ++ ++#define UNIMPLEMENTED(name) \ ++ xla::Unimplemented("StablehloMlxDevice::" #name " is not implemented") ++ ++namespace mlir::stablehlo { ++ ++using xla::LiteralSlice; ++using xla::MutableBorrowingLiteral; ++using xla::PjRtClient; ++using xla::PjRtDevice; ++using xla::PjRtDeviceAttribute; ++using xla::PjRtDeviceDescription; ++using xla::PjRtGlobalDeviceId; ++using xla::PjRtLocalDeviceId; ++using xla::PjRtLocalHardwareId; ++using xla::PjRtMemorySpace; ++using xla::ScopedAsyncTrackingEvent; ++using xla::Unimplemented; ++ ++namespace mx = mlx::core; ++ ++inline std::string mlxDeviceToString(const mx::Device& device) { ++ std::ostringstream os; ++ os << device; ++ return os.str(); ++} ++// Devices need a device description. ++class StablehloMlxDeviceDescription final : public PjRtDeviceDescription { ++ public: ++ explicit StablehloMlxDeviceDescription(int process_id, int local_device_id) ++ : id_(local_device_id), ++ process_index_(process_id), ++ local_hardware_id_(local_device_id), ++ debug_string_(mlxDeviceToString(mx::default_device())), ++ to_string_(absl::StrFormat("%s(id=%d,pid=%d)", debug_string_, ++ id_.value(), process_index_)) { ++ // TRACE_ME_MEMBER; ++ } ++ ++ int id() const override { ++ // TRACE_ME_MEMBER; ++ return id_.value(); ++ } ++ int process_index() const override { ++ // TRACE_ME_MEMBER; ++ return process_index_; ++ } ++ int local_hardware_id() const { ++ // TRACE_ME_MEMBER; ++ return local_hardware_id_; ++ } ++ ++ absl::string_view device_kind() const override { ++ // TRACE_ME_MEMBER; ++ return DebugString(); ++ } ++ ++ absl::string_view DebugString() const override { ++ // TRACE_ME_MEMBER; ++ return debug_string_; ++ } ++ ++ absl::string_view ToString() const override { ++ // TRACE_ME_MEMBER; ++ return to_string_; ++ } ++ ++ const absl::flat_hash_map& Attributes() ++ const override { ++ // TRACE_ME_MEMBER; ++ return attributes_; ++ } ++ ++ private: ++ PjRtGlobalDeviceId id_; ++ int process_index_; ++ int local_hardware_id_; ++ std::string debug_string_; ++ std::string to_string_; ++ absl::flat_hash_map attributes_ = {}; ++}; ++ ++// Clients need devices, and clients own the devices. ++class StablehloMlxDevice : public PjRtDevice { ++ public: ++ explicit StablehloMlxDevice(PjRtClient* client) ++ : PjRtDevice(), client_(client), description_(0, 0) { ++ // TRACE_ME_MEMBER; ++ } ++ ++ absl::string_view DebugString() const override { ++ // TRACE_ME_MEMBER; ++ return "StablehloMlxDevice"; ++ } ++ ++ PjRtLocalDeviceId local_device_id() const override { ++ // TRACE_ME_MEMBER; ++ return PjRtLocalDeviceId(local_hardware_id().value()); ++ } ++ ++ PjRtLocalHardwareId local_hardware_id() const override { ++ // TRACE_ME_MEMBER; ++ return PjRtLocalHardwareId(description_.local_hardware_id()); ++ } ++ ++ PjRtClient* client() const override { ++ // TRACE_ME_MEMBER; ++ return client_; ++ } ++ ++ bool IsAddressable() const override { ++ // TRACE_ME_MEMBER; ++ return process_index() == client()->process_index(); ++ } ++ ++ absl::Status TransferToInfeed(const LiteralSlice& literal) override { ++ // TRACE_ME_MEMBER; ++ return UNIMPLEMENTED(TransferToInfeed); ++ } ++ ++ absl::Status TransferFromOutfeed(MutableBorrowingLiteral literal) override { ++ // TRACE_ME_MEMBER; ++ return UNIMPLEMENTED(TransferFromOutfeed); ++ } ++ ++ void AttachDefaultMemorySpace(PjRtMemorySpace* memory_space) { ++ // TRACE_ME_MEMBER; ++ memory_space_ = memory_space; ++ } ++ ++ absl::Span memory_spaces() const override { ++ // TRACE_ME_MEMBER; ++ return absl::MakeSpan(&memory_space_, 1); ++ } ++ ++ absl::StatusOr default_memory_space() const override { ++ // TRACE_ME_MEMBER; ++ if (!memory_space_) ++ return absl::InternalError("Plugin memory space unset."); ++ ++ return memory_space_; ++ } ++ ++ std::unique_ptr CreateAsyncTrackingEvent( ++ absl::string_view description) const override { ++ // TRACE_ME_MEMBER; ++ LOG(FATAL) << "Plugin does not implement CreateAsyncTrackingEvent."; ++ return nullptr; ++ } ++ ++ const PjRtDeviceDescription& description() const override { ++ // TRACE_ME_MEMBER; ++ return description_; ++ } ++ ++ private: ++ PjRtClient* client_; ++ PjRtMemorySpace* memory_space_; // unpinned memory owned by client ++ StablehloMlxDeviceDescription description_; ++}; ++ ++// Device Description ++std::unique_ptr GetStablehloMlxDeviceDescription( ++ int process_id, int local_device_id) { ++ return std::make_unique(process_id, ++ local_device_id); ++} ++ ++// Mlx Device ++std::unique_ptr GetStablehloMlxDevice(PjRtClient* client) { ++ return std::make_unique(client); ++} ++ ++void AttachStablehloMlxMemorySpace(PjRtDevice* device, ++ PjRtMemorySpace* memory_space) { ++ auto stablehlo_device = dynamic_cast(device); ++ if (stablehlo_device == nullptr) { ++ LOG(FATAL) << "Plugin cannot attach memory space to device of kind " ++ << device->device_kind(); ++ return; ++ } ++ stablehlo_device->AttachDefaultMemorySpace(memory_space); ++} ++ ++} // namespace mlir::stablehlo +\ No newline at end of file +diff --git a/xla/pjrt/plugin/mlx/device.h b/xla/pjrt/plugin/mlx/device.h +new file mode 100644 +index 0000000000..ef42968a3a +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/device.h +@@ -0,0 +1,36 @@ ++/* Copyright 2024 The OpenXLA Authors. ++Licensed under the Apache License, Version 2.0 (the "License"); ++you may not use this file except in compliance with the License. ++You may obtain a copy of the License at ++ http://www.apache.org/licenses/LICENSE-2.0 ++Unless required by applicable law or agreed to in writing, software ++distributed under the License is distributed on an "AS IS" BASIS, ++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++See the License for the specific language governing permissions and ++limitations under the License. ++==============================================================================*/ ++ ++#ifndef XLA_PJRT_PLUGIN_STABLEHLO_MLX_DEVICE_H_ ++#define XLA_PJRT_PLUGIN_STABLEHLO_MLX_DEVICE_H_ ++ ++#include ++ ++#include "xla/pjrt/pjrt_client.h" ++#include "xla/pjrt/pjrt_compiler.h" ++#include "xla/pjrt/pjrt_device_description.h" ++ ++namespace mlir::stablehlo { ++ ++// Device Description ++std::unique_ptr GetStablehloMlxDeviceDescription( ++ int process_id, int local_device_id); ++ ++// MLX Device ++std::unique_ptr GetStablehloMlxDevice(xla::PjRtClient* client); ++ ++void AttachStablehloMlxMemorySpace(xla::PjRtDevice* device, ++ xla::PjRtMemorySpace* memory_space); ++ ++} // namespace mlir::stablehlo ++ ++#endif // XLA_PJRT_PLUGIN_STABLEHLO_MLX_DEVICE_H_ +\ No newline at end of file +diff --git a/xla/pjrt/plugin/mlx/executable.cc b/xla/pjrt/plugin/mlx/executable.cc +new file mode 100644 +index 0000000000..b3dd63d442 +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/executable.cc +@@ -0,0 +1,409 @@ ++/* Copyright 2024 The OpenXLA Authors. ++Licensed under the Apache License, Version 2.0 (the "License"); ++you may not use this file except in compliance with the License. ++You may obtain a copy of the License at ++ http://www.apache.org/licenses/LICENSE-2.0 ++Unless required by applicable law or agreed to in writing, software ++distributed under the License is distributed on an "AS IS" BASIS, ++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++See the License for the specific language governing permissions and ++limitations under the License. ++==============================================================================*/ ++ ++#include "xla/pjrt/plugin/mlx/executable.h" ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++// TODO(@cryptodeal): might need to update `BUILD` ++#include "mlir/IR/Visitors.h" ++#include "absl/log/log.h" ++#include "absl/status/status.h" ++#include "absl/status/statusor.h" ++#include "llvm/ADT/TypeSwitch.h" ++#include "mlir/Bytecode/BytecodeWriter.h" ++#include "mlir/Dialect/Func/IR/FuncOps.h" ++#include "mlir/IR/AsmState.h" ++#include "mlir/IR/BuiltinAttributes.h" ++#include "mlir/IR/MLIRContext.h" ++#include "mlir/IR/Operation.h" ++#include "mlir/IR/OperationSupport.h" ++#include "mlir/IR/OwningOpRef.h" ++#include "mlir/Interfaces/DataLayoutInterfaces.h" ++#include "mlir/Pass/PassManager.h" ++#include "mlir/Support/LLVM.h" ++#include "mlx/compile_impl.h" ++#include "mlx/mlx.h" ++#include "stablehlo/reference/Api.h" ++#include "stablehlo/transforms/optimization/Passes.h" ++#include "xla/hlo/translate/stablehlo.h" ++#include "xla/mlir/utils/error_util.h" ++#include "xla/mlir/utils/type_util.h" ++#include "xla/mlir_hlo/mhlo/transforms/passes.h" ++#include "xla/mlir_hlo/stablehlo_ext/transforms/passes.h" ++#include "xla/pjrt/pjrt_client.h" ++#include "xla/pjrt/pjrt_executable.h" ++#include "xla/pjrt/pjrt_future.h" ++#include "xla/pjrt/plugin/mlx/buffer.h" ++#include "xla/pjrt/plugin/mlx/compile.h" ++#include "xla/pjrt/plugin/mlx/logging.h" ++#include "xla/pjrt/plugin/mlx/utils.h" ++#include "xla/service/computation_placer.h" ++#include "tsl/platform/status.h" ++#include "tsl/platform/statusor.h" ++ ++#define DEBUG_TYPE "stablehlo-pjrt" ++ ++namespace mx = mlx::core; ++ ++namespace mlir::stablehlo { ++ ++#define UNIMPLEMENTED(name) \ ++ xla::Unimplemented("MlxPjRtBuffer::" #name " is not implemented") ++ ++using xla::DeviceAssignment; ++using xla::PjRtBuffer; ++using xla::PjRtClient; ++using xla::PjRtDevice; ++using xla::PjRtFuture; ++using xla::PjRtLoadedExecutable; ++using xla::PjRtMemorySpace; ++ ++mlir::OwningOpRef cloneIntoContext(ModuleOp module, ++ MLIRContext& context) { ++ // Clone the module into the context. MHLO->StableHLO just in case. ++ PassManager pm(module->getContext()); ++ pm.addPass(mhlo::createHloLegalizeToStablehloPass()); ++ if (failed(pm.run(module))) { ++ LOG(ERROR) << "Failed to convert MHLO to StableHLO"; ++ return nullptr; ++ } ++ ++ std::string bytecode; ++ llvm::raw_string_ostream os(bytecode); ++ mlir::BytecodeWriterConfig config; ++ mlir::OwningOpRef cloned = module.clone(); ++ if (mlir::failed(mlir::writeBytecodeToFile(*cloned, os, config))) { ++ LOG(ERROR) << "Failed to write bytecode to string\n"; ++ return nullptr; ++ } ++ return *parseStablehloModule(bytecode, context); ++} ++ ++LogicalResult decomposeChloToStablehlo(ModuleOp module) { ++ PassManager pm(module->getContext()); ++ stablehlo_ext::createChloLegalizeToStablehloPipeline(pm); ++ if (failed(pm.run(module))) { ++ return module->emitError() << "Failed to recompose CHLO"; ++ } ++ return success(); ++} ++ ++LogicalResult runHardwareIndependentOptimizations(ModuleOp module) { ++ PassManager pm(module->getContext()); ++ pm.addNestedPass( ++ stablehlo::createStablehloTargetIndependentOptimizationPass()); ++ if (failed(pm.run(module))) { ++ return module->emitError() ++ << "Failed to run hardware independent optimizations"; ++ } ++ return success(); ++} ++ ++class MlirLoadedExecutable : public PjRtLoadedExecutable { ++ public: ++ MlirLoadedExecutable( ++ ModuleOp module, ++ std::unordered_map mlx_cache, ++ DeviceAssignment assignment, absl::Span devices, ++ PjRtClient* client) ++ : PjRtLoadedExecutable(), ++ name_("MlirLoadedExecutable"), ++ assignment_(assignment), ++ devices_(client->devices()), ++ client_(client), ++ context_(), ++ mlx_cache_(std::move(mlx_cache)), ++ module_(cloneIntoContext(module, context_)) { ++ // TRACE_ME_MEMBER; ++ compiled_module_ = compile::compileModule(module, mlx_cache_); ++ } ++ ++ static absl::StatusOr> CompileAndLoad( ++ ModuleOp module, DeviceAssignment assignment, ++ absl::Span devices, PjRtClient* client) { ++ TRACE_ME; ++ ++ mlir::BaseScopedDiagnosticHandler diagnostic_handler(module->getContext()); ++ if (failed(decomposeChloToStablehlo(module))) { ++ return diagnostic_handler.ConsumeStatus(); ++ } ++ ++ // Simplify the graph using available HWI passes. ++ if (failed(runHardwareIndependentOptimizations(module))) { ++ return diagnostic_handler.ConsumeStatus(); ++ } ++ ++ // Lock mlx mutex whenever a thread modifies the ++ // mlx compilation cache state; since we force compilation ++ // by evaluating with zeroed inputs, the compilation cache ++ // will not be modified by this `Executable` until it is destroyed. ++ std::lock_guard lock(mx::detail::compilation_mutex); ++ ++ auto maybe_mlx_cache = compile::validateAndPrecompile(module); ++ if (!maybe_mlx_cache.ok()) { ++ return maybe_mlx_cache.status(); ++ } ++ ++ // auto device_info = mx::metal::device_info(); ++ // for (const auto& [key, value] : device_info) { ++ // std::cout << key << ": "; ++ // std::visit([](const auto& v) { std::cout << v; }, value); ++ // std::cout << std::endl; ++ // } ++ ++ return std::make_unique( ++ module, std::move(maybe_mlx_cache).value(), assignment, devices, ++ client); ++ } ++ ++ PjRtClient* client() const override { ++ // TRACE_ME_MEMBER; ++ return client_; ++ } ++ ++ const DeviceAssignment& device_assignment() const override { ++ // TRACE_ME_MEMBER; ++ return assignment_; ++ } ++ ++ absl::Span ++ addressable_device_logical_ids() const override { ++ // TRACE_ME_MEMBER; ++ LOG_UNIMPLEMENTED(addressable_device_logical_ids); ++ return {}; ++ } ++ ++ absl::Span addressable_devices() const override { ++ // TRACE_ME_MEMBER; ++ return devices_; ++ } ++ ++ // Helper function to get default mem from device. ++ PjRtMemorySpace* get_default_memory_space() const { ++ // TRACE_ME_MEMBER; ++ return devices_[0]->default_memory_space().value_or(nullptr); ++ } ++ ++ absl::StatusOr ExecuteWithMlxInterpreter( ++ absl::Span argument_handles, ModuleOp module, ++ PjRtDevice* device, bool fill_future) { ++ // TRACE_ME_MEMBER; ++ ++ std::vector inputs; ++ for (auto& buffer : argument_handles) { ++ TF_ASSIGN_OR_RETURN(mx::array input, GetArrayFromBuffer(buffer)); ++ inputs.push_back(std::move(input)); ++ } ++ std::vector mlx_res; ++ { ++ // Lock mlx compilation mutex while calling compiled ++ // functions, since it performs a lookup on MLX compilation ++ // cache, which might be modified by other threads. ++ std::lock_guard lock(mx::detail::compilation_mutex); ++ mlx_res = compiled_module_->call(inputs); ++ } ++ // TODO (@cryptodeal): figure out best practice for returning strided data ++ for (auto i = 0; i < mlx_res.size(); ++i) { ++ mlx_res[i] = mx::contiguous(mlx_res[i]); ++ } ++ mx::eval(mlx_res); ++ ++ // Naive memory space selection, only using CPU global memory. ++ PjRtMemorySpace* memory_space = ++ device->default_memory_space().value_or(nullptr); ++ ++ std::vector> buffer_results(mlx_res.size()); ++ for (auto i = 0; i < mlx_res.size(); i++) { ++ buffer_results[i] = ++ CreateMlirBufferFromMlxArray(mlx_res[i], memory_space); ++ } ++ ++ std::optional> future; ++ if (fill_future) { ++ // Synchronous! To make async, this would need to return a future that ++ // is ready when the computation is done. ++ future = PjRtFuture<>(absl::OkStatus()); ++ } ++ ++ return PjRtLoadedExecutable::Result{future, std::move(buffer_results)}; ++ } ++ ++ absl::StatusOr>>> Execute( ++ absl::Span> argument_handles, ++ const xla::ExecuteOptions& options, ++ std::optional>>& returned_futures) override { ++ // TRACE_ME_MEMBER; ++ if (argument_handles.size() != 1) { ++ // One arg handle per device. ++ return absl::InvalidArgumentError( ++ "MlirLoadedExecutable::Execute only supports a single argument " ++ "vector"); ++ } ++ ++ // Single device, synchronous, can always use 0. ++ PjRtDevice* device = devices_[0]; ++ bool fill_future = returned_futures.has_value(); ++ TF_ASSIGN_OR_RETURN( ++ PjRtLoadedExecutable::Result result, ++ ExecuteWithMlxInterpreter(argument_handles[0], module_.get(), device, ++ fill_future)); ++ std::vector>> results; ++ results.push_back(std::move(result.buffers)); ++ if (returned_futures.has_value()) { ++ returned_futures->push_back(std::move(result.future.value())); ++ } ++ return results; ++ } ++ ++ absl::StatusOr>> ExecuteSharded( ++ absl::Span argument_handles, PjRtDevice* device, ++ const xla::ExecuteOptions& options, ++ std::optional>& returned_future, bool fill_future) override { ++ // TRACE_ME_MEMBER; ++ // Synchronous! To make async, have the device make a buffer with a ready ++ // future that is ready when the computation is done / buffer is ready. ++ TF_ASSIGN_OR_RETURN( ++ PjRtLoadedExecutable::Result result, ++ ExecuteWithMlxInterpreter(argument_handles, module_.get(), device, ++ fill_future)); ++ if (returned_future.has_value() && fill_future) { ++ returned_future = std::move(result.future); ++ } ++ return std::move(result.buffers); ++ } ++ ++ absl::StatusOr>> ExecutePortable( ++ absl::Span argument_handles, PjRtDevice* device, ++ const xla::ExecuteOptions& options, ++ std::optional>& returned_future, bool fill_future) override { ++ // TRACE_ME_MEMBER; ++ // Synchronous! To make async, have the device make a buffer with a ready ++ // future that is ready when the computation is done / buffer is ready. ++ TF_ASSIGN_OR_RETURN( ++ PjRtLoadedExecutable::Result result, ++ ExecuteWithMlxInterpreter(argument_handles, module_.get(), device, ++ fill_future)); ++ if (returned_future.has_value() && fill_future) { ++ returned_future = std::move(result.future); ++ } ++ return std::move(result.buffers); ++ } ++ ++ // TODO (@cryptodeal): Implement `SerializeExecutable` ++ // absl::StatusOr SerializeExecutable() {} ++ ++ void Delete() override { ++ // TRACE_ME_MEMBER; ++ { ++ // lock mutex and clear compiled functions owned by this executable ++ std::lock_guard lock(mx::detail::compilation_mutex); ++ mlx_cache_.clear(); ++ compiled_module_ = nullptr; ++ } ++ module_.release(); ++ module_ = nullptr; ++ } ++ ++ bool IsDeleted() override { ++ // TRACE_ME_MEMBER; ++ return !module_; ++ } ++ ++ // PjRtExecutable API. ++ int num_replicas() const override { ++ // TRACE_ME_MEMBER; ++ return assignment_.replica_count(); ++ } ++ ++ int num_partitions() const override { ++ // TRACE_ME_MEMBER; ++ return assignment_.computation_count(); ++ } ++ ++ int64_t SizeOfGeneratedCodeInBytes() const override { ++ // No generated code.. so just return 1. ++ // TRACE_ME_MEMBER; ++ return 1; ++ } ++ ++ absl::string_view name() const override { ++ // TRACE_ME_MEMBER; ++ return name_; ++ } ++ ++ absl::StatusOr>> GetHloModules() ++ const override { ++ // TODO: This shouldn't be needed for an MLIR plugin, its only used in the ++ // JAX layer for determining output sharding, which exists on the mlir ++ // module. ++ // TRACE_ME_MEMBER; ++ auto moduleClone = llvm::cast(module_.get()->clone()); ++ TF_ASSIGN_OR_RETURN(auto hlo_module, ++ xla::ConvertStablehloToHlo(moduleClone)); ++ return std::vector>{std::move(hlo_module)}; ++ } ++ ++ absl::StatusOr>> ++ GetOutputMemoryKinds() const override { ++ // TRACE_ME_MEMBER; ++ return UNIMPLEMENTED(GetOutputMemoryKinds); ++ } ++ ++ absl::StatusOr FingerprintExecutable() const override { ++ // TRACE_ME_MEMBER; ++ return UNIMPLEMENTED(FingerprintExecutable); ++ } ++ ++ private: ++ std::string name_; ++ DeviceAssignment assignment_; ++ absl::Span devices_; ++ PjRtClient* client_; ++ // MLX compilation context ++ std::unordered_map mlx_cache_; ++ utils::MlxFuncRef* compiled_module_ = nullptr; ++ ++ // MLIR ++ MLIRContext context_; ++ mlir::OwningOpRef module_; ++}; ++ ++absl::StatusOr> StablehloMlxCompile( ++ mlir::ModuleOp module, DeviceAssignment assignment, PjRtClient* client) { ++ TRACE_ME; ++ return MlirLoadedExecutable::CompileAndLoad(module, assignment, ++ client->devices(), client); ++} ++ ++absl::StatusOr> StablehloMlxCompile( ++ xla::XlaComputation const& computation, xla::DeviceAssignment assignment, ++ xla::PjRtClient* client) { ++ TRACE_ME; ++ MLIRContext context; ++ TF_ASSIGN_OR_RETURN(auto module, ++ ConvertHloToStablehlo(context, &computation.proto())); ++ return StablehloMlxCompile(module.get(), assignment, client); ++} ++ ++} // namespace mlir::stablehlo +diff --git a/xla/pjrt/plugin/mlx/executable.h b/xla/pjrt/plugin/mlx/executable.h +new file mode 100644 +index 0000000000..cc11793e50 +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/executable.h +@@ -0,0 +1,36 @@ ++/* Copyright 2024 The OpenXLA Authors. ++Licensed under the Apache License, Version 2.0 (the "License"); ++you may not use this file except in compliance with the License. ++You may obtain a copy of the License at ++ http://www.apache.org/licenses/LICENSE-2.0 ++Unless required by applicable law or agreed to in writing, software ++distributed under the License is distributed on an "AS IS" BASIS, ++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++See the License for the specific language governing permissions and ++limitations under the License. ++==============================================================================*/ ++ ++#ifndef XLA_PJRT_PLUGIN_STABLEHLO_MLX_EXECUTABLE_H_ ++#define XLA_PJRT_PLUGIN_STABLEHLO_MLX_EXECUTABLE_H_ ++#include ++ ++#include "mlir/IR/BuiltinOps.h" ++#include "mlx/mlx.h" ++#include "xla/hlo/builder/xla_computation.h" ++#include "xla/pjrt/pjrt_client.h" ++#include "xla/pjrt/plugin/mlx/utils.h" ++#include "xla/service/computation_placer.h" ++ ++namespace mx = mlx::core; ++namespace mlir::stablehlo { ++ ++absl::StatusOr> StablehloMlxCompile( ++ mlir::ModuleOp module, xla::DeviceAssignment assignment, ++ xla::PjRtClient* client); ++ ++absl::StatusOr> StablehloMlxCompile( ++ xla::XlaComputation const& computation, xla::DeviceAssignment assignment, ++ xla::PjRtClient* client); ++ ++} // namespace mlir::stablehlo ++#endif // XLA_PJRT_PLUGIN_STABLEHLO_MLX_EXECUTABLE_H_ +\ No newline at end of file +diff --git a/xla/pjrt/plugin/mlx/logging.cc b/xla/pjrt/plugin/mlx/logging.cc +new file mode 100644 +index 0000000000..3f21f67baf +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/logging.cc +@@ -0,0 +1,89 @@ ++/* Copyright 2024 The OpenXLA Authors. ++Licensed under the Apache License, Version 2.0 (the "License"); ++you may not use this file except in compliance with the License. ++You may obtain a copy of the License at ++ http://www.apache.org/licenses/LICENSE-2.0 ++Unless required by applicable law or agreed to in writing, software ++distributed under the License is distributed on an "AS IS" BASIS, ++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++See the License for the specific language governing permissions and ++limitations under the License. ++==============================================================================*/ ++ ++#include "xla/pjrt/plugin/mlx/logging.h" ++ ++#include ++ ++#include "absl/base/log_severity.h" ++#include "absl/log/globals.h" ++#include "llvm/Support/raw_ostream.h" ++#include "mlir/IR/Attributes.h" ++#include "mlir/IR/BuiltinAttributes.h" ++#include "mlir/IR/Operation.h" ++ ++namespace mlir::stablehlo { ++ ++std::string ToString(mlir::Attribute attr) { ++ std::string out; ++ llvm::raw_string_ostream os(out); ++ attr.print(os); ++ return out; ++} ++ ++std::string ToString(SmallVector attrs) { ++ std::string out; ++ llvm::raw_string_ostream os(out); ++ os << "["; ++ bool first = true; ++ for (auto attr : attrs) { ++ if (!first) os << ", "; ++ first = false; ++ attr.print(os); ++ } ++ os << "]"; ++ return out; ++} ++std::string ToString(Operation& op) { ++ std::string out; ++ llvm::raw_string_ostream os(out); ++ op.print(os); ++ return out; ++} ++ ++std::string ToString(Operation* op) { ++ std::string out; ++ llvm::raw_string_ostream os(out); ++ os << *op; ++ return out; ++} ++ ++std::string ToString(mlir::Value value) { ++ std::string out; ++ llvm::raw_string_ostream os(out); ++ value.print(os); ++ return out; ++} ++ ++std::string ToString(mlir::Block& block) { ++ std::string out; ++ llvm::raw_string_ostream os(out); ++ block.print(os); ++ return out; ++} ++ ++void SetupLogLevelFromEnv() { ++ absl::SetMinLogLevel(absl::LogSeverityAtLeast::kError); ++ const char* log_env = std::getenv("PJRT_LOG_LEVEL"); ++ if (!log_env) return; ++ if (strcmp(log_env, "INFO") == 0) { ++ absl::SetMinLogLevel(absl::LogSeverityAtLeast::kInfo); ++ } else if (strcmp(log_env, "WARNING") == 0) { ++ absl::SetMinLogLevel(absl::LogSeverityAtLeast::kWarning); ++ } else if (strcmp(log_env, "ERROR") == 0) { ++ absl::SetMinLogLevel(absl::LogSeverityAtLeast::kError); ++ } else { ++ LOG(ERROR) << "Invalid PJRT_LOG_LEVEL: " << log_env; ++ } ++} ++ ++} // namespace mlir::stablehlo +\ No newline at end of file +diff --git a/xla/pjrt/plugin/mlx/logging.h b/xla/pjrt/plugin/mlx/logging.h +new file mode 100644 +index 0000000000..58f296ab2f +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/logging.h +@@ -0,0 +1,42 @@ ++/* Copyright 2024 The OpenXLA Authors. ++Licensed under the Apache License, Version 2.0 (the "License"); ++you may not use this file except in compliance with the License. ++You may obtain a copy of the License at ++ http://www.apache.org/licenses/LICENSE-2.0 ++Unless required by applicable law or agreed to in writing, software ++distributed under the License is distributed on an "AS IS" BASIS, ++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++See the License for the specific language governing permissions and ++limitations under the License. ++==============================================================================*/ ++ ++#ifndef XLA_PJRT_PLUGIN_STABLEHLO_MLX_LOGGING_H_ ++#define XLA_PJRT_PLUGIN_STABLEHLO_MLX_LOGGING_H_ ++ ++// This file has some joint logging to allow LOG and VLOG to play well with ++// MLIR data structures ++ ++#include "absl/log/log.h" ++#include "mlir/IR/Attributes.h" ++#include "mlir/IR/BuiltinAttributes.h" ++ ++#define LOG_UNIMPLEMENTED(name) \ ++ LOG(ERROR) << "MlxPjRtBuffer::" #name " is not implemented" ++ ++#define TRACE_ME LOG(INFO) << __func__ << "\n"; ++ ++#define TRACE_ME_MEMBER LOG(INFO) << __func__ << "(" << (void*)this << ")\n"; ++ ++namespace mlir::stablehlo { ++std::string ToString(mlir::Attribute attr); ++std::string ToString(llvm::SmallVector attrs); ++std::string ToString(Operation* op); ++std::string ToString(Operation& op); ++std::string ToString(mlir::Value op); ++std::string ToString(mlir::Block& block); ++ ++// Looks for `PJRT_LOG_LEVEL = INFO|WARNING|ERROR` in env variables. ++void SetupLogLevelFromEnv(); ++} // namespace mlir::stablehlo ++ ++#endif // XLA_PJRT_PLUGIN_STABLEHLO_MLX_LOGGING_H_ +\ No newline at end of file +diff --git a/xla/pjrt/plugin/mlx/plugin_pjrt_test.cc b/xla/pjrt/plugin/mlx/plugin_pjrt_test.cc +new file mode 100644 +index 0000000000..d56a22565f +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/plugin_pjrt_test.cc +@@ -0,0 +1,33 @@ ++/* Copyright 2022 The OpenXLA Authors. ++Licensed under the Apache License, Version 2.0 (the "License"); ++you may not use this file except in compliance with the License. ++You may obtain a copy of the License at ++ http://www.apache.org/licenses/LICENSE-2.0 ++Unless required by applicable law or agreed to in writing, software ++distributed under the License is distributed on an "AS IS" BASIS, ++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++See the License for the specific language governing permissions and ++limitations under the License. ++==============================================================================*/ ++ ++#include "xla/pjrt/c/pjrt_c_api_test.h" ++#include "xla/pjrt/c/pjrt_c_api_wrapper_impl.h" ++#include "xla/pjrt/pjrt_client_test.h" ++#include "xla/pjrt/plugin/mlx/client_c_pjrt.h" ++#include "xla/pjrt/plugin/mlx/client_cpp_pjrt.h" ++ ++namespace pjrt { ++namespace { ++ ++const bool kUnused = ++ (RegisterPjRtCApiTestFactory([]() { return GetPjrtApi(); }, ++ /*platform_name=*/"stablehlo_mlx"), ++ true); ++ ++const bool kUnused2 = (xla::RegisterTestClientFactory([]() { ++ return mlir::stablehlo::CreateStablehloMlxPjrtClient(); ++ }), ++ true); ++ ++} // namespace ++} // namespace pjrt +\ No newline at end of file +diff --git a/xla/pjrt/plugin/mlx/utils.cc b/xla/pjrt/plugin/mlx/utils.cc +new file mode 100644 +index 0000000000..643a842d46 +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/utils.cc +@@ -0,0 +1,692 @@ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "llvm/ADT/Hashing.h" ++#include "mlir/IR/OperationSupport.h" ++#include "mlir/Support/LLVM.h" ++#include "xla/mlir/utils/type_util.h" ++#include "xla/pjrt/plugin/mlx/utils.h" ++#include "xla/pjrt/plugin/mlx/logging.h" ++#include "xla/shape_util.h" ++#include "xla/util.h" ++ ++std::tuple shapeAndBytes( ++ absl::Span dims, ++ std::optional> byte_strides, mx::Dtype dtype) { ++ mx::Shape shape(dims.size()); ++ mx::Strides strides; ++ mx::Shape raw_buffer_shape = {shape.size() && !byte_strides.has_value() ? 1 ++ : 0}; ++ for (auto i = 0; i < dims.size(); ++i) { ++ shape[i] = static_cast(dims[i]); ++ if (byte_strides.has_value()) { ++ auto stride_value = byte_strides.value()[i]; ++ strides.push_back(stride_value / dtype.size()); ++ if (strides[i] != 0) { ++ raw_buffer_shape[0] = std::max( ++ raw_buffer_shape[0], static_cast(stride_value) * shape[i]); ++ } ++ } else { ++ raw_buffer_shape[0] *= static_cast(dims[i]); ++ } ++ } ++ if (!byte_strides.has_value()) { ++ raw_buffer_shape[0] *= dtype.size(); ++ } else if (raw_buffer_shape[0] == 0 and ++ std::find(dims.begin(), dims.end(), 0) == dims.end()) { ++ raw_buffer_shape[0] += dtype.size(); ++ } ++ return std::make_tuple(shape, raw_buffer_shape, strides); ++} ++ ++namespace utils { ++namespace detail { ++// Returns true if the given `ReduceOp` matches ++// `zml.Tensor.argMax` op implementation. ++bool isArgMaxReduce(mlir::Block& body_block) { ++ unsigned op_count = 0; ++ // TODO (@cryptodeal): This is a lazy verification; maybe check operands ++ // for each operation in the block. ++ for (auto& op : body_block.getOperations()) { ++ switch (op_count) { ++ case 0: { ++ auto compare_op = llvm::dyn_cast(op); ++ if (compare_op && compare_op.getComparisonDirection() == ++ mlir::stablehlo::ComparisonDirection::GT) { ++ break; ++ } else ++ return false; ++ } ++ case 1: { ++ auto compare_op = llvm::dyn_cast(op); ++ if (compare_op && compare_op.getComparisonDirection() == ++ mlir::stablehlo::ComparisonDirection::NE) { ++ break; ++ } else ++ return false; ++ } ++ case 2: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ case 3: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ case 4: { ++ auto compare_op = llvm::dyn_cast(op); ++ if (compare_op && compare_op.getComparisonDirection() == ++ mlir::stablehlo::ComparisonDirection::EQ) { ++ break; ++ } else ++ return false; ++ } ++ case 5: { ++ auto compare_op = llvm::dyn_cast(op); ++ if (compare_op && compare_op.getComparisonDirection() == ++ mlir::stablehlo::ComparisonDirection::LT) { ++ break; ++ } else ++ return false; ++ } ++ case 6: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ case 7: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ case 8: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ case 9: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ default: ++ return false; ++ } ++ op_count++; ++ } ++ return true; ++} ++ ++// Returns true if the given `ReduceOp` matches ++// `zml.Tensor.sum` op implementation. ++bool isSumReduce(mlir::Block& body_block) { ++ unsigned op_count = 0; ++ // TODO (@cryptodeal): This is a lazy verification; maybe check operands ++ // for each operation in the block. ++ for (auto& op : body_block.getOperations()) { ++ switch (op_count) { ++ case 0: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ case 1: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ } ++ op_count++; ++ } ++ return true; ++} ++ ++// Returns true if the given `ReduceOp` matches ++// `zml.Tensor.max` op implementation. ++bool isMaxReduce(mlir::Block& body_block) { ++ unsigned op_count = 0; ++ // TODO (@cryptodeal): This is a lazy verification; maybe check operands ++ // for each operation in the block. ++ for (auto& op : body_block.getOperations()) { ++ switch (op_count) { ++ case 0: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ case 1: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ } ++ op_count++; ++ } ++ return true; ++} ++ ++// Returns true if the given `ReduceOp` matches ++// `zml.Tensor.max` op implementation. ++bool isMinReduce(mlir::Block& body_block) { ++ unsigned op_count = 0; ++ // TODO (@cryptodeal): This is a lazy verification; maybe check operands ++ // for each operation in the block. ++ for (auto& op : body_block.getOperations()) { ++ switch (op_count) { ++ case 0: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ case 1: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ } ++ op_count++; ++ } ++ return true; ++} ++ ++// Returns true if the given `ReduceOp` matches ++// `zml.Tensor.any` op implementation. ++bool isAnyReduce(mlir::Block& body_block) { ++ unsigned op_count = 0; ++ // TODO (@cryptodeal): This is a lazy verification; maybe check operands ++ // for each operation in the block. ++ for (auto& op : body_block.getOperations()) { ++ switch (op_count) { ++ case 0: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ case 1: ++ if (llvm::dyn_cast(op)) { ++ break; ++ } else ++ return false; ++ } ++ op_count++; ++ } ++ return true; ++} ++ ++} // namespace detail ++ ++namespace dtype { ++xla::PrimitiveType asXlaPrimitiveType(mx::Dtype dtype) { ++ switch (dtype) { ++ case mx::bool_: ++ return xla::PrimitiveType::PRED; ++ case mx::uint8: ++ return xla::PrimitiveType::U8; ++ case mx::uint16: ++ return xla::PrimitiveType::U16; ++ case mx::uint32: ++ return xla::PrimitiveType::U32; ++ case mx::uint64: ++ return xla::PrimitiveType::U64; ++ case mx::int8: ++ return xla::PrimitiveType::S8; ++ case mx::int16: ++ return xla::PrimitiveType::S16; ++ case mx::int32: ++ return xla::PrimitiveType::S32; ++ case mx::int64: ++ return xla::PrimitiveType::S64; ++ case mx::float16: ++ return xla::PrimitiveType::F16; ++ case mx::float32: ++ return xla::PrimitiveType::F32; ++ case mx::bfloat16: ++ return xla::PrimitiveType::BF16; ++ case mx::complex64: ++ return xla::PrimitiveType::C64; ++ } ++} ++ ++absl::StatusOr fromXlaPrimitiveType(xla::PrimitiveType dtype) { ++ switch (dtype) { ++ case xla::PrimitiveType::PRED: ++ return mx::bool_; ++ case xla::PrimitiveType::BF16: ++ return mx::bfloat16; ++ case xla::PrimitiveType::F16: ++ return mx::float16; ++ case xla::PrimitiveType::F32: ++ return mx::float32; ++ case xla::PrimitiveType::S8: ++ return mx::int8; ++ case xla::PrimitiveType::S16: ++ return mx::int16; ++ case xla::PrimitiveType::S32: ++ return mx::int32; ++ case xla::PrimitiveType::S64: ++ return mx::int64; ++ case xla::PrimitiveType::U8: ++ return mx::uint8; ++ case xla::PrimitiveType::U16: ++ return mx::uint16; ++ case xla::PrimitiveType::U32: ++ return mx::uint32; ++ case xla::PrimitiveType::U64: ++ return mx::uint64; ++ case xla::PrimitiveType::C64: ++ return mx::complex64; ++ default: ++ return absl::InvalidArgumentError("Unsupported type"); ++ } ++} ++ ++absl::StatusOr fromMlirType(mlir::Type type) { ++ auto primitive_type = xla::ConvertMlirTypeToPrimitiveType(type); ++ if (primitive_type == xla::PrimitiveType::PRIMITIVE_TYPE_INVALID) { ++ return xla::Internal("Unsupported type: %s", ++ xla::PrimitiveType_Name(primitive_type)); ++ } ++ return fromXlaPrimitiveType(primitive_type); ++} ++} // namespace dtype ++ ++namespace array { ++absl::StatusOr fromHostBuffer( ++ const void* data, absl::Span dims, ++ std::optional> byte_strides, ++ xla::PrimitiveType type) { ++ TF_ASSIGN_OR_RETURN(mx::Dtype dtype, dtype::fromXlaPrimitiveType(type)); ++ auto [shape, raw_buffer_shape, strides] = ++ shapeAndBytes(dims, byte_strides, dtype); ++ mx::array typed_view = ++ mx::view(mx::array(reinterpret_cast(data), ++ raw_buffer_shape, mx::uint8), ++ dtype); ++ if (byte_strides.has_value()) { ++ auto res = mx::contiguous(mx::as_strided(typed_view, shape, strides, 0)); ++ res.eval(); ++ return res; ++ } else { ++ return mx::reshape(typed_view, shape); ++ } ++} ++ ++absl::StatusOr fromHostLiteral(const xla::LiteralSlice& literal) { ++ return fromHostBuffer(literal.untyped_data(), literal.shape().dimensions(), ++ std::nullopt, literal.shape().element_type()); ++} ++ ++template ++absl::StatusOr fromDenseElementsAttr(mlir::DenseElementsAttr attr) { ++ auto attr_type = attr.getType(); ++ // convert to mlx shape ++ std::vector shape(attr_type.getShape().begin(), ++ attr_type.getShape().end()); ++ ++ // handle splat values ++ if (attr.isSplat()) { ++ if constexpr (std::is_same::value) { ++ return mx::full( ++ shape, static_cast(attr.getSplatValue())); ++ } else if constexpr (std::is_same::value) { ++ return mx::full( ++ shape, static_cast(attr.getSplatValue())); ++ } else if constexpr (std::is_same>::value) { ++ return mx::full( ++ shape, static_cast(attr.getSplatValue())); ++ } else { ++ return mx::full(shape, attr.getSplatValue()); ++ } ++ } ++ ++ // handle non-splat values that are expanded to fit shape ++ auto it = attr.getValues(); ++ auto buffer = llvm::to_vector(it); ++ if (attr.size() != attr.getNumElements()) { ++ if constexpr (std::is_same::value) { ++ return mx::full( ++ shape, ++ mx::array(reinterpret_cast(buffer.data()), ++ {static_cast(attr.size())})); ++ } else if constexpr (std::is_same::value) { ++ return mx::full( ++ shape, ++ mx::array(reinterpret_cast(buffer.data()), ++ {static_cast(attr.size())})); ++ } else if constexpr (std::is_same>::value) { ++ return mx::full( ++ shape, ++ mx::array(reinterpret_cast(buffer.data()), ++ {static_cast(attr.size())})); ++ } else { ++ return mx::full(shape, mx::array(buffer.begin(), ++ {static_cast(attr.size())})); ++ } ++ } ++ ++ if constexpr (std::is_same::value) { ++ return mx::array(reinterpret_cast(buffer.data()), ++ shape); ++ } else if constexpr (std::is_same::value) { ++ return mx::array(reinterpret_cast(buffer.data()), ++ shape); ++ } else if constexpr (std::is_same>::value) { ++ return mx::array(reinterpret_cast(buffer.data()), ++ shape); ++ } else { ++ return mx::array(buffer.begin(), shape); ++ } ++} ++ ++absl::StatusOr fromDenseElementsAttr(mlir::DenseElementsAttr attr) { ++ auto element_type = ++ xla::ConvertMlirTypeToPrimitiveType(attr.getType().getElementType()); ++ switch (element_type) { ++ case xla::PrimitiveType::PRED: ++ return fromDenseElementsAttr(attr); ++ case xla::PrimitiveType::U8: ++ return fromDenseElementsAttr(attr); ++ case xla::PrimitiveType::S8: ++ return fromDenseElementsAttr(attr); ++ case xla::PrimitiveType::U16: ++ return fromDenseElementsAttr(attr); ++ case xla::PrimitiveType::S16: ++ return fromDenseElementsAttr(attr); ++ case xla::PrimitiveType::U32: ++ return fromDenseElementsAttr(attr); ++ case xla::PrimitiveType::S32: ++ return fromDenseElementsAttr(attr); ++ case xla::PrimitiveType::U64: ++ return fromDenseElementsAttr(attr); ++ case xla::PrimitiveType::S64: ++ return fromDenseElementsAttr(attr); ++ case xla::PrimitiveType::F16: ++ return fromDenseElementsAttr(attr); ++ case xla::PrimitiveType::BF16: ++ return fromDenseElementsAttr(attr); ++ case xla::PrimitiveType::F32: ++ return fromDenseElementsAttr(attr); ++ case xla::PrimitiveType::C64: ++ return fromDenseElementsAttr>(attr); ++ default: ++ return xla::Internal("Unsupported type: %s", ++ xla::PrimitiveType_Name(element_type)); ++ } ++} ++ ++mx::array fromOperand( ++ const mlir::Value& operand, ++ const std::unordered_map>& ++ block_ctx, ++ const std::vector& inputs, bool debug) { ++ if (auto defining_op = operand.getDefiningOp(); defining_op) { ++ if (auto search = block_ctx.find(defining_op); search != block_ctx.end()) { ++ for (auto i = 0; i < defining_op->getNumResults(); i++) { ++ mlir::Value maybe_res = defining_op->getResult(i); ++ if (maybe_res == operand) { ++ if (debug) ++ std::cout << "Operator from block context @ idx:" ++ << std::to_string(i) << std::endl ++ << mlir::stablehlo::ToString(defining_op).c_str() ++ << std::endl ++ << std::endl; ++ return search->second[i]; ++ } ++ } ++ if (debug) ++ std::cout << "Operator from block context not found:" << std::endl ++ << mlir::stablehlo::ToString(defining_op).c_str() << std::endl ++ << std::endl; ++ } else { ++ std::cout << "Defining op not found in block context: " ++ << mlir::stablehlo::ToString(defining_op).c_str() << std::endl; ++ return inputs[llvm::cast(operand).getArgNumber()]; ++ } ++ } ++ auto block_arg = llvm::cast(operand); ++ if (debug) ++ std::cout << "Operator from block argument @ idx:" ++ << std::to_string(block_arg.getArgNumber()) << std::endl ++ << std::endl; ++ return inputs[block_arg.getArgNumber()]; ++} ++ ++} // namespace array ++ ++void printVector(const std::string& name, const std::vector& vec, ++ bool indent) { ++ if (indent) { ++ std::cout << "\t"; ++ } ++ std::cout << name.c_str() << ": { "; ++ for (auto i = 0; i < vec.size(); i++) { ++ std::cout << std::to_string(vec[i]); ++ if (i < vec.size() - 1) { ++ std::cout << ", "; ++ } ++ } ++ std::cout << " }" << std::endl; ++} ++ ++absl::StatusOr getScatterType(mlir::Block& block) { ++ unsigned op_count = 0; ++ utils::ScatterType scatter_type = utils::ScatterType::Replace; ++ for (const auto& op : block.getOperations()) { ++ switch (op_count) { ++ case 0: ++ if (llvm::dyn_cast(op)) { ++ return scatter_type; ++ } ++ if (llvm::dyn_cast(op)) { ++ scatter_type = ScatterType::Add; ++ } else if (llvm::dyn_cast(op)) { ++ scatter_type = ScatterType::Prod; ++ } else if (llvm::dyn_cast(op)) { ++ scatter_type = ScatterType::Prod; ++ } else if (llvm::dyn_cast(op)) { ++ scatter_type = ScatterType::Max; ++ } else if (llvm::dyn_cast(op)) { ++ scatter_type = ScatterType::Min; ++ } else { ++ return xla::Internal("Unsupported comparator: %s", ++ mlir::stablehlo::ToString(block)); ++ } ++ break; ++ case 1: ++ if (llvm::dyn_cast(op)) { ++ return scatter_type; ++ } ++ default: ++ return xla::Internal("Unsupported comparator: %s", ++ mlir::stablehlo::ToString(block)); ++ } ++ } ++} ++ ++absl::StatusOr> getSortInfo( ++ mlir::Block& block) { ++ unsigned op_count = 0; ++ mx::ComparatorType comparator = mx::ComparatorType::LessThan; ++ unsigned input_idx = 0; ++ for (auto& op : block.getOperations()) { ++ switch (op_count) { ++ case 0: { ++ if (auto compare_op = llvm::dyn_cast(op)) { ++ switch (compare_op.getComparisonDirection()) { ++ case mlir::stablehlo::ComparisonDirection::GT: { ++ auto lhs = ++ llvm::dyn_cast(compare_op.getLhs()); ++ auto rhs = ++ llvm::dyn_cast(compare_op.getRhs()); ++ if (lhs && rhs) { ++ unsigned lhs_idx = static_cast(lhs.getArgNumber()); ++ unsigned rhs_idx = static_cast(rhs.getArgNumber()); ++ if (rhs_idx - 1 == lhs_idx) { ++ comparator = mx::ComparatorType::GreaterThan; ++ input_idx = lhs_idx == 0 ? 0 : lhs_idx / 2; ++ break; ++ } ++ } ++ return xla::Internal("Unsupported comparator: %s", ++ mlir::stablehlo::ToString(block)); ++ } ++ case mlir::stablehlo::ComparisonDirection::LT: { ++ auto lhs = ++ llvm::dyn_cast(compare_op.getLhs()); ++ auto rhs = ++ llvm::dyn_cast(compare_op.getRhs()); ++ if (lhs && rhs) { ++ unsigned lhs_idx = static_cast(lhs.getArgNumber()); ++ unsigned rhs_idx = static_cast(rhs.getArgNumber()); ++ if (rhs_idx - 1 == lhs_idx) { ++ comparator = mx::ComparatorType::LessThan; ++ input_idx = lhs_idx == 0 ? 0 : lhs_idx / 2; ++ break; ++ } ++ } ++ return xla::Internal("Unsupported comparator: %s", ++ mlir::stablehlo::ToString(block)); ++ } ++ default: ++ return xla::Internal("Unsupported comparator: %s", ++ mlir::stablehlo::ToString(block)); ++ } ++ break; ++ } ++ } ++ case 1: ++ if (llvm::dyn_cast(op)) break; ++ default: ++ return xla::Internal("Unsupported comparator: %s", ++ mlir::stablehlo::ToString(block)); ++ } ++ op_count++; ++ } ++ return std::make_pair(input_idx, comparator); ++} ++ ++llvm::hash_code computeHash( ++ mlir::Operation* op, ++ llvm::function_ref hashOperands, ++ llvm::function_ref hashResults) { ++ // Hash operations based upon their: ++ // - Operation Name ++ // - Attributes ++ // - Properties ++ ++ // Potentially need further specialization for: ++ // - stablehlo::ScatterOp (requires determining the scatter type) ++ ++ llvm::hash_code hash = llvm::hash_combine( ++ op->getName(), op->getRawDictionaryAttrs(), op->hashProperties()); ++ ++ // - Operands ++ if (op->hasTrait() && ++ op->getNumOperands() > 0) { ++ size_t operandHash = hashOperands(op->getOperand(0)); ++ for (auto operand : op->getOperands().drop_front()) ++ operandHash += hashOperands(operand); ++ hash = llvm::hash_combine(hash, operandHash); ++ } else { ++ for (mlir::Value operand : op->getOperands()) ++ hash = llvm::hash_combine(hash, hashOperands(operand)); ++ } ++ ++ // - Results ++ for (mlir::Value result : op->getResults()) ++ hash = llvm::hash_combine(hash, hashResults(result)); ++ ++ return hash; ++} ++ ++absl::StatusOr getReduceType(mlir::Operation& op, ++ mlir::Block& body_block) { ++ if (detail::isArgMaxReduce(body_block)) { ++ return ReduceType::ArgMax; ++ } else if (detail::isSumReduce(body_block)) { ++ return ReduceType::Sum; ++ } else if (detail::isMaxReduce(body_block)) { ++ return ReduceType::Max; ++ } else if (detail::isMinReduce(body_block)) { ++ return ReduceType::Min; ++ } else if (detail::isAnyReduce(body_block)) { ++ return ReduceType::Any; ++ } else { ++ return xla::Internal("Unsupported reduce op: %s", ++ mlir::stablehlo::ToString(op)); ++ } ++} ++ ++uint64_t getHashValue(mlir::func::FuncOp func_op) { ++ llvm::hash_code hash = {}; ++ for (mlir::NamedAttribute attr : func_op->getAttrs()) { ++ if (attr.getName() == func_op.getSymNameAttrName()) { ++ continue; ++ } ++ hash = llvm::hash_combine(hash, attr); ++ } ++ func_op.getBody().walk([&](mlir::Operation* op) { ++ hash = llvm::hash_combine( ++ hash, computeHash(op, mlir::OperationEquivalence::ignoreHashValue, ++ mlir::OperationEquivalence::ignoreHashValue)); ++ }); ++ return hash; ++} ++ ++uint64_t getHashValue(mlir::Operation* op) { ++ return computeHash(op, mlir::OperationEquivalence::ignoreHashValue, ++ mlir::OperationEquivalence::ignoreHashValue); ++} ++ ++bool isValidType(mlir::Type type) { ++ auto primitive_type = xla::ConvertMlirTypeToPrimitiveType( ++ mlir::cast(type).getElementType()); ++ switch (primitive_type) { ++ case xla::PrimitiveType::PRED: ++ case xla::PrimitiveType::U8: ++ case xla::PrimitiveType::S8: ++ case xla::PrimitiveType::U16: ++ case xla::PrimitiveType::S16: ++ case xla::PrimitiveType::U32: ++ case xla::PrimitiveType::S32: ++ case xla::PrimitiveType::U64: ++ case xla::PrimitiveType::S64: ++ case xla::PrimitiveType::F16: ++ case xla::PrimitiveType::BF16: ++ case xla::PrimitiveType::F32: ++ case xla::PrimitiveType::C64: ++ return true; ++ default: { ++ // std::cout << "Unsupported type: " ++ // << xla::PrimitiveType_Name(primitive_type).c_str() ++ // << std::endl; ++ return false; ++ } ++ } ++} ++ ++bool validateTypeRange(mlir::Operation::operand_range range) { ++ for (auto operand : range) { ++ if (!isValidType(operand.getType())) { ++ return false; ++ } ++ } ++ return true; ++} ++ ++bool validateTypeRange(mlir::Operation::result_range range) { ++ for (auto result : range) { ++ if (!isValidType(result.getType())) { ++ return false; ++ } ++ } ++ return true; ++} ++ ++} // namespace utils +diff --git a/xla/pjrt/plugin/mlx/utils.h b/xla/pjrt/plugin/mlx/utils.h +new file mode 100644 +index 0000000000..6e38b50502 +--- /dev/null ++++ b/xla/pjrt/plugin/mlx/utils.h +@@ -0,0 +1,136 @@ ++/* Copyright 2024 The OpenXLA Authors. ++Licensed under the Apache License, Version 2.0 (the "License"); ++you may not use this file except in compliance with the License. ++You may obtain a copy of the License at ++ http://www.apache.org/licenses/LICENSE-2.0 ++Unless required by applicable law or agreed to in writing, software ++distributed under the License is distributed on an "AS IS" BASIS, ++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++See the License for the specific language governing permissions and ++limitations under the License. ++==============================================================================*/ ++ ++#ifndef XLA_PJRT_PLUGIN_STABLEHLO_MLX_UTILS_H_ ++#define XLA_PJRT_PLUGIN_STABLEHLO_MLX_UTILS_H_ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "absl/status/statusor.h" ++#include "absl/types/span.h" ++#include "mlir/Dialect/Func/IR/FuncOps.h" ++#include "stablehlo/reference/Api.h" ++#include "mlir/IR/OwningOpRef.h" ++#include "mlir/IR/Types.h" ++#include "mlir/IR/Operation.h" ++#include "mlir/IR/Value.h" ++#include "mlx/compile_impl.h" ++#include "mlx/mlx.h" ++#include "xla/literal.h" ++ ++namespace mx = mlx::core; ++ ++using MlxFunc = ++ std::function(const std::vector&)>; ++using OwnedMlxFunc = std::shared_ptr; ++ ++namespace utils { ++class MlxFuncRef { ++ public: ++ MlxFuncRef() : fun_id_(0) {} ++ MlxFuncRef(MlxFunc ref, std::uintptr_t fun_id) ++ : ref_(std::move(ref)), fun_id_(fun_id) {} ++ ++ ~MlxFuncRef() { ++ if (fun_id_) { ++ std::cout << "Destroying MlxFuncRef: " << std::to_string(fun_id_) ++ << std::endl; ++ mx::detail::compile_erase(fun_id_); ++ } ++ } ++ ++ MlxFuncRef(MlxFuncRef&& other) noexcept { ++ ref_ = std::move(other.ref_); ++ fun_id_ = other.fun_id_; ++ } ++ ++ MlxFuncRef& operator=(MlxFuncRef&& other) noexcept { ++ ref_ = std::move(other.ref_); ++ fun_id_ = other.fun_id_; ++ return *this; ++ } ++ ++ std::vector call(const std::vector& inputs) { ++ return ref_(inputs); ++ } ++ ++ bool isInitialized() { return fun_id_ != 0; } ++ ++ void initialize(MlxFunc ref, std::uintptr_t fun_id) { ++ // if (fun_id_ != 0) mx::detail::compile_erase(fun_id_); ++ ref_ = std::move(ref); ++ fun_id_ = fun_id; ++ } ++ ++ private: ++ MlxFunc ref_; ++ std::uintptr_t fun_id_; ++}; ++ ++enum class ScatterType { ++ Replace, ++ Add, ++ Prod, ++ Max, ++ Min, ++}; ++ ++enum class ReduceType { ++ ArgMax, ++ Sum, ++ Max, ++ Min, ++ Any, ++}; ++namespace dtype { ++xla::PrimitiveType asXlaPrimitiveType(mx::Dtype dtype); ++ ++absl::StatusOr fromXlaPrimitiveType(xla::PrimitiveType dtype); ++ ++absl::StatusOr fromMlirType(mlir::Type type); ++ ++} // namespace dtype ++ ++namespace array { ++absl::StatusOr fromHostBuffer( ++ const void* data, absl::Span dims, ++ std::optional> byte_strides, ++ xla::PrimitiveType type); ++ ++absl::StatusOr fromHostLiteral(const xla::LiteralSlice& literal); ++ ++absl::StatusOr fromDenseElementsAttr(mlir::DenseElementsAttr attr); ++mx::array fromOperand( ++ const mlir::Value& operand, ++ const std::unordered_map>& ++ block_ctx, ++ const std::vector& inputs, bool debug = false); ++} // namespace array ++ ++void printVector(const std::string& name, const std::vector& vec, ++ bool indent = false); ++absl::StatusOr getScatterType(mlir::Block& block); ++absl::StatusOr> getSortInfo( ++ mlir::Block& block); ++absl::StatusOr getReduceType(mlir::Operation& op, ++ mlir::Block& body_block); ++uint64_t getHashValue(mlir::func::FuncOp func_op); ++uint64_t getHashValue(mlir::Operation* op); ++bool validateTypeRange(mlir::Operation::operand_range range); ++bool validateTypeRange(mlir::Operation::result_range range); ++} // namespace utils ++ ++#endif // XLA_PJRT_PLUGIN_STABLEHLO_MLX_UTILS_H_ +\ No newline at end of file diff --git a/third_party/modules/xla/20250527.0-cb67f2f/source.json b/third_party/modules/xla/20250527.0-cb67f2f/source.json index 86bdd83a..ee8f90a3 100644 --- a/third_party/modules/xla/20250527.0-cb67f2f/source.json +++ b/third_party/modules/xla/20250527.0-cb67f2f/source.json @@ -1,17 +1,18 @@ { - "strip_prefix": "xla-cb67f2f7ce4787f63f5fc80dc5c30cd3dee8f4e3", - "url": "https://github.com/openxla/xla/archive/cb67f2f7ce4787f63f5fc80dc5c30cd3dee8f4e3.tar.gz", - "integrity": "sha256-SDAAOY6cjcCQ5e1JMob5G4+BYHk7spDb5zZEDrVeA4I=", - "overlay": { - "llvm.bzl": "", - "MODULE.bazel": "", - "workspace_private.bzl": "", - "xla.bzl": "" - }, - "patch_strip": 1, - "patches": { - "0001-bazel-migration-to-bazel-8.1.1.patch": "", - "0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch": "", - "0003-Revert-Add-optional-allowOtherDialects-field-to-stab.patch": "" - } + "strip_prefix": "xla-cb67f2f7ce4787f63f5fc80dc5c30cd3dee8f4e3", + "url": "https://github.com/openxla/xla/archive/cb67f2f7ce4787f63f5fc80dc5c30cd3dee8f4e3.tar.gz", + "integrity": "sha256-SDAAOY6cjcCQ5e1JMob5G4+BYHk7spDb5zZEDrVeA4I=", + "overlay": { + "llvm.bzl": "", + "MODULE.bazel": "", + "workspace_private.bzl": "", + "xla.bzl": "" + }, + "patch_strip": 1, + "patches": { + "0001-bazel-migration-to-bazel-8.1.1.patch": "", + "0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch": "", + "0003-Revert-Add-optional-allowOtherDialects-field-to-stab.patch": "", + "0004-Added-MLX-PjRt-Plugin.patch": "" + } } diff --git a/zml/context.zig b/zml/context.zig index 03de62a3..31c2006b 100644 --- a/zml/context.zig +++ b/zml/context.zig @@ -120,7 +120,7 @@ pub const Context = struct { self.* = undefined; } - const prefered_targets = [_]Target{ .tpu, .neuron, .cuda, .rocm, .cpu }; + const prefered_targets = [_]Target{ .tpu, .neuron, .cuda, .rocm, .cpu, .mlx }; /// Automatically selects the best Platform loaded in the current Context. /// @@ -177,7 +177,9 @@ pub const Context = struct { return error.NoDevicesFound; } - try CustomCall.registerZmlCustomCalls(p); + if (target != .mlx) { + try CustomCall.registerZmlCustomCalls(p); + } self.platforms.set(target, p); return p; diff --git a/zml/module.zig b/zml/module.zig index 798b81cf..c0bc9d1b 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -975,6 +975,7 @@ pub const FnCache = std.AutoHashMapUnmanaged(FnKey, MlirFn); pub const FnKey = struct { fn_ptr: *const anyopaque, input_hash: u64 }; test FnCache { + // TODO(@cryptodeal): Passing; need to implement MLX PjRt plugin cache const zml = @import("zml.zig"); const platform = zml.testing.env(); @@ -1035,6 +1036,7 @@ test FnCache { } test "FnCache with mixed integer/tensor" { + // TODO(@cryptodeal): Passing; need to implement MLX PjRt plugin cache const zml = @import("zml.zig"); const platform = zml.testing.env(); diff --git a/zml/nn.zig b/zml/nn.zig index c6a8560a..1c54365e 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -1095,6 +1095,7 @@ pub fn sdpaChunk(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) PartialSoft } test sdpaMemEfficient { + // TODO(@cryptodeal): MLX PjRt plugin implement `stablehlo::RngOp` const platform = zml.testing.env(); const allocator = std.testing.allocator; @@ -1153,6 +1154,7 @@ test sdpaMemEfficient { } test "sdpaMemEfficient transposed" { + // TODO(@cryptodeal): MLX PjRt plugin implement `stablehlo::RngOp` const platform = zml.testing.env(); const allocator = std.testing.allocator; @@ -1248,6 +1250,7 @@ pub fn sampleTokens(activations: Tensor, opts: SamplingStrategy, rng: Tensor.Rng } test sampleTokens { + // TODO(@cryptodeal): MLX PjRt plugin implement `stablehlo::RandomBitGeneratorOp` const platform = zml.testing.env(); const allocator = std.testing.allocator; @@ -1371,6 +1374,7 @@ fn fixupLogits(logits: Tensor, opts: DynamicSamplingStrategy) [2]Tensor { } test sampleTokensDynamic { + // TODO(@cryptodeal): MLX PjRt plugin implement `stablehlo::ReduceWindowOp` const platform = zml.testing.env(); const allocator = std.testing.allocator; diff --git a/zml/ops.zig b/zml/ops.zig index 1765888a..0eae345b 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -72,6 +72,7 @@ pub fn while_( } test "simple while" { + // TODO(@cryptodeal): MLX PjRt plugin implement `stablehlo::WhileOp` const CountInts = struct { step: Tensor, end: Tensor, @@ -349,6 +350,7 @@ pub fn for_(comptime func: anytype, blk_ctx: BlockSign(func).BlkCtx, num_steps_: } test for_ { + // TODO(@cryptodeal): MLX PjRt plugin implement `stablehlo::WhileOp` const Squares = struct { const Squares = @This(); @@ -389,6 +391,7 @@ test for_ { } test "nested for" { + // TODO(@cryptodeal): MLX PjRt plugin implement `stablehlo::WhileOp` const OuterProd = struct { const OuterProd = @This(); @@ -482,6 +485,7 @@ pub fn if_( } test "if" { + // TODO(@cryptodeal): MLX PjRt plugin implement `stablehlo::IfOp` const zml = @import("zml.zig"); const platform = zml.testing.env(); const allocator = std.testing.allocator; @@ -544,6 +548,7 @@ pub fn case( } test "case" { + // TODO(@cryptodeal): MLX PjRt plugin implement `stablehlo::CaseOp` const zml = @import("zml.zig"); const platform = zml.testing.env(); diff --git a/zml/platform.zig b/zml/platform.zig index ec4ae4c6..4ae4b06d 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -99,6 +99,7 @@ const _CreateOptions = struct { rocm: struct {} = .{}, tpu: struct {} = .{}, neuron: struct {} = .{}, + mlx: struct {} = .{}, pub const Cuda = struct { allocator: Allocator = .{ .bfc = .{} }, diff --git a/zml/tensor.zig b/zml/tensor.zig index ae06a776..c997dcaa 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -386,16 +386,64 @@ pub const Tensor = struct { return binaryOp(@src(), "shiftLeft", dialect.stablehlo.shift_left)(self, other); } + test shiftLeft { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + const lhs: [3]i64 = .{ -1, 0, 1 }; + const rhs: [3]i64 = .{ 1, 2, 3 }; + const expected: [3]i64 = .{ -2, 0, 8 }; + + const input = try zml.Buffer.fromSlice(platform, .{3}, &lhs); + const shift_by = try zml.Buffer.fromSlice(platform, .{3}, &rhs); + + const output = try zml.testing.compileAndCall(platform, Tensor.shiftLeft, .{ input, shift_by }); + + try zml.testing.expectClose(zml.HostBuffer.fromSlice(.{3}, &expected), output, 1e-4); + } + /// Returns a Tensor containing the element-wise arithmetic right-shift operation of 'self' by 'other'. pub fn shiftRightArithmetic(self: Tensor, other: Tensor) Tensor { return binaryOp(@src(), "shiftRightArithmetic", dialect.stablehlo.shift_right_arithmetic)(self, other); } + test shiftRightArithmetic { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + const lhs: [3]i64 = .{ -1, 0, 8 }; + const rhs: [3]i64 = .{ 1, 2, 3 }; + const expected: [3]i64 = .{ -1, 0, 1 }; + + const input = try zml.Buffer.fromSlice(platform, .{3}, &lhs); + const shift_by = try zml.Buffer.fromSlice(platform, .{3}, &rhs); + + const output = try zml.testing.compileAndCall(platform, Tensor.shiftRightArithmetic, .{ input, shift_by }); + + try zml.testing.expectClose(zml.HostBuffer.fromSlice(.{3}, &expected), output, 1e-4); + } + /// Returns a Tensor containing the element-wise logical right-shift operation of 'self' by 'other'. pub fn shiftRightLogical(self: Tensor, other: Tensor) Tensor { return binaryOp(@src(), "shiftRightLogical", dialect.stablehlo.shift_right_logical)(self, other); } + test shiftRightLogical { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + const lhs: [3]i64 = .{ -1, 0, 8 }; + const rhs: [3]i64 = .{ 1, 2, 3 }; + const expected: [3]i64 = .{ 9223372036854775807, 0, 1 }; + + const input = try zml.Buffer.fromSlice(platform, .{3}, &lhs); + const shift_by = try zml.Buffer.fromSlice(platform, .{3}, &rhs); + + const output = try zml.testing.compileAndCall(platform, Tensor.shiftRightLogical, .{ input, shift_by }); + + try zml.testing.expectClose(zml.HostBuffer.fromSlice(.{3}, &expected), output, 1e-4); + } + /// Returns the Cholesky decomposition of the input Tensor. /// /// 'lower' controls the form of the output Tensor. The output will be lower-triangular if 'lower' is true @@ -615,6 +663,7 @@ pub const Tensor = struct { } test uniform { + // TODO(@cryptodeal): MLX PjRt plugin implement `stablehlo::RandomBitGeneratorOp` const zml = @import("zml.zig"); const Stats = struct { const Stats = @This(); @@ -708,6 +757,7 @@ pub const Tensor = struct { } test gumbel { + // TODO(@cryptodeal): MLX PjRt plugin implement `stablehlo::RandomBitGeneratorOp` const zml = @import("zml.zig"); const Stats = struct { const Stats = @This(); @@ -1033,6 +1083,39 @@ pub const Tensor = struct { }; } + test logical { + const zml = @import("zml.zig"); + const Local = struct { + pub fn call(comptime logical_op: LogicalOp) type { + return struct { + fn _logical(a: zml.Tensor, b: zml.Tensor) zml.Tensor { + return a.logical(logical_op, b); + } + + pub fn testCase(comptime T: type, platform: zml.Platform, lhs: [3]T, rhs: [3]T, expected: [3]T) !void { + const actual = try zml.testing.compileAndCall(platform, _logical, .{ try zml.Buffer.fromSlice(platform, .{3}, &lhs), try zml.Buffer.fromSlice(platform, .{3}, &rhs) }); + return zml.testing.expectClose(zml.HostBuffer.fromArray(&expected), actual, 1e-4); + } + }; + } + }.call; + + const platform = zml.testing.env(); + inline for (.{ + .{ i8, .{ 127, -128, -128 }, .{ 0, 127, -128 }, .{ 127, -1, -128 } }, + .{ u8, .{ 0, 127, 255 }, .{ 255, 255, 255 }, .{ 255, 255, 255 } }, + .{ i16, .{ 32767, -32768, -32768 }, .{ 0, 32767, -32768 }, .{ 32767, -1, -32768 } }, + .{ u16, .{ 0, 32767, 65535 }, .{ 65535, 65535, 65535 }, .{ 65535, 65535, 65535 } }, + .{ i32, .{ 2147483647, -2147483648, -2147483648 }, .{ 0, 2147483647, -2147483648 }, .{ 2147483647, -1, -2147483648 } }, + .{ u32, .{ 0, 2147483647, 4294967295 }, .{ 4294967295, 4294967295, 4294967295 }, .{ 4294967295, 4294967295, 4294967295 } }, + .{ i64, .{ 9223372036854775807, -9223372036854775808, -9223372036854775808 }, .{ 0, 9223372036854775807, -9223372036854775808 }, .{ 9223372036854775807, -1, -9223372036854775808 } }, + .{ u64, .{ 0, 9223372036854775807, 18446744073709551615 }, .{ 18446744073709551615, 18446744073709551615, 18446744073709551615 }, .{ 18446744073709551615, 18446744073709551615, 18446744073709551615 } }, + }) |testcase| { + const T, const lhs, const rhs, const expected = testcase; + try Local(.OR).testCase(T, platform, lhs, rhs, expected); + } + } + /// Returns a Tensor containing the element-wise floor operation of the input Tensor. pub fn floor(self: Tensor) Tensor { const loc = self.getContext().mlirCtx().location(@src()); @@ -1375,6 +1458,7 @@ pub const Tensor = struct { } test cumulativeSum { + // TODO(@cryptodeal): MLX PjRt plugin implement `stablehlo::ReduceWindowOp` const zml = @import("zml.zig"); const platform = zml.testing.env(); @@ -2830,6 +2914,7 @@ pub const Tensor = struct { } test argMax { + // TODO(@cryptodeal): fix test case for MLX PjRt plugin const zml = @import("zml.zig"); const platform = zml.testing.env(); const allocator = std.testing.allocator; @@ -3893,6 +3978,7 @@ fn getComparisonType(ctx: mlir.Context, dtype: DataType) dialect.stablehlo.Compa } test "Tensor.maxPool1d" { + // TODO(@cryptodeal): MLX PjRt plugin implement `stablehlo::ReduceWindowOp` const zml = @import("zml.zig"); const platform = zml.testing.env(); @@ -3929,6 +4015,7 @@ test "Tensor.maxPool1d" { } test "Tensor.maxPool2d" { + // TODO(@cryptodeal): MLX PjRt plugin implement `stablehlo::ReduceWindowOp` const zml = @import("zml.zig"); const platform = zml.testing.env(); diff --git a/zml/testing.zig b/zml/testing.zig index 8946f78b..4fb0d312 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -11,11 +11,18 @@ const log = std.log.scoped(.@"zml/testing"); var _platform: ?zml.Platform = null; +pub fn mlxSkipTest(platform: zml.Platform) !void { + if (platform.target == .mlx) { + return error.SkipZigTest; + } +} + pub fn env() zml.Platform { if (!builtin.is_test) @compileError("Cannot use zml.testing.env outside of a test block"); if (_platform == null) { var ctx = zml.Context.init() catch unreachable; - _platform = ctx.autoPlatform(.{}).withCompilationOptions(.{ + var auto_platform = ctx.autoPlatform(.{}); + _platform = auto_platform.withCompilationOptions(if (auto_platform.target == .mlx) .{} else .{ .xla_dump_to = "/tmp/zml/tests/", .sharding_enabled = true, });