From 502792d640ba91ab4aff98e8d92efccf0a6c4102 Mon Sep 17 00:00:00 2001 From: Corentin Kerisit Date: Fri, 25 Jul 2025 10:01:26 +0000 Subject: [PATCH 01/10] clean neuron --- runtimes/neuron/BUILD.bazel | 14 ++++++++++---- runtimes/neuron/libpjrt_neuron.BUILD.bazel | 9 +++++++-- runtimes/neuron/{libpjrt_neuron.h => nrt.h} | 0 3 files changed, 17 insertions(+), 6 deletions(-) rename runtimes/neuron/{libpjrt_neuron.h => nrt.h} (100%) diff --git a/runtimes/neuron/BUILD.bazel b/runtimes/neuron/BUILD.bazel index b65d7a38..89885855 100644 --- a/runtimes/neuron/BUILD.bazel +++ b/runtimes/neuron/BUILD.bazel @@ -107,11 +107,15 @@ cc_library( ) cc_library( - name = "libpjrt_neuron", - hdrs = ["libpjrt_neuron.h"], + name = "zmlxneuron", defines = ["ZML_RUNTIME_NEURON"], +) + +cc_library( + name = "libnrt_headers", + hdrs = ["nrt.h"], deps = [ - "@libpjrt_neuron", + "@libpjrt_neuron//:libnrt_headers", ], ) @@ -130,7 +134,9 @@ zig_library( "//pjrt", ] + select({ "//runtimes:neuron.enabled": [ - ":libpjrt_neuron", + "@libpjrt_neuron", + ":zmlxneuron", + ":libnrt_headers", ":libneuronxla_pyenv", ":libpython", "//async", diff --git a/runtimes/neuron/libpjrt_neuron.BUILD.bazel b/runtimes/neuron/libpjrt_neuron.BUILD.bazel index 5df16d6b..e6905ecf 100644 --- a/runtimes/neuron/libpjrt_neuron.BUILD.bazel +++ b/runtimes/neuron/libpjrt_neuron.BUILD.bazel @@ -77,8 +77,7 @@ copy_to_directory( ) cc_library( - name = "libpjrt_neuron", - data = [":sandbox"], + name = "libnrt_headers", deps = [ "@aws-neuronx-runtime-lib//:libnrt_headers", ], @@ -94,3 +93,9 @@ cc_library( ], visibility = ["@zml//runtimes/neuron:__subpackages__"], ) + +cc_library( + name = "libpjrt_neuron", + data = [":sandbox"], + visibility = ["@zml//runtimes/neuron:__subpackages__"], +) diff --git a/runtimes/neuron/libpjrt_neuron.h b/runtimes/neuron/nrt.h similarity index 100% rename from runtimes/neuron/libpjrt_neuron.h rename to runtimes/neuron/nrt.h From 2050cf2d0879d2313b736208e4178d4498fec5f4 Mon Sep 17 00:00:00 2001 From: Corentin Kerisit Date: Fri, 25 Jul 2025 15:59:28 +0000 Subject: [PATCH 02/10] wip --- BUILD.bazel | 1 + pjrt/pjrt.zig | 4 ++++ runtimes/neuron/neuron.zig | 2 ++ 3 files changed, 7 insertions(+) diff --git a/BUILD.bazel b/BUILD.bazel index 63540f5e..137f1800 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -13,5 +13,6 @@ zls_completion( "//async", "//stdx", "//zml", + "//runtimes/neuron:neuronx-cc_lib", ], ) diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 58d9ac29..92dc4ebf 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -34,6 +34,10 @@ fn pjrtStructSize(comptime T: type) usize { pub inline fn pjrtStruct(v: anytype) @TypeOf(v) { var ret = v; ret.struct_size = pjrtStructSize(@TypeOf(v)); + if (@TypeOf(v) == c.PJRT_Plugin_Attributes_Args) { + std.debug.print("type is {any} and size is {}\n", .{ @TypeOf(v), ret.struct_size }); + ret.struct_size = 0x18; + } return ret; } diff --git a/runtimes/neuron/neuron.zig b/runtimes/neuron/neuron.zig index fc973424..1a0dd4b4 100644 --- a/runtimes/neuron/neuron.zig +++ b/runtimes/neuron/neuron.zig @@ -73,7 +73,9 @@ fn initialize(allocator: std.mem.Allocator, r_: *runfiles.Runfiles) !void { { config.module_search_paths_set = 1; for (libneuronxla_pyenv.modules) |module| { + std.debug.print("module in {s}\n", .{module}); const path = (try r.rlocation(module, &buf)).?; + std.debug.print("path in {s}\n", .{path}); const wline = toWchar(std.fs.path.dirname(path).?, &wbuf); pyErrorOrExit(c.PyWideStringList_Append(&config.module_search_paths, wline.ptr)); } From 4f0b5df257045108cb267584ab3fe74c2a376661 Mon Sep 17 00:00:00 2001 From: Corentin Kerisit Date: Fri, 25 Jul 2025 15:59:42 +0000 Subject: [PATCH 03/10] wip --- runtimes/neuron/BUILD.bazel | 32 ++++++++++++++++++++++++++++---- runtimes/neuron/neuron.zig | 2 +- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/runtimes/neuron/BUILD.bazel b/runtimes/neuron/BUILD.bazel index 89885855..4a168058 100644 --- a/runtimes/neuron/BUILD.bazel +++ b/runtimes/neuron/BUILD.bazel @@ -3,12 +3,13 @@ load("@rules_python//python:pip.bzl", "compile_pip_requirements") load("@rules_python//python/entry_points:py_console_script_binary.bzl", "py_console_script_binary") load("@rules_uv//uv:pip.bzl", "pip_compile") load("@rules_zig//zig:defs.bzl", "BINARY_KIND", "zig_binary", "zig_library") +load("@zml//bazel:zig.bzl", "zig_cc_binary") load(":neuron.bzl", "py_binary_with_script") load(":pyenv.bzl", "pyenv_zig") pyenv_zig( name = "libneuronxla_pyenv_zig", - deps = [":libneuronxla"], + deps = ["@neuron_py_deps//libneuronxla"], ) zig_library( @@ -16,6 +17,16 @@ zig_library( main = ":libneuronxla_pyenv_zig", ) +pyenv_zig( + name = "neuronx_cc_pyenv_zig", + deps = ["@neuron_py_deps//neuronx_cc"], +) + +zig_library( + name = "neuronx_cc_pyenv", + main = ":neuronx_cc_pyenv_zig", +) + # A proxy PJRT Plugin that loads the Neuron PJRT Plugin # and returns the instance from nested GetPjrtApi. # @@ -53,10 +64,23 @@ pip_compile( ], ) -py_console_script_binary( +zig_cc_binary( name = "neuronx-cc", - binary_rule = py_binary_with_script, - pkg = "@neuron_py_deps//neuronx_cc", + main = "neuronx-cc.zig", + copts = ["-lc"], + data = [ + "@neuron_py_deps//neuronx_cc", + ], + deps = [ + ":neuronx_cc_pyenv", + ":libpython", + "//async", + "//stdx", + "@rules_zig//zig/runfiles", + ], + tags = [ + "manual", + ], visibility = ["//visibility:public"], ) diff --git a/runtimes/neuron/neuron.zig b/runtimes/neuron/neuron.zig index 1a0dd4b4..9df4d9eb 100644 --- a/runtimes/neuron/neuron.zig +++ b/runtimes/neuron/neuron.zig @@ -82,7 +82,7 @@ fn initialize(allocator: std.mem.Allocator, r_: *runfiles.Runfiles) !void { } { - const neuronx_cc = (try r.rlocation("zml/runtimes/neuron/neuronx-cc/neuronx-cc", &buf)).?; + const neuronx_cc = (try r.rlocation("zml/runtimes/neuron/neuronx-cc", &buf)).?; const neuronx_cc_path = std.fs.path.dirname(neuronx_cc).?; const path = std.posix.getenv("PATH") orelse ""; const new_path = try std.fmt.allocPrintZ(allocator, "{s}:{s}", .{ neuronx_cc_path, path }); From 0d7dc0c70ca0bea3c341340ad44014eca6aa5cc7 Mon Sep 17 00:00:00 2001 From: Corentin Kerisit Date: Fri, 25 Jul 2025 18:21:50 +0000 Subject: [PATCH 04/10] WIP --- bazel/zig.bzl | 2 + runtimes/neuron/BUILD.bazel | 38 +++++- runtimes/neuron/libpjrt_neuron.BUILD.bazel | 20 +++ runtimes/neuron/libpjrt_neuron.zig | 142 ++++++++++++++++++--- runtimes/neuron/neuron.zig | 95 +------------- runtimes/neuron/neuronx-cc.zig | 88 +++++++++++++ 6 files changed, 275 insertions(+), 110 deletions(-) create mode 100644 runtimes/neuron/neuronx-cc.zig diff --git a/bazel/zig.bzl b/bazel/zig.bzl index 26773a5e..7132b4b8 100644 --- a/bazel/zig.bzl +++ b/bazel/zig.bzl @@ -10,6 +10,7 @@ def zig_cc_binary( data = [], deps = [], tags = [], + linkopts = [], visibility = None, **kwargs): zig_binary( @@ -25,6 +26,7 @@ def zig_cc_binary( args = args, env = env, data = data, + linkopts = linkopts, deps = [":{}_lib".format(name)], tags = tags, visibility = visibility, diff --git a/runtimes/neuron/BUILD.bazel b/runtimes/neuron/BUILD.bazel index 4a168058..f7fe7c94 100644 --- a/runtimes/neuron/BUILD.bazel +++ b/runtimes/neuron/BUILD.bazel @@ -7,6 +7,35 @@ load("@zml//bazel:zig.bzl", "zig_cc_binary") load(":neuron.bzl", "py_binary_with_script") load(":pyenv.bzl", "pyenv_zig") +# load("@aspect_bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") +# load("@zml//bazel:patchelf.bzl", "patchelf") +# load("@zml//bazel:runfiles.bzl", "runfiles_to_default") + + +# runfiles_to_default( +# name = "runfiles", +# deps = ["@neuron_py_deps//neuronx_cc"], +# ) + +# copy_to_directory( +# name = "pysandbox", +# srcs = [ +# ":bin/neuronx-cc", +# "@rules_python//python:current_py_toolchain", +# ":runfiles", +# ], +# allow_overwrites = True, +# add_directory_to_runfiles = True, +# include_external_repositories = ["**"], +# include_srcs_patterns = [ +# "bin/neuronx-cc", +# "lib/libpython3.11*/**", +# "lib/python*/**", +# "site-packages/**", +# ], +# hardlink = "on", +# ) + pyenv_zig( name = "libneuronxla_pyenv_zig", deps = ["@neuron_py_deps//libneuronxla"], @@ -29,7 +58,7 @@ zig_library( # A proxy PJRT Plugin that loads the Neuron PJRT Plugin # and returns the instance from nested GetPjrtApi. -# +# # Additionally, it provides a way to load implicit transitive dependencies # of neuronx-cc (see add_needed of the patchelf target below). # @@ -42,6 +71,7 @@ zig_binary( deps = [ "//stdx", "@rules_zig//zig/runfiles", + ":libpython", ], visibility = ["@libpjrt_neuron//:__subpackages__"], ) @@ -67,12 +97,14 @@ pip_compile( zig_cc_binary( name = "neuronx-cc", main = "neuronx-cc.zig", - copts = ["-lc"], data = [ "@neuron_py_deps//neuronx_cc", ], + linkopts = [ + "-Wl,-rpath,$ORIGIN/../lib", + ], deps = [ - ":neuronx_cc_pyenv", + # ":neuronx_cc_pyenv", ":libpython", "//async", "//stdx", diff --git a/runtimes/neuron/libpjrt_neuron.BUILD.bazel b/runtimes/neuron/libpjrt_neuron.BUILD.bazel index e6905ecf..68843fc6 100644 --- a/runtimes/neuron/libpjrt_neuron.BUILD.bazel +++ b/runtimes/neuron/libpjrt_neuron.BUILD.bazel @@ -51,12 +51,23 @@ patchelf( soname = "libpjrt_neuron.so", ) +runfiles_to_default( + name = "neuronx_cc_runfiles", + deps = [ + "@neuron_py_deps//libneuronxla", + "@neuron_py_deps//neuronx_cc", + ], +) + copy_to_directory( name = "sandbox", srcs = [ ":zmlxneuron_so", ":libneuronpjrt.patchelf", ":libpjrt_neuron_proxy.patchelf", + ":neuronx_cc_runfiles", + "@rules_python//python:current_py_toolchain", + "@zml//runtimes/neuron:neuronx-cc", "@aws-neuronx-runtime-lib//:libnrt.patchelf", "@aws-neuronx-runtime-lib//:libncfw.patchelf", "@aws-neuronx-collectives//:libnccom", @@ -64,6 +75,7 @@ copy_to_directory( "@libgomp1", ], replace_prefixes = { + "runtimes/neuron/neuronx-cc": "bin/neuronx-cc", "libneuronpjrt.patchelf": "lib", "libpjrt_neuron_proxy.patchelf": "lib", "libnrt.patchelf": "lib", @@ -72,8 +84,16 @@ copy_to_directory( "usr/lib/x86_64-linux-gnu": "lib", "opt/neuron": "lib", }, + # include_srcs_patterns = [ + # "bin/neuronx-cc", + # "lib/libpython3.11*/**", + # "lib/python*/**", + # "site-packages/**", + # ], + allow_overwrites = True, add_directory_to_runfiles = True, include_external_repositories = ["**"], + hardlink = "on", ) cc_library( diff --git a/runtimes/neuron/libpjrt_neuron.zig b/runtimes/neuron/libpjrt_neuron.zig index 6485ecd0..e0eedc34 100644 --- a/runtimes/neuron/libpjrt_neuron.zig +++ b/runtimes/neuron/libpjrt_neuron.zig @@ -1,11 +1,107 @@ const std = @import("std"); -const runfiles = @import("runfiles"); + const bazel_builtin = @import("bazel_builtin"); +const c = @import("c"); +const runfiles = @import("runfiles"); const stdx = @import("stdx"); const log = std.log.scoped(.@"zml/runtimes/neuron"); -pub export fn GetPjrtApi() *anyopaque { +fn comptimeStrJoin(comptime separator: [:0]const u8, comptime slices: []const [:0]const u8) [:0]const u8 { + comptime var ret = slices[0]; + inline for (slices[1..]) |slice| { + ret = ret ++ separator ++ slice; + } + return ret; +} + +fn pyErrorOrExit(status: c.PyStatus) void { + if (c.PyStatus_Exception(status) != 0) { + if (c.PyStatus_IsExit(status) != 0) { + std.process.exit(@intCast(status.exitcode)); + } + c.Py_ExitStatusException(status); + } +} + +pub fn toPosixPathW(file_path: []const u8) error{NameTooLong}![std.posix.PATH_MAX - 1:0]c.wchar_t { + if (file_path.len >= std.posix.PATH_MAX) return error.NameTooLong; + + var path_with_null: [std.posix.PATH_MAX - 1:0]c.wchar_t = undefined; + const len = c.mbstowcs(&path_with_null, file_path.ptr, file_path.len); + path_with_null[len] = 0; + return path_with_null; +} + +pub fn setNeuronCCFlags() void { + // See neuronxcc reference: + // https://awsdocs-neuron.readthedocs-hosted.com/en/latest/compiler/neuronx-cc/api-reference-guide/neuron-compiler-cli-reference-guide.html#neuron-compiler-cli-reference-guide + _ = c.setenv("NEURON_CC_FLAGS", comptimeStrJoin(" ", &.{ + // 30% faster, no visible speed difference on llama + "--optlevel=1", + // generic is the default, but it fails on transformers, force it + "--model-type=transformer", + // disable it, we do our own + "--auto-cast=none", + "--enable-fast-loading-neuron-binaries", + }), 1); + + // Enable stochastic rounding + // https://awsdocs-neuron.readthedocs-hosted.com/en/latest/general/arch/neuron-features/rounding-modes.html + _ = c.setenv("NEURON_RT_STOCHASTIC_ROUNDING_EN", "1", 1); +} + +fn setupPythonEnv(sandbox_path: []const u8) !void { + { + var preconfig: c.PyPreConfig = undefined; + c.PyPreConfig_InitIsolatedConfig(&preconfig); + preconfig.utf8_mode = 1; + pyErrorOrExit(c.Py_PreInitialize(&preconfig)); + } + + var config: c.PyConfig = undefined; + c.PyConfig_InitIsolatedConfig(&config); + defer c.PyConfig_Clear(&config); + + { + var bufZ: [std.fs.max_path_bytes:0]u8 = undefined; + pyErrorOrExit(c.PyConfig_SetBytesString(&config, &config.home, try stdx.fs.path.bufJoinZ(&bufZ, &.{ + sandbox_path, + "lib", + "python3.11", + }))); + } + + { + config.module_search_paths_set = 1; + var buf: [std.fs.max_path_bytes]u8 = undefined; + pyErrorOrExit(c.PyWideStringList_Append(&config.module_search_paths, &(try toPosixPathW(try stdx.fs.path.bufJoin(&buf, &.{ + sandbox_path, + "lib", + "python3.11", + }))))); + pyErrorOrExit(c.PyWideStringList_Append(&config.module_search_paths, &(try toPosixPathW(try stdx.fs.path.bufJoin(&buf, &.{ + sandbox_path, + "site-packages", + }))))); + } + + pyErrorOrExit(c.Py_InitializeFromConfig(&config)); + + // release the GIL + _ = c.PyEval_SaveThread(); +} + +fn setupBinPath(allocator: std.mem.Allocator, sandbox_path: []const u8) !void { + var buf: [std.fs.max_path_bytes]u8 = undefined; + const bin_path = try stdx.fs.path.bufJoin(&buf, &.{ sandbox_path, "bin" }); + std.debug.print(">>>>>>>>> {s}\n\n", .{bin_path}); + const old_path = std.posix.getenv("PATH") orelse ""; + const new_path = try std.fmt.allocPrintZ(allocator, "{s}:{s}", .{ bin_path, old_path }); + _ = c.setenv("PATH", new_path, 1); +} + +pub export fn GetPjrtApi() ?*anyopaque { var arena = std.heap.ArenaAllocator.init(std.heap.c_allocator); defer arena.deinit(); @@ -16,24 +112,40 @@ pub export fn GetPjrtApi() *anyopaque { const source_repo = bazel_builtin.current_repository; const r = r_.withSourceRepo(source_repo); - var path_buf: [std.fs.max_path_bytes]u8 = undefined; - const sandbox_path = r.rlocation("libpjrt_neuron/sandbox", &path_buf) catch |err| { + var sandbox_path_buf: [std.fs.max_path_bytes]u8 = undefined; + const sandbox_path = r.rlocation("libpjrt_neuron/sandbox", &sandbox_path_buf) catch |err| { stdx.debug.panic("Failed to find sandbox path for NEURON runtime: {}", .{err}); } orelse stdx.debug.panic("No NEURON sandbox path found", .{}); - var lib_path_buf: [std.fs.max_path_bytes]u8 = undefined; - const library = stdx.fs.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libneuronpjrt.so" }) catch unreachable; + { + setNeuronCCFlags(); + setupBinPath(arena.allocator(), sandbox_path) catch |err| { + log.err("Failed to setup bin path: {}", .{err}); + return null; + }; + setupPythonEnv(sandbox_path) catch |err| { + log.err("Failed to setup Python environment: {}", .{err}); + return null; + }; + } - var lib: std.DynLib = blk: { - const handle = std.c.dlopen(library, .{ .LAZY = true, .GLOBAL = true, .NODELETE = true }) orelse { - stdx.debug.panic("Unable to dlopen plugin: {s}", .{library}); + const GetPjrtApi_inner = blk: { + var lib_path_buf: [std.fs.max_path_bytes]u8 = undefined; + const library = stdx.fs.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libneuronpjrt.so" }) catch unreachable; + + var lib: std.DynLib = blk_lib: { + const handle = std.c.dlopen(library, .{ .LAZY = true, .GLOBAL = true, .NODELETE = true }) orelse { + log.err("Unable to dlopen plugin: {s}", .{library}); + return null; + }; + break :blk_lib .{ .inner = .{ .handle = handle } }; + }; + + break :blk lib.lookup(*const fn () callconv(.C) *anyopaque, "GetPjrtApi") orelse { + log.err("Unable to find symbol GetPjrtApi in plugin: {s}", .{library}); + return null; }; - break :blk .{ .inner = .{ .handle = handle } }; }; - const sym = lib.lookup(*const fn () callconv(.C) *anyopaque, "GetPjrtApi") orelse { - stdx.debug.panic("Unable to find symbol GetPjrtApi in plugin: {s}", .{library}); - }; - - return sym(); + return GetPjrtApi_inner(); } diff --git a/runtimes/neuron/neuron.zig b/runtimes/neuron/neuron.zig index 9df4d9eb..084a4256 100644 --- a/runtimes/neuron/neuron.zig +++ b/runtimes/neuron/neuron.zig @@ -1,5 +1,5 @@ -const builtin = @import("builtin"); const std = @import("std"); +const builtin = @import("builtin"); const asynk = @import("async"); const bazel_builtin = @import("bazel_builtin"); @@ -32,95 +32,6 @@ fn isRunningOnEC2() !bool { return std.mem.eql(u8, &buf, AmazonEC2); } -fn toWchar(str: []const u8, out: [:0]c.wchar_t) [:0]c.wchar_t { - const len = c.mbstowcs(out.ptr, str.ptr, str.len); - out[len] = 0; - return out[0..len :0]; -} - -fn pyErrorOrExit(status: c.PyStatus) void { - if (c.PyStatus_Exception(status) != 0) { - if (c.PyStatus_IsExit(status) != 0) { - std.process.exit(@intCast(status.exitcode)); - } - c.Py_ExitStatusException(status); - } -} - -fn initialize(allocator: std.mem.Allocator, r_: *runfiles.Runfiles) !void { - { - var preconfig: c.PyPreConfig = undefined; - c.PyPreConfig_InitIsolatedConfig(&preconfig); - preconfig.utf8_mode = 1; - pyErrorOrExit(c.Py_PreInitialize(&preconfig)); - } - - var config: c.PyConfig = undefined; - c.PyConfig_InitIsolatedConfig(&config); - defer c.PyConfig_Clear(&config); - - const r = r_.withSourceRepo(bazel_builtin.current_repository); - - var buf: [std.fs.max_path_bytes]u8 = undefined; - var wbuf: [std.fs.max_path_bytes:0]c.wchar_t = undefined; - - { - const path = (try r.rlocation(libneuronxla_pyenv.home, &buf)).?; - const wpath = toWchar(std.fs.path.dirname(path).?, &wbuf); - pyErrorOrExit(c.PyConfig_SetString(&config, &config.home, wpath.ptr)); - } - - { - config.module_search_paths_set = 1; - for (libneuronxla_pyenv.modules) |module| { - std.debug.print("module in {s}\n", .{module}); - const path = (try r.rlocation(module, &buf)).?; - std.debug.print("path in {s}\n", .{path}); - const wline = toWchar(std.fs.path.dirname(path).?, &wbuf); - pyErrorOrExit(c.PyWideStringList_Append(&config.module_search_paths, wline.ptr)); - } - } - - { - const neuronx_cc = (try r.rlocation("zml/runtimes/neuron/neuronx-cc", &buf)).?; - const neuronx_cc_path = std.fs.path.dirname(neuronx_cc).?; - const path = std.posix.getenv("PATH") orelse ""; - const new_path = try std.fmt.allocPrintZ(allocator, "{s}:{s}", .{ neuronx_cc_path, path }); - _ = c.setenv("PATH", new_path.ptr, 1); - } - - pyErrorOrExit(c.Py_InitializeFromConfig(&config)); - - // release the GIL - _ = c.PyEval_SaveThread(); -} - -fn comptimeStrJoin(comptime separator: [:0]const u8, comptime slices: []const [:0]const u8) [:0]const u8 { - comptime var ret = slices[0]; - inline for (slices[1..]) |slice| { - ret = ret ++ separator ++ slice; - } - return ret; -} - -pub fn setNeuronCCFlags() void { - // See neuronxcc reference: - // https://awsdocs-neuron.readthedocs-hosted.com/en/latest/compiler/neuronx-cc/api-reference-guide/neuron-compiler-cli-reference-guide.html#neuron-compiler-cli-reference-guide - _ = c.setenv("NEURON_CC_FLAGS", comptimeStrJoin(" ", &.{ - // 30% faster, no visible speed difference on llama - "--optlevel=1", - // generic is the default, but it fails on transformers, force it - "--model-type=transformer", - // disable it, we do our own - "--auto-cast=none", - "--enable-fast-loading-neuron-binaries", - }), 1); - - // Enable stochastic rounding - // https://awsdocs-neuron.readthedocs-hosted.com/en/latest/general/arch/neuron-features/rounding-modes.html - _ = c.setenv("NEURON_RT_STOCHASTIC_ROUNDING_EN", "1", 1); -} - pub fn load() !*const pjrt.Api { if (comptime !isEnabled()) { return error.Unavailable; @@ -151,8 +62,8 @@ pub fn load() !*const pjrt.Api { return error.FileNotFound; }; - setNeuronCCFlags(); - try initialize(arena.allocator(), &r_); + // setNeuronCCFlags(); + // try initialize(arena.allocator(), &r_); return blk: { var lib_path_buf: [std.fs.max_path_bytes]u8 = undefined; diff --git a/runtimes/neuron/neuronx-cc.zig b/runtimes/neuron/neuronx-cc.zig new file mode 100644 index 00000000..c9f19b34 --- /dev/null +++ b/runtimes/neuron/neuronx-cc.zig @@ -0,0 +1,88 @@ +const std = @import("std"); +const builtin = @import("builtin"); + +const asynk = @import("async"); +const bazel_builtin = @import("bazel_builtin"); +const c = @import("c"); +const runfiles = @import("runfiles"); +const stdx = @import("stdx"); + +fn pyErrorOrExit(status: c.PyStatus) void { + if (c.PyStatus_Exception(status) != 0) { + if (c.PyStatus_IsExit(status) != 0) { + std.process.exit(@intCast(status.exitcode)); + } + c.Py_ExitStatusException(status); + } +} + +pub fn toPosixPathW(file_path: []const u8) error{NameTooLong}![std.posix.PATH_MAX - 1:0]c.wchar_t { + if (file_path.len >= std.posix.PATH_MAX) return error.NameTooLong; + + var path_with_null: [std.posix.PATH_MAX - 1:0]c.wchar_t = undefined; + const len = c.mbstowcs(&path_with_null, file_path.ptr, file_path.len); + path_with_null[len] = 0; + return path_with_null; +} + +pub fn main() !void { + var arena = std.heap.ArenaAllocator.init(std.heap.c_allocator); + defer arena.deinit(); + + { + var preconfig: c.PyPreConfig = undefined; + c.PyPreConfig_InitIsolatedConfig(&preconfig); + preconfig.utf8_mode = 1; + pyErrorOrExit(c.Py_PreInitialize(&preconfig)); + } + + var config: c.PyConfig = undefined; + c.PyConfig_InitIsolatedConfig(&config); + _ = c.PyConfig_SetBytesArgv(&config, @intCast(std.os.argv.len), @ptrCast(std.os.argv)); + defer c.PyConfig_Clear(&config); + + var self_exe_dir_buf: [std.fs.max_path_bytes]u8 = undefined; + const self_exe_dir = try std.fs.selfExeDirPath(&self_exe_dir_buf); + + { + var bufZ: [std.fs.max_path_bytes:0]u8 = undefined; + pyErrorOrExit(c.PyConfig_SetBytesString(&config, &config.home, try stdx.fs.path.bufJoinZ(&bufZ, &.{ + self_exe_dir, + "..", + "lib", + "python3.11", + }))); + } + + { + var buf: [std.fs.max_path_bytes]u8 = undefined; + config.module_search_paths_set = 1; + pyErrorOrExit(c.PyWideStringList_Append(&config.module_search_paths, &(try toPosixPathW(try stdx.fs.path.bufJoin(&buf, &.{ + self_exe_dir, + "..", + "lib", + "python3.11", + }))))); + pyErrorOrExit(c.PyWideStringList_Append(&config.module_search_paths, &(try toPosixPathW(try stdx.fs.path.bufJoin(&buf, &.{ + self_exe_dir, + "..", + "site-packages", + }))))); + } + + std.debug.print(">>>> CORENTIN\n", .{}); + + pyErrorOrExit(c.Py_InitializeFromConfig(&config)); + defer c.Py_Finalize(); + + const neuronxcc_main = blk: { + const module = c.PyImport_ImportModule("neuronxcc.driver.CommandDriver"); + std.debug.print(">>> MODULE: {any}\n", .{module}); + defer c.Py_DecRef(module); + break :blk c.PyObject_GetAttrString(module, "main"); + }; + defer c.Py_DecRef(neuronxcc_main); + + const result = c.PyObject_CallNoArgs(neuronxcc_main); + defer c.Py_DecRef(result); +} From 9a86402c2b3a82f06c6629863431ab2c1599f85d Mon Sep 17 00:00:00 2001 From: Corentin Kerisit Date: Fri, 25 Jul 2025 18:55:33 +0000 Subject: [PATCH 05/10] WIP --- BUILD.bazel | 2 +- pjrt/pjrt.zig | 4 --- runtimes/neuron/BUILD.bazel | 3 +- runtimes/neuron/libpjrt_neuron.zig | 58 ++++++++++++++++++++++-------- 4 files changed, 47 insertions(+), 20 deletions(-) diff --git a/BUILD.bazel b/BUILD.bazel index 137f1800..20b070ad 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -13,6 +13,6 @@ zls_completion( "//async", "//stdx", "//zml", - "//runtimes/neuron:neuronx-cc_lib", + "//runtimes/neuron:bin/neuronx-cc_lib", ], ) diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 92dc4ebf..58d9ac29 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -34,10 +34,6 @@ fn pjrtStructSize(comptime T: type) usize { pub inline fn pjrtStruct(v: anytype) @TypeOf(v) { var ret = v; ret.struct_size = pjrtStructSize(@TypeOf(v)); - if (@TypeOf(v) == c.PJRT_Plugin_Attributes_Args) { - std.debug.print("type is {any} and size is {}\n", .{ @TypeOf(v), ret.struct_size }); - ret.struct_size = 0x18; - } return ret; } diff --git a/runtimes/neuron/BUILD.bazel b/runtimes/neuron/BUILD.bazel index f7fe7c94..aa9a29c3 100644 --- a/runtimes/neuron/BUILD.bazel +++ b/runtimes/neuron/BUILD.bazel @@ -69,9 +69,10 @@ zig_binary( main = "libpjrt_neuron.zig", copts = ["-lc"], deps = [ + ":libpython", "//stdx", "@rules_zig//zig/runfiles", - ":libpython", + "@xla//xla/pjrt/c:pjrt_c_api_hdrs", ], visibility = ["@libpjrt_neuron//:__subpackages__"], ) diff --git a/runtimes/neuron/libpjrt_neuron.zig b/runtimes/neuron/libpjrt_neuron.zig index e0eedc34..ec84c4e9 100644 --- a/runtimes/neuron/libpjrt_neuron.zig +++ b/runtimes/neuron/libpjrt_neuron.zig @@ -7,6 +7,9 @@ const stdx = @import("stdx"); const log = std.log.scoped(.@"zml/runtimes/neuron"); +var api_inner: *c.PJRT_Api = undefined; +var api_proxy: c.PJRT_Api = undefined; + fn comptimeStrJoin(comptime separator: [:0]const u8, comptime slices: []const [:0]const u8) [:0]const u8 { comptime var ret = slices[0]; inline for (slices[1..]) |slice| { @@ -95,13 +98,12 @@ fn setupPythonEnv(sandbox_path: []const u8) !void { fn setupBinPath(allocator: std.mem.Allocator, sandbox_path: []const u8) !void { var buf: [std.fs.max_path_bytes]u8 = undefined; const bin_path = try stdx.fs.path.bufJoin(&buf, &.{ sandbox_path, "bin" }); - std.debug.print(">>>>>>>>> {s}\n\n", .{bin_path}); const old_path = std.posix.getenv("PATH") orelse ""; const new_path = try std.fmt.allocPrintZ(allocator, "{s}:{s}", .{ bin_path, old_path }); _ = c.setenv("PATH", new_path, 1); } -pub export fn GetPjrtApi() ?*anyopaque { +pub export fn GetPjrtApi() ?*c.PJRT_Api { var arena = std.heap.ArenaAllocator.init(std.heap.c_allocator); defer arena.deinit(); @@ -129,23 +131,51 @@ pub export fn GetPjrtApi() ?*anyopaque { }; } - const GetPjrtApi_inner = blk: { - var lib_path_buf: [std.fs.max_path_bytes]u8 = undefined; - const library = stdx.fs.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libneuronpjrt.so" }) catch unreachable; + api_inner = blk: { + const GetPjrtApi_inner = GetPjrtApi_blk: { + var lib: std.DynLib = .{ + .inner = .{ + .handle = handle_blk: { + var lib_path_buf: [std.fs.max_path_bytes]u8 = undefined; + const library = stdx.fs.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libneuronpjrt.so" }) catch |err| { + log.err("Failed to find NEURON PJRT library: {any}", .{err}); + return null; + }; + break :handle_blk std.c.dlopen(library, .{ .LAZY = true, .GLOBAL = false, .NODELETE = true }) orelse { + log.err("Unable to dlopen plugin: {?s}", .{std.mem.span(std.c.dlerror())}); + return null; + }; + }, + }, + }; - var lib: std.DynLib = blk_lib: { - const handle = std.c.dlopen(library, .{ .LAZY = true, .GLOBAL = true, .NODELETE = true }) orelse { - log.err("Unable to dlopen plugin: {s}", .{library}); + break :GetPjrtApi_blk lib.lookup(*const fn () callconv(.c) *c.PJRT_Api, "GetPjrtApi") orelse { + log.err("Unable to find symbol GetPjrtApi in plugin: {?s}", .{std.mem.span(std.c.dlerror())}); return null; }; - break :blk_lib .{ .inner = .{ .handle = handle } }; }; - break :blk lib.lookup(*const fn () callconv(.C) *anyopaque, "GetPjrtApi") orelse { - log.err("Unable to find symbol GetPjrtApi in plugin: {s}", .{library}); - return null; - }; + break :blk GetPjrtApi_inner(); }; - return GetPjrtApi_inner(); + api_proxy = blk: { + var tmp: c.PJRT_Api = .{}; + const api_struct_size = @min(@sizeOf(c.PJRT_Api), api_inner.struct_size); + @memcpy( + std.mem.asBytes(&tmp)[0..api_struct_size], + std.mem.asBytes(api_inner)[0..api_struct_size], + ); + break :blk tmp; + }; + + // Setup the API proxy functions + api_proxy.PJRT_Plugin_Attributes = &struct { + fn call(args: [*c]c.PJRT_Plugin_Attributes_Args) callconv(.c) ?*c.PJRT_Error { + var new_args = args.*; + new_args.struct_size = 0x18; + return api_inner.PJRT_Plugin_Attributes.?(&new_args); + } + }.call; + + return &api_proxy; } From e1dfa5f4ffeaa769373583e9e1e530649f4f60b5 Mon Sep 17 00:00:00 2001 From: Corentin Kerisit Date: Fri, 25 Jul 2025 18:58:04 +0000 Subject: [PATCH 06/10] WIP --- runtimes/neuron/BUILD.bazel | 6 ------ 1 file changed, 6 deletions(-) diff --git a/runtimes/neuron/BUILD.bazel b/runtimes/neuron/BUILD.bazel index aa9a29c3..8aa786b3 100644 --- a/runtimes/neuron/BUILD.bazel +++ b/runtimes/neuron/BUILD.bazel @@ -181,12 +181,6 @@ zig_library( import_name = "runtimes/neuron", main = "neuron.zig", visibility = ["//visibility:public"], - data = select({ - "//runtimes:neuron.enabled": [ - ":libneuronxla", - ], - "//conditions:default": [], - }), deps = [ "//pjrt", ] + select({ From 5961c1290f27fd66356ac14f81719dfaddd211ac Mon Sep 17 00:00:00 2001 From: Corentin Kerisit Date: Fri, 25 Jul 2025 23:40:20 +0000 Subject: [PATCH 07/10] WIP --- runtimes/neuron/BUILD.bazel | 11 +++ runtimes/neuron/libneuronxla.zig | 79 ++++++++++++++++++++++ runtimes/neuron/libpjrt_neuron.BUILD.bazel | 4 +- 3 files changed, 93 insertions(+), 1 deletion(-) create mode 100644 runtimes/neuron/libneuronxla.zig diff --git a/runtimes/neuron/BUILD.bazel b/runtimes/neuron/BUILD.bazel index 8aa786b3..aeeda8b0 100644 --- a/runtimes/neuron/BUILD.bazel +++ b/runtimes/neuron/BUILD.bazel @@ -205,3 +205,14 @@ filegroup( ], visibility = ["//visibility:public"], ) + +zig_binary( + name = "neuronxla", + kind = BINARY_KIND.shared_lib, + main = "libneuronxla.zig", + copts = ["-lc"], + deps = [ + ":libpython", + ], + visibility = ["@libpjrt_neuron//:__subpackages__"], +) diff --git a/runtimes/neuron/libneuronxla.zig b/runtimes/neuron/libneuronxla.zig new file mode 100644 index 00000000..ed9afced --- /dev/null +++ b/runtimes/neuron/libneuronxla.zig @@ -0,0 +1,79 @@ +const std = @import("std"); +const c = @import("c"); + +var module_def: c.PyModuleDef = .{ + .m_base = .{}, + .m_name = "libneuronxla", + .m_doc = "Example module written in Zig.", + .m_size = 0, + .m_methods = @constCast(&[_]c.PyMethodDef{ + .{ + .ml_name = "hook", + .ml_meth = @ptrCast(&hook), + .ml_flags = c.METH_NOARGS, + .ml_doc = "Return a greeting from Zig.", + }, + .{ + .ml_name = "neuronx_cc", + .ml_meth = @ptrCast(&neuronx_cc), + .ml_flags = c.METH_VARARGS, + .ml_doc = "Return a greeting from Zig.", + }, + .{}, + }), + .m_slots = @constCast(&[_]c.PyModuleDef_Slot{ + .{ .slot = c.Py_mod_exec, .value = @constCast(@ptrCast(&module_exec)) }, + .{}, + }), + .m_traverse = null, + .m_clear = null, + .m_free = null, +}; + +fn module_exec(module: ?*c.PyObject) callconv(.c) c_int { + _ = module; + return 0; // 0 = success, -1 = failure +} + +fn hook(self: ?*c.PyObject, args: ?*c.PyObject) callconv(.c) ?*c.PyObject { + _ = self; + _ = args; + std.debug.print("Hello from Zig!\n", .{}); + const none = c.Py_None(); + defer c.Py_IncRef(none); + return none; +} + +fn neuronx_cc(self: ?*c.PyObject, args: ?*c.PyObject) callconv(.c) ?*c.PyObject { + _ = self; + _ = args; + std.debug.print("Hello from Zig! 2222222\n", .{}); + + const none = c.Py_None(); + c.Py_IncRef(none); + + const tuple = c.PyTuple_New(2) orelse { + c.Py_DecRef(none); + return null; + }; + const empty_bytes = c.PyBytes_FromStringAndSize(null, 0) orelse { + c.Py_DecRef(none); + c.Py_DecRef(tuple); + return null; + }; + const py_long = c.PyLong_FromLongLong(0) orelse { + c.Py_DecRef(none); + c.Py_DecRef(tuple); + return null; + }; + + _ = c.PyTuple_SetItem(tuple, 0, py_long); + _ = c.PyTuple_SetItem(tuple, 1, empty_bytes); + + return tuple; +} + + +pub export fn PyInit_libneuronxla() callconv(.c) ?*c.PyObject { + return c.PyModuleDef_Init(&module_def); +} diff --git a/runtimes/neuron/libpjrt_neuron.BUILD.bazel b/runtimes/neuron/libpjrt_neuron.BUILD.bazel index 68843fc6..46545fa9 100644 --- a/runtimes/neuron/libpjrt_neuron.BUILD.bazel +++ b/runtimes/neuron/libpjrt_neuron.BUILD.bazel @@ -54,7 +54,7 @@ patchelf( runfiles_to_default( name = "neuronx_cc_runfiles", deps = [ - "@neuron_py_deps//libneuronxla", + # "@neuron_py_deps//libneuronxla", "@neuron_py_deps//neuronx_cc", ], ) @@ -68,6 +68,7 @@ copy_to_directory( ":neuronx_cc_runfiles", "@rules_python//python:current_py_toolchain", "@zml//runtimes/neuron:neuronx-cc", + "@zml//runtimes/neuron:neuronxla", "@aws-neuronx-runtime-lib//:libnrt.patchelf", "@aws-neuronx-runtime-lib//:libncfw.patchelf", "@aws-neuronx-collectives//:libnccom", @@ -76,6 +77,7 @@ copy_to_directory( ], replace_prefixes = { "runtimes/neuron/neuronx-cc": "bin/neuronx-cc", + "runtimes/neuron/libneuronxla.so": "site-packages/libneuronxla.so", "libneuronpjrt.patchelf": "lib", "libpjrt_neuron_proxy.patchelf": "lib", "libnrt.patchelf": "lib", From b60a0403a4793c21b4cd0b8c3d2b24b2b0e0fe17 Mon Sep 17 00:00:00 2001 From: Corentin Kerisit Date: Sat, 26 Jul 2025 15:44:27 +0000 Subject: [PATCH 08/10] WIP --- BUILD.bazel | 2 +- runtimes/neuron/BUILD.bazel | 1 + runtimes/neuron/libneuronxla.zig | 115 +++++++++++++++++++++++++++++-- 3 files changed, 113 insertions(+), 5 deletions(-) diff --git a/BUILD.bazel b/BUILD.bazel index 20b070ad..855d560a 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -13,6 +13,6 @@ zls_completion( "//async", "//stdx", "//zml", - "//runtimes/neuron:bin/neuronx-cc_lib", + # "//runtimes/neuron:bin/neuronx-cc_lib", ], ) diff --git a/runtimes/neuron/BUILD.bazel b/runtimes/neuron/BUILD.bazel index aeeda8b0..b6362e7f 100644 --- a/runtimes/neuron/BUILD.bazel +++ b/runtimes/neuron/BUILD.bazel @@ -213,6 +213,7 @@ zig_binary( copts = ["-lc"], deps = [ ":libpython", + "//stdx", ], visibility = ["@libpjrt_neuron//:__subpackages__"], ) diff --git a/runtimes/neuron/libneuronxla.zig b/runtimes/neuron/libneuronxla.zig index ed9afced..0e820e92 100644 --- a/runtimes/neuron/libneuronxla.zig +++ b/runtimes/neuron/libneuronxla.zig @@ -1,5 +1,21 @@ const std = @import("std"); + const c = @import("c"); +const stdx = @import("stdx"); + +const log = std.log.scoped(.@"zml/runtimes/neuron/libneuronxla"); + +pub fn makeTempDir(buf: []u8, prefix: []const u8) ![]const u8 { + const tmp_dir = std.posix.getenv("TMPDIR") orelse "/tmp"; + const ret = try std.fmt.bufPrint(buf, "{s}{s}{s}{d}", .{ + tmp_dir, + std.fs.path.sep_str_posix, + prefix, + std.time.microTimestamp(), + }); + try std.fs.makeDirAbsolute(ret); + return ret; +} var module_def: c.PyModuleDef = .{ .m_base = .{}, @@ -16,7 +32,7 @@ var module_def: c.PyModuleDef = .{ .{ .ml_name = "neuronx_cc", .ml_meth = @ptrCast(&neuronx_cc), - .ml_flags = c.METH_VARARGS, + .ml_flags = c.METH_FASTCALL, .ml_doc = "Return a greeting from Zig.", }, .{}, @@ -44,14 +60,90 @@ fn hook(self: ?*c.PyObject, args: ?*c.PyObject) callconv(.c) ?*c.PyObject { return none; } -fn neuronx_cc(self: ?*c.PyObject, args: ?*c.PyObject) callconv(.c) ?*c.PyObject { +pub fn PyBytes_AsStringAndSize(object: *c.PyObject) []u8 { + var buf: [*c]u8 = undefined; + var len: c.Py_ssize_t = undefined; + _ = c.PyBytes_AsStringAndSize(object, &buf, &len); + return buf[0..@intCast(len)]; +} + +fn neuronx_cc_(self: ?*c.PyObject, args_: [*c]*c.PyObject, nargs_: c.Py_ssize_t) !?*c.PyObject { + var arena = std.heap.ArenaAllocator.init(std.heap.c_allocator); + defer arena.deinit(); + _ = self; - _ = args; - std.debug.print("Hello from Zig! 2222222\n", .{}); + const args = args_[0..@intCast(nargs_)]; + + const code = PyBytes_AsStringAndSize(args[0]); + const platform_version = PyBytes_AsStringAndSize(args[2]); + + const target: []const u8 = blk: { + if (std.mem.eql(u8, platform_version, "1.0")) { + break :blk "--target=inf1"; + } else if (std.mem.eql(u8, platform_version, "2.0")) { + break :blk "--target=trn1"; + } else if (std.mem.eql(u8, platform_version, "3.0")) { + break :blk "--target=trn2"; + } else { + log.err("Unknown platform version: {s}\n", .{platform_version}); + return null; + } + }; + + var tmp_dir_buf: [std.fs.max_path_bytes]u8 = undefined; + const tmp_dir = try makeTempDir(&tmp_dir_buf, "zml-neuronxcc-"); + + const code_file = try std.fs.path.join(arena.allocator(), &.{ tmp_dir, "file.code" }); + { + const file = try std.fs.cwd().createFile(code_file, .{ .truncate = true }); + defer file.close(); + try file.writeAll(code); + } + + const wrapped_neff_hlo_file = try std.fs.path.join(arena.allocator(), &.{ tmp_dir, "wrapped_neff.hlo" }); + _ = wrapped_neff_hlo_file; // autofix + const neff_file = try std.fs.path.join(arena.allocator(), &.{ tmp_dir, "file.neff" }); + + var child = std.process.Child.init(&.{ + "neuronx-cc", + "compile", + "--framework=XLA", + target, + "--verbose=35", + "--enable-internal-neff-wrapper", + "--output", + neff_file, + "--optlevel=1", + // generic is the default, but it fails on transformers, force it + "--model-type=transformer", + // disable it, we do our own + "--auto-cast=none", + "--enable-fast-loading-neuron-binaries", + code_file, + }, arena.allocator()); + child.stdin_behavior = .Ignore; + child.stdout_behavior = .Inherit; + child.stderr_behavior = .Inherit; + _ = try child.spawnAndWait(); + + std.debug.print(">>>> {s}\n", .{tmp_dir}); const none = c.Py_None(); c.Py_IncRef(none); + // var arg1: [*c]const u8 = undefined; + // var arg2: c_int = undefined; + // c.Py + // const arg_parse_result = c.PyArg_ParseTuple(args, "s:neuronx_cc", &arg1, &arg2); + + // if (arg_parse_result == 0) { + // std.debug.print("Error parsing arguments\n", .{}); + // c.Py_DecRef(none); + // return null; + // } + + // std.debug.print("arg1: {s}\n", .{arg1}); + const tuple = c.PyTuple_New(2) orelse { c.Py_DecRef(none); return null; @@ -73,6 +165,21 @@ fn neuronx_cc(self: ?*c.PyObject, args: ?*c.PyObject) callconv(.c) ?*c.PyObject return tuple; } +fn neuronx_cc(self: ?*c.PyObject, args_: [*c]*c.PyObject, nargs_: c.Py_ssize_t) callconv(.c) ?*c.PyObject { + return neuronx_cc_(self, args_, nargs_) catch |err| { + log.err("Error in neuronx_cc: {}\n", .{err}); + + const none = c.Py_None(); + c.Py_IncRef(none); + const tuple = c.PyTuple_New(2) orelse { + c.Py_DecRef(none); + return null; + }; + _ = c.PyTuple_SetItem(tuple, 0, c.PyLong_FromLongLong(400).?); + _ = c.PyTuple_SetItem(tuple, 1, none); + return tuple; + }; +} pub export fn PyInit_libneuronxla() callconv(.c) ?*c.PyObject { return c.PyModuleDef_Init(&module_def); From 8d4b5d305fd602a3dc68851b8882f5d7166f9cd9 Mon Sep 17 00:00:00 2001 From: Corentin Kerisit Date: Sun, 27 Jul 2025 20:33:29 +0000 Subject: [PATCH 09/10] WIP --- BUILD.bazel | 2 +- runtimes/neuron/BUILD.bazel | 10 ++- runtimes/neuron/libneuronxla.zig | 114 ++++++++++++++++++++++--------- 3 files changed, 93 insertions(+), 33 deletions(-) diff --git a/BUILD.bazel b/BUILD.bazel index 855d560a..159fef91 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -13,6 +13,6 @@ zls_completion( "//async", "//stdx", "//zml", - # "//runtimes/neuron:bin/neuronx-cc_lib", + "//runtimes/neuron:neuronxla", ], ) diff --git a/runtimes/neuron/BUILD.bazel b/runtimes/neuron/BUILD.bazel index b6362e7f..0f243167 100644 --- a/runtimes/neuron/BUILD.bazel +++ b/runtimes/neuron/BUILD.bazel @@ -6,6 +6,13 @@ load("@rules_zig//zig:defs.bzl", "BINARY_KIND", "zig_binary", "zig_library") load("@zml//bazel:zig.bzl", "zig_cc_binary") load(":neuron.bzl", "py_binary_with_script") load(":pyenv.bzl", "pyenv_zig") +load("@zml//bazel:zig_proto_library.bzl", "zig_proto_library") + +zig_proto_library( + name = "hlo_proto", + import_name = "hlo_proto", + deps = ["@xla//xla/service:hlo_proto"], +) # load("@aspect_bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") # load("@zml//bazel:patchelf.bzl", "patchelf") @@ -212,8 +219,9 @@ zig_binary( main = "libneuronxla.zig", copts = ["-lc"], deps = [ + ":hlo_proto", ":libpython", "//stdx", ], - visibility = ["@libpjrt_neuron//:__subpackages__"], + visibility = ["//visibility:public"], ) diff --git a/runtimes/neuron/libneuronxla.zig b/runtimes/neuron/libneuronxla.zig index 0e820e92..6a76eb26 100644 --- a/runtimes/neuron/libneuronxla.zig +++ b/runtimes/neuron/libneuronxla.zig @@ -1,6 +1,7 @@ const std = @import("std"); const c = @import("c"); +const hlo_proto = @import("hlo_proto"); const stdx = @import("stdx"); const log = std.log.scoped(.@"zml/runtimes/neuron/libneuronxla"); @@ -67,6 +68,68 @@ pub fn PyBytes_AsStringAndSize(object: *c.PyObject) []u8 { return buf[0..@intCast(len)]; } +fn wrapNeffAsCustomCall(allocator: std.mem.Allocator, hlo_code: []const u8, neff_file_path: []const u8) ![]const u8 { + var arena = std.heap.ArenaAllocator.init(allocator); + defer arena.deinit(); + + const hlo_module = try hlo_proto.HloModuleProto.decode(hlo_code, arena.allocator()); + var entry = blk: { + for (hlo_module.computations.items) |*comp| { + if (comp.id == hlo_module.entry_computation_id) { + break :blk comp; + } + } else return error.ComputationNotFound; + }; + + var fused_root: hlo_proto.HloInstructionProto = blk: { + for (entry.instructions.items) |inst| { + if (inst.id == entry.root_id) { + break :blk try inst.dupe(arena.allocator()); + } + } else return error.CustomCallNotFound; + }; + + fused_root.opcode = .{ .Const = "custom-call" }; + fused_root.custom_call_target = .{ .Const = "AwsNeuronNeff" }; + fused_root.backend_config = .{ .Owned = blk: { + const neff_file = try std.fs.openFileAbsolute(neff_file_path, .{}); + defer neff_file.close(); + const stat = try neff_file.stat(); + const neff_buf = try arena.allocator().alloc(u8, @intCast(stat.size)); + _ = try neff_file.readAll(neff_buf); + break :blk neff_buf; + } }; + + const parameters_len = entry.program_shape.?.parameters.items.len; + var new_entry_instructions: std.ArrayListUnmanaged(hlo_proto.HloInstructionProto) = try .initCapacity( + arena.allocator(), + parameters_len + 1, // params + fused_root + ); + + fused_root.operand_ids.clearRetainingCapacity(); + for (entry.instructions.items) |inst| { + if (std.mem.eql(u8, inst.opcode.getSlice(), "parameter")) { + try fused_root.operand_ids.append(arena.allocator(), inst.id); + try new_entry_instructions.append(arena.allocator(), inst); + } + } + + try fused_root.frontend_attributes.?.map.append(arena.allocator(), .{ + .key = .{ .Const = "valid_inputs" }, + .value = .{ .Owned = blk: { + const valid_inputs_value = try arena.allocator().alloc(u8, parameters_len * 2 - 1); + for (valid_inputs_value, 0..) |*char, i| { + char.* = if (i % 2 == 0) '1' else ','; + } + break :blk valid_inputs_value; + } }, + }); + try new_entry_instructions.append(arena.allocator(), fused_root); + entry.instructions = new_entry_instructions; + + return hlo_module.encode(allocator); +} + fn neuronx_cc_(self: ?*c.PyObject, args_: [*c]*c.PyObject, nargs_: c.Py_ssize_t) !?*c.PyObject { var arena = std.heap.ArenaAllocator.init(std.heap.c_allocator); defer arena.deinit(); @@ -77,17 +140,13 @@ fn neuronx_cc_(self: ?*c.PyObject, args_: [*c]*c.PyObject, nargs_: c.Py_ssize_t) const code = PyBytes_AsStringAndSize(args[0]); const platform_version = PyBytes_AsStringAndSize(args[2]); - const target: []const u8 = blk: { - if (std.mem.eql(u8, platform_version, "1.0")) { - break :blk "--target=inf1"; - } else if (std.mem.eql(u8, platform_version, "2.0")) { - break :blk "--target=trn1"; - } else if (std.mem.eql(u8, platform_version, "3.0")) { - break :blk "--target=trn2"; - } else { - log.err("Unknown platform version: {s}\n", .{platform_version}); - return null; - } + const target = std.StaticStringMap([]const u8).initComptime(.{ + .{ "1.0", "inf1" }, + .{ "2.0", "trn1" }, + .{ "3.0", "trn2" }, + }).get(platform_version) orelse { + log.err("Unknown platform version: {s}\n", .{platform_version}); + return error.UnknownPlatformVersion; }; var tmp_dir_buf: [std.fs.max_path_bytes]u8 = undefined; @@ -108,8 +167,9 @@ fn neuronx_cc_(self: ?*c.PyObject, args_: [*c]*c.PyObject, nargs_: c.Py_ssize_t) "neuronx-cc", "compile", "--framework=XLA", + "--target", target, - "--verbose=35", + "--verbose=debug", "--enable-internal-neff-wrapper", "--output", neff_file, @@ -124,10 +184,16 @@ fn neuronx_cc_(self: ?*c.PyObject, args_: [*c]*c.PyObject, nargs_: c.Py_ssize_t) child.stdin_behavior = .Ignore; child.stdout_behavior = .Inherit; child.stderr_behavior = .Inherit; + child.cwd = tmp_dir; _ = try child.spawnAndWait(); std.debug.print(">>>> {s}\n", .{tmp_dir}); + const neff_hlo_bytes = wrapNeffAsCustomCall(arena.allocator(), code, neff_file) catch |err| { + log.err("Error wrapping NEFF as custom call: {}\n", .{err}); + return err; + }; + const none = c.Py_None(); c.Py_IncRef(none); @@ -144,25 +210,11 @@ fn neuronx_cc_(self: ?*c.PyObject, args_: [*c]*c.PyObject, nargs_: c.Py_ssize_t) // std.debug.print("arg1: {s}\n", .{arg1}); - const tuple = c.PyTuple_New(2) orelse { - c.Py_DecRef(none); - return null; - }; - const empty_bytes = c.PyBytes_FromStringAndSize(null, 0) orelse { - c.Py_DecRef(none); - c.Py_DecRef(tuple); - return null; - }; - const py_long = c.PyLong_FromLongLong(0) orelse { - c.Py_DecRef(none); - c.Py_DecRef(tuple); - return null; - }; - - _ = c.PyTuple_SetItem(tuple, 0, py_long); - _ = c.PyTuple_SetItem(tuple, 1, empty_bytes); - - return tuple; + return c.PyTuple_Pack( + 2, + c.PyLong_FromLongLong(0), + c.PyBytes_FromStringAndSize(@ptrCast(neff_hlo_bytes), @intCast(neff_hlo_bytes.len)), + ); } fn neuronx_cc(self: ?*c.PyObject, args_: [*c]*c.PyObject, nargs_: c.Py_ssize_t) callconv(.c) ?*c.PyObject { From abc5ea1a341e85059f04ad2c91c1076fd8eae7c9 Mon Sep 17 00:00:00 2001 From: Corentin Kerisit Date: Mon, 28 Jul 2025 10:11:09 +0000 Subject: [PATCH 10/10] WIP --- runtimes/neuron/libneuronxla.zig | 32 ++++++----------- runtimes/neuron/libpjrt_neuron.BUILD.bazel | 25 ++++--------- runtimes/neuron/libpjrt_neuron.zig | 41 +++++++++------------- 3 files changed, 32 insertions(+), 66 deletions(-) diff --git a/runtimes/neuron/libneuronxla.zig b/runtimes/neuron/libneuronxla.zig index 6a76eb26..fc670bb3 100644 --- a/runtimes/neuron/libneuronxla.zig +++ b/runtimes/neuron/libneuronxla.zig @@ -107,10 +107,11 @@ fn wrapNeffAsCustomCall(allocator: std.mem.Allocator, hlo_code: []const u8, neff ); fused_root.operand_ids.clearRetainingCapacity(); + try fused_root.operand_ids.ensureTotalCapacityPrecise(arena.allocator(), parameters_len); for (entry.instructions.items) |inst| { if (std.mem.eql(u8, inst.opcode.getSlice(), "parameter")) { - try fused_root.operand_ids.append(arena.allocator(), inst.id); - try new_entry_instructions.append(arena.allocator(), inst); + fused_root.operand_ids.appendAssumeCapacity(inst.id); + new_entry_instructions.appendAssumeCapacity(inst); } } @@ -124,6 +125,7 @@ fn wrapNeffAsCustomCall(allocator: std.mem.Allocator, hlo_code: []const u8, neff break :blk valid_inputs_value; } }, }); + try new_entry_instructions.append(arena.allocator(), fused_root); entry.instructions = new_entry_instructions; @@ -131,10 +133,11 @@ fn wrapNeffAsCustomCall(allocator: std.mem.Allocator, hlo_code: []const u8, neff } fn neuronx_cc_(self: ?*c.PyObject, args_: [*c]*c.PyObject, nargs_: c.Py_ssize_t) !?*c.PyObject { + _ = self; + var arena = std.heap.ArenaAllocator.init(std.heap.c_allocator); defer arena.deinit(); - _ = self; const args = args_[0..@intCast(nargs_)]; const code = PyBytes_AsStringAndSize(args[0]); @@ -151,6 +154,9 @@ fn neuronx_cc_(self: ?*c.PyObject, args_: [*c]*c.PyObject, nargs_: c.Py_ssize_t) var tmp_dir_buf: [std.fs.max_path_bytes]u8 = undefined; const tmp_dir = try makeTempDir(&tmp_dir_buf, "zml-neuronxcc-"); + defer std.fs.deleteTreeAbsolute(tmp_dir) catch |err| { + log.err("Error deleting temporary directory {s}: {}\n", .{ tmp_dir, err }); + }; const code_file = try std.fs.path.join(arena.allocator(), &.{ tmp_dir, "file.code" }); { @@ -159,8 +165,6 @@ fn neuronx_cc_(self: ?*c.PyObject, args_: [*c]*c.PyObject, nargs_: c.Py_ssize_t) try file.writeAll(code); } - const wrapped_neff_hlo_file = try std.fs.path.join(arena.allocator(), &.{ tmp_dir, "wrapped_neff.hlo" }); - _ = wrapped_neff_hlo_file; // autofix const neff_file = try std.fs.path.join(arena.allocator(), &.{ tmp_dir, "file.neff" }); var child = std.process.Child.init(&.{ @@ -169,7 +173,7 @@ fn neuronx_cc_(self: ?*c.PyObject, args_: [*c]*c.PyObject, nargs_: c.Py_ssize_t) "--framework=XLA", "--target", target, - "--verbose=debug", + "--verbose=info", "--enable-internal-neff-wrapper", "--output", neff_file, @@ -194,22 +198,6 @@ fn neuronx_cc_(self: ?*c.PyObject, args_: [*c]*c.PyObject, nargs_: c.Py_ssize_t) return err; }; - const none = c.Py_None(); - c.Py_IncRef(none); - - // var arg1: [*c]const u8 = undefined; - // var arg2: c_int = undefined; - // c.Py - // const arg_parse_result = c.PyArg_ParseTuple(args, "s:neuronx_cc", &arg1, &arg2); - - // if (arg_parse_result == 0) { - // std.debug.print("Error parsing arguments\n", .{}); - // c.Py_DecRef(none); - // return null; - // } - - // std.debug.print("arg1: {s}\n", .{arg1}); - return c.PyTuple_Pack( 2, c.PyLong_FromLongLong(0), diff --git a/runtimes/neuron/libpjrt_neuron.BUILD.bazel b/runtimes/neuron/libpjrt_neuron.BUILD.bazel index 46545fa9..9998a87d 100644 --- a/runtimes/neuron/libpjrt_neuron.BUILD.bazel +++ b/runtimes/neuron/libpjrt_neuron.BUILD.bazel @@ -5,18 +5,6 @@ load("@bazel_skylib//rules:select_file.bzl", "select_file") load("@rules_cc//cc:cc_shared_library.bzl", "cc_shared_library") load("@zml//bazel:runfiles.bzl", "runfiles_to_default") -cc_shared_library( - name = "zmlxneuron_so_", - shared_lib_name = "libzmlxneuron.so.0", - deps = ["@zml//runtimes/neuron:zmlxneuron_lib"], -) - -copy_file( - name = "zmlxneuron_so", - src = ":zmlxneuron_so_", - out = "lib/libzmlxneuron.so.0", -) - runfiles_to_default( name = "libneuronxla_files", deps = ["@neuron_py_deps//libneuronxla:pkg"], @@ -30,11 +18,11 @@ select_file( patchelf( name = "libneuronpjrt.patchelf", - add_needed = [ - "libpython3.11.so.1.0", - "libzmlxneuron.so.0", - "libnccom.so.2", - ], + # add_needed = [ + # "libpython3.11.so.1.0", + # # "libzmlxneuron.so.0", + # "libnccom.so.2", + # ], set_rpath = '$ORIGIN', shared_library = ":libneuronpjrt_so", soname = "libneuronpjrt.so", @@ -54,7 +42,6 @@ patchelf( runfiles_to_default( name = "neuronx_cc_runfiles", deps = [ - # "@neuron_py_deps//libneuronxla", "@neuron_py_deps//neuronx_cc", ], ) @@ -62,7 +49,7 @@ runfiles_to_default( copy_to_directory( name = "sandbox", srcs = [ - ":zmlxneuron_so", + # ":zmlxneuron_so", ":libneuronpjrt.patchelf", ":libpjrt_neuron_proxy.patchelf", ":neuronx_cc_runfiles", diff --git a/runtimes/neuron/libpjrt_neuron.zig b/runtimes/neuron/libpjrt_neuron.zig index ec84c4e9..54ba1764 100644 --- a/runtimes/neuron/libpjrt_neuron.zig +++ b/runtimes/neuron/libpjrt_neuron.zig @@ -10,12 +10,16 @@ const log = std.log.scoped(.@"zml/runtimes/neuron"); var api_inner: *c.PJRT_Api = undefined; var api_proxy: c.PJRT_Api = undefined; -fn comptimeStrJoin(comptime separator: [:0]const u8, comptime slices: []const [:0]const u8) [:0]const u8 { - comptime var ret = slices[0]; - inline for (slices[1..]) |slice| { - ret = ret ++ separator ++ slice; - } - return ret; +pub export fn zmlxneuron_dlopen(filename: [*c]const u8, flags: c_int) ?*anyopaque { + const replacements: std.StaticStringMap([:0]const u8) = .initComptime(.{ + .{ "libnccom.so", "libnccom.so.2" }, + .{ "libnrt.so", "libnrt.so.1" }, + .{ "libncfw.so", "libncfw.so.2" }, + }); + return std.c.dlopen( + if (filename) |f| replacements.get(std.mem.span(f)) orelse filename else null, + @bitCast(flags), + ); } fn pyErrorOrExit(status: c.PyStatus) void { @@ -36,24 +40,6 @@ pub fn toPosixPathW(file_path: []const u8) error{NameTooLong}![std.posix.PATH_MA return path_with_null; } -pub fn setNeuronCCFlags() void { - // See neuronxcc reference: - // https://awsdocs-neuron.readthedocs-hosted.com/en/latest/compiler/neuronx-cc/api-reference-guide/neuron-compiler-cli-reference-guide.html#neuron-compiler-cli-reference-guide - _ = c.setenv("NEURON_CC_FLAGS", comptimeStrJoin(" ", &.{ - // 30% faster, no visible speed difference on llama - "--optlevel=1", - // generic is the default, but it fails on transformers, force it - "--model-type=transformer", - // disable it, we do our own - "--auto-cast=none", - "--enable-fast-loading-neuron-binaries", - }), 1); - - // Enable stochastic rounding - // https://awsdocs-neuron.readthedocs-hosted.com/en/latest/general/arch/neuron-features/rounding-modes.html - _ = c.setenv("NEURON_RT_STOCHASTIC_ROUNDING_EN", "1", 1); -} - fn setupPythonEnv(sandbox_path: []const u8) !void { { var preconfig: c.PyPreConfig = undefined; @@ -107,6 +93,8 @@ pub export fn GetPjrtApi() ?*c.PJRT_Api { var arena = std.heap.ArenaAllocator.init(std.heap.c_allocator); defer arena.deinit(); + std.debug.print("ONE\n", .{}); + var r_ = runfiles.Runfiles.create(.{ .allocator = arena.allocator() }) catch |err| { stdx.debug.panic("Unable to find runfiles: {}", .{err}); } orelse stdx.debug.panic("Runfiles not availeabwewefle", .{}); @@ -119,8 +107,9 @@ pub export fn GetPjrtApi() ?*c.PJRT_Api { stdx.debug.panic("Failed to find sandbox path for NEURON runtime: {}", .{err}); } orelse stdx.debug.panic("No NEURON sandbox path found", .{}); + std.debug.print("TWO\n", .{}); + { - setNeuronCCFlags(); setupBinPath(arena.allocator(), sandbox_path) catch |err| { log.err("Failed to setup bin path: {}", .{err}); return null; @@ -158,6 +147,8 @@ pub export fn GetPjrtApi() ?*c.PJRT_Api { break :blk GetPjrtApi_inner(); }; + std.debug.print("THREE\n", .{}); + api_proxy = blk: { var tmp: c.PJRT_Api = .{}; const api_struct_size = @min(@sizeOf(c.PJRT_Api), api_inner.struct_size);