diff --git a/BUILD.bazel b/BUILD.bazel index 63540f5e..159fef91 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -13,5 +13,6 @@ zls_completion( "//async", "//stdx", "//zml", + "//runtimes/neuron:neuronxla", ], ) diff --git a/bazel/zig.bzl b/bazel/zig.bzl index 26773a5e..7132b4b8 100644 --- a/bazel/zig.bzl +++ b/bazel/zig.bzl @@ -10,6 +10,7 @@ def zig_cc_binary( data = [], deps = [], tags = [], + linkopts = [], visibility = None, **kwargs): zig_binary( @@ -25,6 +26,7 @@ def zig_cc_binary( args = args, env = env, data = data, + linkopts = linkopts, deps = [":{}_lib".format(name)], tags = tags, visibility = visibility, diff --git a/runtimes/neuron/BUILD.bazel b/runtimes/neuron/BUILD.bazel index b65d7a38..0f243167 100644 --- a/runtimes/neuron/BUILD.bazel +++ b/runtimes/neuron/BUILD.bazel @@ -3,12 +3,49 @@ load("@rules_python//python:pip.bzl", "compile_pip_requirements") load("@rules_python//python/entry_points:py_console_script_binary.bzl", "py_console_script_binary") load("@rules_uv//uv:pip.bzl", "pip_compile") load("@rules_zig//zig:defs.bzl", "BINARY_KIND", "zig_binary", "zig_library") +load("@zml//bazel:zig.bzl", "zig_cc_binary") load(":neuron.bzl", "py_binary_with_script") load(":pyenv.bzl", "pyenv_zig") +load("@zml//bazel:zig_proto_library.bzl", "zig_proto_library") + +zig_proto_library( + name = "hlo_proto", + import_name = "hlo_proto", + deps = ["@xla//xla/service:hlo_proto"], +) + +# load("@aspect_bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") +# load("@zml//bazel:patchelf.bzl", "patchelf") +# load("@zml//bazel:runfiles.bzl", "runfiles_to_default") + + +# runfiles_to_default( +# name = "runfiles", +# deps = ["@neuron_py_deps//neuronx_cc"], +# ) + +# copy_to_directory( +# name = "pysandbox", +# srcs = [ +# ":bin/neuronx-cc", +# "@rules_python//python:current_py_toolchain", +# ":runfiles", +# ], +# allow_overwrites = True, +# add_directory_to_runfiles = True, +# include_external_repositories = ["**"], +# include_srcs_patterns = [ +# "bin/neuronx-cc", +# "lib/libpython3.11*/**", +# "lib/python*/**", +# "site-packages/**", +# ], +# hardlink = "on", +# ) pyenv_zig( name = "libneuronxla_pyenv_zig", - deps = [":libneuronxla"], + deps = ["@neuron_py_deps//libneuronxla"], ) zig_library( @@ -16,9 +53,19 @@ zig_library( main = ":libneuronxla_pyenv_zig", ) +pyenv_zig( + name = "neuronx_cc_pyenv_zig", + deps = ["@neuron_py_deps//neuronx_cc"], +) + +zig_library( + name = "neuronx_cc_pyenv", + main = ":neuronx_cc_pyenv_zig", +) + # A proxy PJRT Plugin that loads the Neuron PJRT Plugin # and returns the instance from nested GetPjrtApi. -# +# # Additionally, it provides a way to load implicit transitive dependencies # of neuronx-cc (see add_needed of the patchelf target below). # @@ -29,8 +76,10 @@ zig_binary( main = "libpjrt_neuron.zig", copts = ["-lc"], deps = [ + ":libpython", "//stdx", "@rules_zig//zig/runfiles", + "@xla//xla/pjrt/c:pjrt_c_api_hdrs", ], visibility = ["@libpjrt_neuron//:__subpackages__"], ) @@ -53,10 +102,25 @@ pip_compile( ], ) -py_console_script_binary( +zig_cc_binary( name = "neuronx-cc", - binary_rule = py_binary_with_script, - pkg = "@neuron_py_deps//neuronx_cc", + main = "neuronx-cc.zig", + data = [ + "@neuron_py_deps//neuronx_cc", + ], + linkopts = [ + "-Wl,-rpath,$ORIGIN/../lib", + ], + deps = [ + # ":neuronx_cc_pyenv", + ":libpython", + "//async", + "//stdx", + "@rules_zig//zig/runfiles", + ], + tags = [ + "manual", + ], visibility = ["//visibility:public"], ) @@ -107,11 +171,15 @@ cc_library( ) cc_library( - name = "libpjrt_neuron", - hdrs = ["libpjrt_neuron.h"], + name = "zmlxneuron", defines = ["ZML_RUNTIME_NEURON"], +) + +cc_library( + name = "libnrt_headers", + hdrs = ["nrt.h"], deps = [ - "@libpjrt_neuron", + "@libpjrt_neuron//:libnrt_headers", ], ) @@ -120,17 +188,13 @@ zig_library( import_name = "runtimes/neuron", main = "neuron.zig", visibility = ["//visibility:public"], - data = select({ - "//runtimes:neuron.enabled": [ - ":libneuronxla", - ], - "//conditions:default": [], - }), deps = [ "//pjrt", ] + select({ "//runtimes:neuron.enabled": [ - ":libpjrt_neuron", + "@libpjrt_neuron", + ":zmlxneuron", + ":libnrt_headers", ":libneuronxla_pyenv", ":libpython", "//async", @@ -148,3 +212,16 @@ filegroup( ], visibility = ["//visibility:public"], ) + +zig_binary( + name = "neuronxla", + kind = BINARY_KIND.shared_lib, + main = "libneuronxla.zig", + copts = ["-lc"], + deps = [ + ":hlo_proto", + ":libpython", + "//stdx", + ], + visibility = ["//visibility:public"], +) diff --git a/runtimes/neuron/libneuronxla.zig b/runtimes/neuron/libneuronxla.zig new file mode 100644 index 00000000..fc670bb3 --- /dev/null +++ b/runtimes/neuron/libneuronxla.zig @@ -0,0 +1,226 @@ +const std = @import("std"); + +const c = @import("c"); +const hlo_proto = @import("hlo_proto"); +const stdx = @import("stdx"); + +const log = std.log.scoped(.@"zml/runtimes/neuron/libneuronxla"); + +pub fn makeTempDir(buf: []u8, prefix: []const u8) ![]const u8 { + const tmp_dir = std.posix.getenv("TMPDIR") orelse "/tmp"; + const ret = try std.fmt.bufPrint(buf, "{s}{s}{s}{d}", .{ + tmp_dir, + std.fs.path.sep_str_posix, + prefix, + std.time.microTimestamp(), + }); + try std.fs.makeDirAbsolute(ret); + return ret; +} + +var module_def: c.PyModuleDef = .{ + .m_base = .{}, + .m_name = "libneuronxla", + .m_doc = "Example module written in Zig.", + .m_size = 0, + .m_methods = @constCast(&[_]c.PyMethodDef{ + .{ + .ml_name = "hook", + .ml_meth = @ptrCast(&hook), + .ml_flags = c.METH_NOARGS, + .ml_doc = "Return a greeting from Zig.", + }, + .{ + .ml_name = "neuronx_cc", + .ml_meth = @ptrCast(&neuronx_cc), + .ml_flags = c.METH_FASTCALL, + .ml_doc = "Return a greeting from Zig.", + }, + .{}, + }), + .m_slots = @constCast(&[_]c.PyModuleDef_Slot{ + .{ .slot = c.Py_mod_exec, .value = @constCast(@ptrCast(&module_exec)) }, + .{}, + }), + .m_traverse = null, + .m_clear = null, + .m_free = null, +}; + +fn module_exec(module: ?*c.PyObject) callconv(.c) c_int { + _ = module; + return 0; // 0 = success, -1 = failure +} + +fn hook(self: ?*c.PyObject, args: ?*c.PyObject) callconv(.c) ?*c.PyObject { + _ = self; + _ = args; + std.debug.print("Hello from Zig!\n", .{}); + const none = c.Py_None(); + defer c.Py_IncRef(none); + return none; +} + +pub fn PyBytes_AsStringAndSize(object: *c.PyObject) []u8 { + var buf: [*c]u8 = undefined; + var len: c.Py_ssize_t = undefined; + _ = c.PyBytes_AsStringAndSize(object, &buf, &len); + return buf[0..@intCast(len)]; +} + +fn wrapNeffAsCustomCall(allocator: std.mem.Allocator, hlo_code: []const u8, neff_file_path: []const u8) ![]const u8 { + var arena = std.heap.ArenaAllocator.init(allocator); + defer arena.deinit(); + + const hlo_module = try hlo_proto.HloModuleProto.decode(hlo_code, arena.allocator()); + var entry = blk: { + for (hlo_module.computations.items) |*comp| { + if (comp.id == hlo_module.entry_computation_id) { + break :blk comp; + } + } else return error.ComputationNotFound; + }; + + var fused_root: hlo_proto.HloInstructionProto = blk: { + for (entry.instructions.items) |inst| { + if (inst.id == entry.root_id) { + break :blk try inst.dupe(arena.allocator()); + } + } else return error.CustomCallNotFound; + }; + + fused_root.opcode = .{ .Const = "custom-call" }; + fused_root.custom_call_target = .{ .Const = "AwsNeuronNeff" }; + fused_root.backend_config = .{ .Owned = blk: { + const neff_file = try std.fs.openFileAbsolute(neff_file_path, .{}); + defer neff_file.close(); + const stat = try neff_file.stat(); + const neff_buf = try arena.allocator().alloc(u8, @intCast(stat.size)); + _ = try neff_file.readAll(neff_buf); + break :blk neff_buf; + } }; + + const parameters_len = entry.program_shape.?.parameters.items.len; + var new_entry_instructions: std.ArrayListUnmanaged(hlo_proto.HloInstructionProto) = try .initCapacity( + arena.allocator(), + parameters_len + 1, // params + fused_root + ); + + fused_root.operand_ids.clearRetainingCapacity(); + try fused_root.operand_ids.ensureTotalCapacityPrecise(arena.allocator(), parameters_len); + for (entry.instructions.items) |inst| { + if (std.mem.eql(u8, inst.opcode.getSlice(), "parameter")) { + fused_root.operand_ids.appendAssumeCapacity(inst.id); + new_entry_instructions.appendAssumeCapacity(inst); + } + } + + try fused_root.frontend_attributes.?.map.append(arena.allocator(), .{ + .key = .{ .Const = "valid_inputs" }, + .value = .{ .Owned = blk: { + const valid_inputs_value = try arena.allocator().alloc(u8, parameters_len * 2 - 1); + for (valid_inputs_value, 0..) |*char, i| { + char.* = if (i % 2 == 0) '1' else ','; + } + break :blk valid_inputs_value; + } }, + }); + + try new_entry_instructions.append(arena.allocator(), fused_root); + entry.instructions = new_entry_instructions; + + return hlo_module.encode(allocator); +} + +fn neuronx_cc_(self: ?*c.PyObject, args_: [*c]*c.PyObject, nargs_: c.Py_ssize_t) !?*c.PyObject { + _ = self; + + var arena = std.heap.ArenaAllocator.init(std.heap.c_allocator); + defer arena.deinit(); + + const args = args_[0..@intCast(nargs_)]; + + const code = PyBytes_AsStringAndSize(args[0]); + const platform_version = PyBytes_AsStringAndSize(args[2]); + + const target = std.StaticStringMap([]const u8).initComptime(.{ + .{ "1.0", "inf1" }, + .{ "2.0", "trn1" }, + .{ "3.0", "trn2" }, + }).get(platform_version) orelse { + log.err("Unknown platform version: {s}\n", .{platform_version}); + return error.UnknownPlatformVersion; + }; + + var tmp_dir_buf: [std.fs.max_path_bytes]u8 = undefined; + const tmp_dir = try makeTempDir(&tmp_dir_buf, "zml-neuronxcc-"); + defer std.fs.deleteTreeAbsolute(tmp_dir) catch |err| { + log.err("Error deleting temporary directory {s}: {}\n", .{ tmp_dir, err }); + }; + + const code_file = try std.fs.path.join(arena.allocator(), &.{ tmp_dir, "file.code" }); + { + const file = try std.fs.cwd().createFile(code_file, .{ .truncate = true }); + defer file.close(); + try file.writeAll(code); + } + + const neff_file = try std.fs.path.join(arena.allocator(), &.{ tmp_dir, "file.neff" }); + + var child = std.process.Child.init(&.{ + "neuronx-cc", + "compile", + "--framework=XLA", + "--target", + target, + "--verbose=info", + "--enable-internal-neff-wrapper", + "--output", + neff_file, + "--optlevel=1", + // generic is the default, but it fails on transformers, force it + "--model-type=transformer", + // disable it, we do our own + "--auto-cast=none", + "--enable-fast-loading-neuron-binaries", + code_file, + }, arena.allocator()); + child.stdin_behavior = .Ignore; + child.stdout_behavior = .Inherit; + child.stderr_behavior = .Inherit; + child.cwd = tmp_dir; + _ = try child.spawnAndWait(); + + std.debug.print(">>>> {s}\n", .{tmp_dir}); + + const neff_hlo_bytes = wrapNeffAsCustomCall(arena.allocator(), code, neff_file) catch |err| { + log.err("Error wrapping NEFF as custom call: {}\n", .{err}); + return err; + }; + + return c.PyTuple_Pack( + 2, + c.PyLong_FromLongLong(0), + c.PyBytes_FromStringAndSize(@ptrCast(neff_hlo_bytes), @intCast(neff_hlo_bytes.len)), + ); +} + +fn neuronx_cc(self: ?*c.PyObject, args_: [*c]*c.PyObject, nargs_: c.Py_ssize_t) callconv(.c) ?*c.PyObject { + return neuronx_cc_(self, args_, nargs_) catch |err| { + log.err("Error in neuronx_cc: {}\n", .{err}); + + const none = c.Py_None(); + c.Py_IncRef(none); + const tuple = c.PyTuple_New(2) orelse { + c.Py_DecRef(none); + return null; + }; + _ = c.PyTuple_SetItem(tuple, 0, c.PyLong_FromLongLong(400).?); + _ = c.PyTuple_SetItem(tuple, 1, none); + return tuple; + }; +} + +pub export fn PyInit_libneuronxla() callconv(.c) ?*c.PyObject { + return c.PyModuleDef_Init(&module_def); +} diff --git a/runtimes/neuron/libpjrt_neuron.BUILD.bazel b/runtimes/neuron/libpjrt_neuron.BUILD.bazel index 5df16d6b..9998a87d 100644 --- a/runtimes/neuron/libpjrt_neuron.BUILD.bazel +++ b/runtimes/neuron/libpjrt_neuron.BUILD.bazel @@ -5,18 +5,6 @@ load("@bazel_skylib//rules:select_file.bzl", "select_file") load("@rules_cc//cc:cc_shared_library.bzl", "cc_shared_library") load("@zml//bazel:runfiles.bzl", "runfiles_to_default") -cc_shared_library( - name = "zmlxneuron_so_", - shared_lib_name = "libzmlxneuron.so.0", - deps = ["@zml//runtimes/neuron:zmlxneuron_lib"], -) - -copy_file( - name = "zmlxneuron_so", - src = ":zmlxneuron_so_", - out = "lib/libzmlxneuron.so.0", -) - runfiles_to_default( name = "libneuronxla_files", deps = ["@neuron_py_deps//libneuronxla:pkg"], @@ -30,11 +18,11 @@ select_file( patchelf( name = "libneuronpjrt.patchelf", - add_needed = [ - "libpython3.11.so.1.0", - "libzmlxneuron.so.0", - "libnccom.so.2", - ], + # add_needed = [ + # "libpython3.11.so.1.0", + # # "libzmlxneuron.so.0", + # "libnccom.so.2", + # ], set_rpath = '$ORIGIN', shared_library = ":libneuronpjrt_so", soname = "libneuronpjrt.so", @@ -51,12 +39,23 @@ patchelf( soname = "libpjrt_neuron.so", ) +runfiles_to_default( + name = "neuronx_cc_runfiles", + deps = [ + "@neuron_py_deps//neuronx_cc", + ], +) + copy_to_directory( name = "sandbox", srcs = [ - ":zmlxneuron_so", + # ":zmlxneuron_so", ":libneuronpjrt.patchelf", ":libpjrt_neuron_proxy.patchelf", + ":neuronx_cc_runfiles", + "@rules_python//python:current_py_toolchain", + "@zml//runtimes/neuron:neuronx-cc", + "@zml//runtimes/neuron:neuronxla", "@aws-neuronx-runtime-lib//:libnrt.patchelf", "@aws-neuronx-runtime-lib//:libncfw.patchelf", "@aws-neuronx-collectives//:libnccom", @@ -64,6 +63,8 @@ copy_to_directory( "@libgomp1", ], replace_prefixes = { + "runtimes/neuron/neuronx-cc": "bin/neuronx-cc", + "runtimes/neuron/libneuronxla.so": "site-packages/libneuronxla.so", "libneuronpjrt.patchelf": "lib", "libpjrt_neuron_proxy.patchelf": "lib", "libnrt.patchelf": "lib", @@ -72,13 +73,20 @@ copy_to_directory( "usr/lib/x86_64-linux-gnu": "lib", "opt/neuron": "lib", }, + # include_srcs_patterns = [ + # "bin/neuronx-cc", + # "lib/libpython3.11*/**", + # "lib/python*/**", + # "site-packages/**", + # ], + allow_overwrites = True, add_directory_to_runfiles = True, include_external_repositories = ["**"], + hardlink = "on", ) cc_library( - name = "libpjrt_neuron", - data = [":sandbox"], + name = "libnrt_headers", deps = [ "@aws-neuronx-runtime-lib//:libnrt_headers", ], @@ -94,3 +102,9 @@ cc_library( ], visibility = ["@zml//runtimes/neuron:__subpackages__"], ) + +cc_library( + name = "libpjrt_neuron", + data = [":sandbox"], + visibility = ["@zml//runtimes/neuron:__subpackages__"], +) diff --git a/runtimes/neuron/libpjrt_neuron.zig b/runtimes/neuron/libpjrt_neuron.zig index 6485ecd0..54ba1764 100644 --- a/runtimes/neuron/libpjrt_neuron.zig +++ b/runtimes/neuron/libpjrt_neuron.zig @@ -1,14 +1,100 @@ const std = @import("std"); -const runfiles = @import("runfiles"); + const bazel_builtin = @import("bazel_builtin"); +const c = @import("c"); +const runfiles = @import("runfiles"); const stdx = @import("stdx"); const log = std.log.scoped(.@"zml/runtimes/neuron"); -pub export fn GetPjrtApi() *anyopaque { +var api_inner: *c.PJRT_Api = undefined; +var api_proxy: c.PJRT_Api = undefined; + +pub export fn zmlxneuron_dlopen(filename: [*c]const u8, flags: c_int) ?*anyopaque { + const replacements: std.StaticStringMap([:0]const u8) = .initComptime(.{ + .{ "libnccom.so", "libnccom.so.2" }, + .{ "libnrt.so", "libnrt.so.1" }, + .{ "libncfw.so", "libncfw.so.2" }, + }); + return std.c.dlopen( + if (filename) |f| replacements.get(std.mem.span(f)) orelse filename else null, + @bitCast(flags), + ); +} + +fn pyErrorOrExit(status: c.PyStatus) void { + if (c.PyStatus_Exception(status) != 0) { + if (c.PyStatus_IsExit(status) != 0) { + std.process.exit(@intCast(status.exitcode)); + } + c.Py_ExitStatusException(status); + } +} + +pub fn toPosixPathW(file_path: []const u8) error{NameTooLong}![std.posix.PATH_MAX - 1:0]c.wchar_t { + if (file_path.len >= std.posix.PATH_MAX) return error.NameTooLong; + + var path_with_null: [std.posix.PATH_MAX - 1:0]c.wchar_t = undefined; + const len = c.mbstowcs(&path_with_null, file_path.ptr, file_path.len); + path_with_null[len] = 0; + return path_with_null; +} + +fn setupPythonEnv(sandbox_path: []const u8) !void { + { + var preconfig: c.PyPreConfig = undefined; + c.PyPreConfig_InitIsolatedConfig(&preconfig); + preconfig.utf8_mode = 1; + pyErrorOrExit(c.Py_PreInitialize(&preconfig)); + } + + var config: c.PyConfig = undefined; + c.PyConfig_InitIsolatedConfig(&config); + defer c.PyConfig_Clear(&config); + + { + var bufZ: [std.fs.max_path_bytes:0]u8 = undefined; + pyErrorOrExit(c.PyConfig_SetBytesString(&config, &config.home, try stdx.fs.path.bufJoinZ(&bufZ, &.{ + sandbox_path, + "lib", + "python3.11", + }))); + } + + { + config.module_search_paths_set = 1; + var buf: [std.fs.max_path_bytes]u8 = undefined; + pyErrorOrExit(c.PyWideStringList_Append(&config.module_search_paths, &(try toPosixPathW(try stdx.fs.path.bufJoin(&buf, &.{ + sandbox_path, + "lib", + "python3.11", + }))))); + pyErrorOrExit(c.PyWideStringList_Append(&config.module_search_paths, &(try toPosixPathW(try stdx.fs.path.bufJoin(&buf, &.{ + sandbox_path, + "site-packages", + }))))); + } + + pyErrorOrExit(c.Py_InitializeFromConfig(&config)); + + // release the GIL + _ = c.PyEval_SaveThread(); +} + +fn setupBinPath(allocator: std.mem.Allocator, sandbox_path: []const u8) !void { + var buf: [std.fs.max_path_bytes]u8 = undefined; + const bin_path = try stdx.fs.path.bufJoin(&buf, &.{ sandbox_path, "bin" }); + const old_path = std.posix.getenv("PATH") orelse ""; + const new_path = try std.fmt.allocPrintZ(allocator, "{s}:{s}", .{ bin_path, old_path }); + _ = c.setenv("PATH", new_path, 1); +} + +pub export fn GetPjrtApi() ?*c.PJRT_Api { var arena = std.heap.ArenaAllocator.init(std.heap.c_allocator); defer arena.deinit(); + std.debug.print("ONE\n", .{}); + var r_ = runfiles.Runfiles.create(.{ .allocator = arena.allocator() }) catch |err| { stdx.debug.panic("Unable to find runfiles: {}", .{err}); } orelse stdx.debug.panic("Runfiles not availeabwewefle", .{}); @@ -16,24 +102,71 @@ pub export fn GetPjrtApi() *anyopaque { const source_repo = bazel_builtin.current_repository; const r = r_.withSourceRepo(source_repo); - var path_buf: [std.fs.max_path_bytes]u8 = undefined; - const sandbox_path = r.rlocation("libpjrt_neuron/sandbox", &path_buf) catch |err| { + var sandbox_path_buf: [std.fs.max_path_bytes]u8 = undefined; + const sandbox_path = r.rlocation("libpjrt_neuron/sandbox", &sandbox_path_buf) catch |err| { stdx.debug.panic("Failed to find sandbox path for NEURON runtime: {}", .{err}); } orelse stdx.debug.panic("No NEURON sandbox path found", .{}); - var lib_path_buf: [std.fs.max_path_bytes]u8 = undefined; - const library = stdx.fs.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libneuronpjrt.so" }) catch unreachable; + std.debug.print("TWO\n", .{}); - var lib: std.DynLib = blk: { - const handle = std.c.dlopen(library, .{ .LAZY = true, .GLOBAL = true, .NODELETE = true }) orelse { - stdx.debug.panic("Unable to dlopen plugin: {s}", .{library}); + { + setupBinPath(arena.allocator(), sandbox_path) catch |err| { + log.err("Failed to setup bin path: {}", .{err}); + return null; }; - break :blk .{ .inner = .{ .handle = handle } }; + setupPythonEnv(sandbox_path) catch |err| { + log.err("Failed to setup Python environment: {}", .{err}); + return null; + }; + } + + api_inner = blk: { + const GetPjrtApi_inner = GetPjrtApi_blk: { + var lib: std.DynLib = .{ + .inner = .{ + .handle = handle_blk: { + var lib_path_buf: [std.fs.max_path_bytes]u8 = undefined; + const library = stdx.fs.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libneuronpjrt.so" }) catch |err| { + log.err("Failed to find NEURON PJRT library: {any}", .{err}); + return null; + }; + break :handle_blk std.c.dlopen(library, .{ .LAZY = true, .GLOBAL = false, .NODELETE = true }) orelse { + log.err("Unable to dlopen plugin: {?s}", .{std.mem.span(std.c.dlerror())}); + return null; + }; + }, + }, + }; + + break :GetPjrtApi_blk lib.lookup(*const fn () callconv(.c) *c.PJRT_Api, "GetPjrtApi") orelse { + log.err("Unable to find symbol GetPjrtApi in plugin: {?s}", .{std.mem.span(std.c.dlerror())}); + return null; + }; + }; + + break :blk GetPjrtApi_inner(); }; - const sym = lib.lookup(*const fn () callconv(.C) *anyopaque, "GetPjrtApi") orelse { - stdx.debug.panic("Unable to find symbol GetPjrtApi in plugin: {s}", .{library}); + std.debug.print("THREE\n", .{}); + + api_proxy = blk: { + var tmp: c.PJRT_Api = .{}; + const api_struct_size = @min(@sizeOf(c.PJRT_Api), api_inner.struct_size); + @memcpy( + std.mem.asBytes(&tmp)[0..api_struct_size], + std.mem.asBytes(api_inner)[0..api_struct_size], + ); + break :blk tmp; }; - return sym(); + // Setup the API proxy functions + api_proxy.PJRT_Plugin_Attributes = &struct { + fn call(args: [*c]c.PJRT_Plugin_Attributes_Args) callconv(.c) ?*c.PJRT_Error { + var new_args = args.*; + new_args.struct_size = 0x18; + return api_inner.PJRT_Plugin_Attributes.?(&new_args); + } + }.call; + + return &api_proxy; } diff --git a/runtimes/neuron/neuron.zig b/runtimes/neuron/neuron.zig index fc973424..084a4256 100644 --- a/runtimes/neuron/neuron.zig +++ b/runtimes/neuron/neuron.zig @@ -1,5 +1,5 @@ -const builtin = @import("builtin"); const std = @import("std"); +const builtin = @import("builtin"); const asynk = @import("async"); const bazel_builtin = @import("bazel_builtin"); @@ -32,93 +32,6 @@ fn isRunningOnEC2() !bool { return std.mem.eql(u8, &buf, AmazonEC2); } -fn toWchar(str: []const u8, out: [:0]c.wchar_t) [:0]c.wchar_t { - const len = c.mbstowcs(out.ptr, str.ptr, str.len); - out[len] = 0; - return out[0..len :0]; -} - -fn pyErrorOrExit(status: c.PyStatus) void { - if (c.PyStatus_Exception(status) != 0) { - if (c.PyStatus_IsExit(status) != 0) { - std.process.exit(@intCast(status.exitcode)); - } - c.Py_ExitStatusException(status); - } -} - -fn initialize(allocator: std.mem.Allocator, r_: *runfiles.Runfiles) !void { - { - var preconfig: c.PyPreConfig = undefined; - c.PyPreConfig_InitIsolatedConfig(&preconfig); - preconfig.utf8_mode = 1; - pyErrorOrExit(c.Py_PreInitialize(&preconfig)); - } - - var config: c.PyConfig = undefined; - c.PyConfig_InitIsolatedConfig(&config); - defer c.PyConfig_Clear(&config); - - const r = r_.withSourceRepo(bazel_builtin.current_repository); - - var buf: [std.fs.max_path_bytes]u8 = undefined; - var wbuf: [std.fs.max_path_bytes:0]c.wchar_t = undefined; - - { - const path = (try r.rlocation(libneuronxla_pyenv.home, &buf)).?; - const wpath = toWchar(std.fs.path.dirname(path).?, &wbuf); - pyErrorOrExit(c.PyConfig_SetString(&config, &config.home, wpath.ptr)); - } - - { - config.module_search_paths_set = 1; - for (libneuronxla_pyenv.modules) |module| { - const path = (try r.rlocation(module, &buf)).?; - const wline = toWchar(std.fs.path.dirname(path).?, &wbuf); - pyErrorOrExit(c.PyWideStringList_Append(&config.module_search_paths, wline.ptr)); - } - } - - { - const neuronx_cc = (try r.rlocation("zml/runtimes/neuron/neuronx-cc/neuronx-cc", &buf)).?; - const neuronx_cc_path = std.fs.path.dirname(neuronx_cc).?; - const path = std.posix.getenv("PATH") orelse ""; - const new_path = try std.fmt.allocPrintZ(allocator, "{s}:{s}", .{ neuronx_cc_path, path }); - _ = c.setenv("PATH", new_path.ptr, 1); - } - - pyErrorOrExit(c.Py_InitializeFromConfig(&config)); - - // release the GIL - _ = c.PyEval_SaveThread(); -} - -fn comptimeStrJoin(comptime separator: [:0]const u8, comptime slices: []const [:0]const u8) [:0]const u8 { - comptime var ret = slices[0]; - inline for (slices[1..]) |slice| { - ret = ret ++ separator ++ slice; - } - return ret; -} - -pub fn setNeuronCCFlags() void { - // See neuronxcc reference: - // https://awsdocs-neuron.readthedocs-hosted.com/en/latest/compiler/neuronx-cc/api-reference-guide/neuron-compiler-cli-reference-guide.html#neuron-compiler-cli-reference-guide - _ = c.setenv("NEURON_CC_FLAGS", comptimeStrJoin(" ", &.{ - // 30% faster, no visible speed difference on llama - "--optlevel=1", - // generic is the default, but it fails on transformers, force it - "--model-type=transformer", - // disable it, we do our own - "--auto-cast=none", - "--enable-fast-loading-neuron-binaries", - }), 1); - - // Enable stochastic rounding - // https://awsdocs-neuron.readthedocs-hosted.com/en/latest/general/arch/neuron-features/rounding-modes.html - _ = c.setenv("NEURON_RT_STOCHASTIC_ROUNDING_EN", "1", 1); -} - pub fn load() !*const pjrt.Api { if (comptime !isEnabled()) { return error.Unavailable; @@ -149,8 +62,8 @@ pub fn load() !*const pjrt.Api { return error.FileNotFound; }; - setNeuronCCFlags(); - try initialize(arena.allocator(), &r_); + // setNeuronCCFlags(); + // try initialize(arena.allocator(), &r_); return blk: { var lib_path_buf: [std.fs.max_path_bytes]u8 = undefined; diff --git a/runtimes/neuron/neuronx-cc.zig b/runtimes/neuron/neuronx-cc.zig new file mode 100644 index 00000000..c9f19b34 --- /dev/null +++ b/runtimes/neuron/neuronx-cc.zig @@ -0,0 +1,88 @@ +const std = @import("std"); +const builtin = @import("builtin"); + +const asynk = @import("async"); +const bazel_builtin = @import("bazel_builtin"); +const c = @import("c"); +const runfiles = @import("runfiles"); +const stdx = @import("stdx"); + +fn pyErrorOrExit(status: c.PyStatus) void { + if (c.PyStatus_Exception(status) != 0) { + if (c.PyStatus_IsExit(status) != 0) { + std.process.exit(@intCast(status.exitcode)); + } + c.Py_ExitStatusException(status); + } +} + +pub fn toPosixPathW(file_path: []const u8) error{NameTooLong}![std.posix.PATH_MAX - 1:0]c.wchar_t { + if (file_path.len >= std.posix.PATH_MAX) return error.NameTooLong; + + var path_with_null: [std.posix.PATH_MAX - 1:0]c.wchar_t = undefined; + const len = c.mbstowcs(&path_with_null, file_path.ptr, file_path.len); + path_with_null[len] = 0; + return path_with_null; +} + +pub fn main() !void { + var arena = std.heap.ArenaAllocator.init(std.heap.c_allocator); + defer arena.deinit(); + + { + var preconfig: c.PyPreConfig = undefined; + c.PyPreConfig_InitIsolatedConfig(&preconfig); + preconfig.utf8_mode = 1; + pyErrorOrExit(c.Py_PreInitialize(&preconfig)); + } + + var config: c.PyConfig = undefined; + c.PyConfig_InitIsolatedConfig(&config); + _ = c.PyConfig_SetBytesArgv(&config, @intCast(std.os.argv.len), @ptrCast(std.os.argv)); + defer c.PyConfig_Clear(&config); + + var self_exe_dir_buf: [std.fs.max_path_bytes]u8 = undefined; + const self_exe_dir = try std.fs.selfExeDirPath(&self_exe_dir_buf); + + { + var bufZ: [std.fs.max_path_bytes:0]u8 = undefined; + pyErrorOrExit(c.PyConfig_SetBytesString(&config, &config.home, try stdx.fs.path.bufJoinZ(&bufZ, &.{ + self_exe_dir, + "..", + "lib", + "python3.11", + }))); + } + + { + var buf: [std.fs.max_path_bytes]u8 = undefined; + config.module_search_paths_set = 1; + pyErrorOrExit(c.PyWideStringList_Append(&config.module_search_paths, &(try toPosixPathW(try stdx.fs.path.bufJoin(&buf, &.{ + self_exe_dir, + "..", + "lib", + "python3.11", + }))))); + pyErrorOrExit(c.PyWideStringList_Append(&config.module_search_paths, &(try toPosixPathW(try stdx.fs.path.bufJoin(&buf, &.{ + self_exe_dir, + "..", + "site-packages", + }))))); + } + + std.debug.print(">>>> CORENTIN\n", .{}); + + pyErrorOrExit(c.Py_InitializeFromConfig(&config)); + defer c.Py_Finalize(); + + const neuronxcc_main = blk: { + const module = c.PyImport_ImportModule("neuronxcc.driver.CommandDriver"); + std.debug.print(">>> MODULE: {any}\n", .{module}); + defer c.Py_DecRef(module); + break :blk c.PyObject_GetAttrString(module, "main"); + }; + defer c.Py_DecRef(neuronxcc_main); + + const result = c.PyObject_CallNoArgs(neuronxcc_main); + defer c.Py_DecRef(result); +} diff --git a/runtimes/neuron/libpjrt_neuron.h b/runtimes/neuron/nrt.h similarity index 100% rename from runtimes/neuron/libpjrt_neuron.h rename to runtimes/neuron/nrt.h