diff --git a/MODULE.bazel b/MODULE.bazel index 489e6233..29ff309f 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -24,7 +24,7 @@ bazel_dep(name = "rules_zig", version = "20250519.0-233b207") bazel_dep(name = "sentencepiece", version = "20240618.0-d7ace0a") bazel_dep(name = "toolchains_protoc", version = "0.3.7") bazel_dep(name = "with_cfg.bzl", version = "0.9.1") -bazel_dep(name = "xla", version = "20250317.2-71c67e2") +bazel_dep(name = "xla", version = "20250409.0-7167d71") bazel_dep(name = "zig-protobuf", version = "20250318.0-930153e") bazel_dep(name = "zig-yaml", version = "20240903.0-83d5fdf") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index ee52d7cf..e534d743 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -83,7 +83,7 @@ "https://bazel-registry.zml.ai/modules/re2/2024-07-02.bcr.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/re2/2024-07-02/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_android/0.1.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_apple/3.17.0/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/rules_apple/3.20.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.0.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.0.10/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.0.13/MODULE.bazel": "not found", @@ -150,7 +150,7 @@ "https://bazel-registry.zml.ai/modules/rules_shell/0.2.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_shell/0.3.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_swift/1.16.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_swift/2.1.1/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/rules_swift/2.4.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_uv/0.65.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.5.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.5.3/MODULE.bazel": "not found", @@ -269,8 +269,8 @@ "https://bcr.bazel.build/modules/re2/2024-07-02/MODULE.bazel": "0eadc4395959969297cbcf31a249ff457f2f1d456228c67719480205aa306daa", "https://bcr.bazel.build/modules/rules_android/0.1.1/MODULE.bazel": "48809ab0091b07ad0182defb787c4c5328bd3a278938415c00a7b69b50c4d3a8", "https://bcr.bazel.build/modules/rules_android/0.1.1/source.json": "e6986b41626ee10bdc864937ffb6d6bf275bb5b9c65120e6137d56e6331f089e", - "https://bcr.bazel.build/modules/rules_apple/3.17.0/MODULE.bazel": "39ee3d15edbf09f536a52c42b4297fe1f08a239e28964555f578f301159033f3", - "https://bcr.bazel.build/modules/rules_apple/3.17.0/source.json": "57af375068c7292d88062b2a3c2cae546f6e262f653f03388b5d73207f7482e3", + "https://bcr.bazel.build/modules/rules_apple/3.20.1/MODULE.bazel": "d9049cabd0becdf8d26377927dda874e2febc803401269814776f232fde91b7c", + "https://bcr.bazel.build/modules/rules_apple/3.20.1/source.json": "7870121042c518ca6109fa16fb070b4dd6762f42aefdd9ddcc259194ca6627cd", "https://bcr.bazel.build/modules/rules_cc/0.0.1/MODULE.bazel": "cb2aa0747f84c6c3a78dad4e2049c154f08ab9d166b1273835a8174940365647", "https://bcr.bazel.build/modules/rules_cc/0.0.10/MODULE.bazel": "ec1705118f7eaedd6e118508d3d26deba2a4e76476ada7e0e3965211be012002", "https://bcr.bazel.build/modules/rules_cc/0.0.13/MODULE.bazel": "0e8529ed7b323dad0775ff924d2ae5af7640b23553dfcd4d34344c7e7a867191", @@ -350,8 +350,8 @@ "https://bcr.bazel.build/modules/rules_shell/0.3.0/MODULE.bazel": "de4402cd12f4cc8fda2354fce179fdb068c0b9ca1ec2d2b17b3e21b24c1a937b", "https://bcr.bazel.build/modules/rules_shell/0.3.0/source.json": "c55ed591aa5009401ddf80ded9762ac32c358d2517ee7820be981e2de9756cf3", "https://bcr.bazel.build/modules/rules_swift/1.16.0/MODULE.bazel": "4a09f199545a60d09895e8281362b1ff3bb08bbde69c6fc87aff5b92fcc916ca", - "https://bcr.bazel.build/modules/rules_swift/2.1.1/MODULE.bazel": "494900a80f944fc7aa61500c2073d9729dff0b764f0e89b824eb746959bc1046", - "https://bcr.bazel.build/modules/rules_swift/2.1.1/source.json": "40fc69dfaac64deddbb75bd99cdac55f4427d9ca0afbe408576a65428427a186", + "https://bcr.bazel.build/modules/rules_swift/2.4.0/MODULE.bazel": "1639617eb1ede28d774d967a738b4a68b0accb40650beadb57c21846beab5efd", + "https://bcr.bazel.build/modules/rules_swift/2.4.0/source.json": "a6577f57f9febbdc015a01f2a8f3487422032f134d6c61d18ed8e8ca3b9acc7c", "https://bcr.bazel.build/modules/rules_uv/0.65.0/MODULE.bazel": "a700505a80ad0ad65f062c9335458d137f504b98c5937b338807bb5cbe305e73", "https://bcr.bazel.build/modules/rules_uv/0.65.0/source.json": "315237fb5f44ada2d537aabb2f89c6ba2296569ed63fc7340ca66d65f5730081", "https://bcr.bazel.build/modules/stardoc/0.5.1/MODULE.bazel": "1a05d92974d0c122f5ccf09291442580317cdd859f07a8655f1db9a60374f9f8", @@ -380,7 +380,7 @@ "@@apple_support+//crosstool:setup.bzl%apple_cc_configure_extension": { "general": { "bzlTransitiveDigest": "JJV/C5bhYKZip41t/GGZvqIMmmrcwr0Ydybkn97uXxQ=", - "usagesDigest": "wD+r6kULzwkrPbJnhFpRl2hIsJZIwSSaeEUj3z6v0i4=", + "usagesDigest": "ai1Q9Iw9PMxCbfOYikwNfCbFYAEzeR5Pirndq3g0LFo=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, @@ -2066,6 +2066,160 @@ ] ] } + }, + "@@rules_swift+//swift:extensions.bzl%non_module_deps": { + "general": { + "bzlTransitiveDigest": "3iOPDsadEKzPUpfznfK5UYw/yuGpZzq7zrjrB/oEo6g=", + "usagesDigest": "9w18ec4dj90mX/JqpR2tBU2YVc1GU9yNxi/d7q6lLVA=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "com_github_apple_swift_protobuf": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-protobuf/archive/1.20.2.tar.gz" + ], + "sha256": "3fb50bd4d293337f202d917b6ada22f9548a0a0aed9d9a4d791e6fbd8a246ebb", + "strip_prefix": "swift-protobuf-1.20.2/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_protobuf/BUILD.overlay" + } + }, + "com_github_grpc_grpc_swift": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/grpc/grpc-swift/archive/1.16.0.tar.gz" + ], + "sha256": "58b60431d0064969f9679411264b82e40a217ae6bd34e17096d92cc4e47556a5", + "strip_prefix": "grpc-swift-1.16.0/", + "build_file": "@@rules_swift+//third_party:com_github_grpc_grpc_swift/BUILD.overlay" + } + }, + "com_github_apple_swift_docc_symbolkit": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-docc-symbolkit/archive/refs/tags/swift-5.10-RELEASE.tar.gz" + ], + "sha256": "de1d4b6940468ddb53b89df7aa1a81323b9712775b0e33e8254fa0f6f7469a97", + "strip_prefix": "swift-docc-symbolkit-swift-5.10-RELEASE", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_docc_symbolkit/BUILD.overlay" + } + }, + "com_github_apple_swift_nio": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio/archive/2.42.0.tar.gz" + ], + "sha256": "e3304bc3fb53aea74a3e54bd005ede11f6dc357117d9b1db642d03aea87194a0", + "strip_prefix": "swift-nio-2.42.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio/BUILD.overlay" + } + }, + "com_github_apple_swift_nio_http2": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio-http2/archive/1.26.0.tar.gz" + ], + "sha256": "f0edfc9d6a7be1d587e5b403f2d04264bdfae59aac1d74f7d974a9022c6d2b25", + "strip_prefix": "swift-nio-http2-1.26.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_http2/BUILD.overlay" + } + }, + "com_github_apple_swift_nio_transport_services": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio-transport-services/archive/1.15.0.tar.gz" + ], + "sha256": "f3498dafa633751a52b9b7f741f7ac30c42bcbeb3b9edca6d447e0da8e693262", + "strip_prefix": "swift-nio-transport-services-1.15.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_transport_services/BUILD.overlay" + } + }, + "com_github_apple_swift_nio_extras": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio-extras/archive/1.4.0.tar.gz" + ], + "sha256": "4684b52951d9d9937bb3e8ccd6b5daedd777021ef2519ea2f18c4c922843b52b", + "strip_prefix": "swift-nio-extras-1.4.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_extras/BUILD.overlay" + } + }, + "com_github_apple_swift_log": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-log/archive/1.4.4.tar.gz" + ], + "sha256": "48fe66426c784c0c20031f15dc17faf9f4c9037c192bfac2f643f65cb2321ba0", + "strip_prefix": "swift-log-1.4.4/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_log/BUILD.overlay" + } + }, + "com_github_apple_swift_nio_ssl": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio-ssl/archive/2.23.0.tar.gz" + ], + "sha256": "4787c63f61dd04d99e498adc3d1a628193387e41efddf8de19b8db04544d016d", + "strip_prefix": "swift-nio-ssl-2.23.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_ssl/BUILD.overlay" + } + }, + "com_github_apple_swift_collections": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-collections/archive/1.0.4.tar.gz" + ], + "sha256": "d9e4c8a91c60fb9c92a04caccbb10ded42f4cb47b26a212bc6b39cc390a4b096", + "strip_prefix": "swift-collections-1.0.4/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_collections/BUILD.overlay" + } + }, + "com_github_apple_swift_atomics": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-atomics/archive/1.1.0.tar.gz" + ], + "sha256": "1bee7f469f7e8dc49f11cfa4da07182fbc79eab000ec2c17bfdce468c5d276fb", + "strip_prefix": "swift-atomics-1.1.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_atomics/BUILD.overlay" + } + }, + "build_bazel_rules_swift_index_import": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "build_file": "@@rules_swift+//third_party:build_bazel_rules_swift_index_import/BUILD.overlay", + "canonical_id": "index-import-5.8", + "urls": [ + "https://github.com/MobileNativeFoundation/index-import/releases/download/5.8.0.1/index-import.tar.gz" + ], + "sha256": "28c1ffa39d99e74ed70623899b207b41f79214c498c603915aef55972a851a15" + } + }, + "build_bazel_rules_swift_local_config": { + "repoRuleId": "@@rules_swift+//swift/internal:swift_autoconfiguration.bzl%swift_autoconfiguration", + "attributes": {} + } + }, + "recordedRepoMappingEntries": [ + [ + "rules_swift+", + "bazel_tools", + "bazel_tools" + ] + ] + } } } } diff --git a/async/async.zig b/async/async.zig index 5cee9947..91bb06f4 100644 --- a/async/async.zig +++ b/async/async.zig @@ -1,14 +1,15 @@ const std = @import("std"); + const stdx = @import("stdx"); const xev = @import("xev").Dynamic; +const XevThreadPool = @import("xev").ThreadPool; + +const aio = @import("asyncio.zig"); +const channel_mod = @import("channel.zig"); const coro = @import("coro.zig"); const executor = @import("executor.zig"); -const channel_mod = @import("channel.zig"); -const aio = @import("asyncio.zig"); const stack = @import("stack.zig"); -const XevThreadPool = @import("xev").ThreadPool; - pub const Condition = struct { inner: executor.Condition, @@ -67,6 +68,64 @@ pub fn asyncc(comptime func: anytype, args: anytype) !FrameEx(func, @TypeOf(args }; } +pub fn Queue(comptime func: anytype) type { + const Args = stdx.meta.FnArgs(func); + const Ret = stdx.meta.FnResult(func); + + stdx.debug.assertComptime(@sizeOf(Args) <= @sizeOf(stack.Stack.Data), "Queue expects arguments to fit inside {} bytes, got {}", .{ @sizeOf(stack.Stack.Data), @sizeOf(Args) }); + stdx.debug.assertComptime(Ret == void or @typeInfo(Ret) == .error_union, "Queue expects a function returning void or an error, got {}", .{func}); + const Task = struct { + const T = @This(); + + _task: XevThreadPool.Task = .{ .callback = &T.run }, + active: bool = false, + args: Args, + res: ?Ret = null, + + pub fn run(task_: *XevThreadPool.Task) void { + const task: *T = @alignCast(@fieldParentPtr("_task", task_)); + task.res = @call(.auto, func, task.args); + @atomicStore(bool, &task.active, false, .monotonic); + } + }; + + return struct { + data: []Task = &.{}, + next: usize = 0, + + const Q = @This(); + + pub fn initCapacity(allocator: std.mem.Allocator, capacity: usize) !Q { + const data = try allocator.alloc(Task, capacity); + for (data) |*d| d.active = false; + return .{ .data = data }; + } + + pub fn push(q: *Q, args: stdx.meta.FnArgs(func)) void { + const next = q.next; + const task: *Task = &q.data[next]; + while (@atomicLoad(bool, &task.active, .acquire)) {} + + task.* = .{ .active = true, .args = args }; + AsyncThread.current.thread_pool.schedule(XevThreadPool.Batch.from(&task._task)); + q.next = (next + 1) % q.data.len; + } + + /// Busy wait. + pub fn waitSync(q: *Q) Ret { + for (q.data) |*task| { + while (@atomicLoad(bool, &task.active, .acquire)) {} + if (Ret != void) if (task.res) |res| try res; + } + } + + pub fn deinit(q: *Q, allocator: std.mem.Allocator) void { + allocator.free(q.data); + q.data = &.{}; + } + }; +} + pub fn callBlocking(comptime func: anytype, args: anytype) stdx.meta.FnSignature(func, @TypeOf(args)).ReturnT { const Signature = stdx.meta.FnSignature(func, @TypeOf(args)); diff --git a/examples/.vscode/settings.json b/examples/.vscode/settings.json index 3a9f5009..82050af6 100644 --- a/examples/.vscode/settings.json +++ b/examples/.vscode/settings.json @@ -23,7 +23,7 @@ "C_Cpp.clang_format_style": "Microsoft", "zig.formattingProvider": "zls", "zig.path": "x", - "zig.zls.path": "/${workspaceFolder}/tools/zls.sh", + "zig.zls.path": "${workspaceFolder}/tools/zls.sh", "[zig]": { "editor.suggest.insertMode": "replace", "editor.stickyScroll.defaultModel": "foldingProviderModel", diff --git a/examples/MODULE.bazel.lock b/examples/MODULE.bazel.lock index 1008eb86..612efec4 100644 --- a/examples/MODULE.bazel.lock +++ b/examples/MODULE.bazel.lock @@ -83,7 +83,7 @@ "https://bazel-registry.zml.ai/modules/re2/2024-07-02.bcr.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/re2/2024-07-02/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_android/0.1.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_apple/3.17.0/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/rules_apple/3.20.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.0.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.0.10/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.0.13/MODULE.bazel": "not found", @@ -151,7 +151,7 @@ "https://bazel-registry.zml.ai/modules/rules_shell/0.2.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_shell/0.3.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_swift/1.16.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_swift/2.1.1/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/rules_swift/2.4.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_uv/0.65.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.5.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.5.3/MODULE.bazel": "not found", @@ -271,8 +271,8 @@ "https://bcr.bazel.build/modules/re2/2024-07-02/MODULE.bazel": "0eadc4395959969297cbcf31a249ff457f2f1d456228c67719480205aa306daa", "https://bcr.bazel.build/modules/rules_android/0.1.1/MODULE.bazel": "48809ab0091b07ad0182defb787c4c5328bd3a278938415c00a7b69b50c4d3a8", "https://bcr.bazel.build/modules/rules_android/0.1.1/source.json": "e6986b41626ee10bdc864937ffb6d6bf275bb5b9c65120e6137d56e6331f089e", - "https://bcr.bazel.build/modules/rules_apple/3.17.0/MODULE.bazel": "39ee3d15edbf09f536a52c42b4297fe1f08a239e28964555f578f301159033f3", - "https://bcr.bazel.build/modules/rules_apple/3.17.0/source.json": "57af375068c7292d88062b2a3c2cae546f6e262f653f03388b5d73207f7482e3", + "https://bcr.bazel.build/modules/rules_apple/3.20.1/MODULE.bazel": "d9049cabd0becdf8d26377927dda874e2febc803401269814776f232fde91b7c", + "https://bcr.bazel.build/modules/rules_apple/3.20.1/source.json": "7870121042c518ca6109fa16fb070b4dd6762f42aefdd9ddcc259194ca6627cd", "https://bcr.bazel.build/modules/rules_cc/0.0.1/MODULE.bazel": "cb2aa0747f84c6c3a78dad4e2049c154f08ab9d166b1273835a8174940365647", "https://bcr.bazel.build/modules/rules_cc/0.0.10/MODULE.bazel": "ec1705118f7eaedd6e118508d3d26deba2a4e76476ada7e0e3965211be012002", "https://bcr.bazel.build/modules/rules_cc/0.0.13/MODULE.bazel": "0e8529ed7b323dad0775ff924d2ae5af7640b23553dfcd4d34344c7e7a867191", @@ -354,8 +354,8 @@ "https://bcr.bazel.build/modules/rules_shell/0.3.0/MODULE.bazel": "de4402cd12f4cc8fda2354fce179fdb068c0b9ca1ec2d2b17b3e21b24c1a937b", "https://bcr.bazel.build/modules/rules_shell/0.3.0/source.json": "c55ed591aa5009401ddf80ded9762ac32c358d2517ee7820be981e2de9756cf3", "https://bcr.bazel.build/modules/rules_swift/1.16.0/MODULE.bazel": "4a09f199545a60d09895e8281362b1ff3bb08bbde69c6fc87aff5b92fcc916ca", - "https://bcr.bazel.build/modules/rules_swift/2.1.1/MODULE.bazel": "494900a80f944fc7aa61500c2073d9729dff0b764f0e89b824eb746959bc1046", - "https://bcr.bazel.build/modules/rules_swift/2.1.1/source.json": "40fc69dfaac64deddbb75bd99cdac55f4427d9ca0afbe408576a65428427a186", + "https://bcr.bazel.build/modules/rules_swift/2.4.0/MODULE.bazel": "1639617eb1ede28d774d967a738b4a68b0accb40650beadb57c21846beab5efd", + "https://bcr.bazel.build/modules/rules_swift/2.4.0/source.json": "a6577f57f9febbdc015a01f2a8f3487422032f134d6c61d18ed8e8ca3b9acc7c", "https://bcr.bazel.build/modules/rules_uv/0.65.0/MODULE.bazel": "a700505a80ad0ad65f062c9335458d137f504b98c5937b338807bb5cbe305e73", "https://bcr.bazel.build/modules/rules_uv/0.65.0/source.json": "315237fb5f44ada2d537aabb2f89c6ba2296569ed63fc7340ca66d65f5730081", "https://bcr.bazel.build/modules/stardoc/0.5.1/MODULE.bazel": "1a05d92974d0c122f5ccf09291442580317cdd859f07a8655f1db9a60374f9f8", @@ -385,7 +385,7 @@ "@@apple_support+//crosstool:setup.bzl%apple_cc_configure_extension": { "general": { "bzlTransitiveDigest": "JJV/C5bhYKZip41t/GGZvqIMmmrcwr0Ydybkn97uXxQ=", - "usagesDigest": "wD+r6kULzwkrPbJnhFpRl2hIsJZIwSSaeEUj3z6v0i4=", + "usagesDigest": "ai1Q9Iw9PMxCbfOYikwNfCbFYAEzeR5Pirndq3g0LFo=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, @@ -2291,6 +2291,160 @@ ] ] } + }, + "@@rules_swift+//swift:extensions.bzl%non_module_deps": { + "general": { + "bzlTransitiveDigest": "3iOPDsadEKzPUpfznfK5UYw/yuGpZzq7zrjrB/oEo6g=", + "usagesDigest": "9w18ec4dj90mX/JqpR2tBU2YVc1GU9yNxi/d7q6lLVA=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "com_github_apple_swift_protobuf": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-protobuf/archive/1.20.2.tar.gz" + ], + "sha256": "3fb50bd4d293337f202d917b6ada22f9548a0a0aed9d9a4d791e6fbd8a246ebb", + "strip_prefix": "swift-protobuf-1.20.2/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_protobuf/BUILD.overlay" + } + }, + "com_github_grpc_grpc_swift": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/grpc/grpc-swift/archive/1.16.0.tar.gz" + ], + "sha256": "58b60431d0064969f9679411264b82e40a217ae6bd34e17096d92cc4e47556a5", + "strip_prefix": "grpc-swift-1.16.0/", + "build_file": "@@rules_swift+//third_party:com_github_grpc_grpc_swift/BUILD.overlay" + } + }, + "com_github_apple_swift_docc_symbolkit": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-docc-symbolkit/archive/refs/tags/swift-5.10-RELEASE.tar.gz" + ], + "sha256": "de1d4b6940468ddb53b89df7aa1a81323b9712775b0e33e8254fa0f6f7469a97", + "strip_prefix": "swift-docc-symbolkit-swift-5.10-RELEASE", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_docc_symbolkit/BUILD.overlay" + } + }, + "com_github_apple_swift_nio": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio/archive/2.42.0.tar.gz" + ], + "sha256": "e3304bc3fb53aea74a3e54bd005ede11f6dc357117d9b1db642d03aea87194a0", + "strip_prefix": "swift-nio-2.42.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio/BUILD.overlay" + } + }, + "com_github_apple_swift_nio_http2": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio-http2/archive/1.26.0.tar.gz" + ], + "sha256": "f0edfc9d6a7be1d587e5b403f2d04264bdfae59aac1d74f7d974a9022c6d2b25", + "strip_prefix": "swift-nio-http2-1.26.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_http2/BUILD.overlay" + } + }, + "com_github_apple_swift_nio_transport_services": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio-transport-services/archive/1.15.0.tar.gz" + ], + "sha256": "f3498dafa633751a52b9b7f741f7ac30c42bcbeb3b9edca6d447e0da8e693262", + "strip_prefix": "swift-nio-transport-services-1.15.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_transport_services/BUILD.overlay" + } + }, + "com_github_apple_swift_nio_extras": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio-extras/archive/1.4.0.tar.gz" + ], + "sha256": "4684b52951d9d9937bb3e8ccd6b5daedd777021ef2519ea2f18c4c922843b52b", + "strip_prefix": "swift-nio-extras-1.4.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_extras/BUILD.overlay" + } + }, + "com_github_apple_swift_log": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-log/archive/1.4.4.tar.gz" + ], + "sha256": "48fe66426c784c0c20031f15dc17faf9f4c9037c192bfac2f643f65cb2321ba0", + "strip_prefix": "swift-log-1.4.4/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_log/BUILD.overlay" + } + }, + "com_github_apple_swift_nio_ssl": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio-ssl/archive/2.23.0.tar.gz" + ], + "sha256": "4787c63f61dd04d99e498adc3d1a628193387e41efddf8de19b8db04544d016d", + "strip_prefix": "swift-nio-ssl-2.23.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_ssl/BUILD.overlay" + } + }, + "com_github_apple_swift_collections": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-collections/archive/1.0.4.tar.gz" + ], + "sha256": "d9e4c8a91c60fb9c92a04caccbb10ded42f4cb47b26a212bc6b39cc390a4b096", + "strip_prefix": "swift-collections-1.0.4/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_collections/BUILD.overlay" + } + }, + "com_github_apple_swift_atomics": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-atomics/archive/1.1.0.tar.gz" + ], + "sha256": "1bee7f469f7e8dc49f11cfa4da07182fbc79eab000ec2c17bfdce468c5d276fb", + "strip_prefix": "swift-atomics-1.1.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_atomics/BUILD.overlay" + } + }, + "build_bazel_rules_swift_index_import": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "build_file": "@@rules_swift+//third_party:build_bazel_rules_swift_index_import/BUILD.overlay", + "canonical_id": "index-import-5.8", + "urls": [ + "https://github.com/MobileNativeFoundation/index-import/releases/download/5.8.0.1/index-import.tar.gz" + ], + "sha256": "28c1ffa39d99e74ed70623899b207b41f79214c498c603915aef55972a851a15" + } + }, + "build_bazel_rules_swift_local_config": { + "repoRuleId": "@@rules_swift+//swift/internal:swift_autoconfiguration.bzl%swift_autoconfiguration", + "attributes": {} + } + }, + "recordedRepoMappingEntries": [ + [ + "rules_swift+", + "bazel_tools", + "bazel_tools" + ] + ] + } } } } diff --git a/examples/custom_call/BUILD b/examples/custom_call/BUILD new file mode 100644 index 00000000..bc605442 --- /dev/null +++ b/examples/custom_call/BUILD @@ -0,0 +1,10 @@ +load("@zml//bazel:zig.bzl", "zig_cc_binary") + +zig_cc_binary( + name = "custom_call", + main = "main.zig", + deps = [ + "@zml//async", + "@zml//zml", + ], +) diff --git a/examples/custom_call/main.zig b/examples/custom_call/main.zig new file mode 100644 index 00000000..db0c4b7a --- /dev/null +++ b/examples/custom_call/main.zig @@ -0,0 +1,173 @@ +const std = @import("std"); +const zml = @import("zml"); +const asynk = @import("async"); + +const cuda = zml.context.cuda; +const ffi = zml.ffi; + +pub const std_options: std.Options = .{ + .log_level = .info, + .logFn = asynk.logFn(std.log.defaultLog), +}; + +const log = std.log.scoped(.@"examples/custom_call"); + +pub const AddOpHostFuncCtx = struct { + a: zml.Buffer, + b: zml.Buffer, + result: zml.Buffer, +}; + +fn add_op_host_func(data: *const anyopaque) callconv(.c) void { + const ctx = @as(*const AddOpHostFuncCtx, @alignCast(@ptrCast(data))); + + const a_ptr = std.mem.bytesAsValue(f32, ctx.a.asPinnedHostBuffer().bytes()); + const b_ptr = std.mem.bytesAsValue(f32, ctx.b.asPinnedHostBuffer().bytes()); + const result_ptr = std.mem.bytesAsValue(f32, ctx.result.asPinnedHostBuffer().mutBytes()); + result_ptr.* = a_ptr.* + b_ptr.*; +} + +pub const AddOp = struct { + pub var type_id: i64 = undefined; + const Self = @This(); + + a: zml.Buffer, + b: zml.Buffer, + result: zml.Buffer, + + host_func_ctx: AddOpHostFuncCtx = undefined, + + platform: zml.Platform, + results: []*const ffi.FFIBuffer = undefined, + stream: *ffi.FFIStream = undefined, + + pub fn init( + platform: zml.Platform, + host_buffer: zml.HostBuffer, + ) !AddOp { + const a = try zml.Buffer.fromEx(platform, host_buffer, .{ .memory = .host_pinned }); + _ = try a.awaitt(); + + const b = try zml.Buffer.fromEx(platform, host_buffer, .{ .memory = .host_pinned }); + _ = try b.awaitt(); + + const result = try zml.Buffer.fromEx(platform, host_buffer, .{ .memory = .host_pinned }); + _ = try result.awaitt(); + + return .{ + .a = a, + .b = b, + .result = result, + .platform = platform, + }; + } + + pub fn call(self: *Self, a: *const ffi.FFIBuffer, b: *const ffi.FFIBuffer) !void { + cuda.memcpyToHostAsync(self.a.asPinnedHostBuffer().mutBytes(), a.data, self.stream); + cuda.memcpyToHostAsync(self.b.asPinnedHostBuffer().mutBytes(), b.data, self.stream); + + self.host_func_ctx = .{ + .a = self.a, + .b = self.b, + .result = self.result, + }; + + _ = cuda.cuLaunchHostFunc(self.stream, @ptrCast(&add_op_host_func), @ptrCast(&self.host_func_ctx)); + + cuda.memcpyToDeviceAsync(self.results[0].data, self.result.asPinnedHostBuffer().bytes(), self.stream); + } +}; + +/// Model definition +const Layer = struct { + pub fn forward(_: Layer, a: zml.Tensor, b: zml.Tensor) zml.Tensor { + const result = zml.custom_call(AddOp, .{ a, b }, &[_]zml.Shape{a.shape()}); + return result[0]; + } +}; + +pub fn main() !void { + try asynk.AsyncThread.main(std.heap.c_allocator, asyncMain); +} + +pub fn asyncMain() !void { + // Short lived allocations + var gpa = std.heap.GeneralPurposeAllocator(.{}){}; + defer _ = gpa.deinit(); + const allocator = gpa.allocator(); + + // Arena allocator for BufferStore etc. + var arena_state = std.heap.ArenaAllocator.init(allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + + var context = try zml.Context.init(); + defer context.deinit(); + + const platform = context.autoPlatform(.{}); + context.printAvailablePlatforms(platform); + + // const shape = zml.Shape.init(.{ 40, 128 }, .bf16); + const shape = zml.Shape.init(.{}, .f32); + + // We manually produce a BufferStore. You would not normally do that. + // A BufferStore is usually created by loading model data from a file. + const buffers: zml.aio.BufferStore.Buffers = .{}; + + // the actual BufferStore + const buffer_store: zml.aio.BufferStore = .{ + .arena = arena_state, + .buffers = buffers, + }; + + // A clone of our model, consisting of shapes. We only need shapes for compiling. + // We use the BufferStore to infer the shapes. + const model_shapes = try zml.aio.populateModel(Layer, allocator, buffer_store); + + // Start compiling. This uses the inferred shapes from the BufferStore. + // The shape of the input tensor, we have to pass in manually. + var compilation = try asynk.asyncc(zml.compileModel, .{ allocator, Layer.forward, model_shapes, .{ shape, shape }, platform }); + + // Produce a bufferized weights struct from the fake BufferStore. + // This is like the inferred shapes, but with actual values. + // We will need to send those to the computation device later. + var model_weights = try zml.aio.loadModelBuffers(Layer, model_shapes, buffer_store, arena, platform); + defer zml.aio.unloadBuffers(&model_weights); // for good practice + + // Wait for compilation to finish + const compiled = try compilation.awaitt(); + + // pass the model weights to the compiled module to create an executable module + var executable = compiled.prepare(model_weights).withExecutionContext(); + defer executable.deinit(); + + try executable.register(AddOp); + + const result_hb = try zml.HostBuffer.empty(allocator, shape); + defer result_hb.deinit(allocator); + + var add_op: AddOp = try .init(platform, result_hb); + try executable.bind(AddOp, &add_op); + + var input_a = [1]f32{1.0}; + var input_buffer_a = try zml.Buffer.from(platform, zml.HostBuffer.fromSlice(shape, &input_a)); + _ = try input_buffer_a.awaitt(); + defer input_buffer_a.deinit(); + + var input_b = [1]f32{1.0}; + var input_buffer_b = try zml.Buffer.from(platform, zml.HostBuffer.fromSlice(shape, &input_b)); + _ = try input_buffer_b.awaitt(); + defer input_buffer_b.deinit(); + + var result: zml.Buffer = executable.call(.{ input_buffer_a, input_buffer_b }); + defer result.deinit(); + + // fetch the result to CPU memory + var cpu_result = try result.toHostAlloc(arena); + _ = try cpu_result.awaitt(); + + log.warn( + "\nThe result of {d} + {d} = {d}\n", + .{ &input_a, &input_b, cpu_result.items(f32) }, + ); +} diff --git a/examples/mnist/mnist.zig b/examples/mnist/mnist.zig index c49300a9..96221ff1 100644 --- a/examples/mnist/mnist.zig +++ b/examples/mnist/mnist.zig @@ -105,7 +105,7 @@ pub fn asyncMain() !void { defer input.deinit(); printDigit(sample); - var result: zml.Buffer = mnist.call(.{input}); + var result: zml.Buffer = mnist.call(.{try input.awaitt()}); defer result.deinit(); log.info( diff --git a/mlir/dialects/stablehlo.zig b/mlir/dialects/stablehlo.zig index 63283219..d47da487 100644 --- a/mlir/dialects/stablehlo.zig +++ b/mlir/dialects/stablehlo.zig @@ -731,6 +731,14 @@ pub fn convolution( }); } +pub fn optimization_barrier(ctx: mlir.Context, inputs: []const mlir.Value, res_types: []const mlir.Type, location: mlir.Location) mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.optimization_barrier", .{ + .operands = inputs, + .results = res_types, + .location = location, + }); +} + pub const CustomCallOpts = struct { pub const ApiVersion = enum(i32) { original = 1, diff --git a/pjrt/ffi.zig b/pjrt/ffi.zig index 39e7c248..b7f81217 100644 --- a/pjrt/ffi.zig +++ b/pjrt/ffi.zig @@ -91,22 +91,20 @@ fn TransmuteMixin(comptime T: type, comptime InnerT: type) type { pub const Api = opaque { pub const inner = TransmuteMixin(Api, c.XLA_FFI_Api).to; - pub fn getStream(self: *const Api, context: ?*ExecutionContext) ApiError!*anyopaque { + pub fn stream(self: *const Api, context: ?*ExecutionContext) *Stream { var ret = pjrtStruct(c.XLA_FFI_Stream_Get_Args{ .ctx = if (context) |ctx| ctx.inner() else null, }); const result = self.inner().XLA_FFI_Stream_Get.?(&ret); - if (result) |ffi_error| { const err = Error.fromInner(ffi_error); defer err.destroy(self); log.err("[Api.getStream] {s}", .{err.getMessage(self)}); - // TODO(Corentin): Retrieve error code from Error when implemented in XLA. - return error.Unknown; + @panic("failed to get stream"); } - return ret.stream.?; + return @ptrCast(ret.stream.?); } pub fn allocateDeviceMemory(self: *const Api, context: ?*ExecutionContext, size: usize, alignment: usize) ApiError!*anyopaque { @@ -165,54 +163,92 @@ pub const ExecutionStage = enum(c.XLA_FFI_ExecutionStage) { pub const ExecutionContext = opaque { pub const inner = TransmuteMixin(ExecutionContext, c.XLA_FFI_ExecutionContext).to; - // pub fn attach(self: *ExecutionContext, api: *const Api, value: anytype) ApiError!void { - // // register type id ==> typeid - // const typename_ = "zml." ++ @typeName(@TypeOf(value)); + pub fn Context(comptime T: type) type { + return struct { + pub fn get(self: *const ExecutionContext, api: *const Api) ApiError!*T { + const type_id: TypeId = .{ .type_id = T.type_id }; + var ret = pjrtStruct(c.XLA_FFI_ExecutionContext_Get_Args{ + .ctx = @constCast(self.inner()), + .type_id = @constCast(&type_id.toCStruct()), + }); + const result = api.inner().XLA_FFI_ExecutionContext_Get.?(&ret); - // var ret = pjrtStruct(c.XLA_FFI_ExecutionContext_Register_Args{ - // .ctx = self.inner(), - // .handler = @ptrCast(@alignCast(handler)), - // }); - // const result = api.inner().XLA_FFI_ExecutionContext_Register.?(&ret); + if (result) |ffi_error| { + const err = Error.fromInner(ffi_error); + defer err.destroy(api); + log.err("[ExecutionContext.get] {s}", .{err.getMessage(api)}); - // var ret = pjrtStruct(c.XLA_FFI_ExecutionContext_Register_Args{ - // .ctx = self.inner(), - // .handler = @ptrCast(@alignCast(handler)), - // }); - // const result = api.inner().XLA_FFI_ExecutionContext_Register.?(&ret); + // TODO(Corentin): Retrieve error code from Error when implemented in XLA. + return error.Unknown; + } - // if (result) |ffi_error| { - // const err = Error.fromInner(ffi_error); - // defer err.destroy(api); - // log.err("[ExecutionContext.register] {s}", .{err.getMessage(api)}); + if (ret.data == null) return error.NotFound; + return @ptrCast(@alignCast(ret.data.?)); + } + }; + } - // // TODO(Corentin): Retrieve error code from Error when implemented in XLA. - // return error.Unknown; - // } - // } - - pub fn get(self: *ExecutionContext, api: *const Api, type_id: *TypeId) ApiError!*anyopaque { - var ret = pjrtStruct(c.XLA_FFI_ExecutionContext_Get_Args{ - .ctx = self.inner(), - .type_id = @ptrCast(@alignCast(type_id)), + pub fn getDeviceOrdinal(self: *const ExecutionContext, api: *const Api) ApiError!i32 { + var ret = pjrtStruct(c.XLA_FFI_DeviceOrdinal_Get_Args{ + .ctx = @constCast(self.inner()), }); - const result = api.inner().XLA_FFI_ExecutionContext_Get.?(&ret); + const result = api.inner().XLA_FFI_DeviceOrdinal_Get.?(&ret); if (result) |ffi_error| { const err = Error.fromInner(ffi_error); defer err.destroy(api); - log.err("[ExecutionContext.get] {s}", .{err.getMessage(api)}); + log.err("[ExecutionContext.getDeviceOrdinal] {s}", .{err.getMessage(api)}); // TODO(Corentin): Retrieve error code from Error when implemented in XLA. return error.Unknown; } - return ret.data.?; + return ret.device_ordinal; } - // TODO getDeviceOrdinal() + pub fn scheduleTask(self: *const ExecutionContext, api: *const Api, task: *const Task, data: *anyopaque) ApiError!void { + var ret = pjrtStruct(c.XLA_FFI_ThreadPool_Schedule_Args{ + .ctx = @constCast(self.inner()), + .task = @ptrCast(@alignCast(task)), + .data = @ptrCast(@alignCast(data)), + }); + + const result = api.inner().XLA_FFI_ThreadPool_Schedule.?(&ret); + + if (result) |ffi_error| { + const err = Error.fromInner(ffi_error); + defer err.destroy(api); + std.debug.print("error: {any} \n", .{err}); + log.err("[ExecutionContext.get] {s}", .{err.getMessage(api)}); + + // TODO(Corentin): Retrieve error code from Error when implemented in XLA. + return error.Unknown; + } + } + + fn getTypeId(type_name: []const u8) TypeId { + const id: i64 = @bitCast(std.hash.Fnv1a_64.hash(type_name)); + + return .{ + .type_id = id, + }; + } }; +const TypeId = extern struct { + type_id: i64, + + pub fn toCStruct(self: TypeId) c.XLA_FFI_TypeId { + return c.XLA_FFI_TypeId{ + .type_id = self.type_id, + }; + } +}; + +const Task = fn (*anyopaque) void; + +pub const Stream = opaque {}; + const ByteSpan = extern struct { ptr: [*]const u8, len: usize, @@ -222,17 +258,13 @@ const ByteSpan = extern struct { } }; -pub const TypeId = extern struct { - type_id: i64, -}; - pub const DataType = enum(c.XLA_FFI_DataType) { invalid = c.XLA_FFI_DataType_INVALID, pred = c.XLA_FFI_DataType_PRED, - s8 = c.XLA_FFI_DataType_S8, - s16 = c.XLA_FFI_DataType_S16, - s32 = c.XLA_FFI_DataType_S32, - s64 = c.XLA_FFI_DataType_S64, + i8 = c.XLA_FFI_DataType_S8, + i16 = c.XLA_FFI_DataType_S16, + i32 = c.XLA_FFI_DataType_S32, + i64 = c.XLA_FFI_DataType_S64, u8 = c.XLA_FFI_DataType_U8, u16 = c.XLA_FFI_DataType_U16, u32 = c.XLA_FFI_DataType_U32, @@ -253,11 +285,19 @@ pub const DataType = enum(c.XLA_FFI_DataType) { f8e4m3fnuz = c.XLA_FFI_DataType_F8E4M3FNUZ, }; +pub const DevicePtr = enum(usize) { + _, + + pub fn asPtr(ptr: DevicePtr) [*]u8 { + return @ptrFromInt(@intFromEnum(ptr)); + } +}; + pub const Buffer = extern struct { struct_size: usize, extension_start: ?*c.XLA_FFI_Extension_Base, dtype: DataType, - data: [*]u8, + data: DevicePtr, rank: u64, _dims: [*]const i64, @@ -274,7 +314,7 @@ pub const Buffer = extern struct { _ = fmt; _ = options; - try writer.print("FfiBuffer({d}, .{s})@0x{x}", .{ buffer.dims(), @tagName(buffer.dtype), @intFromPtr(buffer.data) }); + try writer.print("FfiBuffer({d}, .{s})@0x{x}", .{ buffer.dims(), @tagName(buffer.dtype), @intFromPtr(buffer.data.asPtr()) }); } }; @@ -289,9 +329,8 @@ pub const Args = extern struct { buffer = c.XLA_FFI_ArgType_BUFFER, }; - pub fn get(self: Args, i: usize) *const Buffer { - std.debug.assert(self.types[0..self.len][i] == .buffer); - return self.ptr[0..self.len][i]; + pub fn buffers(self: Args) []*const Buffer { + return self.ptr[0..self.len]; } }; @@ -306,9 +345,8 @@ pub const Rets = extern struct { buffer = c.XLA_FFI_RetType_BUFFER, }; - pub fn get(self: Rets, i: usize) *const Buffer { - std.debug.assert(self.types[0..self.len][i] == .buffer); - return self.ptr[0..self.len][i]; + pub fn buffers(self: Rets) []*const Buffer { + return self.ptr[0..self.len]; } }; @@ -346,8 +384,18 @@ pub const Attrs = extern struct { dtype: DataType, len: usize, data: [*]const u8, + + pub fn slice(self: Array, T: type) []const T { + const ptr: [*]const T = @alignCast(@ptrCast(self.data)); + return ptr[0..self.len]; + } }; + pub fn slice(self: Array, T: type) []const T { + const ptr: [*]const T = @alignCast(@ptrCast(self.data)); + return ptr[0..self.len]; + } + pub fn getByIndex(self: Attrs, comptime attr_type: AttrType, index: usize) ?*const @FieldType(Attr, @tagName(attr_type)) { const attr = self.ptr[0..self.len][index]; const actual_type = self.types[index]; @@ -370,8 +418,8 @@ pub const Attrs = extern struct { pub const CallFrame = extern struct { struct_size: usize, extension_start: ?*ExtensionBase, - api: ?*const Api, - ctx: ?*const ExecutionContext, + api: *const Api, + ctx: ?*ExecutionContext, stage: ExecutionStage, args: Args, results: Rets, @@ -438,7 +486,7 @@ pub const Error = opaque { pub const inner = TransmuteMixin(Error, c.XLA_FFI_Error).to; pub const fromInner = TransmuteMixin(Error, c.XLA_FFI_Error).from; - pub fn create(api: *const Api, error_code: ErrorCode, message: [:0]const u8) *Error { + pub fn create(api: *const Api, error_code: ErrorCode, message: []const u8) *Error { var ret = pjrtStruct(c.XLA_FFI_Error_Create_Args{ .message = message.ptr, .errc = @intFromEnum(error_code), diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index df1a7d43..f6e642b4 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -31,7 +31,7 @@ fn pjrtStructSize(comptime T: type) usize { return @field(c, typedef_name ++ "_STRUCT_SIZE"); } -inline fn pjrtStruct(v: anytype) @TypeOf(v) { +pub inline fn pjrtStruct(v: anytype) @TypeOf(v) { var ret = v; ret.struct_size = pjrtStructSize(@TypeOf(v)); return ret; @@ -160,9 +160,14 @@ pub const Api = struct { return state.str; } - pub fn customCallRegistry(api: *const Api) ?CustomCallRegistry { + pub fn createExecuteContext(api: *const Api) ApiError!*ExecuteContext { + const ret = try api.call(.PJRT_ExecuteContext_Create, .{}); + return @ptrCast(ret.context.?); + } + + pub fn ffi(api: *const Api) ?FFI { if (api.lookupExtension(c.PJRT_FFI_Extension, c.PJRT_Extension_Type_FFI)) |ext| { - return .{ .inner = ext.register_handler.? }; + return .{ .inner = ext }; } return null; } @@ -410,15 +415,26 @@ pub const Client = opaque { element_type: BufferType, layout: MemoryLayout, device: *const Device, + memory_kind: ?Memory.Kind, on_delete_callback: *const fn (device_buffer_ptr: ?*anyopaque, ctx: ?*anyopaque) callconv(.C) void = &struct { fn call(_: ?*anyopaque, _: ?*anyopaque) callconv(.C) void {} }.call, on_delete_callback_arg: ?*anyopaque = null, - stream: ?isize = null, + stream: ?*const anyopaque = null, }; pub fn createViewOfDeviceBuffer(self: *const Client, api: *const Api, args: CreateViewOfDeviceBufferArgs) ApiError!*Buffer { const layout = args.layout.toCStruct(); + var memory: ?*const Memory = null; + + if (args.memory_kind) |kind| { + for (try args.device.addressableMemories(api)) |m| { + if (m.kind(api) == kind) { + memory = m; + } + } + } + const ret = try api.call(.PJRT_Client_CreateViewOfDeviceBuffer, .{ .client = self.inner(), .device_buffer_ptr = @ptrCast(@constCast(args.data)), @@ -427,9 +443,10 @@ pub const Client = opaque { .element_type = @intFromEnum(args.element_type), .layout = @ptrCast(@constCast(&layout)), .device = @ptrCast(@constCast(args.device)), + .memory = if (memory) |m| @ptrCast(@constCast(m)) else null, .on_delete_callback = args.on_delete_callback, .on_delete_callback_arg = args.on_delete_callback_arg, - .stream = if (args.stream) |stream| stream else 0, + .stream = @bitCast(@intFromPtr(args.stream)), }); return @ptrCast(ret.buffer.?); } @@ -444,20 +461,21 @@ pub const Client = opaque { return &.{}; } - pub fn dmaMap(self: *const Client, api: *const Api, data: []const u8) ApiError!*Buffer { - const ret = try api.call(.PJRT_Client_DMA_Map, .{ + pub fn dmaMap(self: *const Client, api: *const Api, data: []const u8) ApiError!void { + const args: Api.CallFnArgType(.PJRT_Client_DmaMap) = .{ .client = self.inner(), .data = @ptrCast(@constCast(data.ptr)), .size = @intCast(data.len), - }); - return @ptrCast(ret.buffer.?); + }; + std.debug.print("DMA map {any} \n", .{args}); + _ = try api.call(.PJRT_Client_DmaMap, args); } - pub fn dmaUnmap(self: *const Client, api: *const Api, data: []const u8) void { - _ = api.call(.PJRT_Client_DMA_Unmap, .{ + pub fn dmaUnmap(self: *const Client, api: *const Api, data: []const u8) ApiError!void { + _ = try api.call(.PJRT_Client_DmaUnmap, .{ .client = self.inner(), .data = @ptrCast(@constCast(data.ptr)), - }) catch unreachable; + }); } pub const CreateBuffersForAsyncHostToDeviceArgs = struct { @@ -564,6 +582,14 @@ pub const SerializeResult = struct { } }; +pub const ExecuteContext = opaque { + pub fn deinit(self: *ExecuteContext, api: *const Api) void { + _ = api.call(.PJRT_ExecuteContext_Destroy, .{ + .context = @ptrCast(self), + }) catch {}; + } +}; + pub const Executable = opaque { const inner = InnerMixin(c.PJRT_Executable).inner; @@ -630,6 +656,7 @@ pub const LoadedExecutable = opaque { results: []const [*]*Buffer, events: []?*Event, non_donatable_input_indices: []const i64 = &.{}, + context: ?*ExecuteContext, }; pub fn execute(self: *const LoadedExecutable, api: *const Api, args: ExecuteArgs) ApiError!void { var options = pjrtStruct(c.PJRT_ExecuteOptions{ @@ -640,6 +667,7 @@ pub const LoadedExecutable = opaque { .launch_id = 0, .non_donatable_input_indices = @ptrCast(args.non_donatable_input_indices.ptr), .num_non_donatable_input_indices = args.non_donatable_input_indices.len, + .context = @ptrCast(args.context), }); _ = try api.call(.PJRT_LoadedExecutable_Execute, .{ .executable = self.inner(), @@ -653,7 +681,7 @@ pub const LoadedExecutable = opaque { }); } - pub fn getExecutable(self: *LoadedExecutable, api: *const Api) ApiError!*Executable { + pub fn getExecutable(self: *const LoadedExecutable, api: *const Api) ApiError!*Executable { const ret = try api.call(.PJRT_LoadedExecutable_GetExecutable, .{ .loaded_executable = self.inner(), }); @@ -818,7 +846,7 @@ pub const Buffer = opaque { return ret.on_device_size_in_bytes; } - pub fn copyToDevice(self: *const Buffer, api: *const Api, device: Device) ApiError!Buffer { + pub fn copyToDevice(self: *const Buffer, api: *const Api, device: Device) ApiError!*Buffer { const ret = try api.call(.PJRT_Buffer_CopyToDevice, .{ .buffer = self.inner(), .dst_device = device.inner, @@ -850,7 +878,7 @@ pub const Buffer = opaque { return @ptrCast(ret.event); } - pub fn copyToMemory(self: *const Buffer, api: *const Api, dst_memory: *const Memory) ApiError!?*Buffer { + pub fn copyToMemory(self: *const Buffer, api: *const Api, dst_memory: *const Memory) ApiError!*Buffer { const ret = try api.call(.PJRT_Buffer_CopyToMemory, .{ .buffer = self.inner(), .dst_memory = @ptrCast(@constCast(dst_memory)), @@ -932,8 +960,8 @@ pub const Memory = opaque { pub fn kind(self: *const Memory, api: *const Api) Kind { const ret = api.call(.PJRT_Memory_Kind, .{ .memory = self.inner() }) catch unreachable; - const kind_ = ret.kind orelse unreachable[0..ret.kind_size]; - return std.meta.stringToEnum(Kind, kind_) orelse unreachable; + const kind_ = ret.kind orelse unreachable; + return std.meta.stringToEnum(Kind, kind_[0..ret.kind_size]) orelse unreachable; } pub fn kindId(self: *const Memory, api: *const Api) u32 { @@ -1044,21 +1072,6 @@ pub const AsyncHostToDeviceTransferManager = opaque { } }; -pub const ExecutionContext = opaque { - const inner = InnerMixin(c.PJRT_ExecutionContext).inner; - - pub fn init(api: *const Api) ApiError!*ExecutionContext { - const ret = try api.call(.PJRT_ExecutionContext_Create, .{}); - return @ptrCast(ret.context.?); - } - - pub fn deinit(self: *ExecutionContext, api: *const Api) void { - _ = api.call(.PJRT_ExecutionContext_Destroy, .{ - .context = self.inner(), - }) catch unreachable; - } -}; - pub const NamedValue = extern struct { comptime { std.debug.assert(@sizeOf(NamedValue) == @sizeOf(c.PJRT_NamedValue)); @@ -1164,17 +1177,81 @@ pub const NamedValue = extern struct { } }; +pub const FFI = extern struct { + inner: *const c.PJRT_FFI, + + pub const UserData = extern struct { + type_id: i64, + user_data: *anyopaque, + + fn toCStruct(self: UserData) c.PJRT_FFI_UserData { + return .{ + .type_id = self.type_id, + .data = self.user_data, + }; + } + }; + + pub fn customCallRegistry(self: *const FFI) ?CustomCallRegistry { + return .{ .inner = self.inner.register_handler.? }; + } + + pub fn registerTypeId(self: *const FFI, api: *const Api, user_type: anytype) ApiError!void { + const type_name = @typeName(user_type); + var ret = pjrtStruct(c.PJRT_FFI_TypeID_Register_Args{ + .type_name = type_name.ptr, + .type_name_size = type_name.len, + .type_id = undefined, + }); + const result = self.inner.type_id_register.?(&ret); + if (result) |pjrt_c_error| { + const pjrt_error: *Error = @ptrCast(pjrt_c_error); + return pjrt_error.getCode(api).toApiError(); + } + + user_type.type_id = ret.type_id; + } + + pub fn addUserData(self: *const FFI, api: *const Api, context: *ExecuteContext, user_data: UserData) ApiError!void { + var ret = pjrtStruct(c.PJRT_FFI_UserData_Add_Args{ + .context = @ptrCast(context), + .user_data = user_data.toCStruct(), + }); + const result = self.inner.user_data_add.?(&ret); + if (result) |pjrt_c_error| { + const pjrt_error: *Error = @ptrCast(pjrt_c_error); + log.err("[addUserData] {s}", .{pjrt_error.getMessage(api)}); + return pjrt_error.getCode(api).toApiError(); + } + } + + pub fn getTypeId(type_name: []const u8) i64 { + // Use FNV-1a 64-bit hash algorithm + return @bitCast(std.hash.Fnv1a_64.hash(type_name)); + } +}; + +pub const RegisterHandlerTraits = enum(c.PJRT_FFI_Handler_TraitsBits) { + command_buffer_compatible = 0, + _, +}; + // todo : support all missing handlers available in GPU plugin extension: handler_instantiate, handler_prepare, handler_initialize // introduced by https://github.com/openxla/xla/commit/ef85a7bcc308313492ebc50295a8a08b4e51b8f5 pub const CustomCallRegistry = extern struct { inner: *const c.PJRT_FFI_Register_Handler, + pub const RegisterFfiOptions = struct { + traits: RegisterHandlerTraits = @enumFromInt(1), + }; + pub fn registerFfi( self: *const CustomCallRegistry, api: *const Api, target_name: []const u8, platform_name: []const u8, func: *const ffi.Handler, + options: RegisterFfiOptions, ) ApiError!void { var ret = pjrtStruct(c.PJRT_FFI_Register_Handler_Args{ .api_version = 1, @@ -1183,6 +1260,7 @@ pub const CustomCallRegistry = extern struct { .handler = @ptrCast(@constCast(func)), .platform_name = platform_name.ptr, .platform_name_size = platform_name.len, + .traits = @intFromEnum(options.traits), }); const result = self.inner(&ret); if (result) |pjrt_c_error| { diff --git a/runtimes/cpu/cpu.bzl b/runtimes/cpu/cpu.bzl index d3613802..9e0b0374 100644 --- a/runtimes/cpu/cpu.bzl +++ b/runtimes/cpu/cpu.bzl @@ -25,22 +25,22 @@ def _cpu_pjrt_plugin_impl(mctx): http_archive( name = "libpjrt_cpu_linux_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_LINUX, - sha256 = "1cda1325095c12bd0019838d28ee92d811ac478d22ed3c08020d5a0cd2d9f34a", - url = "https://github.com/zml/pjrt-artifacts/releases/download/v7.0.0/pjrt-cpu_linux-amd64.tar.gz", + sha256 = "a799573bc238bfbfa18c415002280902e4547f7e12fb09a9a7e73e8b35b58b9b", + url = "https://github.com/zml/pjrt-artifacts/releases/download/v8.0.0/pjrt-cpu_linux-amd64.tar.gz", ) http_archive( name = "libpjrt_cpu_darwin_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, - sha256 = "35af82d9e5c70d16ac15f4c18024a2dd5ed2faefc89940eafe3d5350d2cbd9e7", - url = "https://github.com/zml/pjrt-artifacts/releases/download/v7.0.0/pjrt-cpu_darwin-amd64.tar.gz", + sha256 = "31f6fcc1a5341ea7cd6eaf5c441189e5f6399a37dfafb4a45699edacf44c742c", + url = "https://github.com/zml/pjrt-artifacts/releases/download/v8.0.0/pjrt-cpu_darwin-amd64.tar.gz", ) http_archive( name = "libpjrt_cpu_darwin_arm64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, - sha256 = "da4deaf850d715997614768b2fc0283595ee8181133ab3243d65635e3439de69", - url = "https://github.com/zml/pjrt-artifacts/releases/download/v7.0.0/pjrt-cpu_darwin-arm64.tar.gz", + sha256 = "91a18c0fb2c82f28953b5134833415638422925099ee6498987d583f178588b5", + url = "https://github.com/zml/pjrt-artifacts/releases/download/v8.0.0/pjrt-cpu_darwin-arm64.tar.gz", ) return mctx.extension_metadata( diff --git a/runtimes/cuda/BUILD.bazel b/runtimes/cuda/BUILD.bazel index 0913c058..4acd97d0 100644 --- a/runtimes/cuda/BUILD.bazel +++ b/runtimes/cuda/BUILD.bazel @@ -1,6 +1,11 @@ -load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("@rules_cc//cc:defs.bzl", "cc_library") load("@rules_zig//zig:defs.bzl", "zig_library") +exports_files( + ["libpjrt_cuda.so"], + visibility = ["//visibility:public"], +) + cc_library( name = "zmlxcuda_lib", srcs = ["zmlxcuda.c"], diff --git a/runtimes/cuda/cuda.bzl b/runtimes/cuda/cuda.bzl index 6cd9bee8..7eb82d1a 100644 --- a/runtimes/cuda/cuda.bzl +++ b/runtimes/cuda/cuda.bzl @@ -214,8 +214,8 @@ def _cuda_impl(mctx): http_archive( name = "libpjrt_cuda", build_file = "libpjrt_cuda.BUILD.bazel", - url = "https://github.com/zml/pjrt-artifacts/releases/download/v7.0.0/pjrt-cuda_linux-amd64.tar.gz", - sha256 = "64029cd3d68118b166198e3246877ed706ed35eb732b1770a9bf530b5b0a8ab4", + url = "https://github.com/zml/pjrt-artifacts/releases/download/v8.0.0/pjrt-cuda_linux-amd64.tar.gz", + sha256 = "9e2d1741de8531274c4c2ffe77f3a82003591827294e2a2773cc09f33cc2137e", ) return mctx.extension_metadata( diff --git a/runtimes/cuda/libpjrt_cuda.BUILD.bazel b/runtimes/cuda/libpjrt_cuda.BUILD.bazel index 8881a016..b5b30f67 100644 --- a/runtimes/cuda/libpjrt_cuda.BUILD.bazel +++ b/runtimes/cuda/libpjrt_cuda.BUILD.bazel @@ -25,7 +25,8 @@ copy_to_directory( cc_import( name = "libpjrt_cuda", data = [":sandbox"], - shared_library = "libpjrt_cuda.so", + shared_library = "@zml//runtimes/cuda:libpjrt_cuda.so", + # shared_library = "libpjrt_cuda.so", add_needed = ["libzmlxcuda.so.0"], rename_dynamic_symbols = { "dlopen": "zmlxcuda_dlopen", diff --git a/runtimes/rocm/rocm.bzl b/runtimes/rocm/rocm.bzl index c190a9ee..9d41c106 100644 --- a/runtimes/rocm/rocm.bzl +++ b/runtimes/rocm/rocm.bzl @@ -127,8 +127,8 @@ def _rocm_impl(mctx): http_archive( name = "libpjrt_rocm", build_file = "libpjrt_rocm.BUILD.bazel", - url = "https://github.com/zml/pjrt-artifacts/releases/download/v7.0.0/pjrt-rocm_linux-amd64.tar.gz", - sha256 = "13a2ced965a44a0e8ad0d752c8ac5aa99107b17b41bd850967d40b82e102ec50", + url = "https://github.com/zml/pjrt-artifacts/releases/download/v8.0.0/pjrt-rocm_linux-amd64.tar.gz", + sha256 = "4bdb7690f6e7ac0faa1c29299663269781c2b5d33fbe28c1ff4bc33ce9ec01ff", ) return mctx.extension_metadata( diff --git a/stdx/meta.zig b/stdx/meta.zig index 7aa90531..ee8d998d 100644 --- a/stdx/meta.zig +++ b/stdx/meta.zig @@ -164,6 +164,14 @@ pub fn FnResult(comptime func: anytype) type { return @typeInfo(@TypeOf(func)).@"fn".return_type orelse @compileError("anytype is not supported"); } +pub fn FnResultNoError(comptime func: anytype) type { + const ReturnT = @typeInfo(@TypeOf(func)).@"fn".return_type orelse @compileError("anytype is not supported"); + return switch (@typeInfo(ReturnT)) { + .error_union => |info| info.payload, + else => ReturnT, + }; +} + pub fn Head(Tuple: type) type { return switch (@typeInfo(Tuple)) { .@"struct" => |struct_info| { diff --git a/third_party/modules/xla/20250317.1-71c67e2/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch b/third_party/modules/xla/20250317.1-71c67e2/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch index 58e3874d..bff748ec 100644 --- a/third_party/modules/xla/20250317.1-71c67e2/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch +++ b/third_party/modules/xla/20250317.1-71c67e2/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch @@ -31,10 +31,14 @@ diff --git a/xla/pjrt/c/pjrt_c_api_ffi_extension.h b/xla/pjrt/c/pjrt_c_api_ffi_e index a33bd4aa9c..3309194538 100644 --- a/xla/pjrt/c/pjrt_c_api_ffi_extension.h +++ b/xla/pjrt/c/pjrt_c_api_ffi_extension.h -@@ -66,12 +66,28 @@ PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_UserData_Add_Args, user_data); +@@ -66,12 +66,33 @@ PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_UserData_Add_Args, user_data); // Adds a user data to the execute context. typedef PJRT_Error* PJRT_FFI_UserData_Add(PJRT_FFI_UserData_Add_Args* args); ++ typedef enum PJRT_FFI_Handler_TraitsBits { ++ PJRT_FFI_HANDLER_TRAITS_COMMAND_BUFFER_COMPATIBLE = 1u << 0, ++} PJRT_FFI_Handler_TraitsBits; ++ +struct PJRT_FFI_Register_Handler_Args { + size_t struct_size; + const char* target_name; @@ -43,8 +47,9 @@ index a33bd4aa9c..3309194538 100644 + void* handler; + const char* platform_name; + size_t platform_name_size; ++ PJRT_FFI_Handler_TraitsBits traits; +}; -+PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_Register_Handler_Args, handler); ++PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_Register_Handler_Args, traits); + +// Registers an FFI call handler for a specific platform. +typedef PJRT_Error* PJRT_FFI_Register_Handler( diff --git a/third_party/modules/xla/20250409.0-7167d71/MODULE.bazel b/third_party/modules/xla/20250409.0-7167d71/MODULE.bazel new file mode 100644 index 00000000..50cafb9e --- /dev/null +++ b/third_party/modules/xla/20250409.0-7167d71/MODULE.bazel @@ -0,0 +1,37 @@ +module( + name = "xla", + version = "20250409.0-7167d71", + compatibility_level = 1, +) + +bazel_dep(name = "platforms", version = "0.0.11") +bazel_dep(name = "bazel_skylib", version = "1.5.0") +bazel_dep(name = "rules_cc", version = "0.0.17") +bazel_dep(name = "rules_apple", version = "3.20.1", repo_name = "build_bazel_rules_apple") +bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "com_google_absl") +bazel_dep(name = "rules_python", version = "0.29.0") +bazel_dep(name = "rules_proto", version = "6.0.0-rc1") +bazel_dep(name = "rules_java", version = "7.3.2") +bazel_dep(name = "rules_pkg", version = "0.9.1") +bazel_dep(name = "zlib", version = "1.2.13") +bazel_dep(name = "re2", version = "2024-07-02.bcr.1", repo_name = "com_googlesource_code_re2") +bazel_dep(name = "rules_license", version = "0.0.8") + +tsl = use_extension("//:tsl.bzl", "tsl") +use_repo(tsl, "tsl", "python_version_repo") + +xla_workspace = use_extension("//:workspace.bzl", "xla_workspace") +use_repo( + xla_workspace, + "com_github_grpc_grpc", + "com_google_protobuf", + "local_config_cuda", + "local_config_remote_execution", + "local_config_rocm", + "local_config_tensorrt", + "llvm-raw", + "stablehlo", +) + +llvm_configure = use_repo_rule("@llvm-raw//utils/bazel:configure.bzl", "llvm_configure") +llvm_configure(name = "llvm-project") diff --git a/third_party/modules/xla/20250409.0-7167d71/overlay/MODULE.bazel b/third_party/modules/xla/20250409.0-7167d71/overlay/MODULE.bazel new file mode 120000 index 00000000..9b599e3a --- /dev/null +++ b/third_party/modules/xla/20250409.0-7167d71/overlay/MODULE.bazel @@ -0,0 +1 @@ +../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/xla/20250409.0-7167d71/overlay/llvm.bzl b/third_party/modules/xla/20250409.0-7167d71/overlay/llvm.bzl new file mode 100644 index 00000000..57e77361 --- /dev/null +++ b/third_party/modules/xla/20250409.0-7167d71/overlay/llvm.bzl @@ -0,0 +1,13 @@ +load("@llvm-raw//utils/bazel:configure.bzl", _llvm_configure = "llvm_configure") + +def _llvm_configure_impl(mctx): + _llvm_configure(name = "llvm-project") + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = "all", + root_module_direct_dev_deps = [], + ) + +llvm_configure = module_extension( + implementation = _llvm_configure_impl, +) diff --git a/third_party/modules/xla/20250409.0-7167d71/overlay/tsl.bzl b/third_party/modules/xla/20250409.0-7167d71/overlay/tsl.bzl new file mode 100644 index 00000000..dc25c9d6 --- /dev/null +++ b/third_party/modules/xla/20250409.0-7167d71/overlay/tsl.bzl @@ -0,0 +1,19 @@ +load("//third_party:repo.bzl", "tf_vendored") +load("//third_party/py:python_init_repositories.bzl", "python_init_repositories") + +def _tsl_impl(mctx): + python_init_repositories( + requirements = { + "3.11": "//:requirements_lock_3_11.txt", + }, + ) + tf_vendored(name = "tsl", relpath = "third_party/tsl") + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = ["tsl"], + root_module_direct_dev_deps = [], + ) + +tsl = module_extension( + implementation = _tsl_impl, +) diff --git a/third_party/modules/xla/20250409.0-7167d71/overlay/workspace.bzl b/third_party/modules/xla/20250409.0-7167d71/overlay/workspace.bzl new file mode 100644 index 00000000..192e06f8 --- /dev/null +++ b/third_party/modules/xla/20250409.0-7167d71/overlay/workspace.bzl @@ -0,0 +1,60 @@ +load("//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls") +load("//third_party/gpus:cuda_configure.bzl", "cuda_configure") +load("//third_party/gpus:rocm_configure.bzl", "rocm_configure") +load("//third_party/llvm:workspace.bzl", llvm = "repo") +load("//third_party/pybind11_bazel:workspace.bzl", pybind11_bazel = "repo") +load("//third_party/stablehlo:workspace.bzl", stablehlo = "repo") +load("//third_party/tensorrt:tensorrt_configure.bzl", "tensorrt_configure") +load("//third_party/triton:workspace.bzl", triton = "repo") +load("//tools/toolchains/remote:configure.bzl", "remote_execution_configure") + +def _xla_workspace_impl(mctx): + cuda_configure(name = "local_config_cuda") + remote_execution_configure(name = "local_config_remote_execution") + rocm_configure(name = "local_config_rocm") + tensorrt_configure(name = "local_config_tensorrt") + pybind11_bazel() + triton() + llvm("llvm-raw") + stablehlo() + tf_http_archive( + name = "com_github_grpc_grpc", + sha256 = "b956598d8cbe168b5ee717b5dafa56563eb5201a947856a6688bbeac9cac4e1f", + strip_prefix = "grpc-b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd", + system_build_file = "//third_party/systemlibs:grpc.BUILD", + patch_file = [ + "//third_party/grpc:generate_cc_env_fix.patch", + "//third_party/grpc:register_go_toolchain.patch", + ], + system_link_files = { + "//third_party/systemlibs:BUILD.bazel": "bazel/BUILD.bazel", + "//third_party/systemlibs:grpc.BUILD": "src/compiler/BUILD", + "//third_party/systemlibs:grpc.bazel.grpc_deps.bzl": "bazel/grpc_deps.bzl", + "//third_party/systemlibs:grpc.bazel.grpc_extra_deps.bzl": "bazel/grpc_extra_deps.bzl", + "//third_party/systemlibs:grpc.bazel.cc_grpc_library.bzl": "bazel/cc_grpc_library.bzl", + "//third_party/systemlibs:grpc.bazel.generate_cc.bzl": "bazel/generate_cc.bzl", + "//third_party/systemlibs:grpc.bazel.protobuf.bzl": "bazel/protobuf.bzl", + }, + urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd.tar.gz"), + ) + tf_http_archive( + name = "com_google_protobuf", + patch_file = ["//third_party/protobuf:protobuf.patch"], + sha256 = "f66073dee0bc159157b0bd7f502d7d1ee0bc76b3c1eac9836927511bdc4b3fc1", + strip_prefix = "protobuf-3.21.9", + system_build_file = "//third_party/systemlibs:protobuf.BUILD", + system_link_files = { + "//third_party/systemlibs:protobuf.bzl": "protobuf.bzl", + "//third_party/systemlibs:protobuf_deps.bzl": "protobuf_deps.bzl", + }, + urls = tf_mirror_urls("https://github.com/protocolbuffers/protobuf/archive/v3.21.9.zip"), + ) + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = "all", + root_module_direct_dev_deps = [], + ) + +xla_workspace = module_extension( + implementation = _xla_workspace_impl, +) diff --git a/third_party/modules/xla/20250409.0-7167d71/patches/0001-bazel-migration-to-bazel-8.1.1.patch b/third_party/modules/xla/20250409.0-7167d71/patches/0001-bazel-migration-to-bazel-8.1.1.patch new file mode 100644 index 00000000..8924cf4b --- /dev/null +++ b/third_party/modules/xla/20250409.0-7167d71/patches/0001-bazel-migration-to-bazel-8.1.1.patch @@ -0,0 +1,41 @@ +From 6cf475b500521c1b8be06f590fdbc1818f0dc44b Mon Sep 17 00:00:00 2001 +From: Jean-Baptiste Dalido +Date: Mon, 6 Jan 2025 13:33:13 +0100 +Subject: [PATCH] bazel: migration to bazel 8.0.1 + +--- + .bazelversion | 2 +- + third_party/tsl/third_party/gpus/cuda_configure.bzl | 4 ++-- + 2 files changed, 3 insertions(+), 3 deletions(-) + +diff --git a/.bazelversion b/.bazelversion +index f22d756da3..fa5fce04b3 100644 +--- a/.bazelversion ++++ b/.bazelversion +@@ -1 +1 @@ +-7.4.1 ++8.1.1 +\ No newline at end of file +diff --git a/third_party/gpus/cuda_configure.bzl b/third_party/gpus/cuda_configure.bzl +index d62531152d..71d80a5a99 100644 +--- a/third_party/gpus/cuda_configure.bzl ++++ b/third_party/gpus/cuda_configure.bzl +@@ -33,14 +33,14 @@ NB: DEPRECATED! Use `hermetic/cuda_configure` rule instead. + load( + "@bazel_tools//tools/cpp:lib_cc_configure.bzl", + "escape_string", +- "get_env_var", + ) + load( + "@bazel_tools//tools/cpp:windows_cc_configure.bzl", +- "find_msvc_tool", + "find_vc_path", + "setup_vc_env_vars", + ) ++load("@rules_cc//cc/private/toolchain:windows_cc_configure.bzl", "find_msvc_tool") ++load("@rules_cc//cc/private/toolchain:lib_cc_configure.bzl", "get_env_var") + load("//third_party/clang_toolchain:download_clang.bzl", "download_clang") + load( + "//third_party/remote_config:common.bzl", +-- +2.39.3 (Apple Git-146) diff --git a/third_party/modules/xla/20250409.0-7167d71/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch b/third_party/modules/xla/20250409.0-7167d71/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch new file mode 100644 index 00000000..89cee870 --- /dev/null +++ b/third_party/modules/xla/20250409.0-7167d71/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch @@ -0,0 +1,137 @@ +From 490985facbe2fe6462b2006a99514b1c7e8a0775 Mon Sep 17 00:00:00 2001 +From: Hugo Mano +Date: Wed, 5 Feb 2025 19:25:03 +0100 +Subject: [PATCH 1/7] Added FFI handler registration API to the FFI PjRt + +PR: https://github.com/openxla/xla/pull/13420 +--- + xla/pjrt/c/BUILD | 5 ++++ + xla/pjrt/c/pjrt_c_api_ffi_extension.h | 16 +++++++++++++ + xla/pjrt/c/pjrt_c_api_ffi_internal.cc | 33 ++++++++++++++++++++++++++- + 3 files changed, 53 insertions(+), 1 deletion(-) + +diff --git a/xla/pjrt/c/BUILD b/xla/pjrt/c/BUILD +index 8e8a4baa53..551c35b461 100644 +--- a/xla/pjrt/c/BUILD ++++ b/xla/pjrt/c/BUILD +@@ -69,8 +69,13 @@ cc_library( + ":pjrt_c_api_wrapper_impl", + "//xla/ffi:execution_context", + "//xla/ffi:type_id_registry", ++ "//xla/ffi:ffi_api", ++ "//xla/ffi/api:c_api", ++ "//xla/ffi/api:ffi", ++ "//xla/service:custom_call_target_registry", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings:string_view", ++ "@com_google_absl//absl/strings:str_format", + ], + ) + +diff --git a/xla/pjrt/c/pjrt_c_api_ffi_extension.h b/xla/pjrt/c/pjrt_c_api_ffi_extension.h +index d9f6d163d8..3dbd4df089 100644 +--- a/xla/pjrt/c/pjrt_c_api_ffi_extension.h ++++ b/xla/pjrt/c/pjrt_c_api_ffi_extension.h +@@ -69,12 +69,33 @@ PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_UserData_Add_Args, user_data); + // Adds a user data to the execute context. + typedef PJRT_Error* PJRT_FFI_UserData_Add(PJRT_FFI_UserData_Add_Args* args); + ++ typedef enum PJRT_FFI_Handler_TraitsBits { ++ PJRT_FFI_HANDLER_TRAITS_COMMAND_BUFFER_COMPATIBLE = 1u << 0, ++} PJRT_FFI_Handler_TraitsBits; ++ ++struct PJRT_FFI_Register_Handler_Args { ++ size_t struct_size; ++ const char* target_name; ++ size_t target_name_size; ++ int api_version; // 0 for an untyped call, 1 -- for typed ++ void* handler; ++ const char* platform_name; ++ size_t platform_name_size; ++ PJRT_FFI_Handler_TraitsBits traits; ++}; ++PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_Register_Handler_Args, traits); ++ ++// Registers an FFI call handler for a specific platform. ++typedef PJRT_Error* PJRT_FFI_Register_Handler( ++ PJRT_FFI_Register_Handler_Args* args); ++ + typedef struct PJRT_FFI_Extension { + size_t struct_size; + PJRT_Extension_Type type; + PJRT_Extension_Base* next; + PJRT_FFI_TypeID_Register* type_id_register; + PJRT_FFI_UserData_Add* user_data_add; ++ PJRT_FFI_Register_Handler* register_handler; + } PJRT_FFI; + PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_Extension, user_data_add); + +diff --git a/xla/pjrt/c/pjrt_c_api_ffi_internal.cc b/xla/pjrt/c/pjrt_c_api_ffi_internal.cc +index 8898088a23..746861f697 100644 +--- a/xla/pjrt/c/pjrt_c_api_ffi_internal.cc ++++ b/xla/pjrt/c/pjrt_c_api_ffi_internal.cc +@@ -13,16 +13,21 @@ See the License for the specific language governing permissions and + limitations under the License. + ==============================================================================*/ + +-#include "xla/pjrt/c/pjrt_c_api_ffi_internal.h" ++#include + + #include "absl/status/status.h" + #include "absl/strings/string_view.h" ++#include "absl/strings/str_format.h" ++#include "xla/ffi/api/c_api.h" ++#include "xla/ffi/api/ffi.h" + #include "xla/ffi/execution_context.h" + #include "xla/ffi/type_id_registry.h" ++#include "xla/ffi/ffi_api.h" + #include "xla/pjrt/c/pjrt_c_api.h" + #include "xla/pjrt/c/pjrt_c_api_ffi_extension.h" + #include "xla/pjrt/c/pjrt_c_api_helpers.h" + #include "xla/pjrt/c/pjrt_c_api_wrapper_impl.h" ++#include "xla/service/custom_call_target_registry.h" + + namespace pjrt { + +@@ -68,6 +73,31 @@ static PJRT_Error* PJRT_FFI_UserData_Add(PJRT_FFI_UserData_Add_Args* args) { + return nullptr; + } + ++static PJRT_Error* PJRT_FFI_Register_Handler( ++ PJRT_FFI_Register_Handler_Args* args) { ++ PJRT_RETURN_IF_ERROR(ActualStructSizeIsGreaterOrEqual( ++ "PJRT_FFI_Register_Handler_Args", ++ PJRT_FFI_Register_Handler_Args_STRUCT_SIZE, args->struct_size)); ++ std::string target_name(args->target_name, args->target_name_size); ++ std::string platform_name(args->platform_name, args->platform_name_size); ++ switch (args->api_version) { ++ case 0: ++ xla::CustomCallTargetRegistry::Global()->Register( ++ target_name, args->handler, platform_name); ++ return nullptr; ++ case 1: ++ xla::ffi::Ffi::RegisterStaticHandler( ++ xla::ffi::GetXlaFfiApi(), target_name, platform_name, ++ reinterpret_cast(args->handler)); ++ return nullptr; ++ default: ++ return new PJRT_Error{absl::UnimplementedError( ++ absl::StrFormat("API version %d not supported for PJRT GPU plugin. " ++ "Supported versions are 0 and 1.", ++ args->api_version))}; ++ } ++} ++ + PJRT_FFI_Extension CreateFfiExtension(PJRT_Extension_Base* next) { + return { + /*struct_size=*/PJRT_FFI_Extension_STRUCT_SIZE, +@@ -75,6 +105,7 @@ PJRT_FFI_Extension CreateFfiExtension(PJRT_Extension_Base* next) { + /*next=*/next, + /*type_id_register=*/PJRT_FFI_TypeID_Register, + /*user_data_add=*/PJRT_FFI_UserData_Add, ++ /*register_handler=*/PJRT_FFI_Register_Handler, + }; + } + +-- +2.39.5 (Apple Git-154) diff --git a/third_party/modules/xla/20250409.0-7167d71/source.json b/third_party/modules/xla/20250409.0-7167d71/source.json new file mode 100644 index 00000000..9f9bf0a7 --- /dev/null +++ b/third_party/modules/xla/20250409.0-7167d71/source.json @@ -0,0 +1,16 @@ +{ + "strip_prefix": "xla-7167d711e572e1da17d6c4a87ab1f3661c8f2ac0", + "url": "https://github.com/openxla/xla/archive/7167d711e572e1da17d6c4a87ab1f3661c8f2ac0.tar.gz", + "integrity": "sha256-QxwcMAFT4lQLHp+mHh3nhWo9qwbTf9+kiOMWmNuXoVI=", + "overlay": { + "llvm.bzl": "", + "tsl.bzl": "", + "workspace.bzl": "", + "MODULE.bazel": "" + }, + "patch_strip": 1, + "patches": { + "0001-bazel-migration-to-bazel-8.1.1.patch": "", + "0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch": "" + } +} diff --git a/third_party/modules/xla/metadata.json b/third_party/modules/xla/metadata.json index d0304d21..01c87137 100644 --- a/third_party/modules/xla/metadata.json +++ b/third_party/modules/xla/metadata.json @@ -19,8 +19,7 @@ "20250123.0-cc075be", "20250204-0-6789523", "20250317.0-71c67e2", - "20250317.1-71c67e2", - "20250317.2-71c67e2" + "20250409.0-7167d71" ], "yanked_versions": {} } diff --git a/zml/buffer.zig b/zml/buffer.zig index 1fc57ef3..71d74400 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -4,6 +4,8 @@ const testing = std.testing; const asynk = @import("async"); const stdx = @import("stdx"); +const ffi = @import("ffi.zig"); +const ops = @import("ops.zig"); const Context = @import("context.zig").Context; const Data = @import("dtype.zig").Data; const DataType = @import("dtype.zig").DataType; @@ -73,38 +75,72 @@ pub const Buffer = struct { const buffer_type = bufferTypeFromDtype(host_buffer.shape().dtype()); const byte_strides = host_buffer.strides(); - var frames: std.BoundedArray(asynk.Frame(pjrt.Client.bufferFromHostBuffer), MAX_NUM_SHARDS) = .{}; const devices = platform.getDevices(); for (0..n_partitions) |i| { - // If no sharding if found, the given buffer is replicated on all devices. const buf = if (sharding_ax) |ax| buf: { const start: i64 = @as(i64, @intCast(i)) * chunk_size; break :buf host_buffer.slice1d(ax, .{ .start = start, .end = start + chunk_size }); } else host_buffer; - const frame = try asynk.asyncc(pjrt.Client.bufferFromHostBuffer, .{ - platform.pjrt_client, - platform.pjrt_api, - pjrt.Client.BufferFromHostBufferArgs{ - .data = buf._data, - .buffer_type = buffer_type, - .dims = buf.shape().dims(), - .byte_strides = byte_strides, - .device = devices[i], - .host_buffer_semantics = .ImmutableOnlyDuringCall, - }, + const pjrt_buffer, const event = try platform.pjrt_client.bufferFromHostBuffer(platform.pjrt_api, .{ + .data = buf._data, + .buffer_type = buffer_type, + .dims = buf.shape().dims(), + .byte_strides = byte_strides, + .device = devices[i], + .host_buffer_semantics = .ImmutableOnlyDuringCall, }); - - frames.appendAssumeCapacity(frame); - } - - for (frames.slice()) |*frame| { - const pjrt_buffer = try frame.awaitt(); + if (event) |ev| { + ev.deinit(platform.pjrt_api); + } res._shards.appendAssumeCapacity(pjrt_buffer); } + return res; } + pub const FromExOpts = struct { + memory: Memory = .device, + }; + pub fn fromEx(platform: Platform, host_buffer: HostBuffer, opts: FromExOpts) !Buffer { + var res: Buffer = .{ + ._api = platform.pjrt_api, + ._shape = host_buffer.shape(), + ._shards = .{}, + }; + + const buffer_type = bufferTypeFromDtype(host_buffer.shape().dtype()); + const byte_strides = host_buffer.strides(); + + const devices = platform.getDevices(); + for (devices) |device| { + const pjrt_buffer, const event = try platform.pjrt_client.bufferFromHostBuffer(platform.pjrt_api, .{ + .data = host_buffer._data, + .buffer_type = buffer_type, + .dims = host_buffer.shape().dims(), + .byte_strides = byte_strides, + .device = device, + .host_buffer_semantics = .ImmutableOnlyDuringCall, + .memory = @constCast(platform.pjrt_client.memoryByKind(platform.pjrt_api, opts.memory.toPjrtMemory())), + }); + if (event) |ev| { + ev.deinit(platform.pjrt_api); + } + res._shards.appendAssumeCapacity(pjrt_buffer); + } + + return res; + } + + pub fn awaitt(self: Buffer) !Buffer { + for (self._shards.constSlice()) |buffer| { + if (buffer.getReadyEvent(self._api)) |ev| { + try ev.awaitt(self._api); + } + } + return self; + } + /// Wraps pre-exisiting `pjrt.Buffer` shards into one `zml.Buffer`. pub fn fromPjrtBuffers(platform: Platform, shape_: Shape, pjrt_buffers: []const *pjrt.Buffer) Buffer { stdx.debug.assert(pjrt_buffers.len <= MAX_NUM_SHARDS, "ZML doesn't support having more than {} shards. Received {} shards for one buffer.", .{ MAX_NUM_SHARDS, pjrt_buffers.len }); @@ -224,24 +260,18 @@ pub const Buffer = struct { /// Creates a Buffer from a pointer into device memory. /// This allows to interface with other libraries producing buffers. - pub fn asViewOfDeviceBuffer(platform: Platform, shape_: Shape, stream: ?isize, device_data: *anyopaque) Buffer { - const minor_to_major: [Shape.MAX_RANK]i64 = comptime blk: { - var res: [Shape.MAX_RANK]i64 = undefined; - for (0..Shape.MAX_RANK) |i| { - res[i] = @intCast(Shape.MAX_RANK - i - 1); - } - break :blk res; - }; - + pub fn asViewOfDeviceBuffer(platform: Platform, shape_: Shape, stream: ?*const anyopaque, device_data: *anyopaque) Buffer { const pjrt_buffer = platform.pjrt_client.createViewOfDeviceBuffer(platform.pjrt_api, .{ .data = device_data, .element_type = bufferTypeFromDtype(shape_.dtype()), .dims = shape_.dims(), // TODO: exposes sharding in the API. .device = platform.getDevices()[0], + // TODO: expose memory in the API. + .memory_kind = null, .layout = .{ .tiled = .{ - .minor_to_major = minor_to_major[Shape.MAX_RANK - shape_.rank() ..], + .minor_to_major = ops.minorToMajor(shape_.rank()), .tile_dims = &.{}, .tile_dims_sizes = &.{}, }, @@ -265,7 +295,7 @@ pub const Buffer = struct { stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); const maybe_event = try self._shards.get(0).toHostBuffer(self._api, std.mem.asBytes(&res)); if (maybe_event) |event| { - try event.await_(self._api); + try event.awaitt(self._api); } return res; } @@ -276,21 +306,22 @@ pub const Buffer = struct { pub fn toHost(self: Buffer, output: []u8) !HostBuffer { stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); const maybe_event = try self._shards.get(0).toHostBuffer(self._api, output); - if (maybe_event) |event| { - try event.await_(self._api); - } - return HostBuffer.fromBytes(self.shape(), output); + var hb = HostBuffer.fromBytes(self.shape(), output); + hb._event = maybe_event; + hb._api = self._api; + hb._ready = if (maybe_event) |ev| ev.isReady(self._api) else true; + return hb; } /// Copies the content of the Buffer to the host. /// The returned `HostBuffer` does own the memory. pub fn toHostAlloc(self: Buffer, allocator: std.mem.Allocator) !HostBuffer { - const output = try HostBuffer.empty(allocator, self.shape()); + var output = try HostBuffer.empty(allocator, self.shape()); stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); const maybe_event = try self._shards.get(0).toHostBuffer(self._api, @constCast(output.bytes())); - if (maybe_event) |event| { - try event.await_(self._api); - } + output._event = maybe_event; + output._api = self._api; + output._ready = if (maybe_event) |ev| ev.isReady(self._api) else true; return output; } @@ -348,6 +379,29 @@ pub const Buffer = struct { return shard.memory(self._api); } + pub fn copyToMemory(self: Buffer, platform: Platform, kind: Memory) !Buffer { + const device = try self._shards.get(0).getDevice(self._api); + const memories = try device.addressableMemories(self._api); + var selected_mem: *const pjrt.Memory = undefined; + for (memories) |mem| { + if (mem.kind(platform.pjrt_api) == kind.toPjrtMemory()) { + selected_mem = mem; + } + } + + var shards: Shards = .{}; + for (self._shards.constSlice()) |shard| { + const pjrt_buffer = try shard.copyToMemory(self._api, selected_mem); + shards.appendAssumeCapacity(pjrt_buffer); + } + + return .{ + ._api = platform.pjrt_api, + ._shape = self._shape, + ._shards = shards, + }; + } + fn hasShardedAxis(self: Buffer) bool { if (self._shards.len == 1) return false; return @reduce(.Or, self._shape._sharding_info); diff --git a/zml/context.zig b/zml/context.zig index 03de62a3..39d7cfeb 100644 --- a/zml/context.zig +++ b/zml/context.zig @@ -24,6 +24,8 @@ test { std.testing.refAllDecls(Context); } +var _active_platform: Platform = undefined; + /// Every program using ZML must start with a `zml.Context.init(.{});` /// The ZML context contains global state to interact with the different /// devices available on your system. @@ -37,6 +39,7 @@ pub const Context = struct { inline for (comptime std.enums.values(runtimes.Platform)) |t| { if (runtimes.load(t)) |api| { Context.apis.set(t, api); + if (t == .cuda) cuda.init(); } else |_| {} } } @@ -128,7 +131,10 @@ pub const Context = struct { pub fn autoPlatform(self: *Context, opts: Platform.CreateOptions) Platform { stdx.debug.assert(prefered_targets.len == apis.values.len, "New target need to be inserted inside `zml.Context.preferred_targets`", .{}); - return self.platformByPreferences(opts, &prefered_targets); + const p = self.platformByPreferences(opts, &prefered_targets); + // TODO properly propagate the platform to the callbacks + _active_platform = p; + return p; } /// Given a list of preferred targets to select the best Platform @@ -177,8 +183,6 @@ pub const Context = struct { return error.NoDevicesFound; } - try CustomCall.registerZmlCustomCalls(p); - self.platforms.set(target, p); return p; } @@ -212,69 +216,71 @@ pub const Context = struct { } } } - - pub const HostCallback = fn (?*anyopaque, []const HostBuffer, []const HostBuffer) void; }; -const CustomCall = struct { - pub fn registerZmlCustomCalls(platform: Platform) !void { - const registry = platform.pjrt_api.customCallRegistry(); +pub const cuda = struct { + var _memcpyAsync: MemcpyAsync = @ptrFromInt(0xdeadc00da00); + var _memcpyBlocking: MemcpyBlocking = @ptrFromInt(0xdeadc00da00); + pub var streamSynchronize: StreamSynchronize = @ptrFromInt(0xdeadc00da00); + pub var cuLaunchHostFunc: CuLaunchHostFunc = @ptrFromInt(0xdeadc00da00); - if (registry) |reg| { - try reg.registerFfi(platform.pjrt_api, "zmlHostBufferCallback", @tagName(platform.target), &hostBufferCallback); - } else { - stdx.debug.panic("Registering custom calls failed", .{}); - } - } - - fn hostBufferCallback(call_frame: *pjrt.ffi.CallFrame) callconv(.C) ?*pjrt.ffi.Error { - if (call_frame.registeringHook()) return null; - - const callback_attr = call_frame.attrs.getByName(.scalar, "callback") orelse unreachable; - std.debug.assert(callback_attr.dtype == .u64); - const callback: *const Context.HostCallback = @ptrFromInt(callback_attr.get(usize)); - - const user_ctx_ptr = call_frame.attrs.getByName(.scalar, "user_context") orelse unreachable; - std.debug.assert(user_ctx_ptr.dtype == .u64); - const user_ctx: ?*anyopaque = @ptrFromInt(user_ctx_ptr.get(usize)); - - const input_buffers = stdx.stackSlice(8, HostBuffer, call_frame.args.len); - for (input_buffers, 0..) |*b, i| { - b.* = hostBufferFromPinnedBuffer(call_frame.args.get(i)); - } - - const output_buffers = stdx.stackSlice(8, HostBuffer, call_frame.results.len); - for (output_buffers, 0..) |*b, i| { - b.* = hostBufferFromPinnedBuffer(call_frame.results.get(i)); - } - - callback(user_ctx, input_buffers, output_buffers); - return null; - } -}; - -fn getShape(buffer_desc: *const pjrt.ffi.Buffer) Shape { - // log.warn("received buffer {}", .{buffer_desc}); - const dt: DataType = switch (buffer_desc.dtype) { - .invalid => @panic("invalid ffi"), - .pred => .bool, - .s8 => .i8, - .s16 => .i16, - .s32 => .i32, - .s64 => .i64, - .token, .f8e4m3, .f8e3m4 => @panic("Unsupported ffi type"), - inline else => |t| @field(DataType, @tagName(t)), + pub const MemcpyKind = enum(c_int) { + host_to_host = 0, + host_to_device = 1, + device_to_host = 2, + device_to_device = 3, + inferred = 4, }; - return Shape.init(buffer_desc.dims(), dt); -} -/// Create a HostBuffer from a ffi description of a buffer. -/// Normally the ffi describe device buffer but we assume they are located in pinned memory, -/// and therefore the data pointer is readable both from host and from device. -fn hostBufferFromPinnedBuffer(buffer_desc: *const pjrt.ffi.Buffer) HostBuffer { - const buffer_shape = getShape(buffer_desc); - return HostBuffer.fromBytes( - buffer_shape, - buffer_desc.data[0..buffer_shape.byteSize()], - ); -} + const MemcpyAsync = *const fn (dst: *anyopaque, src: *const anyopaque, count: usize, kind: MemcpyKind, stream: ?*pjrt.ffi.Stream) callconv(.C) c_int; + const MemcpyBlocking = *const fn (dst: *anyopaque, src: *const anyopaque, count: usize, kind: MemcpyKind) callconv(.C) c_int; + const StreamSynchronize = *const fn (stream: ?*pjrt.ffi.Stream) callconv(.C) c_int; + const CuLaunchHostFunc = *const fn (stream: *anyopaque, host_func: *const fn (user_data: *const anyopaque) callconv(.c) void, user_data: *anyopaque) callconv(.c) c_int; + + pub fn init() void { + var cudart = std.DynLib.open("libcudart.so.12") catch { + log.err("cudart not found, callback will segfault", .{}); + return; + }; + defer cudart.close(); + + _memcpyAsync = cudart.lookup(MemcpyAsync, "cudaMemcpyAsync") orelse { + @panic("cudaMemcpyAsync not found"); + }; + _memcpyBlocking = cudart.lookup(MemcpyBlocking, "cudaMemcpy") orelse { + @panic("cudaMemcpy not found"); + }; + streamSynchronize = cudart.lookup(StreamSynchronize, "cudaStreamSynchronize") orelse { + @panic("cudaStreamSynchronize not found"); + }; + cuLaunchHostFunc = cudart.lookup(CuLaunchHostFunc, "cudaLaunchHostFunc") orelse { + @panic("cudaLaunchHostFunc not found"); + }; + } + + pub fn memcpyToHostBlocking(dst: []u8, src: pjrt.ffi.DevicePtr) void { + const err = _memcpyBlocking(dst.ptr, src.asPtr(), dst.len, .device_to_host); + check(err); + } + + pub fn memcpyToDeviceBlocking(dst: pjrt.ffi.DevicePtr, src: []const u8) void { + const err = _memcpyBlocking(dst.asPtr(), src.ptr, src.len, .host_to_device); + check(err); + } + + pub fn memcpyToDeviceAsync(dst: pjrt.ffi.DevicePtr, src: []const u8, stream: ?*pjrt.ffi.Stream) void { + const err = _memcpyAsync(dst.asPtr(), src.ptr, src.len, .host_to_device, stream); + check(err); + } + + pub fn memcpyToHostAsync(dst: []u8, src: pjrt.ffi.DevicePtr, stream: ?*pjrt.ffi.Stream) void { + const err = _memcpyAsync(dst.ptr, src.asPtr(), dst.len, .device_to_host, stream); + check(err); + } + + pub fn check(err: c_int) void { + if (err == 0) return; + log.err("CUDA ERROR {d}", .{err}); + @panic("CUDA error"); + } +}; diff --git a/zml/exe.zig b/zml/exe.zig index 2292b70b..1d98173f 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -12,7 +12,7 @@ const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; const ShapeOf = @import("tensor.zig").ShapeOf; -const log = std.log.scoped(.zml); +const log = std.log.scoped(.@"zml/exe"); test { std.testing.refAllDecls(@This()); @@ -156,6 +156,8 @@ pub const BaseExe = struct { /// Allocator backing memory _arena: std.heap.ArenaAllocator, + _context: ?*pjrt.ExecuteContext = null, + pub fn init( parent_allocator: std.mem.Allocator, platform: Platform, @@ -199,6 +201,9 @@ pub const BaseExe = struct { } pub fn deinit(self: BaseExe) void { + if (self._context) |ctx| { + ctx.deinit(self.platform.pjrt_api); + } self._arena.deinit(); } @@ -211,6 +216,7 @@ pub const BaseExe = struct { var events = [_]?*pjrt.Event{null} ** Platform.MAX_NUM_DEVICES; const sharding = self.platform.sharding(); + const trace = self.platform.tracer.frameStart("_unsafeCall execute"); self.exe.execute(self.platform.pjrt_api, .{ .arguments = self.input_per_device, .num_args = self.input_buffer_count, @@ -220,15 +226,17 @@ pub const BaseExe = struct { // even if it has been marked as "can be donated" during compilation. // TODO: expose it ? .non_donatable_input_indices = &.{}, - }) catch |err| { - std.debug.panic("PJRT_LoadedExecutable_Execute failed with: {}", .{err}); - }; + .context = if (self._context) |ctx| ctx else null, + }) catch unreachable; + self.platform.tracer.frameEnd(trace, "_unsafeCall execute"); - for (events[0..sharding.num_partitions]) |e| { - if (e) |ev| { - ev.await_(self.platform.pjrt_api) catch unreachable; - } - } + // const trace_events_await = self.platform.tracer.frameStart("_unsafeCall await events"); + // for (events[0..sharding.num_partitions]) |e| { + // if (e) |ev| { + // ev.awaitt(self.platform.pjrt_api) catch unreachable; + // } + // } + // self.platform.tracer.frameEnd(trace_events_await, "_unsafeCall await events"); } pub fn serialize(self: BaseExe, writer: anytype) !void { @@ -245,10 +253,29 @@ pub const BaseExe = struct { // } pub fn prepare(self: *BaseExe, x: anytype) void { - const n = fillBuffers(&x, self.input_shapes, self.input_per_device, self.ready_buffer_count); + const n = fillBuffers(&x, self.input_shapes, self.platform, self.input_per_device, self.ready_buffer_count); self.ready_buffer_count += n; } + pub fn register(self: BaseExe, comptime T: type) !void { + stdx.debug.assert(self._context != null, "Exe doesn't have an execution context", .{}); + const pjrt_api = self.platform.pjrt_api; + const ffi = pjrt_api.ffi().?; + try ffi.registerTypeId(pjrt_api, T); + log.info("Registered type id {d} for {s}", .{ T.type_id, @typeName(T) }); + } + + pub fn bind(self: BaseExe, comptime T: type, value: *T) !void { + stdx.debug.assert(self._context != null, "Exe doesn't have an execution context", .{}); + const pjrt_api = self.platform.pjrt_api; + const ffi = pjrt_api.ffi().?; + const type_id = T.type_id; + const user_data: *anyopaque = @ptrCast(@constCast(value)); + + try ffi.addUserData(pjrt_api, self._context.?, .{ .type_id = type_id, .user_data = user_data }); + log.info("Attached {s}@{x} with type id {d} on {any}", .{ @typeName(T), user_data, T.type_id, self._context.? }); + } + pub fn getOutputBuffer(self: BaseExe, i: usize) Buffer { var shards: Buffer.Shards = .{}; for (self.output_per_device) |dev_out| { @@ -259,11 +286,13 @@ pub const BaseExe = struct { } pub fn clone(self: BaseExe, parent_allocator: std.mem.Allocator) !BaseExe { - return .init(parent_allocator, self.platform, self.exe, .{ + var exe: BaseExe = try .init(parent_allocator, self.platform, self.exe, .{ .input_shapes = self.input_shapes, .result_shapes = self.result_shapes, .n_devices = self.num_devices, }); + exe._context = self._context; + return exe; } }; @@ -292,6 +321,23 @@ pub fn Exe(ArgsT: type, ReturnT: type) type { return new; } + pub fn register(self: Self, comptime T: type) !void { + try self.inner.register(T); + } + + pub fn bind(self: Self, comptime T: type, value: *T) !void { + try self.inner.bind(T, value); + } + + pub fn withExecutionContext(self: Self) Self { + stdx.debug.assert(self.inner._context == null, "Exe already has an execution context", .{}); + var new: Self = .{ .inner = self.inner }; + const pjrt_execute_context = self.inner.platform.pjrt_api.createExecuteContext() catch unreachable; + new.inner._context = pjrt_execute_context; + log.info("Created context execution {*} for {*}", .{ pjrt_execute_context, self.inner.exe }); + return new; + } + pub fn serialize(self: Self, writer: anytype) !void { return try self.inner.serialize(writer); } @@ -301,7 +347,7 @@ pub fn Exe(ArgsT: type, ReturnT: type) type { } pub fn call(self: Self, args: Bufferized(ArgsT)) Bufferized(ReturnT) { - const total_ready = fillBuffers(&args, self.inner.input_shapes, self.inner.input_per_device, self.inner.ready_buffer_count); + const total_ready = fillBuffers(&args, self.inner.input_shapes, self.inner.platform, self.inner.input_per_device, self.inner.ready_buffer_count); std.debug.assert(total_ready == self.inner.input_buffer_count); self.inner._unsafeCall(); var result: Bufferized(ReturnT) = undefined; @@ -323,7 +369,10 @@ fn splitBuffer(T: type, buffer: []T, lengths: anytype) [lengths.len][]T { } /// Visit the given struct and fill the `buffers` slice with the buffer associated with encountered Tensor. -fn fillBuffers(v: anytype, shapes: []const Shape, buffers: []const [*]*pjrt.Buffer, start: u32) u32 { +fn fillBuffers(v: anytype, shapes: []const Shape, platform: Platform, buffers: []const [*]*pjrt.Buffer, start: u32) u32 { + const trace = platform.tracer.frameStart("fillBuffers"); + defer platform.tracer.frameEnd(trace, "fillBuffers"); + const LocalContext = struct { index: u32, buffers: []const [*]*pjrt.Buffer, @@ -351,6 +400,9 @@ fn fillBuffers(v: anytype, shapes: []const Shape, buffers: []const [*]*pjrt.Buff /// Visit the given struct and override tensors by creating a new one using the provided PJRT buffers. fn assignRawBuffers(v: anytype, platform: Platform, buffers: []const [*]*pjrt.Buffer, buffer_shapes: []Shape) void { + const trace = platform.tracer.frameStart("assignRawBuffers"); + defer platform.tracer.frameEnd(trace, "assignRawBuffers"); + const LocalContext = struct { index: u32, platform: Platform, diff --git a/zml/ffi.zig b/zml/ffi.zig new file mode 100644 index 00000000..2ef0e645 --- /dev/null +++ b/zml/ffi.zig @@ -0,0 +1,242 @@ +const std = @import("std"); +const testing = std.testing; +const builtin = @import("builtin"); + +const asynk = @import("async"); +const stdx = @import("stdx"); + +const context = @import("context.zig"); +const dialect = struct { + const stablehlo = @import("mlir/dialects").stablehlo; +}; +const dtype = @import("dtype.zig"); +const pjrt = @import("pjrtx.zig"); +const buffer = @import("buffer.zig"); +const hostbuffer = @import("hostbuffer.zig"); +const mlir = @import("mlir.zig"); +const module = @import("module.zig"); +const ops = @import("ops.zig"); +const platform = @import("platform.zig"); +const tensor = @import("tensor.zig"); +const shape = @import("shape.zig"); +const cuda = context.cuda; +const Buffer = buffer.Buffer; +const HostBuffer = hostbuffer.HostBuffer; +const CompilationContext = module.CompilationContext; +const DataType = dtype.DataType; +const Platform = platform.Platform; +const Shape = shape.Shape; +const Tensor = tensor.Tensor; + +const log = std.log.scoped(.@"zml/custom_call"); + +pub const std_options: std.Options = .{ + .log_level = .info, + .logFn = asynk.logFn(std.log.defaultLog), +}; + +pub const FFIBuffer = pjrt.ffi.Buffer; +pub const FFIRets = pjrt.ffi.Rets; +pub const FFIStream = pjrt.ffi.Stream; +// todo: check deallocs + +pub fn CustomCallCompilationInputType(custom_op: type) type { + const ArgsT = stdx.meta.FnArgs(custom_op.compile); + if (@typeInfo(ArgsT) != .@"struct") { + @compileError("Expected struct type"); + } + // todo check typeof first input then others args + return [@typeInfo(ArgsT).@"struct".fields.len]Tensor; +} + +pub fn CustomCallCompilationOutputType(custom_op: type) type { + const ReturnT = stdx.meta.FnResultNoError(custom_op.call); + + if (@typeInfo(ReturnT) != .@"struct") { + @compileError("Expected struct type"); + } + + if (ReturnT == Shape) { + return Tensor; + } + return [@typeInfo(ReturnT).@"struct".fields.len]Tensor; +} + +pub fn lenOutputBeforeCustomCall(custom_op: type) usize { + const ReturnT = stdx.meta.FnResultNoError(custom_op.compile); + + if (@typeInfo(ReturnT) != .@"struct") { + @compileError("Expected struct type"); + } + + if (ReturnT == Tensor) { + return 1; + } + return @typeInfo(ReturnT).@"struct".fields.len; +} + +pub fn CustomCallInputsType(custom_op: type) type { + const ArgsT = stdx.meta.FnArgs(custom_op.call); + if (@typeInfo(ArgsT) != .@"struct") { + @compileError("Expected struct type"); + } + // todo check typeof first input then others args + return [@typeInfo(ArgsT).@"struct".fields.len - 1]Tensor; +} + +pub fn custom_call( + comptime custom_op: type, + inputs: CustomCallInputsType(custom_op), + res_shapes_: []const Shape, +) []Tensor { + stdx.debug.assert(@hasDecl(custom_op, "call"), "custom_op must have a call method", .{}); + const op_name = @typeName(custom_op); + + const ctx = CompilationContext.current(); + const allocator = ctx.allocator(); + const mlir_ctx = ctx.mlirCtx(); + const platform_ = ctx._platform; + const pjrt_api = platform_.pjrt_api; + const ffi = pjrt_api.ffi().?; + const registry = ffi.customCallRegistry().?; + const ffi_func = proxy(custom_op); + const target_name = "callback_" ++ op_name; + + registry.registerFfi(pjrt_api, target_name, @tagName(platform_.target), &ffi_func, .{}) catch unreachable; + log.info("{s} / Registered custom call with target_name \"{s}\" with proxy func {*}", .{ op_name, target_name, &ffi_func }); + + // var custom_call_inputs = allocator.alloc(mlir.Value, 8) catch unreachable; + // var res_shapes = allocator.alloc(Shape, 8) catch unreachable; + // var res_types = allocator.alloc(mlir.Type, 8) catch unreachable; + + // if (@typeInfo(@TypeOf(custom_op.compile)) != .@"fn") { + // stdx.debug.panic("compile must be a function"); + // } + + // var args: std.meta.ArgsTuple(@TypeOf(custom_op.compile)) = undefined; + // inline for (0..args.len) |i| { + // args[i] = inputs[i]; + // } + + // const compile_ouputs = ctx.callFunc(@typeName(custom_op), custom_op.compile, args) catch |err| { + // stdx.debug.panic("Error in {any} beforeCustomCall func: {any}\n", .{ @typeName(custom_op), err }); + // }; + // var compile_ouputs_array = allocator.alloc(Shape, 8) catch unreachable; + + // if (@TypeOf(compile_ouputs) == Shape) { + // compile_ouputs_array[0] = compile_ouputs; + // } else { + // inline for (@typeInfo(@TypeOf(compile_ouputs)).@"struct".fields, 0..) |field, i| { + // compile_ouputs_array[i] = @field(compile_ouputs, field.name); + // } + // } + + // for (compile_ouputs_array[0..res_shapes_.len]) |*rest_t| { + // log.warn("compile_ouputs_array: {any}", .{rest_t}); + // } + + // for (inputs, 0..) |t, i| { + // custom_call_inputs[i] = t.value(); + // } + + // for (compile_ouputs_array[0..res_shapes_.len], 0..) |sh, i| { + // res_shapes[i] = sh; + // res_types[i] = mlir.ext.RankedTensorType.fromShape(mlir_ctx, sh).as(mlir.Type); + // } + + const custom_call_inputs = allocator.alloc(mlir.Value, 8) catch unreachable; + for (inputs, 0..) |t, i| { + custom_call_inputs[i] = t.value(); + } + + const res_types = allocator.alloc(mlir.Type, 8) catch unreachable; + for (res_shapes_, 0..) |sh, i| { + res_types[i] = mlir.ext.RankedTensorType.fromShape(mlir_ctx, sh).as(mlir.Type); + } + + const frontend_attributes = mlir.Attribute.dict(mlir_ctx, &.{}); + + const op = dialect.stablehlo.custom_call( + mlir_ctx, + custom_call_inputs[0..inputs.len], + .{ + .call_target_name = target_name, + .api_version = .typed_ffi, + .backend_config = mlir.Attribute.dict(mlir_ctx, &.{}), + .additional_attributes = &.{.{ "mhlo.frontend_attributes", frontend_attributes }}, + .has_side_effect = true, + .output_operand_aliases = &.{}, + }, + res_types[0..res_shapes_.len], + mlir_ctx.location(@src()), + ); + + // if (CustomCallCompilationOutputType(custom_op) == Tensor) { + // return Tensor._result(res_shapes_[0], op.result(0)); + // } + + var custom_call_outputs = allocator.alloc(Tensor, 8) catch unreachable; + + for (custom_call_outputs[0..res_shapes_.len], res_shapes_, 0..) |*t, sh, i| { + t.* = Tensor._result(sh, op.result(i)); + } + + return custom_call_outputs[0..res_shapes_.len]; +} + +fn proxy(comptime custom_op: type) pjrt.ffi.Handler { + const op_name = @typeName(custom_op); + _ = op_name; // autofix + + return struct { + const Self = @This(); + + pub fn proxy(call_frame: *pjrt.ffi.CallFrame) callconv(.C) ?*pjrt.ffi.Error { + if (call_frame.registeringHook()) return null; + const execution_context = call_frame.ctx.?; + // const device_id = execution_context.getDeviceOrdinal(call_frame.api) catch unreachable; + const user_ctx: *custom_op = pjrt.ffi.ExecutionContext.Context(custom_op).get(execution_context, call_frame.api) catch unreachable; + // const platform_: Platform = user_ctx.platform; + // const pjrt_api = platform_.pjrt_api; + // const pjrt_client = platform_.pjrt_client; + // const tracer = platform_.tracer; + // _ = tracer; // autofix + + // const device = pjrt_client.getAddressableDevices(pjrt_api)[@as(u32, @intCast(device_id))]; + // _ = device; // autofix + const ctx = call_frame.ctx; + const stream = call_frame.api.stream(@constCast(ctx)); + + var callback_args: std.meta.ArgsTuple(@TypeOf(custom_op.call)) = undefined; + callback_args[0] = user_ctx; + + inline for (1..callback_args.len) |i| { + const ffi_buffer = call_frame.args.buffers()[i - 1]; + callback_args[i] = ffi_buffer; + } + + user_ctx.*.results = call_frame.results.buffers(); + user_ctx.*.stream = stream; + + @call(.auto, custom_op.call, callback_args) catch |err| { + stdx.debug.panic("Error while calling {s} call func: {any}\n", .{ @typeName(custom_op), err }); + }; + + return null; + } + }.proxy; +} + +pub fn getShape(ffi_buffer: *const pjrt.ffi.Buffer) Shape { + const dt: DataType = switch (ffi_buffer.dtype) { + .invalid => stdx.debug.panic("Invalid FFI dtype {any} used by {any}", .{ ffi_buffer.dtype, ffi_buffer }), + .pred => .bool, + .i8 => .i8, + .i16 => .i16, + .i32 => .i32, + .i64 => .i64, + .token, .f8e4m3, .f8e3m4 => stdx.debug.panic("Unsupported FFI dtype {any} used by {any}", .{ ffi_buffer.dtype, ffi_buffer }), + inline else => |t| @field(DataType, @tagName(t)), + }; + return Shape.init(ffi_buffer.dims(), dt); +} diff --git a/zml/hostbuffer.zig b/zml/hostbuffer.zig index 5e8697c0..45dc8a55 100644 --- a/zml/hostbuffer.zig +++ b/zml/hostbuffer.zig @@ -2,6 +2,8 @@ const std = @import("std"); const stdx = @import("stdx"); +const pjrt = @import("pjrtx.zig"); + const Buffer = @import("buffer.zig").Buffer; const Data = @import("dtype.zig").Data; const DataType = @import("dtype.zig").DataType; @@ -24,6 +26,9 @@ pub const HostBuffer = struct { managed: std.mem.Alignment, unmanaged, } = .unmanaged, + _ready: bool = true, + _api: ?*const pjrt.Api = null, + _event: ?*pjrt.Event = null, /// Allocates a HostBuffer with the given shape. /// The memory is left undefined. @@ -158,6 +163,8 @@ pub const HostBuffer = struct { /// Copies this HostBuffer to the given accelerator. pub fn toDevice(self: HostBuffer, platform_: Platform) !Buffer { + const frame = platform_.tracer.frameStart("HostBuffer toDevice"); + defer platform_.tracer.frameEnd(frame, "HostBuffer toDevice"); return try Buffer.from(platform_, self); } @@ -317,6 +324,20 @@ pub const HostBuffer = struct { }; } + pub fn awaitt(self: *HostBuffer) !*HostBuffer { + if (self._ready) { + return self; + } + if (self._event) |ev| { + if (self._api) |api| { + try ev.awaitt(api); + } else { + stdx.debug.panic("Got an event but no API. This should not happen.", .{}); + } + } + return self; + } + pub fn format( self: HostBuffer, comptime fmt: []const u8, diff --git a/zml/module.zig b/zml/module.zig index 798b81cf..49f31c83 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -1159,9 +1159,9 @@ pub fn hash(hasher: *std.hash.Wyhash, key: anytype, comptime strat: std.hash.Str .@"anyframe", .@"fn" => hash(hasher, @intFromPtr(key), strat), .pointer => |info| switch (info.size) { .one => switch (strat) { - .shallow => hash(hasher, @intFromPtr(key), .Shallow), - .deep => hash(hasher, key.*, .Shallow), - .deeprecursive => switch (@typeInfo(info.child)) { + .Shallow => hash(hasher, @intFromPtr(key), .Shallow), + .Deep => hash(hasher, key.*, .Shallow), + .DeepRecursive => switch (@typeInfo(info.child)) { .@"opaque", .@"fn" => hash(hasher, @intFromPtr(key), .Shallow), else => hash(hasher, key.*, .DeepRecursive), }, @@ -1177,7 +1177,7 @@ pub fn hash(hasher: *std.hash.Wyhash, key: anytype, comptime strat: std.hash.Str .many, .c, => switch (strat) { - .shallow => hash(hasher, @intFromPtr(key), .Shallow), + .Shallow => hash(hasher, @intFromPtr(key), .Shallow), else => @compileError( \\ unknown-length pointers and C pointers cannot be hashed deeply. \\ Consider providing your own hash function. diff --git a/zml/ops.zig b/zml/ops.zig index 1765888a..3f70997b 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -769,58 +769,6 @@ pub fn fromMlirOperationWithTags(op: mlir.Operation, base: anytype) @TypeOf(base return res; } -pub const HostCallbackOpt = struct { - has_side_effect: bool = false, - output_operand_aliases: []const i64 = &.{}, -}; - -pub fn addHostCallback( - callback: *const Context.HostCallback, - blkctx: ?*anyopaque, - inputs: []const Tensor, - output_shapes: []const Shape, - opts: HostCallbackOpt, -) []Tensor { - const ctx = CompilationContext.current(); - - const mlir_ctx = ctx.mlirCtx(); - const backend_config = mlir.Attribute.dict(mlir_ctx, &.{ - .{ "callback", .int(mlir_ctx, .u64, @bitCast(@intFromPtr(callback))) }, - .{ "user_context", .int(mlir_ctx, .u64, @bitCast(@intFromPtr(blkctx))) }, - }); - - const values = stdx.stackSlice(8, mlir.Value, inputs.len); - for (inputs, values) |i, *v| { - v.* = ctx.getValue(i.toMemory(.host_pinned)); - } - const res_types = stdx.stackSlice(8, mlir.Type, output_shapes.len); - for (res_types, output_shapes) |*r, o| { - r.* = mlir.ext.RankedTensorType.fromShape(mlir_ctx, o).as(mlir.Type); - } - - const loc = ctx.mlirCtx().location(@src()); - const op = dialect.stablehlo.custom_call( - ctx.mlirCtx(), - values, - .{ - .call_target_name = "zmlHostBufferCallback", - .api_version = .typed_ffi, - .backend_config = backend_config, - .has_side_effect = opts.has_side_effect, - .output_operand_aliases = opts.output_operand_aliases, - }, - res_types, - loc, - ); - - const res = ctx.allocator().alloc(Tensor, output_shapes.len) catch @panic("OOM"); - for (res, output_shapes, 0..) |*r, o, i| { - r.* = Tensor._result(o, op.result(i)).toMemory(.device); - } - - return res; -} - pub const TritonOps = struct { debug: bool = false, name: [:0]const u8, @@ -1253,13 +1201,13 @@ inline fn toI64(values: anytype) []i64 { } const _MINOR_TO_MAJOR = blk: { - var ret: [Shape.MAX_RANK]usize = undefined; + var ret: [Shape.MAX_RANK]i64 = undefined; for (0..Shape.MAX_RANK) |i| { ret[i] = @intCast(Shape.MAX_RANK - i - 1); } break :blk ret; }; -fn minorToMajor(rank: u8) []const usize { +pub fn minorToMajor(rank: u8) []const i64 { return _MINOR_TO_MAJOR[_MINOR_TO_MAJOR.len - rank ..]; } diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index bd094ead..aa40de12 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -8,6 +8,7 @@ pub const ffi = pjrt.ffi; pub const Profiler = pjrt.Profiler; pub const ApiError = pjrt.ApiError; pub const ErrorCode = pjrt.ErrorCode; +pub const ExecuteContext = pjrt.ExecuteContext; pub const BufferType = pjrt.BufferType; pub const Device = pjrt.Device; pub const DeviceDescription = pjrt.DeviceDescription; @@ -57,13 +58,16 @@ pub const Client = opaque { } pub const BufferFromHostBufferArgs = pjrt.Client.BufferFromHostBufferArgs; - pub fn bufferFromHostBuffer(self: *const Client, api: *const Api, args: BufferFromHostBufferArgs) ApiError!*Buffer { - const buffer, const event_ = try asynk.callBlocking(pjrt.Client.bufferFromHostBuffer, .{ self.inner(), api, args }); - if (event_) |event__| { - const event: *Event = @ptrCast(event__); - try event.await_(api); - } - return @ptrCast(buffer); + pub fn bufferFromHostBuffer(self: *const Client, api: *const Api, args: BufferFromHostBufferArgs) !struct { *Buffer, ?*Event } { + const buffer, const event_ = try self.inner().bufferFromHostBuffer(api, args); + return .{ @ptrCast(buffer), @ptrCast(event_) }; + } + + pub fn dmaMap(self: *const Client, api: *const Api, data: []const u8) ApiError!void { + try self.inner().dmaMap(api, data); + } + pub fn dmaUnmap(self: *const Client, api: *const Api, data: []const u8) ApiError!void { + try self.inner().dmaUnmap(api, data); } pub fn deserializeAndLoad(self: *const Client, api: *const Api, bytes: []const u8) ApiError!*LoadedExecutable { @@ -120,7 +124,7 @@ pub const Client = opaque { return self.inner().addressableMemories(api); } - pub fn memoryByKind(self: *const Client, api: *const Api, kind: Memory.Kind) ?*Memory { + pub fn memoryByKind(self: *const Client, api: *const Api, kind: Memory.Kind) ?*const Memory { for (self.addressableMemories(api)) |mem| { if (mem.kind(api) == kind) { return mem; @@ -153,10 +157,6 @@ pub const Buffer = opaque { return self.inner().isOnCpu(api); } - pub fn memory(self: *const Buffer, api: *const Api) *const Memory { - return self.inner().memory(api); - } - pub fn toHostBuffer(self: *const Buffer, api: *const Api, dst: []u8) ApiError!?*Event { return @ptrCast(try self.inner().toHostBuffer(api, dst)); } @@ -182,7 +182,7 @@ pub const Buffer = opaque { } pub fn copyToMemory(self: *const Buffer, api: *const Api, memory_: *const Memory) ApiError!*Buffer { - return @ptrCast(self.inner().copyToMemory(api, memory_)); + return @ptrCast(try self.inner().copyToMemory(api, memory_)); } pub fn getReadyEvent(self: *const Buffer, api: *const Api) ?*Event { @@ -209,7 +209,12 @@ pub const Event = opaque { return self.inner().getEventError(api); } - pub fn await_(self: *Event, api: *const Api) ApiError!void { + pub fn awaitt2(self: *Event, api: *const Api) !void { + return self.inner().await_(api); + } + + pub const await_ = awaitt; + pub fn awaitt(self: *Event, api: *const Api) !void { defer self.deinit(api); if (self.isReady(api)) { @@ -262,16 +267,18 @@ pub const LoadedExecutable = opaque { results: []const [*]*Buffer, events: []?*Event, non_donatable_input_indices: []const i64 = &.{}, + context: ?*ExecuteContext, }; pub fn execute(self: *const LoadedExecutable, api: *const Api, args: ExecuteArgs) ExecuteError!void { - try asynk.callBlocking(pjrt.LoadedExecutable.execute, .{ self.inner(), api, pjrt.LoadedExecutable.ExecuteArgs{ + try self.inner().execute(api, pjrt.LoadedExecutable.ExecuteArgs{ .num_args = args.num_args, .arguments = @ptrCast(args.arguments), .results = @ptrCast(args.results), .events = @ptrCast(args.events), .non_donatable_input_indices = args.non_donatable_input_indices, - } }); + .context = args.context, + }); } pub fn getExecutable(self: *LoadedExecutable, api: *const Api) ApiError!*Executable { diff --git a/zml/platform.zig b/zml/platform.zig index ec4ae4c6..f71efbe5 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -7,6 +7,7 @@ const stdx = @import("stdx"); const meta = @import("meta.zig"); const module = @import("module.zig"); const pjrt = @import("pjrtx.zig"); +const Tracer = @import("tools/tracer.zig").Tracer; const log = std.log.scoped(.zml); @@ -27,6 +28,7 @@ pub const Platform = struct { pjrt_api: *const pjrt.Api, pjrt_client: *pjrt.Client, compilation_options: CompilationOptions = .{}, + tracer: Tracer, pub const MAX_NUM_DEVICES: u8 = 32; pub const CreateOptions = _CreateOptions; @@ -43,6 +45,7 @@ pub const Platform = struct { .pjrt_api = api, .pjrt_client = pjrt_client, .compilation_options = .{}, + .tracer = Tracer.init("ai.zml.platform"), }; } @@ -125,6 +128,7 @@ const _CreateOptions = struct { }; pub fn writeNamedValues(self: Cuda, values: *std.ArrayListUnmanaged(pjrt.NamedValue)) void { + values.appendAssumeCapacity(pjrt.NamedValue.fromBool("should_stage_host_to_device_transfers", false)); switch (self.allocator) { .platform => { values.appendAssumeCapacity(pjrt.NamedValue.fromString("allocator", "platform")); diff --git a/zml/tensor.zig b/zml/tensor.zig index ae06a776..629040ea 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -8,8 +8,10 @@ const stdx = @import("stdx"); const Buffer = @import("buffer.zig").Buffer; const Data = @import("dtype.zig").Data; const DataType = @import("dtype.zig").DataType; +const custom_call = @import("ffi.zig").custom_call; const HostBuffer = @import("hostbuffer.zig").HostBuffer; const Memory = @import("buffer.zig").Buffer.Memory; +const ffi = @import("ffi.zig"); const meta = @import("meta.zig"); const mlir = @import("mlir.zig"); const Location = mlir.Location; @@ -234,6 +236,39 @@ pub const Tensor = struct { }; } + pub fn optimizationBarrier(tensors: []const Tensor) []Tensor { + const ctx = CompilationContext.current(); + const mlir_ctx = ctx.mlirCtx(); + const allocator = ctx.allocator(); + + var inputs: [16]mlir.Value = undefined; + var res_types: [16]mlir.Type = undefined; + + for (tensors, 0..) |t, i| { + inputs[i] = t.value(); + } + + for (tensors, 0..) |t, i| { + res_types[i] = mlir.ext.RankedTensorType.fromShape(mlir_ctx, t.shape()).as(mlir.Type); + } + + const res = allocator.alloc(Tensor, tensors.len) catch @panic("OOM"); + errdefer allocator.dealloc(res); + + const op = dialect.stablehlo.optimization_barrier( + mlir_ctx, + inputs[0..tensors.len], + res_types[0..tensors.len], + mlir_ctx.location(@src()), + ); + + for (res, 0..) |*t, i| { + t.* = _result(tensors[i]._shape, op.result(i)); + } + + return res; + } + /// Returns a Tensor with new tag names. pub fn rename(self: Tensor, renames: anytype) Tensor { var res = self; @@ -3827,28 +3862,6 @@ pub const Tensor = struct { } }.binaryOpHelper; } - - /// Insert code that will print the content of the given buffer at runtime. - /// Only for debug purpose, it inserts device to host synchronization - /// so it will slow down the program execution. - pub fn print(input: Tensor) Tensor { - // TODO: find a way of doing print that doesn't involve a H2D copy. - return ops.addHostCallback( - &printCallback, - null, - &.{input}, - &.{input.shape()}, - .{ .output_operand_aliases = &.{0} }, - )[0]; - } - - fn printCallback(_: ?*anyopaque, inputs: []const HostBuffer, outputs: []const HostBuffer) void { - const host_buffer = inputs[0]; - std.log.defaultLog(.info, .zml, "Device buffer: {}: {}", .{ host_buffer.shape(), host_buffer.pretty() }); - // This is true because of the operand aliases. - // Since the result is already pointing to the input we don't need to modify the buffer. - std.debug.assert(host_buffer._data == outputs[0]._data); - } }; fn initPoolArg(rank: usize, data: []const i64) [Tensor.MAX_RANK]i64 { diff --git a/zml/zml.zig b/zml/zml.zig index 18282b93..40ec6069 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -3,56 +3,56 @@ //! compiling it for various accelerators and targets, and executing it. //! -pub const Buffer = @import("buffer.zig").Buffer; -pub const Bufferized = @import("tensor.zig").Bufferized; -pub const CompilationOptions = @import("platform.zig").CompilationOptions; -pub const Context = @import("context.zig").Context; -pub const Data = @import("dtype.zig").Data; -pub const DataType = @import("dtype.zig").DataType; -pub const HostBuffer = @import("hostbuffer.zig").HostBuffer; -pub const Platform = @import("platform.zig").Platform; -pub const Shape = @import("shape.zig").Shape; -pub const ShapeOf = @import("tensor.zig").ShapeOf; -pub const Target = @import("platform.zig").Target; -pub const Tensor = @import("tensor.zig").Tensor; +const std = @import("std"); -// Namespaces -pub const context = @import("context.zig"); -pub const exe = @import("exe.zig"); -pub const floats = @import("floats.zig"); -pub const helpers = @import("helpers.zig"); -pub const nn = @import("nn.zig"); -pub const module = @import("module.zig"); -pub const meta = @import("meta.zig"); -pub const platform = @import("platform.zig"); -pub const mlir = @import("mlir.zig"); -pub const pjrt = @import("pjrtx.zig"); -pub const testing = @import("testing.zig"); -pub const torch = @import("torch.zig"); - -// pub const tokenizer = @import("tokenizer.zig"); pub const tokenizer = @import("zml/tokenizer"); -pub const call = ops.call; +pub const aio = @import("aio.zig"); +pub const sentencepiece = @import("aio/sentencepiece.zig"); +pub const Buffer = @import("buffer.zig").Buffer; +pub const Bufferized = @import("tensor.zig").Bufferized; +pub const bufferTypeFromDtype = @import("buffer.zig").bufferTypeFromDtype; +pub const CompilationOptions = @import("platform.zig").CompilationOptions; +pub const Context = @import("context.zig").Context; +pub const context = @import("context.zig"); +pub const Data = @import("dtype.zig").Data; +pub const DataType = @import("dtype.zig").DataType; +pub const exe = @import("exe.zig"); pub const compile = exe.compile; pub const compileFn = exe.compileFn; pub const compileModel = exe.compileModel; pub const FnExe = exe.FnExe; pub const ModuleExe = exe.ModuleExe; pub const ModuleSignature = exe.ModuleSignature; - +pub const floats = @import("floats.zig"); +pub const helpers = @import("helpers.zig"); +pub const HostBuffer = @import("hostbuffer.zig").HostBuffer; +pub const meta = @import("meta.zig"); +pub const mlir = @import("mlir.zig"); +pub const module = @import("module.zig"); +pub const nn = @import("nn.zig"); pub const ops = @import("ops.zig"); +pub const ffi = @import("ffi.zig"); +pub const custom_call = ffi.custom_call; +pub const call = ops.call; +pub const pjrt = @import("pjrtx.zig"); +pub const Platform = @import("platform.zig").Platform; +pub const platform = @import("platform.zig"); +pub const Shape = @import("shape.zig").Shape; +pub const ShapeOf = @import("tensor.zig").ShapeOf; +pub const Target = @import("platform.zig").Target; +pub const Tensor = @import("tensor.zig").Tensor; +pub const testing = @import("testing.zig"); +pub const torch = @import("torch.zig"); + +// Namespaces +// pub const tokenizer = @import("tokenizer.zig"); pub const tools = struct { pub const Tracer = @import("tools/tracer.zig").Tracer; }; -pub const aio = @import("aio.zig"); -pub const sentencepiece = @import("aio/sentencepiece.zig"); - pub const log = std.log.scoped(.zml); -const std = @import("std"); - test { // NOTE : testing entrypoint. // Don't forget to import your module if you want to declare tests declarations that will be run by //zml:test