diff --git a/MODULE.bazel b/MODULE.bazel index 1bcf2717..0311cca8 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -24,7 +24,7 @@ bazel_dep(name = "rules_zig", version = "20250314.0-b9739c6") bazel_dep(name = "sentencepiece", version = "20240618.0-d7ace0a") bazel_dep(name = "toolchains_protoc", version = "0.3.7") bazel_dep(name = "with_cfg.bzl", version = "0.9.1") -bazel_dep(name = "xla", version = "20250317.1-71c67e2") +bazel_dep(name = "xla", version = "20250409.0-7167d71") bazel_dep(name = "zig-protobuf", version = "20250318.0-930153e") bazel_dep(name = "zig-yaml", version = "20240903.0-83d5fdf") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index f6792256..22e6de13 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -83,7 +83,7 @@ "https://bazel-registry.zml.ai/modules/re2/2024-07-02.bcr.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/re2/2024-07-02/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_android/0.1.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_apple/3.17.0/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/rules_apple/3.20.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.0.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.0.10/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.0.13/MODULE.bazel": "not found", @@ -150,7 +150,7 @@ "https://bazel-registry.zml.ai/modules/rules_shell/0.2.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_shell/0.3.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_swift/1.16.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_swift/2.1.1/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/rules_swift/2.4.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_uv/0.53.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.5.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.5.3/MODULE.bazel": "not found", @@ -269,8 +269,8 @@ "https://bcr.bazel.build/modules/re2/2024-07-02/MODULE.bazel": "0eadc4395959969297cbcf31a249ff457f2f1d456228c67719480205aa306daa", "https://bcr.bazel.build/modules/rules_android/0.1.1/MODULE.bazel": "48809ab0091b07ad0182defb787c4c5328bd3a278938415c00a7b69b50c4d3a8", "https://bcr.bazel.build/modules/rules_android/0.1.1/source.json": "e6986b41626ee10bdc864937ffb6d6bf275bb5b9c65120e6137d56e6331f089e", - "https://bcr.bazel.build/modules/rules_apple/3.17.0/MODULE.bazel": "39ee3d15edbf09f536a52c42b4297fe1f08a239e28964555f578f301159033f3", - "https://bcr.bazel.build/modules/rules_apple/3.17.0/source.json": "57af375068c7292d88062b2a3c2cae546f6e262f653f03388b5d73207f7482e3", + "https://bcr.bazel.build/modules/rules_apple/3.20.1/MODULE.bazel": "d9049cabd0becdf8d26377927dda874e2febc803401269814776f232fde91b7c", + "https://bcr.bazel.build/modules/rules_apple/3.20.1/source.json": "7870121042c518ca6109fa16fb070b4dd6762f42aefdd9ddcc259194ca6627cd", "https://bcr.bazel.build/modules/rules_cc/0.0.1/MODULE.bazel": "cb2aa0747f84c6c3a78dad4e2049c154f08ab9d166b1273835a8174940365647", "https://bcr.bazel.build/modules/rules_cc/0.0.10/MODULE.bazel": "ec1705118f7eaedd6e118508d3d26deba2a4e76476ada7e0e3965211be012002", "https://bcr.bazel.build/modules/rules_cc/0.0.13/MODULE.bazel": "0e8529ed7b323dad0775ff924d2ae5af7640b23553dfcd4d34344c7e7a867191", @@ -350,8 +350,8 @@ "https://bcr.bazel.build/modules/rules_shell/0.3.0/MODULE.bazel": "de4402cd12f4cc8fda2354fce179fdb068c0b9ca1ec2d2b17b3e21b24c1a937b", "https://bcr.bazel.build/modules/rules_shell/0.3.0/source.json": "c55ed591aa5009401ddf80ded9762ac32c358d2517ee7820be981e2de9756cf3", "https://bcr.bazel.build/modules/rules_swift/1.16.0/MODULE.bazel": "4a09f199545a60d09895e8281362b1ff3bb08bbde69c6fc87aff5b92fcc916ca", - "https://bcr.bazel.build/modules/rules_swift/2.1.1/MODULE.bazel": "494900a80f944fc7aa61500c2073d9729dff0b764f0e89b824eb746959bc1046", - "https://bcr.bazel.build/modules/rules_swift/2.1.1/source.json": "40fc69dfaac64deddbb75bd99cdac55f4427d9ca0afbe408576a65428427a186", + "https://bcr.bazel.build/modules/rules_swift/2.4.0/MODULE.bazel": "1639617eb1ede28d774d967a738b4a68b0accb40650beadb57c21846beab5efd", + "https://bcr.bazel.build/modules/rules_swift/2.4.0/source.json": "a6577f57f9febbdc015a01f2a8f3487422032f134d6c61d18ed8e8ca3b9acc7c", "https://bcr.bazel.build/modules/rules_uv/0.53.0/MODULE.bazel": "795c21aaad2574f8cb8163a03808ef4a4a33c0b0a81c0749ca489563561b83c1", "https://bcr.bazel.build/modules/rules_uv/0.53.0/source.json": "bd665e54519407989760844261e02b184ea4f141e65e399f75cd6a3ebf02b649", "https://bcr.bazel.build/modules/stardoc/0.5.1/MODULE.bazel": "1a05d92974d0c122f5ccf09291442580317cdd859f07a8655f1db9a60374f9f8", @@ -380,7 +380,7 @@ "@@apple_support+//crosstool:setup.bzl%apple_cc_configure_extension": { "general": { "bzlTransitiveDigest": "JJV/C5bhYKZip41t/GGZvqIMmmrcwr0Ydybkn97uXxQ=", - "usagesDigest": "wD+r6kULzwkrPbJnhFpRl2hIsJZIwSSaeEUj3z6v0i4=", + "usagesDigest": "ai1Q9Iw9PMxCbfOYikwNfCbFYAEzeR5Pirndq3g0LFo=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, @@ -2029,6 +2029,160 @@ ] ] } + }, + "@@rules_swift+//swift:extensions.bzl%non_module_deps": { + "general": { + "bzlTransitiveDigest": "3iOPDsadEKzPUpfznfK5UYw/yuGpZzq7zrjrB/oEo6g=", + "usagesDigest": "9w18ec4dj90mX/JqpR2tBU2YVc1GU9yNxi/d7q6lLVA=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "com_github_apple_swift_protobuf": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-protobuf/archive/1.20.2.tar.gz" + ], + "sha256": "3fb50bd4d293337f202d917b6ada22f9548a0a0aed9d9a4d791e6fbd8a246ebb", + "strip_prefix": "swift-protobuf-1.20.2/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_protobuf/BUILD.overlay" + } + }, + "com_github_grpc_grpc_swift": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/grpc/grpc-swift/archive/1.16.0.tar.gz" + ], + "sha256": "58b60431d0064969f9679411264b82e40a217ae6bd34e17096d92cc4e47556a5", + "strip_prefix": "grpc-swift-1.16.0/", + "build_file": "@@rules_swift+//third_party:com_github_grpc_grpc_swift/BUILD.overlay" + } + }, + "com_github_apple_swift_docc_symbolkit": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-docc-symbolkit/archive/refs/tags/swift-5.10-RELEASE.tar.gz" + ], + "sha256": "de1d4b6940468ddb53b89df7aa1a81323b9712775b0e33e8254fa0f6f7469a97", + "strip_prefix": "swift-docc-symbolkit-swift-5.10-RELEASE", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_docc_symbolkit/BUILD.overlay" + } + }, + "com_github_apple_swift_nio": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio/archive/2.42.0.tar.gz" + ], + "sha256": "e3304bc3fb53aea74a3e54bd005ede11f6dc357117d9b1db642d03aea87194a0", + "strip_prefix": "swift-nio-2.42.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio/BUILD.overlay" + } + }, + "com_github_apple_swift_nio_http2": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio-http2/archive/1.26.0.tar.gz" + ], + "sha256": "f0edfc9d6a7be1d587e5b403f2d04264bdfae59aac1d74f7d974a9022c6d2b25", + "strip_prefix": "swift-nio-http2-1.26.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_http2/BUILD.overlay" + } + }, + "com_github_apple_swift_nio_transport_services": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio-transport-services/archive/1.15.0.tar.gz" + ], + "sha256": "f3498dafa633751a52b9b7f741f7ac30c42bcbeb3b9edca6d447e0da8e693262", + "strip_prefix": "swift-nio-transport-services-1.15.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_transport_services/BUILD.overlay" + } + }, + "com_github_apple_swift_nio_extras": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio-extras/archive/1.4.0.tar.gz" + ], + "sha256": "4684b52951d9d9937bb3e8ccd6b5daedd777021ef2519ea2f18c4c922843b52b", + "strip_prefix": "swift-nio-extras-1.4.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_extras/BUILD.overlay" + } + }, + "com_github_apple_swift_log": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-log/archive/1.4.4.tar.gz" + ], + "sha256": "48fe66426c784c0c20031f15dc17faf9f4c9037c192bfac2f643f65cb2321ba0", + "strip_prefix": "swift-log-1.4.4/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_log/BUILD.overlay" + } + }, + "com_github_apple_swift_nio_ssl": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio-ssl/archive/2.23.0.tar.gz" + ], + "sha256": "4787c63f61dd04d99e498adc3d1a628193387e41efddf8de19b8db04544d016d", + "strip_prefix": "swift-nio-ssl-2.23.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_ssl/BUILD.overlay" + } + }, + "com_github_apple_swift_collections": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-collections/archive/1.0.4.tar.gz" + ], + "sha256": "d9e4c8a91c60fb9c92a04caccbb10ded42f4cb47b26a212bc6b39cc390a4b096", + "strip_prefix": "swift-collections-1.0.4/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_collections/BUILD.overlay" + } + }, + "com_github_apple_swift_atomics": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-atomics/archive/1.1.0.tar.gz" + ], + "sha256": "1bee7f469f7e8dc49f11cfa4da07182fbc79eab000ec2c17bfdce468c5d276fb", + "strip_prefix": "swift-atomics-1.1.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_atomics/BUILD.overlay" + } + }, + "build_bazel_rules_swift_index_import": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "build_file": "@@rules_swift+//third_party:build_bazel_rules_swift_index_import/BUILD.overlay", + "canonical_id": "index-import-5.8", + "urls": [ + "https://github.com/MobileNativeFoundation/index-import/releases/download/5.8.0.1/index-import.tar.gz" + ], + "sha256": "28c1ffa39d99e74ed70623899b207b41f79214c498c603915aef55972a851a15" + } + }, + "build_bazel_rules_swift_local_config": { + "repoRuleId": "@@rules_swift+//swift/internal:swift_autoconfiguration.bzl%swift_autoconfiguration", + "attributes": {} + } + }, + "recordedRepoMappingEntries": [ + [ + "rules_swift+", + "bazel_tools", + "bazel_tools" + ] + ] + } } } } diff --git a/async/async.zig b/async/async.zig index 5cee9947..23aab5af 100644 --- a/async/async.zig +++ b/async/async.zig @@ -219,6 +219,50 @@ pub const AsyncThread = struct { return try aio.run(&executor_, mainFunc, .{}, mainStack); } + + pub fn mainWithArgs(allocator: std.mem.Allocator, comptime mainFunc: anytype, args: anytype) !stdx.meta.FnResultNoError(mainFunc) { + if (xev.dynamic) try xev.detect(); + var thread_pool = XevThreadPool.init(.{}); + defer { + thread_pool.shutdown(); + thread_pool.deinit(); + } + + var loop = try xev.Loop.init(.{ + .thread_pool = &thread_pool, + }); + defer loop.deinit(); + + var executor_ = aio.Executor.init(&loop); + + var async_notifier = try xev.Async.init(); + defer async_notifier.deinit(); + + var waiters_queue: threading.WaiterQueue = undefined; + waiters_queue.init(); + + var stack_allocator = stack.StackAllocator.init(allocator); + defer stack_allocator.deinit(); + + var asyncThread: AsyncThread = .{ + .executor = &executor_, + .stack_allocator = &stack_allocator, + .loop = &loop, + .thread_pool = &thread_pool, + .async_notifier = &async_notifier, + .waiters_queue = &waiters_queue, + }; + AsyncThread.current = &asyncThread; + + var c2: xev.Completion = undefined; + async_notifier.wait(AsyncThread.current.loop, &c2, AsyncThread, AsyncThread.current, &AsyncThread.wakerCallback); + + // allocate the main coroutine stack, on the current thread's stack! + var mainStackData: stack.Stack.Data = undefined; + const mainStack = stack.Stack.init(&mainStackData); + + return try aio.run(&executor_, mainFunc, args, mainStack); + } }; pub fn getStdIn() !File { diff --git a/examples/MODULE.bazel b/examples/MODULE.bazel index 2fcb0573..6a60ea53 100644 --- a/examples/MODULE.bazel +++ b/examples/MODULE.bazel @@ -4,7 +4,7 @@ bazel_dep(name = "bazel_skylib", version = "1.7.1") bazel_dep(name = "rules_zig", version = "20250314.0-b9739c6") bazel_dep(name = "platforms", version = "0.0.11") bazel_dep(name = "zml", version = "0.1.0") -bazel_dep(name = "aspect_bazel_lib", version = "2.11.0") +bazel_dep(name = "aspect_bazel_lib", version = "2.14.0") bazel_dep(name = "rules_oci", version = "2.0.0") non_module_deps = use_extension("//:third_party/non_module_deps.bzl", "non_module_deps") diff --git a/examples/MODULE.bazel.lock b/examples/MODULE.bazel.lock index 4c09f13c..060ab953 100644 --- a/examples/MODULE.bazel.lock +++ b/examples/MODULE.bazel.lock @@ -15,7 +15,6 @@ "https://bazel-registry.zml.ai/modules/apple_support/1.15.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/apple_support/1.17.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.10.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.11.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.14.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.7.2/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.8.1/MODULE.bazel": "not found", @@ -84,7 +83,7 @@ "https://bazel-registry.zml.ai/modules/re2/2024-07-02.bcr.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/re2/2024-07-02/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_android/0.1.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_apple/3.17.0/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/rules_apple/3.20.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.0.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.0.10/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_cc/0.0.13/MODULE.bazel": "not found", @@ -152,7 +151,7 @@ "https://bazel-registry.zml.ai/modules/rules_shell/0.2.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_shell/0.3.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_swift/1.16.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_swift/2.1.1/MODULE.bazel": "not found", + "https://bazel-registry.zml.ai/modules/rules_swift/2.4.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/rules_uv/0.53.0/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.5.1/MODULE.bazel": "not found", "https://bazel-registry.zml.ai/modules/stardoc/0.5.3/MODULE.bazel": "not found", @@ -187,7 +186,6 @@ "https://bcr.bazel.build/modules/apple_support/1.17.1/MODULE.bazel": "655c922ab1209978a94ef6ca7d9d43e940cd97d9c172fb55f94d91ac53f8610b", "https://bcr.bazel.build/modules/apple_support/1.17.1/source.json": "6b2b8c74d14e8d485528a938e44bdb72a5ba17632b9e14ef6e68a5ee96c8347f", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.10.0/MODULE.bazel": "ae12288421b12faeb09bc4cfb3c05945ce66df5c55fca8a2f2f66ae9f21c7acb", - "https://bcr.bazel.build/modules/aspect_bazel_lib/2.11.0/MODULE.bazel": "cb1ba9f9999ed0bc08600c221f532c1ddd8d217686b32ba7d45b0713b5131452", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.14.0/MODULE.bazel": "2b31ffcc9bdc8295b2167e07a757dbbc9ac8906e7028e5170a3708cecaac119f", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.14.0/source.json": "0cf1826853b0bef8b5cd19c0610d717500f5521aa2b38b72b2ec302ac5e7526c", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.7.2/MODULE.bazel": "780d1a6522b28f5edb7ea09630748720721dfe27690d65a2d33aa7509de77e07", @@ -273,8 +271,8 @@ "https://bcr.bazel.build/modules/re2/2024-07-02/MODULE.bazel": "0eadc4395959969297cbcf31a249ff457f2f1d456228c67719480205aa306daa", "https://bcr.bazel.build/modules/rules_android/0.1.1/MODULE.bazel": "48809ab0091b07ad0182defb787c4c5328bd3a278938415c00a7b69b50c4d3a8", "https://bcr.bazel.build/modules/rules_android/0.1.1/source.json": "e6986b41626ee10bdc864937ffb6d6bf275bb5b9c65120e6137d56e6331f089e", - "https://bcr.bazel.build/modules/rules_apple/3.17.0/MODULE.bazel": "39ee3d15edbf09f536a52c42b4297fe1f08a239e28964555f578f301159033f3", - "https://bcr.bazel.build/modules/rules_apple/3.17.0/source.json": "57af375068c7292d88062b2a3c2cae546f6e262f653f03388b5d73207f7482e3", + "https://bcr.bazel.build/modules/rules_apple/3.20.1/MODULE.bazel": "d9049cabd0becdf8d26377927dda874e2febc803401269814776f232fde91b7c", + "https://bcr.bazel.build/modules/rules_apple/3.20.1/source.json": "7870121042c518ca6109fa16fb070b4dd6762f42aefdd9ddcc259194ca6627cd", "https://bcr.bazel.build/modules/rules_cc/0.0.1/MODULE.bazel": "cb2aa0747f84c6c3a78dad4e2049c154f08ab9d166b1273835a8174940365647", "https://bcr.bazel.build/modules/rules_cc/0.0.10/MODULE.bazel": "ec1705118f7eaedd6e118508d3d26deba2a4e76476ada7e0e3965211be012002", "https://bcr.bazel.build/modules/rules_cc/0.0.13/MODULE.bazel": "0e8529ed7b323dad0775ff924d2ae5af7640b23553dfcd4d34344c7e7a867191", @@ -356,8 +354,8 @@ "https://bcr.bazel.build/modules/rules_shell/0.3.0/MODULE.bazel": "de4402cd12f4cc8fda2354fce179fdb068c0b9ca1ec2d2b17b3e21b24c1a937b", "https://bcr.bazel.build/modules/rules_shell/0.3.0/source.json": "c55ed591aa5009401ddf80ded9762ac32c358d2517ee7820be981e2de9756cf3", "https://bcr.bazel.build/modules/rules_swift/1.16.0/MODULE.bazel": "4a09f199545a60d09895e8281362b1ff3bb08bbde69c6fc87aff5b92fcc916ca", - "https://bcr.bazel.build/modules/rules_swift/2.1.1/MODULE.bazel": "494900a80f944fc7aa61500c2073d9729dff0b764f0e89b824eb746959bc1046", - "https://bcr.bazel.build/modules/rules_swift/2.1.1/source.json": "40fc69dfaac64deddbb75bd99cdac55f4427d9ca0afbe408576a65428427a186", + "https://bcr.bazel.build/modules/rules_swift/2.4.0/MODULE.bazel": "1639617eb1ede28d774d967a738b4a68b0accb40650beadb57c21846beab5efd", + "https://bcr.bazel.build/modules/rules_swift/2.4.0/source.json": "a6577f57f9febbdc015a01f2a8f3487422032f134d6c61d18ed8e8ca3b9acc7c", "https://bcr.bazel.build/modules/rules_uv/0.53.0/MODULE.bazel": "795c21aaad2574f8cb8163a03808ef4a4a33c0b0a81c0749ca489563561b83c1", "https://bcr.bazel.build/modules/rules_uv/0.53.0/source.json": "bd665e54519407989760844261e02b184ea4f141e65e399f75cd6a3ebf02b649", "https://bcr.bazel.build/modules/stardoc/0.5.1/MODULE.bazel": "1a05d92974d0c122f5ccf09291442580317cdd859f07a8655f1db9a60374f9f8", @@ -387,7 +385,7 @@ "@@apple_support+//crosstool:setup.bzl%apple_cc_configure_extension": { "general": { "bzlTransitiveDigest": "JJV/C5bhYKZip41t/GGZvqIMmmrcwr0Ydybkn97uXxQ=", - "usagesDigest": "wD+r6kULzwkrPbJnhFpRl2hIsJZIwSSaeEUj3z6v0i4=", + "usagesDigest": "ai1Q9Iw9PMxCbfOYikwNfCbFYAEzeR5Pirndq3g0LFo=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, @@ -2508,6 +2506,160 @@ ] ] } + }, + "@@rules_swift+//swift:extensions.bzl%non_module_deps": { + "general": { + "bzlTransitiveDigest": "3iOPDsadEKzPUpfznfK5UYw/yuGpZzq7zrjrB/oEo6g=", + "usagesDigest": "9w18ec4dj90mX/JqpR2tBU2YVc1GU9yNxi/d7q6lLVA=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "com_github_apple_swift_protobuf": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-protobuf/archive/1.20.2.tar.gz" + ], + "sha256": "3fb50bd4d293337f202d917b6ada22f9548a0a0aed9d9a4d791e6fbd8a246ebb", + "strip_prefix": "swift-protobuf-1.20.2/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_protobuf/BUILD.overlay" + } + }, + "com_github_grpc_grpc_swift": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/grpc/grpc-swift/archive/1.16.0.tar.gz" + ], + "sha256": "58b60431d0064969f9679411264b82e40a217ae6bd34e17096d92cc4e47556a5", + "strip_prefix": "grpc-swift-1.16.0/", + "build_file": "@@rules_swift+//third_party:com_github_grpc_grpc_swift/BUILD.overlay" + } + }, + "com_github_apple_swift_docc_symbolkit": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-docc-symbolkit/archive/refs/tags/swift-5.10-RELEASE.tar.gz" + ], + "sha256": "de1d4b6940468ddb53b89df7aa1a81323b9712775b0e33e8254fa0f6f7469a97", + "strip_prefix": "swift-docc-symbolkit-swift-5.10-RELEASE", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_docc_symbolkit/BUILD.overlay" + } + }, + "com_github_apple_swift_nio": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio/archive/2.42.0.tar.gz" + ], + "sha256": "e3304bc3fb53aea74a3e54bd005ede11f6dc357117d9b1db642d03aea87194a0", + "strip_prefix": "swift-nio-2.42.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio/BUILD.overlay" + } + }, + "com_github_apple_swift_nio_http2": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio-http2/archive/1.26.0.tar.gz" + ], + "sha256": "f0edfc9d6a7be1d587e5b403f2d04264bdfae59aac1d74f7d974a9022c6d2b25", + "strip_prefix": "swift-nio-http2-1.26.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_http2/BUILD.overlay" + } + }, + "com_github_apple_swift_nio_transport_services": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio-transport-services/archive/1.15.0.tar.gz" + ], + "sha256": "f3498dafa633751a52b9b7f741f7ac30c42bcbeb3b9edca6d447e0da8e693262", + "strip_prefix": "swift-nio-transport-services-1.15.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_transport_services/BUILD.overlay" + } + }, + "com_github_apple_swift_nio_extras": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio-extras/archive/1.4.0.tar.gz" + ], + "sha256": "4684b52951d9d9937bb3e8ccd6b5daedd777021ef2519ea2f18c4c922843b52b", + "strip_prefix": "swift-nio-extras-1.4.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_extras/BUILD.overlay" + } + }, + "com_github_apple_swift_log": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-log/archive/1.4.4.tar.gz" + ], + "sha256": "48fe66426c784c0c20031f15dc17faf9f4c9037c192bfac2f643f65cb2321ba0", + "strip_prefix": "swift-log-1.4.4/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_log/BUILD.overlay" + } + }, + "com_github_apple_swift_nio_ssl": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio-ssl/archive/2.23.0.tar.gz" + ], + "sha256": "4787c63f61dd04d99e498adc3d1a628193387e41efddf8de19b8db04544d016d", + "strip_prefix": "swift-nio-ssl-2.23.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_ssl/BUILD.overlay" + } + }, + "com_github_apple_swift_collections": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-collections/archive/1.0.4.tar.gz" + ], + "sha256": "d9e4c8a91c60fb9c92a04caccbb10ded42f4cb47b26a212bc6b39cc390a4b096", + "strip_prefix": "swift-collections-1.0.4/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_collections/BUILD.overlay" + } + }, + "com_github_apple_swift_atomics": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-atomics/archive/1.1.0.tar.gz" + ], + "sha256": "1bee7f469f7e8dc49f11cfa4da07182fbc79eab000ec2c17bfdce468c5d276fb", + "strip_prefix": "swift-atomics-1.1.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_atomics/BUILD.overlay" + } + }, + "build_bazel_rules_swift_index_import": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "build_file": "@@rules_swift+//third_party:build_bazel_rules_swift_index_import/BUILD.overlay", + "canonical_id": "index-import-5.8", + "urls": [ + "https://github.com/MobileNativeFoundation/index-import/releases/download/5.8.0.1/index-import.tar.gz" + ], + "sha256": "28c1ffa39d99e74ed70623899b207b41f79214c498c603915aef55972a851a15" + } + }, + "build_bazel_rules_swift_local_config": { + "repoRuleId": "@@rules_swift+//swift/internal:swift_autoconfiguration.bzl%swift_autoconfiguration", + "attributes": {} + } + }, + "recordedRepoMappingEntries": [ + [ + "rules_swift+", + "bazel_tools", + "bazel_tools" + ] + ] + } } } } diff --git a/examples/custom_call/BUILD b/examples/custom_call/BUILD new file mode 100644 index 00000000..bc605442 --- /dev/null +++ b/examples/custom_call/BUILD @@ -0,0 +1,10 @@ +load("@zml//bazel:zig.bzl", "zig_cc_binary") + +zig_cc_binary( + name = "custom_call", + main = "main.zig", + deps = [ + "@zml//async", + "@zml//zml", + ], +) diff --git a/examples/custom_call/main.zig b/examples/custom_call/main.zig new file mode 100644 index 00000000..099c6e4e --- /dev/null +++ b/examples/custom_call/main.zig @@ -0,0 +1,208 @@ +const std = @import("std"); +const zml = @import("zml"); +const asynk = @import("async"); + +pub const std_options: std.Options = .{ + .log_level = .warn, + .logFn = asynk.logFn(std.log.defaultLog), +}; + +const log = std.log.scoped(.@"examples/custom_call"); + +pub const AddOp = struct { + const Self = @This(); + + _platform: *const zml.Platform, + + pub fn beforeCustomCall(a: zml.Tensor, b: zml.Tensor) struct { zml.Tensor, zml.Tensor, zml.Tensor } { + return .{ a, b, zml.Tensor.scalar(0, .f32) }; + } + + pub fn call(self: *Self, a: zml.Buffer, b: zml.Buffer, result: zml.Buffer) !struct { zml.Buffer, zml.Buffer, zml.Buffer } { + _ = self; // autofix + // const a_ = try a.getValue(f32); + // const b_ = try b.getValue(f32); + + // log.warn("{s}@{*} a value: {d}", .{ @typeName(Self), self, a_ }); + // log.warn("{s}@{*} b value: {d}", .{ @typeName(Self), self, b_ }); + + // const add_result = a_ + b_; + // try result.setValue(f32, add_result); + // log.warn("{s}@{*} result value: {d}", .{ @typeName(Self), self, try result.getValue(f32) }); + + return .{ a, b, result }; + } + + fn getPlatform(self: *Self) zml.Platform { + return self._platform.*; + } +}; + +pub const LogResultOp = struct { + const Self = @This(); + + _platform: *const zml.Platform, + + pub fn beforeCustomCall(value: zml.Tensor) zml.Tensor { + return value; + } + + pub fn call(self: *Self, value: zml.Buffer) !zml.Buffer { + _ = self; // autofix + // log.warn("{s}@{*} value: {d}", .{ @typeName(Self), self, try value.getValue(f32) }); + return value; + } + + fn getPlatform(self: *Self) zml.Platform { + return self._platform.*; + } +}; + +pub const LogValuesVoidOp = struct { + const Self = @This(); + + _platform: *const zml.Platform, + + pub fn call(self: *Self, result: zml.HostBuffer, a: zml.HostBuffer, b: zml.HostBuffer) !void { + _ = self; // autofix + log.info("LogValuesVoidOp mem result: {any}", .{result.items(f32)}); + log.info("LogValuesVoidOp mem a: {any}", .{a.items(f32)}); + log.info("LogValuesVoidOp mem b: {any}", .{b.items(f32)}); + } + + fn getPlatform(self: *Self) zml.Platform { + return self._platform.*; + } +}; + +/// Model definition +const Layer = struct { + // a = 40x128 bf16 ou f32 + pub fn forward(_: Layer, a: zml.Tensor, b: zml.Tensor) zml.Tensor { + var result = a.add(b); + // const results = zml.custom_call(AddOp, .{ a, b }); + // result = results[0].mul(results[1]); + result = zml.custom_call(LogResultOp, .{result}); + return result.add(a); + } +}; + +pub fn main() !void { + try asynk.AsyncThread.main(std.heap.c_allocator, asyncMain); +} + +pub fn asyncMain() !void { + // Short lived allocations + var gpa = std.heap.GeneralPurposeAllocator(.{}){}; + defer _ = gpa.deinit(); + const allocator = gpa.allocator(); + + // Arena allocator for BufferStore etc. + var arena_state = std.heap.ArenaAllocator.init(allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + + var context = try zml.Context.init(); + defer context.deinit(); + + const platform = context.autoPlatform(.{}); + context.printAvailablePlatforms(platform); + + const shape = zml.Shape.init(.{ 40, 128 }, .bf16); + + // We manually produce a BufferStore. You would not normally do that. + // A BufferStore is usually created by loading model data from a file. + const buffers: zml.aio.BufferStore.Buffers = .{}; + + // the actual BufferStore + const buffer_store: zml.aio.BufferStore = .{ + .arena = arena_state, + .buffers = buffers, + }; + + // A clone of our model, consisting of shapes. We only need shapes for compiling. + // We use the BufferStore to infer the shapes. + const model_shapes = try zml.aio.populateModel(Layer, allocator, buffer_store); + + // Start compiling. This uses the inferred shapes from the BufferStore. + // The shape of the input tensor, we have to pass in manually. + var compilation = try asynk.asyncc(zml.compileModel, .{ allocator, Layer.forward, model_shapes, .{ shape, shape }, platform }); + + // Produce a bufferized weights struct from the fake BufferStore. + // This is like the inferred shapes, but with actual values. + // We will need to send those to the computation device later. + var model_weights = try zml.aio.loadModelBuffers(Layer, model_shapes, buffer_store, arena, platform); + defer zml.aio.unloadBuffers(&model_weights); // for good practice + + // Wait for compilation to finish + const compiled = try compilation.awaitt(); + + // pass the model weights to the compiled module to create an executable module + var executable = compiled.prepare(model_weights).withExecutionContext(); + defer executable.deinit(); + + const add_op_ctx: AddOp = .{ ._platform = &platform }; + try executable.attach(&add_op_ctx); + const log_result_op_ctx: LogResultOp = .{ ._platform = &platform }; + try executable.attach(&log_result_op_ctx); + // const log_values_op_ctx: LogValuesVoidOp = .{ ._platform = &platform }; + // try executable.attach(&log_values_op_ctx); + + var rng = std.Random.DefaultPrng.init(0); + const random = rng.random(); + + for (0..2) |i| { + log.warn("Iteration {d}", .{i}); + + // prepare input buffers + var input_buffer_a = try createRandomBuffer(allocator, platform, shape, random); + defer input_buffer_a.deinit(); + var input_buffer_b = try createRandomBuffer(allocator, platform, shape, random); + defer input_buffer_b.deinit(); + + var result: zml.Buffer = executable.call(.{ input_buffer_a, input_buffer_b }); + defer result.deinit(); + + // fetch the result to CPU memory + // const cpu_result = try result.toHostAlloc(arena); + // log.warn( + // "\nThe result of {d} + {d} = {d}\n", + // .{ &input_a, &input_b, cpu_result.items(f32) }, + // ); + + std.time.sleep(1 * std.time.ns_per_s); + } +} + +fn createRandomBuffer(allocator: std.mem.Allocator, platform: zml.Platform, shape: zml.Shape, random: std.Random) !zml.Buffer { + const data = try allocator.alloc(u8, shape.byteSize()); + defer allocator.free(data); + + switch (shape.dtype()) { + inline else => |v| { + const ZigType = v.toZigType(); + switch (comptime v.class()) { + .bool => unreachable, + .integer => { + for (std.mem.bytesAsSlice(ZigType, data)) |*e| e.* = random.int(ZigType); + }, + .float => { + const value = random.float(f64); + for (std.mem.bytesAsSlice(ZigType, data)) |*e| e.* = if (ZigType == f64) + value + else if (ZigType == f32) + @floatCast(value) + else if (ZigType == f16) + @floatCast(value) + else + @bitCast(random.int(std.meta.Int(.unsigned, @bitSizeOf(ZigType)))); + }, + .complex => unreachable, + } + }, + } + + var host_buffer = zml.HostBuffer.fromBytes(shape, data); + errdefer host_buffer.deinit(allocator); + return zml.Buffer.from(platform, host_buffer); +} diff --git a/examples/mnist/mnist.zig b/examples/mnist/mnist.zig index c49300a9..816dba39 100644 --- a/examples/mnist/mnist.zig +++ b/examples/mnist/mnist.zig @@ -2,14 +2,40 @@ const std = @import("std"); const asynk = @import("async"); const zml = @import("zml"); - +const pjrt = zml.pjrt; const log = std.log.scoped(.mnist); pub const std_options: std.Options = .{ .log_level = .info, + .log_scope_levels = &[_]std.log.ScopeLevel{ + .{ .scope = .@"zml/module", .level = .debug }, + }, .logFn = asynk.logFn(std.log.defaultLog), }; +pub const PrintOp = struct { + const Self = @This(); + + _platform: *const zml.Platform, + + pub fn beforeCustomCall(input: zml.Tensor, another_input: zml.Tensor) struct { zml.Tensor, zml.Tensor } { + log.info("Compiling PrintOp for target: {s}", .{@tagName(input.getContext().target())}); + return .{ input, another_input.add(another_input) }; + } + + pub fn call(self: *Self, input: zml.Buffer, another_input: zml.Buffer) !struct { zml.Buffer, zml.Buffer } { + var output = input; + log.info("Input 0: {any}", .{output.getMemory().kind(self._platform.pjrt_api)}); + output = try output.copyToMemory(self.getPlatform(), .host_pinned); + log.info("After copy 0: {any}", .{output.getMemory().kind(self._platform.pjrt_api)}); + return .{ try output.copyToMemory(self.getPlatform(), .device), another_input }; + } + + fn getPlatform(self: *Self) zml.Platform { + return self._platform.*; + } +}; + /// Model definition const Mnist = struct { fc1: Layer, @@ -26,13 +52,36 @@ const Mnist = struct { /// just two linear layers + relu activation pub fn forward(self: Mnist, input: zml.Tensor) zml.Tensor { - // std.log.info("Compiling for target: {s}", .{@tagName(input.getContext().target())}); + log.info("Compiling for target: {s}", .{@tagName(input.getContext().target())}); var x = input.flattenAll().convert(.f32); const layers: []const Layer = &.{ self.fc1, self.fc2 }; for (layers) |layer| { x = zml.call(layer, .forward, .{x}); } - return x.argMax(0).indices.convert(.u8); + return relu(x).argMax(0).indices.convert(.u8); + } + + pub fn relu(input: zml.Tensor) zml.Tensor { + return zml.ops.addHostCallback( + &reluCallback, + null, + &.{input}, + &.{input.shape()}, + .{}, + )[0]; + } + + fn reluCallback(_: ?*anyopaque, inputs: []const zml.HostBuffer, outputs: []const zml.HostBuffer) void { + const in = inputs[0].items(f32); + const out = outputs[0].mutItems(f32); + std.log.defaultLog(.info, .zml, "in: {*}[0..{}] = {e:.3}", .{ in.ptr, in.len, in }); + + var i: usize = 0; + for (in) |x| { + out[i] = @max(x, 0); + i += 1; + } + std.log.defaultLog(.info, .zml, "out: {*}[0..{}] = {e:.3}", .{ out.ptr, out.len, out }); } }; @@ -49,7 +98,7 @@ pub fn asyncMain() !void { // log.info("\n===========================\n== ZML MNIST Example ==\n===========================\n\n", .{}); - // // Auto-select platform + // Auto-select platform const platform = context.autoPlatform(.{}); context.printAvailablePlatforms(platform); @@ -85,7 +134,7 @@ pub fn asyncMain() !void { const compiled_mnist = try compilation.awaitt(); log.info("✅ Compiled model in {d}ms", .{start_time.read() / std.time.ns_per_ms}); - const mnist = compiled_mnist.prepare(model_weights); + const mnist = compiled_mnist.prepare(model_weights).withExecutionContext(); defer mnist.deinit(); log.info("✅ Weights transferred in {d}ms", .{start_time.read() / std.time.ns_per_ms}); @@ -98,6 +147,10 @@ pub fn asyncMain() !void { // inference - can be looped { + const print_op_ctx: PrintOp = .{ ._platform = &platform }; + try mnist.attach(&print_op_ctx); + std.debug.print("Attached print op to model: {}\n", .{print_op_ctx}); + const idx = rng.random().intRangeAtMost(u64, 0, 10000 - 1); var sample: [28 * 28]u8 align(16) = undefined; _ = try dataset.pread(&sample, 16 + (idx * 28 * 28)); diff --git a/pjrt/ffi.zig b/pjrt/ffi.zig index 39e7c248..13b3df1a 100644 --- a/pjrt/ffi.zig +++ b/pjrt/ffi.zig @@ -91,22 +91,20 @@ fn TransmuteMixin(comptime T: type, comptime InnerT: type) type { pub const Api = opaque { pub const inner = TransmuteMixin(Api, c.XLA_FFI_Api).to; - pub fn getStream(self: *const Api, context: ?*ExecutionContext) ApiError!*anyopaque { + pub fn stream(self: *const Api, context: ?*ExecutionContext) *Stream { var ret = pjrtStruct(c.XLA_FFI_Stream_Get_Args{ .ctx = if (context) |ctx| ctx.inner() else null, }); const result = self.inner().XLA_FFI_Stream_Get.?(&ret); - if (result) |ffi_error| { const err = Error.fromInner(ffi_error); defer err.destroy(self); log.err("[Api.getStream] {s}", .{err.getMessage(self)}); - // TODO(Corentin): Retrieve error code from Error when implemented in XLA. - return error.Unknown; + @panic("failed to get stream"); } - return ret.stream.?; + return @ptrCast(ret.stream.?); } pub fn allocateDeviceMemory(self: *const Api, context: ?*ExecutionContext, size: usize, alignment: usize) ApiError!*anyopaque { @@ -165,54 +163,92 @@ pub const ExecutionStage = enum(c.XLA_FFI_ExecutionStage) { pub const ExecutionContext = opaque { pub const inner = TransmuteMixin(ExecutionContext, c.XLA_FFI_ExecutionContext).to; - // pub fn attach(self: *ExecutionContext, api: *const Api, value: anytype) ApiError!void { - // // register type id ==> typeid - // const typename_ = "zml." ++ @typeName(@TypeOf(value)); + pub fn Context(comptime T: type) type { + return struct { + pub fn get(self: *const ExecutionContext, api: *const Api) ApiError!*T { + const type_id = getTypeId(@typeName(T)); + var ret = pjrtStruct(c.XLA_FFI_ExecutionContext_Get_Args{ + .ctx = @constCast(self.inner()), + .type_id = @constCast(&type_id.toCStruct()), + }); + const result = api.inner().XLA_FFI_ExecutionContext_Get.?(&ret); - // var ret = pjrtStruct(c.XLA_FFI_ExecutionContext_Register_Args{ - // .ctx = self.inner(), - // .handler = @ptrCast(@alignCast(handler)), - // }); - // const result = api.inner().XLA_FFI_ExecutionContext_Register.?(&ret); + if (result) |ffi_error| { + const err = Error.fromInner(ffi_error); + defer err.destroy(api); + log.err("[ExecutionContext.get] {s}", .{err.getMessage(api)}); - // var ret = pjrtStruct(c.XLA_FFI_ExecutionContext_Register_Args{ - // .ctx = self.inner(), - // .handler = @ptrCast(@alignCast(handler)), - // }); - // const result = api.inner().XLA_FFI_ExecutionContext_Register.?(&ret); + // TODO(Corentin): Retrieve error code from Error when implemented in XLA. + return error.Unknown; + } - // if (result) |ffi_error| { - // const err = Error.fromInner(ffi_error); - // defer err.destroy(api); - // log.err("[ExecutionContext.register] {s}", .{err.getMessage(api)}); + if (ret.data == null) return error.NotFound; + return @ptrCast(@alignCast(ret.data.?)); + } + }; + } - // // TODO(Corentin): Retrieve error code from Error when implemented in XLA. - // return error.Unknown; - // } - // } - - pub fn get(self: *ExecutionContext, api: *const Api, type_id: *TypeId) ApiError!*anyopaque { - var ret = pjrtStruct(c.XLA_FFI_ExecutionContext_Get_Args{ - .ctx = self.inner(), - .type_id = @ptrCast(@alignCast(type_id)), + pub fn getDeviceOrdinal(self: *const ExecutionContext, api: *const Api) ApiError!i32 { + var ret = pjrtStruct(c.XLA_FFI_DeviceOrdinal_Get_Args{ + .ctx = @constCast(self.inner()), }); - const result = api.inner().XLA_FFI_ExecutionContext_Get.?(&ret); + const result = api.inner().XLA_FFI_DeviceOrdinal_Get.?(&ret); if (result) |ffi_error| { const err = Error.fromInner(ffi_error); defer err.destroy(api); - log.err("[ExecutionContext.get] {s}", .{err.getMessage(api)}); + log.err("[ExecutionContext.getDeviceOrdinal] {s}", .{err.getMessage(api)}); // TODO(Corentin): Retrieve error code from Error when implemented in XLA. return error.Unknown; } - return ret.data.?; + return ret.device_ordinal; } - // TODO getDeviceOrdinal() + pub fn scheduleTask(self: *const ExecutionContext, api: *const Api, task: *const Task, data: *anyopaque) ApiError!void { + var ret = pjrtStruct(c.XLA_FFI_ThreadPool_Schedule_Args{ + .ctx = @constCast(self.inner()), + .task = @ptrCast(@alignCast(task)), + .data = @ptrCast(@alignCast(data)), + }); + + const result = api.inner().XLA_FFI_ThreadPool_Schedule.?(&ret); + + if (result) |ffi_error| { + const err = Error.fromInner(ffi_error); + defer err.destroy(api); + std.debug.print("error: {any} \n", .{err}); + log.err("[ExecutionContext.get] {s}", .{err.getMessage(api)}); + + // TODO(Corentin): Retrieve error code from Error when implemented in XLA. + return error.Unknown; + } + } + + fn getTypeId(type_name: []const u8) TypeId { + const id: i64 = @bitCast(std.hash.Fnv1a_64.hash(type_name)); + + return .{ + .type_id = id, + }; + } }; +const TypeId = extern struct { + type_id: i64, + + pub fn toCStruct(self: TypeId) c.XLA_FFI_TypeId { + return c.XLA_FFI_TypeId{ + .type_id = self.type_id, + }; + } +}; + +const Task = fn (*anyopaque) void; + +pub const Stream = opaque {}; + const ByteSpan = extern struct { ptr: [*]const u8, len: usize, @@ -222,10 +258,6 @@ const ByteSpan = extern struct { } }; -pub const TypeId = extern struct { - type_id: i64, -}; - pub const DataType = enum(c.XLA_FFI_DataType) { invalid = c.XLA_FFI_DataType_INVALID, pred = c.XLA_FFI_DataType_PRED, @@ -253,11 +285,19 @@ pub const DataType = enum(c.XLA_FFI_DataType) { f8e4m3fnuz = c.XLA_FFI_DataType_F8E4M3FNUZ, }; +pub const DevicePtr = enum(usize) { + _, + + pub fn asPtr(ptr: DevicePtr) [*]u8 { + return @ptrFromInt(@intFromEnum(ptr)); + } +}; + pub const Buffer = extern struct { struct_size: usize, extension_start: ?*c.XLA_FFI_Extension_Base, dtype: DataType, - data: [*]u8, + data: DevicePtr, rank: u64, _dims: [*]const i64, @@ -274,7 +314,7 @@ pub const Buffer = extern struct { _ = fmt; _ = options; - try writer.print("FfiBuffer({d}, .{s})@0x{x}", .{ buffer.dims(), @tagName(buffer.dtype), @intFromPtr(buffer.data) }); + try writer.print("FfiBuffer({d}, .{s})@0x{x}", .{ buffer.dims(), @tagName(buffer.dtype), @intFromPtr(buffer.data.asPtr()) }); } }; @@ -289,9 +329,8 @@ pub const Args = extern struct { buffer = c.XLA_FFI_ArgType_BUFFER, }; - pub fn get(self: Args, i: usize) *const Buffer { - std.debug.assert(self.types[0..self.len][i] == .buffer); - return self.ptr[0..self.len][i]; + pub fn buffers(self: Args) []*const Buffer { + return self.ptr[0..self.len]; } }; @@ -306,9 +345,8 @@ pub const Rets = extern struct { buffer = c.XLA_FFI_RetType_BUFFER, }; - pub fn get(self: Rets, i: usize) *const Buffer { - std.debug.assert(self.types[0..self.len][i] == .buffer); - return self.ptr[0..self.len][i]; + pub fn buffers(self: Rets) []*const Buffer { + return self.ptr[0..self.len]; } }; @@ -348,6 +386,11 @@ pub const Attrs = extern struct { data: [*]const u8, }; + pub fn slice(self: Array, T: type) []const T { + const ptr: [*]const T = @alignCast(@ptrCast(self.data)); + return ptr[0..self.len]; + } + pub fn getByIndex(self: Attrs, comptime attr_type: AttrType, index: usize) ?*const @FieldType(Attr, @tagName(attr_type)) { const attr = self.ptr[0..self.len][index]; const actual_type = self.types[index]; @@ -438,7 +481,7 @@ pub const Error = opaque { pub const inner = TransmuteMixin(Error, c.XLA_FFI_Error).to; pub const fromInner = TransmuteMixin(Error, c.XLA_FFI_Error).from; - pub fn create(api: *const Api, error_code: ErrorCode, message: [:0]const u8) *Error { + pub fn create(api: *const Api, error_code: ErrorCode, message: []const u8) *Error { var ret = pjrtStruct(c.XLA_FFI_Error_Create_Args{ .message = message.ptr, .errc = @intFromEnum(error_code), diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 9dbd5aac..4a7b7339 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -31,7 +31,7 @@ fn pjrtStructSize(comptime T: type) usize { return @field(c, typedef_name ++ "_STRUCT_SIZE"); } -inline fn pjrtStruct(v: anytype) @TypeOf(v) { +pub inline fn pjrtStruct(v: anytype) @TypeOf(v) { var ret = v; ret.struct_size = pjrtStructSize(@TypeOf(v)); return ret; @@ -160,9 +160,14 @@ pub const Api = struct { return state.str; } - pub fn customCallRegistry(api: *const Api) ?CustomCallRegistry { + pub fn createExecuteContext(api: *const Api) ApiError!*ExecuteContext { + const ret = try api.call(.PJRT_ExecuteContext_Create, .{}); + return @ptrCast(ret.context.?); + } + + pub fn ffi(api: *const Api) ?FFI { if (api.lookupExtension(c.PJRT_FFI_Extension, c.PJRT_Extension_Type_FFI)) |ext| { - return .{ .inner = ext.register_handler.? }; + return .{ .inner = ext }; } return null; } @@ -410,6 +415,7 @@ pub const Client = opaque { element_type: BufferType, layout: MemoryLayout, device: *const Device, + memory_kind: ?Memory.Kind, on_delete_callback: *const fn (device_buffer_ptr: ?*anyopaque, ctx: ?*anyopaque) callconv(.C) void = &struct { fn call(_: ?*anyopaque, _: ?*anyopaque) callconv(.C) void {} }.call, @@ -419,6 +425,16 @@ pub const Client = opaque { pub fn createViewOfDeviceBuffer(self: *const Client, api: *const Api, args: CreateViewOfDeviceBufferArgs) ApiError!*Buffer { const layout = args.layout.toCStruct(); + var memory: ?*const Memory = null; + + if (args.memory_kind) |kind| { + for (args.device.addressableMemories(api)) |m| { + if (m.kind(api) == kind) { + memory = m; + } + } + } + const ret = try api.call(.PJRT_Client_CreateViewOfDeviceBuffer, .{ .client = self.inner(), .device_buffer_ptr = @ptrCast(@constCast(args.data)), @@ -427,6 +443,7 @@ pub const Client = opaque { .element_type = @intFromEnum(args.element_type), .layout = @ptrCast(@constCast(&layout)), .device = @ptrCast(@constCast(args.device)), + .memory = if (memory) |m| @ptrCast(@constCast(m)) else null, .on_delete_callback = args.on_delete_callback, .on_delete_callback_arg = args.on_delete_callback_arg, .stream = if (args.stream) |stream| stream else 0, @@ -444,20 +461,21 @@ pub const Client = opaque { return &.{}; } - pub fn dmaMap(self: *const Client, api: *const Api, data: []const u8) ApiError!*Buffer { - const ret = try api.call(.PJRT_Client_DMA_Map, .{ + pub fn dmaMap(self: *const Client, api: *const Api, data: []const u8) ApiError!void { + const args: Api.CallFnArgType(.PJRT_Client_DmaMap) = .{ .client = self.inner(), .data = @ptrCast(@constCast(data.ptr)), .size = @intCast(data.len), - }); - return @ptrCast(ret.buffer.?); + }; + std.debug.print("DMA map {any} \n", .{args}); + _ = try api.call(.PJRT_Client_DmaMap, args); } - pub fn dmaUnmap(self: *const Client, api: *const Api, data: []const u8) void { - _ = api.call(.PJRT_Client_DMA_Unmap, .{ + pub fn dmaUnmap(self: *const Client, api: *const Api, data: []const u8) ApiError!void { + _ = try api.call(.PJRT_Client_DmaUnmap, .{ .client = self.inner(), .data = @ptrCast(@constCast(data.ptr)), - }) catch unreachable; + }); } pub const CreateBuffersForAsyncHostToDeviceArgs = struct { @@ -502,6 +520,16 @@ pub const Device = opaque { }) catch unreachable; return @intCast(ret.local_hardware_id); } + + pub fn addressableMemories(self: *const Device, api: *const Api) []*const Memory { + const ret = api.call(.PJRT_Device_AddressableMemories, .{ + .device = self.inner(), + }) catch unreachable; + if (ret.memories) |memories| { + return @constCast(@ptrCast(memories[0..ret.num_memories])); + } + return &.{}; + } }; pub const DeviceDescription = opaque { @@ -557,6 +585,14 @@ pub const SerializeResult = struct { } }; +pub const ExecuteContext = opaque { + pub fn deinit(self: *ExecuteContext, api: *const Api) void { + _ = api.call(.PJRT_ExecuteContext_Destroy, .{ + .context = @ptrCast(self), + }) catch {}; + } +}; + pub const Executable = opaque { const inner = InnerMixin(c.PJRT_Executable).inner; @@ -623,6 +659,7 @@ pub const LoadedExecutable = opaque { results: []const [*]*Buffer, events: []?*Event, non_donatable_input_indices: []const i64 = &.{}, + context: ?*ExecuteContext, }; pub fn execute(self: *const LoadedExecutable, api: *const Api, args: ExecuteArgs) ApiError!void { var options = pjrtStruct(c.PJRT_ExecuteOptions{ @@ -633,6 +670,7 @@ pub const LoadedExecutable = opaque { .launch_id = 0, .non_donatable_input_indices = @ptrCast(args.non_donatable_input_indices.ptr), .num_non_donatable_input_indices = args.non_donatable_input_indices.len, + .context = @ptrCast(args.context), }); _ = try api.call(.PJRT_LoadedExecutable_Execute, .{ .executable = self.inner(), @@ -843,7 +881,7 @@ pub const Buffer = opaque { return @ptrCast(ret.event); } - pub fn copyToMemory(self: *const Buffer, api: *const Api, dst_memory: *const Memory) ApiError!?*Buffer { + pub fn copyToMemory(self: *const Buffer, api: *const Api, dst_memory: *const Memory) ApiError!*Buffer { const ret = try api.call(.PJRT_Buffer_CopyToMemory, .{ .buffer = self.inner(), .dst_memory = @ptrCast(@constCast(dst_memory)), @@ -925,8 +963,8 @@ pub const Memory = opaque { pub fn kind(self: *const Memory, api: *const Api) Kind { const ret = api.call(.PJRT_Memory_Kind, .{ .memory = self.inner() }) catch unreachable; - const kind_ = ret.kind orelse unreachable[0..ret.kind_size]; - return std.meta.stringToEnum(Kind, kind_) orelse unreachable; + const kind_ = ret.kind orelse unreachable; + return std.meta.stringToEnum(Kind, kind_[0..ret.kind_size]) orelse unreachable; } pub fn kindId(self: *const Memory, api: *const Api) u32 { @@ -1037,21 +1075,6 @@ pub const AsyncHostToDeviceTransferManager = opaque { } }; -pub const ExecutionContext = opaque { - const inner = InnerMixin(c.PJRT_ExecutionContext).inner; - - pub fn init(api: *const Api) ApiError!*ExecutionContext { - const ret = try api.call(.PJRT_ExecutionContext_Create, .{}); - return @ptrCast(ret.context.?); - } - - pub fn deinit(self: *ExecutionContext, api: *const Api) void { - _ = api.call(.PJRT_ExecutionContext_Destroy, .{ - .context = self.inner(), - }) catch unreachable; - } -}; - pub const NamedValue = extern struct { comptime { std.debug.assert(@sizeOf(NamedValue) == @sizeOf(c.PJRT_NamedValue)); @@ -1157,6 +1180,59 @@ pub const NamedValue = extern struct { } }; +pub const FFI = extern struct { + inner: *const c.PJRT_FFI, + + pub const UserData = extern struct { + type_id: i64, + user_data: *anyopaque, + + fn toCStruct(self: UserData) c.PJRT_FFI_UserData { + return .{ + .type_id = self.type_id, + .data = self.user_data, + }; + } + }; + + pub fn customCallRegistry(self: *const FFI) ?CustomCallRegistry { + return .{ .inner = self.inner.register_handler.? }; + } + + pub fn registerTypeId(self: *const FFI, api: *const Api, type_name: []const u8) ApiError!i64 { + var ret = pjrtStruct(c.PJRT_FFI_TypeID_Register_Args{ + .type_name = type_name.ptr, + .type_name_size = type_name.len, + .type_id = getTypeId(type_name), + }); + const result = self.inner.type_id_register.?(&ret); + if (result) |pjrt_c_error| { + const pjrt_error: *Error = @ptrCast(pjrt_c_error); + return pjrt_error.getCode(api).toApiError(); + } + + return ret.type_id; + } + + pub fn addUserData(self: *const FFI, api: *const Api, context: *ExecuteContext, user_data: UserData) ApiError!void { + var ret = pjrtStruct(c.PJRT_FFI_UserData_Add_Args{ + .context = @ptrCast(context), + .user_data = user_data.toCStruct(), + }); + const result = self.inner.user_data_add.?(&ret); + if (result) |pjrt_c_error| { + const pjrt_error: *Error = @ptrCast(pjrt_c_error); + log.err("[addUserData] {s}", .{pjrt_error.getMessage(api)}); + return pjrt_error.getCode(api).toApiError(); + } + } + + pub fn getTypeId(type_name: []const u8) i64 { + // Use FNV-1a 64-bit hash algorithm + return @bitCast(std.hash.Fnv1a_64.hash(type_name)); + } +}; + // todo : support all missing handlers available in GPU plugin extension: handler_instantiate, handler_prepare, handler_initialize // introduced by https://github.com/openxla/xla/commit/ef85a7bcc308313492ebc50295a8a08b4e51b8f5 pub const CustomCallRegistry = extern struct { diff --git a/runtimes/cpu/cpu.bzl b/runtimes/cpu/cpu.bzl index d3613802..9e0b0374 100644 --- a/runtimes/cpu/cpu.bzl +++ b/runtimes/cpu/cpu.bzl @@ -25,22 +25,22 @@ def _cpu_pjrt_plugin_impl(mctx): http_archive( name = "libpjrt_cpu_linux_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_LINUX, - sha256 = "1cda1325095c12bd0019838d28ee92d811ac478d22ed3c08020d5a0cd2d9f34a", - url = "https://github.com/zml/pjrt-artifacts/releases/download/v7.0.0/pjrt-cpu_linux-amd64.tar.gz", + sha256 = "a799573bc238bfbfa18c415002280902e4547f7e12fb09a9a7e73e8b35b58b9b", + url = "https://github.com/zml/pjrt-artifacts/releases/download/v8.0.0/pjrt-cpu_linux-amd64.tar.gz", ) http_archive( name = "libpjrt_cpu_darwin_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, - sha256 = "35af82d9e5c70d16ac15f4c18024a2dd5ed2faefc89940eafe3d5350d2cbd9e7", - url = "https://github.com/zml/pjrt-artifacts/releases/download/v7.0.0/pjrt-cpu_darwin-amd64.tar.gz", + sha256 = "31f6fcc1a5341ea7cd6eaf5c441189e5f6399a37dfafb4a45699edacf44c742c", + url = "https://github.com/zml/pjrt-artifacts/releases/download/v8.0.0/pjrt-cpu_darwin-amd64.tar.gz", ) http_archive( name = "libpjrt_cpu_darwin_arm64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, - sha256 = "da4deaf850d715997614768b2fc0283595ee8181133ab3243d65635e3439de69", - url = "https://github.com/zml/pjrt-artifacts/releases/download/v7.0.0/pjrt-cpu_darwin-arm64.tar.gz", + sha256 = "91a18c0fb2c82f28953b5134833415638422925099ee6498987d583f178588b5", + url = "https://github.com/zml/pjrt-artifacts/releases/download/v8.0.0/pjrt-cpu_darwin-arm64.tar.gz", ) return mctx.extension_metadata( diff --git a/runtimes/cuda/BUILD.bazel b/runtimes/cuda/BUILD.bazel index a7f8bf94..ed50ebf5 100644 --- a/runtimes/cuda/BUILD.bazel +++ b/runtimes/cuda/BUILD.bazel @@ -1,5 +1,11 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") load("@rules_zig//zig:defs.bzl", "zig_library") +exports_files( + ["libpjrt_cuda.so"], + visibility = ["//visibility:public"], +) + cc_library( name = "zmlxcuda_lib", srcs = ["zmlxcuda.c"], diff --git a/runtimes/cuda/cuda.bzl b/runtimes/cuda/cuda.bzl index 82d1e448..2c116ef0 100644 --- a/runtimes/cuda/cuda.bzl +++ b/runtimes/cuda/cuda.bzl @@ -202,8 +202,8 @@ def _cuda_impl(mctx): http_archive( name = "libpjrt_cuda", build_file = "libpjrt_cuda.BUILD.bazel", - url = "https://github.com/zml/pjrt-artifacts/releases/download/v7.0.0/pjrt-cuda_linux-amd64.tar.gz", - sha256 = "64029cd3d68118b166198e3246877ed706ed35eb732b1770a9bf530b5b0a8ab4", + url = "https://github.com/zml/pjrt-artifacts/releases/download/v8.0.0/pjrt-cuda_linux-amd64.tar.gz", + sha256 = "9e2d1741de8531274c4c2ffe77f3a82003591827294e2a2773cc09f33cc2137e", ) return mctx.extension_metadata( diff --git a/runtimes/cuda/cuda.zig b/runtimes/cuda/cuda.zig index bc749693..37a81480 100644 --- a/runtimes/cuda/cuda.zig +++ b/runtimes/cuda/cuda.zig @@ -62,6 +62,9 @@ pub fn load() !*const pjrt.Api { log.warn("Detected {s} in LD_LIBRARY_PATH. This can lead to undefined behaviors and crashes", .{nvidiaLibsPath}); } + _ = c.setenv("XLA_PJRT_GPU_HOST_MEMORY_LIMIT_GB", "8", 1); + _ = c.setenv("XLA_PJRT_GPU_HOST_MEMORY_PREALLOCATE", "true", 1); + // CUDA path has to be set _before_ loading the PJRT plugin. // See https://github.com/openxla/xla/issues/21428 try setupXlaGpuCudaDirFlag(); diff --git a/runtimes/rocm/rocm.bzl b/runtimes/rocm/rocm.bzl index c190a9ee..9d41c106 100644 --- a/runtimes/rocm/rocm.bzl +++ b/runtimes/rocm/rocm.bzl @@ -127,8 +127,8 @@ def _rocm_impl(mctx): http_archive( name = "libpjrt_rocm", build_file = "libpjrt_rocm.BUILD.bazel", - url = "https://github.com/zml/pjrt-artifacts/releases/download/v7.0.0/pjrt-rocm_linux-amd64.tar.gz", - sha256 = "13a2ced965a44a0e8ad0d752c8ac5aa99107b17b41bd850967d40b82e102ec50", + url = "https://github.com/zml/pjrt-artifacts/releases/download/v8.0.0/pjrt-rocm_linux-amd64.tar.gz", + sha256 = "4bdb7690f6e7ac0faa1c29299663269781c2b5d33fbe28c1ff4bc33ce9ec01ff", ) return mctx.extension_metadata( diff --git a/stdx/meta.zig b/stdx/meta.zig index 7aa90531..ee8d998d 100644 --- a/stdx/meta.zig +++ b/stdx/meta.zig @@ -164,6 +164,14 @@ pub fn FnResult(comptime func: anytype) type { return @typeInfo(@TypeOf(func)).@"fn".return_type orelse @compileError("anytype is not supported"); } +pub fn FnResultNoError(comptime func: anytype) type { + const ReturnT = @typeInfo(@TypeOf(func)).@"fn".return_type orelse @compileError("anytype is not supported"); + return switch (@typeInfo(ReturnT)) { + .error_union => |info| info.payload, + else => ReturnT, + }; +} + pub fn Head(Tuple: type) type { return switch (@typeInfo(Tuple)) { .@"struct" => |struct_info| { diff --git a/third_party/modules/xla/20250409.0-7167d71/MODULE.bazel b/third_party/modules/xla/20250409.0-7167d71/MODULE.bazel new file mode 100644 index 00000000..50cafb9e --- /dev/null +++ b/third_party/modules/xla/20250409.0-7167d71/MODULE.bazel @@ -0,0 +1,37 @@ +module( + name = "xla", + version = "20250409.0-7167d71", + compatibility_level = 1, +) + +bazel_dep(name = "platforms", version = "0.0.11") +bazel_dep(name = "bazel_skylib", version = "1.5.0") +bazel_dep(name = "rules_cc", version = "0.0.17") +bazel_dep(name = "rules_apple", version = "3.20.1", repo_name = "build_bazel_rules_apple") +bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "com_google_absl") +bazel_dep(name = "rules_python", version = "0.29.0") +bazel_dep(name = "rules_proto", version = "6.0.0-rc1") +bazel_dep(name = "rules_java", version = "7.3.2") +bazel_dep(name = "rules_pkg", version = "0.9.1") +bazel_dep(name = "zlib", version = "1.2.13") +bazel_dep(name = "re2", version = "2024-07-02.bcr.1", repo_name = "com_googlesource_code_re2") +bazel_dep(name = "rules_license", version = "0.0.8") + +tsl = use_extension("//:tsl.bzl", "tsl") +use_repo(tsl, "tsl", "python_version_repo") + +xla_workspace = use_extension("//:workspace.bzl", "xla_workspace") +use_repo( + xla_workspace, + "com_github_grpc_grpc", + "com_google_protobuf", + "local_config_cuda", + "local_config_remote_execution", + "local_config_rocm", + "local_config_tensorrt", + "llvm-raw", + "stablehlo", +) + +llvm_configure = use_repo_rule("@llvm-raw//utils/bazel:configure.bzl", "llvm_configure") +llvm_configure(name = "llvm-project") diff --git a/third_party/modules/xla/20250409.0-7167d71/overlay/MODULE.bazel b/third_party/modules/xla/20250409.0-7167d71/overlay/MODULE.bazel new file mode 120000 index 00000000..3ef3b743 --- /dev/null +++ b/third_party/modules/xla/20250409.0-7167d71/overlay/MODULE.bazel @@ -0,0 +1 @@ +/mnt/hugo/zml/third_party/modules/xla/20250409.0-7167d71/MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/xla/20250409.0-7167d71/overlay/tsl.bzl b/third_party/modules/xla/20250409.0-7167d71/overlay/tsl.bzl new file mode 100644 index 00000000..dc25c9d6 --- /dev/null +++ b/third_party/modules/xla/20250409.0-7167d71/overlay/tsl.bzl @@ -0,0 +1,19 @@ +load("//third_party:repo.bzl", "tf_vendored") +load("//third_party/py:python_init_repositories.bzl", "python_init_repositories") + +def _tsl_impl(mctx): + python_init_repositories( + requirements = { + "3.11": "//:requirements_lock_3_11.txt", + }, + ) + tf_vendored(name = "tsl", relpath = "third_party/tsl") + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = ["tsl"], + root_module_direct_dev_deps = [], + ) + +tsl = module_extension( + implementation = _tsl_impl, +) diff --git a/third_party/modules/xla/20250409.0-7167d71/overlay/workspace.bzl b/third_party/modules/xla/20250409.0-7167d71/overlay/workspace.bzl new file mode 100644 index 00000000..192e06f8 --- /dev/null +++ b/third_party/modules/xla/20250409.0-7167d71/overlay/workspace.bzl @@ -0,0 +1,60 @@ +load("//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls") +load("//third_party/gpus:cuda_configure.bzl", "cuda_configure") +load("//third_party/gpus:rocm_configure.bzl", "rocm_configure") +load("//third_party/llvm:workspace.bzl", llvm = "repo") +load("//third_party/pybind11_bazel:workspace.bzl", pybind11_bazel = "repo") +load("//third_party/stablehlo:workspace.bzl", stablehlo = "repo") +load("//third_party/tensorrt:tensorrt_configure.bzl", "tensorrt_configure") +load("//third_party/triton:workspace.bzl", triton = "repo") +load("//tools/toolchains/remote:configure.bzl", "remote_execution_configure") + +def _xla_workspace_impl(mctx): + cuda_configure(name = "local_config_cuda") + remote_execution_configure(name = "local_config_remote_execution") + rocm_configure(name = "local_config_rocm") + tensorrt_configure(name = "local_config_tensorrt") + pybind11_bazel() + triton() + llvm("llvm-raw") + stablehlo() + tf_http_archive( + name = "com_github_grpc_grpc", + sha256 = "b956598d8cbe168b5ee717b5dafa56563eb5201a947856a6688bbeac9cac4e1f", + strip_prefix = "grpc-b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd", + system_build_file = "//third_party/systemlibs:grpc.BUILD", + patch_file = [ + "//third_party/grpc:generate_cc_env_fix.patch", + "//third_party/grpc:register_go_toolchain.patch", + ], + system_link_files = { + "//third_party/systemlibs:BUILD.bazel": "bazel/BUILD.bazel", + "//third_party/systemlibs:grpc.BUILD": "src/compiler/BUILD", + "//third_party/systemlibs:grpc.bazel.grpc_deps.bzl": "bazel/grpc_deps.bzl", + "//third_party/systemlibs:grpc.bazel.grpc_extra_deps.bzl": "bazel/grpc_extra_deps.bzl", + "//third_party/systemlibs:grpc.bazel.cc_grpc_library.bzl": "bazel/cc_grpc_library.bzl", + "//third_party/systemlibs:grpc.bazel.generate_cc.bzl": "bazel/generate_cc.bzl", + "//third_party/systemlibs:grpc.bazel.protobuf.bzl": "bazel/protobuf.bzl", + }, + urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd.tar.gz"), + ) + tf_http_archive( + name = "com_google_protobuf", + patch_file = ["//third_party/protobuf:protobuf.patch"], + sha256 = "f66073dee0bc159157b0bd7f502d7d1ee0bc76b3c1eac9836927511bdc4b3fc1", + strip_prefix = "protobuf-3.21.9", + system_build_file = "//third_party/systemlibs:protobuf.BUILD", + system_link_files = { + "//third_party/systemlibs:protobuf.bzl": "protobuf.bzl", + "//third_party/systemlibs:protobuf_deps.bzl": "protobuf_deps.bzl", + }, + urls = tf_mirror_urls("https://github.com/protocolbuffers/protobuf/archive/v3.21.9.zip"), + ) + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = "all", + root_module_direct_dev_deps = [], + ) + +xla_workspace = module_extension( + implementation = _xla_workspace_impl, +) diff --git a/third_party/modules/xla/20250409.0-7167d71/patches/0001-bazel-migration-to-bazel-8.1.1.patch b/third_party/modules/xla/20250409.0-7167d71/patches/0001-bazel-migration-to-bazel-8.1.1.patch new file mode 100644 index 00000000..8924cf4b --- /dev/null +++ b/third_party/modules/xla/20250409.0-7167d71/patches/0001-bazel-migration-to-bazel-8.1.1.patch @@ -0,0 +1,41 @@ +From 6cf475b500521c1b8be06f590fdbc1818f0dc44b Mon Sep 17 00:00:00 2001 +From: Jean-Baptiste Dalido +Date: Mon, 6 Jan 2025 13:33:13 +0100 +Subject: [PATCH] bazel: migration to bazel 8.0.1 + +--- + .bazelversion | 2 +- + third_party/tsl/third_party/gpus/cuda_configure.bzl | 4 ++-- + 2 files changed, 3 insertions(+), 3 deletions(-) + +diff --git a/.bazelversion b/.bazelversion +index f22d756da3..fa5fce04b3 100644 +--- a/.bazelversion ++++ b/.bazelversion +@@ -1 +1 @@ +-7.4.1 ++8.1.1 +\ No newline at end of file +diff --git a/third_party/gpus/cuda_configure.bzl b/third_party/gpus/cuda_configure.bzl +index d62531152d..71d80a5a99 100644 +--- a/third_party/gpus/cuda_configure.bzl ++++ b/third_party/gpus/cuda_configure.bzl +@@ -33,14 +33,14 @@ NB: DEPRECATED! Use `hermetic/cuda_configure` rule instead. + load( + "@bazel_tools//tools/cpp:lib_cc_configure.bzl", + "escape_string", +- "get_env_var", + ) + load( + "@bazel_tools//tools/cpp:windows_cc_configure.bzl", +- "find_msvc_tool", + "find_vc_path", + "setup_vc_env_vars", + ) ++load("@rules_cc//cc/private/toolchain:windows_cc_configure.bzl", "find_msvc_tool") ++load("@rules_cc//cc/private/toolchain:lib_cc_configure.bzl", "get_env_var") + load("//third_party/clang_toolchain:download_clang.bzl", "download_clang") + load( + "//third_party/remote_config:common.bzl", +-- +2.39.3 (Apple Git-146) diff --git a/third_party/modules/xla/20250409.0-7167d71/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch b/third_party/modules/xla/20250409.0-7167d71/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch new file mode 100644 index 00000000..f367b12b --- /dev/null +++ b/third_party/modules/xla/20250409.0-7167d71/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch @@ -0,0 +1,132 @@ +From 490985facbe2fe6462b2006a99514b1c7e8a0775 Mon Sep 17 00:00:00 2001 +From: Hugo Mano +Date: Wed, 5 Feb 2025 19:25:03 +0100 +Subject: [PATCH 1/7] Added FFI handler registration API to the FFI PjRt + +PR: https://github.com/openxla/xla/pull/13420 +--- + xla/pjrt/c/BUILD | 5 ++++ + xla/pjrt/c/pjrt_c_api_ffi_extension.h | 16 +++++++++++++ + xla/pjrt/c/pjrt_c_api_ffi_internal.cc | 33 ++++++++++++++++++++++++++- + 3 files changed, 53 insertions(+), 1 deletion(-) + +diff --git a/xla/pjrt/c/BUILD b/xla/pjrt/c/BUILD +index 8e8a4baa53..551c35b461 100644 +--- a/xla/pjrt/c/BUILD ++++ b/xla/pjrt/c/BUILD +@@ -69,8 +69,13 @@ cc_library( + ":pjrt_c_api_wrapper_impl", + "//xla/ffi:execution_context", + "//xla/ffi:type_id_registry", ++ "//xla/ffi:ffi_api", ++ "//xla/ffi/api:c_api", ++ "//xla/ffi/api:ffi", ++ "//xla/service:custom_call_target_registry", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings:string_view", ++ "@com_google_absl//absl/strings:str_format", + ], + ) + +diff --git a/xla/pjrt/c/pjrt_c_api_ffi_extension.h b/xla/pjrt/c/pjrt_c_api_ffi_extension.h +index d9f6d163d8..3dbd4df089 100644 +--- a/xla/pjrt/c/pjrt_c_api_ffi_extension.h ++++ b/xla/pjrt/c/pjrt_c_api_ffi_extension.h +@@ -69,12 +69,28 @@ PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_UserData_Add_Args, user_data); + // Adds a user data to the execute context. + typedef PJRT_Error* PJRT_FFI_UserData_Add(PJRT_FFI_UserData_Add_Args* args); + ++struct PJRT_FFI_Register_Handler_Args { ++ size_t struct_size; ++ const char* target_name; ++ size_t target_name_size; ++ int api_version; // 0 for an untyped call, 1 -- for typed ++ void* handler; ++ const char* platform_name; ++ size_t platform_name_size; ++}; ++PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_Register_Handler_Args, handler); ++ ++// Registers an FFI call handler for a specific platform. ++typedef PJRT_Error* PJRT_FFI_Register_Handler( ++ PJRT_FFI_Register_Handler_Args* args); ++ + typedef struct PJRT_FFI_Extension { + size_t struct_size; + PJRT_Extension_Type type; + PJRT_Extension_Base* next; + PJRT_FFI_TypeID_Register* type_id_register; + PJRT_FFI_UserData_Add* user_data_add; ++ PJRT_FFI_Register_Handler* register_handler; + } PJRT_FFI; + PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_Extension, user_data_add); + +diff --git a/xla/pjrt/c/pjrt_c_api_ffi_internal.cc b/xla/pjrt/c/pjrt_c_api_ffi_internal.cc +index 8898088a23..746861f697 100644 +--- a/xla/pjrt/c/pjrt_c_api_ffi_internal.cc ++++ b/xla/pjrt/c/pjrt_c_api_ffi_internal.cc +@@ -13,16 +13,21 @@ See the License for the specific language governing permissions and + limitations under the License. + ==============================================================================*/ + +-#include "xla/pjrt/c/pjrt_c_api_ffi_internal.h" ++#include + + #include "absl/status/status.h" + #include "absl/strings/string_view.h" ++#include "absl/strings/str_format.h" ++#include "xla/ffi/api/c_api.h" ++#include "xla/ffi/api/ffi.h" + #include "xla/ffi/execution_context.h" + #include "xla/ffi/type_id_registry.h" ++#include "xla/ffi/ffi_api.h" + #include "xla/pjrt/c/pjrt_c_api.h" + #include "xla/pjrt/c/pjrt_c_api_ffi_extension.h" + #include "xla/pjrt/c/pjrt_c_api_helpers.h" + #include "xla/pjrt/c/pjrt_c_api_wrapper_impl.h" ++#include "xla/service/custom_call_target_registry.h" + + namespace pjrt { + +@@ -68,6 +73,31 @@ static PJRT_Error* PJRT_FFI_UserData_Add(PJRT_FFI_UserData_Add_Args* args) { + return nullptr; + } + ++static PJRT_Error* PJRT_FFI_Register_Handler( ++ PJRT_FFI_Register_Handler_Args* args) { ++ PJRT_RETURN_IF_ERROR(ActualStructSizeIsGreaterOrEqual( ++ "PJRT_FFI_Register_Handler_Args", ++ PJRT_FFI_Register_Handler_Args_STRUCT_SIZE, args->struct_size)); ++ std::string target_name(args->target_name, args->target_name_size); ++ std::string platform_name(args->platform_name, args->platform_name_size); ++ switch (args->api_version) { ++ case 0: ++ xla::CustomCallTargetRegistry::Global()->Register( ++ target_name, args->handler, platform_name); ++ return nullptr; ++ case 1: ++ xla::ffi::Ffi::RegisterStaticHandler( ++ xla::ffi::GetXlaFfiApi(), target_name, platform_name, ++ reinterpret_cast(args->handler)); ++ return nullptr; ++ default: ++ return new PJRT_Error{absl::UnimplementedError( ++ absl::StrFormat("API version %d not supported for PJRT GPU plugin. " ++ "Supported versions are 0 and 1.", ++ args->api_version))}; ++ } ++} ++ + PJRT_FFI_Extension CreateFfiExtension(PJRT_Extension_Base* next) { + return { + /*struct_size=*/PJRT_FFI_Extension_STRUCT_SIZE, +@@ -75,6 +105,7 @@ PJRT_FFI_Extension CreateFfiExtension(PJRT_Extension_Base* next) { + /*next=*/next, + /*type_id_register=*/PJRT_FFI_TypeID_Register, + /*user_data_add=*/PJRT_FFI_UserData_Add, ++ /*register_handler=*/PJRT_FFI_Register_Handler, + }; + } + +-- +2.39.5 (Apple Git-154) diff --git a/third_party/modules/xla/20250409.0-7167d71/source.json b/third_party/modules/xla/20250409.0-7167d71/source.json new file mode 100644 index 00000000..36797b83 --- /dev/null +++ b/third_party/modules/xla/20250409.0-7167d71/source.json @@ -0,0 +1,15 @@ +{ + "strip_prefix": "xla-7167d711e572e1da17d6c4a87ab1f3661c8f2ac0", + "url": "https://github.com/openxla/xla/archive/7167d711e572e1da17d6c4a87ab1f3661c8f2ac0.tar.gz", + "integrity": "sha256-QxwcMAFT4lQLHp+mHh3nhWo9qwbTf9+kiOMWmNuXoVI=", + "overlay": { + "tsl.bzl": "", + "workspace.bzl": "", + "MODULE.bazel": "" + }, + "patch_strip": 1, + "patches": { + "0001-bazel-migration-to-bazel-8.1.1.patch": "", + "0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch": "" + } +} diff --git a/third_party/modules/xla/metadata.json b/third_party/modules/xla/metadata.json index 3916998f..01c87137 100644 --- a/third_party/modules/xla/metadata.json +++ b/third_party/modules/xla/metadata.json @@ -18,7 +18,8 @@ "20250122.0-cc075be", "20250123.0-cc075be", "20250204-0-6789523", - "20250317.0-71c67e2" + "20250317.0-71c67e2", + "20250409.0-7167d71" ], "yanked_versions": {} } diff --git a/zml/buffer.zig b/zml/buffer.zig index e0829d39..c038636a 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -4,6 +4,7 @@ const testing = std.testing; const asynk = @import("async"); const stdx = @import("stdx"); +const ops = @import("ops.zig"); const Context = @import("context.zig").Context; const Data = @import("dtype.zig").Data; const DataType = @import("dtype.zig").DataType; @@ -44,23 +45,6 @@ pub const Buffer = struct { } }; - pub const Shard = struct { - api: *const pjrt.Api, - buffer: *pjrt.Buffer, - ready_event: ?*pjrt.Event = null, - ready: bool = false, - - pub fn awaitt(self: *Shard) !void { - if (self.ready) { - return; - } - if (self.ready_event orelse self.buffer.getReadyEvent(self.api)) |ev| { - try ev.awaitt(self.api); - } - self.ready = true; - } - }; - _shape: Shape, _api: *const pjrt.Api, _shards: Shards, @@ -108,7 +92,7 @@ pub const Buffer = struct { .dims = buf.shape().dims(), .byte_strides = byte_strides, .device = devices[i], - .host_buffer_semantics = .ImmutableOnlyDuringCall, + .host_buffer_semantics = .MutableZeroCopy, }, }); @@ -234,23 +218,17 @@ pub const Buffer = struct { /// Creates a Buffer from a pointer into device memory. /// This allows to interface with other libraries producing buffers. pub fn asViewOfDeviceBuffer(platform: Platform, shape_: Shape, stream: ?*const anyopaque, device_data: *anyopaque) Buffer { - const minor_to_major: [Shape.MAX_RANK]i64 = comptime blk: { - var res: [Shape.MAX_RANK]i64 = undefined; - for (0..Shape.MAX_RANK) |i| { - res[i] = @intCast(Shape.MAX_RANK - i - 1); - } - break :blk res; - }; - const pjrt_buffer = platform.pjrt_client.createViewOfDeviceBuffer(platform.pjrt_api, .{ .data = device_data, .element_type = bufferTypeFromDtype(shape_.dtype()), .dims = shape_.dims(), // TODO: exposes sharding in the API. .device = platform.getDevices()[0], + // TODO: expose memory in the API. + .memory_kind = null, .layout = .{ .tiled = .{ - .minor_to_major = minor_to_major[Shape.MAX_RANK - shape_.rank() ..], + .minor_to_major = ops.minorToMajor(shape_.rank()), .tile_dims = &.{}, .tile_dims_sizes = &.{}, }, @@ -279,6 +257,31 @@ pub const Buffer = struct { return res; } + pub fn setValue(self: Buffer, T: type, value: anytype) !void { + stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); + const shard_buffer = self._shards.get(0); + const opaqueDataPointer = try shard_buffer.getOpaqueDeviceMemoryDataPointer(self._api); + const sizeInBytes = try shard_buffer.getOnDeviceSizeInBytes(self._api); + stdx.debug.assert(sizeInBytes == @sizeOf(T), "Buffer size {d} does not match type size {d}", .{ sizeInBytes, @sizeOf(T) }); + const data = @as(*T, @ptrFromInt(@intFromPtr(opaqueDataPointer))); + data.* = value; + } + + pub fn getValueFromDataInMemory(self: Buffer, T: type) !T { + stdx.debug.assert(self._shape.byteSize() == @sizeOf(T), "Buffer {} has {d} bytes of data, can't load it to a {s} with {d} bytes", .{ self, self._shape.byteSize(), @typeName(T), @sizeOf(T) }); + const data = try self.dataInMemory(); + const value = std.mem.bytesAsValue(T, @constCast(data)); + return value.*; + } + + pub fn dataInMemory(self: Buffer) ![]const u8 { + const shard_buffer = self._shards.get(0); + const opaqueDataPointer = try shard_buffer.getOpaqueDeviceMemoryDataPointer(self._api); + const sizeInbytes = try shard_buffer.getOnDeviceSizeInBytes(self._api); + const data = @as([*]const u8, @ptrFromInt(@intFromPtr(opaqueDataPointer))); + const end: usize = @intCast(sizeInbytes); + return data[0..end]; + } /// Copies the content of the Buffer back to host, in the given buffer, /// and return a new `HostBuffer` object with the same shape. /// The returned `HostBuffer` doesn't own the memory. @@ -357,6 +360,29 @@ pub const Buffer = struct { return shard.memory(self._api); } + pub fn copyToMemory(self: Buffer, platform: Platform, kind: Memory) !Buffer { + const device = try self._shards.get(0).getDevice(self._api); + const memories = device.addressableMemories(self._api); + var selected_mem: *const pjrt.Memory = undefined; + for (memories) |mem| { + if (mem.kind(platform.pjrt_api) == kind.toPjrtMemory()) { + selected_mem = mem; + } + } + + var shards: Shards = .{}; + for (self._shards.constSlice()) |shard| { + const pjrt_buffer = try shard.copyToMemory(self._api, selected_mem); + shards.appendAssumeCapacity(pjrt_buffer); + } + + return .{ + ._api = platform.pjrt_api, + ._shape = self._shape, + ._shards = shards, + }; + } + fn hasShardedAxis(self: Buffer) bool { if (self._shards.len == 1) return false; return @reduce(.Or, self._shape._sharding_info); diff --git a/zml/context.zig b/zml/context.zig index 03de62a3..65c6f381 100644 --- a/zml/context.zig +++ b/zml/context.zig @@ -37,6 +37,7 @@ pub const Context = struct { inline for (comptime std.enums.values(runtimes.Platform)) |t| { if (runtimes.load(t)) |api| { Context.apis.set(t, api); + if (t == .cuda) cuda.init(); } else |_| {} } } @@ -177,8 +178,6 @@ pub const Context = struct { return error.NoDevicesFound; } - try CustomCall.registerZmlCustomCalls(p); - self.platforms.set(target, p); return p; } @@ -212,69 +211,66 @@ pub const Context = struct { } } } - - pub const HostCallback = fn (?*anyopaque, []const HostBuffer, []const HostBuffer) void; }; -const CustomCall = struct { - pub fn registerZmlCustomCalls(platform: Platform) !void { - const registry = platform.pjrt_api.customCallRegistry(); +pub const cuda = struct { + var _memcpyAsync: MemcpyAsync = @ptrFromInt(0xdeadc00da00); + var _memcpyBlocking: MemcpyBlocking = @ptrFromInt(0xdeadc00da00); + pub var streamSynchronize: StreamSynchronize = @ptrFromInt(0xdeadc00da00); - if (registry) |reg| { - try reg.registerFfi(platform.pjrt_api, "zmlHostBufferCallback", @tagName(platform.target), &hostBufferCallback); - } else { - stdx.debug.panic("Registering custom calls failed", .{}); - } - } - - fn hostBufferCallback(call_frame: *pjrt.ffi.CallFrame) callconv(.C) ?*pjrt.ffi.Error { - if (call_frame.registeringHook()) return null; - - const callback_attr = call_frame.attrs.getByName(.scalar, "callback") orelse unreachable; - std.debug.assert(callback_attr.dtype == .u64); - const callback: *const Context.HostCallback = @ptrFromInt(callback_attr.get(usize)); - - const user_ctx_ptr = call_frame.attrs.getByName(.scalar, "user_context") orelse unreachable; - std.debug.assert(user_ctx_ptr.dtype == .u64); - const user_ctx: ?*anyopaque = @ptrFromInt(user_ctx_ptr.get(usize)); - - const input_buffers = stdx.stackSlice(8, HostBuffer, call_frame.args.len); - for (input_buffers, 0..) |*b, i| { - b.* = hostBufferFromPinnedBuffer(call_frame.args.get(i)); - } - - const output_buffers = stdx.stackSlice(8, HostBuffer, call_frame.results.len); - for (output_buffers, 0..) |*b, i| { - b.* = hostBufferFromPinnedBuffer(call_frame.results.get(i)); - } - - callback(user_ctx, input_buffers, output_buffers); - return null; - } -}; - -fn getShape(buffer_desc: *const pjrt.ffi.Buffer) Shape { - // log.warn("received buffer {}", .{buffer_desc}); - const dt: DataType = switch (buffer_desc.dtype) { - .invalid => @panic("invalid ffi"), - .pred => .bool, - .s8 => .i8, - .s16 => .i16, - .s32 => .i32, - .s64 => .i64, - .token, .f8e4m3, .f8e3m4 => @panic("Unsupported ffi type"), - inline else => |t| @field(DataType, @tagName(t)), + pub const MemcpyKind = enum(c_int) { + host_to_host = 0, + host_to_device = 1, + device_to_host = 2, + device_to_device = 3, + inferred = 4, }; - return Shape.init(buffer_desc.dims(), dt); -} -/// Create a HostBuffer from a ffi description of a buffer. -/// Normally the ffi describe device buffer but we assume they are located in pinned memory, -/// and therefore the data pointer is readable both from host and from device. -fn hostBufferFromPinnedBuffer(buffer_desc: *const pjrt.ffi.Buffer) HostBuffer { - const buffer_shape = getShape(buffer_desc); - return HostBuffer.fromBytes( - buffer_shape, - buffer_desc.data[0..buffer_shape.byteSize()], - ); -} + const MemcpyAsync = *const fn (dst: *anyopaque, src: *const anyopaque, count: usize, kind: MemcpyKind, stream: ?*pjrt.ffi.Stream) callconv(.C) c_int; + const MemcpyBlocking = *const fn (dst: *anyopaque, src: *const anyopaque, count: usize, kind: MemcpyKind) callconv(.C) c_int; + const StreamSynchronize = *const fn (stream: ?*pjrt.ffi.Stream) callconv(.C) c_int; + + pub fn init() void { + var cudart = std.DynLib.open("libcudart.so.12") catch { + log.err("cudart not found, callback will segfault", .{}); + return; + }; + defer cudart.close(); + + _memcpyAsync = cudart.lookup(MemcpyAsync, "cudaMemcpyAsync") orelse { + @panic("cudaMemcpyAsync not found"); + }; + _memcpyBlocking = cudart.lookup(MemcpyBlocking, "cudaMemcpy") orelse { + @panic("cudaMemcpy not found"); + }; + streamSynchronize = cudart.lookup(StreamSynchronize, "cudaStreamSynchronize") orelse { + @panic("cudaStreamSynchronize not found"); + }; + } + + pub fn memcpyToHostBlocking(dst: []u8, src: pjrt.ffi.DevicePtr) void { + const err = _memcpyBlocking(dst.ptr, src.asPtr(), dst.len, .device_to_host); + check(err); + } + + pub fn memcpyToDeviceBlocking(dst: pjrt.ffi.DevicePtr, src: []const u8) void { + const err = _memcpyBlocking(dst.asPtr(), src.ptr, src.len, .host_to_device); + check(err); + } + + pub fn memcpyToDeviceAsync(dst: pjrt.ffi.DevicePtr, src: []const u8, stream: ?*pjrt.ffi.Stream) void { + const err = _memcpyAsync(dst.asPtr(), src.ptr, src.len, .host_to_device, stream); + check(err); + } + + pub fn memcpyToHostAsync(dst: []u8, src: pjrt.ffi.DevicePtr, stream: ?*pjrt.ffi.Stream) void { + const err = _memcpyAsync(dst.ptr, src.asPtr(), dst.len, .device_to_host, stream); + check(err); + } + + pub fn check(err: c_int) void { + if (err == 0) return; + log.err("CUDA ERROR {d}", .{err}); + @panic("CUDA error"); + } +}; diff --git a/zml/exe.zig b/zml/exe.zig index 946410c6..1d6f263f 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -12,7 +12,7 @@ const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; const ShapeOf = @import("tensor.zig").ShapeOf; -const log = std.log.scoped(.zml); +const log = std.log.scoped(.@"zml/exe"); test { std.testing.refAllDecls(@This()); @@ -155,6 +155,8 @@ pub const BaseExe = struct { /// Allocator backing memory _arena: std.heap.ArenaAllocator, + _context: ?*pjrt.ExecuteContext = null, + pub fn init(parent_allocator: std.mem.Allocator, platform: Platform, exe: *pjrt.LoadedExecutable, args: struct { n_in: u32, result_shapes: []const Shape, n_devices: u8 }) !BaseExe { var arena = std.heap.ArenaAllocator.init(parent_allocator); errdefer arena.deinit(); @@ -188,6 +190,9 @@ pub const BaseExe = struct { } pub fn deinit(self: BaseExe) void { + if (self._context) |ctx| { + ctx.deinit(self.platform.pjrt_api); + } self._arena.deinit(); } @@ -209,6 +214,7 @@ pub const BaseExe = struct { // even if it has been marked as "can be donated" during compilation. // TODO: expose it ? .non_donatable_input_indices = &.{}, + .context = if (self._context) |ctx| ctx else null, }) catch unreachable; for (events[0..sharding.num_partitions]) |e| { @@ -271,6 +277,32 @@ pub fn Exe(ArgsT: type, ReturnT: type) type { return new; } + pub fn withExecutionContext(self: Self) Self { + stdx.debug.assert(self.inner._context == null, "Exe already has an execution context", .{}); + var new: Self = .{ .inner = self.inner }; + const pjrt_execute_context = self.inner.platform.pjrt_api.createExecuteContext() catch unreachable; + new.inner._context = pjrt_execute_context; + log.info("Created context execution {*} for {*}", .{ pjrt_execute_context, self.inner.exe }); + return new; + } + + pub fn attach(self: Self, value: anytype) !void { + stdx.debug.assert(self.inner._context != null, "Exe doesn't have an execution context", .{}); + const pjrt_api = self.inner.platform.pjrt_api; + const ffi = pjrt_api.ffi().?; + const ValueT = @TypeOf(value); + const type_name = switch (@typeInfo(ValueT)) { + .pointer => @typeName(std.meta.Child(ValueT)), + else => @typeName(ValueT), // todo error + }; + + const type_id = try ffi.registerTypeId(pjrt_api, type_name); + + const user_data: *anyopaque = @ptrCast(@constCast(value)); + log.info("Attached {s}@{x} with type id {d} on {any}", .{ type_name, user_data, type_id, self.inner._context.? }); + try ffi.addUserData(pjrt_api, self.inner._context.?, .{ .type_id = type_id, .user_data = user_data }); + } + pub fn serialize(self: Self, writer: anytype) !void { return try self.inner.serialize(writer); } diff --git a/zml/ffi.zig b/zml/ffi.zig new file mode 100644 index 00000000..5fff04cf --- /dev/null +++ b/zml/ffi.zig @@ -0,0 +1,276 @@ +const std = @import("std"); +const testing = std.testing; +const builtin = @import("builtin"); + +const asynk = @import("async"); +const stdx = @import("stdx"); + +const context = @import("context.zig"); +const dialect = struct { + const stablehlo = @import("mlir/dialects").stablehlo; +}; +const dtype = @import("dtype.zig"); +const pjrt = @import("pjrtx.zig"); +const buffer = @import("buffer.zig"); +const hostbuffer = @import("hostbuffer.zig"); +const mlir = @import("mlir.zig"); +const module = @import("module.zig"); +const ops = @import("ops.zig"); +const platform = @import("platform.zig"); +const tensor = @import("tensor.zig"); +const shape = @import("shape.zig"); +const cuda = context.cuda; +const Buffer = buffer.Buffer; +const HostBuffer = hostbuffer.HostBuffer; +const CompilationContext = module.CompilationContext; +const DataType = dtype.DataType; +const Platform = platform.Platform; +const Shape = shape.Shape; +const Tensor = tensor.Tensor; + +const log = std.log.scoped(.@"zml/custom_call"); + +pub const std_options: std.Options = .{ + .log_level = .info, + .logFn = asynk.logFn(std.log.defaultLog), +}; + +// todo: check deallocs + +pub fn CustomCallInputType(custom_op: type) type { + const ArgsT = stdx.meta.FnArgs(custom_op.beforeCustomCall); + if (@typeInfo(ArgsT) != .@"struct") { + @compileError("Expected struct type"); + } + // todo check typeof first input then others args + return [@typeInfo(ArgsT).@"struct".fields.len]Tensor; +} + +pub fn CustomCallOutputType(custom_op: type) type { + const ReturnT = stdx.meta.FnResultNoError(custom_op.call); + + if (@typeInfo(ReturnT) != .@"struct") { + @compileError("Expected struct type"); + } + + if (ReturnT == Buffer) { + return Tensor; + } + return [@typeInfo(ReturnT).@"struct".fields.len]Tensor; +} + +pub fn lenOutputBeforeCustomCall(custom_op: type) usize { + const ReturnT = stdx.meta.FnResultNoError(custom_op.beforeCustomCall); + + if (@typeInfo(ReturnT) != .@"struct") { + @compileError("Expected struct type"); + } + + if (ReturnT == Tensor) { + return 1; + } + return @typeInfo(ReturnT).@"struct".fields.len; +} + +pub fn custom_call( + comptime custom_op: type, + inputs: CustomCallInputType(custom_op), +) CustomCallOutputType(custom_op) { + stdx.debug.assert(@hasDecl(custom_op, "call"), "custom_op must have a call method", .{}); + const op_name = @typeName(custom_op); + + const ctx = inputs[0].getContext(); + const mlir_ctx = ctx.mlirCtx(); + const platform_ = ctx._platform; + const pjrt_api = platform_.pjrt_api; + const ffi = pjrt_api.ffi().?; + const registry = ffi.customCallRegistry().?; + const ffi_func = proxy(custom_op); + const target_name = "callback_" ++ op_name; + + registry.registerFfi(pjrt_api, target_name, @tagName(platform_.target), &ffi_func) catch unreachable; + log.info("{s} / Registered custom call with target_name \"{s}\" with proxy func {*}", .{ op_name, target_name, &ffi_func }); + + var custom_call_inputs: []mlir.Value = undefined; + var res_shapes: []Shape = undefined; + var res_types: []mlir.Type = undefined; + + if (@hasDecl(custom_op, "beforeCustomCall")) { + const before_custom_call = custom_op.beforeCustomCall; + if (@typeInfo(@TypeOf(before_custom_call)) != .@"fn") { + stdx.debug.panic("beforeCustomCall must be a function"); + } + + var args: std.meta.ArgsTuple(@TypeOf(before_custom_call)) = undefined; + inline for (0..args.len) |i| { + args[i] = inputs[i]; + } + + const before_custom_call_outputs = ctx.callFunc(@typeName(custom_op), before_custom_call, args) catch |err| { + stdx.debug.panic("Error in {any} beforeCustomCall func: {any}\n", .{ @typeName(custom_op), err }); + }; + var before_custom_call_outputs_array: []Tensor = undefined; + + if (@TypeOf(before_custom_call_outputs) == Tensor) { + before_custom_call_outputs_array = stdx.stackSlice(8, Tensor, 1); + before_custom_call_outputs_array[0] = before_custom_call_outputs; + } else { + before_custom_call_outputs_array = stdx.stackSlice(8, Tensor, @typeInfo(@TypeOf(before_custom_call_outputs)).@"struct".fields.len); + inline for (@typeInfo(@TypeOf(before_custom_call_outputs)).@"struct".fields, 0..) |field, i| { + before_custom_call_outputs_array[i] = @field(before_custom_call_outputs, field.name); + } + } + + custom_call_inputs = stdx.stackSlice(8, mlir.Value, lenOutputBeforeCustomCall(custom_op)); + res_shapes = stdx.stackSlice(8, Shape, before_custom_call_outputs_array.len); + res_types = stdx.stackSlice(8, mlir.Type, before_custom_call_outputs_array.len); + + for (before_custom_call_outputs_array, 0..) |o, i| { + res_shapes[i] = o.shape(); + custom_call_inputs[i] = o.value(); + res_types[i] = mlir.ext.RankedTensorType.fromShape(mlir_ctx, o.shape()).as(mlir.Type); + } + } else { + // todo : check if args len != outputs len + log.warn("{s} / No beforeCustomCall function found, we will expect that the return type of call func will be like args type", .{op_name}); + custom_call_inputs = stdx.stackSlice(8, mlir.Value, inputs.len); + + for (custom_call_inputs, inputs) |*input, tensor_| { + input.* = tensor_.value(); + } + + res_shapes = stdx.stackSlice(8, Shape, inputs.len); + res_types = stdx.stackSlice(8, mlir.Type, inputs.len); + inline for (inputs, 0..) |input, i| { + res_shapes[i] = input.shape(); + res_types[i] = mlir.ext.RankedTensorType.fromShape(mlir_ctx, input.shape()).as(mlir.Type); + } + } + + const frontend_attributes = mlir.Attribute.dict(mlir_ctx, &.{ + .{ "_xla_compute_type", .string(mlir_ctx, "host") }, + .{ "_xla_buffer_placement", .string(mlir_ctx, @tagName(Buffer.Memory.host_pinned.toPjrtMemory())) }, + }); + + const op = dialect.stablehlo.custom_call( + mlir_ctx, + custom_call_inputs, + .{ + .call_target_name = target_name, + .api_version = .typed_ffi, + .backend_config = mlir.Attribute.dict(mlir_ctx, &.{}), + .additional_attributes = &.{.{ "mhlo.frontend_attributes", frontend_attributes }}, + .has_side_effect = true, + .output_operand_aliases = &.{}, + }, + res_types, + mlir_ctx.location(@src()), + ); + + if (CustomCallOutputType(custom_op) == Tensor) { + return Tensor._result(res_shapes[0], op.result(0)); + } + + var custom_call_outputs: CustomCallOutputType(custom_op) = undefined; + + for (&custom_call_outputs, res_shapes, 0..) |*r, shape_, i| { + r.* = Tensor._result(shape_, op.result(i)); + } + + return custom_call_outputs; +} + +fn proxy(comptime custom_op: type) pjrt.ffi.Handler { + const op_name = @typeName(custom_op); + _ = op_name; // autofix + + return struct { + const Self = @This(); + + pub fn proxy(call_frame: *pjrt.ffi.CallFrame) callconv(.C) ?*pjrt.ffi.Error { + if (call_frame.registeringHook()) return null; + const execution_context = call_frame.ctx.?; + const device_id = execution_context.getDeviceOrdinal(call_frame.api.?) catch unreachable; + const user_ctx: *custom_op = pjrt.ffi.ExecutionContext.Context(custom_op).get(execution_context, call_frame.api.?) catch unreachable; + + const platform_: *const Platform = user_ctx._platform; + const pjrt_api = platform_.pjrt_api; + const pjrt_client = platform_.pjrt_client; + + const device = pjrt_client.getAddressableDevices(pjrt_api)[@as(u32, @intCast(device_id))]; + _ = device; // autofix + const ffi_buffer = call_frame.args.buffers()[0]; + // const buffer_on_device = Buffer.asViewOfDeviceBuffer(platform_.*, getShape(ffi_buffer), null, ffi_buffer.data.asPtr()); + // const buffer_pinned = buffer_on_device.copyToMemory(platform_.*, .host_pinned) catch unreachable; + // _ = buffer_pinned; // autofix + + const ctx = call_frame.ctx; + const stream = call_frame.api.?.stream(@constCast(ctx)); + _ = stream; // autofix + const host = HostBuffer.empty(std.heap.smp_allocator, getShape(ffi_buffer)) catch unreachable; + cuda.memcpyToHostAsync(@constCast(host.data), ffi_buffer.data, null); + _ = cuda.streamSynchronize(null); + // log.info("{s} / Proxy call_frame {*} on device {} (ordinal: {d})", .{ op_name, call_frame, device, device_id }); + + // var callback_args: std.meta.ArgsTuple(@TypeOf(custom_op.call)) = undefined; + // callback_args[0] = user_ctx; + + // inline for (1..callback_args.len) |i| { + // const ffi_buffer = call_frame.args.buffers()[i - 1]; + // log.info("{s} / FFI Buffer arg: {} ({d}/{d})", .{ op_name, ffi_buffer, i, call_frame.args.len }); + + // const buffer_on_device = Buffer.asViewOfDeviceBuffer(platform_.*, getShape(ffi_buffer), null, ffi_buffer.data.asPtr()); + // const buffer_device_pjrt_buffer = buffer_on_device._shards.get(0).getOpaqueDeviceMemoryDataPointer(pjrt_api); + // log.info("{s} / ZML Buffer on device: {} {any} ({d}/{d})", .{ op_name, buffer_on_device, buffer_device_pjrt_buffer, i, call_frame.args.len }); + + // const buffer_pinned = buffer_on_device.copyToMemory(platform_.*, .host_pinned) catch unreachable; + // const buffer_pinned_pjrt_buffer = buffer_pinned._shards.get(0).getOpaqueDeviceMemoryDataPointer(pjrt_api); + // log.info("{s} / ZML Buffer pinned: {} {any} ({d}/{d})", .{ op_name, buffer_pinned, buffer_pinned_pjrt_buffer, i, call_frame.args.len }); + + // callback_args[i] = buffer_pinned; + // } + + // const zml_results = @call(.auto, custom_op.call, callback_args) catch |err| { + // stdx.debug.panic("Error while calling {any} call func: {any}\n", .{ @typeName(custom_op), err }); + // }; + + // var results_array: []Buffer = undefined; + // if (@TypeOf(zml_results) == Buffer) { + // results_array = stdx.stackSlice(8, Buffer, 1); + // results_array[0] = zml_results; + // } else { + // results_array = stdx.stackSlice(8, Buffer, @typeInfo(@TypeOf(zml_results)).@"struct".fields.len); + // inline for (@typeInfo(@TypeOf(zml_results)).@"struct".fields, 0..) |field, i| { + // results_array[i] = @field(zml_results, field.name); + // } + // } + + // for (results_array, 0..) |zml_buffer, i| { + // const ffi_buffer = call_frame.results.buffers()[i]; + // _ = ffi_buffer; // autofix + // log.info("{s} / FFI Buffer result ptr: {} ({d}/{d})", .{ op_name, call_frame.results.ptr[i], i + 1, call_frame.results.len }); + + // const zml_buffer_pjrt_buffer = zml_buffer._shards.get(0).getOpaqueDeviceMemoryDataPointer(pjrt_api) catch unreachable; + // log.info("{s} / ZML Buffer result: {} {any} ({d}/{d})", .{ op_name, zml_buffer, zml_buffer_pjrt_buffer, i + 1, call_frame.results.len }); + + // // cuda.memcpyToDeviceBlocking(ffi_buffer.data, zml_buffer.dataInMemory() catch unreachable); + // } + + return null; + } + }.proxy; +} + +fn getShape(ffi_buffer: *const pjrt.ffi.Buffer) Shape { + const dt: DataType = switch (ffi_buffer.dtype) { + .invalid => stdx.debug.panic("Invalid FFI dtype {any} used by {any}", .{ ffi_buffer.dtype, ffi_buffer }), + .pred => .bool, + .s8 => .i8, + .s16 => .i16, + .s32 => .i32, + .s64 => .i64, + .token, .f8e4m3, .f8e3m4 => stdx.debug.panic("Unsupported FFI dtype {any} used by {any}", .{ ffi_buffer.dtype, ffi_buffer }), + inline else => |t| @field(DataType, @tagName(t)), + }; + return Shape.init(ffi_buffer.dims(), dt); +} diff --git a/zml/module.zig b/zml/module.zig index 011e6fe6..1fad6d37 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -196,6 +196,9 @@ pub const CompilationContext = struct { pjrt_location = try std.fs.path.joinZ(arena, &.{ module_dir.?, "module.pjrt" }); } + log.info("******** ZML generated MLIR ********", .{}); + log.info("{}", .{module.op().mlirFormatter(.{})}); + const loaded_executable: *pjrt.LoadedExecutable = blk: { if (pjrt_location) |pjrt_loc| { if (loadPjrtExecutable(arena, self._platform, pjrt_loc)) |exe| { @@ -220,9 +223,6 @@ pub const CompilationContext = struct { break :blk loaded_executable; }; - log.debug("******** ZML generated MLIR ********", .{}); - log.debug("{}", .{module.op().mlirFormatter(.{})}); - if (timer) |*t| { const time_ms = @divFloor(t.lap(), std.time.ns_per_ms); if (time_ms > 1000) log.info("Compilation took {d:.3}s", .{stdx.math.divFloat(f32, time_ms, 1000)}); @@ -1154,9 +1154,9 @@ pub fn hash(hasher: *std.hash.Wyhash, key: anytype, comptime strat: std.hash.Str .@"anyframe", .@"fn" => hash(hasher, @intFromPtr(key), strat), .pointer => |info| switch (info.size) { .one => switch (strat) { - .shallow => hash(hasher, @intFromPtr(key), .Shallow), - .deep => hash(hasher, key.*, .Shallow), - .deeprecursive => switch (@typeInfo(info.child)) { + .Shallow => hash(hasher, @intFromPtr(key), .Shallow), + .Deep => hash(hasher, key.*, .Shallow), + .DeepRecursive => switch (@typeInfo(info.child)) { .@"opaque", .@"fn" => hash(hasher, @intFromPtr(key), .Shallow), else => hash(hasher, key.*, .DeepRecursive), }, @@ -1172,7 +1172,7 @@ pub fn hash(hasher: *std.hash.Wyhash, key: anytype, comptime strat: std.hash.Str .many, .c, => switch (strat) { - .shallow => hash(hasher, @intFromPtr(key), .Shallow), + .Shallow => hash(hasher, @intFromPtr(key), .Shallow), else => @compileError( \\ unknown-length pointers and C pointers cannot be hashed deeply. \\ Consider providing your own hash function. diff --git a/zml/ops.zig b/zml/ops.zig index 1765888a..3f70997b 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -769,58 +769,6 @@ pub fn fromMlirOperationWithTags(op: mlir.Operation, base: anytype) @TypeOf(base return res; } -pub const HostCallbackOpt = struct { - has_side_effect: bool = false, - output_operand_aliases: []const i64 = &.{}, -}; - -pub fn addHostCallback( - callback: *const Context.HostCallback, - blkctx: ?*anyopaque, - inputs: []const Tensor, - output_shapes: []const Shape, - opts: HostCallbackOpt, -) []Tensor { - const ctx = CompilationContext.current(); - - const mlir_ctx = ctx.mlirCtx(); - const backend_config = mlir.Attribute.dict(mlir_ctx, &.{ - .{ "callback", .int(mlir_ctx, .u64, @bitCast(@intFromPtr(callback))) }, - .{ "user_context", .int(mlir_ctx, .u64, @bitCast(@intFromPtr(blkctx))) }, - }); - - const values = stdx.stackSlice(8, mlir.Value, inputs.len); - for (inputs, values) |i, *v| { - v.* = ctx.getValue(i.toMemory(.host_pinned)); - } - const res_types = stdx.stackSlice(8, mlir.Type, output_shapes.len); - for (res_types, output_shapes) |*r, o| { - r.* = mlir.ext.RankedTensorType.fromShape(mlir_ctx, o).as(mlir.Type); - } - - const loc = ctx.mlirCtx().location(@src()); - const op = dialect.stablehlo.custom_call( - ctx.mlirCtx(), - values, - .{ - .call_target_name = "zmlHostBufferCallback", - .api_version = .typed_ffi, - .backend_config = backend_config, - .has_side_effect = opts.has_side_effect, - .output_operand_aliases = opts.output_operand_aliases, - }, - res_types, - loc, - ); - - const res = ctx.allocator().alloc(Tensor, output_shapes.len) catch @panic("OOM"); - for (res, output_shapes, 0..) |*r, o, i| { - r.* = Tensor._result(o, op.result(i)).toMemory(.device); - } - - return res; -} - pub const TritonOps = struct { debug: bool = false, name: [:0]const u8, @@ -1253,13 +1201,13 @@ inline fn toI64(values: anytype) []i64 { } const _MINOR_TO_MAJOR = blk: { - var ret: [Shape.MAX_RANK]usize = undefined; + var ret: [Shape.MAX_RANK]i64 = undefined; for (0..Shape.MAX_RANK) |i| { ret[i] = @intCast(Shape.MAX_RANK - i - 1); } break :blk ret; }; -fn minorToMajor(rank: u8) []const usize { +pub fn minorToMajor(rank: u8) []const i64 { return _MINOR_TO_MAJOR[_MINOR_TO_MAJOR.len - rank ..]; } diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index bd094ead..287cc3b2 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -8,6 +8,7 @@ pub const ffi = pjrt.ffi; pub const Profiler = pjrt.Profiler; pub const ApiError = pjrt.ApiError; pub const ErrorCode = pjrt.ErrorCode; +pub const ExecuteContext = pjrt.ExecuteContext; pub const BufferType = pjrt.BufferType; pub const Device = pjrt.Device; pub const DeviceDescription = pjrt.DeviceDescription; @@ -66,6 +67,13 @@ pub const Client = opaque { return @ptrCast(buffer); } + pub fn dmaMap(self: *const Client, api: *const Api, data: []const u8) ApiError!void { + try self.inner().dmaMap(api, data); + } + pub fn dmaUnmap(self: *const Client, api: *const Api, data: []const u8) ApiError!void { + try self.inner().dmaUnmap(api, data); + } + pub fn deserializeAndLoad(self: *const Client, api: *const Api, bytes: []const u8) ApiError!*LoadedExecutable { return @ptrCast(try asynk.callBlocking(pjrt.Client.deserializeAndLoad, .{ self.inner(), api, bytes })); } @@ -182,7 +190,7 @@ pub const Buffer = opaque { } pub fn copyToMemory(self: *const Buffer, api: *const Api, memory_: *const Memory) ApiError!*Buffer { - return @ptrCast(self.inner().copyToMemory(api, memory_)); + return @ptrCast(try self.inner().copyToMemory(api, memory_)); } pub fn getReadyEvent(self: *const Buffer, api: *const Api) ?*Event { @@ -262,6 +270,7 @@ pub const LoadedExecutable = opaque { results: []const [*]*Buffer, events: []?*Event, non_donatable_input_indices: []const i64 = &.{}, + context: ?*ExecuteContext, }; pub fn execute(self: *const LoadedExecutable, api: *const Api, args: ExecuteArgs) ExecuteError!void { @@ -271,6 +280,7 @@ pub const LoadedExecutable = opaque { .results = @ptrCast(args.results), .events = @ptrCast(args.events), .non_donatable_input_indices = args.non_donatable_input_indices, + .context = args.context, } }); } diff --git a/zml/platform.zig b/zml/platform.zig index ec4ae4c6..ba6f3ac3 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -125,6 +125,7 @@ const _CreateOptions = struct { }; pub fn writeNamedValues(self: Cuda, values: *std.ArrayListUnmanaged(pjrt.NamedValue)) void { + values.appendAssumeCapacity(pjrt.NamedValue.fromBool("should_stage_host_to_device_transfers", false)); switch (self.allocator) { .platform => { values.appendAssumeCapacity(pjrt.NamedValue.fromString("allocator", "platform")); diff --git a/zml/tensor.zig b/zml/tensor.zig index b248ea20..72a30324 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -8,8 +8,10 @@ const stdx = @import("stdx"); const Buffer = @import("buffer.zig").Buffer; const Data = @import("dtype.zig").Data; const DataType = @import("dtype.zig").DataType; +const custom_call = @import("ffi.zig").custom_call; const HostBuffer = @import("hostbuffer.zig").HostBuffer; const Memory = @import("buffer.zig").Buffer.Memory; +const ffi = @import("ffi.zig"); const meta = @import("meta.zig"); const mlir = @import("mlir.zig"); const Location = mlir.Location; @@ -206,8 +208,6 @@ pub const Tensor = struct { const ctx = self.getContext(); const mlir_ctx = ctx.mlirCtx(); if (ctx.target() == .cpu) return self; - var res = self; - res._output_memory_kind = kind; const memory_kind = @tagName(kind.toPjrtMemory()); @@ -223,7 +223,9 @@ pub const Tensor = struct { .api_version = .original, }, &.{self.value().getType()}, mlir_ctx.location(@src())); - return _result(res._shape, op.result(0)); + var res = _result(self._shape, op.result(0)); + res._output_memory_kind = kind; + return res; }, .buffer_id => { var res = self; @@ -3782,23 +3784,8 @@ pub const Tensor = struct { }.binaryOpHelper; } - /// Insert code that will print the content of the given buffer at runtime. - /// Only for debug purpose, it inserts device to host synchronization - /// so it will slow down the program execution. - pub fn print(input: Tensor) Tensor { - return ops.addHostCallback( - &printCallback, - null, - &.{input}, - &.{input.shape()}, - .{ .output_operand_aliases = &.{0} }, - )[0]; - } - - fn printCallback(_: ?*anyopaque, inputs: []const HostBuffer, outputs: []const HostBuffer) void { - const host_buffer = inputs[0]; - std.debug.print("Device buffer: {}: {}", .{ host_buffer.shape(), host_buffer.pretty() }); - std.debug.assert(host_buffer.data.ptr == outputs[0].data.ptr); + pub fn print(self: Tensor, print_op: type, another_tensor: Tensor) ffi.CustomCallOutputType(print_op) { + return custom_call(print_op, .{ self, another_tensor }); } }; diff --git a/zml/test_buffer.zig b/zml/test_buffer.zig new file mode 100644 index 00000000..2028e3ff --- /dev/null +++ b/zml/test_buffer.zig @@ -0,0 +1,73 @@ +const std = @import("std"); +const testing = std.testing; +const zml = @import("zml.zig"); +const Buffer = zml.Buffer; +const Shape = zml.Shape; +const DataType = zml.DataType; + +test "Buffer.fromSlice basic usage" { + const platform = try zml.testing.env(); + defer platform.deinit(); + + const data = [_]f32{ 1.0, 2.0, 3.0, 4.0, 5.0, 6.0 }; + var buf = try Buffer.fromSlice(platform, .{ 2, 3 }, &data); + defer buf.deinit(); + + try testing.expectEqual(Shape.init(.{ 2, 3 }, .f32), buf.shape()); + + const result = try buf.getValue([6]f32); + try testing.expectEqualSlices(f32, &data, &result); +} + +test "Buffer.constant creation" { + const platform = try zml.testing.env(); + defer platform.deinit(); + + // Test scalar constant + { + var buf = try Buffer.constant(platform, Shape.init(.{}, .f32), 42.0); + defer buf.deinit(); + + const val = try buf.getValue(f32); + try testing.expectEqual(@as(f32, 42.0), val); + } + + // Test multi-dimensional constant + { + var buf = try Buffer.constant(platform, Shape.init(.{ 2, 3 }, .i32), 7); + defer buf.deinit(); + + const result = try buf.getValue([6]i32); + try testing.expectEqualSlices(i32, &[_]i32{7} ** 6, &result); + } +} + +test "Buffer.fromArray" { + const platform = try zml.testing.env(); + defer platform.deinit(); + + const arr = [2][3]u8{ + [_]u8{ 1, 2, 3 }, + [_]u8{ 4, 5, 6 }, + }; + + var buf = try Buffer.fromArray(platform, arr); + defer buf.deinit(); + + try testing.expectEqual(Shape.init(.{ 2, 3 }, .u8), buf.shape()); + + const result = try buf.getValue([6]u8); + try testing.expectEqualSlices(u8, &[_]u8{ 1, 2, 3, 4, 5, 6 }, &result); +} + +test "Buffer dtype and shape operations" { + const platform = try zml.testing.env(); + defer platform.deinit(); + + var buf = try Buffer.constant(platform, Shape.init(.{ 2, 3, 4 }, .f32), 1.0); + defer buf.deinit(); + + try testing.expectEqual(DataType.f32, buf.dtype()); + try testing.expectEqual(@as(u4, 3), buf.rank()); + try testing.expectEqualSlices(i64, &[_]i64{ 2, 3, 4 }, buf.dims()); +} diff --git a/zml/zml.zig b/zml/zml.zig index 18282b93..38e600a9 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -18,6 +18,7 @@ pub const Tensor = @import("tensor.zig").Tensor; // Namespaces pub const context = @import("context.zig"); +pub const ffi = @import("ffi.zig"); pub const exe = @import("exe.zig"); pub const floats = @import("floats.zig"); pub const helpers = @import("helpers.zig"); @@ -34,6 +35,7 @@ pub const torch = @import("torch.zig"); pub const tokenizer = @import("zml/tokenizer"); pub const call = ops.call; +pub const custom_call = ffi.custom_call; pub const compile = exe.compile; pub const compileFn = exe.compileFn; pub const compileModel = exe.compileModel;