diff --git a/examples/dma/BUILD.bazel b/examples/dma/BUILD.bazel new file mode 100644 index 00000000..7145de12 --- /dev/null +++ b/examples/dma/BUILD.bazel @@ -0,0 +1,11 @@ +load("@zml//bazel:zig.bzl", "zig_cc_binary") + +zig_cc_binary( + name = "dma", + main = "main7.zig", + deps = [ + "@zml//async", + "@zml//stdx", + "@zml//zml", + ], +) diff --git a/examples/dma/main.zig b/examples/dma/main.zig new file mode 100644 index 00000000..671afbbd --- /dev/null +++ b/examples/dma/main.zig @@ -0,0 +1,185 @@ +const std = @import("std"); +const zml = @import("zml"); +const asynk = @import("async"); +const stdx = @import("stdx"); + +// set log level to debug to print the generated IR +pub const std_options = .{ + .log_level = .info, +}; + +pub const Runtime = struct { + cudaProfilerStart: CudaProfilerStart, + cudaProfilerStop: CudaProfilerStop, + + const CudaProfilerStart = *const fn () callconv(.C) c_int; + const CudaProfilerStop = *const fn () callconv(.C) c_int; + + pub fn init() !Runtime { + var cudart = try std.DynLib.open("libcudart.so.12"); + defer cudart.close(); + + return .{ + .cudaProfilerStart = cudart.lookup(Runtime.CudaProfilerStart, "cudaProfilerStart") orelse return error.NotFound, + .cudaProfilerStop = cudart.lookup(Runtime.CudaProfilerStop, "cudaProfilerStop") orelse return error.NotFound, + }; + } +}; + +// __host__​cudaError_t cudaMallocHost ( void** ptr, size_t size ) + +extern fn cudaMallocHost(ptr: **anyopaque, size: usize) c_int; +extern fn cudaMemcpy(dst: ?*anyopaque, src: ?*anyopaque, count: usize, kind: usize) c_int; + +pub fn cudaMallocHostZig(size: usize) []u8 { + var res: []u8 = undefined; + _ = cudaMallocHost(@ptrCast(&(res.ptr)), size); + return res[0..size]; +} + +pub fn toHost(buf: zml.Buffer, dst: []u8) !void { + const shard = buf._shards.get(0); + const pjrt_buf = shard.buffer; + const odp = pjrt_buf.getOpaqueDeviceMemoryDataPointer(shard.api) catch unreachable; + // const src = @ptrCast(odp); + _ = cudaMemcpy(dst.ptr, odp, dst.len, 2); +} + +pub fn benchmark(a: zml.Tensor, b: zml.Tensor) zml.Tensor { + return a.withSharding(.{.k}).dot(b.withSharding(.{.k}), .{.k}).withSharding(.{.m}); +} + +pub fn main() !void { + try asynk.AsyncThread.main(std.heap.c_allocator, asyncMain); +} + +pub fn asyncMain() !void { + // Short lived allocations + var gpa = std.heap.GeneralPurposeAllocator(.{}){}; + defer _ = gpa.deinit(); + const allocator = gpa.allocator(); + + var context = try zml.Context.init(); + defer context.deinit(); + + // Auto-select platform + const platform = context.autoPlatform(.{}).withCompilationOptions(.{ + .sharding_enabled = false, + }); + context.printAvailablePlatforms(platform); + + const size = 8192; + const dtype: zml.DataType = .bf16; + + // const profile: ?[]const u8 = "/home/zml/monorepo-hugo/profile.json"; + // var profiler = if (profile != null) platform.getProfiler(null) else undefined; + // defer profiler.deinit(); + + // if (profile != null) profiler.start(); + + const a_shape = zml.Shape.init(.{ size, size }, dtype).withTags(.{ .m, .k }).withSharding(.{.k}); + const b_shape = a_shape.withTags(.{ .k, .n }).withSharding(.{.k}); + + var compilation = try asynk.asyncc(zml.compileFn, .{ allocator, benchmark, .{ a_shape, b_shape }, platform }); + + // Wait for compilation to finish + const executable = try compilation.awaitt(); + defer executable.deinit(); + + var rng = std.Random.DefaultPrng.init(0); + const random = rng.random(); + + var a_buffer = try createRandomBuffer(allocator, platform, a_shape, random); + defer a_buffer.deinit(); + var b_buffer = try createRandomBuffer(allocator, platform, b_shape, random); + defer b_buffer.deinit(); + + const result_host_buffer_data = cudaMallocHostZig(a_shape.byteSize()); + + // const result_host_buffer_data = try allocator.alloc(u8, a_shape.byteSize()); + // defer allocator.free(result_host_buffer_data); + + const result_host_buffer2_data = try allocator.alloc(u8, a_shape.byteSize()); + defer allocator.free(result_host_buffer2_data); + const result_host_buffer3_data = try allocator.alloc(u8, a_shape.byteSize()); + defer allocator.free(result_host_buffer3_data); + + // Ignore first run + { + var result: zml.Buffer = executable.call(.{ a_buffer, b_buffer }); + defer result.deinit(); + try asynk.sleep(1000); // to see more quickly stuffs in the profiling ui + } + + // for (0..10) |i| { + // _ = i; // autofix + // bazel run -c opt --@zml//runtimes:cuda=true --@zml//runtimes:cpu=false --run_under="sudo /opt/nvidia/nsight-systems-cli/2025.1.1/bin/nsys profile -t cuda,nvtx,cublas,cublas-verbose,cusparse,cusparse-verbose,cudnn --gpu-metrics-devices='cuda-visible' --cuda-memory-usage true --cuda-graph-trace=node --capture-range=cudaProfilerApi --capture-range-end=stop" //dma + const runtime = Runtime.init() catch unreachable; + + { + var result: zml.Buffer = executable.call(.{ a_buffer, b_buffer }); + // var result2: zml.Buffer = executable.call(.{ result, b_buffer }); + // var result3: zml.Buffer = executable.call(.{ result, result2 }); + // var timer = try stdx.time.Timer.start(); + _ = runtime.cudaProfilerStart(); + // var hb = try result.toHost(result_host_buffer_data); + try toHost(result, result_host_buffer_data); + // std.log.info("Time result.toHost: {}\n", .{timer.read()}); + // var hb2 = try result2.toHost(result_host_buffer2_data); + // std.log.info("Time result2.toHost: {}\n", .{timer.read()}); + // var hb3 = try result3.toHost(result_host_buffer3_data); + // std.log.info("Time result3.toHost: {}\n", .{timer.read()}); + // _ = try hb.awaitt(); + // std.log.info("Time hb.awaitt: {}\n", .{timer.read()}); + // _ = try hb2.awaitt(); + // std.log.info("Time hb2.awaitt: {}\n", .{timer.read()}); + // _ = try hb3.awaitt(); + // std.log.info("Time hb3.awaitt: {}\n", .{timer.read()}); + _ = runtime.cudaProfilerStop(); + defer result.deinit(); + // defer result2.deinit(); + // defer result3.deinit(); + } + // } + std.log.info("Time hb3.awaitt: \n", .{}); + + try asynk.sleep(1000); // to see more quickly stuffs in the profiling ui + + // if (profile) |profile_file| { + // profiler.stop(); + // try profiler.dumpAsJsonTo(allocator, std.fs.cwd(), profile_file); + // } +} + +fn createRandomBuffer(allocator: std.mem.Allocator, platform: zml.Platform, shape: zml.Shape, random: std.Random) !zml.Buffer { + const data = try allocator.alloc(u8, shape.byteSize()); + defer allocator.free(data); + + switch (shape.dtype()) { + inline else => |v| { + const ZigType = v.toZigType(); + switch (comptime v.class()) { + .bool => unreachable, + .integer => { + for (std.mem.bytesAsSlice(ZigType, data)) |*e| e.* = random.int(ZigType); + }, + .float => { + const value = random.float(f64); + for (std.mem.bytesAsSlice(ZigType, data)) |*e| e.* = if (ZigType == f64) + value + else if (ZigType == f32) + @floatCast(value) + else if (ZigType == f16) + @floatCast(value) + else + @bitCast(random.int(std.meta.Int(.unsigned, @bitSizeOf(ZigType)))); + }, + .complex => unreachable, + } + }, + } + + var host_buffer = zml.HostBuffer.fromBytes(shape, data); + errdefer host_buffer.deinit(allocator); + return zml.Buffer.from(platform, host_buffer); +} diff --git a/examples/dma/main2.zig b/examples/dma/main2.zig new file mode 100644 index 00000000..63541044 --- /dev/null +++ b/examples/dma/main2.zig @@ -0,0 +1,156 @@ +const std = @import("std"); +const zml = @import("zml"); +const asynk = @import("async"); +const stdx = @import("stdx"); + +// set log level to debug to print the generated IR +pub const std_options = .{ + .log_level = .info, +}; + +pub const Runtime = struct { + cudaProfilerStart: CudaProfilerStart, + cudaProfilerStop: CudaProfilerStop, + + const CudaProfilerStart = *const fn () callconv(.C) c_int; + const CudaProfilerStop = *const fn () callconv(.C) c_int; + + pub fn init() !Runtime { + var cudart = try std.DynLib.open("libcudart.so.12"); + defer cudart.close(); + + return .{ + .cudaProfilerStart = cudart.lookup(Runtime.CudaProfilerStart, "cudaProfilerStart") orelse return error.NotFound, + .cudaProfilerStop = cudart.lookup(Runtime.CudaProfilerStop, "cudaProfilerStop") orelse return error.NotFound, + }; + } +}; + +extern fn cudaMallocHost(ptr: **anyopaque, size: usize) c_int; +extern fn cudaMemcpy(dst: ?*anyopaque, src: ?*anyopaque, count: usize, kind: usize) c_int; +extern fn cudaFreeHost(ptr: *anyopaque) c_int; + +pub fn cudaMallocHostZig(size: usize) []u8 { + var res: []u8 = undefined; + _ = cudaMallocHost(@ptrCast(&(res.ptr)), size); + return res[0..size]; +} + +pub fn cudaFreeHostZig(ptr: *anyopaque) void { + _ = cudaFreeHost(ptr); +} + +pub fn toHost(buf: zml.Buffer, dst: []u8) !void { + const shard = buf._shards.get(0); + const pjrt_buf = shard.buffer; + const odp = pjrt_buf.getOpaqueDeviceMemoryDataPointer(shard.api) catch unreachable; + _ = cudaMemcpy(dst.ptr, odp, dst.len, 2); +} + +pub fn benchmark(a: zml.Tensor, b: zml.Tensor) zml.Tensor { + return a.withSharding(.{.k}).dot(b.withSharding(.{.k}), .{.k}).withSharding(.{.m}); +} + +pub fn main() !void { + try asynk.AsyncThread.main(std.heap.c_allocator, asyncMain); +} + +pub fn asyncMain() !void { + // Short lived allocations + var gpa = std.heap.GeneralPurposeAllocator(.{}){}; + defer _ = gpa.deinit(); + const allocator = gpa.allocator(); + + var context = try zml.Context.init(); + defer context.deinit(); + + // Auto-select platform + const platform = context.autoPlatform(.{}).withCompilationOptions(.{ + .sharding_enabled = false, + }); + context.printAvailablePlatforms(platform); + + const size = 8192; + const dtype: zml.DataType = .bf16; + + const a_shape = zml.Shape.init(.{ size, size }, dtype).withTags(.{ .m, .k }).withSharding(.{.k}); + const b_shape = a_shape.withTags(.{ .k, .n }).withSharding(.{.k}); + + var compilation = try asynk.asyncc(zml.compileFn, .{ allocator, benchmark, .{ a_shape, b_shape }, platform }); + + // Wait for compilation to finish + const executable = try compilation.awaitt(); + defer executable.deinit(); + + var rng = std.Random.DefaultPrng.init(0); + const random = rng.random(); + + var a_buffer = try createRandomBuffer(allocator, platform, a_shape, random); + defer a_buffer.deinit(); + var b_buffer = try createRandomBuffer(allocator, platform, b_shape, random); + defer b_buffer.deinit(); + + { + var result: zml.Buffer = executable.call(.{ a_buffer, b_buffer }); + defer result.deinit(); + } + + // bazel run -c opt --@zml//runtimes:cuda=true --@zml//runtimes:cpu=false --run_under="sudo /opt/nvidia/nsight-systems-cli/2025.1.1/bin/nsys profile -t cuda,nvtx,cublas,cublas-verbose,cusparse,cusparse-verbose,cudnn --gpu-metrics-devices='cuda-visible' --cuda-memory-usage true --cuda-graph-trace=node --capture-range=cudaProfilerApi --capture-range-end=stop" //dma + const runtime = Runtime.init() catch unreachable; + + _ = runtime.cudaProfilerStart(); + + const result_host_buffer_data = cudaMallocHostZig(a_shape.byteSize()); + defer cudaFreeHostZig(result_host_buffer_data.ptr); + const result_host_buffer2_data = cudaMallocHostZig(a_shape.byteSize()); + defer cudaFreeHostZig(result_host_buffer2_data.ptr); + const result_host_buffer3_data = cudaMallocHostZig(a_shape.byteSize()); + defer cudaFreeHostZig(result_host_buffer3_data.ptr); + + { + var result: zml.Buffer = executable.call(.{ a_buffer, b_buffer }); + _ = try result.awaitt(); + try toHost(result, result_host_buffer_data); + var result2: zml.Buffer = executable.call(.{ result, b_buffer }); + var result3: zml.Buffer = executable.call(.{ result, result2 }); + try toHost(result2, result_host_buffer2_data); + try toHost(result3, result_host_buffer3_data); + defer result.deinit(); + defer result2.deinit(); + defer result3.deinit(); + } + _ = runtime.cudaProfilerStop(); +} + +fn createRandomBuffer(allocator: std.mem.Allocator, platform: zml.Platform, shape: zml.Shape, random: std.Random) !zml.Buffer { + const data = try allocator.alloc(u8, shape.byteSize()); + defer allocator.free(data); + + switch (shape.dtype()) { + inline else => |v| { + const ZigType = v.toZigType(); + switch (comptime v.class()) { + .bool => unreachable, + .integer => { + for (std.mem.bytesAsSlice(ZigType, data)) |*e| e.* = random.int(ZigType); + }, + .float => { + const value = random.float(f64); + for (std.mem.bytesAsSlice(ZigType, data)) |*e| e.* = if (ZigType == f64) + value + else if (ZigType == f32) + @floatCast(value) + else if (ZigType == f16) + @floatCast(value) + else + @bitCast(random.int(std.meta.Int(.unsigned, @bitSizeOf(ZigType)))); + }, + .complex => unreachable, + } + }, + } + + var host_buffer = zml.HostBuffer.fromBytes(shape, data); + errdefer host_buffer.deinit(allocator); + return zml.Buffer.from(platform, host_buffer); +} diff --git a/examples/dma/main3.zig b/examples/dma/main3.zig new file mode 100644 index 00000000..3d971994 --- /dev/null +++ b/examples/dma/main3.zig @@ -0,0 +1,170 @@ +const std = @import("std"); +const zml = @import("zml"); +const asynk = @import("async"); +const stdx = @import("stdx"); + +// set log level to debug to print the generated IR +pub const std_options = .{ + .log_level = .info, +}; + +pub const Runtime = struct { + cudaProfilerStart: CudaProfilerStart, + cudaProfilerStop: CudaProfilerStop, + + const CudaProfilerStart = *const fn () callconv(.C) c_int; + const CudaProfilerStop = *const fn () callconv(.C) c_int; + + pub fn init() !Runtime { + var cudart = try std.DynLib.open("libcudart.so.12"); + defer cudart.close(); + + return .{ + .cudaProfilerStart = cudart.lookup(Runtime.CudaProfilerStart, "cudaProfilerStart") orelse return error.NotFound, + .cudaProfilerStop = cudart.lookup(Runtime.CudaProfilerStop, "cudaProfilerStop") orelse return error.NotFound, + }; + } +}; + +extern fn cudaMallocHost(ptr: **anyopaque, size: usize) c_int; +extern fn cudaFreeHost(ptr: *anyopaque) c_int; + +pub fn cudaMallocHostZig(size: usize) []u8 { + var res: []u8 = undefined; + _ = cudaMallocHost(@ptrCast(&(res.ptr)), size); + return res[0..size]; +} + +pub fn cudaFreeHostZig(ptr: *anyopaque) void { + _ = cudaFreeHost(ptr); +} + +pub fn benchmark(a: zml.Tensor, b: zml.Tensor) zml.Tensor { + return a.withSharding(.{.k}).dot(b.withSharding(.{.k}), .{.k}).withSharding(.{.m}); +} + +pub fn main() !void { + try asynk.AsyncThread.main(std.heap.c_allocator, asyncMain); +} + +pub fn asyncMain() !void { + // Short lived allocations + var gpa = std.heap.GeneralPurposeAllocator(.{}){}; + defer _ = gpa.deinit(); + const allocator = gpa.allocator(); + + var context = try zml.Context.init(); + defer context.deinit(); + + // Auto-select platform + const platform = context.autoPlatform(.{}).withCompilationOptions(.{ + .sharding_enabled = false, + }); + context.printAvailablePlatforms(platform); + + const size = 8192; + const dtype: zml.DataType = .bf16; + + const a_shape = zml.Shape.init(.{ size, size }, dtype).withTags(.{ .m, .k }).withSharding(.{.k}); + const b_shape = a_shape.withTags(.{ .k, .n }).withSharding(.{.k}); + + var compilation = try asynk.asyncc(zml.compileFn, .{ allocator, benchmark, .{ a_shape, b_shape }, platform }); + + // Wait for compilation to finish + const executable = try compilation.awaitt(); + defer executable.deinit(); + + var rng = std.Random.DefaultPrng.init(0); + const random = rng.random(); + + var a_buffer = try createRandomBuffer(allocator, platform, a_shape, random); + defer a_buffer.deinit(); + var b_buffer = try createRandomBuffer(allocator, platform, b_shape, random); + defer b_buffer.deinit(); + + const result_host_buffer_data = cudaMallocHostZig(a_shape.byteSize()); + + var result: zml.Buffer = executable.call(.{ a_buffer, b_buffer }); + var hb = try result.toHostEx(result_host_buffer_data, .{}); + _ = try hb.awaitt(); + defer result.deinit(); + + { + // bazel run -c opt --@zml//runtimes:cuda=true --@zml//runtimes:cpu=false --run_under="sudo /opt/nvidia/nsight-systems-cli/2025.1.1/bin/nsys profile -t cuda,nvtx,cublas,cublas-verbose,cusparse,cusparse-verbose,cudnn --gpu-metrics-devices='cuda-visible' --cuda-memory-usage true --cuda-graph-trace=node --capture-range=cudaProfilerApi --capture-range-end=stop" //dma + const runtime = Runtime.init() catch unreachable; + _ = runtime; // autofix + + // _ = runtime.cudaProfilerStart(); + + const profile: ?[]const u8 = "/home/zml/monorepo-hugo/profileXX.json"; + var profiler = if (profile != null) platform.getProfiler(null) else undefined; + defer profiler.deinit(); + + const result_host_buffer1_data = cudaMallocHostZig(a_shape.byteSize()); + defer cudaFreeHostZig(result_host_buffer1_data.ptr); + const result_host_buffer2_data = cudaMallocHostZig(a_shape.byteSize()); + defer cudaFreeHostZig(result_host_buffer2_data.ptr); + const result_host_buffer3_data = cudaMallocHostZig(a_shape.byteSize()); + defer cudaFreeHostZig(result_host_buffer3_data.ptr); + + if (profile != null) profiler.start(); + + var result1: zml.Buffer = executable.call(.{ a_buffer, b_buffer }); + var hb1 = try result.toHostEx(result_host_buffer1_data, .{}); + var result2: zml.Buffer = executable.call(.{ result1, b_buffer }); + var result3: zml.Buffer = executable.call(.{ a_buffer, result1 }); + var hb2 = try result2.toHostEx(result_host_buffer2_data, .{}); + _ = try hb1.awaitt(); + _ = try hb2.awaitt(); + var result4: zml.Buffer = executable.call(.{ result, result2 }); + var hb3 = try result3.toHostEx(result_host_buffer3_data, .{}); + var hb4 = try result4.toHostEx(result_host_buffer3_data, .{}); + _ = try hb3.awaitt(); + _ = try hb4.awaitt(); + + defer result1.deinit(); + defer result2.deinit(); + defer result3.deinit(); + defer result4.deinit(); + + if (profile) |profile_file| { + profiler.stop(); + try profiler.dumpAsJsonTo(allocator, std.fs.cwd(), profile_file); + } + + // _ = runtime.cudaProfilerStop(); + } +} + +fn createRandomBuffer(allocator: std.mem.Allocator, platform: zml.Platform, shape: zml.Shape, random: std.Random) !zml.Buffer { + const data = try allocator.alloc(u8, shape.byteSize()); + defer allocator.free(data); + + switch (shape.dtype()) { + inline else => |v| { + const ZigType = v.toZigType(); + switch (comptime v.class()) { + .bool => unreachable, + .integer => { + for (std.mem.bytesAsSlice(ZigType, data)) |*e| e.* = random.int(ZigType); + }, + .float => { + const value = 1; + for (std.mem.bytesAsSlice(ZigType, data)) |*e| e.* = if (ZigType == f64) + value + else if (ZigType == f32) + @floatCast(value) + else if (ZigType == f16) + @floatCast(value) + else + @bitCast(random.int(std.meta.Int(.unsigned, @bitSizeOf(ZigType)))); + }, + .complex => unreachable, + } + }, + } + + var host_buffer = zml.HostBuffer.fromBytes(shape, data); + errdefer host_buffer.deinit(allocator); + return zml.Buffer.from(platform, host_buffer); +} diff --git a/examples/dma/main4.zig b/examples/dma/main4.zig new file mode 100644 index 00000000..4adcdbbe --- /dev/null +++ b/examples/dma/main4.zig @@ -0,0 +1,120 @@ +const std = @import("std"); +const zml = @import("zml"); +const asynk = @import("async"); +const stdx = @import("stdx"); + +// set log level to debug to print the generated IR +pub const std_options = .{ + .log_level = .info, +}; + +pub const Runtime = struct { + cudaProfilerStart: CudaProfilerStart, + cudaProfilerStop: CudaProfilerStop, + + const CudaProfilerStart = *const fn () callconv(.C) c_int; + const CudaProfilerStop = *const fn () callconv(.C) c_int; + + pub fn init() !Runtime { + var cudart = try std.DynLib.open("libcudart.so.12"); + defer cudart.close(); + + return .{ + .cudaProfilerStart = cudart.lookup(Runtime.CudaProfilerStart, "cudaProfilerStart") orelse return error.NotFound, + .cudaProfilerStop = cudart.lookup(Runtime.CudaProfilerStop, "cudaProfilerStop") orelse return error.NotFound, + }; + } +}; + +extern fn cudaHostRegister(ptr: **anyopaque, size: usize, flags: c_int) c_int; +extern fn cudaMallocHost(ptr: **anyopaque, size: usize) c_int; +extern fn cudaMemcpy(dst: ?*anyopaque, src: ?*anyopaque, count: usize, kind: usize) c_int; +extern fn cudaFreeHost(ptr: *anyopaque) c_int; + +pub fn cudaMallocHostZig(size: usize) []u8 { + var res: []u8 = undefined; + _ = cudaMallocHost(@ptrCast(&(res.ptr)), size); + return res[0..size]; +} + +pub fn cudaFreeHostZig(ptr: *anyopaque) void { + _ = cudaFreeHost(ptr); +} + +pub fn benchmark(a: zml.Tensor, b: zml.Tensor) zml.Tensor { + return a.withSharding(.{.k}).dot(b.withSharding(.{.k}), .{.k}).withSharding(.{.m}); +} + +pub fn main() !void { + try asynk.AsyncThread.main(std.heap.c_allocator, asyncMain); +} + +pub fn asyncMain() !void { + const HAllocator = @import("halloc.zig"); + const halloc: std.mem.Allocator = .{ + .ptr = undefined, + .vtable = &HAllocator.vtable, + }; + + // Short lived allocations + var gpa = std.heap.GeneralPurposeAllocator(.{}){}; + defer _ = gpa.deinit(); + const allocator = gpa.allocator(); + + var context = try zml.Context.init(); + defer context.deinit(); + + // Auto-select platform + const platform = context.autoPlatform(.{}).withCompilationOptions(.{ + .sharding_enabled = false, + }); + context.printAvailablePlatforms(platform); + + const size = 4096; + const dtype: zml.DataType = .f32; + + const a_shape = zml.Shape.init(.{ size, size }, dtype).withTags(.{ .m, .k }).withSharding(.{.k}); + const b_shape = a_shape.withTags(.{ .k, .n }).withSharding(.{.k}); + + var rng = std.Random.DefaultPrng.init(0); + const random = rng.random(); + + { + const data_a = try allocator.alloc(u8, a_shape.byteSize()); + const data_b = try allocator.alloc(u8, a_shape.byteSize()); + + for (std.mem.bytesAsSlice(f32, data_a)) |*e| e.* = random.float(f32); + for (std.mem.bytesAsSlice(f32, data_b)) |*e| e.* = random.float(f32); + + var host_buffer_a = zml.HostBuffer.fromBytes(a_shape, data_a); + defer host_buffer_a.deinit(allocator); + var buffer_a = try zml.Buffer.from(platform, host_buffer_a); + defer buffer_a.deinit(); + _ = try buffer_a.awaitt(); + std.debug.print("buffer_a: {any}\n", .{buffer_a}); + + var host_buffer_b = zml.HostBuffer.fromBytes(a_shape, data_a); + defer host_buffer_b.deinit(allocator); + var buffer_b = try zml.Buffer.from(platform, host_buffer_b); + _ = try buffer_b.awaitt(); + std.debug.print("buffer_b: {any}\n", .{buffer_b}); + buffer_b.deinit(); + + const result_host_buffer_data = cudaMallocHostZig(a_shape.byteSize()); + // defer result_host_buffer_data.deinit(); + // _ = cudaHostRegister(@ptrCast(&(result_host_buffer_data.ptr)), result_host_buffer_data.len, 2); + + var executable = try zml.compileFn(allocator, halloc, benchmark, .{ a_shape, b_shape }, platform); + // Wait for compilation to finish + // const executable = try compilation.awaitt(); + defer executable.deinit(); + + var result: zml.Buffer = executable.call(.{ buffer_a, buffer_b }); + + var result_host_buffer = try result.toHostEx(result_host_buffer_data, .{}); + _ = try result_host_buffer.awaitt(); + std.debug.print("result_host_buffer: {any}\n", .{result_host_buffer}); + } + + std.debug.print("end\n", .{}); +} diff --git a/examples/dma/main5.zig b/examples/dma/main5.zig new file mode 100644 index 00000000..b905a7d4 --- /dev/null +++ b/examples/dma/main5.zig @@ -0,0 +1,192 @@ +const std = @import("std"); +const zml = @import("zml"); +const asynk = @import("async"); +const stdx = @import("stdx"); + +// set log level to debug to print the generated IR +pub const std_options = .{ + .log_level = .info, +}; + +pub const Runtime = struct { + cudaProfilerStart: CudaProfilerStart, + cudaProfilerStop: CudaProfilerStop, + + const CudaProfilerStart = *const fn () callconv(.C) c_int; + const CudaProfilerStop = *const fn () callconv(.C) c_int; + + pub fn init() !Runtime { + var cudart = try std.DynLib.open("libcudart.so.12"); + defer cudart.close(); + + return .{ + .cudaProfilerStart = cudart.lookup(Runtime.CudaProfilerStart, "cudaProfilerStart") orelse return error.NotFound, + .cudaProfilerStop = cudart.lookup(Runtime.CudaProfilerStop, "cudaProfilerStop") orelse return error.NotFound, + }; + } +}; + +extern fn cudaHostRegister(ptr: **anyopaque, size: usize, flags: c_int) c_int; +extern fn cudaMallocHost(ptr: **anyopaque, size: usize) c_int; +extern fn cudaMemcpy(dst: ?*anyopaque, src: ?*anyopaque, count: usize, kind: usize) c_int; +extern fn cudaFreeHost(ptr: *anyopaque) c_int; + +pub fn cudaMallocHostZig(size: usize) []u8 { + var res: []u8 = undefined; + _ = cudaMallocHost(@ptrCast(&(res.ptr)), size); + return res[0..size]; +} + +pub fn cudaFreeHostZig(ptr: *anyopaque) void { + _ = cudaFreeHost(ptr); +} + +pub fn benchmark(a: zml.Tensor, b: zml.Tensor) zml.Tensor { + // return a.add(b); + return a.withSharding(.{.k}).dot(b.withSharding(.{.k}), .{.k}).withSharding(.{.m}); +} + +pub fn main() !void { + try asynk.AsyncThread.main(std.heap.c_allocator, asyncMain); +} + +pub fn asyncMain() !void { + // Short lived allocations + var gpa = std.heap.GeneralPurposeAllocator(.{}){}; + defer _ = gpa.deinit(); + const allocator = gpa.allocator(); + + const HAllocator = @import("halloc.zig"); + const halloc: std.mem.Allocator = .{ + .ptr = undefined, + .vtable = &HAllocator.vtable, + }; + + var context = try zml.Context.init(); + defer context.deinit(); + + // Auto-select platform + const platform = context.autoPlatform(.{}).withCompilationOptions(.{ + .sharding_enabled = false, + }); + context.printAvailablePlatforms(platform); + + const size = 8192; + const dtype: zml.DataType = .f16; + + const a_shape = zml.Shape.init(.{ size, size }, dtype).withTags(.{ .m, .k }).withSharding(.{.k}); + const b_shape = a_shape.withTags(.{ .k, .n }).withSharding(.{.k}); + + var compilation = try asynk.asyncc(zml.compileFn, .{ allocator, halloc, benchmark, .{ a_shape, b_shape }, platform }); + + // Wait for compilation to finish + const executable = try compilation.awaitt(); + defer executable.deinit(); + + var rng = std.Random.DefaultPrng.init(0); + const random = rng.random(); + + // bazel run -c opt --@zml//runtimes:cuda=true --@zml//runtimes:cpu=false --run_under="sudo /opt/nvidia/nsight-systems-cli/2025.1.1/bin/nsys profile -t cuda,nvtx,cublas,cublas-verbose,cusparse,cusparse-verbose,cudnn --gpu-metrics-devices='cuda-visible' --cuda-memory-usage true --cuda-graph-trace=node --capture-range=cudaProfilerApi --capture-range-end=stop" //dma + const runtime = Runtime.init() catch unreachable; + + _ = runtime.cudaProfilerStart(); + + var a_host_buffer = try createRandomBuffer(allocator, a_shape, random); + defer a_host_buffer.deinit(); + + var b_host_buffer = try createRandomBuffer(allocator, platform, b_shape, random); + defer b_host_buffer.deinit(); + + var a_buffer = try zml.Buffer.from(platform, a_host_buffer); + defer a_buffer.deinit(); + var b_buffer = try zml.Buffer.from(platform, a_host_buffer); + defer b_buffer.deinit(); + + var result: zml.Buffer = executable.call(.{ a_buffer, b_buffer }); + _ = try result.awaitt(); + _ = runtime.cudaProfilerStop(); + + const result_host_buffer_data = cudaMallocHostZig(a_shape.byteSize()); + var hb = try result.toHost2(result_host_buffer_data); + _ = try hb.awaitt(); + defer result.deinit(); + + { + const result_host_buffer1_data = cudaMallocHostZig(a_shape.byteSize()); + const result_host_buffer2_data = cudaMallocHostZig(a_shape.byteSize()); + // const result_host_buffer2_data = try halloc.alloc(u8, a_shape.byteSize()); + // const result_host_buffer3_data = cudaMallocHostZig(a_shape.byteSize()); + + // var result_host_buffer1_data = try allocator.alloc(u8, a_shape.byteSize()); + // defer allocator.free(result_host_buffer1_data); + // const result_host_buffer2_data = try allocator.alloc(u8, a_shape.byteSize()); + // defer allocator.free(result_host_buffer2_data); + // const result_host_buffer3_data = try allocator.alloc(u8, a_shape.byteSize()); + // defer allocator.free(result_host_buffer3_data); + + // _ = runtime.cudaProfilerStart(); + // const profile: ?[]const u8 = "/home/zml/monorepo-hugo/profileXX.json"; + // var profiler = if (profile != null) platform.getProfiler(null) else undefined; + // defer profiler.deinit(); + + // if (profile != null) profiler.start(); + + // _ = cudaHostRegister(@ptrCast(&(result_host_buffer1_data.ptr)), result_host_buffer1_data.len, 2); + + var result1: zml.Buffer = executable.call(.{ a_buffer, b_buffer }); + var hb1 = try result1.toHost2(result_host_buffer1_data); + var result2: zml.Buffer = executable.call(.{ result1, b_buffer }); + var hb2 = try result2.toHost2(result_host_buffer2_data); + // var result3: zml.Buffer = executable.call(.{ result, result2 }); + // var hb3 = try result3.toHost2(result_host_buffer3_data); + // var result4: zml.Buffer = executable.call(.{ a_buffer, result1 }); + _ = try hb1.awaitt(); + _ = try hb2.awaitt(); + // _ = try hb3.awaitt(); + // var result3: zml.Buffer = executable.call(.{ result, result2 }); + // var hb4 = try result4.toHost2(result_host_buffer3_data); + // _ = try hb4.awaitt(); + defer result1.deinit(); + // defer result2.deinit(); + // defer result3.deinit(); + // defer result4.deinit(); + // if (profile) |profile_file| { + // profiler.stop(); + // try profiler.dumpAsJsonTo(allocator, std.fs.cwd(), profile_file); + // } + } +} + +fn createRandomBuffer(allocator: std.mem.Allocator, shape: zml.Shape, random: std.Random) !zml.HostBuffer { + const data = try allocator.alloc(u8, shape.byteSize()); + defer allocator.free(data); + + switch (shape.dtype()) { + inline else => |v| { + const ZigType = v.toZigType(); + switch (comptime v.class()) { + .bool => unreachable, + .integer => { + for (std.mem.bytesAsSlice(ZigType, data)) |*e| e.* = random.int(ZigType); + }, + .float => { + const value = 1; + for (std.mem.bytesAsSlice(ZigType, data)) |*e| e.* = if (ZigType == f64) + value + else if (ZigType == f32) + @floatCast(value) + else if (ZigType == f16) + @floatCast(value) + else + @bitCast(random.int(std.meta.Int(.unsigned, @bitSizeOf(ZigType)))); + }, + .complex => unreachable, + } + }, + } + + // std.debug.print("data: {any}\n", .{std.mem.bytesAsSlice(f16, data)[0..100]}); + var host_buffer = zml.HostBuffer.fromBytes(shape, data); + errdefer host_buffer.deinit(allocator); + return host_buffer; +} diff --git a/examples/dma/main6.zig b/examples/dma/main6.zig new file mode 100644 index 00000000..f9130187 --- /dev/null +++ b/examples/dma/main6.zig @@ -0,0 +1,284 @@ +const std = @import("std"); +const zml = @import("zml"); +const asynk = @import("async"); +const stdx = @import("stdx"); + +// set log level to debug to print the generated IR +pub const std_options = .{ + .log_level = .info, +}; + +pub const Runtime = struct { + cudaProfilerStart: CudaProfilerStart, + cudaProfilerStop: CudaProfilerStop, + + const CudaProfilerStart = *const fn () callconv(.C) c_int; + const CudaProfilerStop = *const fn () callconv(.C) c_int; + + pub fn init() !Runtime { + var cudart = try std.DynLib.open("libcudart.so.12"); + defer cudart.close(); + + return .{ + .cudaProfilerStart = cudart.lookup(Runtime.CudaProfilerStart, "cudaProfilerStart") orelse return error.NotFound, + .cudaProfilerStop = cudart.lookup(Runtime.CudaProfilerStop, "cudaProfilerStop") orelse return error.NotFound, + }; + } +}; + +pub fn benchmark(a: zml.Tensor, b: zml.Tensor) zml.Tensor { + var output = a; + // for (0..10) |_| { + // output = a.mul(b); + // } + // return output.toMemory(.pinned_host); + for (0..10) |_| { + output = output.add(a.dot(b, .{.k})); + } + return output.toMemory(.pinned_host); +} + +pub fn main() !void { + try asynk.AsyncThread.main(std.heap.c_allocator, asyncMain); +} + +pub fn asyncMain() !void { + // Short lived allocations + var gpa = std.heap.GeneralPurposeAllocator(.{}){}; + defer _ = gpa.deinit(); + const allocator = gpa.allocator(); + + var context = try zml.Context.init(); + defer context.deinit(); + + // Auto-select platform + const platform = context.autoPlatform(.{}).withCompilationOptions(.{ + .sharding_enabled = false, + }); + context.printAvailablePlatforms(platform); + + // const profile: ?[]const u8 = "/home/zml/monorepo-hugo/profile-hugo.json"; + // var profiler = if (profile != null) platform.getProfiler(null) else undefined; + // defer profiler.deinit(); + + const runtime = Runtime.init() catch unreachable; + + const size = 8192; + const dtype: zml.DataType = .bf16; + + const a_shape = zml.Shape.init(.{ size, size }, dtype).withTags(.{ .m, .k }).withSharding(.{.k}); + const b_shape = a_shape.withTags(.{ .k, .n }).withSharding(.{.k}); + + // Wait for compilation to finish + const executable = try zml.compileFn(allocator, benchmark, .{ a_shape, b_shape }, platform); + defer executable.deinit(); + + var rng = std.Random.DefaultPrng.init(0); + const random = rng.random(); + + const random_buffer_a = try createRandomBuffer(allocator, a_shape, random); + defer allocator.free(random_buffer_a); + var buffer_a = try zml.Buffer.fromToMemory(platform, random_buffer_a, a_shape, .device); + defer buffer_a.deinit(); + _ = try buffer_a.awaitt(); + std.debug.print("buffer_a: {any}\n", .{buffer_a}); + + const random_buffer_b = try createRandomBuffer(allocator, a_shape, random); + defer allocator.free(random_buffer_b); + var buffer_b = try zml.Buffer.fromToMemory(platform, random_buffer_a, a_shape, .device); + defer buffer_b.deinit(); + _ = try buffer_b.awaitt(); + std.debug.print("buffer_b: {any}\n", .{buffer_b}); + + const result_host_buffer_data = try allocator.alloc(u8, a_shape.byteSize()); + defer allocator.free(result_host_buffer_data); + _ = runtime.cudaProfilerStart(); + var result: zml.Buffer = executable.call(.{ buffer_a, buffer_b }); + std.debug.print("result _memory: {s}\n", .{result.getMemory().debugString(platform.pjrt_api)}); + const hb = try result.copyToMemory(platform, zml.Buffer.MemoryKind.device); + var result2: zml.Buffer = executable.call(.{ hb, buffer_b }); + std.debug.print("result2 _memory: {s}\n", .{result2.getMemory().debugString(platform.pjrt_api)}); + // _ = try hb.awaitt(); + defer result.deinit(); + _ = runtime.cudaProfilerStop(); + std.debug.print("Warmup finished\n", .{}); + + // try asynk.sleep(10_000); + + // { + // const result_host_buffer1_data = try allocator.alloc(u8, a_shape.byteSize()); + // defer allocator.free(result_host_buffer1_data); + // const result_host_buffer2_data = try allocator.alloc(u8, a_shape.byteSize()); + // defer allocator.free(result_host_buffer2_data); + // const result_host_buffer3_data = try allocator.alloc(u8, a_shape.byteSize()); + // defer allocator.free(result_host_buffer3_data); + + // if (profile != null) profiler.start(); + + // var result1: zml.Buffer = executable.call(.{ buffer_a, buffer_b }); + // defer result1.deinit(); + + // std.debug.print("result1 _memory: {s}\n", .{result1._memory.debugString(platform.pjrt_api)}); + // var hb1 = try result1.copyToMemory(platform, zml.Buffer.MemoryKind.pinned_host); + // _ = try hb1.awaitt(); + // std.debug.print("hb1 _memory: {s}\n", .{hb1._memory.debugString(platform.pjrt_api)}); + + // if (profile) |profile_file| { + // profiler.stop(); + // try profiler.dumpAsJsonTo(allocator, std.fs.cwd(), profile_file); + // } + // } + + // { + // const result_host_buffer1_data = try allocator.alloc(u8, a_shape.byteSize()); + // defer allocator.free(result_host_buffer1_data); + // const result_host_buffer2_data = try allocator.alloc(u8, a_shape.byteSize()); + // defer allocator.free(result_host_buffer2_data); + // const result_host_buffer3_data = try allocator.alloc(u8, a_shape.byteSize()); + // defer allocator.free(result_host_buffer3_data); + + // // if (profile != null) profiler.start(); + // _ = runtime.cudaProfilerStart(); + + // var result1: zml.Buffer = executable.call(.{ buffer_a, buffer_b }); + // defer result1.deinit(); + // var result2: zml.Buffer = executable.call(.{ result1, buffer_b }); + // defer result2.deinit(); + // var result3: zml.Buffer = executable.call(.{ buffer_a, buffer_b }); + // defer result3.deinit(); + // var result4: zml.Buffer = executable.call(.{ result1, result2 }); + // defer result4.deinit(); + + // var hb1 = try result1.copyToMemory(platform, zml.Buffer.MemoryKind.pinned_host); + // var hb2 = try result2.copyToMemory(platform, zml.Buffer.MemoryKind.pinned_host); + // var hb3 = try result3.copyToMemory(platform, zml.Buffer.MemoryKind.pinned_host); + // var hb4 = try result4.copyToMemory(platform, zml.Buffer.MemoryKind.pinned_host); + + // _ = try hb1.awaitt(); + // _ = try hb2.awaitt(); + // _ = try hb3.awaitt(); + // _ = try hb4.awaitt(); + + // _ = runtime.cudaProfilerStop(); + + // // if (profile) |profile_file| { + // // profiler.stop(); + // // try profiler.dumpAsJsonTo(allocator, std.fs.cwd(), profile_file); + // // } + // } + + // { + // const result_host_buffer1_data = try allocator.alloc(u8, a_shape.byteSize()); + // defer allocator.free(result_host_buffer1_data); + // const result_host_buffer2_data = try allocator.alloc(u8, a_shape.byteSize()); + // defer allocator.free(result_host_buffer2_data); + // const result_host_buffer3_data = try allocator.alloc(u8, a_shape.byteSize()); + // defer allocator.free(result_host_buffer3_data); + + // // if (profile != null) profiler.start(); + // _ = runtime.cudaProfilerStart(); + + // var result1: zml.Buffer = executable.call(.{ buffer_a, buffer_b }); + // defer result1.deinit(); + + // var hb1 = try result1.copyToMemory(platform, zml.Buffer.MemoryKind.pinned_host); + + // var result2: zml.Buffer = executable.call(.{ result1, buffer_b }); + // defer result2.deinit(); + + // var hb2 = try result2.copyToMemory(platform, zml.Buffer.MemoryKind.pinned_host); + + // var result3: zml.Buffer = executable.call(.{ buffer_a, buffer_b }); + // defer result3.deinit(); + + // _ = try hb1.awaitt(); + // _ = try hb2.awaitt(); + + // var result4: zml.Buffer = executable.call(.{ result1, result2 }); + // defer result4.deinit(); + + // var hb3 = try result3.copyToMemory(platform, zml.Buffer.MemoryKind.pinned_host); + // var hb4 = try result4.copyToMemory(platform, zml.Buffer.MemoryKind.pinned_host); + + // _ = try hb3.awaitt(); + // _ = try hb4.awaitt(); + + // _ = runtime.cudaProfilerStop(); + + // // if (profile) |profile_file| { + // // profiler.stop(); + // // try profiler.dumpAsJsonTo(allocator, std.fs.cwd(), profile_file); + // // } + // } + + // { + // const result_host_buffer1_data = try allocator.alloc(u8, a_shape.byteSize()); + // defer allocator.free(result_host_buffer1_data); + // const result_host_buffer2_data = try allocator.alloc(u8, a_shape.byteSize()); + // defer allocator.free(result_host_buffer2_data); + // const result_host_buffer3_data = try allocator.alloc(u8, a_shape.byteSize()); + // defer allocator.free(result_host_buffer3_data); + + // if (profile != null) profiler.start(); + + // var result1: zml.Buffer = executable.call(.{ buffer_a, buffer_b }); + // defer result1.deinit(); + + // var hb1 = try result1.toHostEx(result_host_buffer1_data, .{}); + + // var result2: zml.Buffer = executable.call(.{ result1, buffer_b }); + // defer result2.deinit(); + + // var result3: zml.Buffer = executable.call(.{ buffer_a, buffer_b }); + // defer result3.deinit(); + + // var hb2 = try result2.toHostEx(result_host_buffer2_data, .{}); + + // _ = try hb1.awaitt(); + // _ = try hb2.awaitt(); + + // var result4: zml.Buffer = executable.call(.{ result1, result2 }); + // defer result4.deinit(); + + // var hb3 = try result3.toHostEx(result_host_buffer3_data, .{}); + // var hb4 = try result4.toHostEx(result_host_buffer3_data, .{}); + + // _ = try hb3.awaitt(); + // _ = try hb4.awaitt(); + + // if (profile) |profile_file| { + // profiler.stop(); + // try profiler.dumpAsJsonTo(allocator, std.fs.cwd(), profile_file); + // } + // } +} + +fn createRandomBuffer(allocator: std.mem.Allocator, shape: zml.Shape, random: std.Random) ![]u8 { + const data = try allocator.alloc(u8, shape.byteSize()); + + switch (shape.dtype()) { + inline else => |v| { + const ZigType = v.toZigType(); + switch (comptime v.class()) { + .bool => unreachable, + .integer => { + for (std.mem.bytesAsSlice(ZigType, data)) |*e| e.* = random.int(ZigType); + }, + .float => { + const value = 1; + for (std.mem.bytesAsSlice(ZigType, data)) |*e| e.* = if (ZigType == f64) + value + else if (ZigType == f32) + @floatCast(value) + else if (ZigType == f16) + @floatCast(value) + else + @bitCast(random.int(std.meta.Int(.unsigned, @bitSizeOf(ZigType)))); + }, + .complex => unreachable, + } + }, + } + + return data; +} diff --git a/examples/dma/main7.zig b/examples/dma/main7.zig new file mode 100644 index 00000000..b9a75f30 --- /dev/null +++ b/examples/dma/main7.zig @@ -0,0 +1,201 @@ +const std = @import("std"); +const os = std.os; +const zml = @import("zml"); +const asynk = @import("async"); +const stdx = @import("stdx"); + +// set log level to debug to print the generated IR +pub const std_options = .{ + .log_level = .debug, +}; + +pub const Runtime = struct { + cudaProfilerStart: CudaProfilerStart, + cudaProfilerStop: CudaProfilerStop, + + const CudaProfilerStart = *const fn () callconv(.C) c_int; + const CudaProfilerStop = *const fn () callconv(.C) c_int; + + pub fn init() !Runtime { + var cudart = try std.DynLib.open("libcudart.so.12"); + defer cudart.close(); + + return .{ + .cudaProfilerStart = cudart.lookup(Runtime.CudaProfilerStart, "cudaProfilerStart") orelse return error.NotFound, + .cudaProfilerStop = cudart.lookup(Runtime.CudaProfilerStop, "cudaProfilerStop") orelse return error.NotFound, + }; + } +}; + +pub fn main() !void { + try asynk.AsyncThread.main(std.heap.c_allocator, asyncMain); +} + +const Input = struct { + a: zml.Buffer, + b: zml.Buffer, +}; + +const Execute = asynk.Channel(Input, 16); +const PopResult = asynk.Channel(zml.Buffer, 1); + +pub fn benchmark(a: zml.Tensor, b: zml.Tensor) zml.Tensor { + var output = a; + for (0..10) |_| { + output = output.add(a.dot(b, .{.k})); + } + return output; // .toMemory(.pinned_host) + // var output = a; + // for (0..10) |_| { + // output = a.mul(b); + // } + // return output.toMemory(.pinned_host); + // return a.mul(b); +} + +fn inference_loop(executable: anytype, platform: zml.Platform, popResultChannel: *PopResult, input: Input) !void { + var current = input.a; + while (true) { + const result = executable.call(.{ current, input.b }); + const buffer_pinned = try result.copyToMemory(platform, zml.Buffer.MemoryKind.pinned_host); + popResultChannel.send(buffer_pinned); + current.deinit(); + current = result; + } +} + +fn results_loop(platform: zml.Platform, runtime_: Runtime, popResultChannel: *PopResult) !void { + var i: usize = 0; + while (popResultChannel.recv()) |buffer| { + var buffer_pinned = buffer; + defer buffer_pinned.deinit(); + _ = try buffer_pinned.awaitt(); + std.debug.print("--------------------------------\n", .{}); + std.debug.print("result _memory: {s}\n", .{buffer_pinned._memory.debugString(platform.pjrt_api)}); + // const data = try buffer_pinned.dataInMemory(); + // std.debug.print("result: {any}\n", .{data[0..100]}); + // std.debug.print("i: {d}\n", .{i}); + i += 1; + + if (i == 100) { + _ = runtime_.cudaProfilerStop(); + } + } +} + +pub fn asyncMain() !void { + // Short lived allocations + var gpa = std.heap.GeneralPurposeAllocator(.{}){}; + defer _ = gpa.deinit(); + const allocator = gpa.allocator(); + + var context = try zml.Context.init(); + defer context.deinit(); + + // Auto-select platform + const platform = context.autoPlatform(.{}).withCompilationOptions(.{ + .sharding_enabled = false, + }); + context.printAvailablePlatforms(platform); + + var runtime = Runtime.init() catch unreachable; + + const size = 2048; + const dtype: zml.DataType = .bf16; + + const a_shape = zml.Shape.init(.{ size, size }, dtype).withTags(.{ .m, .k }).withSharding(.{.k}); + const b_shape = a_shape.withTags(.{ .k, .n }).withSharding(.{.k}); + + // Wait for compilation to finish + const executable = try zml.compileFn(allocator, benchmark, .{ a_shape, b_shape }, platform); + defer executable.deinit(); + + var rng = std.Random.DefaultPrng.init(0); + const random = rng.random(); + + const random_buffer_a = try createRandomBuffer(allocator, a_shape, random); + defer allocator.free(random_buffer_a); + var buffer_a = try zml.Buffer.fromToMemory(platform, random_buffer_a, a_shape, .device); + defer buffer_a.deinit(); + _ = try buffer_a.awaitt(); + + const random_buffer_b = try createRandomBuffer(allocator, a_shape, random); + defer allocator.free(random_buffer_b); + var buffer_b = try zml.Buffer.fromToMemory(platform, random_buffer_b, a_shape, .device); + defer buffer_b.deinit(); + _ = try buffer_b.awaitt(); + + std.debug.print("buffer_a _memory: {s}\n", .{buffer_a.getMemory().debugString(platform.pjrt_api)}); + std.debug.print("buffer_b _memory: {s}\n", .{buffer_b.getMemory().debugString(platform.pjrt_api)}); + + var result: zml.Buffer = executable.call(.{ buffer_a, buffer_b }); + std.debug.print("result _memory: {s}\n", .{buffer_a.getMemory().debugString(platform.pjrt_api)}); + var hb = try result.copyToMemory(platform, zml.Buffer.MemoryKind.pinned_host); + _ = try hb.awaitt(); + hb.deinit(); + result.deinit(); + + std.debug.print("Warmup finished\n", .{}); + + // var popResultChannel = PopResult.init(); + // _ = popResultChannel; // autofix + + std.debug.print("buffer_a _memory: {s}\n", .{buffer_a.getMemory().debugString(platform.pjrt_api)}); + std.debug.print("buffer_b _memory: {s}\n", .{buffer_b.getMemory().debugString(platform.pjrt_api)}); + + _ = runtime.cudaProfilerStart(); + + var result2: zml.Buffer = executable.call(.{ buffer_a, buffer_b }); + var result_2_to_device = try result2.copyToMemory(platform, .pinned_host); + var result3: zml.Buffer = executable.call(.{ result2, buffer_b }); + _ = try result3.awaitt(); + _ = try result_2_to_device.awaitt(); + // var result2_pinned = try result2.copyToMemory(platform, zml.Buffer.MemoryKind.pinned_host); + // _ = try result2.awaitt(); + // std.debug.print("result _memory: {s}\n", .{result2.getMemory().debugString(platform.pjrt_api)}); + // std.debug.print("result _memory kindId: {any}\n", .{result2.getMemory().id(platform.pjrt_api)}); + // const data = try result2.dataInMemory(); + + // std.debug.print("result _memory value: {any}\n", .{std.mem.bytesAsSlice(f16, data)[0..100]}); + defer result2.deinit(); + defer result3.deinit(); + + _ = runtime.cudaProfilerStop(); + + try asynk.sleep(100); + + // var frame1 = try asynk.asyncc(inference_loop, .{ &executable, platform, &popResultChannel, .{ .a = buffer_a, .b = buffer_b } }); + // var frame2 = try asynk.asyncc(results_loop, .{ platform, runtime, &popResultChannel }); + // _ = frame2.awaitt() catch unreachable; + // _ = frame1.awaitt() catch unreachable; +} + +fn createRandomBuffer(allocator: std.mem.Allocator, shape: zml.Shape, random: std.Random) ![]u8 { + const data = try allocator.alloc(u8, shape.byteSize()); + + switch (shape.dtype()) { + inline else => |v| { + const ZigType = v.toZigType(); + switch (comptime v.class()) { + .bool => unreachable, + .integer => { + for (std.mem.bytesAsSlice(ZigType, data)) |*e| e.* = random.int(ZigType); + }, + .float => { + const value = 1; + for (std.mem.bytesAsSlice(ZigType, data)) |*e| e.* = if (ZigType == f64) + value + else if (ZigType == f32) + @floatCast(value) + else if (ZigType == f16) + @floatCast(value) + else + @bitCast(random.int(std.meta.Int(.unsigned, @bitSizeOf(ZigType)))); + }, + .complex => unreachable, + } + }, + } + + return data; +} diff --git a/examples/dma/main8.zig b/examples/dma/main8.zig new file mode 100644 index 00000000..a42bc5f8 --- /dev/null +++ b/examples/dma/main8.zig @@ -0,0 +1,225 @@ +const std = @import("std"); +const os = std.os; +const zml = @import("zml"); +const asynk = @import("async"); +const stdx = @import("stdx"); + +// set log level to debug to print the generated IR +pub const std_options = .{ + .log_level = .info, +}; + +pub const Runtime = struct { + cudaProfilerStart: CudaProfilerStart, + cudaProfilerStop: CudaProfilerStop, + + const CudaProfilerStart = *const fn () callconv(.C) c_int; + const CudaProfilerStop = *const fn () callconv(.C) c_int; + + pub fn init() !Runtime { + var cudart = try std.DynLib.open("libcudart.so.12"); + defer cudart.close(); + + return .{ + .cudaProfilerStart = cudart.lookup(Runtime.CudaProfilerStart, "cudaProfilerStart") orelse return error.NotFound, + .cudaProfilerStop = cudart.lookup(Runtime.CudaProfilerStop, "cudaProfilerStop") orelse return error.NotFound, + }; + } +}; + +pub fn main() !void { + try asynk.AsyncThread.main(std.heap.c_allocator, asyncMain); +} + +const Input = struct { + slice: zml.Buffer, + index: ?zml.Buffer, + token: ?zml.Buffer, +}; + +const OutputTensors = struct { + slice: zml.Tensor, + index: zml.Tensor, + token: zml.Tensor, +}; + +const Buffers = struct { + slice: zml.Buffer, + index: zml.Buffer, + token: zml.Buffer, +}; + +const Execute = asynk.Channel(Input, 16); +const PopResult = asynk.Channel(zml.Buffer, 1); + +pub fn program(input: zml.Tensor, index: ?zml.Tensor, token: ?zml.Tensor) OutputTensors { + const idx: zml.Tensor = index orelse zml.Tensor.scalar(0, .i32); + const tok: zml.Tensor = token orelse zml.Tensor.scalar(0, .i32); + + var updated_slice = input.dynamicUpdateSlice(.{ .s = idx, .m = idx }, tok.convert(.bf16)); + + const new_tok = tok.convert(.bf16).broadcastLeft(updated_slice.shape()); + for (0..10) |_| { + updated_slice = updated_slice.matmul(new_tok); + } + + return .{ + .slice = updated_slice.reuseBuffer(input), + .index = idx.addConstant(1), + .token = tok.addConstant(2), + }; +} + +fn inference_loop(executable: anytype, platform: zml.Platform, popResultChannel: *PopResult, input: Buffers) !void { + const slice = input.slice; + var index = input.index; + var token = input.token; + + while (true) { + const result = executable.call(.{ slice, index, token }); + const buffer_pinned = try result.token.copyToMemory(platform, zml.Buffer.MemoryKind.pinned_host); + try asynk.sleep(0); + index = result.index; + token = result.token; + popResultChannel.send(buffer_pinned); + } +} + +fn results_loop(platform: zml.Platform, runtime_: Runtime, popResultChannel: *PopResult) !void { + var i: usize = 0; + while (popResultChannel.recv()) |buffer| { + var buffer_pinned = buffer; + defer buffer_pinned.deinit(); + _ = try buffer_pinned.awaitt(); + std.debug.print("result _memory: {s}\n", .{buffer_pinned.getMemory().debugString(platform.pjrt_api)}); + // const inMem = try buffer_pinned.getValueFromDataInMemory(i32); + // std.debug.print("result: {any}\n", .{inMem}); + i += 1; + + if (i == 128) { + _ = runtime_.cudaProfilerStop(); + } + } +} + +pub fn asyncMain() !void { + // Short lived allocations + var gpa = std.heap.GeneralPurposeAllocator(.{}){}; + defer _ = gpa.deinit(); + const allocator = gpa.allocator(); + + var context = try zml.Context.init(); + defer context.deinit(); + + // Auto-select platform + const platform = context.autoPlatform(.{}).withCompilationOptions(.{ + .sharding_enabled = false, + }); + context.printAvailablePlatforms(platform); + + const runtime = Runtime.init() catch unreachable; + + const size = 2048; + const dtype: zml.DataType = .bf16; + const i_dtype: zml.DataType = .i32; + + const slice_shape = zml.Shape.init(.{ .s = size, .m = size }, dtype); + const index_shape = zml.Shape.init(.{}, i_dtype); + const token_shape = zml.Shape.init(.{ .s = size, .m = 1 }, i_dtype); + + // Wait for compilation to finish + const executable = try zml.compileFn(allocator, program, .{ slice_shape, index_shape, token_shape }, platform); + defer executable.deinit(); + + var rng = std.Random.DefaultPrng.init(0); + const random = rng.random(); + + const slice_random = try createRandomBuffer(allocator, slice_shape, random, 1); + defer allocator.free(slice_random); + var slice_buffer = try zml.Buffer.fromToMemory(platform, slice_random, slice_shape, .device); + defer slice_buffer.deinit(); + _ = try slice_buffer.awaitt(); + + const index_random = try createRandomBuffer(allocator, index_shape, random, 0); + defer allocator.free(index_random); + var index_buffer = try zml.Buffer.fromToMemory(platform, index_random, index_shape, .device); + defer index_buffer.deinit(); + _ = try index_buffer.awaitt(); + + const token_random = try createRandomBuffer(allocator, token_shape, random, 0); + defer allocator.free(token_random); + var token_buffer = try zml.Buffer.fromToMemory(platform, token_random, token_shape, .device); + defer token_buffer.deinit(); + _ = try token_buffer.awaitt(); + + const input: Buffers = .{ .slice = slice_buffer, .index = index_buffer, .token = token_buffer }; + + var result = executable.call(.{ input.slice, input.index, input.token }); + var result_slice_in_pinned_memory = try result.slice.copyToMemory(platform, zml.Buffer.MemoryKind.pinned_host); + _ = try result_slice_in_pinned_memory.awaitt(); + defer { + result.slice.deinit(); + result.index.deinit(); + result.token.deinit(); + } + + std.debug.print("Warmup finished\n", .{}); + + // var slice_buffer_2 = try slice_buffer.copyToMemory(platform, zml.Buffer.MemoryKind.pinned_host); + // _ = try slice_buffer_2.awaitt(); + + // var index_buffer2 = try index_buffer.copyToMemory(platform, zml.Buffer.MemoryKind.pinned_host); + // _ = try index_buffer2.awaitt(); + + // var token_buffer2 = try index_buffer.copyToMemory(platform, zml.Buffer.MemoryKind.pinned_host); + // _ = try token_buffer2.awaitt(); + + // _ = runtime.cudaProfilerStart(); + + // const input2: Buffers = .{ .slice = slice_buffer_2, .index = index_buffer2, .token = token_buffer2 }; + // _ = input2; // autofix + + // std.debug.print("slice: {s}\n", .{slice_buffer_2._memory.debugString(platform.pjrt_api)}); + // std.debug.print("index: {s}\n", .{index_buffer2._memory.debugString(platform.pjrt_api)}); + // std.debug.print("token: {s}\n", .{token_buffer2._memory.debugString(platform.pjrt_api)}); + + var popResultChannel = PopResult.init(); + + _ = runtime.cudaProfilerStart(); + + var frame1 = try asynk.asyncc(inference_loop, .{ &executable, platform, &popResultChannel, input }); + var frame2 = try asynk.asyncc(results_loop, .{ platform, runtime, &popResultChannel }); + _ = frame2.awaitt() catch unreachable; + _ = frame1.awaitt() catch unreachable; +} + +fn createRandomBuffer(allocator: std.mem.Allocator, shape: zml.Shape, random: std.Random, given_value: f32) ![]u8 { + const data = try allocator.alloc(u8, shape.byteSize()); + + switch (shape.dtype()) { + inline else => |v| { + const ZigType = v.toZigType(); + switch (comptime v.class()) { + .bool => unreachable, + .integer => { + const value = given_value; + for (std.mem.bytesAsSlice(ZigType, data)) |*e| e.* = @intFromFloat(value); + }, + .float => { + const value = given_value; + for (std.mem.bytesAsSlice(ZigType, data)) |*e| e.* = if (ZigType == f64) + value + else if (ZigType == f32) + @floatCast(value) + else if (ZigType == f16) + @floatCast(value) + else + @bitCast(random.int(std.meta.Int(.unsigned, @bitSizeOf(ZigType)))); + }, + .complex => unreachable, + } + }, + } + + return data; +} diff --git a/examples/dma/main9.zig b/examples/dma/main9.zig new file mode 100644 index 00000000..ee14fae8 --- /dev/null +++ b/examples/dma/main9.zig @@ -0,0 +1,192 @@ +const std = @import("std"); +const os = std.os; +const zml = @import("zml"); +const asynk = @import("async"); +const stdx = @import("stdx"); + +// set log level to debug to print the generated IR +pub const std_options = .{ + .log_level = .debug, +}; + +pub const Runtime = struct { + cudaProfilerStart: CudaProfilerStart, + cudaProfilerStop: CudaProfilerStop, + + const CudaProfilerStart = *const fn () callconv(.C) c_int; + const CudaProfilerStop = *const fn () callconv(.C) c_int; + + pub fn init() !Runtime { + var cudart = try std.DynLib.open("libcudart.so.12"); + defer cudart.close(); + + return .{ + .cudaProfilerStart = cudart.lookup(Runtime.CudaProfilerStart, "cudaProfilerStart") orelse return error.NotFound, + .cudaProfilerStop = cudart.lookup(Runtime.CudaProfilerStop, "cudaProfilerStop") orelse return error.NotFound, + }; + } +}; + +pub fn main() !void { + try asynk.AsyncThread.main(std.heap.c_allocator, asyncMain); +} + +const Input = struct { + a: zml.Buffer, + b: zml.Buffer, +}; + +const Execute = asynk.Channel(Input, 16); +const PopResult = asynk.Channel(zml.Buffer, 1); + +pub fn benchmark(a: zml.Tensor, b: zml.Tensor) zml.Tensor { + var output = a; + for (0..10) |_| { + output = a.mul(b); + } + return output.toMemory(.pinned_host); + // return a.mul(b); +} + +fn inference_loop(executable: anytype, platform: zml.Platform, popResultChannel: *PopResult, input: Input) !void { + var current = input.a; + while (true) { + const result = executable.call(.{ current, input.b }); + const buffer_pinned = try result.copyToMemory(platform, zml.Buffer.MemoryKind.pinned_host); + popResultChannel.send(buffer_pinned); + current.deinit(); + current = result; + } +} + +fn results_loop(platform: zml.Platform, runtime_: Runtime, popResultChannel: *PopResult) !void { + var i: usize = 0; + while (popResultChannel.recv()) |buffer| { + var buffer_pinned = buffer; + defer buffer_pinned.deinit(); + _ = try buffer_pinned.awaitt(); + std.debug.print("--------------------------------\n", .{}); + std.debug.print("result _memory: {s}\n", .{buffer_pinned._memory.debugString(platform.pjrt_api)}); + // const data = try buffer_pinned.dataInMemory(); + // std.debug.print("result: {any}\n", .{data[0..100]}); + // std.debug.print("i: {d}\n", .{i}); + i += 1; + + if (i == 100) { + _ = runtime_.cudaProfilerStop(); + } + } +} + +pub fn asyncMain() !void { + // Short lived allocations + var gpa = std.heap.GeneralPurposeAllocator(.{}){}; + defer _ = gpa.deinit(); + const allocator = gpa.allocator(); + + var context = try zml.Context.init(); + defer context.deinit(); + + // Auto-select platform + const platform = context.autoPlatform(.{}).withCompilationOptions(.{ + .sharding_enabled = false, + }); + context.printAvailablePlatforms(platform); + + var runtime = Runtime.init() catch unreachable; + + const size = 2048; + const dtype: zml.DataType = .f16; + + const a_shape = zml.Shape.init(.{ 1, size }, dtype).withTags(.{ .m, .k }).withSharding(.{.k}); + const b_shape = a_shape.withTags(.{ .k, .n }).withSharding(.{.k}); + + // Wait for compilation to finish + const executable = try zml.compileFn(allocator, benchmark, .{ a_shape, b_shape }, platform); + defer executable.deinit(); + + var rng = std.Random.DefaultPrng.init(0); + const random = rng.random(); + + const random_buffer_a = try createRandomBuffer(allocator, a_shape, random); + defer allocator.free(random_buffer_a); + var buffer_a = try zml.Buffer.fromToMemory(platform, random_buffer_a, a_shape, .device); + defer buffer_a.deinit(); + _ = try buffer_a.awaitt(); + + const random_buffer_b = try createRandomBuffer(allocator, a_shape, random); + defer allocator.free(random_buffer_b); + var buffer_b = try zml.Buffer.fromToMemory(platform, random_buffer_b, a_shape, .device); + defer buffer_b.deinit(); + _ = try buffer_b.awaitt(); + + std.debug.print("buffer_a _memory: {s}\n", .{buffer_a.getMemory().debugString(platform.pjrt_api)}); + std.debug.print("buffer_b _memory: {s}\n", .{buffer_b.getMemory().debugString(platform.pjrt_api)}); + + var result: zml.Buffer = executable.call(.{ buffer_a, buffer_b }); + std.debug.print("result _memory: {s}\n", .{buffer_a.getMemory().debugString(platform.pjrt_api)}); + // var hb = try result.copyToMemory(platform, zml.Buffer.MemoryKind.pinned_host); + // _ = try hb.awaitt(); + defer result.deinit(); + + std.debug.print("Warmup finished\n", .{}); + + // var popResultChannel = PopResult.init(); + // _ = popResultChannel; // autofix + + std.debug.print("buffer_a _memory: {s}\n", .{buffer_a.getMemory().debugString(platform.pjrt_api)}); + std.debug.print("buffer_b _memory: {s}\n", .{buffer_b.getMemory().debugString(platform.pjrt_api)}); + _ = runtime.cudaProfilerStart(); + + var result2: zml.Buffer = executable.call(.{ buffer_a, buffer_b }); + const result_2_to_device = try result2.copyToMemory(platform, .device); + var result3: zml.Buffer = executable.call(.{ result_2_to_device, buffer_b }); + // var result2_pinned = try result2.copyToMemory(platform, zml.Buffer.MemoryKind.pinned_host); + // _ = try result2.awaitt(); + // std.debug.print("result _memory: {s}\n", .{result2.getMemory().debugString(platform.pjrt_api)}); + // std.debug.print("result _memory kindId: {any}\n", .{result2.getMemory().id(platform.pjrt_api)}); + // const data = try result2.dataInMemory(); + + // std.debug.print("result _memory value: {any}\n", .{std.mem.bytesAsSlice(f16, data)[0..100]}); + defer result2.deinit(); + defer result3.deinit(); + + _ = runtime.cudaProfilerStop(); + + try asynk.sleep(100); + + // var frame1 = try asynk.asyncc(inference_loop, .{ &executable, platform, &popResultChannel, .{ .a = buffer_a, .b = buffer_b } }); + // var frame2 = try asynk.asyncc(results_loop, .{ platform, runtime, &popResultChannel }); + // _ = frame2.awaitt() catch unreachable; + // _ = frame1.awaitt() catch unreachable; +} + +fn createRandomBuffer(allocator: std.mem.Allocator, shape: zml.Shape, random: std.Random) ![]u8 { + const data = try allocator.alloc(u8, shape.byteSize()); + + switch (shape.dtype()) { + inline else => |v| { + const ZigType = v.toZigType(); + switch (comptime v.class()) { + .bool => unreachable, + .integer => { + for (std.mem.bytesAsSlice(ZigType, data)) |*e| e.* = random.int(ZigType); + }, + .float => { + const value = 1; + for (std.mem.bytesAsSlice(ZigType, data)) |*e| e.* = if (ZigType == f64) + value + else if (ZigType == f32) + @floatCast(value) + else if (ZigType == f16) + @floatCast(value) + else + @bitCast(random.int(std.meta.Int(.unsigned, @bitSizeOf(ZigType)))); + }, + .complex => unreachable, + } + }, + } + + return data; +} diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 9a17c404..3874cb18 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -282,7 +282,10 @@ pub fn asyncMain() !void { }); log.info("\tLoading Llama weights from {?s}...", .{res.args.weights}); + var runtime = Runtime.init() catch unreachable; + _ = runtime.cudaProfilerStart(); var llama_weights = try zml.aio.loadBuffers(llama.LlamaLM, .{ config, llama_options }, ts, model_arena.allocator(), platform); + _ = runtime.cudaProfilerStop(); defer zml.aio.unloadBuffers(&llama_weights); log.info("✅\tLoaded weights in {}", .{std.fmt.fmtDuration(start.read())}); @@ -314,7 +317,10 @@ pub fn asyncMain() !void { const seed = res.args.seed orelse @as(u128, @bitCast(std.time.nanoTimestamp())); const skip_llama3_encoding = res.args.@"no-llama3" orelse false; - const generated_text = try generateText(config, model_instance, llama_module_prefill, llama_module, kv_cache, tokenizer, allocator, seed, prompt[0..], skip_llama3_encoding); - // generated text will be printed token by token. - defer allocator.free(generated_text); + for (0..2) |_| { + log.warn("----------------------------------------------", .{}); + const generated_text = try generateText(config, model_instance, llama_module_prefill, llama_module, kv_cache, tokenizer, allocator, seed, prompt[0..], skip_llama3_encoding); + // generated text will be printed token by token. + defer allocator.free(generated_text); + } } diff --git a/main.py b/main.py new file mode 100644 index 00000000..f3bf55bd --- /dev/null +++ b/main.py @@ -0,0 +1,39 @@ +import jax +import jax.numpy as jnp +from ctypes import cdll +libcudart = cdll.LoadLibrary('libcudart.so') + +jax.config.update("jax_traceback_in_locations_limit", -1) + +@jax.jit +def benchmark(a, b): + return jnp.dot(a, b) + +# Main function +def main(): + size = 8192 + dtype = jnp.bfloat16 + + rng_key = jax.random.PRNGKey(0) + a = jax.random.uniform(rng_key, shape=(size, size), dtype=dtype) + b = jax.random.uniform(rng_key, shape=(size, size), dtype=dtype) + + print("Compiling the benchmark function...") + compiled_benchmark = jax.jit(benchmark) + + print("Running warm-up execution...") + compiled_benchmark(a, b) + + print("Running benchmark...") + + # with jax.profiler.trace("/home/zml/zml-hugo/jax-trace", create_perfetto_link=True): + libcudart.cudaProfilerStart() + result = compiled_benchmark(a, b) + result2 = compiled_benchmark(result, b) + jax.device_get(result) + jax.device_get(result2) + libcudart.cudaProfilerStop() + +if __name__ == "__main__": + main() + diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index f48b6361..f4cc0948 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -502,6 +502,26 @@ pub const Device = opaque { }) catch unreachable; return @intCast(ret.local_hardware_id); } + + pub fn addressableMemories(self: *const Device, api: *const Api) []*const Memory { + const ret = api.call(.PJRT_Device_AddressableMemories, .{ + .device = self.inner(), + }) catch unreachable; + if (ret.memories) |memories| { + return @ptrCast(@constCast(memories[0..ret.num_memories])); + } + return &.{}; + } + + pub fn getMemoryByKind(self: *const Device, api: *const Api, kind: Memory.Kind) ?*const Memory { + const memories = self.addressableMemories(api); + for (memories) |m| { + if (m.kind(api) == kind) { + return m; + } + } + return null; + } }; pub const DeviceDescription = opaque { @@ -780,6 +800,31 @@ pub const Buffer = opaque { return @ptrCast(ret.event); } + pub fn copyRawToHost(self: *const Buffer, api: *const Api, dst: []u8, offset: i64, size: i64) ApiError!?*Event { + const ret = try api.call(.PJRT_Buffer_CopyRawToHost, .{ + .buffer = self.inner(), + .dst = @ptrCast(dst.ptr), + .offset = offset, + .transfer_size = size, + }); + return @ptrCast(ret.event); + } + + pub fn copyToMemory(self: *const Buffer, api: *const Api, dst_memory: *const Memory) ApiError!?*Buffer { + const ret = try api.call(.PJRT_Buffer_CopyToMemory, .{ + .buffer = self.inner(), + .dst_memory = @ptrCast(@constCast(dst_memory)), + }); + return @ptrCast(ret.dst_buffer); + } + + pub fn getMemory(self: *const Buffer, api: *const Api) *const Memory { + const ret = api.call(.PJRT_Buffer_Memory, .{ + .buffer = self.inner(), + }) catch unreachable; + return @ptrCast(ret.memory); + } + pub fn getElementType(self: *const Buffer, api: *const Api) BufferType { const ret = api.call(.PJRT_Buffer_ElementType, .{ .buffer = self.inner(), diff --git a/runtimes/cuda/BUILD.bazel b/runtimes/cuda/BUILD.bazel index a7f8bf94..6b50f575 100644 --- a/runtimes/cuda/BUILD.bazel +++ b/runtimes/cuda/BUILD.bazel @@ -1,5 +1,10 @@ load("@rules_zig//zig:defs.bzl", "zig_library") +exports_files( + ["libpjrt_cuda.so"], + visibility = ["//visibility:public"], +) + cc_library( name = "zmlxcuda_lib", srcs = ["zmlxcuda.c"], diff --git a/runtimes/cuda/cuda.zig b/runtimes/cuda/cuda.zig index bc749693..79aa477e 100644 --- a/runtimes/cuda/cuda.zig +++ b/runtimes/cuda/cuda.zig @@ -62,6 +62,10 @@ pub fn load() !*const pjrt.Api { log.warn("Detected {s} in LD_LIBRARY_PATH. This can lead to undefined behaviors and crashes", .{nvidiaLibsPath}); } + _ = c.setenv("XLA_PJRT_GPU_HOST_MEMORY_LIMIT_GB", "2", 1); + _ = c.setenv("XLA_PJRT_GPU_HOST_MEMORY_PREALLOCATE", "true", 1); + _ = c.setenv("CUDA_VISIBLE_DEVICES", "0", 1); + // CUDA path has to be set _before_ loading the PJRT plugin. // See https://github.com/openxla/xla/issues/21428 try setupXlaGpuCudaDirFlag(); diff --git a/zml/aio.zig b/zml/aio.zig index 27abbd8a..dd556165 100644 --- a/zml/aio.zig +++ b/zml/aio.zig @@ -550,6 +550,7 @@ pub fn loadModelBuffersWithPrefix( // We copy the shape, and let visitStructAndLoadBuffer write the other fields. // to write them just afterward. var res: zml.Bufferized(Model) = undefined; + try zml.meta.mapAlloc(struct { pub fn initBuffer(_: void, tensor: zml.Tensor) zml.Buffer { return .{ ._shape = tensor.shape(), ._api = undefined, ._shards = undefined }; diff --git a/zml/buffer.zig b/zml/buffer.zig index 20a6fcb7..cda0a9aa 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -59,12 +59,6 @@ pub const Buffer = struct { /// Copies the content of the given buffer from host memory to the accelerator memory. pub fn from(platform: Platform, host_buffer: HostBuffer) !Buffer { - var res: Buffer = .{ - ._api = platform.pjrt_api, - ._shape = host_buffer.shape(), - ._shards = .{}, - }; - // We shard only on the first axis so that the chunks are still contiguous. // TODO: support more advanced sharding specs stdx.debug.assert(platform.sharding().num_replicas == 1, "ZML doesn't support num_replicas > 1 for now, got: {}", .{platform.sharding()}); @@ -81,6 +75,13 @@ pub const Buffer = struct { var frames: std.BoundedArray(asynk.Frame(pjrt.Client.bufferFromHostBuffer), MAX_NUM_SHARDS) = .{}; const devices = platform.getDevices(); + + var res: Buffer = .{ + ._api = platform.pjrt_api, + ._shape = host_buffer.shape(), + ._shards = .{}, + }; + for (0..n_partitions) |i| { // If no sharding if found, the given buffer is replicated on all devices. const buf = if (sharding_ax) |ax| buf: { @@ -124,6 +125,74 @@ pub const Buffer = struct { }; } + pub fn fromToMemory(platform: Platform, buffer_: []const u8, shape_: Shape, memory_kind: MemoryKind) !Buffer { + // We shard only on the first axis so that the chunks are still contiguous. + // TODO: support more advanced sharding specs + stdx.debug.assert(platform.sharding().num_replicas == 1, "ZML doesn't support num_replicas > 1 for now, got: {}", .{platform.sharding()}); + const sharding_ax: ?u3 = std.simd.firstTrue(shape_._sharding_info); + _ = sharding_ax; // autofix + const n_partitions = platform.sharding().num_partitions; + const chunk_size = 0; + _ = chunk_size; // autofix + + const buffer_type = bufferTypeFromDtype(shape_.dtype()); + const byte_strides = shape_.computeStrides().constSlice(); + + const devices = platform.getDevices(); + + var res: Buffer = .{ + ._api = platform.pjrt_api, + ._shape = shape_, + ._shards = .{}, + }; + + for (0..n_partitions) |i| { + const pjrt_buffer, const event = try platform.pjrt_client.bufferFromHostBuffer(platform.pjrt_api, .{ + .data = buffer_, + .buffer_type = buffer_type, + .dims = shape_.dims(), + .byte_strides = byte_strides, + .device = devices[i], + .host_buffer_semantics = .ImmutableUntilTransferCompletes, + .memory = devices[i].getMemoryByKind(platform.pjrt_api, memory_kind), + }); + + res._shards.appendAssumeCapacity(.{ + .buffer = pjrt_buffer, + .ready_event = event, + }); + } + + return res; + } + + pub fn getMemory(self: Buffer) *const Memory { + return self._shards.get(0).buffer.getMemory(self._api); + } + + pub fn dataInMemory(self: Buffer) ![]const u8 { + const shard_buffer = self._shards.get(0).buffer; + const opaqueDataPointer = try shard_buffer.getOpaqueDeviceMemoryDataPointer(self._api); + const sizeInbytes = try shard_buffer.getOnDeviceSizeInBytes(self._api); + const data = @as([*]const u8, @ptrFromInt(@intFromPtr(opaqueDataPointer))); + const end: usize = @intCast(sizeInbytes); + return data[0..end]; + } + + pub fn getValueFromDataInMemory(self: Buffer, T: type) !T { + stdx.debug.assert(self._shape.byteSize() == @sizeOf(T), "Buffer {} has {d} bytes of data, can't load it to a {s} with {d} bytes", .{ self, self._shape.byteSize(), @typeName(T), @sizeOf(T) }); + const data = try self.dataInMemory(); + const value = std.mem.bytesAsValue(T, @constCast(data)); + return value.*; + } + + pub fn awaitt(self: *Buffer) !*Buffer { + for (self._shards.slice()) |*shard| { + try shard.awaitt(self._api); + } + return self; + } + /// Copies the given Zig slice to the accelerator memory and /// return a Buffer with the given dimensions. pub fn fromSlice(platform: Platform, dimz: anytype, s: anytype) !Buffer { @@ -281,6 +350,49 @@ pub const Buffer = struct { return HostBuffer.fromBytes(self.shape(), output); } + const ToHostExOpts = struct { + offset: i64 = 0, + // If not set, the whole buffer starting from offset (default: 0) on device is copied. + size: ?i64 = null, + }; + pub fn toHostEx(self: Buffer, output: []u8, opts: ToHostExOpts) !HostBuffer { + var host_buffer = HostBuffer.fromBytes(self.shape(), output); + stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); + const size: i64 = opts.size orelse @as(i64, @intCast(output.len)) - opts.offset; + const maybe_event = try self._shards.get(0).buffer.copyRawToHost(self._api, @constCast(host_buffer.data), opts.offset, size); + host_buffer._event = maybe_event; + host_buffer._api = self._api; + host_buffer._ready = if (maybe_event) |ev| ev.isReady(self._api) else true; + return host_buffer; + } + + pub const MemoryKind = pjrt.Memory.Kind; + pub fn copyToMemory(self: Buffer, platform: Platform, kind: pjrt.Memory.Kind) !Buffer { + const memories = platform.pjrt_client.addressableMemories(platform.pjrt_api); // todo : adresse via device + var selected_mem: *const pjrt.Memory = undefined; + for (memories) |mem| { + if (mem.kind(platform.pjrt_api) == kind) { + selected_mem = mem; + } + } + + var shards: Shards = .{}; + for (self._shards.constSlice()) |shard| { + const pjrt_buffer = try shard.buffer.copyToMemory(self._api, selected_mem); + const event = if (pjrt_buffer) |buf| buf.getReadyEvent(self._api) else null; + shards.appendAssumeCapacity(.{ + .buffer = pjrt_buffer.?, + .ready_event = event, + .ready = if (event) |ev| ev.isReady(self._api) else true, + }); + } + return .{ + ._api = self._api, + ._shape = self._shape, + ._shards = shards, + }; + } + /// Copies the content of the Buffer to the host. /// The returned `HostBuffer` does own the memory. pub fn toHostAlloc(self: Buffer, allocator: std.mem.Allocator) !HostBuffer { diff --git a/zml/module.zig b/zml/module.zig index 453f0cb0..afbbd61f 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -404,6 +404,7 @@ pub const CompilationContext = struct { const fn_res_types = try res_allocator.alloc(mlir.Type, out_tensor_count); const fn_res_shapes = try res_allocator.alloc(Shape, out_tensor_count); const fn_res_donations = try res_allocator.alloc(Tensor._Donation, out_tensor_count); + const fn_res_output_memory_kind = try res_allocator.alloc(Tensor._MemoryKind, out_tensor_count); var fn_body = self.openBlock(.hermetic, input_types, locations) catch unreachable; { defer self.closeBlock(fn_body); @@ -419,7 +420,7 @@ pub const CompilationContext = struct { }; var fn_res_values: [out_tensor_count]mlir.Value = undefined; - self.extractValuesAndTypes(fn_res, &fn_res_values, fn_res_types, fn_res_shapes, fn_res_donations); + self.extractValuesAndTypes(fn_res, &fn_res_values, fn_res_types, fn_res_shapes, fn_res_donations, fn_res_output_memory_kind); const fn_ret = dialect.func.return_(mlir_ctx, &fn_res_values, loc); fn_body.appendOperationRecursive(fn_ret); @@ -433,11 +434,14 @@ pub const CompilationContext = struct { if (opts.kind == .main) { self.addDonationsAttributes(arg_attrs, fn_res_donations); + self.addOutputMemoryKindAttributes(res_attrs, fn_res_output_memory_kind); if (self._platform.sharding().num_partitions > 1) { self.addShardingAttributes(arg_attrs, res_attrs, input_shapes.items, fn_res_shapes); } } + std.debug.print("res_attrs : {any}", .{res_attrs}); + const mlir_fn = dialect.func.func(self.mlirCtx(), .{ .sym_name = opts.name, .args = input_types, @@ -470,6 +474,18 @@ pub const CompilationContext = struct { }; } + fn addOutputMemoryKindAttributes(self: CompilationContext, attributes: []AttributeList, output_memory_kind: []const Tensor._MemoryKind) void { + const mlir_ctx = self.mlirCtx(); + for (attributes, output_memory_kind) |*attr, memory_kind| { + if (memory_kind) |mk| { + attr.appendAssumeCapacity(mlir.NamedAttribute.init( + mlir.Identifier.get(mlir_ctx, "mhlo.memory_kind"), + mlir.StringAttribute.init(mlir_ctx, @tagName(mk)).as(mlir.Attribute).?, + )); + } + } + } + /// Given a list of donations mapping output buffers to input buffers, /// generate donation attribute for each `n_args` input argument. fn addDonationsAttributes(self: CompilationContext, attributes: []AttributeList, donations: []const Tensor._Donation) void { @@ -770,7 +786,7 @@ pub const CompilationContext = struct { } /// Visit the given struct and extract the mlir.Value and mlir.Type associated with each tensor found. - pub fn extractValuesAndTypes(self: *const CompilationContext, v: anytype, values: []mlir.Value, types: []mlir.Type, shapes: []Shape, donations: []Tensor._Donation) void { + pub fn extractValuesAndTypes(self: *const CompilationContext, v: anytype, values: []mlir.Value, types: []mlir.Type, shapes: []Shape, donations: []Tensor._Donation, output_memory_kind: []Tensor._MemoryKind) void { std.debug.assert(values.len == types.len); const LocalContext = struct { self: *const CompilationContext, @@ -779,8 +795,9 @@ pub const CompilationContext = struct { types: []mlir.Type, shapes: []Shape, donations: []Tensor._Donation, + output_memory_kind: []Tensor._MemoryKind, }; - var context = LocalContext{ .self = self, .values = values, .types = types, .shapes = shapes, .donations = donations }; + var context = LocalContext{ .self = self, .values = values, .types = types, .shapes = shapes, .donations = donations, .output_memory_kind = output_memory_kind }; meta.visit((struct { fn cb(ctx: *LocalContext, tensor: *const Tensor) void { const value, const donation = ctx.self.getValueAndDonation(tensor.*); @@ -788,6 +805,7 @@ pub const CompilationContext = struct { ctx.types[ctx.index] = value.getType(); ctx.shapes[ctx.index] = tensor._shape; ctx.donations[ctx.index] = donation; + ctx.output_memory_kind[ctx.index] = tensor._output_memory_kind; ctx.index += 1; } }).cb, &context, v); diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index a642979c..3a03f6bc 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -20,6 +20,7 @@ pub const ErrorCode = pjrt.ErrorCode; pub const BufferType = pjrt.BufferType; pub const Device = pjrt.Device; pub const DeviceDescription = pjrt.DeviceDescription; +pub const Memory = pjrt.Memory; pub const Api = pjrt.Api; pub const NamedValue = pjrt.NamedValue; pub const ClientInitError = pjrt.ClientInitError; @@ -163,6 +164,18 @@ pub const Buffer = opaque { return @ptrCast(try self.inner().toHostBuffer(api, dst)); } + pub fn copyRawToHost(self: *const Buffer, api: *const Api, dst: []u8, offset: i64, size: i64) ApiError!?*Event { + return @ptrCast(try self.inner().copyRawToHost(api, dst, offset, size)); + } + + pub fn copyToMemory(self: *const Buffer, api: *const Api, dst_memory: *const Memory) ApiError!?*Buffer { + return @ptrCast(try self.inner().copyToMemory(api, dst_memory)); + } + + pub fn getMemory(self: *const Buffer, api: *const Api) *const Memory { + return self.inner().getMemory(api); + } + pub fn getElementType(self: *const Buffer, api: *const Api) BufferType { return self.inner().getElementType(api); } @@ -194,6 +207,14 @@ pub const Buffer = opaque { pub fn getOpaqueDeviceMemoryDataPointer(self: *const Buffer, api: *const Api) ApiError!*anyopaque { return try self.inner().getOpaqueDeviceMemoryDataPointer(api); } + + pub fn increaseExternalReferenceCount(self: *Buffer, api: *const Api) ApiError!void { + try self.inner().increaseExternalReferenceCount(api); + } + + pub fn decreaseExternalReferenceCount(self: *Buffer, api: *const Api) ApiError!void { + try self.inner().decreaseExternalReferenceCount(api); + } }; pub const Event = opaque { @@ -233,6 +254,7 @@ pub const Event = opaque { ctx.event.wait(); if (ctx.err) |e| { + log.err("Error waiting for event: {}", .{e}); defer e.deinit(api); return e.getCode(api).toApiError(); } diff --git a/zml/platform.zig b/zml/platform.zig index ec4ae4c6..2cd6b0f5 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -7,6 +7,7 @@ const stdx = @import("stdx"); const meta = @import("meta.zig"); const module = @import("module.zig"); const pjrt = @import("pjrtx.zig"); +const Shape = @import("shape.zig").Shape; const log = std.log.scoped(.zml); @@ -87,6 +88,96 @@ pub const Platform = struct { } }; +// struct PJRT_AsyncHostToDeviceTransferManager_TransferData_Args { +// size_t struct_size; +// PJRT_Extension_Base* extension_start; +// PJRT_AsyncHostToDeviceTransferManager* transfer_manager; +// int buffer_index; +// const void* data; +// int64_t offset; +// int64_t transfer_size; +// bool is_last_transfer; +// PJRT_Event* done_with_h2d_transfer; // out +// }; + +// struct PJRT_AsyncHostToDeviceTransferManager_RetrieveBuffer_Args { +// size_t struct_size; +// PJRT_Extension_Base* extension_start; +// PJRT_AsyncHostToDeviceTransferManager* transfer_manager; +// int buffer_index; +// PJRT_Buffer* buffer_out; // out +// }; + +// struct PJRT_AsyncHostToDeviceTransferManager_Device_Args { +// size_t struct_size; +// PJRT_Extension_Base* extension_start; +// PJRT_AsyncHostToDeviceTransferManager* transfer_manager; +// PJRT_Device* device_out; // out +// }; + +// struct PJRT_AsyncHostToDeviceTransferManager_BufferCount_Args { +// size_t struct_size; +// PJRT_Extension_Base* extension_start; +// PJRT_AsyncHostToDeviceTransferManager* transfer_manager; +// size_t buffer_count; // out +// }; + +// struct PJRT_AsyncHostToDeviceTransferManager_BufferSize_Args { +// size_t struct_size; +// PJRT_Extension_Base* extension_start; +// PJRT_AsyncHostToDeviceTransferManager* transfer_manager; +// int buffer_index; +// size_t buffer_size; // out +// }; + +// struct PJRT_AsyncHostToDeviceTransferManager_AddMetadata_Args { +// size_t struct_size; +// PJRT_Extension_Base* extension_start; +// PJRT_AsyncHostToDeviceTransferManager* transfer_manager; +// const PJRT_NamedValue* transfer_metadata; +// size_t num_metadata; +// }; + +// struct PJRT_Client_CreateBuffersForAsyncHostToDevice_Args { +// size_t struct_size; +// PJRT_Extension_Base* extension_start; +// PJRT_Client* client; +// PJRT_ShapeSpec* shape_specs; +// size_t num_shape_specs; +// PJRT_Buffer_MemoryLayout** device_layouts; // optional +// size_t num_device_layouts; +// PJRT_Memory* memory; +// PJRT_AsyncHostToDeviceTransferManager* transfer_manager; // out +// }; + +pub const TransferManager = struct { + pjrt_client: *pjrt.Client, + pjrt_api: *pjrt.Api, + pjrt_transfer_manager: []*pjrt.AsyncHostToDeviceTransferManager, + shape_specs: []const Shape, + memory: *pjrt.Memory, + + pub fn init(platform: Platform, memory_kind: pjrt.Memory.Kind, shapes: []Shape) !TransferManager { + const device = platform.getDevices()[0]; + const memory = device.getMemoryByKind(memory_kind); + if (memory == null) { + stdx.debug.panic("Device {s} doesn't have memory of kind {s}", .{ device.getName(), @tagName(memory_kind) }); + } + + return .{ + .pjrt_client = platform.pjrt_client, + .pjrt_api = platform.pjrt_api, + .pjrt_transfer_manager = try pjrt.Client.createBuffersForAsyncHostToDevice(platform.pjrt_client, memory, null), + .shape_specs = shapes, + .memory = memory, + }; + } + + pub fn deinit(self: *TransferManager) void { + pjrt.AsyncHostToDeviceTransferManager.deinit(self.pjrt_transfer_manager, self.pjrt_api); + } +}; + const _CreateOptions = struct { // XLA CPU client doesn't read options // https://github.com/openxla/xla/blob/42496a28c374bd35f493cc5dbde74805407245dc/xla/pjrt/c/pjrt_c_api_cpu_internal.cc#L33-L46 @@ -108,6 +199,7 @@ const _CreateOptions = struct { // num_nodes // enable_mock_nccl // mock_gpu_topology + should_stage_host_to_device_transfers: bool = false, pub const Allocator = union(enum) { /// "Best-Fit with Coalescing" algorithm @@ -125,6 +217,8 @@ const _CreateOptions = struct { }; pub fn writeNamedValues(self: Cuda, values: *std.ArrayListUnmanaged(pjrt.NamedValue)) void { + values.appendAssumeCapacity(pjrt.NamedValue.fromBool("should_stage_host_to_device_transfers", self.should_stage_host_to_device_transfers)); + switch (self.allocator) { .platform => { values.appendAssumeCapacity(pjrt.NamedValue.fromString("allocator", "platform")); diff --git a/zml/tensor.zig b/zml/tensor.zig index 15543d7f..391115bf 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -40,9 +40,11 @@ pub const Tensor = struct { _shape: Shape, _id: _Id, _donation: _Donation = .no_buffer, + _output_memory_kind: _MemoryKind = undefined, pub const _Donation = union(enum) { no_buffer, input_buffer, arg: u16 }; pub const _Id = union(enum) { mlir: mlir.Value, buffer_id: u64, arg_id: u64 }; + pub const _MemoryKind = ?Buffer.Memory.Kind; pub const MAX_RANK = Shape.MAX_RANK; @@ -193,6 +195,33 @@ pub const Tensor = struct { }; } + pub fn toMemory(self: Tensor, kind: Buffer.Memory.Kind) Tensor { + return switch (self._id) { + .arg_id, .mlir => { + const ctx = self.getContext(); + var res = self; + res._output_memory_kind = kind; + + const memory_kind = mlir.StringAttribute.init(ctx.mlirCtx(), @tagName(kind)); + + const op = dialect.stablehlo.annotate_device_placement( + ctx.mlirCtx(), + &.{self.value()}, + memory_kind, + &.{self.value().getType()}, + ctx.mlirCtx().location(@src()), + ); + + return _result(res._shape, op.result(0)); + }, + .buffer_id => { + var res = self; + res._output_memory_kind = kind; + return res; + }, + }; + } + /// Returns a Tensor with new tag names. pub fn rename(self: Tensor, renames: anytype) Tensor { var res = self;