From a6dfa1d48b68149fc2f8b9108cdad5f651972fe8 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Thu, 6 Mar 2025 15:17:16 +0000 Subject: [PATCH 1/7] test pa triton kernel --- examples/simple_layer/BUILD.bazel | 11 ++ examples/simple_layer/kernel.bc | Bin 0 -> 788 bytes examples/simple_layer/kernel.mlir | 12 ++ examples/simple_layer/main.zig | 37 ++++- examples/simple_layer/pa.zig | 125 ++++++++++++++ examples/simple_layer/pa_kernel.mlir | 236 +++++++++++++++++++++++++++ mlir/dialects/stablehlo.zig | 33 ++++ zml/module.zig | 6 +- zml/ops.zig | 43 +++++ 9 files changed, 491 insertions(+), 12 deletions(-) create mode 100644 examples/simple_layer/kernel.bc create mode 100644 examples/simple_layer/kernel.mlir create mode 100644 examples/simple_layer/pa.zig create mode 100644 examples/simple_layer/pa_kernel.mlir diff --git a/examples/simple_layer/BUILD.bazel b/examples/simple_layer/BUILD.bazel index 8d747677..9af8a468 100644 --- a/examples/simple_layer/BUILD.bazel +++ b/examples/simple_layer/BUILD.bazel @@ -5,6 +5,7 @@ load("@zml//bazel:zig.bzl", "zig_cc_binary") zig_cc_binary( name = "simple_layer", + extra_srcs = ["kernel.mlir"], main = "main.zig", deps = [ "@zml//async", @@ -12,6 +13,16 @@ zig_cc_binary( ], ) +zig_cc_binary( + name = "pa", + extra_srcs = ["pa_kernel.mlir"], + main = "pa.zig", + deps = [ + "@zml//async", + "@zml//zml", + ], +) + # Manifest created from the simple_layer binary and friends mtree_spec( name = "mtree", diff --git a/examples/simple_layer/kernel.bc b/examples/simple_layer/kernel.bc new file mode 100644 index 0000000000000000000000000000000000000000..1afeae47a1ecf3b7acebb7f6e8c41f5085435618 GIT binary patch literal 788 zcmZXS!D`z;5I}b{UT?)BmPHt0L=Z-}CYac9LmEn4hq$dnXhPHU5OOF+iREOeEd^NVcdz!CjM z4Cpe77yz_n8Xy2faCsH6hJ8=)b^u%u1L6zuoA}`bAR>)A0GB>I02u|~H3CqCCVZkn z1Vobvi8Uf3T0sCY(Iz^?I?*LIWB`2`Cxx-sy?IjFWn!;~S)QdM_K?kQ(_+5JKR#r^ zalomC$X)Jw6ySo}vbJ|)i{X6mbj1E0Y?_sMF&x}g*%7NfMh+(mM9rb+a`z-eQTyp& z|I7{7!YFKoG1KEVN6_PSfvCqDE}|UwDdX1%h^QKqEs4mb#`-pT95tWpc&OZl@3~^~ zOd-br>$_CqRC->O!u!%*Ray%s`_wnyjIjQK93baa<;3^SLFKfYlf%ksb+G=L96`=; z<#fsMUcT}9#=2r}>+}F%-wKOK3gqPmds|< zBDPFEq|?8xnO5gTp3K;Mm0hx{)J|`T{HjQnQ!`_w&5M*3sa+KryUerFCYfdFoh=P3 zR~LpYSBrUd>zoZsJ2Tmce%YIuTT_|~voQ7(U0a$wo0{d#f;~EDZ~z(8XaIWH!7hG7 Pp?yJfCTIY^3flewB=Uo= literal 0 HcmV?d00001 diff --git a/examples/simple_layer/kernel.mlir b/examples/simple_layer/kernel.mlir new file mode 100644 index 00000000..ae457cb9 --- /dev/null +++ b/examples/simple_layer/kernel.mlir @@ -0,0 +1,12 @@ +module { + tt.func public @add_one(%arg0: !tt.ptr {tt.divisibility = 32 : i32}, %arg1: !tt.ptr {tt.divisibility = 32 : i32}, %arg2: !tt.ptr {tt.divisibility = 32 : i32}, %arg3: !tt.ptr {tt.divisibility = 32 : i32}) { + %0 = tt.get_program_id x : i32 + %1 = tt.load %arg0 {cache = 1 : i32, evict = 1 : i32, isVolatile = false} : !tt.ptr + %2 = tt.load %arg1 {cache = 1 : i32, evict = 1 : i32, isVolatile = false} : !tt.ptr + %cst = arith.constant 1.000000e+00 : f32 + %3 = arith.addf %1, %cst : f32 + tt.store %arg2, %3 {cache = 1 : i32, evict = 1 : i32} : !tt.ptr + tt.store %arg3, %2 {cache = 1 : i32, evict = 1 : i32} : !tt.ptr + tt.return + } +} diff --git a/examples/simple_layer/main.zig b/examples/simple_layer/main.zig index 6f61c0b0..eb8d82a3 100644 --- a/examples/simple_layer/main.zig +++ b/examples/simple_layer/main.zig @@ -2,16 +2,36 @@ const std = @import("std"); const zml = @import("zml"); const asynk = @import("async"); +pub const std_options = .{ + .log_level = .info, + .logFn = asynk.logFn(std.log.defaultLog), +}; + /// Model definition const Layer = struct { bias: ?zml.Tensor = null, weight: zml.Tensor, pub fn forward(self: Layer, x: zml.Tensor) zml.Tensor { - var y = self.weight.mul(x); - if (self.bias) |bias| { - y = y.add(bias); - } + _ = self; // autofix + _ = x; // autofix + // var y = self.weight.mul(x); + + const xx = zml.Tensor.scalar(3, .f32); + const xo = zml.Tensor.scalar(5, .f32); + + const y = zml.ops.triton(.{ xx, xo }, xx.shape(), xo.shape(), .{ + .name = "add_one", + .ir = @embedFile("kernel.mlir"), + .grid = .{ 1, 1, 1 }, + .num_stages = 3, + .num_warps = 4, + .debug = true, + }); + + // if (self.bias) |bias| { + // y = y.add(bias); + // } return y; } }; @@ -63,16 +83,15 @@ pub fn asyncMain() !void { // Start compiling. This uses the inferred shapes from the BufferStore. // The shape of the input tensor, we have to pass in manually. var compilation = try asynk.asyncc(zml.compileModel, .{ allocator, Layer.forward, model_shapes, .{input_shape}, platform }); - + // Wait for compilation to finish + const compiled = try compilation.awaitt(); + std.debug.print("Compilation finished\n", .{}); // Produce a bufferized weights struct from the fake BufferStore. // This is like the inferred shapes, but with actual values. // We will need to send those to the computation device later. var model_weights = try zml.aio.loadModelBuffers(Layer, model_shapes, buffer_store, arena, platform); defer zml.aio.unloadBuffers(&model_weights); // for good practice - // Wait for compilation to finish - const compiled = try compilation.awaitt(); - // pass the model weights to the compiled module to create an executable module var executable = compiled.prepare(model_weights); defer executable.deinit(); @@ -93,6 +112,6 @@ pub fn asyncMain() !void { const cpu_result = try result.toHostAlloc(arena); std.debug.print( "\nThe result of {d} * {d} + {d} = {d}\n", - .{ &weights, &input, &bias, cpu_result.items(f16) }, + .{ &weights, &input, &bias, cpu_result.items(f32) }, ); } diff --git a/examples/simple_layer/pa.zig b/examples/simple_layer/pa.zig new file mode 100644 index 00000000..de552aeb --- /dev/null +++ b/examples/simple_layer/pa.zig @@ -0,0 +1,125 @@ +const std = @import("std"); +const zml = @import("zml"); +const asynk = @import("async"); + +pub const std_options = .{ + .log_level = .info, + .logFn = asynk.logFn(std.log.defaultLog), +}; + +pub fn forward( + q: zml.Tensor, + k: zml.Tensor, + v: zml.Tensor, + block_tables: zml.Tensor, + seq_lens: zml.Tensor, +) zml.Tensor { + const output_shape = zml.Shape.init(.{ 256, 32, 128 }, .bf16); + const strides_o = output_shape.computeStrides(); + const strides_q = q.shape().computeStrides(); + const strides_k = k.shape().computeStrides(); + const strides_bt = block_tables.shape().computeStrides(); + // Arguments to kernel: + // 'out_ptr': '*bf16', + // 'q_ptr': '*bf16', + // 'k_cache_ptr': '*bf16', + // 'v_cache_ptr': '*bf16', + // 'blk_tables_ptr': '*i32', + // 'seq_lens_ptr': '*i64', + // 'alibi_slopes': '*i8', + // 'scale': 'fp32', + // 'k_scale': 'fp32', + // 'v_scale': 'fp32', + // 'stride_o_s': 'i32', + // 'stride_o_nh': 'i32', + // 'stride_o_hs': 'i32', + // 'stride_q_s': 'i32', + // 'stride_q_nh': 'i32', + // 'stride_q_hs': 'i32', + // 'stride_k_b': 'i32', + // 'stride_k_nh': 'i32', + // 'stride_k_kb': 'i32', + // 'stride_k_hs': 'i32', + // 'stride_bt_s': 'i32', + // 'stride_bt_nb': 'i32' + + const y = zml.ops.triton(.{ + q, + k, + v, + seq_lens, + block_tables, + zml.Tensor.constant(.{8 * 4}, zml.Data.init(.i8, 0)), // check actually it's none + zml.Tensor.scalar(0.08838834765, .f32), + // zml.Tensor.scalar(1.0, .f32), + // zml.Tensor.scalar(1.0, .f32), + zml.Tensor.scalar(strides_o.get(0), .i32), + zml.Tensor.scalar(strides_o.get(1), .i32), + // zml.Tensor.scalar(strides_o.get(2), .i32), + zml.Tensor.scalar(strides_q.get(0), .i32), + zml.Tensor.scalar(strides_q.get(1), .i32), + zml.Tensor.scalar(strides_q.get(2), .i32), + zml.Tensor.scalar(strides_k.get(0), .i32), + zml.Tensor.scalar(strides_k.get(1), .i32), + zml.Tensor.scalar(strides_k.get(2), .i32), + zml.Tensor.scalar(strides_k.get(3), .i32), + zml.Tensor.scalar(strides_bt.get(0), .i32), + // zml.Tensor.scalar(strides_bt.get(1), .i32), + }, output_shape, .{ + .name = "_paged_attn_decode_v1_w_dot_kernel", + .ir = @embedFile("pa_kernel.mlir"), + .grid = .{ 256, 8, 1 }, + .num_stages = 3, + .num_warps = 4, + .debug = false, + }); + + return y; +} + +pub fn main() !void { + try asynk.AsyncThread.main(std.heap.c_allocator, asyncMain); +} + +pub fn asyncMain() !void { + // Short lived allocations + var gpa = std.heap.GeneralPurposeAllocator(.{}){}; + defer _ = gpa.deinit(); + const allocator = gpa.allocator(); + + // Arena allocator for BufferStore etc. + var arena_state = std.heap.ArenaAllocator.init(allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + _ = arena; // autofix + + var context = try zml.Context.init(); + defer context.deinit(); + + const platform = context.autoPlatform(.{}); + context.printAvailablePlatforms(platform); + + const query = zml.Shape.init(.{ 256, 8 * 4, 128 }, .bf16); + const key_cache = zml.Shape.init(.{ 16384, 8, 16, 128 }, .bf16); + const value_cache = zml.Shape.init(.{ 16384, 8, 16, 128 }, .bf16); + const seq_lens = zml.Shape.init(.{256}, .i64); + const block_tables = zml.Shape.init(.{ 256, 128 }, .i32); + + var compilation = try asynk.asyncc(zml.compileFn, .{ allocator, forward, .{ query, key_cache, value_cache, block_tables, seq_lens }, platform }); + const compiled = try compilation.awaitt(); + _ = compiled; // autofix + std.debug.print("Compilation finished\n", .{}); + + // var model_weights = try zml.aio.loadModelBuffers(PagedAttention, model_shapes, buffer_store, arena, platform); + // defer zml.aio.unloadBuffers(&model_weights); // for good practice + + // var executable = compiled.prepare(model_weights); + // defer executable.deinit(); + + // var input_buffer = try zml.Buffer.from(platform, zml.HostBuffer.fromSlice(input_shape, &input)); + // defer input_buffer.deinit(); + + // call our executable module + // var result: zml.Buffer = executable.call(.{input_buffer}); + // defer result.deinit(); +} diff --git a/examples/simple_layer/pa_kernel.mlir b/examples/simple_layer/pa_kernel.mlir new file mode 100644 index 00000000..917f3150 --- /dev/null +++ b/examples/simple_layer/pa_kernel.mlir @@ -0,0 +1,236 @@ +#loc = loc("/workspace/pa_decode_rework.py":157:0) +#loc1 = loc(unknown) +#loc54 = loc("/workspace/pa_decode_rework.py":271:42) +#loc71 = loc("/workspace/pa_decode_rework.py":286:43) +#loc89 = loc(callsite(#loc1 at #loc54)) +#loc93 = loc(callsite(#loc1 at #loc71)) +module { + tt.func public @_paged_attn_decode_v1_w_dot_kernel(%arg0: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32} loc("/workspace/pa_decode_rework.py":157:0), %arg1: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32} loc("/workspace/pa_decode_rework.py":157:0), %arg2: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32} loc("/workspace/pa_decode_rework.py":157:0), %arg3: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32} loc("/workspace/pa_decode_rework.py":157:0), %arg4: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32} loc("/workspace/pa_decode_rework.py":157:0), %arg5: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32} loc("/workspace/pa_decode_rework.py":157:0), %arg6: f32 loc("/workspace/pa_decode_rework.py":157:0), %arg7: f32 loc("/workspace/pa_decode_rework.py":157:0), %arg8: f32 loc("/workspace/pa_decode_rework.py":157:0), %arg9: i32 {tt.divisibility = 16 : i32} loc("/workspace/pa_decode_rework.py":157:0), %arg10: i32 {tt.divisibility = 16 : i32} loc("/workspace/pa_decode_rework.py":157:0), %arg11: i32 {tt.divisibility = 16 : i32} loc("/workspace/pa_decode_rework.py":157:0), %arg12: i32 {tt.divisibility = 16 : i32} loc("/workspace/pa_decode_rework.py":157:0), %arg13: i32 {tt.divisibility = 16 : i32} loc("/workspace/pa_decode_rework.py":157:0), %arg14: i32 {tt.divisibility = 16 : i32} loc("/workspace/pa_decode_rework.py":157:0), %arg15: i32 {tt.divisibility = 16 : i32} loc("/workspace/pa_decode_rework.py":157:0), %arg16: i32 {tt.divisibility = 16 : i32} loc("/workspace/pa_decode_rework.py":157:0)) attributes {noinline = false} { + %cst = arith.constant dense<0xFF800000> : tensor<16xf32> loc(#loc1) + %cst_0 = arith.constant dense<0.000000e+00> : tensor<16xf32> loc(#loc1) + %c15_i64 = arith.constant 15 : i64 loc(#loc1) + %c1_i64 = arith.constant 1 : i64 loc(#loc1) + %c0_i64 = arith.constant 0 : i64 loc(#loc1) + %cst_1 = arith.constant dense<0.000000e+00> : tensor<16x128xbf16> loc(#loc1) + %cst_2 = arith.constant dense<1.44269502> : tensor<16xf32> loc(#loc1) + %cst_3 = arith.constant dense<1.44269502> : tensor<16x16xf32> loc(#loc1) + %cst_4 = arith.constant dense<0xFF800000> : tensor<16x16xf32> loc(#loc1) + %cst_5 = arith.constant dense<0.000000e+00> : tensor<16x16xf32> loc(#loc1) + %cst_6 = arith.constant dense<16> : tensor<16x1xi32> loc(#loc1) + %c16_i64 = arith.constant 16 : i64 loc(#loc1) + %cst_7 = arith.constant dense<0.000000e+00> : tensor<16x128xf32> loc(#loc1) + %cst_8 = arith.constant dense<128> : tensor<1x128xi32> loc(#loc1) + %cst_9 = arith.constant dense<4> : tensor<16x1xi32> loc(#loc1) + %c4_i32 = arith.constant 4 : i32 loc(#loc1) + %0 = tt.get_program_id x : i32 loc(#loc2) + %1 = tt.get_program_id y : i32 loc(#loc3) + %2 = tt.addptr %arg5, %0 : !tt.ptr, i32 loc(#loc4) + %3 = tt.load %2 : !tt.ptr loc(#loc5) + %4 = arith.addi %3, %c15_i64 : i64 loc(#loc86) + %5 = arith.divsi %4, %c16_i64 : i64 loc(#loc87) + %6 = tt.make_range {end = 16 : i32, start = 0 : i32} : tensor<16xi32> loc(#loc9) + %7 = tt.make_range {end = 128 : i32, start = 0 : i32} : tensor<128xi32> loc(#loc10) + %8 = arith.muli %0, %arg11 : i32 loc(#loc11) + %9 = arith.muli %1, %c4_i32 : i32 loc(#loc12) + %10 = tt.expand_dims %6 {axis = 1 : i32} : tensor<16xi32> -> tensor<16x1xi32> loc(#loc13) + %11 = tt.splat %9 : i32 -> tensor<16x1xi32> loc(#loc14) + %12 = arith.addi %11, %10 : tensor<16x1xi32> loc(#loc14) + %13 = tt.splat %arg12 : i32 -> tensor<16x1xi32> loc(#loc15) + %14 = arith.muli %12, %13 : tensor<16x1xi32> loc(#loc15) + %15 = tt.splat %8 : i32 -> tensor<16x1xi32> loc(#loc16) + %16 = arith.addi %15, %14 : tensor<16x1xi32> loc(#loc16) + %17 = tt.expand_dims %7 {axis = 0 : i32} : tensor<128xi32> -> tensor<1x128xi32> loc(#loc17) + %18 = tt.broadcast %16 : tensor<16x1xi32> -> tensor<16x128xi32> loc(#loc18) + %19 = tt.broadcast %17 : tensor<1x128xi32> -> tensor<16x128xi32> loc(#loc18) + %20 = arith.addi %18, %19 : tensor<16x128xi32> loc(#loc18) + %21 = arith.cmpi slt, %10, %cst_9 : tensor<16x1xi32> loc(#loc19) + %22 = arith.cmpi slt, %17, %cst_8 : tensor<1x128xi32> loc(#loc20) + %23 = tt.broadcast %21 : tensor<16x1xi1> -> tensor<16x128xi1> loc(#loc21) + %24 = tt.broadcast %22 : tensor<1x128xi1> -> tensor<16x128xi1> loc(#loc21) + %25 = arith.andi %23, %24 : tensor<16x128xi1> loc(#loc21) + %26 = tt.splat %arg1 : !tt.ptr -> tensor<16x128x!tt.ptr> loc(#loc22) + %27 = tt.addptr %26, %20 : tensor<16x128x!tt.ptr>, tensor<16x128xi32> loc(#loc22) + %28 = tt.load %27, %25, %cst_1 : tensor<16x128x!tt.ptr> loc(#loc23) + %29 = arith.extf %28 : tensor<16x128xbf16> to tensor<16x128xf32> loc(#loc24) + %30 = tt.splat %arg6 : f32 -> tensor<16x128xf32> loc(#loc24) + %31 = arith.mulf %29, %30 : tensor<16x128xf32> loc(#loc24) + %32 = arith.truncf %31 : tensor<16x128xf32> to tensor<16x128xbf16> loc(#loc25) + %33 = arith.muli %1, %arg14 : i32 loc(#loc26) + %34 = tt.splat %arg15 : i32 -> tensor<16x1xi32> loc(#loc27) + %35 = arith.muli %10, %34 : tensor<16x1xi32> loc(#loc27) + %36 = tt.splat %33 : i32 -> tensor<16x1xi32> loc(#loc28) + %37 = arith.addi %36, %35 : tensor<16x1xi32> loc(#loc28) + %38 = tt.broadcast %37 : tensor<16x1xi32> -> tensor<16x128xi32> loc(#loc29) + %39 = arith.addi %38, %19 : tensor<16x128xi32> loc(#loc29) + %40 = arith.muli %0, %arg16 : i32 loc(#loc30) + %41 = tt.addptr %arg4, %40 : !tt.ptr, i32 loc(#loc31) + %42 = arith.extsi %6 : tensor<16xi32> to tensor<16xi64> loc(#loc32) + %43 = tt.splat %3 : i64 -> tensor<16x1xi64> loc(#loc33) + %44 = arith.cmpi slt, %10, %cst_6 : tensor<16x1xi32> loc(#loc34) + %45 = tt.splat %arg2 : !tt.ptr -> tensor<16x128x!tt.ptr> loc(#loc35) + %46 = tt.splat %3 : i64 -> tensor<1x16xi64> loc(#loc36) + %47 = tt.broadcast %21 : tensor<16x1xi1> -> tensor<16x16xi1> loc(#loc37) + %48 = tt.splat %arg3 : !tt.ptr -> tensor<16x128x!tt.ptr> loc(#loc38) + %49:3 = scf.for %arg17 = %c0_i64 to %5 step %c1_i64 iter_args(%arg18 = %cst_7, %arg19 = %cst_0, %arg20 = %cst) -> (tensor<16x128xf32>, tensor<16xf32>, tensor<16xf32>) : i64 { + %63 = tt.addptr %41, %arg17 : !tt.ptr, i64 loc(#loc40) + %64 = tt.load %63 : !tt.ptr loc(#loc41) + %65 = arith.muli %64, %arg13 : i32 loc(#loc42) + %66 = tt.splat %65 : i32 -> tensor<16x128xi32> loc(#loc43) + %67 = arith.addi %66, %39 : tensor<16x128xi32> loc(#loc43) + %68 = arith.muli %arg17, %c16_i64 : i64 loc(#loc44) + %69 = tt.splat %68 : i64 -> tensor<16xi64> loc(#loc32) + %70 = arith.addi %69, %42 : tensor<16xi64> loc(#loc32) + %71 = tt.expand_dims %70 {axis = 1 : i32} : tensor<16xi64> -> tensor<16x1xi64> loc(#loc45) + %72 = arith.cmpi slt, %71, %43 : tensor<16x1xi64> loc(#loc33) + %73 = arith.andi %72, %44 : tensor<16x1xi1> loc(#loc46) + %74 = tt.broadcast %73 : tensor<16x1xi1> -> tensor<16x128xi1> loc(#loc47) + %75 = arith.andi %74, %24 : tensor<16x128xi1> loc(#loc47) + %76 = tt.addptr %45, %67 : tensor<16x128x!tt.ptr>, tensor<16x128xi32> loc(#loc35) + %77 = tt.load %76, %75, %cst_1 : tensor<16x128x!tt.ptr> loc(#loc48) + %78 = tt.trans %77 {order = array} : tensor<16x128xbf16> -> tensor<128x16xbf16> loc(#loc49) + %79 = tt.dot %32, %78, %cst_5 : tensor<16x128xbf16> * tensor<128x16xbf16> -> tensor<16x16xf32> loc(#loc49) + %80 = tt.expand_dims %70 {axis = 0 : i32} : tensor<16xi64> -> tensor<1x16xi64> loc(#loc50) + %81 = arith.cmpi slt, %80, %46 : tensor<1x16xi64> loc(#loc36) + %82 = tt.broadcast %81 : tensor<1x16xi1> -> tensor<16x16xi1> loc(#loc37) + %83 = arith.andi %47, %82 : tensor<16x16xi1> loc(#loc37) + %84 = arith.select %83, %79, %cst_4 : tensor<16x16xi1>, tensor<16x16xf32> loc(#loc51) + %85 = arith.select %83, %84, %cst_4 : tensor<16x16xi1>, tensor<16x16xf32> loc(#loc52) + %86 = "tt.reduce"(%85) <{axis = 1 : i32}> ({ + ^bb0(%arg21: f32 loc(callsite(#loc1 at #loc54)), %arg22: f32 loc(callsite(#loc1 at #loc54))): + %107 = arith.maxnumf %arg21, %arg22 : f32 loc(#loc95) + tt.reduce.return %107 : f32 loc(#loc88) + }) : (tensor<16x16xf32>) -> tensor<16xf32> loc(#loc88) + %87 = arith.maxnumf %86, %arg20 : tensor<16xf32> loc(#loc56) + %88 = tt.expand_dims %87 {axis = 1 : i32} : tensor<16xf32> -> tensor<16x1xf32> loc(#loc57) + %89 = tt.broadcast %88 : tensor<16x1xf32> -> tensor<16x16xf32> loc(#loc58) + %90 = arith.subf %85, %89 : tensor<16x16xf32> loc(#loc58) + %91 = arith.mulf %90, %cst_3 : tensor<16x16xf32> loc(#loc59) + %92 = math.exp2 %91 : tensor<16x16xf32> loc(#loc60) + %93 = arith.subf %arg20, %87 : tensor<16xf32> loc(#loc61) + %94 = arith.mulf %93, %cst_2 : tensor<16xf32> loc(#loc62) + %95 = math.exp2 %94 : tensor<16xf32> loc(#loc63) + %96 = tt.expand_dims %95 {axis = 1 : i32} : tensor<16xf32> -> tensor<16x1xf32> loc(#loc64) + %97 = tt.broadcast %96 : tensor<16x1xf32> -> tensor<16x128xf32> loc(#loc65) + %98 = arith.mulf %arg18, %97 : tensor<16x128xf32> loc(#loc65) + %99 = tt.addptr %48, %67 : tensor<16x128x!tt.ptr>, tensor<16x128xi32> loc(#loc38) + %100 = tt.load %99, %75, %cst_1 : tensor<16x128x!tt.ptr> loc(#loc66) + %101 = arith.truncf %92 : tensor<16x16xf32> to tensor<16x16xbf16> loc(#loc67) + %102 = tt.dot %101, %100, %98 : tensor<16x16xbf16> * tensor<16x128xbf16> -> tensor<16x128xf32> loc(#loc68) + %103 = arith.mulf %arg19, %95 : tensor<16xf32> loc(#loc69) + %104 = arith.extf %101 : tensor<16x16xbf16> to tensor<16x16xf32> loc(#loc91) + %105 = "tt.reduce"(%104) <{axis = 1 : i32}> ({ + ^bb0(%arg21: f32 loc(callsite(#loc1 at #loc71)), %arg22: f32 loc(callsite(#loc1 at #loc71))): + %107 = arith.addf %arg21, %arg22 : f32 loc(#loc96) + tt.reduce.return %107 : f32 loc(#loc92) + }) : (tensor<16x16xf32>) -> tensor<16xf32> loc(#loc92) + %106 = arith.addf %103, %105 : tensor<16xf32> loc(#loc74) + scf.yield %102, %106, %87 : tensor<16x128xf32>, tensor<16xf32>, tensor<16xf32> loc(#loc75) + } loc(#loc39) + %50 = tt.expand_dims %49#1 {axis = 1 : i32} : tensor<16xf32> -> tensor<16x1xf32> loc(#loc76) + %51 = tt.broadcast %50 : tensor<16x1xf32> -> tensor<16x128xf32> loc(#loc77) + %52 = arith.divf %49#0, %51 : tensor<16x128xf32> loc(#loc77) + %53 = arith.muli %0, %arg9 : i32 loc(#loc78) + %54 = tt.splat %arg10 : i32 -> tensor<16x1xi32> loc(#loc79) + %55 = arith.muli %12, %54 : tensor<16x1xi32> loc(#loc79) + %56 = tt.splat %53 : i32 -> tensor<16x1xi32> loc(#loc80) + %57 = arith.addi %56, %55 : tensor<16x1xi32> loc(#loc80) + %58 = tt.broadcast %57 : tensor<16x1xi32> -> tensor<16x128xi32> loc(#loc81) + %59 = arith.addi %58, %19 : tensor<16x128xi32> loc(#loc81) + %60 = tt.splat %arg0 : !tt.ptr -> tensor<16x128x!tt.ptr> loc(#loc82) + %61 = tt.addptr %60, %59 : tensor<16x128x!tt.ptr>, tensor<16x128xi32> loc(#loc82) + %62 = arith.truncf %52 : tensor<16x128xf32> to tensor<16x128xbf16> loc(#loc83) + tt.store %61, %62, %25 : tensor<16x128x!tt.ptr> loc(#loc84) + tt.return loc(#loc85) + } loc(#loc) +} loc(#loc) +#loc2 = loc("/workspace/pa_decode_rework.py":192:28) +#loc3 = loc("/workspace/pa_decode_rework.py":193:32) +#loc4 = loc("/workspace/pa_decode_rework.py":197:37) +#loc5 = loc("/workspace/pa_decode_rework.py":197:22) +#loc6 = loc("/usr/local/lib/python3.12/dist-packages/triton/language/standard.py":40:22) +#loc7 = loc("/workspace/pa_decode_rework.py":199:35) +#loc8 = loc("/usr/local/lib/python3.12/dist-packages/triton/language/standard.py":40:28) +#loc9 = loc("/workspace/pa_decode_rework.py":201:28) +#loc10 = loc("/workspace/pa_decode_rework.py":202:32) +#loc11 = loc("/workspace/pa_decode_rework.py":216:18) +#loc12 = loc("/workspace/pa_decode_rework.py":217:25) +#loc13 = loc("/workspace/pa_decode_rework.py":217:51) +#loc14 = loc("/workspace/pa_decode_rework.py":217:40) +#loc15 = loc("/workspace/pa_decode_rework.py":217:63) +#loc16 = loc("/workspace/pa_decode_rework.py":217:10) +#loc17 = loc("/workspace/pa_decode_rework.py":218:23) +#loc18 = loc("/workspace/pa_decode_rework.py":218:10) +#loc19 = loc("/workspace/pa_decode_rework.py":222:36) +#loc20 = loc("/workspace/pa_decode_rework.py":222:77) +#loc21 = loc("/workspace/pa_decode_rework.py":222:53) +#loc22 = loc("/workspace/pa_decode_rework.py":224:24) +#loc23 = loc("/workspace/pa_decode_rework.py":224:16) +#loc24 = loc("/workspace/pa_decode_rework.py":225:13) +#loc25 = loc("/workspace/pa_decode_rework.py":225:23) +#loc26 = loc("/workspace/pa_decode_rework.py":232:22) +#loc27 = loc("/workspace/pa_decode_rework.py":233:30) +#loc28 = loc("/workspace/pa_decode_rework.py":233:10) +#loc29 = loc("/workspace/pa_decode_rework.py":234:10) +#loc30 = loc("/workspace/pa_decode_rework.py":236:51) +#loc31 = loc("/workspace/pa_decode_rework.py":236:41) +#loc32 = loc("/workspace/pa_decode_rework.py":241:39) +#loc33 = loc("/workspace/pa_decode_rework.py":243:37) +#loc34 = loc("/workspace/pa_decode_rework.py":244:35) +#loc35 = loc("/workspace/pa_decode_rework.py":249:36) +#loc36 = loc("/workspace/pa_decode_rework.py":256:76) +#loc37 = loc("/workspace/pa_decode_rework.py":256:52) +#loc38 = loc("/workspace/pa_decode_rework.py":279:36) +#loc39 = loc("/workspace/pa_decode_rework.py":238:19) +#loc40 = loc("/workspace/pa_decode_rework.py":239:50) +#loc41 = loc("/workspace/pa_decode_rework.py":239:30) +#loc42 = loc("/workspace/pa_decode_rework.py":240:36) +#loc43 = loc("/workspace/pa_decode_rework.py":240:49) +#loc44 = loc("/workspace/pa_decode_rework.py":241:27) +#loc45 = loc("/workspace/pa_decode_rework.py":243:26) +#loc46 = loc("/workspace/pa_decode_rework.py":244:15) +#loc47 = loc("/workspace/pa_decode_rework.py":245:15) +#loc48 = loc("/workspace/pa_decode_rework.py":249:22) +#loc49 = loc("/workspace/pa_decode_rework.py":254:23) +#loc50 = loc("/workspace/pa_decode_rework.py":256:65) +#loc51 = loc("/workspace/pa_decode_rework.py":258:12) +#loc52 = loc("/workspace/pa_decode_rework.py":269:12) +#loc53 = loc("/usr/local/lib/python3.12/dist-packages/triton/language/standard.py":184:40) +#loc55 = loc("/usr/local/lib/python3.12/dist-packages/triton/language/standard.py":163:27) +#loc56 = loc("/workspace/pa_decode_rework.py":271:55) +#loc57 = loc("/workspace/pa_decode_rework.py":274:45) +#loc58 = loc("/workspace/pa_decode_rework.py":274:31) +#loc59 = loc("/workspace/pa_decode_rework.py":274:57) +#loc60 = loc("/workspace/pa_decode_rework.py":274:25) +#loc61 = loc("/workspace/pa_decode_rework.py":275:42) +#loc62 = loc("/workspace/pa_decode_rework.py":275:59) +#loc63 = loc("/workspace/pa_decode_rework.py":275:29) +#loc64 = loc("/workspace/pa_decode_rework.py":276:21) +#loc65 = loc("/workspace/pa_decode_rework.py":276:15) +#loc66 = loc("/workspace/pa_decode_rework.py":279:22) +#loc67 = loc("/workspace/pa_decode_rework.py":283:17) +#loc68 = loc("/workspace/pa_decode_rework.py":284:25) +#loc69 = loc("/workspace/pa_decode_rework.py":286:28) +#loc70 = loc("/usr/local/lib/python3.12/dist-packages/triton/language/standard.py":266:46) +#loc72 = loc("/usr/local/lib/python3.12/dist-packages/triton/language/standard.py":267:36) +#loc73 = loc("/usr/local/lib/python3.12/dist-packages/triton/language/standard.py":256:15) +#loc74 = loc("/workspace/pa_decode_rework.py":286:36) +#loc75 = loc("/workspace/pa_decode_rework.py":287:8) +#loc76 = loc("/workspace/pa_decode_rework.py":289:24) +#loc77 = loc("/workspace/pa_decode_rework.py":289:16) +#loc78 = loc("/workspace/pa_decode_rework.py":292:18) +#loc79 = loc("/workspace/pa_decode_rework.py":293:63) +#loc80 = loc("/workspace/pa_decode_rework.py":293:10) +#loc81 = loc("/workspace/pa_decode_rework.py":294:10) +#loc82 = loc("/workspace/pa_decode_rework.py":298:23) +#loc83 = loc("/workspace/pa_decode_rework.py":298:40) +#loc84 = loc("/workspace/pa_decode_rework.py":298:33) +#loc85 = loc("/workspace/pa_decode_rework.py":298:4) +#loc86 = loc(callsite(#loc6 at #loc7)) +#loc87 = loc(callsite(#loc8 at #loc7)) +#loc88 = loc(callsite(#loc53 at #loc54)) +#loc90 = loc(callsite(#loc55 at #loc53)) +#loc91 = loc(callsite(#loc70 at #loc71)) +#loc92 = loc(callsite(#loc72 at #loc71)) +#loc94 = loc(callsite(#loc73 at #loc72)) +#loc95 = loc(callsite(#loc90 at #loc54)) +#loc96 = loc(callsite(#loc94 at #loc71)) diff --git a/mlir/dialects/stablehlo.zig b/mlir/dialects/stablehlo.zig index d7d8673c..f6062c8f 100644 --- a/mlir/dialects/stablehlo.zig +++ b/mlir/dialects/stablehlo.zig @@ -764,6 +764,39 @@ pub fn custom_call(ctx: mlir.Context, inputs: []const mlir.Value, opts: CustomCa }); } +pub const CustomCallOpts2 = struct { + call_target_name: [:0]const u8, + has_side_effect: bool, + backend_config: mlir.DictionaryAttribute, + // api_version: i32, + output_operand_aliases: []const i64, +}; + +pub fn custom_call2(ctx: mlir.Context, inputs: []const mlir.Value, opts: CustomCallOpts2, res_types: []const mlir.Type, location: mlir.Location) mlir.Operation { + var buffer: [1024]u8 = undefined; + var fba = std.heap.FixedBufferAllocator.init(&buffer); + const allocator = fba.allocator(); + + const output_operand_aliases = allocator.alloc(mlir.Attribute, opts.output_operand_aliases.len) catch unreachable; + for (opts.output_operand_aliases, 0..) |alias, i| { + output_operand_aliases[i] = OutputOperandAliasAttribute.init(ctx, &.{}, alias, &.{}).as(mlir.Attribute).?; + } + + return mlir.Operation.make(ctx, "stablehlo.custom_call", .{ + .operands = inputs, + .results = res_types, + .attributes = &.{ + // .{ "api_version", mlir.IntegerAttribute(.i32).init(ctx, opts.api_version).as(mlir.Attribute).? }, + .{ "call_target_name", mlir.StringAttribute.init(ctx, opts.call_target_name).as(mlir.Attribute).? }, + .{ "has_side_effect", mlir.BoolAttribute.init(ctx, opts.has_side_effect).as(mlir.Attribute).? }, + .{ "backend_config", mlir.StringAttribute.init(ctx, "").as(mlir.Attribute).? }, + .{ "output_operand_aliases", mlir.ArrayAttribute.init(ctx, output_operand_aliases).as(mlir.Attribute).? }, + .{ "mhlo.backend_config", opts.backend_config.as(mlir.Attribute).? }, + }, + .location = location, + }); +} + pub fn sharding(ctx: mlir.Context, inputs: []const mlir.Value, sharding_spec: mlir.StringAttribute, res_types: []const mlir.Type, location: mlir.Location) mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.custom_call", .{ .operands = inputs, diff --git a/zml/module.zig b/zml/module.zig index 2377449d..58f30cbc 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -915,9 +915,9 @@ fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, platform: Platform, m .cuda => { // NVIDIA recommends these settings // https://github.com/NVIDIA/JAX-Toolbox?tab=readme-ov-file#environment-variables - setFlag(&options, "xla_gpu_enable_triton_gemm", false); - setFlag(&options, "xla_gpu_enable_latency_hiding_scheduler", true); - setFlag(&options, "xla_gpu_enable_llvm_module_compilation_parallelism", true); + // setFlag(&options, "xla_gpu_enable_triton_gemm", false); + // setFlag(&options, "xla_gpu_enable_latency_hiding_scheduler", true); + // setFlag(&options, "xla_gpu_enable_llvm_module_compilation_parallelism", true); setFlag(&options, "xla_gpu_enable_libnvptxcompiler", true); // setFlag(&options, "xla_gpu_enable_cudnn_fmha", true); // setFlag(&options, "xla_gpu_fused_attention_use_cudnn_rng", true); diff --git a/zml/ops.zig b/zml/ops.zig index 99c7ace2..22e40b90 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -1160,3 +1160,46 @@ inline fn toI64(values: anytype) []i64 { for (values, 0..) |val, i| res[i] = @intCast(val); return res[0..values.len]; } + +pub const TritonOps = struct { + debug: bool = false, + name: []const u8, + ir: []const u8, + grid: [3]i32, + num_stages: i32, + num_warps: i32, +}; + +/// Generate an MLIR call to the given member function with the given tensors. +pub fn triton(args: anytype, res_shape: anytype, opts: TritonOps) Tensor { + const ctx = CompilationContext.current(); + + var values: [args.len]mlir.Value = undefined; + ctx.extractValues(&args, &values); + + const attrs = mlir.DictionaryAttribute.init(ctx.mlirCtx(), &.{ + mlir.NamedAttribute.init(mlir.Identifier.get(ctx.mlirCtx(), "name"), mlir.StringAttribute.init(ctx.mlirCtx(), opts.name).as(mlir.Attribute).?), + mlir.NamedAttribute.init(mlir.Identifier.get(ctx.mlirCtx(), "ir"), mlir.StringAttribute.init(ctx.mlirCtx(), opts.ir).as(mlir.Attribute).?), + mlir.NamedAttribute.init(mlir.Identifier.get(ctx.mlirCtx(), "grid_x"), mlir.IntegerAttribute(.i32).init(ctx.mlirCtx(), @intCast(opts.grid[0])).as(mlir.Attribute).?), + mlir.NamedAttribute.init(mlir.Identifier.get(ctx.mlirCtx(), "grid_y"), mlir.IntegerAttribute(.i32).init(ctx.mlirCtx(), @intCast(opts.grid[1])).as(mlir.Attribute).?), + mlir.NamedAttribute.init(mlir.Identifier.get(ctx.mlirCtx(), "grid_z"), mlir.IntegerAttribute(.i32).init(ctx.mlirCtx(), @intCast(opts.grid[2])).as(mlir.Attribute).?), + mlir.NamedAttribute.init(mlir.Identifier.get(ctx.mlirCtx(), "num_stages"), mlir.IntegerAttribute(.i32).init(ctx.mlirCtx(), @intCast(opts.num_stages)).as(mlir.Attribute).?), + mlir.NamedAttribute.init(mlir.Identifier.get(ctx.mlirCtx(), "num_warps"), mlir.IntegerAttribute(.i32).init(ctx.mlirCtx(), @intCast(opts.num_warps)).as(mlir.Attribute).?), + }); + + const op = dialect.stablehlo.custom_call2( + ctx.mlirCtx(), + &values, + .{ + .call_target_name = "__gpu$xla.gpu.triton", + .backend_config = attrs, + // .api_version = 4, + .has_side_effect = false, + .output_operand_aliases = &.{}, + }, + &.{mlir.ext.mlirType(ctx.mlirCtx(), res_shape)}, + ctx.mlirCtx().location(@src()), + ); + + return Tensor._result(res_shape, op.result(0)); +} From 0794fe560dc34be6cbe1e306c1d81aa940ea8942 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Fri, 7 Mar 2025 18:11:07 +0000 Subject: [PATCH 2/7] pa decode triton --- examples/simple_layer/load_q_in_o.mlir | 24 ++++ examples/simple_layer/pa.zig | 132 ++++++++++++++++--- examples/simple_layer/pa_kernel.mlir | 175 ++++++++++++++----------- examples/simple_layer/pour.mlir | 10 ++ 4 files changed, 243 insertions(+), 98 deletions(-) create mode 100644 examples/simple_layer/load_q_in_o.mlir create mode 100644 examples/simple_layer/pour.mlir diff --git a/examples/simple_layer/load_q_in_o.mlir b/examples/simple_layer/load_q_in_o.mlir new file mode 100644 index 00000000..bbe637e6 --- /dev/null +++ b/examples/simple_layer/load_q_in_o.mlir @@ -0,0 +1,24 @@ +module { + tt.func public @_paged_attn_decode_v1_w_dot_kernel_tt_load_only(%ptr1: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32}, %ptr2: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32}, %ptr3: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32}, %ptr4: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32}, %ptr5: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32}, %ptr6: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32}, %ptr7: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32}, %ptr8: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32}, %ptr9: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32}, %ptr10: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32}, %ptr11: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32}, %ptr12: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32}, %ptr13: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32}, %ptr14: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32}, %ptr15: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32}, %ptr16: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32}, %ptr0: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32}) { + + %0 = tt.load %ptr6 : !tt.ptr + %1 = tt.load %ptr7 : !tt.ptr + %2 = tt.load %ptr8 : !tt.ptr + %3 = tt.load %ptr9 : !tt.ptr + %4 = tt.load %ptr10 : !tt.ptr + %5 = tt.load %ptr11 : !tt.ptr + %6 = tt.load %ptr12 : !tt.ptr + %7 = tt.load %ptr13 : !tt.ptr + %8 = tt.load %ptr14 : !tt.ptr + %9 = tt.load %ptr15 : !tt.ptr + %10 = tt.load %ptr16 : !tt.ptr + tt.call @_paged_attn_decode_v1_w_dot_kernel(%ptr0, %ptr1, %ptr2, %ptr3, %ptr4, %ptr5, %0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10) : (!tt.ptr, !tt.ptr, !tt.ptr, !tt.ptr, !tt.ptr, !tt.ptr, f32, f32, f32, i32, i32, i32, i32, i32, i32, i32, i32) -> () + tt.return + } + + tt.func private @_paged_attn_decode_v1_w_dot_kernel(%arg0: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32} loc("/workspace/pa_decode_rework.py":159:0), %arg1: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32} loc("/workspace/pa_decode_rework.py":159:0), %arg2: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32} loc("/workspace/pa_decode_rework.py":159:0), %arg3: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32} loc("/workspace/pa_decode_rework.py":159:0), %arg4: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32} loc("/workspace/pa_decode_rework.py":159:0), %arg5: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32} loc("/workspace/pa_decode_rework.py":159:0), %arg6: f32 loc("/workspace/pa_decode_rework.py":159:0), %arg7: f32 loc("/workspace/pa_decode_rework.py":159:0), %arg8: f32 loc("/workspace/pa_decode_rework.py":159:0), %arg9: i32 {tt.divisibility = 16 : i32} loc("/workspace/pa_decode_rework.py":159:0), %arg10: i32 {tt.divisibility = 16 : i32} loc("/workspace/pa_decode_rework.py":159:0), %arg11: i32 {tt.divisibility = 16 : i32} loc("/workspace/pa_decode_rework.py":159:0), %arg12: i32 {tt.divisibility = 16 : i32} loc("/workspace/pa_decode_rework.py":159:0), %arg13: i32 {tt.divisibility = 16 : i32} loc("/workspace/pa_decode_rework.py":159:0), %arg14: i32 {tt.divisibility = 16 : i32} loc("/workspace/pa_decode_rework.py":159:0), %arg15: i32 {tt.divisibility = 16 : i32} loc("/workspace/pa_decode_rework.py":159:0), %arg16: i32 {tt.divisibility = 16 : i32} loc("/workspace/pa_decode_rework.py":159:0)) attributes {noinline = false} { + %0 = tt.load %arg1 : !tt.ptr + tt.store %arg0, %0 : !tt.ptr + tt.return + } +} diff --git a/examples/simple_layer/pa.zig b/examples/simple_layer/pa.zig index de552aeb..b5e3f803 100644 --- a/examples/simple_layer/pa.zig +++ b/examples/simple_layer/pa.zig @@ -2,9 +2,12 @@ const std = @import("std"); const zml = @import("zml"); const asynk = @import("async"); -pub const std_options = .{ +pub const std_options: std.Options = .{ .log_level = .info, - .logFn = asynk.logFn(std.log.defaultLog), + // .logFn = asynk.logFn(std.log.defaultLog), + // .log_scope_levels = .{ + // .@"zml/async" = .warn, + // }, }; pub fn forward( @@ -43,13 +46,16 @@ pub fn forward( // 'stride_bt_s': 'i32', // 'stride_bt_nb': 'i32' + // _paged_attn_decode_v1_w_dot_kernel.kd + // _paged_attn_decode_v1_w_dot_kernel_tt_load_only__1.kd + const y = zml.ops.triton(.{ q, k, v, - seq_lens, block_tables, - zml.Tensor.constant(.{8 * 4}, zml.Data.init(.i8, 0)), // check actually it's none + seq_lens, + // zml.Tensor.empty(.{8 * 4}, .i8), // check actually it's none zml.Tensor.scalar(0.08838834765, .f32), // zml.Tensor.scalar(1.0, .f32), // zml.Tensor.scalar(1.0, .f32), @@ -66,10 +72,10 @@ pub fn forward( zml.Tensor.scalar(strides_bt.get(0), .i32), // zml.Tensor.scalar(strides_bt.get(1), .i32), }, output_shape, .{ - .name = "_paged_attn_decode_v1_w_dot_kernel", + .name = "_paged_attn_decode_v1_w_dot_kernel_tt_load_only", .ir = @embedFile("pa_kernel.mlir"), .grid = .{ 256, 8, 1 }, - .num_stages = 3, + .num_stages = 2, .num_warps = 4, .debug = false, }); @@ -93,33 +99,121 @@ pub fn asyncMain() !void { const arena = arena_state.allocator(); _ = arena; // autofix + var rng = std.Random.DefaultPrng.init(0); + const random = rng.random(); + var context = try zml.Context.init(); defer context.deinit(); - const platform = context.autoPlatform(.{}); - context.printAvailablePlatforms(platform); + const platform = context.autoPlatform(.{}).withCompilationOptions(.{ + .sharding_enabled = false, + }); + const max_seq_len = 2048; const query = zml.Shape.init(.{ 256, 8 * 4, 128 }, .bf16); const key_cache = zml.Shape.init(.{ 16384, 8, 16, 128 }, .bf16); const value_cache = zml.Shape.init(.{ 16384, 8, 16, 128 }, .bf16); const seq_lens = zml.Shape.init(.{256}, .i64); - const block_tables = zml.Shape.init(.{ 256, 128 }, .i32); + const block_tables = zml.Shape.init(.{ 256, max_seq_len / 16 }, .i32); var compilation = try asynk.asyncc(zml.compileFn, .{ allocator, forward, .{ query, key_cache, value_cache, block_tables, seq_lens }, platform }); const compiled = try compilation.awaitt(); - _ = compiled; // autofix + defer compiled.deinit(); std.debug.print("Compilation finished\n", .{}); - // var model_weights = try zml.aio.loadModelBuffers(PagedAttention, model_shapes, buffer_store, arena, platform); - // defer zml.aio.unloadBuffers(&model_weights); // for good practice + const seqlen_host = try allocator.alloc(i64, 256); + defer allocator.free(seqlen_host); + @memset(seqlen_host, 1); + const page_table_host = try allocator.alloc(i32, 256 * (max_seq_len / 16)); + defer allocator.free(page_table_host); + for (0..256) |i| { + for (0..(max_seq_len / 16)) |j| { + page_table_host[i * (max_seq_len / 16) + j] = 0; + } + } - // var executable = compiled.prepare(model_weights); - // defer executable.deinit(); + { + var q_buffer = try createRandomBuffer(allocator, platform, query, random); + defer q_buffer.deinit(); + var k_buffer = try createRandomBuffer(allocator, platform, key_cache, random); + defer k_buffer.deinit(); + var v_buffer = try createRandomBuffer(allocator, platform, value_cache, random); + defer v_buffer.deinit(); + var bt_buffer = try zml.Buffer.fromBytes(platform, block_tables, std.mem.sliceAsBytes(page_table_host)); + defer bt_buffer.deinit(); + var sq_buffer = try zml.Buffer.fromBytes(platform, seq_lens, std.mem.sliceAsBytes(seqlen_host)); + defer sq_buffer.deinit(); - // var input_buffer = try zml.Buffer.from(platform, zml.HostBuffer.fromSlice(input_shape, &input)); - // defer input_buffer.deinit(); + // var profiler = platform.getProfiler(null); + // defer profiler.deinit(); - // call our executable module - // var result: zml.Buffer = executable.call(.{input_buffer}); - // defer result.deinit(); + // profiler.start(); + var result: zml.Buffer = compiled.call(.{ q_buffer, k_buffer, v_buffer, bt_buffer, sq_buffer }); + defer result.deinit(); + + var r = try result.toHostAlloc(allocator); + defer r.deinit(allocator); + // profiler.stop(); + // try profiler.dumpAsJsonTo(allocator, std.fs.cwd(), "profile_file.json"); + std.debug.print("Result: {}\n", .{r.shape()}); + } + + { + var q_buffer = try createRandomBuffer(allocator, platform, query, random); + defer q_buffer.deinit(); + var k_buffer = try createRandomBuffer(allocator, platform, key_cache, random); + defer k_buffer.deinit(); + var v_buffer = try createRandomBuffer(allocator, platform, value_cache, random); + defer v_buffer.deinit(); + var bt_buffer = try zml.Buffer.fromBytes(platform, block_tables, std.mem.sliceAsBytes(page_table_host)); + defer bt_buffer.deinit(); + var sq_buffer = try zml.Buffer.fromBytes(platform, seq_lens, std.mem.sliceAsBytes(seqlen_host)); + defer sq_buffer.deinit(); + + // var profiler = platform.getProfiler(null); + // defer profiler.deinit(); + + // profiler.start(); + var result: zml.Buffer = compiled.call(.{ q_buffer, k_buffer, v_buffer, bt_buffer, sq_buffer }); + defer result.deinit(); + + var r = try result.toHostAlloc(allocator); + defer r.deinit(allocator); + // profiler.stop(); + // try profiler.dumpAsJsonTo(allocator, std.fs.cwd(), "profile_file.json"); + std.debug.print("Result: {}\n", .{r.shape()}); + } +} + +fn createRandomBuffer(allocator: std.mem.Allocator, platform: zml.Platform, shape: zml.Shape, random: std.Random) !zml.Buffer { + const data = try allocator.alloc(u8, shape.byteSize()); + defer allocator.free(data); + + switch (shape.dtype()) { + inline else => |v| { + const ZigType = v.toZigType(); + switch (comptime v.class()) { + .bool => unreachable, + .integer => { + for (std.mem.bytesAsSlice(ZigType, data)) |*e| e.* = random.int(ZigType); + }, + .float => { + const value = random.float(f64); + for (std.mem.bytesAsSlice(ZigType, data)) |*e| e.* = if (ZigType == f64) + value + else if (ZigType == f32) + @floatCast(value) + else if (ZigType == f16) + @floatCast(value) + else + @bitCast(random.int(std.meta.Int(.unsigned, @bitSizeOf(ZigType)))); + }, + .complex => unreachable, + } + }, + } + + var host_buffer = zml.HostBuffer.fromBytes(shape, data); + errdefer host_buffer.deinit(allocator); + return zml.Buffer.from(platform, host_buffer); } diff --git a/examples/simple_layer/pa_kernel.mlir b/examples/simple_layer/pa_kernel.mlir index 917f3150..9c7614f8 100644 --- a/examples/simple_layer/pa_kernel.mlir +++ b/examples/simple_layer/pa_kernel.mlir @@ -1,11 +1,28 @@ -#loc = loc("/workspace/pa_decode_rework.py":157:0) +#loc = loc("/workspace/pa_decode_rework.py":159:0) #loc1 = loc(unknown) -#loc54 = loc("/workspace/pa_decode_rework.py":271:42) -#loc71 = loc("/workspace/pa_decode_rework.py":286:43) +#loc54 = loc("/workspace/pa_decode_rework.py":273:42) +#loc71 = loc("/workspace/pa_decode_rework.py":288:43) #loc89 = loc(callsite(#loc1 at #loc54)) #loc93 = loc(callsite(#loc1 at #loc71)) module { - tt.func public @_paged_attn_decode_v1_w_dot_kernel(%arg0: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32} loc("/workspace/pa_decode_rework.py":157:0), %arg1: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32} loc("/workspace/pa_decode_rework.py":157:0), %arg2: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32} loc("/workspace/pa_decode_rework.py":157:0), %arg3: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32} loc("/workspace/pa_decode_rework.py":157:0), %arg4: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32} loc("/workspace/pa_decode_rework.py":157:0), %arg5: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32} loc("/workspace/pa_decode_rework.py":157:0), %arg6: f32 loc("/workspace/pa_decode_rework.py":157:0), %arg7: f32 loc("/workspace/pa_decode_rework.py":157:0), %arg8: f32 loc("/workspace/pa_decode_rework.py":157:0), %arg9: i32 {tt.divisibility = 16 : i32} loc("/workspace/pa_decode_rework.py":157:0), %arg10: i32 {tt.divisibility = 16 : i32} loc("/workspace/pa_decode_rework.py":157:0), %arg11: i32 {tt.divisibility = 16 : i32} loc("/workspace/pa_decode_rework.py":157:0), %arg12: i32 {tt.divisibility = 16 : i32} loc("/workspace/pa_decode_rework.py":157:0), %arg13: i32 {tt.divisibility = 16 : i32} loc("/workspace/pa_decode_rework.py":157:0), %arg14: i32 {tt.divisibility = 16 : i32} loc("/workspace/pa_decode_rework.py":157:0), %arg15: i32 {tt.divisibility = 16 : i32} loc("/workspace/pa_decode_rework.py":157:0), %arg16: i32 {tt.divisibility = 16 : i32} loc("/workspace/pa_decode_rework.py":157:0)) attributes {noinline = false} { + tt.func public @_paged_attn_decode_v1_w_dot_kernel_tt_load_only(%ptr1: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32}, %ptr2: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32}, %ptr3: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32}, %ptr4: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32}, %ptr5: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32}, %ptr6: !tt.ptr {tt.divisibility = 16 : i32}, %ptr7: !tt.ptr {tt.divisibility = 16 : i32}, %ptr8: !tt.ptr {tt.divisibility = 16 : i32}, %ptr9: !tt.ptr {tt.divisibility = 16 : i32}, %ptr10: !tt.ptr {tt.divisibility = 16 : i32}, %ptr11: !tt.ptr {tt.divisibility = 16 : i32}, %ptr12: !tt.ptr {tt.divisibility = 16 : i32}, %ptr13: !tt.ptr {tt.divisibility = 16 : i32}, %ptr14: !tt.ptr {tt.divisibility = 16 : i32}, %ptr15: !tt.ptr {tt.divisibility = 16 : i32}, %ptr16: !tt.ptr {tt.divisibility = 16 : i32}, %ptr0: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32}) { + + %0 = tt.load %ptr6 : !tt.ptr + %1 = tt.load %ptr7 : !tt.ptr + %2 = tt.load %ptr8 : !tt.ptr + %3 = tt.load %ptr9 : !tt.ptr + %4 = tt.load %ptr10 : !tt.ptr + %5 = tt.load %ptr11 : !tt.ptr + %6 = tt.load %ptr12 : !tt.ptr + %7 = tt.load %ptr13 : !tt.ptr + %8 = tt.load %ptr14 : !tt.ptr + %9 = tt.load %ptr15 : !tt.ptr + %10 = tt.load %ptr16 : !tt.ptr + tt.call @_paged_attn_decode_v1_w_dot_kernel(%ptr0, %ptr1, %ptr2, %ptr3, %ptr4, %ptr5, %0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10) : (!tt.ptr, !tt.ptr, !tt.ptr, !tt.ptr, !tt.ptr, !tt.ptr, f32, f32, f32, i32, i32, i32, i32, i32, i32, i32, i32) -> () + tt.return + } + + tt.func private @_paged_attn_decode_v1_w_dot_kernel(%arg0: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32} loc("/workspace/pa_decode_rework.py":159:0), %arg1: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32} loc("/workspace/pa_decode_rework.py":159:0), %arg2: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32} loc("/workspace/pa_decode_rework.py":159:0), %arg3: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32} loc("/workspace/pa_decode_rework.py":159:0), %arg4: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32} loc("/workspace/pa_decode_rework.py":159:0), %arg5: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32} loc("/workspace/pa_decode_rework.py":159:0), %arg6: f32 loc("/workspace/pa_decode_rework.py":159:0), %arg7: f32 loc("/workspace/pa_decode_rework.py":159:0), %arg8: f32 loc("/workspace/pa_decode_rework.py":159:0), %arg9: i32 {tt.divisibility = 16 : i32} loc("/workspace/pa_decode_rework.py":159:0), %arg10: i32 {tt.divisibility = 16 : i32} loc("/workspace/pa_decode_rework.py":159:0), %arg11: i32 {tt.divisibility = 16 : i32} loc("/workspace/pa_decode_rework.py":159:0), %arg12: i32 {tt.divisibility = 16 : i32} loc("/workspace/pa_decode_rework.py":159:0), %arg13: i32 {tt.divisibility = 16 : i32} loc("/workspace/pa_decode_rework.py":159:0), %arg14: i32 {tt.divisibility = 16 : i32} loc("/workspace/pa_decode_rework.py":159:0), %arg15: i32 {tt.divisibility = 16 : i32} loc("/workspace/pa_decode_rework.py":159:0), %arg16: i32 {tt.divisibility = 16 : i32} loc("/workspace/pa_decode_rework.py":159:0)) attributes {noinline = false} { %cst = arith.constant dense<0xFF800000> : tensor<16xf32> loc(#loc1) %cst_0 = arith.constant dense<0.000000e+00> : tensor<16xf32> loc(#loc1) %c15_i64 = arith.constant 15 : i64 loc(#loc1) @@ -143,88 +160,88 @@ module { tt.return loc(#loc85) } loc(#loc) } loc(#loc) -#loc2 = loc("/workspace/pa_decode_rework.py":192:28) -#loc3 = loc("/workspace/pa_decode_rework.py":193:32) -#loc4 = loc("/workspace/pa_decode_rework.py":197:37) -#loc5 = loc("/workspace/pa_decode_rework.py":197:22) +#loc2 = loc("/workspace/pa_decode_rework.py":194:28) +#loc3 = loc("/workspace/pa_decode_rework.py":195:32) +#loc4 = loc("/workspace/pa_decode_rework.py":199:37) +#loc5 = loc("/workspace/pa_decode_rework.py":199:22) #loc6 = loc("/usr/local/lib/python3.12/dist-packages/triton/language/standard.py":40:22) -#loc7 = loc("/workspace/pa_decode_rework.py":199:35) +#loc7 = loc("/workspace/pa_decode_rework.py":201:35) #loc8 = loc("/usr/local/lib/python3.12/dist-packages/triton/language/standard.py":40:28) -#loc9 = loc("/workspace/pa_decode_rework.py":201:28) -#loc10 = loc("/workspace/pa_decode_rework.py":202:32) -#loc11 = loc("/workspace/pa_decode_rework.py":216:18) -#loc12 = loc("/workspace/pa_decode_rework.py":217:25) -#loc13 = loc("/workspace/pa_decode_rework.py":217:51) -#loc14 = loc("/workspace/pa_decode_rework.py":217:40) -#loc15 = loc("/workspace/pa_decode_rework.py":217:63) -#loc16 = loc("/workspace/pa_decode_rework.py":217:10) -#loc17 = loc("/workspace/pa_decode_rework.py":218:23) -#loc18 = loc("/workspace/pa_decode_rework.py":218:10) -#loc19 = loc("/workspace/pa_decode_rework.py":222:36) -#loc20 = loc("/workspace/pa_decode_rework.py":222:77) -#loc21 = loc("/workspace/pa_decode_rework.py":222:53) -#loc22 = loc("/workspace/pa_decode_rework.py":224:24) -#loc23 = loc("/workspace/pa_decode_rework.py":224:16) -#loc24 = loc("/workspace/pa_decode_rework.py":225:13) -#loc25 = loc("/workspace/pa_decode_rework.py":225:23) -#loc26 = loc("/workspace/pa_decode_rework.py":232:22) -#loc27 = loc("/workspace/pa_decode_rework.py":233:30) -#loc28 = loc("/workspace/pa_decode_rework.py":233:10) -#loc29 = loc("/workspace/pa_decode_rework.py":234:10) -#loc30 = loc("/workspace/pa_decode_rework.py":236:51) -#loc31 = loc("/workspace/pa_decode_rework.py":236:41) -#loc32 = loc("/workspace/pa_decode_rework.py":241:39) -#loc33 = loc("/workspace/pa_decode_rework.py":243:37) -#loc34 = loc("/workspace/pa_decode_rework.py":244:35) -#loc35 = loc("/workspace/pa_decode_rework.py":249:36) -#loc36 = loc("/workspace/pa_decode_rework.py":256:76) -#loc37 = loc("/workspace/pa_decode_rework.py":256:52) -#loc38 = loc("/workspace/pa_decode_rework.py":279:36) -#loc39 = loc("/workspace/pa_decode_rework.py":238:19) -#loc40 = loc("/workspace/pa_decode_rework.py":239:50) -#loc41 = loc("/workspace/pa_decode_rework.py":239:30) -#loc42 = loc("/workspace/pa_decode_rework.py":240:36) -#loc43 = loc("/workspace/pa_decode_rework.py":240:49) -#loc44 = loc("/workspace/pa_decode_rework.py":241:27) -#loc45 = loc("/workspace/pa_decode_rework.py":243:26) -#loc46 = loc("/workspace/pa_decode_rework.py":244:15) -#loc47 = loc("/workspace/pa_decode_rework.py":245:15) -#loc48 = loc("/workspace/pa_decode_rework.py":249:22) -#loc49 = loc("/workspace/pa_decode_rework.py":254:23) -#loc50 = loc("/workspace/pa_decode_rework.py":256:65) -#loc51 = loc("/workspace/pa_decode_rework.py":258:12) -#loc52 = loc("/workspace/pa_decode_rework.py":269:12) +#loc9 = loc("/workspace/pa_decode_rework.py":203:28) +#loc10 = loc("/workspace/pa_decode_rework.py":204:32) +#loc11 = loc("/workspace/pa_decode_rework.py":218:18) +#loc12 = loc("/workspace/pa_decode_rework.py":219:25) +#loc13 = loc("/workspace/pa_decode_rework.py":219:51) +#loc14 = loc("/workspace/pa_decode_rework.py":219:40) +#loc15 = loc("/workspace/pa_decode_rework.py":219:63) +#loc16 = loc("/workspace/pa_decode_rework.py":219:10) +#loc17 = loc("/workspace/pa_decode_rework.py":220:23) +#loc18 = loc("/workspace/pa_decode_rework.py":220:10) +#loc19 = loc("/workspace/pa_decode_rework.py":224:36) +#loc20 = loc("/workspace/pa_decode_rework.py":224:77) +#loc21 = loc("/workspace/pa_decode_rework.py":224:53) +#loc22 = loc("/workspace/pa_decode_rework.py":226:24) +#loc23 = loc("/workspace/pa_decode_rework.py":226:16) +#loc24 = loc("/workspace/pa_decode_rework.py":227:13) +#loc25 = loc("/workspace/pa_decode_rework.py":227:23) +#loc26 = loc("/workspace/pa_decode_rework.py":234:22) +#loc27 = loc("/workspace/pa_decode_rework.py":235:30) +#loc28 = loc("/workspace/pa_decode_rework.py":235:10) +#loc29 = loc("/workspace/pa_decode_rework.py":236:10) +#loc30 = loc("/workspace/pa_decode_rework.py":238:51) +#loc31 = loc("/workspace/pa_decode_rework.py":238:41) +#loc32 = loc("/workspace/pa_decode_rework.py":243:39) +#loc33 = loc("/workspace/pa_decode_rework.py":245:37) +#loc34 = loc("/workspace/pa_decode_rework.py":246:35) +#loc35 = loc("/workspace/pa_decode_rework.py":251:36) +#loc36 = loc("/workspace/pa_decode_rework.py":258:76) +#loc37 = loc("/workspace/pa_decode_rework.py":258:52) +#loc38 = loc("/workspace/pa_decode_rework.py":281:36) +#loc39 = loc("/workspace/pa_decode_rework.py":240:19) +#loc40 = loc("/workspace/pa_decode_rework.py":241:50) +#loc41 = loc("/workspace/pa_decode_rework.py":241:30) +#loc42 = loc("/workspace/pa_decode_rework.py":242:36) +#loc43 = loc("/workspace/pa_decode_rework.py":242:49) +#loc44 = loc("/workspace/pa_decode_rework.py":243:27) +#loc45 = loc("/workspace/pa_decode_rework.py":245:26) +#loc46 = loc("/workspace/pa_decode_rework.py":246:15) +#loc47 = loc("/workspace/pa_decode_rework.py":247:15) +#loc48 = loc("/workspace/pa_decode_rework.py":251:22) +#loc49 = loc("/workspace/pa_decode_rework.py":256:23) +#loc50 = loc("/workspace/pa_decode_rework.py":258:65) +#loc51 = loc("/workspace/pa_decode_rework.py":260:12) +#loc52 = loc("/workspace/pa_decode_rework.py":271:12) #loc53 = loc("/usr/local/lib/python3.12/dist-packages/triton/language/standard.py":184:40) #loc55 = loc("/usr/local/lib/python3.12/dist-packages/triton/language/standard.py":163:27) -#loc56 = loc("/workspace/pa_decode_rework.py":271:55) -#loc57 = loc("/workspace/pa_decode_rework.py":274:45) -#loc58 = loc("/workspace/pa_decode_rework.py":274:31) -#loc59 = loc("/workspace/pa_decode_rework.py":274:57) -#loc60 = loc("/workspace/pa_decode_rework.py":274:25) -#loc61 = loc("/workspace/pa_decode_rework.py":275:42) -#loc62 = loc("/workspace/pa_decode_rework.py":275:59) -#loc63 = loc("/workspace/pa_decode_rework.py":275:29) -#loc64 = loc("/workspace/pa_decode_rework.py":276:21) -#loc65 = loc("/workspace/pa_decode_rework.py":276:15) -#loc66 = loc("/workspace/pa_decode_rework.py":279:22) -#loc67 = loc("/workspace/pa_decode_rework.py":283:17) -#loc68 = loc("/workspace/pa_decode_rework.py":284:25) -#loc69 = loc("/workspace/pa_decode_rework.py":286:28) +#loc56 = loc("/workspace/pa_decode_rework.py":273:55) +#loc57 = loc("/workspace/pa_decode_rework.py":276:45) +#loc58 = loc("/workspace/pa_decode_rework.py":276:31) +#loc59 = loc("/workspace/pa_decode_rework.py":276:57) +#loc60 = loc("/workspace/pa_decode_rework.py":276:25) +#loc61 = loc("/workspace/pa_decode_rework.py":277:42) +#loc62 = loc("/workspace/pa_decode_rework.py":277:59) +#loc63 = loc("/workspace/pa_decode_rework.py":277:29) +#loc64 = loc("/workspace/pa_decode_rework.py":278:21) +#loc65 = loc("/workspace/pa_decode_rework.py":278:15) +#loc66 = loc("/workspace/pa_decode_rework.py":281:22) +#loc67 = loc("/workspace/pa_decode_rework.py":285:17) +#loc68 = loc("/workspace/pa_decode_rework.py":286:25) +#loc69 = loc("/workspace/pa_decode_rework.py":288:28) #loc70 = loc("/usr/local/lib/python3.12/dist-packages/triton/language/standard.py":266:46) #loc72 = loc("/usr/local/lib/python3.12/dist-packages/triton/language/standard.py":267:36) #loc73 = loc("/usr/local/lib/python3.12/dist-packages/triton/language/standard.py":256:15) -#loc74 = loc("/workspace/pa_decode_rework.py":286:36) -#loc75 = loc("/workspace/pa_decode_rework.py":287:8) -#loc76 = loc("/workspace/pa_decode_rework.py":289:24) -#loc77 = loc("/workspace/pa_decode_rework.py":289:16) -#loc78 = loc("/workspace/pa_decode_rework.py":292:18) -#loc79 = loc("/workspace/pa_decode_rework.py":293:63) -#loc80 = loc("/workspace/pa_decode_rework.py":293:10) -#loc81 = loc("/workspace/pa_decode_rework.py":294:10) -#loc82 = loc("/workspace/pa_decode_rework.py":298:23) -#loc83 = loc("/workspace/pa_decode_rework.py":298:40) -#loc84 = loc("/workspace/pa_decode_rework.py":298:33) -#loc85 = loc("/workspace/pa_decode_rework.py":298:4) +#loc74 = loc("/workspace/pa_decode_rework.py":288:36) +#loc75 = loc("/workspace/pa_decode_rework.py":289:8) +#loc76 = loc("/workspace/pa_decode_rework.py":291:24) +#loc77 = loc("/workspace/pa_decode_rework.py":291:16) +#loc78 = loc("/workspace/pa_decode_rework.py":294:18) +#loc79 = loc("/workspace/pa_decode_rework.py":295:63) +#loc80 = loc("/workspace/pa_decode_rework.py":295:10) +#loc81 = loc("/workspace/pa_decode_rework.py":296:10) +#loc82 = loc("/workspace/pa_decode_rework.py":300:23) +#loc83 = loc("/workspace/pa_decode_rework.py":300:40) +#loc84 = loc("/workspace/pa_decode_rework.py":300:33) +#loc85 = loc("/workspace/pa_decode_rework.py":300:4) #loc86 = loc(callsite(#loc6 at #loc7)) #loc87 = loc(callsite(#loc8 at #loc7)) #loc88 = loc(callsite(#loc53 at #loc54)) diff --git a/examples/simple_layer/pour.mlir b/examples/simple_layer/pour.mlir new file mode 100644 index 00000000..3101ed9f --- /dev/null +++ b/examples/simple_layer/pour.mlir @@ -0,0 +1,10 @@ + %0 = tt.load %arg6 : !tt.ptr loc(#loc2) + %1 = tt.load %arg9 : !tt.ptr loc(#loc3) + %2 = tt.load %arg10 : !tt.ptr loc(#loc4) + %3 = tt.load %arg11 : !tt.ptr loc(#loc5) + %4 = tt.load %arg12 : !tt.ptr loc(#loc6) + %5 = tt.load %arg13 : !tt.ptr loc(#loc7) + %6 = tt.load %arg14 : !tt.ptr loc(#loc8) + %7 = tt.load %arg15 : !tt.ptr loc(#loc9) + %8 = tt.load %arg16 : !tt.ptr loc(#loc10) + %9 = tt.load %arg17 : !tt.ptr loc(#loc11) From 84b66c213d5e77c286a91396e12dd0f72bb55f7a Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Mon, 10 Mar 2025 11:30:49 +0000 Subject: [PATCH 3/7] WORKS --- examples/simple_layer/pa.zig | 67 +++++++++++----------------- examples/simple_layer/pa_kernel.mlir | 10 +++-- 2 files changed, 31 insertions(+), 46 deletions(-) diff --git a/examples/simple_layer/pa.zig b/examples/simple_layer/pa.zig index b5e3f803..23f553e1 100644 --- a/examples/simple_layer/pa.zig +++ b/examples/simple_layer/pa.zig @@ -56,28 +56,30 @@ pub fn forward( block_tables, seq_lens, // zml.Tensor.empty(.{8 * 4}, .i8), // check actually it's none + zml.Tensor.scalar(0.08838834765, .f32), - // zml.Tensor.scalar(1.0, .f32), - // zml.Tensor.scalar(1.0, .f32), - zml.Tensor.scalar(strides_o.get(0), .i32), - zml.Tensor.scalar(strides_o.get(1), .i32), - // zml.Tensor.scalar(strides_o.get(2), .i32), - zml.Tensor.scalar(strides_q.get(0), .i32), - zml.Tensor.scalar(strides_q.get(1), .i32), - zml.Tensor.scalar(strides_q.get(2), .i32), - zml.Tensor.scalar(strides_k.get(0), .i32), - zml.Tensor.scalar(strides_k.get(1), .i32), - zml.Tensor.scalar(strides_k.get(2), .i32), - zml.Tensor.scalar(strides_k.get(3), .i32), - zml.Tensor.scalar(strides_bt.get(0), .i32), + zml.Tensor.scalar(1.0, .f32), + zml.Tensor.scalar(1.0, .f32), + + zml.Tensor.scalar(@divExact(strides_o.get(0), 2), .i32), + zml.Tensor.scalar(@divExact(strides_o.get(1), 2), .i32), + // zml.Tensor.scalar(strides_o.get(2), .i32), stride_o_hs + zml.Tensor.scalar(@divExact(strides_q.get(0), 2), .i32), + zml.Tensor.scalar(@divExact(strides_q.get(1), 2), .i32), + // zml.Tensor.scalar(strides_q.get(2), .i32), + zml.Tensor.scalar(@divExact(strides_k.get(0), 2), .i32), + zml.Tensor.scalar(@divExact(strides_k.get(1), 2), .i32), + zml.Tensor.scalar(@divExact(strides_k.get(2), 2), .i32), + // zml.Tensor.scalar(strides_k.get(3), .i32), + zml.Tensor.scalar(@divExact(strides_bt.get(0), 4), .i32), // zml.Tensor.scalar(strides_bt.get(1), .i32), }, output_shape, .{ .name = "_paged_attn_decode_v1_w_dot_kernel_tt_load_only", .ir = @embedFile("pa_kernel.mlir"), .grid = .{ 256, 8, 1 }, - .num_stages = 2, + .num_stages = 1, .num_warps = 4, - .debug = false, + .debug = true, }); return y; @@ -123,39 +125,17 @@ pub fn asyncMain() !void { const seqlen_host = try allocator.alloc(i64, 256); defer allocator.free(seqlen_host); - @memset(seqlen_host, 1); + @memset(seqlen_host, 2048); const page_table_host = try allocator.alloc(i32, 256 * (max_seq_len / 16)); defer allocator.free(page_table_host); for (0..256) |i| { for (0..(max_seq_len / 16)) |j| { - page_table_host[i * (max_seq_len / 16) + j] = 0; + page_table_host[i * (max_seq_len / 16) + j] = @intCast(j); } } - { - var q_buffer = try createRandomBuffer(allocator, platform, query, random); - defer q_buffer.deinit(); - var k_buffer = try createRandomBuffer(allocator, platform, key_cache, random); - defer k_buffer.deinit(); - var v_buffer = try createRandomBuffer(allocator, platform, value_cache, random); - defer v_buffer.deinit(); - var bt_buffer = try zml.Buffer.fromBytes(platform, block_tables, std.mem.sliceAsBytes(page_table_host)); - defer bt_buffer.deinit(); - var sq_buffer = try zml.Buffer.fromBytes(platform, seq_lens, std.mem.sliceAsBytes(seqlen_host)); - defer sq_buffer.deinit(); - - // var profiler = platform.getProfiler(null); - // defer profiler.deinit(); - - // profiler.start(); - var result: zml.Buffer = compiled.call(.{ q_buffer, k_buffer, v_buffer, bt_buffer, sq_buffer }); - defer result.deinit(); - - var r = try result.toHostAlloc(allocator); - defer r.deinit(allocator); - // profiler.stop(); - // try profiler.dumpAsJsonTo(allocator, std.fs.cwd(), "profile_file.json"); - std.debug.print("Result: {}\n", .{r.shape()}); + for (seqlen_host, 0..) |v, i| { + std.debug.print("seq_lens[{}] = {}\n", .{ i, v }); } { @@ -177,8 +157,11 @@ pub fn asyncMain() !void { var result: zml.Buffer = compiled.call(.{ q_buffer, k_buffer, v_buffer, bt_buffer, sq_buffer }); defer result.deinit(); + var result2: zml.Buffer = compiled.call(.{ q_buffer, k_buffer, v_buffer, bt_buffer, sq_buffer }); + defer result2.deinit(); + var r = try result.toHostAlloc(allocator); - defer r.deinit(allocator); + // defer r.deinit(allocator); // profiler.stop(); // try profiler.dumpAsJsonTo(allocator, std.fs.cwd(), "profile_file.json"); std.debug.print("Result: {}\n", .{r.shape()}); diff --git a/examples/simple_layer/pa_kernel.mlir b/examples/simple_layer/pa_kernel.mlir index 9c7614f8..384ebdaf 100644 --- a/examples/simple_layer/pa_kernel.mlir +++ b/examples/simple_layer/pa_kernel.mlir @@ -5,10 +5,9 @@ #loc89 = loc(callsite(#loc1 at #loc54)) #loc93 = loc(callsite(#loc1 at #loc71)) module { - tt.func public @_paged_attn_decode_v1_w_dot_kernel_tt_load_only(%ptr1: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32}, %ptr2: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32}, %ptr3: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32}, %ptr4: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32}, %ptr5: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32}, %ptr6: !tt.ptr {tt.divisibility = 16 : i32}, %ptr7: !tt.ptr {tt.divisibility = 16 : i32}, %ptr8: !tt.ptr {tt.divisibility = 16 : i32}, %ptr9: !tt.ptr {tt.divisibility = 16 : i32}, %ptr10: !tt.ptr {tt.divisibility = 16 : i32}, %ptr11: !tt.ptr {tt.divisibility = 16 : i32}, %ptr12: !tt.ptr {tt.divisibility = 16 : i32}, %ptr13: !tt.ptr {tt.divisibility = 16 : i32}, %ptr14: !tt.ptr {tt.divisibility = 16 : i32}, %ptr15: !tt.ptr {tt.divisibility = 16 : i32}, %ptr16: !tt.ptr {tt.divisibility = 16 : i32}, %ptr0: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32}) { - + tt.func public @_paged_attn_decode_v1_w_dot_kernel_tt_load_only(%ptr1: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32}, %ptr2: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32}, %ptr3: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32}, %ptr4: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32}, %ptr5: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32}, %ptr6: !tt.ptr {tt.divisibility = 16 : i32}, %ptr7: !tt.ptr {tt.divisibility = 16 : i32}, %ptr8: !tt.ptr {tt.divisibility = 16 : i32}, %ptr9: !tt.ptr {tt.divisibility = 16 : i32}, %ptr10: !tt.ptr {tt.divisibility = 16 : i32}, %ptr11: !tt.ptr {tt.divisibility = 16 : i32}, %ptr12: !tt.ptr {tt.divisibility = 16 : i32}, %ptr13: !tt.ptr {tt.divisibility = 16 : i32}, %ptr14: !tt.ptr {tt.divisibility = 16 : i32}, %ptr15: !tt.ptr {tt.divisibility = 16 : i32}, %ptr16: !tt.ptr {tt.divisibility = 16 : i32}, %ptr0: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32}) { %0 = tt.load %ptr6 : !tt.ptr - %1 = tt.load %ptr7 : !tt.ptr + %1 = tt.load %ptr7 : !tt.ptr %2 = tt.load %ptr8 : !tt.ptr %3 = tt.load %ptr9 : !tt.ptr %4 = tt.load %ptr10 : !tt.ptr @@ -18,11 +17,14 @@ module { %8 = tt.load %ptr14 : !tt.ptr %9 = tt.load %ptr15 : !tt.ptr %10 = tt.load %ptr16 : !tt.ptr + // tt.print "stride 0 " {hex = false, isSigned = array} : %3, %4, %5 : i32, i32, i32 + // tt.print "stride 2" {hex = false, isSigned = array} : %4 : i32 + // tt.print "stride 2" {hex = false, isSigned = array} : %5 : i32 tt.call @_paged_attn_decode_v1_w_dot_kernel(%ptr0, %ptr1, %ptr2, %ptr3, %ptr4, %ptr5, %0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10) : (!tt.ptr, !tt.ptr, !tt.ptr, !tt.ptr, !tt.ptr, !tt.ptr, f32, f32, f32, i32, i32, i32, i32, i32, i32, i32, i32) -> () tt.return } - tt.func private @_paged_attn_decode_v1_w_dot_kernel(%arg0: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32} loc("/workspace/pa_decode_rework.py":159:0), %arg1: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32} loc("/workspace/pa_decode_rework.py":159:0), %arg2: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32} loc("/workspace/pa_decode_rework.py":159:0), %arg3: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32} loc("/workspace/pa_decode_rework.py":159:0), %arg4: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32} loc("/workspace/pa_decode_rework.py":159:0), %arg5: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32} loc("/workspace/pa_decode_rework.py":159:0), %arg6: f32 loc("/workspace/pa_decode_rework.py":159:0), %arg7: f32 loc("/workspace/pa_decode_rework.py":159:0), %arg8: f32 loc("/workspace/pa_decode_rework.py":159:0), %arg9: i32 {tt.divisibility = 16 : i32} loc("/workspace/pa_decode_rework.py":159:0), %arg10: i32 {tt.divisibility = 16 : i32} loc("/workspace/pa_decode_rework.py":159:0), %arg11: i32 {tt.divisibility = 16 : i32} loc("/workspace/pa_decode_rework.py":159:0), %arg12: i32 {tt.divisibility = 16 : i32} loc("/workspace/pa_decode_rework.py":159:0), %arg13: i32 {tt.divisibility = 16 : i32} loc("/workspace/pa_decode_rework.py":159:0), %arg14: i32 {tt.divisibility = 16 : i32} loc("/workspace/pa_decode_rework.py":159:0), %arg15: i32 {tt.divisibility = 16 : i32} loc("/workspace/pa_decode_rework.py":159:0), %arg16: i32 {tt.divisibility = 16 : i32} loc("/workspace/pa_decode_rework.py":159:0)) attributes {noinline = false} { + tt.func private @_paged_attn_decode_v1_w_dot_kernel(%arg0: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32}, %arg1: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32}, %arg2: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32}, %arg3: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32}, %arg4: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32}, %arg5: !tt.ptr {tt.divisibility = 16 : i32, tt.pointer_range = 32 : i32}, %arg6: f32, %arg7: f32, %arg8: f32, %arg9: i32 {tt.divisibility = 16 : i32}, %arg10: i32 {tt.divisibility = 16 : i32}, %arg11: i32 {tt.divisibility = 16 : i32}, %arg12: i32 {tt.divisibility = 16 : i32}, %arg13: i32 {tt.divisibility = 16 : i32}, %arg14: i32 {tt.divisibility = 16 : i32}, %arg15: i32 {tt.divisibility = 16 : i32}, %arg16: i32 {tt.divisibility = 16 : i32}) attributes {noinline = false} { %cst = arith.constant dense<0xFF800000> : tensor<16xf32> loc(#loc1) %cst_0 = arith.constant dense<0.000000e+00> : tensor<16xf32> loc(#loc1) %c15_i64 = arith.constant 15 : i64 loc(#loc1) From d38c9d780648c43c62f3ae5b9884d72077e6fd49 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Wed, 12 Mar 2025 15:25:29 +0100 Subject: [PATCH 4/7] validate buffer shapes in call --- zml/exe.zig | 73 +++++++++++++++++++++++++++----------------------- zml/module.zig | 7 +++-- 2 files changed, 44 insertions(+), 36 deletions(-) diff --git a/zml/exe.zig b/zml/exe.zig index 946410c6..d356e885 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -135,6 +135,10 @@ pub const BaseExe = struct { /// The PJRT executable representing the compiled module. exe: *pjrt.LoadedExecutable, + name: []const u8, + + input_shapes: []const Shape, + /// Pre-allocated slice of buffers to use as inputs when the module is called. input_per_device: []const [*]*pjrt.Buffer, @@ -155,22 +159,23 @@ pub const BaseExe = struct { /// Allocator backing memory _arena: std.heap.ArenaAllocator, - pub fn init(parent_allocator: std.mem.Allocator, platform: Platform, exe: *pjrt.LoadedExecutable, args: struct { n_in: u32, result_shapes: []const Shape, n_devices: u8 }) !BaseExe { + pub fn init(parent_allocator: std.mem.Allocator, platform: Platform, exe: *pjrt.LoadedExecutable, args: struct { name: []const u8, input_shapes: []const Shape, result_shapes: []const Shape, n_devices: u8 }) !BaseExe { var arena = std.heap.ArenaAllocator.init(parent_allocator); errdefer arena.deinit(); const allocator = arena.allocator(); + const n_in = args.input_shapes.len; const n_out = args.result_shapes.len; const n_devices = args.n_devices; // Allocate once for all the *pjrt.Buffer we need to store ... - const all_buffers = try allocator.alloc(*pjrt.Buffer, (args.n_in + n_out) * n_devices); - const all_input_buffers, const all_output_buffers = splitBuffer(*pjrt.Buffer, all_buffers, .{ args.n_in * n_devices, n_out * n_devices }); + const all_buffers = try allocator.alloc(*pjrt.Buffer, (n_in + n_out) * n_devices); + const all_input_buffers, const all_output_buffers = splitBuffer(*pjrt.Buffer, all_buffers, .{ n_in * n_devices, n_out * n_devices }); // ... and once for all the [*]*pjrt.Buffer. const all_per_device = try allocator.alloc([*]*pjrt.Buffer, 2 * n_devices); const input_per_device, const output_per_device = splitBuffer([*]*pjrt.Buffer, all_per_device, .{ n_devices, n_devices }); for (0..n_devices) |i| { - input_per_device[i] = all_input_buffers[i * args.n_in ..].ptr; + input_per_device[i] = all_input_buffers[i * n_in ..].ptr; output_per_device[i] = all_output_buffers[i * n_out ..].ptr; } @@ -178,7 +183,9 @@ pub const BaseExe = struct { .platform = platform, .exe = exe, .ready_buffer_count = 0, - .input_buffer_count = args.n_in, + .name = try allocator.dupe(u8, args.name), + .input_shapes = try allocator.dupe(Shape, args.input_shapes), + .input_buffer_count = @intCast(args.input_shapes.len), .num_devices = args.n_devices, .input_per_device = input_per_device, .output_per_device = output_per_device, @@ -231,9 +238,8 @@ pub const BaseExe = struct { // return platform.pjrt_client.deserializeAndLoad(platform.pjrt_api, bytes); // } - pub fn prepare(self: *BaseExe, x: anytype) void { - const n = fillBuffers(&x, self.input_per_device, self.ready_buffer_count); - self.ready_buffer_count += n; + pub fn prepare(self: *BaseExe, buffers_struct_ptr: anytype) void { + self.ready_buffer_count = self.fillInputBuffers(buffers_struct_ptr); } pub fn getOutputBuffer(self: BaseExe, i: usize) Buffer { @@ -244,6 +250,29 @@ pub const BaseExe = struct { return Buffer.fromPjrtBuffers(self.platform, self.result_shapes[i], shards.constSlice()); } + + /// Visit the given struct and fill the `buffers` slice with the buffer associated with encountered Tensor. + fn fillInputBuffers(self: *const BaseExe, buffers_struct_ptr: anytype) u32 { + const LocalContext = struct { index: u32, exe: *const BaseExe }; + var context: LocalContext = .{ + .index = self.ready_buffer_count, + .exe = self, + }; + meta.visit((struct { + fn cb(ctx: *LocalContext, buffer: *const Buffer) void { + const i = ctx.index; + const exe = ctx.exe; + // stdx.debug.assert(!buffer._data.isDeleted(), "Can't use {} (argument buffer {}) because its pjrt buffer has been donated", .{ buffer, ctx.index }); + stdx.debug.assert(buffer.shape().eql(exe.input_shapes[i]), "Executable {s} expected a {} buffer for argument {}, got: {}", .{ exe.name, exe.input_shapes[i], i, buffer.shape() }); + stdx.debug.assert(buffer._shards.len == exe.num_devices, "Can't feed a {}-sharded buffer into a {}-sharded model", .{ buffer._shards.len, exe.num_devices }); + for (buffer._shards.constSlice(), 0..exe.num_devices) |shard, d| { + exe.input_per_device[d][i] = shard; + } + ctx.index += 1; + } + }).cb, &context, buffers_struct_ptr); + return context.index; + } }; /// Represents a ZML function, compiled into a PJRT executable. @@ -267,7 +296,7 @@ pub fn Exe(ArgsT: type, ReturnT: type) type { /// The caller is responsible to come up with a strategy to call `deinit` exactly once. pub fn prepare(self: Self, first_arg: Bufferized(stdx.meta.Head(ArgsT))) Exe(stdx.meta.Tail(ArgsT), ReturnT) { var new: Exe(stdx.meta.Tail(ArgsT), ReturnT) = .{ .inner = self.inner }; - new.inner.prepare(first_arg); + new.inner.prepare(&first_arg); return new; } @@ -280,7 +309,7 @@ pub fn Exe(ArgsT: type, ReturnT: type) type { } pub fn call(self: Self, args: Bufferized(ArgsT)) Bufferized(ReturnT) { - const total_ready = fillBuffers(&args, self.inner.input_per_device, self.inner.ready_buffer_count); + const total_ready = self.inner.fillInputBuffers(&args); std.debug.assert(total_ready == self.inner.input_buffer_count); self.inner._unsafeCall(); var result: Bufferized(ReturnT) = undefined; @@ -301,30 +330,6 @@ fn splitBuffer(T: type, buffer: []T, lengths: anytype) [lengths.len][]T { return res; } -/// Visit the given struct and fill the `buffers` slice with the buffer associated with encountered Tensor. -fn fillBuffers(v: anytype, buffers: []const [*]*pjrt.Buffer, start: u32) u32 { - const LocalContext = struct { - index: u32, - buffers: []const [*]*pjrt.Buffer, - }; - var context: LocalContext = .{ - .index = start, - .buffers = buffers, - }; - meta.visit((struct { - fn cb(ctx: *LocalContext, buffer: *const Buffer) void { - // stdx.debug.assert(!buffer._data.isDeleted(), "Can't use {} (argument buffer {}) because its pjrt buffer has been donated", .{ buffer, ctx.index }); - const model_sharding = ctx.buffers.len; - stdx.debug.assert(buffer._shards.len == model_sharding, "Can't feed a {}-sharded tensor into a {}-sharded model", .{ buffer._shards.len, ctx.buffers.len }); - for (buffer._shards.constSlice(), 0..) |shard, d| { - ctx.buffers[d][ctx.index] = shard; - } - ctx.index += 1; - } - }).cb, &context, v); - return context.index; -} - /// Visit the given struct and override tensors by creating a new one using the provided PJRT buffers. fn assignRawBuffers(v: anytype, platform: Platform, buffers: []const [*]*pjrt.Buffer, buffer_shapes: []Shape) void { const LocalContext = struct { diff --git a/zml/module.zig b/zml/module.zig index 58f30cbc..25e74479 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -71,6 +71,7 @@ const Block = union(BlockKind) { pub const MlirFn = struct { name: []const u8, num_args: u32, + input_shapes: []Shape, res_tensors: *const anyopaque, res_types: []mlir.Type, res_shapes: []Shape, @@ -273,7 +274,8 @@ pub const CompilationContext = struct { self._platform, loaded_executable, .{ - .n_in = f.num_args, + .name = f.name, + .input_shapes = f.input_shapes, .result_shapes = f.res_shapes, .n_devices = sharding.num_replicas * sharding.num_partitions, }, @@ -380,7 +382,7 @@ pub const CompilationContext = struct { const locations = try arena.alloc(mlir.Location, tensor_count); @memset(locations, mlir.Location.unknown(mlir_ctx)); - var input_shapes = try std.ArrayList(Shape).initCapacity(arena, tensor_count); + var input_shapes = try std.ArrayList(Shape).initCapacity(res_allocator, tensor_count); meta.collect(Tensor.shape, {}, &input_shapes, args) catch unreachable; stdx.debug.internalAssert(input_shapes.items.len == tensor_count, "args have changed ?", .{}); @@ -465,6 +467,7 @@ pub const CompilationContext = struct { .mlir_fn = mlir_fn, .name = opts.name, .num_args = @intCast(tensor_count), + .input_shapes = input_shapes.items, .res_tensors = fn_res, .res_types = fn_res_types, .res_shapes = fn_res_shapes, From 1348a5bcff6e3cf32530d0961699ea291c202502 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Thu, 13 Mar 2025 14:28:29 +0000 Subject: [PATCH 5/7] zml: add operand layouts --- mlir/dialects/stablehlo.zig | 84 +++++++++++++++++++++++++++++-------- mlir/mlir.zig | 19 +++++++++ zml/nn/cuda.zig | 2 +- zml/ops.zig | 14 +++---- 4 files changed, 93 insertions(+), 26 deletions(-) diff --git a/mlir/dialects/stablehlo.zig b/mlir/dialects/stablehlo.zig index f6062c8f..ab393473 100644 --- a/mlir/dialects/stablehlo.zig +++ b/mlir/dialects/stablehlo.zig @@ -735,9 +735,12 @@ pub fn convolution( pub const CustomCallOpts = struct { call_target_name: [:0]const u8, has_side_effect: bool, - backend_config: [:0]const u8 = &.{}, + backend_config: union(enum) { + str: [:0]const u8, + dict: mlir.DictionaryAttribute, + }, api_version: i32, - output_operand_aliases: []const i64, + output_operand_aliases: []const i64 = &.{}, }; pub fn custom_call(ctx: mlir.Context, inputs: []const mlir.Value, opts: CustomCallOpts, res_types: []const mlir.Type, location: mlir.Location) mlir.Operation { @@ -757,41 +760,86 @@ pub fn custom_call(ctx: mlir.Context, inputs: []const mlir.Value, opts: CustomCa .{ "api_version", mlir.IntegerAttribute(.i32).init(ctx, opts.api_version).as(mlir.Attribute).? }, .{ "call_target_name", mlir.StringAttribute.init(ctx, opts.call_target_name).as(mlir.Attribute).? }, .{ "has_side_effect", mlir.BoolAttribute.init(ctx, opts.has_side_effect).as(mlir.Attribute).? }, - .{ "backend_config", mlir.StringAttribute.init(ctx, opts.backend_config).as(mlir.Attribute).? }, + .{ + "backend_config", switch (opts.backend_config) { + .str => |v| mlir.StringAttribute.init(ctx, v).as(mlir.Attribute).?, + .dict => |v| v.as(mlir.Attribute).?, + }, + }, .{ "output_operand_aliases", mlir.ArrayAttribute.init(ctx, output_operand_aliases).as(mlir.Attribute).? }, }, .location = location, }); } -pub const CustomCallOpts2 = struct { - call_target_name: [:0]const u8, - has_side_effect: bool, - backend_config: mlir.DictionaryAttribute, - // api_version: i32, - output_operand_aliases: []const i64, -}; +pub fn custom_call_alloc(allocator: std.mem.Allocator, ctx: mlir.Context, inputs: []const mlir.Value, opts: CustomCallOpts, res_types: []const mlir.Type, location: mlir.Location) mlir.Operation { + var arena_state = std.heap.ArenaAllocator.init(allocator); + defer arena_state.deinit(); -pub fn custom_call2(ctx: mlir.Context, inputs: []const mlir.Value, opts: CustomCallOpts2, res_types: []const mlir.Type, location: mlir.Location) mlir.Operation { - var buffer: [1024]u8 = undefined; - var fba = std.heap.FixedBufferAllocator.init(&buffer); - const allocator = fba.allocator(); + var arena = arena_state.allocator(); - const output_operand_aliases = allocator.alloc(mlir.Attribute, opts.output_operand_aliases.len) catch unreachable; + const output_operand_aliases = arena.alloc(mlir.Attribute, opts.output_operand_aliases.len) catch unreachable; for (opts.output_operand_aliases, 0..) |alias, i| { output_operand_aliases[i] = OutputOperandAliasAttribute.init(ctx, &.{}, alias, &.{}).as(mlir.Attribute).?; } + const indices_of_shape_operands = b: { + var indices = std.ArrayList(i64).initCapacity(arena, res_types.len) catch unreachable; + for (0..res_types.len) |indice| { + indices.appendAssumeCapacity(@intCast(indice)); + } + const indices_of_shape_type = mlir.RankedTensorType.init(&.{@intCast(res_types.len)}, mlir.IntegerType(.i32).init(ctx).as(mlir.Type).?).as(mlir.Type).?; + break :b mlir.DenseIntOrFPElementsAttribute(.i32).init(indices_of_shape_type, std.mem.sliceAsBytes(indices.items)); + }; + _ = indices_of_shape_operands; + + const operand_layouts = b: { + var layouts = std.ArrayList(mlir.Attribute).initCapacity(arena, inputs.len) catch unreachable; + const MINOR_TO_MAJOR = [8]i64{ 7, 6, 5, 4, 3, 2, 1, 0 }; + for (inputs) |input| { + const len = MINOR_TO_MAJOR.len; + const input_type = input.getType().as(mlir.ShapedType).?; + const layout = MINOR_TO_MAJOR[len - input_type.rank() ..]; + const layout_shape = mlir.RankedTensorType.init(&.{@intCast(layout.len)}, mlir.IndexType.init(ctx).as(mlir.Type).?).as(mlir.Type).?; + const layout_attribute = mlir.DenseIntOrFPElementsAttribute(.i64).fromRaw(layout_shape, std.mem.sliceAsBytes(layout)).as(mlir.Attribute).?; + layouts.appendAssumeCapacity(layout_attribute); + } + + const operand_layouts = mlir.ArrayAttribute.init(ctx, layouts.items).as(mlir.Attribute).?; + break :b operand_layouts; + }; + const result_layouts = b: { + var layouts = std.ArrayList(mlir.Attribute).initCapacity(arena, res_types.len) catch unreachable; + const MINOR_TO_MAJOR = [8]i64{ 7, 6, 5, 4, 3, 2, 1, 0 }; + for (res_types) |res_raw_type| { + const len = MINOR_TO_MAJOR.len; + const res_type = res_raw_type.as(mlir.ShapedType).?; + const layout = MINOR_TO_MAJOR[len - res_type.rank() ..]; + const layout_shape = mlir.RankedTensorType.init(&.{@intCast(layout.len)}, mlir.IndexType.init(ctx).as(mlir.Type).?).as(mlir.Type).?; + const layout_attribute = mlir.DenseIntOrFPElementsAttribute(.i64).fromRaw(layout_shape, std.mem.sliceAsBytes(layout)).as(mlir.Attribute).?; + layouts.appendAssumeCapacity(layout_attribute); + } + + const result_layouts = mlir.ArrayAttribute.init(ctx, layouts.items).as(mlir.Attribute).?; + break :b result_layouts; + }; + return mlir.Operation.make(ctx, "stablehlo.custom_call", .{ .operands = inputs, .results = res_types, .attributes = &.{ - // .{ "api_version", mlir.IntegerAttribute(.i32).init(ctx, opts.api_version).as(mlir.Attribute).? }, + .{ "api_version", mlir.IntegerAttribute(.i32).init(ctx, opts.api_version).as(mlir.Attribute).? }, .{ "call_target_name", mlir.StringAttribute.init(ctx, opts.call_target_name).as(mlir.Attribute).? }, .{ "has_side_effect", mlir.BoolAttribute.init(ctx, opts.has_side_effect).as(mlir.Attribute).? }, - .{ "backend_config", mlir.StringAttribute.init(ctx, "").as(mlir.Attribute).? }, + .{ + "backend_config", switch (opts.backend_config) { + .str => |v| mlir.StringAttribute.init(ctx, v).as(mlir.Attribute).?, + .dict => |v| v.as(mlir.Attribute).?, + }, + }, .{ "output_operand_aliases", mlir.ArrayAttribute.init(ctx, output_operand_aliases).as(mlir.Attribute).? }, - .{ "mhlo.backend_config", opts.backend_config.as(mlir.Attribute).? }, + .{ "operand_layouts", operand_layouts }, + .{ "result_layouts", result_layouts }, }, .location = location, }); diff --git a/mlir/mlir.zig b/mlir/mlir.zig index 0916ad76..4bb2eed3 100644 --- a/mlir/mlir.zig +++ b/mlir/mlir.zig @@ -649,6 +649,10 @@ pub fn DenseIntOrFPElementsAttribute(comptime dt: DenseElementsAttributeTypes) t return Attr.wrap(initFn(shaped_type.inner(), @intCast(values.len), @ptrCast(@alignCast(values.ptr)))); } + pub fn fromRaw(shaped_type: Type, raw: []const u8) Attr { + return Attr.wrap(c.mlirDenseElementsAttrRawBufferGet(shaped_type.inner(), raw.len, raw.ptr)); + } + pub fn get(self: Attr, pos: usize) ZigOutDataType { return getValue(self.inner(), @intCast(pos)); } @@ -1239,6 +1243,21 @@ pub const Type = struct { } }; +pub const IndexType = struct { + _inner: c.MlirType, + + pub usingnamespace MlirHelpers(IndexType, .{ + .is_null_fn = c.mlirTypeIsNull, + .dump_fn = c.mlirTypeDump, + .equal_fn = c.mlirTypeEqual, + .print_fn = c.mlirTypePrint, + }); + + pub fn init(ctx: Context) IndexType { + return IndexType.wrap(c.mlirIndexTypeGet(ctx.inner())); + } +}; + pub const IntegerTypes = enum { i1, i4, diff --git a/zml/nn/cuda.zig b/zml/nn/cuda.zig index b1960bbc..a119c0e3 100644 --- a/zml/nn/cuda.zig +++ b/zml/nn/cuda.zig @@ -125,7 +125,7 @@ pub fn sdpa(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) Tensor { &.{ q.value(), k.value(), v.value(), bias.value() }, .{ .call_target_name = "__cudnn$fmhaScaleBiasSoftmax", - .backend_config = backend_config, + .backend_config = .{ .str = backend_config }, .api_version = 2, .has_side_effect = false, .output_operand_aliases = &.{}, diff --git a/zml/ops.zig b/zml/ops.zig index 22e40b90..199e1418 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -1163,15 +1163,15 @@ inline fn toI64(values: anytype) []i64 { pub const TritonOps = struct { debug: bool = false, - name: []const u8, - ir: []const u8, + name: [:0]const u8, + ir: [:0]const u8, grid: [3]i32, num_stages: i32, num_warps: i32, }; /// Generate an MLIR call to the given member function with the given tensors. -pub fn triton(args: anytype, res_shape: anytype, opts: TritonOps) Tensor { +pub fn triton(args: anytype, res_shape: Shape, opts: TritonOps) Tensor { const ctx = CompilationContext.current(); var values: [args.len]mlir.Value = undefined; @@ -1187,15 +1187,15 @@ pub fn triton(args: anytype, res_shape: anytype, opts: TritonOps) Tensor { mlir.NamedAttribute.init(mlir.Identifier.get(ctx.mlirCtx(), "num_warps"), mlir.IntegerAttribute(.i32).init(ctx.mlirCtx(), @intCast(opts.num_warps)).as(mlir.Attribute).?), }); - const op = dialect.stablehlo.custom_call2( + const op = dialect.stablehlo.custom_call_alloc( + ctx.allocator(), ctx.mlirCtx(), &values, .{ .call_target_name = "__gpu$xla.gpu.triton", - .backend_config = attrs, - // .api_version = 4, + .backend_config = .{ .dict = attrs }, + .api_version = 4, .has_side_effect = false, - .output_operand_aliases = &.{}, }, &.{mlir.ext.mlirType(ctx.mlirCtx(), res_shape)}, ctx.mlirCtx().location(@src()), From 7fb12004a9075dc421872a9d29a126809d2d2036 Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Thu, 13 Mar 2025 23:43:17 +0100 Subject: [PATCH 6/7] fix addHostCallback --- zml/ops.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zml/ops.zig b/zml/ops.zig index 199e1418..1bbc3fd9 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -838,7 +838,7 @@ pub fn addHostCallback( .api_version = 1, .has_side_effect = false, .call_target_name = "zmlHostBufferCallback", - .backend_config = @ptrCast(std.mem.sliceAsBytes(&backend_config)), + .backend_config = .{ .str = @ptrCast(std.mem.sliceAsBytes(&backend_config)) }, .output_operand_aliases = &.{0}, }, &.{input.value().getType()}, From ee8ca4d71e1ab84a4ee0e22c43a096b07bbd5d7c Mon Sep 17 00:00:00 2001 From: Guillaume Wenzek Date: Mon, 17 Mar 2025 13:03:16 +0100 Subject: [PATCH 7/7] fix identityCustomCall --- zml/ops.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zml/ops.zig b/zml/ops.zig index 1bbc3fd9..e0a2eaa7 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -792,7 +792,7 @@ pub fn identityCustomCall(name: [:0]const u8, input: Tensor, context: *anyopaque .api_version = 1, .has_side_effect = false, .call_target_name = name, - .backend_config = backend_config[0..], + .backend_config = .{ .str = backend_config[0..] }, .output_operand_aliases = &.{0}, }, &.{input.value().getType()},