From 8950d5fac010e8859e7781522e8d615c8bc72f63 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 25 Feb 2025 17:49:25 +0000 Subject: [PATCH 1/7] zml/nn: sdpaPaged --- zml/nn.zig | 2 +- zml/nn/cuda.zig | 118 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 119 insertions(+), 1 deletion(-) diff --git a/zml/nn.zig b/zml/nn.zig index 07f4c9f7..3cc340df 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -2,7 +2,7 @@ const std = @import("std"); const stdx = @import("stdx"); -const cuda = @import("nn/cuda.zig"); +pub const cuda = @import("nn/cuda.zig"); const helpers = @import("helpers.zig"); const meta = @import("meta.zig"); const ops = @import("ops.zig"); diff --git a/zml/nn/cuda.zig b/zml/nn/cuda.zig index 81ff6000..c7cfe409 100644 --- a/zml/nn/cuda.zig +++ b/zml/nn/cuda.zig @@ -138,3 +138,121 @@ pub fn sdpa(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) Tensor { const result = Tensor._result(q.shape(), op.result(0)).transpose(q_.shape()); return result; } + +pub const SdpaPagedOpts = struct { + attn_mask: ?Tensor = null, + scale: ?Tensor = null, + max_seq_len_kv: usize, +}; + +pub fn sdpaPaged(q_: Tensor, k_: Tensor, v_: Tensor, sequence_length_q: Tensor, sequence_length_kv: Tensor, page_table: Tensor, opts: SdpaPagedOpts) Tensor { + const ctx = CompilationContext.current(); + const q = q_.transpose(.{ .b, .h, .q, .hd }); + var k = k_.transpose(.{ .page, .h, .k_chunk, .hd }); + const v = v_.transpose(.{ .page, .h, .k_chunk, .hd }); + + const sqrtHeadDim: f32 = 1.0 / std.math.sqrt(@as(f32, @floatFromInt(q.dim(.hd)))); + const scale: f32 = if (opts.scale) |_| 1.0 else sqrtHeadDim; + + if (opts.scale) |s| k = k.mul(s); + + std.debug.print("k.shape: {}\n", .{k.shape()}); + + var buffer: [4096]u8 = undefined; + var fba = std.heap.FixedBufferAllocator.init(&buffer); + const allocator = fba.allocator(); + + const backend_config = + std.fmt.allocPrintZ( + allocator, + \\{{ + \\ "operation_queue_id":"0", + \\ "wait_on_operation_queues":[], + \\ "cudnn_fmha_backend_config": {{ + \\ "algorithm": {{ + \\ "algo_id": "0", + \\ "math_type": "TENSOR_OP_MATH", + \\ "tuning_knobs": {{"17": "1", "24": "0"}}, + \\ "is_cudnn_frontend": true, + \\ "workspace_size": "0", + \\ }}, + \\ "fmha_scale": {d}, + \\ "dropout_rate": 0, + \\ "intermediate_tensor_shape": {{ + \\ "element_type": "{s}", + \\ "dimensions": ["{}", "{}", "{}", "{}"], + \\ "tuple_shapes": [], + \\ "layout": {{ + \\ "dim_level_types": [], + \\ "dim_unique": [], + \\ "dim_ordered": [], + \\ "minor_to_major": ["3", "2", "1", "0"], + \\ "tiles": [], + \\ "element_size_in_bits": "0", + \\ "memory_space": "0", + \\ "index_primitive_type": "PRIMITIVE_TYPE_INVALID", + \\ "pointer_primitive_type": "PRIMITIVE_TYPE_INVALID", + \\ "dynamic_shape_metadata_prefix_bytes": "0", + \\ }}, + \\ "is_dynamic_dimension": [false, false, false, false], + \\ }}, + \\ "seed": 0, + \\ "is_flash_attention": true, + \\ "mask_type": "NO_MASK", + \\ "bmm1_dot_dimension_numbers": {{ + \\ "lhs_contracting_dimensions": ["3"], + \\ "rhs_contracting_dimensions": ["3"], + \\ "lhs_batch_dimensions": ["0", "1"], + \\ "rhs_batch_dimensions": ["0", "1"], + \\ }}, + \\ "bmm2_dot_dimension_numbers": {{ + \\ "lhs_contracting_dimensions": ["3"], + \\ "rhs_contracting_dimensions": ["2"], + \\ "lhs_batch_dimensions": ["0", "1"], + \\ "rhs_batch_dimensions": ["0", "1"], + \\ }}, + \\ "max_sequence_length_kv": {}, + \\ }} + \\}} + , + .{ + scale, + elementTypeFromDataType(q.dtype()), + q.dim(.b), + q.dim(.h), + q.dim(.q), + k.dim(.k_chunk), + opts.max_seq_len_kv, + }, + ) catch unreachable; + + std.debug.print("backend_config: {s}\n", .{backend_config}); + + var bias = Tensor.constant(Shape.init(.{ .b = q.dim(.b), .h = q.dim(.h), .q = q.dim(.q), .k = opts.max_seq_len_kv }, q.dtype()), Data.init(q.dtype(), 0)); + + if (opts.attn_mask) |attn_mask| { + bias = bias.add(attn_mask.broad(bias.shape())); + } + + const mlir_ctx = ctx.mlirCtx(); + const loc = mlir_ctx.location(@src()); + const op = dialect.stablehlo.custom_call( + mlir_ctx, + &.{ q.value(), k.value(), v.value(), bias.value(), sequence_length_q.value(), sequence_length_kv.value(), page_table.value(), page_table.value() }, + .{ + .call_target_name = "__cudnn$fmhaScaleBiasSoftmax", + .backend_config = backend_config, + .api_version = 2, + .has_side_effect = false, + .output_operand_aliases = &.{}, + }, + &.{ + mlir.ext.mlirType(mlir_ctx, q.shape()), + mlir.RankedTensorType.init(&.{0}, mlir.IntegerType(.u8).init(mlir_ctx).as(mlir.Type).?).asType(), + }, + loc, + ); + const result = Tensor._result(q.shape(), op.result(0)).transpose(q_.shape()); + return result; +} + From 1819ae9e0be90516c66c374380d577b3748400c1 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Wed, 26 Feb 2025 16:24:24 +0000 Subject: [PATCH 2/7] zml/nn/cuda.zig: sdpa should scale correctly by default --- zml/nn/cuda.zig | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/zml/nn/cuda.zig b/zml/nn/cuda.zig index c7cfe409..4e67aab1 100644 --- a/zml/nn/cuda.zig +++ b/zml/nn/cuda.zig @@ -43,9 +43,8 @@ pub fn sdpa(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) Tensor { const v = v_.transpose(.{ .b, .h, .k, .hd }); const sqrtHeadDim: f32 = 1.0 / std.math.sqrt(@as(f32, @floatFromInt(q.dim(.hd)))); - const scale: f32 = if (opts.scale) |_| 1.0 else sqrtHeadDim; - - if (opts.scale) |s| k = k.mul(s); + const head_scaling = if (opts.scale) |s| s else Tensor.scalar(sqrtHeadDim, k.dtype()); + k = k.mul(head_scaling); var buffer: [4096]u8 = undefined; var fba = std.heap.FixedBufferAllocator.init(&buffer); @@ -64,7 +63,7 @@ pub fn sdpa(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) Tensor { \\ "is_cudnn_frontend": true, \\ "workspace_size": "0", \\ }}, - \\ "fmha_scale": {d}, + \\ "fmha_scale": 0, \\ "dropout_rate": 0, \\ "intermediate_tensor_shape": {{ \\ "element_type": "{s}", @@ -103,7 +102,6 @@ pub fn sdpa(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) Tensor { \\}} , .{ - scale, elementTypeFromDataType(q.dtype()), q.dim(.b), q.dim(.h), From faa729dc89b6d6ea39dcabac8790fe284eb8c0cf Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Wed, 26 Feb 2025 16:24:55 +0000 Subject: [PATCH 3/7] zml/nn/cuda.zig: fix scaling in sdpaPaged --- zml/nn/cuda.zig | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/zml/nn/cuda.zig b/zml/nn/cuda.zig index 4e67aab1..c26ef0aa 100644 --- a/zml/nn/cuda.zig +++ b/zml/nn/cuda.zig @@ -150,11 +150,8 @@ pub fn sdpaPaged(q_: Tensor, k_: Tensor, v_: Tensor, sequence_length_q: Tensor, const v = v_.transpose(.{ .page, .h, .k_chunk, .hd }); const sqrtHeadDim: f32 = 1.0 / std.math.sqrt(@as(f32, @floatFromInt(q.dim(.hd)))); - const scale: f32 = if (opts.scale) |_| 1.0 else sqrtHeadDim; - - if (opts.scale) |s| k = k.mul(s); - - std.debug.print("k.shape: {}\n", .{k.shape()}); + const head_scaling = if (opts.scale) |s| s else Tensor.scalar(sqrtHeadDim, k.dtype()); + k = k.mul(head_scaling); var buffer: [4096]u8 = undefined; var fba = std.heap.FixedBufferAllocator.init(&buffer); @@ -174,7 +171,7 @@ pub fn sdpaPaged(q_: Tensor, k_: Tensor, v_: Tensor, sequence_length_q: Tensor, \\ "is_cudnn_frontend": true, \\ "workspace_size": "0", \\ }}, - \\ "fmha_scale": {d}, + \\ "fmha_scale": 0, \\ "dropout_rate": 0, \\ "intermediate_tensor_shape": {{ \\ "element_type": "{s}", @@ -214,7 +211,6 @@ pub fn sdpaPaged(q_: Tensor, k_: Tensor, v_: Tensor, sequence_length_q: Tensor, \\}} , .{ - scale, elementTypeFromDataType(q.dtype()), q.dim(.b), q.dim(.h), @@ -232,6 +228,7 @@ pub fn sdpaPaged(q_: Tensor, k_: Tensor, v_: Tensor, sequence_length_q: Tensor, bias = bias.add(attn_mask.broad(bias.shape())); } + const output_shape = Shape.init(.{ .b = q.dim(.b), .h = q.dim(.h), .q = q.dim(.q), .hd = q.dim(.hd) }, q.dtype()); const mlir_ctx = ctx.mlirCtx(); const loc = mlir_ctx.location(@src()); const op = dialect.stablehlo.custom_call( @@ -245,12 +242,11 @@ pub fn sdpaPaged(q_: Tensor, k_: Tensor, v_: Tensor, sequence_length_q: Tensor, .output_operand_aliases = &.{}, }, &.{ - mlir.ext.mlirType(mlir_ctx, q.shape()), + mlir.ext.mlirType(mlir_ctx, output_shape), mlir.RankedTensorType.init(&.{0}, mlir.IntegerType(.u8).init(mlir_ctx).as(mlir.Type).?).asType(), }, loc, ); - const result = Tensor._result(q.shape(), op.result(0)).transpose(q_.shape()); + const result = Tensor._result(output_shape, op.result(0)).transpose(q_.shape()); return result; } - From 0bf42f4baf913f5e0a7418d221e71c58b43dfdd0 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Thu, 27 Feb 2025 10:10:55 +0000 Subject: [PATCH 4/7] zml/nn/cuda.zig: re-enable in-kernel scaling --- zml/nn/cuda.zig | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/zml/nn/cuda.zig b/zml/nn/cuda.zig index c26ef0aa..f4ccda74 100644 --- a/zml/nn/cuda.zig +++ b/zml/nn/cuda.zig @@ -43,8 +43,10 @@ pub fn sdpa(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) Tensor { const v = v_.transpose(.{ .b, .h, .k, .hd }); const sqrtHeadDim: f32 = 1.0 / std.math.sqrt(@as(f32, @floatFromInt(q.dim(.hd)))); - const head_scaling = if (opts.scale) |s| s else Tensor.scalar(sqrtHeadDim, k.dtype()); - k = k.mul(head_scaling); + const kernel_scale = if (opts.scale) |_| 1.0 else sqrtHeadDim; + if (opts.scale) |s| { + k = k.mul(s); + } var buffer: [4096]u8 = undefined; var fba = std.heap.FixedBufferAllocator.init(&buffer); @@ -63,7 +65,7 @@ pub fn sdpa(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) Tensor { \\ "is_cudnn_frontend": true, \\ "workspace_size": "0", \\ }}, - \\ "fmha_scale": 0, + \\ "fmha_scale": {}, \\ "dropout_rate": 0, \\ "intermediate_tensor_shape": {{ \\ "element_type": "{s}", @@ -102,6 +104,7 @@ pub fn sdpa(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) Tensor { \\}} , .{ + kernel_scale, elementTypeFromDataType(q.dtype()), q.dim(.b), q.dim(.h), @@ -150,8 +153,10 @@ pub fn sdpaPaged(q_: Tensor, k_: Tensor, v_: Tensor, sequence_length_q: Tensor, const v = v_.transpose(.{ .page, .h, .k_chunk, .hd }); const sqrtHeadDim: f32 = 1.0 / std.math.sqrt(@as(f32, @floatFromInt(q.dim(.hd)))); - const head_scaling = if (opts.scale) |s| s else Tensor.scalar(sqrtHeadDim, k.dtype()); - k = k.mul(head_scaling); + const kernel_scale = if (opts.scale) |_| 1.0 else sqrtHeadDim; + if (opts.scale) |s| { + k = k.mul(s); + } var buffer: [4096]u8 = undefined; var fba = std.heap.FixedBufferAllocator.init(&buffer); @@ -171,7 +176,7 @@ pub fn sdpaPaged(q_: Tensor, k_: Tensor, v_: Tensor, sequence_length_q: Tensor, \\ "is_cudnn_frontend": true, \\ "workspace_size": "0", \\ }}, - \\ "fmha_scale": 0, + \\ "fmha_scale": {}, \\ "dropout_rate": 0, \\ "intermediate_tensor_shape": {{ \\ "element_type": "{s}", @@ -211,6 +216,7 @@ pub fn sdpaPaged(q_: Tensor, k_: Tensor, v_: Tensor, sequence_length_q: Tensor, \\}} , .{ + kernel_scale, elementTypeFromDataType(q.dtype()), q.dim(.b), q.dim(.h), @@ -233,7 +239,7 @@ pub fn sdpaPaged(q_: Tensor, k_: Tensor, v_: Tensor, sequence_length_q: Tensor, const loc = mlir_ctx.location(@src()); const op = dialect.stablehlo.custom_call( mlir_ctx, - &.{ q.value(), k.value(), v.value(), bias.value(), sequence_length_q.value(), sequence_length_kv.value(), page_table.value(), page_table.value() }, + &.{ q.value(), k.value(), v.value(), sequence_length_q.value(), sequence_length_kv.value(), page_table.value(), page_table.value() }, .{ .call_target_name = "__cudnn$fmhaScaleBiasSoftmax", .backend_config = backend_config, From 454e245e057327a8b9078511060194743e41628f Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Fri, 7 Mar 2025 16:45:43 +0000 Subject: [PATCH 5/7] runtimes/cuda: bumped cudnn to 9.7.1 --- runtimes/cuda/cudnn.redistrib_9.7.1.json | 77 ++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 runtimes/cuda/cudnn.redistrib_9.7.1.json diff --git a/runtimes/cuda/cudnn.redistrib_9.7.1.json b/runtimes/cuda/cudnn.redistrib_9.7.1.json new file mode 100644 index 00000000..c2d02afb --- /dev/null +++ b/runtimes/cuda/cudnn.redistrib_9.7.1.json @@ -0,0 +1,77 @@ +{ + "release_date": "2025-02-06", + "release_label": "9.7.1", + "release_product": "cudnn", + "cudnn": { + "name": "NVIDIA CUDA Deep Neural Network library", + "license": "cudnn", + "license_path": "cudnn/LICENSE.txt", + "version": "9.7.1.26", + "linux-x86_64": { + "cuda11": { + "relative_path": "cudnn/linux-x86_64/cudnn-linux-x86_64-9.7.1.26_cuda11-archive.tar.xz", + "sha256": "73aadf2d1b721a34bd7b90908d02906257f2addbd246cf8db58e21d68d855ea2", + "md5": "0a38a905dffec48e1701b059accba573", + "size": "673620800" + }, + "cuda12": { + "relative_path": "cudnn/linux-x86_64/cudnn-linux-x86_64-9.7.1.26_cuda12-archive.tar.xz", + "sha256": "109a5e5c6bcdf4396e6f63f3f862ed2dcf16ee93ee034dc704a1b11bdf00c0b1", + "md5": "6d6b02a4f3ed294771bb379d80ad8cff", + "size": "969539424" + } + }, + "cuda_variant": [ + "11", + "12" + ], + "linux-sbsa": { + "cuda11": { + "relative_path": "cudnn/linux-sbsa/cudnn-linux-sbsa-9.7.1.26_cuda11-archive.tar.xz", + "sha256": "25ca58ff295401a8f7edbceb2657ab483c4ee4a80f9a92f8d0cbe5f15aae1cde", + "md5": "a3ae7f24043d2dc76d741f1a968c5842", + "size": "672867120" + }, + "cuda12": { + "relative_path": "cudnn/linux-sbsa/cudnn-linux-sbsa-9.7.1.26_cuda12-archive.tar.xz", + "sha256": "92824550a95eb275b06c90998410e12ce05140e0508f0905644c654c9ed7a764", + "md5": "092e915a2bd8b625b1525d4f64f3fb9a", + "size": "968065560" + } + }, + "windows-x86_64": { + "cuda11": { + "relative_path": "cudnn/windows-x86_64/cudnn-windows-x86_64-9.7.1.26_cuda11-archive.zip", + "sha256": "bbc429272aeb87c838c7ba98f46ad4593a40b737ea6bc0ca6824e8afe0a2e504", + "md5": "618d2c5669f3cd236242608f4519f1b6", + "size": "508778239" + }, + "cuda12": { + "relative_path": "cudnn/windows-x86_64/cudnn-windows-x86_64-9.7.1.26_cuda12-archive.zip", + "sha256": "fecb6b911b6fc74cce40180bc6cb932a66bd03f1ae678987b92ce0dc896ef52b", + "md5": "013c774a296b5268477db15b8831c769", + "size": "706241604" + } + }, + "linux-aarch64": { + "cuda12": { + "relative_path": "cudnn/linux-aarch64/cudnn-linux-aarch64-9.7.1.26_cuda12-archive.tar.xz", + "sha256": "8c33d601728e8896296e53f08391544a1ec5d03820e7d2cb9ddfa95fdcbbaf5c", + "md5": "5f0281fa6b4d2886434f9676c0697975", + "size": "1017901016" + } + } + }, + "cudnn_samples": { + "name": "NVIDIA cuDNN samples", + "license": "cudnn", + "license_path": "cudnn_samples/LICENSE.txt", + "version": "9.7.1.26", + "source": { + "relative_path": "cudnn_samples/source/cudnn_samples-source-9.7.1.26-archive.tar.xz", + "sha256": "5a2141fd430bff59a337c972b3bdcd188de758caab87b0810ce457a632150d99", + "md5": "7189f6715d7f7bb8647be656b1bca80d", + "size": "1666504" + } + } +} From d4e1553c84d8108e8fd822cd8414a3772efdc8f7 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Fri, 7 Mar 2025 16:46:25 +0000 Subject: [PATCH 6/7] wip: operand layouts --- zml/nn/cuda.zig | 45 +++++++++++++++++++++++++++++++++++++++------ 1 file changed, 39 insertions(+), 6 deletions(-) diff --git a/zml/nn/cuda.zig b/zml/nn/cuda.zig index f4ccda74..5516e434 100644 --- a/zml/nn/cuda.zig +++ b/zml/nn/cuda.zig @@ -121,7 +121,8 @@ pub fn sdpa(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) Tensor { const mlir_ctx = ctx.mlirCtx(); const loc = mlir_ctx.location(@src()); - const op = dialect.stablehlo.custom_call( + const op = dialect.stablehlo.custom_call_alloc( + ctx.allocator(), mlir_ctx, &.{ q.value(), k.value(), v.value(), bias.value() }, .{ @@ -146,8 +147,37 @@ pub const SdpaPagedOpts = struct { max_seq_len_kv: usize, }; -pub fn sdpaPaged(q_: Tensor, k_: Tensor, v_: Tensor, sequence_length_q: Tensor, sequence_length_kv: Tensor, page_table: Tensor, opts: SdpaPagedOpts) Tensor { +pub fn layout_constraint(input: Tensor, output_shape: Shape) Tensor { const ctx = CompilationContext.current(); + const mlir_ctx = ctx.mlirCtx(); + const loc = mlir_ctx.location(@src()); + const op = dialect.stablehlo.custom_call( + mlir_ctx, + &.{input.value()}, + .{ + .call_target_name = "LayoutConstraint", + .api_version = 2, + .has_side_effect = false, + .output_operand_aliases = &.{}, + }, + &.{ + mlir.ext.mlirType(mlir_ctx, output_shape), + //mlir.RankedTensorType.init(&.{0}, mlir.IntegerType(.u8).init(mlir_ctx).as(mlir.Type).?).asType(), + }, + loc, + ); + const result = Tensor._result(output_shape, op.result(0)); + return result; +} + +pub fn sdpaPaged(q_: Tensor, k_: Tensor, v_: Tensor, sequence_length_q: Tensor, sequence_length_kv: Tensor, page_table: Tensor, seq_offset: Tensor, opts: SdpaPagedOpts) Tensor { + const ctx = CompilationContext.current(); + //var q = q_; + //var k = k_; + //var v = v_; + //const q = layout_constraint(q_, q_.shape().transpose(.{ .b, .h, .q, .hd })); + //var k = layout_constraint(k_, k_.shape().transpose(.{ .page, .h, .k_chunk, .hd })); + //const v = layout_constraint(v_, v_.shape().transpose(.{ .page, .h, .k_chunk, .hd })); const q = q_.transpose(.{ .b, .h, .q, .hd }); var k = k_.transpose(.{ .page, .h, .k_chunk, .hd }); const v = v_.transpose(.{ .page, .h, .k_chunk, .hd }); @@ -226,20 +256,23 @@ pub fn sdpaPaged(q_: Tensor, k_: Tensor, v_: Tensor, sequence_length_q: Tensor, }, ) catch unreachable; - std.debug.print("backend_config: {s}\n", .{backend_config}); - var bias = Tensor.constant(Shape.init(.{ .b = q.dim(.b), .h = q.dim(.h), .q = q.dim(.q), .k = opts.max_seq_len_kv }, q.dtype()), Data.init(q.dtype(), 0)); if (opts.attn_mask) |attn_mask| { bias = bias.add(attn_mask.broad(bias.shape())); } + //q = q_.transpose(.{ .b, .h, .q, .hd }); + //q = layout_constraint(q, q.shape()); + //k = layout_constraint(k, k.shape().transpose(.{ .page, .h, .k_chunk, .hd })); + //v = layout_constraint(v, v.shape().transpose(.{ .page, .h, .k_chunk, .hd })); const output_shape = Shape.init(.{ .b = q.dim(.b), .h = q.dim(.h), .q = q.dim(.q), .hd = q.dim(.hd) }, q.dtype()); const mlir_ctx = ctx.mlirCtx(); const loc = mlir_ctx.location(@src()); - const op = dialect.stablehlo.custom_call( + const op = dialect.stablehlo.custom_call_alloc( + ctx.allocator(), mlir_ctx, - &.{ q.value(), k.value(), v.value(), sequence_length_q.value(), sequence_length_kv.value(), page_table.value(), page_table.value() }, + &.{ q.value(), k.value(), v.value(), sequence_length_q.value(), sequence_length_kv.value(), page_table.value(), page_table.value(), seq_offset.value() }, .{ .call_target_name = "__cudnn$fmhaScaleBiasSoftmax", .backend_config = backend_config, From 561e4602a3c62ce72c66d9483b811e8ad72bed93 Mon Sep 17 00:00:00 2001 From: Corentin Godeau Date: Tue, 18 Mar 2025 17:47:52 +0000 Subject: [PATCH 7/7] zml/nn/cuda.zig: fix sdpaPaged `backend_config` --- zml/nn/cuda.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zml/nn/cuda.zig b/zml/nn/cuda.zig index 5516e434..71d89f21 100644 --- a/zml/nn/cuda.zig +++ b/zml/nn/cuda.zig @@ -275,7 +275,7 @@ pub fn sdpaPaged(q_: Tensor, k_: Tensor, v_: Tensor, sequence_length_q: Tensor, &.{ q.value(), k.value(), v.value(), sequence_length_q.value(), sequence_length_kv.value(), page_table.value(), page_table.value(), seq_offset.value() }, .{ .call_target_name = "__cudnn$fmhaScaleBiasSoftmax", - .backend_config = backend_config, + .backend_config = .{ .str = backend_config }, .api_version = 2, .has_side_effect = false, .output_operand_aliases = &.{},