diff --git a/runtimes/cuda/cudnn.redistrib_9.7.1.json b/runtimes/cuda/cudnn.redistrib_9.7.1.json new file mode 100644 index 00000000..c2d02afb --- /dev/null +++ b/runtimes/cuda/cudnn.redistrib_9.7.1.json @@ -0,0 +1,77 @@ +{ + "release_date": "2025-02-06", + "release_label": "9.7.1", + "release_product": "cudnn", + "cudnn": { + "name": "NVIDIA CUDA Deep Neural Network library", + "license": "cudnn", + "license_path": "cudnn/LICENSE.txt", + "version": "9.7.1.26", + "linux-x86_64": { + "cuda11": { + "relative_path": "cudnn/linux-x86_64/cudnn-linux-x86_64-9.7.1.26_cuda11-archive.tar.xz", + "sha256": "73aadf2d1b721a34bd7b90908d02906257f2addbd246cf8db58e21d68d855ea2", + "md5": "0a38a905dffec48e1701b059accba573", + "size": "673620800" + }, + "cuda12": { + "relative_path": "cudnn/linux-x86_64/cudnn-linux-x86_64-9.7.1.26_cuda12-archive.tar.xz", + "sha256": "109a5e5c6bcdf4396e6f63f3f862ed2dcf16ee93ee034dc704a1b11bdf00c0b1", + "md5": "6d6b02a4f3ed294771bb379d80ad8cff", + "size": "969539424" + } + }, + "cuda_variant": [ + "11", + "12" + ], + "linux-sbsa": { + "cuda11": { + "relative_path": "cudnn/linux-sbsa/cudnn-linux-sbsa-9.7.1.26_cuda11-archive.tar.xz", + "sha256": "25ca58ff295401a8f7edbceb2657ab483c4ee4a80f9a92f8d0cbe5f15aae1cde", + "md5": "a3ae7f24043d2dc76d741f1a968c5842", + "size": "672867120" + }, + "cuda12": { + "relative_path": "cudnn/linux-sbsa/cudnn-linux-sbsa-9.7.1.26_cuda12-archive.tar.xz", + "sha256": "92824550a95eb275b06c90998410e12ce05140e0508f0905644c654c9ed7a764", + "md5": "092e915a2bd8b625b1525d4f64f3fb9a", + "size": "968065560" + } + }, + "windows-x86_64": { + "cuda11": { + "relative_path": "cudnn/windows-x86_64/cudnn-windows-x86_64-9.7.1.26_cuda11-archive.zip", + "sha256": "bbc429272aeb87c838c7ba98f46ad4593a40b737ea6bc0ca6824e8afe0a2e504", + "md5": "618d2c5669f3cd236242608f4519f1b6", + "size": "508778239" + }, + "cuda12": { + "relative_path": "cudnn/windows-x86_64/cudnn-windows-x86_64-9.7.1.26_cuda12-archive.zip", + "sha256": "fecb6b911b6fc74cce40180bc6cb932a66bd03f1ae678987b92ce0dc896ef52b", + "md5": "013c774a296b5268477db15b8831c769", + "size": "706241604" + } + }, + "linux-aarch64": { + "cuda12": { + "relative_path": "cudnn/linux-aarch64/cudnn-linux-aarch64-9.7.1.26_cuda12-archive.tar.xz", + "sha256": "8c33d601728e8896296e53f08391544a1ec5d03820e7d2cb9ddfa95fdcbbaf5c", + "md5": "5f0281fa6b4d2886434f9676c0697975", + "size": "1017901016" + } + } + }, + "cudnn_samples": { + "name": "NVIDIA cuDNN samples", + "license": "cudnn", + "license_path": "cudnn_samples/LICENSE.txt", + "version": "9.7.1.26", + "source": { + "relative_path": "cudnn_samples/source/cudnn_samples-source-9.7.1.26-archive.tar.xz", + "sha256": "5a2141fd430bff59a337c972b3bdcd188de758caab87b0810ce457a632150d99", + "md5": "7189f6715d7f7bb8647be656b1bca80d", + "size": "1666504" + } + } +} diff --git a/zml/nn.zig b/zml/nn.zig index 07f4c9f7..3cc340df 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -2,7 +2,7 @@ const std = @import("std"); const stdx = @import("stdx"); -const cuda = @import("nn/cuda.zig"); +pub const cuda = @import("nn/cuda.zig"); const helpers = @import("helpers.zig"); const meta = @import("meta.zig"); const ops = @import("ops.zig"); diff --git a/zml/nn/cuda.zig b/zml/nn/cuda.zig index 81ff6000..71d89f21 100644 --- a/zml/nn/cuda.zig +++ b/zml/nn/cuda.zig @@ -43,9 +43,10 @@ pub fn sdpa(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) Tensor { const v = v_.transpose(.{ .b, .h, .k, .hd }); const sqrtHeadDim: f32 = 1.0 / std.math.sqrt(@as(f32, @floatFromInt(q.dim(.hd)))); - const scale: f32 = if (opts.scale) |_| 1.0 else sqrtHeadDim; - - if (opts.scale) |s| k = k.mul(s); + const kernel_scale = if (opts.scale) |_| 1.0 else sqrtHeadDim; + if (opts.scale) |s| { + k = k.mul(s); + } var buffer: [4096]u8 = undefined; var fba = std.heap.FixedBufferAllocator.init(&buffer); @@ -64,7 +65,7 @@ pub fn sdpa(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) Tensor { \\ "is_cudnn_frontend": true, \\ "workspace_size": "0", \\ }}, - \\ "fmha_scale": {d}, + \\ "fmha_scale": {}, \\ "dropout_rate": 0, \\ "intermediate_tensor_shape": {{ \\ "element_type": "{s}", @@ -103,7 +104,7 @@ pub fn sdpa(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) Tensor { \\}} , .{ - scale, + kernel_scale, elementTypeFromDataType(q.dtype()), q.dim(.b), q.dim(.h), @@ -120,7 +121,8 @@ pub fn sdpa(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) Tensor { const mlir_ctx = ctx.mlirCtx(); const loc = mlir_ctx.location(@src()); - const op = dialect.stablehlo.custom_call( + const op = dialect.stablehlo.custom_call_alloc( + ctx.allocator(), mlir_ctx, &.{ q.value(), k.value(), v.value(), bias.value() }, .{ @@ -138,3 +140,152 @@ pub fn sdpa(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) Tensor { const result = Tensor._result(q.shape(), op.result(0)).transpose(q_.shape()); return result; } + +pub const SdpaPagedOpts = struct { + attn_mask: ?Tensor = null, + scale: ?Tensor = null, + max_seq_len_kv: usize, +}; + +pub fn layout_constraint(input: Tensor, output_shape: Shape) Tensor { + const ctx = CompilationContext.current(); + const mlir_ctx = ctx.mlirCtx(); + const loc = mlir_ctx.location(@src()); + const op = dialect.stablehlo.custom_call( + mlir_ctx, + &.{input.value()}, + .{ + .call_target_name = "LayoutConstraint", + .api_version = 2, + .has_side_effect = false, + .output_operand_aliases = &.{}, + }, + &.{ + mlir.ext.mlirType(mlir_ctx, output_shape), + //mlir.RankedTensorType.init(&.{0}, mlir.IntegerType(.u8).init(mlir_ctx).as(mlir.Type).?).asType(), + }, + loc, + ); + const result = Tensor._result(output_shape, op.result(0)); + return result; +} + +pub fn sdpaPaged(q_: Tensor, k_: Tensor, v_: Tensor, sequence_length_q: Tensor, sequence_length_kv: Tensor, page_table: Tensor, seq_offset: Tensor, opts: SdpaPagedOpts) Tensor { + const ctx = CompilationContext.current(); + //var q = q_; + //var k = k_; + //var v = v_; + //const q = layout_constraint(q_, q_.shape().transpose(.{ .b, .h, .q, .hd })); + //var k = layout_constraint(k_, k_.shape().transpose(.{ .page, .h, .k_chunk, .hd })); + //const v = layout_constraint(v_, v_.shape().transpose(.{ .page, .h, .k_chunk, .hd })); + const q = q_.transpose(.{ .b, .h, .q, .hd }); + var k = k_.transpose(.{ .page, .h, .k_chunk, .hd }); + const v = v_.transpose(.{ .page, .h, .k_chunk, .hd }); + + const sqrtHeadDim: f32 = 1.0 / std.math.sqrt(@as(f32, @floatFromInt(q.dim(.hd)))); + const kernel_scale = if (opts.scale) |_| 1.0 else sqrtHeadDim; + if (opts.scale) |s| { + k = k.mul(s); + } + + var buffer: [4096]u8 = undefined; + var fba = std.heap.FixedBufferAllocator.init(&buffer); + const allocator = fba.allocator(); + + const backend_config = + std.fmt.allocPrintZ( + allocator, + \\{{ + \\ "operation_queue_id":"0", + \\ "wait_on_operation_queues":[], + \\ "cudnn_fmha_backend_config": {{ + \\ "algorithm": {{ + \\ "algo_id": "0", + \\ "math_type": "TENSOR_OP_MATH", + \\ "tuning_knobs": {{"17": "1", "24": "0"}}, + \\ "is_cudnn_frontend": true, + \\ "workspace_size": "0", + \\ }}, + \\ "fmha_scale": {}, + \\ "dropout_rate": 0, + \\ "intermediate_tensor_shape": {{ + \\ "element_type": "{s}", + \\ "dimensions": ["{}", "{}", "{}", "{}"], + \\ "tuple_shapes": [], + \\ "layout": {{ + \\ "dim_level_types": [], + \\ "dim_unique": [], + \\ "dim_ordered": [], + \\ "minor_to_major": ["3", "2", "1", "0"], + \\ "tiles": [], + \\ "element_size_in_bits": "0", + \\ "memory_space": "0", + \\ "index_primitive_type": "PRIMITIVE_TYPE_INVALID", + \\ "pointer_primitive_type": "PRIMITIVE_TYPE_INVALID", + \\ "dynamic_shape_metadata_prefix_bytes": "0", + \\ }}, + \\ "is_dynamic_dimension": [false, false, false, false], + \\ }}, + \\ "seed": 0, + \\ "is_flash_attention": true, + \\ "mask_type": "NO_MASK", + \\ "bmm1_dot_dimension_numbers": {{ + \\ "lhs_contracting_dimensions": ["3"], + \\ "rhs_contracting_dimensions": ["3"], + \\ "lhs_batch_dimensions": ["0", "1"], + \\ "rhs_batch_dimensions": ["0", "1"], + \\ }}, + \\ "bmm2_dot_dimension_numbers": {{ + \\ "lhs_contracting_dimensions": ["3"], + \\ "rhs_contracting_dimensions": ["2"], + \\ "lhs_batch_dimensions": ["0", "1"], + \\ "rhs_batch_dimensions": ["0", "1"], + \\ }}, + \\ "max_sequence_length_kv": {}, + \\ }} + \\}} + , + .{ + kernel_scale, + elementTypeFromDataType(q.dtype()), + q.dim(.b), + q.dim(.h), + q.dim(.q), + k.dim(.k_chunk), + opts.max_seq_len_kv, + }, + ) catch unreachable; + + var bias = Tensor.constant(Shape.init(.{ .b = q.dim(.b), .h = q.dim(.h), .q = q.dim(.q), .k = opts.max_seq_len_kv }, q.dtype()), Data.init(q.dtype(), 0)); + + if (opts.attn_mask) |attn_mask| { + bias = bias.add(attn_mask.broad(bias.shape())); + } + + //q = q_.transpose(.{ .b, .h, .q, .hd }); + //q = layout_constraint(q, q.shape()); + //k = layout_constraint(k, k.shape().transpose(.{ .page, .h, .k_chunk, .hd })); + //v = layout_constraint(v, v.shape().transpose(.{ .page, .h, .k_chunk, .hd })); + const output_shape = Shape.init(.{ .b = q.dim(.b), .h = q.dim(.h), .q = q.dim(.q), .hd = q.dim(.hd) }, q.dtype()); + const mlir_ctx = ctx.mlirCtx(); + const loc = mlir_ctx.location(@src()); + const op = dialect.stablehlo.custom_call_alloc( + ctx.allocator(), + mlir_ctx, + &.{ q.value(), k.value(), v.value(), sequence_length_q.value(), sequence_length_kv.value(), page_table.value(), page_table.value(), seq_offset.value() }, + .{ + .call_target_name = "__cudnn$fmhaScaleBiasSoftmax", + .backend_config = .{ .str = backend_config }, + .api_version = 2, + .has_side_effect = false, + .output_operand_aliases = &.{}, + }, + &.{ + mlir.ext.mlirType(mlir_ctx, output_shape), + mlir.RankedTensorType.init(&.{0}, mlir.IntegerType(.u8).init(mlir_ctx).as(mlir.Type).?).asType(), + }, + loc, + ); + const result = Tensor._result(output_shape, op.result(0)).transpose(q_.shape()); + return result; +}