diff --git a/BUILD.bazel b/BUILD.bazel index aa1656d4..84d72ab8 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -12,6 +12,8 @@ zls_completion( deps = [ "//async", "//examples/llama", + "//examples/mixtral", + "//examples/mixtral:test", "//stdx", "//zml", ], diff --git a/docs/tutorials/write_first_model.md b/docs/tutorials/write_first_model.md index 7cea735c..5a6f91df 100644 --- a/docs/tutorials/write_first_model.md +++ b/docs/tutorials/write_first_model.md @@ -65,14 +65,10 @@ Module. For compilation, only the _Shapes_ of all tensors must be known. No actual tensor data is needed at this step. This is important for large models: we can compile them while the actual weight data is being fetched from disk. -To accomplish this, ZML uses a _BufferStore_. The _BufferStore_ knows how to -only load shapes and when to load actual tensor data. In our example, we will -fake the _BufferStore_ a bit: we won't load from disk; we'll use float arrays -instead. - -After compilation is done (and the _BufferStore_ has finished loading weights), -we can send the weights from the _BufferStore_ to our computation device. That -produces an _executable_ module which we can call with different _inputs_. +To accomplish this, most ZML code uses a _BufferStore_. +The _BufferStore_ knows how to only load shapes and when to load actual tensor data. +In our example, we wont' use _BufferStore_ at all, +and manually fill out the shapes first then the weights. In our example, we then copy the result from the computation device to CPU memory and print it. @@ -80,11 +76,10 @@ memory and print it. **So the steps for us are:** - describe the computation as ZML _Module_, using tensor operations -- create a _BufferStore_ that provides _Shapes_ and data of weights and bias - (ca. 5 lines of code). +- describe the shapes of our model - compile the _Module_ **asynchronously** -- make the compiled _Module_ send the weights (and bias) to the computation - device utilizing the _BufferStore_, producing an _executable_ module +- send the weights of the model to the computation device +- bind the model weights to the _Module_ producing an _executable_ module - prepare input tensor and call the _executable_ module. - get the result back to CPU memory and print it @@ -100,9 +95,6 @@ Let's start by writing some Zig code, importing ZML and often-used modules: const std = @import("std"); const zml = @import("zml"); const asynk = @import("async"); - -// shortcut to the asyncc function in the asynk module -const asyncc = asynk.asyncc; ``` You will use above lines probably in all ZML projects. Also, note that **ZML is @@ -110,11 +102,10 @@ async** and comes with its own async runtime, thanks to [zigcoro](https://github.com/rsepassi/zigcoro). - ### Defining our Model -We will start with a very simple "Model". One that resembles a "multiply and -add" operation. +We will start with a very simple "Model". +One that resembles a "multiply and add" operation. ```zig /// Model definition @@ -151,6 +142,15 @@ ZML code is async. Hence, We need to provide an async main function. It works like this: ```zig +pub const std_options: std.Options = .{ + .log_level = .info, + .logFn = asynk.logFn(std.log.defaultLog), +}; + +pub fn main() !void { + try asynk.AsyncThread.main(std.heap.smp_allocator, asyncMain); +} + pub fn main() !void { var gpa = std.heap.GeneralPurposeAllocator(.{}){}; defer _ = gpa.deinit(); @@ -170,166 +170,143 @@ So, let's start with the async main function: ```zig pub fn asyncMain() !void { - // Short lived allocations - var gpa = std.heap.GeneralPurposeAllocator(.{}){}; - defer _ = gpa.deinit(); - const allocator = gpa.allocator(); + var gpa_state = std.heap.DebugAllocator(.{}){}; + defer _ = gpa_state.deinit(); + const gpa = gpa_state.allocator(); - // Arena allocator for BufferStore etc. - var arena_state = std.heap.ArenaAllocator.init(allocator); - defer arena_state.deinit(); - const arena = arena_state.allocator(); - - // Create ZML context + // Start ZML, detect available devices and chose a platform. var context = try zml.Context.init(); defer context.deinit(); - const platform = context.autoPlatform(.{}); + context.printAvailablePlatforms(platform); + ... } ``` -This is boilerplate code that provides a general-purpose allocator and, for -convenience, an arena allocator that we will use later. The advantage of arena -allocators is that you don't need to deallocate individual allocations; you -simply call `.deinit()` to deinitialize the entire arena instead! - -We also initialize the ZML context `context` and get our CPU `platform` -automatically. +This is boilerplate code that provides a debug allocator, +initialize the ZML context `context`, discover available device +and chose a `platform` to use. +A `platform` is an homogenous set of devices that can work together, +eg 2 Nivdia 4090 GPUs. -### The BufferStore +### Compilation -Next, we need to set up the concrete weight and bias tensors for our model. -Typically, we would load them from disk. But since our example works without -stored weights, we are going to create a BufferStore manually, containing -_HostBuffers_ (buffers on the CPU) for both the `weight` and the `bias` tensor. +We already described the model logic, but we didn't specify any sizes. +To compile an executable specialized to our usecase we are going to provide +the shapes of the individual tensors in the model as well as input shapes. -A BufferStore basically contains a dictionary with string keys that match the -name of the struct fields of our `Layer` struct. So, let's create this -dictionary: +To do that we need to instantiate the `Layer` struct. +Note that `zml.Shape` always takes the data type associated with the tensor. +In our example, we will use half-precision floats `f16`, expressed as the enum value `.f16`. + +We also set give a `.buffer_id` to each `Tensor` to let the compiler know those are different objects despite the same shape. ```zig -// Our weights and bias to use -var weights = [3]f16{ 2.0, 2.0, 2.0 }; -var bias = [3]f16{ 1.0, 2.0, 3.0 }; -const input_shape = zml.Shape.init(.{3}, .f16); + const model_shapes: Layer = .{ + .bias = zml.Tensor{ + ._shape = zml.Shape.init(.{4}, .f16).withSharding(.{-1}), + ._id = .{ .buffer_id = 0 }, + }, + .weight = zml.Tensor{ + ._shape = zml.Shape.init(.{4}, .f16).withSharding(.{-1}), + ._id = .{ .buffer_id = 1 }, + }, + }; -// We manually produce a BufferStore. You would not normally do that. -// A BufferStore is usually created by loading model data from a file. -var buffers: zml.aio.BufferStore.Buffers = .{}; -try buffers.put(arena, "weight", zml.HostBuffer.fromArray(&weights)); -try buffers.put(arena, "bias", zml.HostBuffer.fromArray(&bias)); - -// the actual BufferStore -const bs: zml.aio.BufferStore = .{ - .arena = arena_state, - .buffers = buffers, -}; + const input_shape = zml.Shape.init(.{4}, .f16); ``` -Our weights are `{2.0, 2.0, 2.0}`, and our bias is just `{1.0, 2.0, 3.0}`. The -shape of the weight and bias tensors is `{3}`, and because of that, the **shape -of the input tensor** is also going to be `{3}`! +Note: for real models the shape are generally loaded from a config file or directly from the +metadata of the weight file using `zml.aio.BufferStore`. -Note that `zml.Shape` always takes the data type associated with the tensor. In -our example, that is `f16`, expressed as the enum value `.f16`. - - - -### Compiling our Module for the accelerator - -We're only going to use the CPU for our simple model, but we need to compile the -`forward()` function nonetheless. This compilation is usually done -asynchronously. That means, we can continue doing other things while the module -is compiling: +Then we can compile the model asynchronously so we can do other things in parallel, +typically loading the weights on the device. ```zig -// A clone of our model, consisting of shapes. We only need shapes for compiling. -// We use the BufferStore to infer the shapes. -const model_shapes = try zml.aio.populateModel(Layer, allocator, bs); - -// Start compiling. This uses the inferred shapes from the BufferStore. -// The shape of the input tensor, we have to pass in manually. -var compilation = try asyncc( +var compilation = try asynk.asyncc( zml.compileModel, - .{ allocator, Layer.forward, model_shapes, .{input_shape}, platform }, + .{ gpa, Layer.forward, model_shapes, .{input_shape}, platform }, ); - -// Produce a bufferized weights struct from the fake BufferStore. -// This is like the inferred shapes, but with actual values. -// We will need to send those to the computation device later. -var model_weights = try zml.aio.loadBuffers(Layer, .{}, bs, arena, platform); -defer zml.aio.unloadBuffers(&model_weights); // for good practice - -// Wait for compilation to finish -const compiled = try compilation.awaitt(); ``` Compiling is happening in the background via the `asyncc` function. We call `asyncc` with the `zml.compileModel` function and its arguments -separately. The arguments themselves are basically the shapes of the weights in -the BufferStore, the `.forward` function name in order to compile -`Layer.forward`, the shape of the input tensor(s), and the platform for which to -compile (we used auto platform). +separately. +The arguments themselves are: +- an allocator to allocate memory needed for the returned executable. +- `Layer.forward`: the function to be compiled +- the shapes of the model +- the shape of the input tensors (`Layer.forward` only got one argument, so it's a 1-tuple) +- the platform for which to compile +### Loading the weights -### Creating the Executable Model +Now the model is compiling, we can load the model data to the GPU. +To represent the loaded weights, we are going to instantiate a `zml.Bufferized(Layer)` struct. +This is a struct that looks like our `Layer` struct, +but where all `Tensor`s have been replaced by the `zml.Buffer` type, +and where all non-`Tensor` fields have been straight up removed. -Now that we have compiled the module utilizing the shapes, we turn it into an -executable. +``` + // Now we need to create a model instance with actual weights. +const weights = [4]f16{ 2.0, -2.0, 1.0, -1.0 }; +const bias = [4]f16{ 1.0, 2.0, 3.0, 4.0 }; + +var model_weights: zml.Bufferized(Layer) = .{ + .bias = try zml.Buffer.fromSlice(platform, .{4}, &bias), + .weight = try zml.Buffer.fromSlice(platform, .{4}, &weights), +}; +defer zml.aio.unloadBuffers(&model_weights); +``` + +Since `zml.Bufferized(Layer)` is a normal struct we have type checking enabled, +and the Zig compiler will tell us if we forget to load a struct. +Here we hard-coded the weights in the code, +but typically weights would be read from disk at this point using `BufferStore`. + +### Putting it all together + +Now we want to bind the weights we just loaded, +with the executable we started to compile earlier on. ```zig -// pass the model weights to the compiled module to create an executable module -// all required memory has been allocated in `compile`. -var executable = compiled.prepare(model_weights); -defer executable.deinit(); +// Wait for compilation to finish +const compiled = try compilation.awaitt(); +defer compiled.deinit(); + +// pass the model weights to the compiled module to create an executable module. +const executable = compiled.prepare(model_weights); ``` +`prepare` will check that the shapes of the buffers in `model_weights` match +with the shapes given to the compiler. +Also note we don't call `executable.deinit()` since it re-uses the memory of `compiled`. + ### Calling / running the Model The executable can now be invoked with an input of our choice. -To create the `input`, we directly use `zml.Buffer` by calling -`zml.Buffer.fromArray()`. It's important to note that `Buffer`s reside in -_accelerator_ (or _device_) memory, which is precisely where the input needs to -be for the executable to process it on the device. - -For clarity, let's recap the distinction: `HostBuffer`s are located in standard -_host_ memory, which is accessible by the CPU. When we initialized the weights, -we used `HostBuffers` to set up the `BufferStore`. This is because the -`BufferStore` typically loads weights from disk into `HostBuffer`s, and then -converts them into `Buffer`s when we call `loadBuffers()`. - -However, for inputs, we bypass the `BufferStore` and create `Buffer`s directly -in device memory. - - ```zig -// prepare an input buffer -// Here, we use zml.HostBuffer.fromSlice to show how you would create a -// HostBuffer with a specific shape from an array. -// For situations where e.g. you have an [4]f16 array but need a .{2, 2} input -// shape. -var input = [3]f16{ 5.0, 5.0, 5.0 }; -var input_buffer = try zml.Buffer.from( - platform, - zml.HostBuffer.fromSlice(input_shape, &input), -); -defer input_buffer.deinit(); +// prepare the input buffer + const input = [4]f16{ 5.0, 5.0, 5.0, 5.0 }; + var input_buffer = try zml.Buffer.fromSlice(platform, input_shape, &input); + defer input_buffer.deinit(); -// call our executable module -var result: zml.Buffer = executable.call(.{input_buffer}); -defer result.deinit(); + // call our executable module, the result is still on the device. + var result: zml.Buffer = executable.call(.{input_buffer}); + defer result.deinit(); -// fetch the result buffer to CPU memory -const cpu_result = try result.toHostAlloc(arena); -std.debug.print( - "\n\nThe result of {d} * {d} + {d} = {d}\n", - .{ &weights, &input, &bias, cpu_result.items(f16) }, -); + // copy the result to CPU memory + const cpu_result: zml.HostBuffer = try result.toHostAlloc(gpa); + defer cpu_result.deinit(gpa); + std.debug.print( + "\nThe result of {d} * {d} + {d} = {d}\n", + .{ &weights, &input, &bias, cpu_result.items(f16) }, + ); ``` Note that the result of a computation is usually residing in the memory of the @@ -344,8 +321,6 @@ And that's it! Now, let's have a look at building and actually running this example! - - ## Building it As mentioned already, ZML uses Bazel; so to build our model, we just need to @@ -417,10 +392,9 @@ You can access the complete source code of this walkthrough here: ```zig const std = @import("std"); -const zml = @import("zml"); -const asynk = @import("async"); -const asyncc = asynk.asyncc; +const asynk = @import("async"); +const zml = @import("zml"); /// Model definition const Layer = struct { @@ -436,88 +410,84 @@ const Layer = struct { } }; +pub const std_options: std.Options = .{ + .log_level = .info, + .logFn = asynk.logFn(std.log.defaultLog), +}; + pub fn main() !void { - var gpa = std.heap.GeneralPurposeAllocator(.{}){}; - defer _ = gpa.deinit(); - try asynk.AsyncThread.main(gpa.allocator(), asyncMain); + try asynk.AsyncThread.main(std.heap.smp_allocator, asyncMain); } pub fn asyncMain() !void { - // Short lived allocations - var gpa = std.heap.GeneralPurposeAllocator(.{}){}; - defer _ = gpa.deinit(); - const allocator = gpa.allocator(); + var gpa_state = std.heap.DebugAllocator(.{}){}; + defer _ = gpa_state.deinit(); + const gpa = gpa_state.allocator(); - // Arena allocator for BufferStore etc. - var arena_state = std.heap.ArenaAllocator.init(allocator); - defer arena_state.deinit(); - const arena = arena_state.allocator(); - - // Create ZML context + // Start ZML, detect available devices and chose a platform. var context = try zml.Context.init(); defer context.deinit(); - const platform = context.autoPlatform(.{}); + context.printAvailablePlatforms(platform); - // Our weights and bias to use - var weights = [3]f16{ 2.0, 2.0, 2.0 }; - var bias = [3]f16{ 1.0, 2.0, 3.0 }; - const input_shape = zml.Shape.init(.{3}, .f16); - - // We manually produce a BufferStore. You would not normally do that. - // A BufferStore is usually created by loading model data from a file. - var buffers: zml.aio.BufferStore.Buffers = .{}; - try buffers.put(arena, "weight", zml.HostBuffer.fromArray(&weights)); - try buffers.put(arena, "bias", zml.HostBuffer.fromArray(&bias)); - - // the actual BufferStore - const bs: zml.aio.BufferStore = .{ - .arena = arena_state, - .buffers = buffers, + // Create a skeleton for our model with only the shapes. + // The `buffer_id` are typically not set manually + // but instead set by a zml.BufferStore + // to map abstract Tensors to actual part of the model weight file. + const model_shapes: Layer = .{ + .bias = zml.Tensor{ + ._shape = zml.Shape.init(.{4}, .f16).withSharding(.{-1}), + ._id = .{ .buffer_id = 0 }, + }, + .weight = zml.Tensor{ + ._shape = zml.Shape.init(.{4}, .f16).withSharding(.{-1}), + ._id = .{ .buffer_id = 1 }, + }, }; - // A clone of our model, consisting of shapes. We only need shapes for - // compiling. We use the BufferStore to infer the shapes. - const model_shapes = try zml.aio.populateModel(Layer, allocator, bs); + // Start compilation in a different thread. + // We already specified the shape of the model weights, + // but we still need to specifiy the shape of the input tensor. + const input_shape = zml.Shape.init(.{4}, .f16); + var compilation = try asynk.asyncc( + zml.compileModel, + .{ gpa, Layer.forward, model_shapes, .{input_shape}, platform }, + ); - // Start compiling. This uses the inferred shapes from the BufferStore. - // The shape of the input tensor, we have to pass in manually. - var compilation = try asyncc(zml.compileModel, .{ allocator, Layer.forward, model_shapes, .{input_shape}, platform }); + // Now we need to create a model instance with actual weights. + const weights = [4]f16{ 2.0, -2.0, 1.0, -1.0 }; + const bias = [4]f16{ 1.0, 2.0, 3.0, 4.0 }; - // Produce a bufferized weights struct from the fake BufferStore. - // This is like the inferred shapes, but with actual values. - // We will need to send those to the computation device later. - var model_weights = try zml.aio.loadBuffers(Layer, .{}, bs, arena, platform); - defer zml.aio.unloadBuffers(&model_weights); // for good practice + var model_weights: zml.Bufferized(Layer) = .{ + .bias = try zml.Buffer.fromSlice(platform, .{4}, &bias), + .weight = try zml.Buffer.fromSlice(platform, .{4}, &weights), + }; + defer zml.aio.unloadBuffers(&model_weights); // Wait for compilation to finish const compiled = try compilation.awaitt(); + defer compiled.deinit(); - // pass the model weights to the compiled module to create an executable - // module - var executable = compiled.prepare(model_weights); - defer executable.deinit(); + // pass the model weights to the compiled module to create an executable module. + // This is where the shapes of the buffers will be compared to the + // shape expected by the executable. + // Note: we don't call `executable.deinit()` since it uses the same memory than `compiled`. + const executable = compiled.prepare(model_weights); - // prepare an input buffer - // Here, we use zml.HostBuffer.fromSlice to show how you would create a - // HostBuffer with a specific shape from an array. - // For situations where e.g. you have an [4]f16 array but need a .{2, 2} - // input shape. - var input = [3]f16{ 5.0, 5.0, 5.0 }; - var input_buffer = try zml.Buffer.from( - platform, - zml.HostBuffer.fromSlice(input_shape, &input), - ); + // prepare the input buffer + const input = [4]f16{ 5.0, 5.0, 5.0, 5.0 }; + var input_buffer = try zml.Buffer.fromSlice(platform, input_shape, &input); defer input_buffer.deinit(); - // call our executable module + // call our executable module, the result is still on the device. var result: zml.Buffer = executable.call(.{input_buffer}); defer result.deinit(); - // fetch the result to CPU memory - const cpu_result = try result.toHostAlloc(arena); + // copy the result to CPU memory + const cpu_result: zml.HostBuffer = try result.toHostAlloc(gpa); + defer cpu_result.deinit(gpa); std.debug.print( - "\n\nThe result of {d} * {d} + {d} = {d}\n", + "\nThe result of {d} * {d} + {d} = {d}\n", .{ &weights, &input, &bias, cpu_result.items(f16) }, ); } diff --git a/examples/benchmark/main.zig b/examples/benchmark/main.zig index 6428eda9..cfd5fcd5 100644 --- a/examples/benchmark/main.zig +++ b/examples/benchmark/main.zig @@ -1,8 +1,9 @@ const std = @import("std"); -const zml = @import("zml"); -const stdx = @import("stdx"); + const asynk = @import("async"); +const stdx = @import("stdx"); const flags = stdx.flags; +const zml = @import("zml"); // set log level to debug to print the generated IR pub const std_options: std.Options = .{ @@ -10,12 +11,27 @@ pub const std_options: std.Options = .{ .logFn = asynk.logFn(std.log.defaultLog), }; -pub fn benchmark(a: zml.Tensor, b: zml.Tensor) zml.Tensor { - return a.withSharding(.{.k}).dot(b.withSharding(.{.k}), .{.k}).withSharding(.{.m}); +pub fn benchmark(x: zml.Tensor, q_w: zml.Tensor, q_scale: zml.Tensor) zml.Tensor { + return x.dot(dequantize(q_w, q_scale), .{.d}); +} + +fn dequantize(quantized_weight: zml.Tensor, q_scale: zml.Tensor) zml.Tensor { + var fp4_encoding_buf: [16]zml.floats.BFloat16 = undefined; + for (0..16) |i| { + const i_u4: u4 = @truncate(i); + const f4: zml.floats.Float4E2M1 = @bitCast(i_u4); + fp4_encoding_buf[i] = .fromF32(f4.toF32()); + } + std.log.warn("fp4: {d}", .{fp4_encoding_buf}); + const fp4_encoding: zml.Tensor = .constantTensor(.fromSlice(.{ .fp4 = 16 }, &fp4_encoding_buf)); + + const scale: zml.Tensor = .pow(.constant(q_scale.shape(), .init(.bf16, 2)), q_scale.convert(.bf16)); + const actual_weight = fp4_encoding.gatherValues(.fp4, quantized_weight, .{}); + return actual_weight.mul(scale.broad(actual_weight.shape())).merge(.{ .d = .{ .block, .d } }); } pub fn main() !void { - try asynk.AsyncThread.main(std.heap.c_allocator, asyncMain); + try asynk.AsyncThread.main(std.heap.smp_allocator, asyncMain); } pub fn asyncMain() !void { @@ -38,22 +54,25 @@ pub fn asyncMain() !void { // Auto-select platform const platform = context.autoPlatform(.{}).withCompilationOptions(.{ .sharding_enabled = true, + .xla_dump_to = "/tmp/zml/benchmark", + .xla_dump_fusion_visualization = true, }); context.printAvailablePlatforms(platform); var args = std.process.args(); const cli_args = flags.parse(&args, CliArgs); - const a_shape = zml.Shape.init(.{ cli_args.size, cli_args.size }, cli_args.dtype).withTags(.{ .m, .k }).withSharding(.{.k}); - const b_shape = a_shape.withTags(.{ .k, .n }).withSharding(.{.k}); - var timer = try std.time.Timer.start(); + const x_shape: zml.Shape = .init(.{ .b = 16, .d = cli_args.size }, .bf16); + const w_shape: zml.Shape = .init(.{ .d_out = cli_args.size, .block = @divExact(cli_args.size, 32), .d = 32 }, .i4); + const s_shape: zml.Shape = .init(.{ .d_out = cli_args.size, .block = @divExact(cli_args.size, 32) }, .i8); + var timer = try std.time.Timer.start(); std.debug.print("\nCompiling model to MLIR....\n", .{}); std.debug.print("-" ** 160 ++ "\n", .{}); // Start compiling. // The shape of the input tensor, we have to pass in manually. timer.reset(); - var compilation = try asynk.asyncc(zml.compileFn, .{ allocator, benchmark, .{ a_shape, b_shape }, platform }); + var compilation = try asynk.asyncc(zml.compileFn, .{ allocator, benchmark, .{ x_shape, w_shape, s_shape }, platform }); // Wait for compilation to finish const executable = try compilation.awaitt(); @@ -65,22 +84,24 @@ pub fn asyncMain() !void { var rng = std.Random.DefaultPrng.init(0); const random = rng.random(); - var a_buffer = try createRandomBuffer(allocator, platform, a_shape, random); - defer a_buffer.deinit(); - var b_buffer = try createRandomBuffer(allocator, platform, b_shape, random); - defer b_buffer.deinit(); + var x = try createRandomBuffer(allocator, platform, x_shape, random); + defer x.deinit(); + var w = try createRandomBuffer(allocator, platform, w_shape, random); + defer w.deinit(); + var s = try createRandomBuffer(allocator, platform, s_shape, random); + defer s.deinit(); std.debug.print("\nRunning benchmark....\n", .{}); // Ignore first run { - var result: zml.Buffer = executable.call(.{ a_buffer, b_buffer }); + var result: zml.Buffer = executable.call(.{ x, w, s }); defer result.deinit(); } // call our executable module timer.reset(); - var result: zml.Buffer = executable.call(.{ a_buffer, b_buffer }); + var result: zml.Buffer = executable.call(.{ x, w, s }); defer result.deinit(); const elapsed_ns = timer.lap(); const elapsed_ms = @as(f64, @floatFromInt(elapsed_ns)) / std.time.ns_per_ms; diff --git a/examples/callback/BUILD.bazel b/examples/callback/BUILD.bazel new file mode 100644 index 00000000..3f55040f --- /dev/null +++ b/examples/callback/BUILD.bazel @@ -0,0 +1,11 @@ +load("@rules_zig//zig:defs.bzl", "zig_binary") + +zig_binary( + name = "callback", + main = "main.zig", + deps = [ + "@zml//async", + "@zml//zml", + "@zml//runtimes", + ], +) diff --git a/examples/callback/main.zig b/examples/callback/main.zig new file mode 100644 index 00000000..be3b3095 --- /dev/null +++ b/examples/callback/main.zig @@ -0,0 +1,218 @@ +const std = @import("std"); + +const asynk = @import("async"); +const runtimes = @import("runtimes"); +const zml = @import("zml"); +const cu = zml.platform_specific; + +pub const std_options: std.Options = .{ + .log_level = .info, + .logFn = asynk.logFn(std.log.defaultLog), +}; + +const log = std.log.scoped(.@"examples/custom_call"); + +/// Demonstration of the custom_call mechanism. +/// +/// * defines a function to compute the grayscale version of an image. +/// * using simple for loop on CPU +/// * using a custom PTX kernel on Cuda, with manual cuLaunchKernel +/// * Use `zml.customCall` to create an executable calling our callback and our kernel. +pub const GrayScale = struct { + // Mandatory fields to work with ZML custom call api + pub var type_id: zml.pjrt.ffi.TypeId = undefined; + pub const callback_config: zml.callback.Config = .{}; + platform: zml.Platform, + + // Custom field to store our cuda module and function. + cu_data: [2]*anyopaque, + + // Set by ZML before `call` is entered + results: [1]zml.Buffer = undefined, + stream: *zml.pjrt.Stream = undefined, + + pub fn init(platform: zml.Platform) !GrayScale { + var cu_data: [2]*anyopaque = undefined; + if (comptime runtimes.isEnabled(.cuda)) { + var module: cu.CUmodule = undefined; + try cuda.check(cuda.moduleLoadData.?(&module, grayscale_ptx)); + log.info("Loaded Grayscale cuda module", .{}); + var function: cu.CUfunction = undefined; + try cuda.check(cuda.moduleGetFunction.?(&function, module, "rgba_to_grayscale")); + log.info("Found Grayscale cuda function", .{}); + cu_data = .{ module.?, function.? }; + } + return .{ + .platform = platform, + .cu_data = cu_data, + }; + } + + pub fn deinit(self: *GrayScale) void { + if (comptime runtimes.isEnabled(.cuda)) { + const module: cu.CUmodule = @ptrCast(self.cu_data[0]); + _ = cuda.moduleUnload.?(module); + } + } + + pub fn call(self: *GrayScale, rgb_d: zml.Buffer) !void { + switch (self.platform.target) { + .cpu => grayScaleCpu(rgb_d, self.results[0]), + // Only try to compile `grayScaleCuda` if we have cuda symbols. + .cuda => if (comptime runtimes.isEnabled(.cuda)) + try self.grayScaleCuda(rgb_d, self.results[0]) + else + unreachable, + else => @panic("Platform not supported"), + } + } + + pub fn grayScaleCpu(rgb_d: zml.Buffer, gray_d: zml.Buffer) void { + const rgb_h = rgb_d.asHostBuffer().items(u8); + const gray_h = gray_d.asHostBuffer().mutItems(u8); + + for (gray_h, 0..) |*gray, i| { + const px = rgb_h[i * 3 .. i * 3 + 3]; + const R: u32 = @intCast(px[0]); + const G: u32 = @intCast(px[1]); + const B: u32 = @intCast(px[2]); + const gray_u32: u32 = @divFloor(299 * R + 587 * G + 114 * B, 1000); + gray.* = @intCast(gray_u32); + } + } + + pub fn grayScaleCuda(self: GrayScale, rgb_d: zml.Buffer, gray_d: zml.Buffer) !void { + var args: [2][]u8 = .{ + rgb_d.opaqueDeviceMemoryDataPointer()[0..rgb_d.shape().byteSize()], + gray_d.opaqueDeviceMemoryDataPointer()[0..gray_d.shape().byteSize()], + }; + var args_ptr: [2:null]?*anyopaque = .{ @ptrCast(&args[0]), @ptrCast(&args[1]) }; + // This is a naive kernel with one block per pixel. + try cuda.check(cuda.launchKernel.?( + @ptrCast(self.cu_data[1]), // function + @intCast(rgb_d.shape().count() / 3), // num blocks x + 1, // num blocks y + 1, // num blocks z + 1, // num grids x + 1, // num grids y + 1, // num grids z + 0, // shared mem + @ptrCast(self.stream), + &args_ptr, + null, + )); + // Note: no explicit synchronization, we just enqueue work in the stream. + } + + const cuda = struct { + // Here we leverage ZML sandboxing to access cuda symbols and their definitions. + const moduleLoadData = @extern(*const @TypeOf(cu.cuModuleLoadData), .{ .name = "cuModuleLoadData", .linkage = .weak }); + const moduleUnload = @extern(*const @TypeOf(cu.cuModuleUnload), .{ .name = "cuModuleUnload", .linkage = .weak }); + const moduleGetFunction = @extern(*const @TypeOf(cu.cuModuleGetFunction), .{ .name = "cuModuleGetFunction", .linkage = .weak }); + const launchKernel = @extern(*const @TypeOf(cu.cuLaunchKernel), .{ .name = "cuLaunchKernel", .linkage = .weak }); + + pub fn check(result: cu.CUresult) error{CudaError}!void { + if (result == cu.CUDA_SUCCESS) return; + std.log.err("cuda error: {}", .{result}); + return error.CudaError; + } + }; +}; + +pub fn grayscale(rgb: zml.Tensor) zml.Tensor { + const gray_shape = rgb.shape().setDim(0, @divExact(rgb.dim(0), 3)); + const result = zml.callback.call(GrayScale, .{rgb.print()}, &.{gray_shape}); + return result[0]; +} + +pub fn main() !void { + try asynk.AsyncThread.main(std.heap.smp_allocator, asyncMain); +} + +pub fn asyncMain() !void { + var gpa = std.heap.GeneralPurposeAllocator(.{}){}; + defer _ = gpa.deinit(); + const allocator = gpa.allocator(); + + var context = try zml.Context.init(); + defer context.deinit(); + + const platform = context.autoPlatform(.{}); + context.printAvailablePlatforms(platform); + + // Register our custom call + try zml.callback.register(GrayScale, platform); + + // Compile MLIR code containing our custom call. + const rgb_shape = zml.Shape.init(.{12 * 3}, .u8); + const exe = try zml.compileFn(allocator, grayscale, .{rgb_shape}, platform); + defer exe.deinit(); + + // Provide runtime information needed by our custom call. + var gray_op: GrayScale = try .init(platform); + defer gray_op.deinit(); + try exe.bind(GrayScale, &gray_op); + + // Load data and run the executable. + const rgb_h: [12][3]u8 = @splat(.{ 0xFF, 0xAA, 0x00 }); + const rgb_d = try zml.Buffer.fromBytes(platform, rgb_shape, @ptrCast(&rgb_h)); + defer rgb_d.deinit(); + + var gray_d: zml.Buffer = exe.call(.{rgb_d}); + defer gray_d.deinit(); + + // Inspect results + const gray_h = try gray_d.toHostAlloc(allocator); + defer gray_h.deinit(allocator); + + std.debug.print("Grayscale conversion of {any} -> {d}\n", .{ rgb_h, gray_h }); + try std.testing.expectEqualSlices(u8, &@as([12]u8, @splat(0xB0)), gray_h.items(u8)); +} + +// Compiled with Zig and a little help from `https://github.com/gwenzek/cudaz` +const grayscale_ptx = + \\.version 4.0 + \\.target sm_32 + \\.address_size 64 + \\ + \\.visible .entry rgba_to_grayscale( + \\ .param .align 8 .b8 rgba_to_grayscale_param_0[16], + \\ .param .align 8 .b8 rgba_to_grayscale_param_1[16] + \\) + \\{ + \\ .reg .pred %p<2>; + \\ .reg .b16 %rs<4>; + \\ .reg .b32 %r<13>; + \\ .reg .b64 %rd<9>; + \\ + \\ ld.param.u64 %rd5, [rgba_to_grayscale_param_1+8]; + \\ mov.u32 %r1, %tid.x; + \\ mov.u32 %r3, %ntid.x; + \\ mov.u32 %r2, %ctaid.x; + \\ mad.lo.s32 %r4, %r2, %r3, %r1; + \\ cvt.u64.u32 %rd1, %r4; + \\ setp.gt.u64 %p1, %rd5, %rd1; + \\ @%p1 bra $L__BB0_2; + \\ bra.uni $L__BB0_1; + \\$L__BB0_2: + \\ ld.param.u64 %rd4, [rgba_to_grayscale_param_1]; + \\ ld.param.u64 %rd2, [rgba_to_grayscale_param_0]; + \\ cvt.u32.u64 %r5, %rd1; + \\ mul.lo.s32 %r6, %r5, 3; + \\ cvt.u64.u32 %rd6, %r6; + \\ add.s64 %rd7, %rd2, %rd6; + \\ ld.u8 %rs1, [%rd7]; + \\ ld.u8 %rs2, [%rd7+1]; + \\ ld.u8 %rs3, [%rd7+2]; + \\ mul.wide.u16 %r7, %rs1, 299; + \\ mul.wide.u16 %r8, %rs2, 587; + \\ add.s32 %r9, %r8, %r7; + \\ mul.wide.u16 %r10, %rs3, 114; + \\ add.s32 %r11, %r9, %r10; + \\ mul.hi.u32 %r12, %r11, 4294968; + \\ add.s64 %rd8, %rd4, %rd1; + \\ st.u8 [%rd8], %r12; + \\$L__BB0_1: + \\ ret; + \\} +; diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig index da487bb3..efe7f220 100644 --- a/examples/llama/llama.zig +++ b/examples/llama/llama.zig @@ -234,8 +234,8 @@ const Mlp = struct { pub fn forward(self: Mlp, x: Tensor) Tensor { const proj = zml.call(self.up_proj, .forward, .{x}); - var output = zml.call(self.gate_proj, .forward, .{x}); - output = output.silu().mul(proj); + const gate = zml.call(self.gate_proj, .forward, .{x}); + const output = gate.silu().mul(proj); return zml.call(self.down_proj, .forward, .{output}); } }; diff --git a/examples/mixtral/BUILD.bazel b/examples/mixtral/BUILD.bazel new file mode 100644 index 00000000..ebe38f94 --- /dev/null +++ b/examples/mixtral/BUILD.bazel @@ -0,0 +1,28 @@ +load("@rules_zig//zig:defs.bzl", "zig_binary") + +zig_binary( + name = "mixtral", + srcs = [ + "Mixtral.zig", + ], + main = "main.zig", + deps = [ + "@com_github_hejsil_clap//:clap", + "@zml//async", + "@zml//stdx", + "@zml//zml", + ], + visibility = ["//visibility:public"], +) + +zig_binary( + name = "test", + srcs = ["Mixtral.zig"], + main = "test.zig", + deps = [ + "//async", + "//stdx", + "//zml", + ], + visibility = ["//visibility:public"], + ) diff --git a/examples/mixtral/Mixtral.zig b/examples/mixtral/Mixtral.zig new file mode 100644 index 00000000..3826e3de --- /dev/null +++ b/examples/mixtral/Mixtral.zig @@ -0,0 +1,567 @@ +///! Mixtral architecture, using huggingface transformers naming. +///! Dimensions of activations: {.b, .s, .d} +const std = @import("std"); + +const stdx = @import("stdx"); +const zml = @import("zml"); + +const Mixtral = @This(); +const log = std.log.scoped(.Mixtral); + +pub const Config = struct { + bos_token_id: u32 = 199998, + eos_token_id: stdx.json.Union(union(enum) { + int: u32, + ints: []u32, + }), + num_hidden_layers: usize, + num_attention_heads: usize, + num_key_value_heads: usize, + rope_theta: f32, + max_position_embeddings: usize, + rms_norm_eps: f32, + hf_rope_impl: bool = true, + rope_scaling: zml.nn.RopeOpts.Scaling = .{ .default = {} }, +}; + +pub const Options = struct { + sampling_strategy: ?zml.nn.SamplingStrategy, + max_seq_len: usize, +}; + +lm_head: ?zml.nn.Linear, +model: Model, + +// Options controlling generation +gen_opts: zml.nn.SamplingStrategy = .{}, +config: Config, + +pub fn init(allocator: std.mem.Allocator, store: zml.aio.BufferStore, config: Config, options: Options) !Mixtral { + var self: Mixtral = .{ + .config = config, + .gen_opts = options.sampling_strategy orelse .{}, + .model = .{ + .max_seq_len = @intCast(options.max_seq_len), + .num_heads = @intCast(config.num_attention_heads), + .num_kv_heads = @intCast(config.num_key_value_heads), + .rope_opts = .{ + .layout = if (config.hf_rope_impl) .sequential else .interleaved, + .freq_base = config.rope_theta, + .scaling = config.rope_scaling, + }, + + .embed_tokens = .{ + .weight = store.getTensor("model.embed_tokens.weight").withSharding(.{1}), + }, + .layers = try allocator.alloc(TransformerLayer, config.num_hidden_layers), + .norm = .{ + .weight = store.getTensor("model.norm.weight"), + .eps = config.rms_norm_eps, + }, + }, + .lm_head = .{ .weight = store.getTensor("lm_head.weight").withSharding(.{0}) }, + }; + + var prefix: zml.aio.PrefixBuilder = try .initCapacity(allocator, 1024); + try prefix.push(stdx.noalloc, "model.layers"); + for (self.model.layers, 0..) |*layer, i| { + try prefix.pushDigit(stdx.noalloc, i); + defer prefix.pop(); + + // TODO: safer layer init + var self_attn = try zml.aio.populateModelWithPrefix(SelfAttn, allocator, store, prefix.concat("self_attn")); + { + self_attn.num_heads = self.model.num_heads; + self_attn.num_kv_heads = self.model.num_kv_heads; + self_attn.rope_opts = self.model.rope_opts; + self_attn.q_proj.weight = self_attn.q_proj.weight.withSharding(.{0}); + self_attn.k_proj.weight = self_attn.k_proj.weight.withSharding(.{0}); + self_attn.v_proj.weight = self_attn.v_proj.weight.withSharding(.{0}); + self_attn.o_proj.weight = self_attn.o_proj.weight.withSharding(.{1}); + } + + const on_disk_moe = try zml.aio.populateModelWithPrefix(MoE.OnDisk, allocator, store, prefix.concat("mlp")); + var moe = on_disk_moe.rewrite(); + { + moe.experts.gate_up_proj.blocks = moe.experts.gate_up_proj.blocks.withSharding(.{.expert}); + moe.experts.down_proj.blocks = moe.experts.down_proj.blocks.withSharding(.{.expert}); + } + + layer.* = .{ + .input_layernorm = .{ + .weight = store.getTensor(prefix.concat("input_layernorm.weight")), + .eps = config.rms_norm_eps, + }, + .post_attention_layernorm = .{ + .weight = store.getTensor(prefix.concat("post_attention_layernorm.weight")), + .eps = config.rms_norm_eps, + }, + .self_attn = self_attn, + .mlp = moe, + }; + } + + // TODO(Corentin): Fix lm_head sharding when top-k sampling is enabled. + // It currently crashes/compilation fails + if (self.gen_opts.topk == 1 and self.lm_head != null) { + self.lm_head.?.weight = self.lm_head.?.weight.withSharding(.{0}); + } + + return self; +} + +/// Predicts the token at `token_index` position. +/// Returns: +/// - updated `tokens`, +/// - updated KV cache +/// - a Rng state to allow for probabilistic generation +pub fn forward( + self: Mixtral, + tokens_: zml.Tensor, + token_index: zml.Tensor, + kv_cache: KvCache, + rng: zml.Tensor.Rng, +) struct { zml.Tensor, KvCache, zml.Tensor.Rng } { + stdx.debug.assert(tokens_.dtype() == .u32 and tokens_.rank() >= 1 and token_index.dtype() == .u32 and token_index.rank() <= 1, "Can't run Mixtral ! Expected >=1d tokens and 0d token_index, got: {f} and {f}", .{ tokens_, token_index }); + const tokens = tokens_.withPartialTags(.{.s}); + const out, const updated_kv_cache = zml.call(self.model, .forward, .{ tokens, token_index, kv_cache }); + const new_tokens, const new_rng = self.sampleTokens(self.lm_head, out, rng, self.gen_opts); + return .{ new_tokens.convert(tokens.dtype()).reuseBuffer(tokens), updated_kv_cache, new_rng }; +} + +pub fn sampleTokens( + self: Mixtral, + lm_head_: ?zml.nn.Linear, + out_: zml.Tensor, + rng: zml.Tensor.Rng, + opts: zml.nn.SamplingStrategy, +) struct { zml.Tensor, zml.Tensor.Rng } { + const out = out_.withPartialTags(.{ .s, .d }); + + var logits = blk: { + if (lm_head_) |lm_head| { + break :blk zml.call(lm_head, .forward, .{out}); + } else { + break :blk self.model.embed_tokens.weight.withTags(.{ .voc, .d }).dot(out, .{.d}); + } + }; + + if (logits.shape().hasTag(.voc) == null) + logits = logits.rename(.{ .d = .voc }); + + const next_tokens, const new_rng = zml.nn.sampleTokens(logits, opts, rng); + return .{ next_tokens, new_rng }; +} + +pub fn loadBuffers(self: Mixtral, allocator: std.mem.Allocator, store: zml.aio.BufferStore, platform: zml.Platform) !zml.Bufferized(Mixtral) { + var prefix: zml.aio.PrefixBuilder = try .initCapacity(allocator, 256); + defer prefix.deinit(allocator); + + const noalloc = stdx.noalloc; + const loaded: zml.Bufferized(Mixtral) = .{ + .model = .{ + .embed_tokens = try store.loadModelById(zml.nn.TokenEmbedding, noalloc, self.model.embed_tokens, platform), + .layers = try allocator.alloc(zml.Bufferized(TransformerLayer), self.model.layers.len), + .norm = try store.loadModelById(RmsNorm, noalloc, self.model.norm, platform), + }, + .lm_head = try store.loadModelById(?zml.nn.Linear, noalloc, self.lm_head, platform), + }; + + prefix.push(noalloc, "model.layers") catch unreachable; + for (loaded.model.layers, self.model.layers, 0..) |*d_layer, layer, layer_id| { + const ckpt = prefix.checkpoint(); + defer prefix.restore(ckpt); + + prefix.pushDigit(noalloc, layer_id) catch unreachable; + d_layer.* = .{ + .input_layernorm = try store.loadModelById(RmsNorm, noalloc, layer.input_layernorm, platform), + .self_attn = try store.loadModelById(SelfAttn, noalloc, layer.self_attn, platform), + .post_attention_layernorm = try store.loadModelById(RmsNorm, noalloc, layer.post_attention_layernorm, platform), + .mlp = try store.loadModelById(MoE, noalloc, layer.mlp, platform), + }; + } + + return loaded; +} + +pub const Model = struct { + embed_tokens: zml.nn.TokenEmbedding, + norm: RmsNorm, + layers: []TransformerLayer, + + max_seq_len: u32 = 0, + num_heads: i64 = 32, + num_kv_heads: i64 = 32, + rope_opts: zml.nn.RopeOpts = .{ + .layout = .interleaved, + .freq_base = 10_000, + }, + + const Shape = struct { + s: u32, + layer: u16, + hd: u16, + nh: u16, + nkvh: u16, + dtype: zml.DataType, + }; + + pub fn shape(self: Model) Shape { + const key_dim = self.layers[0].self_attn.k_proj.weight.dim(0); + const num_kv_heads = if (self.num_kv_heads > 0) self.num_kv_heads else self.num_heads; + + return .{ + .s = self.max_seq_len, + .layer = @intCast(self.layers.len), + .hd = @intCast(@divExact(key_dim, num_kv_heads)), + .nh = @intCast(self.num_heads), + .nkvh = @intCast(num_kv_heads), + .dtype = self.embed_tokens.weight.dtype(), + }; + } + + /// Forward one token, using KV cache for previous tokens. + /// Returns result and updated KV cache. + pub fn forward(self: Model, tokens: zml.Tensor, token_index: zml.Tensor, kv_cache: KvCache) struct { zml.Tensor, KvCache } { + const embeds = embed(self.embed_tokens, tokens); + var hidden = embeds; + + var updated_kv_cache = kv_cache; + for (self.layers, 0..) |layer, i| { + hidden, updated_kv_cache = zml.call(layer, .forward, .{ hidden, token_index, updated_kv_cache.atLayer(i) }); + } + const output = zml.call(self.norm, .forward, .{hidden}); + + return .{ output, updated_kv_cache.reuseBuffer(kv_cache) }; + } + + pub fn embed(embed_tokens_: zml.nn.TokenEmbedding, tokens_: zml.Tensor) zml.Tensor { + return zml.call(embed_tokens_, .forward, .{tokens_}).withPartialTags(.{.d}); + } +}; + +pub const TransformerLayer = struct { + input_layernorm: RmsNorm, + self_attn: SelfAttn, + post_attention_layernorm: RmsNorm, + mlp: MoE, + + pub fn forward( + self: TransformerLayer, + x0: zml.Tensor, + token_index: zml.Tensor, + kv_cache: KvCache, + ) struct { zml.Tensor, KvCache } { + // Self Attention + //log.debug("TransformerLayer({}) -> {}", .{ x0, self.input_layernorm.forward(x0) }); + stdx.debug.assert(x0.rank() >= 2 and x0.shape().hasTags(.{ .s, .d }), "TransformerLayer expected input shape: {{..., .s, .d}}, received: {f}", .{x0}); + + const x0_normalized = zml.call(self.input_layernorm, .forward, .{x0}); + const delta0, const updated_kv_cache = zml.call(self.self_attn, .forward, .{ x0_normalized, token_index, kv_cache }); + const x1 = x0.add(delta0); + + // Fully Connected + const x1_normalized = zml.call(self.post_attention_layernorm, .forward, .{x1}); + const x2 = zml.call(self.mlp, .forward, .{x1_normalized}).add(x1); + + return .{ x2.reuseBuffer(x0), updated_kv_cache }; + } +}; + +const RmsNorm = struct { + weight: zml.Tensor, + eps: f32 = 1e-6, + + /// L2 normalization of input tensor along `.d` axis. + pub fn forward(self: RmsNorm, input: zml.Tensor) zml.Tensor { + const x = if (input.shape().isFullyTagged()) input else input.withPartialTags(.{.d}); + // Note: contrary to Llama here the full layer is done in .f32, not just the variance computation. + const normalized = zml.nn.rmsNorm(x.convert(.f32), .d, self.eps); + return normalized.mul(self.weight.convert(.f32).withTags(.{.d}).broad(x.shape())).convert(input.dtype()); + } +}; + +pub const Router = struct { + weight: zml.Tensor, + bias: zml.Tensor, + const expert_per_token: u32 = 4; + + pub fn forward(self: Router, input: zml.Tensor) zml.Tensor { + const linear: zml.nn.Linear = .{ .weight = self.weight, .bias = self.bias }; + const gating = linear.forward(input).convert(.f32).softmax(-1); + + return zml.nn.moe.hardGating(gating, expert_per_token).convert(input.dtype()); + } +}; + +const MoE = struct { + experts: Mlp, + router: Router, + + pub const OnDisk = struct { + router: Router, + experts: struct { + down_proj_bias: zml.Tensor, + down_proj_blocks: zml.Tensor, + down_proj_scales: zml.Tensor, + gate_up_proj_bias: zml.Tensor, + gate_up_proj_blocks: zml.Tensor, + gate_up_proj_scales: zml.Tensor, + }, + + pub fn rewrite(on_disk: OnDisk) MoE { + const e = on_disk.experts; + const experts: Mlp = .{ + .gate_up_proj = .{ + // We need to bitcast the scale cause safetensors doesn't encode f8 types correctly + .scale = e.gate_up_proj_scales.withTags(.{ .expert, .out, .d }), + // We don't bitcast here because PJRT doesn't handle packed host buffers + .blocks = e.gate_up_proj_blocks.withTags(.{ .expert, .out, .d, .d_block }), + .blocks_dtype = .f4e2m1, + .bias = e.gate_up_proj_bias.withTags(.{ .expert, .d }), + }, + .down_proj = .{ + .blocks = e.down_proj_blocks.withTags(.{ .expert, .out, .d, .d_block }), + .blocks_dtype = .f4e2m1, + .scale = e.down_proj_scales.withTags(.{ .expert, .out, .d }), + .bias = e.down_proj_bias.withTags(.{ .expert, .d }), + }, + }; + + var router = on_disk.router; + router.weight = router.weight.withTags(.{ .expert, .d }); + + return .{ .router = router, .experts = experts }; + } + }; + + pub fn forward(self: MoE, input: zml.Tensor) zml.Tensor { + log.warn("compiling moe with {f}", .{input}); + const gating = self.router.forward(input); + return zml.nn.mixtureOfExperts(Mlp, self.experts, input, gating, .{ .experts_per_token = Router.expert_per_token, .tokens_per_expert_ratio = 1.5 }); + } + + pub fn loadSharded(self: MoE, store: zml.aio.BufferStore, prefix: *zml.aio.PrefixBuilder, platform: zml.Platform) !zml.Bufferized(MoE) { + prefix.push(stdx.noalloc, "block_sparse_moe.experts") catch unreachable; + + return .{ + .router = try store.loadModelById(zml.nn.Linear, stdx.noalloc, self.router, platform), + .experts = .{ + .gate_up_proj = .{ .blocks = try loadShardedWeight(store, prefix, "w1", platform, self.experts.gate_up_proj.blocks) }, + .down_proj = .{ .blocks = try loadShardedWeight(store, prefix, "w2", platform, self.experts.down_proj.blocks) }, + }, + }; + } + + fn loadShardedWeight(store: zml.aio.BufferStore, prefix: *zml.aio.PrefixBuilder, name: []const u8, platform: zml.Platform, weight: zml.Tensor) !zml.Buffer { + const devices = platform.getDevices(); + const num_experts = weight.dim(.expert); + const num_shards = platform.sharding().num_partitions; + + // Note: this requires num_devices == num_partitions, this is overly restrictive. + if (devices.len == num_shards and num_experts % num_shards == 0 and num_experts >= num_shards) { + const num_experts_per_shard = @divExact(num_experts, num_shards); + const tmp_buf: []u8 = try std.heap.smp_allocator.alloc(u8, 4096); + defer std.heap.smp_allocator.free(tmp_buf); + var fba: std.heap.FixedBufferAllocator = .init(tmp_buf); + + const allocator = fba.allocator(); + const transfer = try platform.batchedTransfer(allocator, &.{weight.shape()}, .device); + + for (0..num_experts) |expert_id| { + const part_id = expert_id % num_experts_per_shard; + const shard_id = @divFloor(expert_id, num_experts_per_shard); + const ckpt = prefix.checkpoint(); + defer prefix.restore(ckpt); + try prefix.pushDigit(stdx.noalloc, expert_id); + try prefix.push(stdx.noalloc, name); + try prefix.push(stdx.noalloc, "weight"); + + const expert = store.get(prefix.items()) orelse { + log.err("Buffer not found: {s}", .{prefix.items()}); + store.findSimilarBufferKeys(std.heap.smp_allocator, prefix.items()); + @panic("Buffer not found"); + }; + + const ev = transfer.transferData( + .{ .buffer_id = 0, .device_id = @intCast(shard_id), .offset = expert.bytes().len * part_id }, + expert.bytes(), + part_id + 1 == num_experts_per_shard, + ); + _ = ev; + } + return transfer.buffers[0]; + } else { + @panic("TODO"); + } + } +}; + +pub const Mlp = struct { + gate_up_proj: zml.nn.BlockScaledLinear, // {.out = intermediate_size * 2, .d = hidden_size / block_size, .d_block = block_size } + down_proj: zml.nn.BlockScaledLinear, // {.out = hidden_size * 2, .d = intermediate_size / block_size, .d_block = block_size } + + pub fn forward(self: Mlp, x: zml.Tensor) zml.Tensor { + const dt = x.dtype(); + var gate, var up = zml.nn.splitRealImg(self.gate_up_proj.forward(x), .interleaved); + gate = .minimum(gate, .scalar(7, dt)); + up = .clamp(up, .scalar(-7, dt), .scalar(7, dt)); + + const out = gate.quickGelu().mul(up.addConstant(1)); + return zml.call(self.down_proj, .forward, .{out}); + } + + pub fn format(self: Mlp, writer: *std.Io.Writer) std.Io.Writer.Error!void { + try writer.print("Mlp(gate_up_proj=.{f}, down_proj=.{f})", .{ self.gate_up_proj, self.down_proj }); + } +}; + +pub const SelfAttn = struct { + q_proj: zml.nn.Linear, + k_proj: zml.nn.Linear, + v_proj: zml.nn.Linear, + sinks: zml.Tensor, + + o_proj: zml.nn.Linear, + + num_heads: i64 = undefined, + num_kv_heads: i64 = 0, + rope_opts: zml.nn.RopeOpts = undefined, + + /// Self Attention. + /// - If token_index is set, x is assumed to be the representation of one new token, + /// and kv_cache will be read for the previous tokens. + /// - If token_index is not set, x is assumed to be the representation of all tokens + /// since the beginning of the sequence, and kv_cache won't be read. + /// In both case, kv_cache will be updated with the computed key and value. + /// x: {.b, .s, .d } -> .{.b, .s, .d} + pub fn forward( + self: SelfAttn, + x: zml.Tensor, + token_index: zml.Tensor, + kv_cache: KvCache, + ) struct { zml.Tensor, KvCache } { + const num_kv_heads = if (self.num_kv_heads > 0) self.num_kv_heads else self.num_heads; + var q = zml.call(self.q_proj, .forward, .{x}).splitAxis(-1, .{ .h = self.num_heads, .hd = .auto }).withSharding(.{.h}); + var k = zml.call(self.k_proj, .forward, .{x}).splitAxis(-1, .{ .h = num_kv_heads, .hd = .auto }).withSharding(.{.h}); + var v = zml.call(self.v_proj, .forward, .{x}).splitAxis(-1, .{ .h = num_kv_heads, .hd = .auto }).withSharding(.{.h}); + + // Generate the attention mask. + const seq_len = kv_cache.k.dim(.k); + var attn_mask = zml.nn.causalAttnMask(.{ .q = seq_len, .k = seq_len }, x.dtype(), null); + + // Note: in Pytorch it would be very inefficient to generate the full attn_mask, + // then slice into it, but XLA is able to optimize this correctly. + attn_mask = attn_mask.gatherSlices(zml.Shape.init(.{ .q = x.dim(.s) }, attn_mask.dtype()), token_index.reshape(.{ .coord = 1 }), .{}); + + // In self-attention, .s axis is used both for keys and queries. + const pos_index = b: { + const temp = zml.Tensor.arange(.{ .end = x.dim(.s) }, token_index.dtype()).withTags(.{.s}).broad(zml.Shape.init(.{ .s = x.dim(.s) }, token_index.dtype())); + break :b temp.add(token_index.broad(temp.shape())); + }; + + q = zml.nn.rope(q, pos_index, self.rope_opts); + k = zml.nn.rope(k, pos_index, self.rope_opts); + q = q.rename(.{ .s = .q }); + k = k.rename(.{ .s = .k }); + v = v.rename(.{ .s = .k }); + + const dtype = q.dtype(); + const new_kv_cache = kv_cache.update(k, v, token_index); + k = new_kv_cache.keys().convert(dtype); + v = new_kv_cache.values().convert(dtype); + + const softmax_bias = self.sinks.withTags(.{.h}); + const attn_output = zml.nn.sdpa(q, k, v, .{ .attn_mask = attn_mask, .allow_cudnn = true, .softmax_bias = softmax_bias }); + // const attn_output = zml.nn.sdpaMemEfficient(q, k, v, .{ .attn_mask = attn_mask }, .{ .q_chunk_size = 4096, .k_chunk_size = 1024 }); + const attn = attn_output.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); + return .{ zml.call(self.o_proj, .forward, .{attn}), new_kv_cache }; + } +}; + +pub const KvCache = struct { + k: zml.Tensor, + v: zml.Tensor, + layer_index: zml.Tensor, + + pub fn init(kv_shape: zml.Shape) KvCache { + // The KV-cache is initialized with ones to detect reads of uninitialized memory. + return .{ + .k = .constant(kv_shape, kv_shape.dtype().one()).withSharding(.{.h}), + .v = .constant(kv_shape, kv_shape.dtype().one()).withSharding(.{.h}), + .layer_index = .scalar(-1, .u32), + }; + } + + pub fn initShape(kv_shape: zml.Shape) zml.ShapeOf(KvCache) { + return .{ + .k = kv_shape, + .v = kv_shape, + .layer_index = zml.Shape.init(.{}, .u32), + }; + } + + pub fn initBuffer(kv_shape: zml.Shape, platform: zml.Platform) !zml.Bufferized(KvCache) { + return .{ + .k = try zml.Buffer.uninitialized(platform, kv_shape, .{}), + .v = try zml.Buffer.uninitialized(platform, kv_shape, .{}), + .layer_index = try zml.Buffer.uninitialized(platform, .scalar(.u32), .{}), + }; + } + + pub fn keys(self: KvCache) zml.Tensor { + return self.k.dynamicSlice(.{ .layer = zml.Tensor.DynSlice{ .start = self.layer_index, .len = 1 } }).squeeze(.layer); + } + + pub fn values(self: KvCache) zml.Tensor { + return self.v.dynamicSlice(.{ .layer = zml.Tensor.DynSlice{ .start = self.layer_index, .len = 1 } }).squeeze(.layer); + } + + pub fn update(self: KvCache, new_k: zml.Tensor, new_v: zml.Tensor, token_index: ?zml.Tensor) KvCache { + const k_shape = self.k.shape().drop(.layer); + var layer = self.layer_index; + layer = if (token_index) |idx| layer.broad(idx.shape()) else layer; + + return if (token_index) |idx| .{ + .k = self.k.scatterSlices( + .{ .layer = layer, .k = idx }, + new_k.convert(self.k.dtype()).transpose(k_shape), + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(self.k), + .v = self.v.scatterSlices( + .{ .layer = layer, .k = idx }, + new_v.convert(self.v.dtype()).transpose(k_shape), + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(self.v), + .layer_index = self.layer_index, + } else .{ + .k = self.k.scatterSlices( + .{ .layer = layer }, + new_k.convert(self.k.dtype()).transpose(k_shape), + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(self.k), + .v = self.v.scatterSlices( + .{ .layer = layer }, + new_v.convert(self.v.dtype()).transpose(k_shape), + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(self.v), + .layer_index = self.layer_index, + }; + } + + pub fn atLayer(self: KvCache, layer_index: usize) KvCache { + return .{ + .k = self.k, + .v = self.v, + .layer_index = .scalar(layer_index, .u32), + }; + } + + pub fn reuseBuffer(self: KvCache, other: KvCache) KvCache { + return .{ + .k = self.k.reuseBuffer(other.k), + .v = self.v.reuseBuffer(other.v), + .layer_index = self.layer_index.reuseBuffer(other.layer_index), + }; + } +}; diff --git a/examples/mixtral/gpt_oss.py b/examples/mixtral/gpt_oss.py new file mode 100644 index 00000000..a4860809 --- /dev/null +++ b/examples/mixtral/gpt_oss.py @@ -0,0 +1,39 @@ +import torch +import transformers +import sys +import safetensors + +sys.path.append("/Users/guw/Documents/zml/tools") +import zml_utils + +model_path = "/Users/guw/models/openai/gpt-oss-20b" + +pipeline = transformers.pipeline( + "text-generation", + model=model_path, + # model_kwargs={"torch_dtype": torch.bfloat16}, + # device="cuda", + device="cpu", +) +model, tokenizer = pipeline.model, pipeline.tokenizer + +prompt = "What is the largest animal?" +# Wrap the pipeline, and extract activations. +# Activations files can be huge for big models, +# so let's stop collecting after 1000 layers. +pipeline = zml_utils.ActivationCollector(pipeline, max_layers=1000, stop_after_first_step=True) +output, activations = pipeline(prompt) + +# `output` can be `None` if activations collection +# has stopped before the end of the inference +if output: + print(output) + +# Save activations to a file. +filename = model_path.split("/")[-1] + ".activations.pt" +torch.save(activations, filename) +print(f"Saved {len(activations)} activations to {filename}") + +filename = model_path.split("/")[-1] + ".activations.safetensors" +safetensors.torch.save_file(activations, filename) +print(f"Saved {len(activations)} activations to {filename}") diff --git a/examples/mixtral/main.zig b/examples/mixtral/main.zig new file mode 100644 index 00000000..333fc76e --- /dev/null +++ b/examples/mixtral/main.zig @@ -0,0 +1,341 @@ +const std = @import("std"); + +const asynk = @import("async"); +const clap = @import("clap"); +const stdx = @import("stdx"); +const zml = @import("zml"); +const Buffer = zml.Buffer; +const Tensor = zml.Tensor; +const ShapeOf = zml.ShapeOf; + +const Mixtral = @import("Mixtral.zig"); + +const log = std.log.scoped(.mixtral); + +pub const std_options: std.Options = .{ + .log_level = .info, + .logFn = asynk.logFn(std.log.defaultLog), +}; + +const cli_params = clap.parseParamsComptime( + \\--help print this help + \\--prompt the prompt + \\--hf-model-path path to the directory containing model weights, config and tokenizer + \\--seed random seed (optional) + \\--seq-len sequence length + \\--create-options platform creation options JSON, defaults to {} + \\--no-llama3 skip prompt template + \\--sharding default: true: sharding on or off +); + +pub fn main() !void { + try asynk.AsyncThread.main(std.heap.smp_allocator, asyncMain); +} + +pub fn asyncMain() !void { + log.info(" mixtral was compiled with {}", .{@import("builtin").mode}); + + const allocator = std.heap.smp_allocator; + const cli = ClapBoilerplate.parseCli(allocator); + defer cli.deinit(); + + const hf_model_path = cli.args.@"hf-model-path" orelse { + log.err("Missing --hf-model-path", .{}); + return; + }; + + const model_config_path = try std.fs.path.join(allocator, &.{ hf_model_path, "config.json" }); + defer allocator.free(model_config_path); + + const model_weights_path = b: { + const simple_path = try std.fs.path.join(allocator, &.{ hf_model_path, "model.safetensors" }); + if (asynk.File.access(simple_path, .{})) { + break :b simple_path; + } else |_| { + allocator.free(simple_path); + } + + const sharded_path = try std.fs.path.join(allocator, &.{ hf_model_path, "model.safetensors.index.json" }); + break :b sharded_path; + }; + defer allocator.free(model_weights_path); + + const model_tokenizer_path = try std.fs.path.join(allocator, &.{ hf_model_path, "tokenizer.json" }); + defer allocator.free(model_tokenizer_path); + + const config = blk: { + var config_json_file = try asynk.File.open(model_config_path, .{ .mode = .read_only }); + defer config_json_file.close() catch unreachable; + var config_json_buffer: [256]u8 = undefined; + var config_reader = config_json_file.reader(&config_json_buffer); + var reader = std.json.Reader.init(allocator, &config_reader.interface); + defer reader.deinit(); + const config_obj = try std.json.parseFromTokenSourceLeaky(Mixtral.Config, allocator, &reader, .{ .ignore_unknown_fields = true }); + break :blk config_obj; + }; + + var context = try zml.Context.init(); + defer context.deinit(); + + const compilation_options = zml.CompilationOptions{ + .xla_dump_to = "/tmp/zml/mixtral", + .sharding_enabled = cli.args.sharding orelse true, + }; + + // initialize ZML platform with optional create options + // eg: --create-options='{"cuda":{"allocator":{"bfc":{"memory_fraction": 0.99}}}}' + // or: --create-options='{"cpu":{"device_count":8}}' + const create_opts_json = cli.args.@"create-options" orelse "{}"; + const create_opts = std.json.parseFromSlice(zml.Platform.CreateOptions, allocator, create_opts_json, .{}) catch |err| { + log.err("Failed to parse --create-options as json ({}): {s}", .{ err, create_opts_json }); + return err; + }; + + const platform = context.autoPlatform(create_opts.value).withCompilationOptions(compilation_options); + create_opts.deinit(); + context.printAvailablePlatforms(platform); + + var store = try zml.aio.detectFormatAndOpen(allocator, model_weights_path); + defer store.deinit(); + + var model_arena = std.heap.ArenaAllocator.init(allocator); + const llama_options: Mixtral.Options = .{ + .max_seq_len = @intCast(cli.args.@"seq-len" orelse 256), + .sampling_strategy = .{ + .topk = 1, + .temperature = 1.0, + }, + }; + const model_instance = try Mixtral.init(model_arena.allocator(), store, config, llama_options); + + const dims = model_instance.model.shape(); + const dtype = model_instance.model.embed_tokens.weight.dtype(); + + const tokens_shape_prefill = zml.Shape.init(.{ .s = llama_options.max_seq_len }, .u32); + const tokens_shape = zml.Shape.init(.{ .s = 1 }, .u32); + const token_idx_shape = zml.Shape.init(.{}, .u32); + + const kv_shape = zml.Shape.init(.{ .layer = model_instance.model.layers.len, .k = dims.s, .h = dims.nkvh, .hd = dims.hd }, dtype).withSharding(.{.h}); + + const kv_cache_shape: zml.ShapeOf(Mixtral.KvCache) = Mixtral.KvCache.initShape(kv_shape); + const rng_shape = zml.Tensor.Rng.shape(); + + var start = try std.time.Timer.start(); + var fut_mod_prefill = try asynk.asyncc(zml.compileModel, .{ + allocator, Mixtral.forward, model_instance, + .{ + tokens_shape_prefill, + token_idx_shape, + kv_cache_shape, + rng_shape, + }, + platform, + }); + + var fut_mod = try asynk.asyncc(zml.compileModel, .{ + allocator, Mixtral.forward, model_instance, + .{ + tokens_shape, + token_idx_shape, + kv_cache_shape, + rng_shape, + }, + platform, + }); + + log.info("\tLoading mixtral weights from {s}...", .{model_weights_path}); + var mixtral_weights = try model_instance.loadBuffers(model_arena.allocator(), store, platform); + defer zml.aio.unloadBuffers(&mixtral_weights); + log.info("✅\tLoaded weights in {D}", .{start.read()}); + + var mixtral_module_prefill = (try fut_mod_prefill.awaitt()).prepare(mixtral_weights); + defer mixtral_module_prefill.deinit(); + var mixtral_module = (try fut_mod.awaitt()).prepare(mixtral_weights); + defer mixtral_module.deinit(); + log.info("✅\tCompiled model in {D}", .{start.read()}); + + log.info("Creating KvCache", .{}); + const kv_cache = try Mixtral.KvCache.initBuffer(kv_shape, platform); + + var tokenizer = blk: { + log.info("Loading tokenizer from {s}", .{model_tokenizer_path}); + var timer = try stdx.time.Timer.start(); + defer log.info("Loaded tokenizer from {s} [{f}]", .{ model_tokenizer_path, timer.read() }); + + break :blk try zml.tokenizer.Tokenizer.fromFile(model_arena.allocator(), model_tokenizer_path); + }; + errdefer tokenizer.deinit(); + + const prompt = cli.args.prompt orelse "What is the capital of France?"; + log.info("✅\tPrompt: {s}", .{prompt}); + + const seed = cli.args.seed orelse @as(u128, @bitCast(std.time.nanoTimestamp())); + const skip_llama3_encoding = cli.args.@"no-llama3" orelse false; + + // Unbuffered writing of the tokens to stdout. + var output = std.fs.File.stdout().writer(&.{}); + + try generateText(config, model_instance, mixtral_module_prefill, mixtral_module, kv_cache, tokenizer, allocator, seed, prompt[0..], skip_llama3_encoding, &output.interface); + // generated text will be printed token by token. +} + +pub fn tokenizePrompt(allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, config: Mixtral.Config, prompt: []const u8, skip_llama3_encoding: bool) ![]u32 { + _ = config; // autofix + var tokens = std.array_list.Managed(u32).init(allocator); + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + if (skip_llama3_encoding) { + // Copy to the arraylist so the ownership is the same in both branches. + try tokens.appendSlice(try encoder.encode(prompt)); + return tokens.toOwnedSlice(); + } + + const start_header = tokenizer.tokenToId("<|start|>") orelse return error.NoSuchToken; + const end_header_start_message = tokenizer.tokenToId("<|message|>") orelse return error.NoSuchToken; + const end_message = tokenizer.tokenToId("<|end|>") orelse return error.NoSuchToken; + + { + try tokens.appendSlice(&.{ start_header, tokenizer.tokenToId("system").?, end_header_start_message }); + try tokens.appendSlice(try encoder.encode("You are ChatGPT, a large language model trained by OpenAI.\n")); + try tokens.append(end_message); + } + + { + try tokens.appendSlice(&.{ start_header, tokenizer.tokenToId("user").?, end_header_start_message }); + try tokens.appendSlice(try encoder.encode(prompt)); + try tokens.append(end_message); + } + + try tokens.appendSlice(&.{ start_header, tokenizer.tokenToId("assistant").?, end_header_start_message }); + + return tokens.toOwnedSlice(); +} + +pub fn generateText( + config: Mixtral.Config, + llama_: Mixtral, + mod_prefill: zml.ModuleExe(Mixtral.forward), + mod_generate: zml.ModuleExe(Mixtral.forward), + kv_cache_: zml.Bufferized(Mixtral.KvCache), + tokenizer: zml.tokenizer.Tokenizer, + allocator: std.mem.Allocator, + seed: u128, + prompt: []const u8, + skip_llama3_encoding: bool, + output: *std.Io.Writer, +) !void { + const prompt_tok: []const u32 = try tokenizePrompt(allocator, tokenizer, config, prompt, skip_llama3_encoding); + log.info("\t Tokenized prompt: {any}", .{prompt_tok}); + defer allocator.free(prompt_tok); + + var tokenizer_decoder = try tokenizer.decoder(); + defer tokenizer_decoder.deinit(); + + const platform = mod_generate.platform(); + const max_seq_len = llama_.model.shape().s; + + // init RNG and buffers + var rng = try zml.Tensor.Rng.init(platform, seed); + var generated_token_buffer = [_]u32{undefined}; + + var kv_cache = prefill: { + // prepare device buffers for the prefill tokens and their positions + const prefill_buffer = try allocator.alloc(u32, max_seq_len); + @memcpy(prefill_buffer[0..prompt_tok.len], prompt_tok); + + var prefill_tokens = try zml.Buffer.fromSlice(platform, .{max_seq_len}, prefill_buffer); + defer prefill_tokens.deinit(); + var prefill_token_pos = try zml.Buffer.constant(platform, zml.Shape.init(.{}, .u32), 0); + defer prefill_token_pos.deinit(); + + const prefilled_tokens, const kv_cache, rng = mod_prefill.call(.{ prefill_tokens, prefill_token_pos, kv_cache_, rng }); + _ = try prefilled_tokens.toHost(std.mem.sliceAsBytes(prefill_buffer)); + generated_token_buffer[0] = prefill_buffer[prompt_tok.len - 1]; + break :prefill kv_cache; + }; + defer zml.aio.unloadBuffers(&kv_cache); + + // Prepare for token-by-token generation, + // start with the token generated based on the full prompt. + var current_token = try zml.Buffer.fromSlice(platform, .{1}, &generated_token_buffer); + defer current_token.deinit(); + + const output_tokens_len = max_seq_len - prompt_tok.len - 1; + const start = std.time.microTimestamp(); + + // One token has alreadyh been generated by the prefill. + var num_tokens_generated: usize = 1; + + generation: for (0..output_tokens_len + 1) |i| { + // collect and print generated sequence + num_tokens_generated += 1; + const generated_token = generated_token_buffer[0]; + if (try tokenizer_decoder.next(generated_token)) |chunk| { + try output.writeAll(chunk); + } + + // check for eos + if (i == output_tokens_len) break :generation; + switch (config.eos_token_id.value) { + .int => |eos| if (generated_token == @as(u32, @intCast(eos))) break :generation, + .ints => |eos_list| { + for (eos_list) |eos| { + if (generated_token == @as(u32, @intCast(eos))) break :generation; + } + }, + } + + // current token pos needs to go into a zml.Buffer + const token_pos_buffer = &[_]u32{@intCast(prompt_tok.len + i)}; + const token_pos = try zml.Buffer.fromSlice(platform, .{}, token_pos_buffer); + defer token_pos.deinit(); + + // call to generate the next token + current_token, kv_cache, rng = mod_generate.call(.{ current_token, token_pos, kv_cache, rng }); + + // extract the generated token from the buffer + _ = try current_token.toHost(std.mem.sliceAsBytes(&generated_token_buffer)); + } + const end = std.time.microTimestamp(); + const duration = stdx.math.divFloat(f64, end - start, std.time.us_per_s); + const speed = @as(f64, @floatFromInt(num_tokens_generated)) / duration; + + log.info("✅ Generated {d} tokens in {:.3}s: {d:.3}tok/s", .{ num_tokens_generated, duration, speed }); +} + +const ClapBoilerplate = struct { + pub const Cli = clap.Result(clap.Help, &cli_params, parsers); + + fn bool_parser(in: []const u8) error{}!bool { + return std.mem.indexOfScalar(u8, "tTyY1", in[0]) != null; + } + + const parsers = .{ + .BOOL = bool_parser, + .UINT = clap.parsers.int(usize, 0), + .STRING = clap.parsers.string, + .PATH = clap.parsers.string, + }; + + pub fn parseCli(allocator: std.mem.Allocator) Cli { + var diag: clap.Diagnostic = .{}; + var stderr_buffer: [1024]u8 = undefined; + var stderr = std.fs.File.stderr().writer(&stderr_buffer); + const cli = clap.parse(clap.Help, &cli_params, parsers, .{ + .diagnostic = &diag, + .allocator = allocator, + }) catch |err| { + diag.report(&stderr.interface, err) catch {}; + stderr.interface.print("usage: ", .{}) catch {}; + clap.usage(&stderr.interface, clap.Help, &cli_params) catch {}; + stderr.interface.print("\n", .{}) catch {}; + std.process.exit(1); + }; + if (cli.args.help != 0) { + clap.help(&stderr.interface, clap.Help, &cli_params, .{}) catch {}; + std.process.exit(0); + } + return cli; + } +}; diff --git a/examples/mixtral/test.zig b/examples/mixtral/test.zig new file mode 100644 index 00000000..32b31761 --- /dev/null +++ b/examples/mixtral/test.zig @@ -0,0 +1,236 @@ +const std = @import("std"); + +const asynk = @import("async"); +const stdx = @import("stdx"); +const flags = stdx.flags; +const zml = @import("zml"); + +const Mixtral = @import("./Mixtral.zig"); + +const log = std.log.scoped(.mixtral); + +pub const std_options: std.Options = .{ + .log_level = .info, + .logFn = asynk.logFn(std.log.defaultLog), +}; + +pub fn main() !void { + try asynk.AsyncThread.main(std.heap.smp_allocator, asyncMain); +} + +pub fn asyncMain() !void { + const CliArgs = struct { + pub const help = + \\ test-implementation --model-path=mixtral/ --config=config.json --reference=activation.pt + ; + model_path: []const u8, + activations: []const u8, + }; + var gpa = std.heap.GeneralPurposeAllocator(.{ .thread_safe = true }){}; + defer _ = gpa.deinit(); + const allocator = gpa.allocator(); + + // Create ZML context + var context = try zml.Context.init(); + defer context.deinit(); + + // Select platform + const platform = context.autoPlatform(.{}).withCompilationOptions(.{ + .xla_dump_to = "/tmp/zml/mixtral.test", + .sharding_enabled = false, + }); + + // Resolve model files + var args = std.process.args(); + const cli = flags.parse(&args, CliArgs); + + const model_tokenizer_path = try std.fs.path.join(allocator, &.{ cli.model_path, "tokenizer.json" }); + defer allocator.free(model_tokenizer_path); + + // Memory arena dedicated to model shapes and weights + var arena_state = std.heap.ArenaAllocator.init(allocator); + defer arena_state.deinit(); + const model_arena = arena_state.allocator(); + + // Parse model config + const config: Mixtral.Config = cfg: { + const model_config_path = try std.fs.path.join(allocator, &.{ cli.model_path, "config.json" }); + defer allocator.free(model_config_path); + var config_json_file = try asynk.File.open(model_config_path, .{ .mode = .read_only }); + defer config_json_file.close() catch unreachable; + var config_json_buffer: [256]u8 = undefined; + var config_reader = config_json_file.reader(&config_json_buffer); + var reader = std.json.Reader.init(model_arena, &config_reader.interface); + defer reader.deinit(); + const cfg = try std.json.parseFromTokenSourceLeaky(Mixtral.Config, model_arena, &reader, .{ .ignore_unknown_fields = true }); + + std.log.info("Parsed mixtral config: {}", .{cfg}); + break :cfg cfg; + }; + + // Read model shapes. + var store: zml.aio.BufferStore = st: { + const simple_path = try std.fs.path.join(allocator, &.{ cli.model_path, "model.safetensors" }); + defer allocator.free(simple_path); + + if (asynk.File.access(simple_path, .{})) { + log.info("\tLoading mixtral weights from {s}...", .{simple_path}); + break :st try zml.aio.detectFormatAndOpen(allocator, simple_path); + } else |_| {} + + const sharded_path = try std.fs.path.join(allocator, &.{ cli.model_path, "model.safetensors.index.json" }); + defer allocator.free(sharded_path); + log.info("\tLoading mixtral weights from {s}...", .{sharded_path}); + break :st try zml.aio.detectFormatAndOpen(allocator, sharded_path); + }; + defer store.deinit(); + + // Create the model and configure it. + const options: Mixtral.Options = .{ + .max_seq_len = 256, + .sampling_strategy = .{ + .topk = 1, + .temperature = 1.0, + }, + }; + const mixtral = try Mixtral.init(model_arena, store, config, options); + + // Load the weights. + var mixtral_weights = try mixtral.loadBuffers(model_arena, store, platform); + defer zml.aio.unloadBuffers(&mixtral_weights); + + // Load the activations. + var activation_store = try zml.aio.detectFormatAndOpen(allocator, cli.activations); + defer activation_store.deinit(); + for (activation_store.buffers.keys(), activation_store.buffers.values()) |k, *v| { + if (std.mem.indexOf(u8, k, ".layers.") != null and (std.mem.endsWith(u8, k, ".in.0") or std.mem.endsWith(u8, k, ".out.0"))) { + if (v.dim(0) == 1 and v.rank() == 3) { + v.* = v.squeeze(0).withTags(.{ .s, .d }); + } + } + + // log.info("Loaded activations: {s} -> {f}", .{ k, v }); + if (std.mem.indexOf(u8, k, ".22.self_attn.in.") != null) { + log.info("Self attn input {s}: {d:32.3}", .{ k, v.* }); + } + + // if (std.mem.endsWith(u8, k, "layers.23.mlp.experts.in.1")) log.info("Routing {s}: {d:32.2}", .{k, v.*}); + // if (std.mem.endsWith(u8, k, "layers.23.mlp.experts.in.2")) log.info("Routing weights {s}: {d:32.2}", .{k, v.*}); + // if (std.mem.endsWith(u8, k, ".0.mlp.experts.in.2")) log.info("Routing weights {s}: {d:32.2}", .{ k, v.* }); + } + + // Test implementation + try testImplementation(platform, mixtral, mixtral_weights, activation_store); +} + +fn testImplementation( + platform: zml.Platform, + mixtral: Mixtral, + mixtral_weights: zml.Bufferized(Mixtral), + store: zml.aio.BufferStore, +) !void { + try zml.testing.testLayer(platform, store, "model.model.embed_tokens", mixtral.model.embed_tokens, mixtral_weights.model.embed_tokens, 1e-3); + try zml.testing.testLayer(platform, store, "model.model.layers.0.self_attn.v_proj", mixtral.model.layers[0].self_attn.v_proj, mixtral_weights.model.layers[0].self_attn.v_proj, 2e-2); + try zml.testing.testLayer(platform, store, "model.model.layers.0.self_attn.q_proj", mixtral.model.layers[0].self_attn.q_proj, mixtral_weights.model.layers[0].self_attn.q_proj, 2e-2); + try zml.testing.testLayer(platform, store, "model.model.layers.0.self_attn.k_proj", mixtral.model.layers[0].self_attn.k_proj, mixtral_weights.model.layers[0].self_attn.k_proj, 2e-2); + try zml.testing.testLayer(platform, store, "model.model.layers.0.self_attn.o_proj", mixtral.model.layers[0].self_attn.o_proj, mixtral_weights.model.layers[0].self_attn.o_proj, 2e-2); + try zml.testing.testLayer(platform, store, "model.model.layers.0.input_layernorm", mixtral.model.layers[0].input_layernorm, mixtral_weights.model.layers[0].input_layernorm, 1e-3); + try zml.testing.testLayer(platform, store, "model.model.layers.0.post_attention_layernorm", mixtral.model.layers[0].post_attention_layernorm, mixtral_weights.model.layers[0].post_attention_layernorm, 1e-3); + try zml.testing.testLayer(platform, store, "model.model.layers.0.mlp.router", mixtral.model.layers[0].mlp.router, mixtral_weights.model.layers[0].mlp.router, 1e-2); + + const allocator = std.heap.smp_allocator; + { + const input: zml.HostBuffer = store.buffers.get("model.model.layers.22.self_attn.in.0").?; + const kv_shape = zml.Shape.init(.{ .layer = 1, .k = input.dim(.s), .h = 8, .hd = 64 }, .bf16); + const self_attn_mod = try zml.compileModel( + allocator, + Mixtral.SelfAttn.forward, + mixtral.model.layers[22].self_attn, + .{ + store.buffers.get("model.model.layers.22.self_attn.in.0").?.shape(), + zml.Shape.scalar(.u32), + Mixtral.KvCache.initShape(kv_shape), + }, + platform, + ); + defer self_attn_mod.deinit(); + + const self_attn_exe = self_attn_mod.prepare(mixtral_weights.model.layers[22].self_attn); + const out, _ = self_attn_exe.call(.{ + try input.toDevice(platform), + try .scalar(platform, 0, .u32), + try Mixtral.KvCache.initBuffer(kv_shape, platform), + }); + + const expected = store.buffers.get("model.model.layers.22.self_attn.out.0").?; + try zml.testing.expectClose(expected, out, 1e-2); + log.info("All good for self_attn", .{}); + } + + { + const w = mixtral.model.layers[22].mlp.experts.gate_up_proj; + const dequant_mod = try zml.compileFn(allocator, dequantize, .{.{ + .blocks = w.blocks.shape(), + .scale = w.scale.shape(), + .bias = w.bias.?.shape(), + .blocks_dtype = w.blocks_dtype, + }}, platform); + defer dequant_mod.deinit(); + + const dequant_weight = dequant_mod.call(.{mixtral_weights.model.layers[22].mlp.experts.gate_up_proj}); + const expected = store.buffers.get("model.model.layers.22.mlp.experts.gate_up_proj.bf16").?; + + try zml.testing.expectClose(expected, dequant_weight, 1e-4); + log.info("All good for dequantize gate_up_proj", .{}); + } + + { + const w = mixtral.model.layers[22].mlp.experts.down_proj; + const dequant_mod = try zml.compileFn(allocator, dequantize, .{.{ + .blocks = w.blocks.shape(), + .scale = w.scale.shape(), + .bias = w.bias.?.shape(), + .blocks_dtype = w.blocks_dtype, + }}, platform); + defer dequant_mod.deinit(); + + const dequant_weight = dequant_mod.call(.{mixtral_weights.model.layers[22].mlp.experts.down_proj}); + const expected = store.buffers.get("model.model.layers.22.mlp.experts.down_proj.bf16").?; + + try zml.testing.expectClose(expected, dequant_weight, 1e-4); + log.info("All good for dequantize down_proj", .{}); + } + + { + const moe: MoeAllToAll = .{ .experts = mixtral.model.layers[0].mlp.experts }; + + try zml.testing.testLayer(platform, store, "model.model.layers.0.mlp.experts", moe, .{ .experts = mixtral_weights.model.layers[0].mlp.experts }, 5e-2); + } + + try zml.testing.testLayer(platform, store, "model.model.layers.0.mlp", mixtral.model.layers[0].mlp, mixtral_weights.model.layers[0].mlp, 1e-2); +} + +fn dequantize(self: zml.nn.BlockScaledLinear) zml.Tensor { + // Bitcast to our actual type. This allows to load weights in a packed layout. + const blocks_0 = self.blocks.bitCast(self.blocks_dtype); + const blocks = blocks_0.merge(.{ .d_block = .{ .d_block, .bitcast } }); + + const scale = self.scale.bitCast(.f8e8m0); + + const dequantized_weight: zml.Tensor = .mul( + blocks.convert(.bf16), + scale.convert(.bf16).appendAxes(.{.d_block}), + ); + return dequantized_weight.merge(.{ .d = .{ .d, .d_block } }).contiguous(.{ .d, .out }); +} + +const MoeAllToAll = struct { + experts: Mixtral.Mlp, + + /// Fork of Mixtral.Moe where we also feed the gating. + pub fn forward(self: MoeAllToAll, input: zml.Tensor, routing: zml.Tensor, gating: zml.Tensor) zml.Tensor { + _ = routing; + + return zml.nn.mixtureOfExpertsAllToAll(Mixtral.Mlp, self.experts, input, gating.withTags(.{ .s, .expert })); + } +}; diff --git a/examples/mnist/mnist.zig b/examples/mnist/mnist.zig index 5c85f5b7..241a3a0e 100644 --- a/examples/mnist/mnist.zig +++ b/examples/mnist/mnist.zig @@ -27,7 +27,7 @@ const Mnist = struct { /// just two linear layers + relu activation pub fn forward(self: Mnist, input: zml.Tensor) zml.Tensor { // std.log.info("Compiling for target: {s}", .{@tagName(input.getContext().target())}); - var x = input.flattenAll().convert(.f32); + var x = input.flatten().convert(.f32); const layers: []const Layer = &.{ self.fc1, self.fc2 }; for (layers) |layer| { x = layer.forward(x); diff --git a/examples/modernbert/modernbert.zig b/examples/modernbert/modernbert.zig index 79585f14..7894d5a8 100644 --- a/examples/modernbert/modernbert.zig +++ b/examples/modernbert/modernbert.zig @@ -43,7 +43,7 @@ pub const ModernBertForMaskedLM = struct { const biased_logits = logits.add(self.decoder.bias.withTags(.{.voc}).broad(logits.shape())); const probabilities = biased_logits.softmax(.voc); - return probabilities.topK(5, .voc, .{ .descending = true }); + return probabilities.topK(.{ .best_token = .voc }, 5, .{ .descending = true }); } }; diff --git a/examples/simple_layer/main.zig b/examples/simple_layer/main.zig index a84ab056..1621fd6b 100644 --- a/examples/simple_layer/main.zig +++ b/examples/simple_layer/main.zig @@ -1,3 +1,8 @@ +//! This examples show how to specify a model, +//! compile it, and execute it with specific weights. +//! +//! This code is very explicit and doesn't use zml.aio helper functions +//! in order to show what is happening behind the scene. const std = @import("std"); const asynk = @import("async"); @@ -17,81 +22,82 @@ const Layer = struct { } }; +pub const std_options: std.Options = .{ + .log_level = .info, + .logFn = asynk.logFn(std.log.defaultLog), +}; + pub fn main() !void { - try asynk.AsyncThread.main(std.heap.c_allocator, asyncMain); + try asynk.AsyncThread.main(std.heap.smp_allocator, asyncMain); } pub fn asyncMain() !void { - // Short lived allocations - var gpa = std.heap.GeneralPurposeAllocator(.{}){}; - defer _ = gpa.deinit(); - const allocator = gpa.allocator(); - - // Arena allocator for BufferStore etc. - var arena_state = std.heap.ArenaAllocator.init(allocator); - defer arena_state.deinit(); - const arena = arena_state.allocator(); + var gpa_state = std.heap.DebugAllocator(.{}){}; + defer _ = gpa_state.deinit(); + const gpa = gpa_state.allocator(); + // Start ZML, detect available devices and chose a platform. var context = try zml.Context.init(); defer context.deinit(); - const platform = context.autoPlatform(.{}); context.printAvailablePlatforms(platform); - // Our weights and bias to use - var weights = [4]f32{ 2.0, 2.0, 2.0, 2.0 }; - var bias = [4]f32{ 1.0, 2.0, 3.0, 4.0 }; - const input_shape = zml.Shape.init(.{4}, .f32); - - // We manually produce a BufferStore. You would not normally do that. - // A BufferStore is usually created by loading model data from a file. - var buffers: zml.aio.BufferStore.Buffers = .{}; - try buffers.put(arena, "weight", zml.HostBuffer.fromArray(&weights)); - try buffers.put(arena, "bias", zml.HostBuffer.fromArray(&bias)); - - // the actual BufferStore - const buffer_store: zml.aio.BufferStore = .{ - .arena = arena_state, - .buffers = buffers, + // Create a skeleton for our model with only the shapes. + // The `buffer_id` are typically not set manually + // but instead set by a zml.BufferStore + // to map abstract Tensors to actual part of the model weight file. + const model_shapes: Layer = .{ + .bias = zml.Tensor{ + ._shape = zml.Shape.init(.{4}, .f16).withSharding(.{-1}), + ._id = .{ .buffer_id = 0 }, + }, + .weight = zml.Tensor{ + ._shape = zml.Shape.init(.{4}, .f16).withSharding(.{-1}), + ._id = .{ .buffer_id = 1 }, + }, }; - // A clone of our model, consisting of shapes. We only need shapes for compiling. - // We use the BufferStore to infer the shapes. - var model_shapes = try zml.aio.populateModel(Layer, allocator, buffer_store); - model_shapes.weight = model_shapes.weight.withSharding(.{-1}); - model_shapes.bias = model_shapes.bias.?.withSharding(.{-1}); + // Start compilation in a different thread. + // We already specified the shape of the model weights, + // but we still need to specifiy the shape of the input tensor. + const input_shape = zml.Shape.init(.{4}, .f16); + var compilation = try asynk.asyncc( + zml.compileModel, + .{ gpa, Layer.forward, model_shapes, .{input_shape}, platform }, + ); - // Start compiling. This uses the inferred shapes from the BufferStore. - // The shape of the input tensor, we have to pass in manually. - var compilation = try asynk.asyncc(zml.compileModel, .{ allocator, Layer.forward, model_shapes, .{input_shape}, platform }); + // Now we need to create a model instance with actual weights. + const weights = [4]f16{ 2.0, -2.0, 1.0, -1.0 }; + const bias = [4]f16{ 1.0, 2.0, 3.0, 4.0 }; - // Produce a bufferized weights struct from the fake BufferStore. - // This is like the inferred shapes, but with actual values. - // We will need to send those to the computation device later. - var model_weights = try zml.aio.loadModelBuffers(Layer, model_shapes, buffer_store, arena, platform); - defer zml.aio.unloadBuffers(&model_weights); // for good practice + var model_weights: zml.Bufferized(Layer) = .{ + .bias = try zml.Buffer.fromSlice(platform, .{4}, &bias), + .weight = try zml.Buffer.fromSlice(platform, .{4}, &weights), + }; + defer zml.aio.unloadBuffers(&model_weights); // Wait for compilation to finish const compiled = try compilation.awaitt(); + defer compiled.deinit(); - // pass the model weights to the compiled module to create an executable module - var executable = compiled.prepare(model_weights); - defer executable.deinit(); + // pass the model weights to the compiled module to create an executable module. + // This is where the shapes of the buffers will be compared to the + // shape expected by the executable. + // Note: we don't call `executable.deinit()` since it uses the same memory than `compiled`. + const executable = compiled.prepare(model_weights); - // prepare an input buffer - // Here, we use zml.HostBuffer.fromSlice to show how you would create a HostBuffer - // with a specific shape from an array. - // For situations where e.g. you have an [4]f32 array but need a .{2, 2} input shape. - var input = [4]f32{ 5.0, 5.0, 5.0, 5.0 }; - var input_buffer = try zml.Buffer.from(platform, zml.HostBuffer.fromSlice(input_shape, &input), .{}); + // prepare the input buffer + const input = [4]f16{ 5.0, 5.0, 5.0, 5.0 }; + var input_buffer = try zml.Buffer.fromSlice(platform, input_shape, &input); defer input_buffer.deinit(); - // call our executable module + // call our executable module, the result is still on the device. var result: zml.Buffer = executable.call(.{input_buffer}); defer result.deinit(); - // fetch the result to CPU memory - const cpu_result = try result.toHostAlloc(arena); + // copy the result to CPU memory + const cpu_result: zml.HostBuffer = try result.toHostAlloc(gpa); + defer cpu_result.deinit(gpa); std.debug.print( "\nThe result of {any} * {any} + {any} = {any}\n", .{ &weights, &input, &bias, cpu_result.items(f32) }, diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index 0c60fd01..e06a5617 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -296,21 +296,23 @@ pub fn select(ctx: mlir.Context, condition: mlir.Value, then: mlir.Value, else_: }); } +pub const GatherArgs = struct { + offset_dims: []const i64, + collapsed_slice_dims: []const i64, + operand_batching_dims: []const i64, + start_indices_batching_dims: []const i64, + start_index_map: []const i64, + index_vector_dim: i64, + indices_are_sorted: bool = false, +}; + pub fn gather( ctx: mlir.Context, value: mlir.Value, indices: mlir.Value, slice_sizes: []const i64, location: mlir.Location, - args: struct { - offset_dims: []const i64, - collapsed_slice_dims: []const i64, - operand_batching_dims: []const i64, - start_indices_batching_dims: []const i64, - start_index_map: []const i64, - index_vector_dim: i64, - indices_are_sorted: bool = false, - }, + args: GatherArgs, ) mlir.Operation { return mlir.Operation.make( ctx, @@ -742,6 +744,7 @@ pub const CustomCallOpts = struct { api_version: ApiVersion, }; +// TODO move the call_target_name to second argument pub fn custom_call(ctx: mlir.Context, inputs: []const mlir.Value, opts: CustomCallOpts, res_types: []const mlir.Type, location: mlir.Location) mlir.Operation { const MAX_OPERANDS = 64; const MAX_RESULTS = 16; diff --git a/mlir/mlir.zig b/mlir/mlir.zig index a6721914..8dc63606 100755 --- a/mlir/mlir.zig +++ b/mlir/mlir.zig @@ -416,9 +416,9 @@ pub const ArrayAttribute = struct { pub fn IntegerAttribute(comptime it: IntegerTypes) type { const ZigType, const getter = comptime switch (it) { - .i1, .i4, .i8, .i16, .i32, .i64 => .{ i64, c.mlirIntegerAttrGetValueInt }, + .i1, .i2, .i4, .i8, .i16, .i32, .i64 => .{ i64, c.mlirIntegerAttrGetValueInt }, .si4, .si8, .si16, .si32, .si64 => .{ i64, c.mlirIntegerAttrGetValueSInt }, - .u4, .u8, .u16, .u32, .u64 => .{ u64, c.mlirIntegerAttrGetValueUInt }, + .u2, .u4, .u8, .u16, .u32, .u64 => .{ u64, c.mlirIntegerAttrGetValueUInt }, .unknown => @compileError("IntegerAttribute(unknown)"), }; @@ -1289,6 +1289,7 @@ pub const IndexType = struct { pub const IntegerTypes = enum { i1, + i2, i4, i8, i16, @@ -1299,6 +1300,7 @@ pub const IntegerTypes = enum { si16, si32, si64, + u2, u4, u8, u16, @@ -1311,6 +1313,7 @@ pub const IntegerTypes = enum { pub fn IntegerType(comptime it: IntegerTypes) type { const Config = switch (it) { .i1 => .{ 1, c.mlirIntegerTypeGet, c.mlirIntegerTypeIsSignless }, + .i2 => .{ 2, c.mlirIntegerTypeGet, c.mlirIntegerTypeIsSignless }, .i4 => .{ 4, c.mlirIntegerTypeGet, c.mlirIntegerTypeIsSignless }, .i8 => .{ 8, c.mlirIntegerTypeGet, c.mlirIntegerTypeIsSignless }, .i16 => .{ 16, c.mlirIntegerTypeGet, c.mlirIntegerTypeIsSignless }, @@ -1321,6 +1324,7 @@ pub fn IntegerType(comptime it: IntegerTypes) type { .si16 => .{ 16, c.mlirIntegerTypeSignedGet, c.mlirIntegerTypeIsSigned }, .si32 => .{ 32, c.mlirIntegerTypeSignedGet, c.mlirIntegerTypeIsSigned }, .si64 => .{ 64, c.mlirIntegerTypeSignedGet, c.mlirIntegerTypeIsSigned }, + .u2 => .{ 2, c.mlirIntegerTypeUnsignedGet, c.mlirIntegerTypeIsUnsigned }, .u4 => .{ 4, c.mlirIntegerTypeUnsignedGet, c.mlirIntegerTypeIsUnsigned }, .u8 => .{ 8, c.mlirIntegerTypeUnsignedGet, c.mlirIntegerTypeIsUnsigned }, .u16 => .{ 16, c.mlirIntegerTypeUnsignedGet, c.mlirIntegerTypeIsUnsigned }, @@ -1360,11 +1364,15 @@ pub fn IntegerType(comptime it: IntegerTypes) type { } pub const FloatTypes = enum { + f4e2m1fn, + f8e3m4, + f8e4m3, f8e4m3b11fnuz, f8e4m3fn, f8e4m3fnuz, f8e5m2, f8e5m2fnuz, + f8e8m0fnu, bf16, f16, f32, @@ -1379,11 +1387,15 @@ pub const FloatTypes = enum { pub fn FloatType(comptime ft: FloatTypes) type { const Config = switch (ft) { + .f4e2m1fn => .{ c.mlirTypeIsAFloat4E2M1FN, c.mlirFloat4E2M1FNTypeGet }, + .f8e3m4 => .{ c.mlirTypeIsAFloat8E3M4, c.mlirFloat8E3M4TypeGet }, + .f8e4m3 => .{ c.mlirTypeIsAFloat8E4M3, c.mlirFloat8E4M3TypeGet }, .f8e4m3b11fnuz => .{ c.mlirTypeIsAFloat8E4M3B11FNUZ, c.mlirFloat8E4M3B11FNUZTypeGet }, .f8e4m3fn => .{ c.mlirTypeIsAFloat8E4M3FN, c.mlirFloat8E4M3FNTypeGet }, .f8e4m3fnuz => .{ c.mlirTypeIsAFloat8E4M3FNUZ, c.mlirFloat8E4M3FNUZTypeGet }, .f8e5m2 => .{ c.mlirTypeIsAFloat8E5M2, c.mlirFloat8E5M2TypeGet }, .f8e5m2fnuz => .{ c.mlirTypeIsAFloat8E5M2FNUZ, c.mlirFloat8E5M2FNUZTypeGet }, + .f8e8m0fnu => .{ c.mlirTypeIsAFloat8E8M0FNU, c.mlirFloat8E8M0FNUTypeGet }, .bf16 => .{ c.mlirTypeIsABF16, c.mlirBF16TypeGet }, .f16 => .{ c.mlirTypeIsAF16, c.mlirF16TypeGet }, .f32 => .{ c.mlirTypeIsAF32, c.mlirF32TypeGet }, diff --git a/pjrt/ffi.zig b/pjrt/ffi.zig index d2517a04..2ca639a7 100644 --- a/pjrt/ffi.zig +++ b/pjrt/ffi.zig @@ -2,12 +2,21 @@ const std = @import("std"); const c = @import("c"); +pub const TypeId = c.XLA_FFI_TypeId; const stdx = @import("stdx"); -const pjrtStruct = @import("pjrt.zig").pjrtStruct; +const pjrt = @import("pjrt.zig"); +const Stream = @import("pjrt.zig").Stream; const log = std.log.scoped(.pjrt); +comptime { + if (@typeInfo(TypeId).@"struct".fields.len != 1) @compileError("TypeId has changed"); +} + +/// The signature of a generic custom call. +pub const Handler = fn (*CallFrame) callconv(.c) ?*Error; + pub const ApiVersion = extern struct { pub const major = c.XLA_FFI_API_MAJOR; pub const minor = c.XLA_FFI_API_MINOR; @@ -29,13 +38,13 @@ pub const ExtensionBase = extern struct { }; // Based of https://github.com/openxla/xla/blob/145f836bd5175dc5dd262f716a0c59af2b0297a0/xla/ffi/api/c_api.h#L449 -pub const HandlerTraits = packed struct(u32) { +pub const HandlerTraits = packed struct(c_uint) { /// Calls to FFI handler are safe to trace into the command buffer. /// It means that calls to FFI handler always launch exactly the same device operations (can depend on attribute values) /// that can be captured and then replayed. - command_buffer_compatible: u1, + command_buffer_compatible: bool, - __unassigned__: u31, + __unassigned__: u31 = 0, }; pub const Metadata = extern struct { @@ -49,25 +58,6 @@ pub const MetadataExtension = extern struct { metadata: ?*Metadata, }; -pub const ApiError = error{ - Cancelled, - Unknown, - InvalidArgument, - DeadlineExceeded, - NotFound, - AlreadyExists, - PermissionDenied, - ResourceExhausted, - FailedPrecondition, - Aborted, - OutOfRange, - Unimplemented, - Internal, - Unavailable, - DataLoss, - Unauthenticated, -}; - fn TransmuteMixin(comptime T: type, comptime InnerT: type) type { return struct { pub fn to(self: anytype) switch (@TypeOf(self)) { @@ -91,8 +81,8 @@ fn TransmuteMixin(comptime T: type, comptime InnerT: type) type { pub const Api = opaque { pub const inner = TransmuteMixin(Api, c.XLA_FFI_Api).to; - pub fn stream(self: *const Api, context: *const ExecutionContext) *Stream { - var ret = pjrtStruct(c.XLA_FFI_Stream_Get_Args{ + pub fn stream(self: *const Api, context: *const ExecutionContext) *pjrt.Stream { + var ret = pjrt.pjrtStruct(c.XLA_FFI_Stream_Get_Args{ .ctx = @constCast(context.inner()), }); const result = self.inner().XLA_FFI_Stream_Get.?(&ret); @@ -107,8 +97,8 @@ pub const Api = opaque { return @ptrCast(ret.stream.?); } - pub fn allocateDeviceMemory(self: *const Api, context: *const ExecutionContext, size: usize, alignment: usize) ApiError!*anyopaque { - var ret = pjrtStruct(c.XLA_FFI_DeviceMemory_Allocate_Args{ + pub fn allocateDeviceMemory(self: *const Api, context: *const ExecutionContext, size: usize, alignment: usize) pjrt.ApiError!*anyopaque { + var ret = pjrt.pjrtStruct(c.XLA_FFI_DeviceMemory_Allocate_Args{ .ctx = @constCast(context.inner()), .size = size, .alignment = alignment, @@ -127,8 +117,8 @@ pub const Api = opaque { return ret.data.?; } - pub fn freeDeviceMemory(self: *const Api, context: *const ExecutionContext, data: *anyopaque, size: usize) ApiError!void { - var ret = pjrtStruct(c.XLA_FFI_DeviceMemory_Free_Args{ + pub fn freeDeviceMemory(self: *const Api, context: *const ExecutionContext, data: *anyopaque, size: usize) pjrt.ApiError!void { + var ret = pjrt.pjrtStruct(c.XLA_FFI_DeviceMemory_Free_Args{ .ctx = @constCast(context.inner()), .size = size, .data = data, @@ -163,33 +153,31 @@ pub const ExecutionStage = enum(c.XLA_FFI_ExecutionStage) { pub const ExecutionContext = opaque { pub const inner = TransmuteMixin(ExecutionContext, c.XLA_FFI_ExecutionContext).to; - pub fn Context(comptime T: type) type { - return struct { - pub fn get(self: *const ExecutionContext, api: *const Api) ApiError!*T { - const type_id: TypeId = .{ .type_id = T.type_id }; - var ret = pjrtStruct(c.XLA_FFI_ExecutionContext_Get_Args{ - .ctx = @constCast(self.inner()), - .type_id = @constCast(&type_id.toCStruct()), - }); - const result = api.inner().XLA_FFI_ExecutionContext_Get.?(&ret); - - if (result) |ffi_error| { - const err = Error.fromInner(ffi_error); - defer err.destroy(api); - log.err("[ExecutionContext.get] {s}", .{err.getMessage(api)}); - - // TODO(Corentin): Retrieve error code from Error when implemented in XLA. - return error.Unknown; - } - - if (ret.data == null) return error.NotFound; - return @ptrCast(@alignCast(ret.data.?)); - } + pub fn getContext(self: *const ExecutionContext, type_id: TypeId, api: *const Api) pjrt.ApiError!*anyopaque { + var ret: c.XLA_FFI_ExecutionContext_Get_Args = .{ + .struct_size = pjrt.pjrtStructSize(c.XLA_FFI_ExecutionContext_Get_Args), + .extension_start = api.inner().extension_start, + .ctx = @ptrCast(@constCast(self)), + .type_id = @constCast(&type_id), + .data = undefined, // set by XLA_FFI_ExecutionContext_Get. }; + const maybe_err = api.inner().XLA_FFI_ExecutionContext_Get.?(&ret); + + if (maybe_err) |ffi_error| { + const err = Error.fromInner(ffi_error); + defer err.destroy(api); + log.err("[ExecutionContext.get] {s}", .{err.getMessage(api)}); + + // TODO(Corentin): Retrieve error code from Error when implemented in XLA. + return error.Unknown; + } + + if (ret.data == null) return error.NotFound; + return ret.data.?; } - pub fn getDeviceOrdinal(self: *const ExecutionContext, api: *const Api) ApiError!i32 { - var ret = pjrtStruct(c.XLA_FFI_DeviceOrdinal_Get_Args{ + pub fn getDeviceOrdinal(self: *const ExecutionContext, api: *const Api) pjrt.ApiError!i32 { + var ret = pjrt.pjrtStruct(c.XLA_FFI_DeviceOrdinal_Get_Args{ .ctx = @constCast(self.inner()), }); const result = api.inner().XLA_FFI_DeviceOrdinal_Get.?(&ret); @@ -206,8 +194,10 @@ pub const ExecutionContext = opaque { return ret.device_ordinal; } - pub fn scheduleTask(self: *const ExecutionContext, api: *const Api, task: *const Task, data: *anyopaque) ApiError!void { - var ret = pjrtStruct(c.XLA_FFI_ThreadPool_Schedule_Args{ + const Task = fn (*anyopaque) void; + + pub fn scheduleTask(self: *const ExecutionContext, api: *const Api, task: *const Task, data: *anyopaque) pjrt.ApiError!void { + var ret = pjrt.pjrtStruct(c.XLA_FFI_ThreadPool_Schedule_Args{ .ctx = @constCast(self.inner()), .task = @ptrCast(@alignCast(task)), .data = @ptrCast(@alignCast(data)), @@ -225,23 +215,9 @@ pub const ExecutionContext = opaque { return error.Unknown; } } - - fn getTypeId(type_name: []const u8) TypeId { - const id: i64 = @bitCast(std.hash.Fnv1a_64.hash(type_name)); - - return .{ - .type_id = id, - }; - } }; -const TypeId = c.XLA_FFI_TypeId; - -const Task = fn (*anyopaque) void; - -const Stream = @import("pjrt.zig").Stream; - -const ByteSpan = extern struct { +pub const ByteSpan = extern struct { ptr: [*]const u8, len: usize, @@ -252,7 +228,7 @@ const ByteSpan = extern struct { pub const DataType = enum(c.XLA_FFI_DataType) { invalid = c.XLA_FFI_DataType_INVALID, - pred = c.XLA_FFI_DataType_PRED, + bool = c.XLA_FFI_DataType_PRED, i8 = c.XLA_FFI_DataType_S8, i16 = c.XLA_FFI_DataType_S16, i32 = c.XLA_FFI_DataType_S32, @@ -399,6 +375,8 @@ pub const Attrs = extern struct { } }; +/// All informations needed by the user callback, +/// including the list of input/ouput buffers to work on. pub const CallFrame = extern struct { struct_size: usize, extension_start: ?*ExtensionBase, @@ -422,9 +400,11 @@ pub const CallFrame = extern struct { } return false; } -}; -pub const Handler = fn (*CallFrame) callconv(.c) ?*Error; + pub fn stream(call_frame: CallFrame) ?*const pjrt.Stream { + return call_frame.api.stream(call_frame.ctx); + } +}; pub const ErrorCode = enum(c.XLA_FFI_Error_Code) { cancelled = c.XLA_FFI_Error_Code_CANCELLED, @@ -444,7 +424,7 @@ pub const ErrorCode = enum(c.XLA_FFI_Error_Code) { data_loss = c.XLA_FFI_Error_Code_DATA_LOSS, unauthenticated = c.XLA_FFI_Error_Code_UNAUTHENTICATED, - pub fn toApiError(code: ErrorCode) ApiError { + pub fn toApiError(code: ErrorCode) pjrt.ApiError { return switch (code) { .cancelled => error.Cancelled, .unknown => error.Unknown, @@ -470,8 +450,10 @@ pub const Error = opaque { pub const inner = TransmuteMixin(Error, c.XLA_FFI_Error).to; pub const fromInner = TransmuteMixin(Error, c.XLA_FFI_Error).from; + pub const ok: ?*Error = null; + pub fn create(api: *const Api, error_code: ErrorCode, message: []const u8) *Error { - var ret = pjrtStruct(c.XLA_FFI_Error_Create_Args{ + var ret = pjrt.pjrtStruct(c.XLA_FFI_Error_Create_Args{ .message = message.ptr, .errc = @intFromEnum(error_code), }); @@ -479,12 +461,12 @@ pub const Error = opaque { } pub fn destroy(err: *Error, api: *const Api) void { - var ret = pjrtStruct(c.XLA_FFI_Error_Destroy_Args{ .@"error" = err.inner() }); + var ret = pjrt.pjrtStruct(c.XLA_FFI_Error_Destroy_Args{ .@"error" = err.inner() }); api.inner().XLA_FFI_Error_Destroy.?(&ret); } pub fn getMessage(err: *Error, api: *const Api) [:0]const u8 { - var ret = pjrtStruct(c.XLA_FFI_Error_GetMessage_Args{ + var ret = pjrt.pjrtStruct(c.XLA_FFI_Error_GetMessage_Args{ .@"error" = err.inner(), }); api.inner().XLA_FFI_Error_GetMessage.?(&ret); @@ -496,8 +478,8 @@ pub const Future = opaque { pub const inner = TransmuteMixin(Future, c.XLA_FFI_Future).to; pub const fromInner = TransmuteMixin(Future, c.XLA_FFI_Future).from; - pub fn create(api: *const Api) ApiError!*Future { - var ret = pjrtStruct(c.XLA_FFI_Future_Create_Args{}); + pub fn create(api: *const Api) pjrt.ApiError!*Future { + var ret = pjrt.pjrtStruct(c.XLA_FFI_Future_Create_Args{}); const result = api.inner().XLA_FFI_Future_Create.?(&ret); if (result) |ffi_error| { @@ -512,8 +494,8 @@ pub const Future = opaque { return fromInner(ret.future.?); } - pub fn setAvailable(self: *Future, api: *const Api) ApiError!void { - var ret = pjrtStruct(c.XLA_FFI_Future_SetAvailable_Args{ + pub fn setAvailable(self: *Future, api: *const Api) pjrt.ApiError!void { + var ret = pjrt.pjrtStruct(c.XLA_FFI_Future_SetAvailable_Args{ .future = self.inner(), }); @@ -529,8 +511,8 @@ pub const Future = opaque { } } - pub fn setError(self: *Future, api: *const Api, err: *Error) ApiError!void { - var ret = pjrtStruct(c.XLA_FFI_Future_SetError_Args{ + pub fn setError(self: *Future, api: *const Api, err: *Error) pjrt.ApiError!void { + var ret = pjrt.pjrtStruct(c.XLA_FFI_Future_SetError_Args{ .future = self.inner(), .@"error" = err.inner(), }); diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 6e358562..e57b4575 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -20,7 +20,7 @@ test { // as the way PJRT does it is not very robust. // // 1. https://github.com/openxla/xla/issues/10032 -fn pjrtStructSize(comptime T: type) usize { +pub fn pjrtStructSize(comptime T: type) usize { // unsafe on purpose, we want this to fail if that ever changes const typedef_name = comptime blk: { const needle = ".struct_"; @@ -164,7 +164,7 @@ pub const Api = struct { return @ptrCast(ret.context.?); } - pub fn ffi(api: *const Api) ?FFI { + pub fn ffi(api: *const Api) ?Ffi { if (api.lookupExtension(c.PJRT_FFI_Extension, c.PJRT_Extension_Type_FFI)) |ext| { return .{ .inner = ext }; } @@ -793,11 +793,13 @@ pub const LoadedExecutable = opaque { pub const BufferType = enum(c.PJRT_Buffer_Type) { invalid = c.PJRT_Buffer_Type_INVALID, bool = c.PJRT_Buffer_Type_PRED, + i2 = c.PJRT_Buffer_Type_S2, i4 = c.PJRT_Buffer_Type_S4, i8 = c.PJRT_Buffer_Type_S8, i16 = c.PJRT_Buffer_Type_S16, i32 = c.PJRT_Buffer_Type_S32, i64 = c.PJRT_Buffer_Type_S64, + u2 = c.PJRT_Buffer_Type_U2, u4 = c.PJRT_Buffer_Type_U4, u8 = c.PJRT_Buffer_Type_U8, u16 = c.PJRT_Buffer_Type_U16, @@ -814,6 +816,10 @@ pub const BufferType = enum(c.PJRT_Buffer_Type) { f8e4m3b11fnuz = c.PJRT_Buffer_Type_F8E4M3B11FNUZ, f8e5m2fnuz = c.PJRT_Buffer_Type_F8E5M2FNUZ, f8e4m3fnuz = c.PJRT_Buffer_Type_F8E4M3FNUZ, + f8e4m3 = c.PJRT_Buffer_Type_F8E4M3, + f8e3m4 = c.PJRT_Buffer_Type_F8E3M4, + f8e8m0 = c.PJRT_Buffer_Type_F8E8M0FNU, + f4e2m1 = c.PJRT_Buffer_Type_F4E2M1FN, }; pub const MemoryLayoutType = enum(c.PJRT_Buffer_MemoryLayout_Type) { @@ -1278,7 +1284,7 @@ pub const NamedValue = extern struct { } }; -pub const FFI = extern struct { +pub const Ffi = extern struct { inner: *const c.PJRT_FFI, pub const UserData = extern struct { @@ -1293,19 +1299,15 @@ pub const FFI = extern struct { } }; - pub const RegisterFfiOptions = struct { - traits: RegisterHandlerTraits = @enumFromInt(0), - }; - // todo : support all missing handlers available in GPU plugin extension: handler_instantiate, handler_prepare, handler_initialize // introduced by https://github.com/openxla/xla/commit/ef85a7bcc308313492ebc50295a8a08b4e51b8f5 pub fn register( - self: *const FFI, + self: *const Ffi, api: *const Api, target_name: []const u8, platform_name: []const u8, func: *const ffi.Handler, - options: RegisterFfiOptions, + traits: ffi.HandlerTraits, ) ApiError!void { var ret = pjrtStruct(c.PJRT_FFI_Register_Handler_Args{ .target_name = target_name.ptr, @@ -1313,7 +1315,7 @@ pub const FFI = extern struct { .handler = @ptrCast(@constCast(func)), .platform_name = platform_name.ptr, .platform_name_size = platform_name.len, - .traits = @intFromEnum(options.traits), + .traits = @bitCast(traits), }); const result = self.inner.register_handler.?(&ret); if (result) |pjrt_c_error| { @@ -1323,8 +1325,7 @@ pub const FFI = extern struct { } } - pub fn registerTypeId(self: *const FFI, api: *const Api, T: type) ApiError!void { - const type_name = @typeName(T); + pub fn registerTypeId(self: *const Ffi, api: *const Api, type_name: []const u8) ApiError!ffi.TypeId { var ret = pjrtStruct(c.PJRT_FFI_TypeID_Register_Args{ .type_name = type_name.ptr, .type_name_size = type_name.len, @@ -1336,10 +1337,10 @@ pub const FFI = extern struct { return pjrt_error.getCode(api).toApiError(); } - T.type_id = ret.type_id; + return .{ .type_id = ret.type_id }; } - pub fn addUserData(self: *const FFI, api: *const Api, context: *ExecuteContext, user_data: UserData) ApiError!void { + pub fn addUserData(self: *const Ffi, api: *const Api, context: *ExecuteContext, user_data: UserData) ApiError!void { var ret = pjrtStruct(c.PJRT_FFI_UserData_Add_Args{ .context = @ptrCast(context), .user_data = user_data.toCStruct(), @@ -1352,12 +1353,3 @@ pub const FFI = extern struct { } } }; - -pub const RegisterHandlerTraits = enum(c.PJRT_FFI_Handler_TraitsBits) { - command_buffer_compatible = c.PJRT_FFI_HANDLER_TRAITS_COMMAND_BUFFER_COMPATIBLE, - _, -}; - -pub const CustomCallRegistry = extern struct { - inner: *const c.PJRT_FFI_Register_Handler, -}; diff --git a/runtimes/cuda/BUILD.bazel b/runtimes/cuda/BUILD.bazel index 96cc5879..83109cf0 100644 --- a/runtimes/cuda/BUILD.bazel +++ b/runtimes/cuda/BUILD.bazel @@ -5,7 +5,7 @@ zig_shared_library( name = "zmlxcuda", # Use Clang's compiler-rt, but disable stack checking # to avoid requiring on the _zig_probe_stack symbol. - copts = ["-fno-stack-check"], + copts = ["-fno-stack-check", "-fllvm"], main = "zmlxcuda.zig", shared_lib_name = "libzmlxcuda.so.0", visibility = ["@libpjrt_cuda//:__subpackages__"], diff --git a/stdx/fmt.zig b/stdx/fmt.zig index 259b3cd8..8662bfe2 100644 --- a/stdx/fmt.zig +++ b/stdx/fmt.zig @@ -1,145 +1,119 @@ const std = @import("std"); -pub const Fmt = union(enum) { - int: IntFmt, - float: FloatFmt, - generic: void, +pub fn slice(T: type, any_slice: []const T) FmtSlice(T) { + return .{ .slice = any_slice }; +} - pub fn parse(T: type, comptime fmt_: []const u8) Fmt { - return switch (@typeInfo(T)) { - .float, .comptime_float => .{ .float = FloatFmt.parseComptime(fmt_) }, - .int, .comptime_int => .{ .int = IntFmt.parseComptime(fmt_) }, - else => .{ .generic = {} }, - }; - } -}; +fn FmtSlice(T: type) type { + return struct { + slice: []const T, -pub const FullFormatOptions = struct { - fmt: Fmt, - options: std.fmt.FormatOptions, -}; - -pub const IntFmt = struct { - base: u8, - case: std.fmt.Case = .lower, - - pub fn parseComptime(comptime fmt_: []const u8) IntFmt { - return parse(fmt_) catch @panic("invalid fmt for int: " ++ fmt_); - } - - pub fn parse(fmt_: []const u8) error{InvalidArgument}!IntFmt { - return if (fmt_.len == 0 or std.mem.eql(u8, fmt_, "d")) - .{ .base = 10, .case = .lower } - else if (std.mem.eql(u8, fmt_, "x")) - .{ .base = 16, .case = .lower } - else if (std.mem.eql(u8, fmt_, "X")) - .{ .base = 16, .case = .upper } - else if (std.mem.eql(u8, fmt_, "o")) - .{ .base = 8, .case = .upper } - else - // TODO: unicode/ascii - error.InvalidArgument; - } -}; - -pub const FloatFmt = enum(u8) { - scientific = @intFromEnum(std.fmt.Number.Mode.scientific), - decimal = @intFromEnum(std.fmt.Number.Mode.decimal), - hex, - - pub fn parseComptime(comptime fmt_: []const u8) FloatFmt { - return parse(fmt_) catch @panic("invalid fmt for float: " ++ fmt_); - } - - pub fn parse(fmt_: []const u8) error{InvalidArgument}!FloatFmt { - return if (fmt_.len == 0 or std.mem.eql(u8, fmt_, "e")) - .scientific - else if (std.mem.eql(u8, fmt_, "d")) - .decimal - else if (std.mem.eql(u8, fmt_, "x")) - .hex - else - error.InvalidArgument; - } -}; - -pub fn formatValue(value: anytype, full: FullFormatOptions, writer: anytype) !void { - return switch (@typeInfo(@TypeOf(value))) { - .comptime_float, .float => try formatFloatValue(value, full, writer), - .comptime_int, .int => try formatIntValue(value, full, writer), - else => try formatAnyValue(value, full, writer), + pub fn formatNumber(f: @This(), writer: *std.io.Writer, n: std.fmt.Number) std.io.Writer.Error!void { + return switch (@typeInfo(T)) { + .comptime_float, .float => try formatFloatSlice(f.slice, n, 1, writer), + .comptime_int, .int => try formatIntSlice(f.slice, n, 1, writer), + .bool => try formatBoolSlice(f.slice, n, 1, writer), + .@"struct" => if (@hasField(T, "re") and @hasField(T, "im")) { + try formatComplexSlice(f.slice, n, 1, writer); + } else if (@hasDecl(T, "toF32")) { + try formatFloatSlice(f.slice, n, 1, writer); + } else { + try formatSliceAny(f.slice, n, 1, writer); + }, + else => @compileError("FmtSlice doesn't support type: " ++ @typeName(T)), + }; + } }; } -pub fn formatFloatValue(value: anytype, full: FullFormatOptions, writer: *std.Io.Writer) !void { +pub fn formatFloat(value: anytype, spec: std.fmt.Number, writer: *std.Io.Writer) !void { const x = switch (@typeInfo(@TypeOf(value))) { .@"struct" => value.toF32(), .float => value, - else => @compileError("formatFloatValue expects a float, got: " ++ @typeName(@TypeOf(value))), - }; - try switch (full.fmt.float) { - .scientific => writer.printFloat(x, .{ .mode = .scientific, .precision = full.options.precision }), - .decimal => writer.printFloat(x, .{ .mode = .decimal, .precision = full.options.precision }), - .hex => writer.printFloatHexOptions(x, .{ .mode = .hex }), + else => @compileError("formatFloat expects a float, got: " ++ @typeName(@TypeOf(value))), }; + return writer.printFloat(x, spec); } -pub fn formatIntValue(value: anytype, full: FullFormatOptions, writer: *std.Io.Writer) !void { +pub fn formatInt(value: anytype, spec: std.fmt.Number, writer: *std.Io.Writer) !void { switch (@typeInfo(@TypeOf(value))) { .int => {}, - else => @compileError("formatIntValue expects an int, got: " ++ @typeName(@TypeOf(value))), + else => @compileError("formatInt expects an int, got: " ++ @typeName(@TypeOf(value))), } - return writer.printInt(value, full.fmt.int.base, full.fmt.int.case, full.options); + return writer.printInt(value, spec.mode.base().?, spec.case, .{ .alignment = spec.alignment, .fill = spec.fill }); } -pub fn formatAnyValue(value: anytype, full: FullFormatOptions, writer: *std.Io.Writer) !void { +pub fn formatComplex(value: anytype, spec: std.fmt.Number, writer: *std.Io.Writer) !void { + try writer.writeAll(".{.re="); + try writer.printFloat(value.re, spec); + try writer.writeAll(", .im="); + try writer.printFloat(value.im, spec); + try writer.writeAll("}"); +} + +pub fn formatBool(value: bool, spec: std.fmt.Number, writer: *std.Io.Writer) !void { + try writer.alignBufferOptions(if (value) "1" else "0", .{ .alignment = spec.alignment, .fill = spec.fill }); +} + +pub fn formatAny(value: anytype, spec: std.fmt.Number, writer: *std.Io.Writer) !void { var buf: [48]u8 = undefined; - const s = std.fmt.bufPrint(&buf, "{any}", .{value}) catch blk: { + const T = @TypeOf(value); + const fmt = if (@hasDecl(T, "formatNumber")) "{d}" else "{f}"; + + const s = std.fmt.bufPrint(&buf, fmt, .{value}) catch blk: { buf[45..].* = "...".*; break :blk buf[0..]; }; - return try writer.alignBufferOptions(s, full.options); + return try writer.alignBufferOptions(s, .{ .alignment = spec.alignment, .fill = spec.fill }); } -pub fn formatSliceCustom(fmt_func: anytype, values: anytype, full: FullFormatOptions, writer: anytype) !void { +pub fn formatSliceCustom(fmt_func: anytype, values: anytype, spec: std.fmt.Number, stride: i64, writer: *std.Io.Writer) !void { + // use the format "width" for the number of columns instead of individual width. + const num_cols: usize = spec.width orelse 12; + var my_options = spec; + my_options.width = null; + // TODO: handle negative strides + const strd: usize = @intCast(stride); + const n: usize = @divTrunc(values.len, strd); - // Write first rows - const num_cols: usize = full.options.width orelse 12; - const n: usize = values.len; _ = try writer.write("{"); if (n <= num_cols) { - for (values, 0..) |v, i| { + for (0..n) |i| { // Force inlining so that the switch and the buffer can be done once. - try @call(.always_inline, fmt_func, .{ v, full, writer }); + try @call(.always_inline, fmt_func, .{ values[i * strd], my_options, writer }); if (i < n - 1) _ = try writer.write(","); } } else { const half = @divFloor(num_cols, 2); - for (values[0..half]) |v| { - try @call(.always_inline, fmt_func, .{ v, full, writer }); + for (0..half) |i| { + try @call(.always_inline, fmt_func, .{ values[i * strd], my_options, writer }); _ = try writer.write(","); } _ = try writer.write(" ..., "); - for (values[n - half ..], 0..) |v, i| { - try @call(.always_inline, fmt_func, .{ v, full, writer }); - if (i < half - 1) _ = try writer.write(","); + for (n - half..n) |i| { + try @call(.always_inline, fmt_func, .{ values[i * strd], my_options, writer }); + if (i < n - 1) _ = try writer.write(","); } } _ = try writer.write("}"); } -pub fn formatAny(values: anytype, full: FullFormatOptions, writer: anytype) !void { - return try formatSliceCustom(formatAnyValue, values, full, writer); +pub fn formatSliceAny(values: anytype, spec: std.fmt.Number, stride: i64, writer: *std.Io.Writer) !void { + return try formatSliceCustom(formatAny, values, spec, stride, writer); } -pub fn formatFloatSlice(values: anytype, full: FullFormatOptions, writer: anytype) !void { - return try formatSliceCustom(formatFloatValue, values, full, writer); +pub fn formatFloatSlice(values: anytype, spec: std.fmt.Number, stride: i64, writer: *std.Io.Writer) !void { + return try formatSliceCustom(formatFloat, values, spec, stride, writer); } -pub fn formatIntSlice(values: anytype, full: FullFormatOptions, writer: anytype) !void { - return try formatSliceCustom(formatIntValue, values, full, writer); +pub fn formatIntSlice(values: anytype, spec: std.fmt.Number, stride: i64, writer: *std.Io.Writer) !void { + return try formatSliceCustom(formatInt, values, spec, stride, writer); } -pub fn formatAnySlice(values: anytype, full: FullFormatOptions, writer: anytype) !void { - return try formatSliceCustom(formatAnyValue, values, full, writer); +pub fn formatComplexSlice(values: anytype, spec: std.fmt.Number, stride: i64, writer: *std.Io.Writer) !void { + return try formatSliceCustom(formatComplex, values, spec, stride, writer); +} + +pub fn formatBoolSlice(values: anytype, spec: std.fmt.Number, stride: i64, writer: *std.Io.Writer) !void { + return try formatSliceCustom(formatBool, values, spec, stride, writer); } diff --git a/stdx/stdx.zig b/stdx/stdx.zig index 9bfa532a..ea14b0f1 100644 --- a/stdx/stdx.zig +++ b/stdx/stdx.zig @@ -1,3 +1,6 @@ +const std = @import("std"); +const builtin = @import("builtin"); + pub const BoundedArray = @import("bounded_array.zig").BoundedArray; pub const BoundedArrayAligned = @import("bounded_array.zig").BoundedArrayAligned; pub const debug = @import("debug.zig"); @@ -11,7 +14,6 @@ pub const queue = @import("queue.zig"); pub const time = @import("time.zig"); test { - const std = @import("std"); std.testing.refAllDecls(@This()); } @@ -20,3 +22,53 @@ pub inline fn stackSlice(comptime max_len: usize, T: type, len: usize) []T { var storage: [max_len]T = undefined; return storage[0..len]; } + +pub const noalloc: std.mem.Allocator = if (builtin.mode == .ReleaseFast) undefined else std.testing.failing_allocator; + +pub const mem = struct { + pub fn groupedAlloc( + SliceTuple: type, + allocator: std.mem.Allocator, + len: [@typeInfo(SliceTuple).@"struct".fields.len]usize, + ) error{OutOfMemory}!SliceTuple { + var res: SliceTuple = undefined; + var full_alloc_len: usize = 0; + + const SliceFields = @typeInfo(SliceTuple).@"struct".fields; + inline for (SliceFields, len) |field, l| { + // needed because full_alloc_len starts at 0. + @setRuntimeSafety(false); + + const T = std.meta.Child(field.type); + full_alloc_len = std.mem.alignForward(usize, full_alloc_len, @alignOf(T)); + @field(res, field.name).ptr = @ptrFromInt(full_alloc_len); + @field(res, field.name).len = l; + full_alloc_len += @sizeOf(T) * l; + } + + const bytes = try allocator.alignedAlloc(u8, .of(SliceFields[0].type), full_alloc_len); + // std.log.warn("groupedAlloc({s}) -> {*}[0..{d}] align({})", .{ @typeName(SliceTuple), bytes.ptr, bytes.len, alignment }); + + inline for (SliceFields) |field| { + @field(res, field.name).ptr = @ptrFromInt(@intFromPtr(bytes.ptr) + @intFromPtr(@field(res, field.name).ptr)); + } + + return res; + } + + pub fn groupedFree(SliceTuple: type, allocator: std.mem.Allocator, slice_tuple: SliceTuple) void { + const SliceFields = @typeInfo(SliceTuple).@"struct".fields; + + const slice_start: [*]u8 = @ptrCast(slice_tuple[0].ptr); + const N = SliceFields.len; + const FirstT = std.meta.Child(SliceFields[0].type); + const LastT = std.meta.Child(SliceFields[N - 1].type); + + const last_slice = slice_tuple[N - 1]; + const slice_end: usize = @intFromPtr(last_slice.ptr) + @sizeOf(LastT) * last_slice.len; + const slice_len = slice_end - @intFromPtr(slice_start); + + // std.log.warn("groupedFree({s}) -> {*}[0..{d}] align({})", .{ @typeName(SliceTuple), slice_start, slice_len, @alignOf(FirstT) }); + allocator.rawFree(slice_start[0..slice_len], .fromByteUnits(@alignOf(FirstT)), @returnAddress()); + } +}; diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index aaa44f04..b917f9ec 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -31,6 +31,7 @@ zig_library( "aio/torch/py.zig", "buffer.zig", "context.zig", + "callback.zig", "dtype.zig", "exe.zig", "floats.zig", @@ -53,7 +54,7 @@ zig_library( "torch.zig", "zml.zig", ], - copts = ["-lc"], + copts = ["-lc", "-freference-trace=20"], main = "zml.zig", visibility = ["//visibility:public"], deps = [ @@ -80,7 +81,7 @@ zig_test( data = [ "aio/torch/simple.pt", "aio/torch/simple_test_4.pickle", - ], + ] + glob(["test_data/**"]), test_runner = ":test_runner", deps = [":zml"], ) diff --git a/zml/aio.zig b/zml/aio.zig index 1b926a91..8d13585a 100644 --- a/zml/aio.zig +++ b/zml/aio.zig @@ -32,7 +32,8 @@ pub fn detectFormatAndOpen(allocator: std.mem.Allocator, model_path: []const u8) // else if (std.mem.endsWith(u8, model_path, ".tinyllama")) // try tinyllama.open(allocator, model_path) else { - std.debug.panic("File extension not recognized: {s}", .{model_path}); + log.err("File extension not recognized: {s}", .{model_path}); + return error.NotSupported; }; } @@ -43,8 +44,8 @@ pub fn detectFormatAndOpen(allocator: std.mem.Allocator, model_path: []const u8) /// whose shape is read from the "a.b" tensor. /// * If `Model` contains a list of layers, then the field: /// `Model.layers[2].a.b` will be populated from the "layers.2.a.b" tensor. -pub fn populateModel(comptime Model: type, allocator: std.mem.Allocator, buffer_store: BufferStore) !Model { - return populateModelWithPrefix(Model, allocator, buffer_store, ""); +pub fn populateModel(comptime Model: type, allocator: std.mem.Allocator, store: BufferStore) !Model { + return populateModelWithPrefix(Model, allocator, store, ""); } /// Creates a Model struct with tensor shapes read from the given TensorStore, @@ -62,8 +63,7 @@ pub fn populateModelWithPrefix(comptime Model: type, allocator: std.mem.Allocato try prefix_builder.push(allocator, prefix); defer prefix_builder.deinit(allocator); - const unique_id = zml.Tensor._reserveIdRange(@intCast(store.buffers.count())); - const ok = _populateStruct(allocator, &prefix_builder, unique_id, store, &model, true) catch |err| { + const ok = _populateStruct(allocator, &prefix_builder, store, &model, true) catch |err| { std.debug.panic("Can't populate model of type {s}: {s}", .{ @typeName(type), @errorName(err) }); }; if (!ok) return error.TensorNotFound; @@ -74,17 +74,28 @@ pub fn populateModelWithPrefix(comptime Model: type, allocator: std.mem.Allocato pub const BufferStore = struct { pub const Buffers = std.StringArrayHashMapUnmanaged(HostBuffer); pub const Metadatas = std.StringArrayHashMapUnmanaged(Metadata); + var _unique_store_id: std.atomic.Value(u64) = .init(0); + const _store_id_range: u64 = 1024 * 1024 * 1024; arena: std.heap.ArenaAllocator, files: []MemoryMappedFile = &.{}, buffers: Buffers = .{}, _metadata: Metadatas = .{}, + _unique_id: u64, - /// Create an empty BufferStore. Takes owneship of the given files. - pub fn init(allocator: std.mem.Allocator, files: []const MemoryMappedFile) error{OutOfMemory}!BufferStore { - var self: zml.aio.BufferStore = .{ + /// Create an empty BufferStore. + /// Takes owneship of the given files. + pub fn init(allocator: std.mem.Allocator) BufferStore { + return .{ .arena = std.heap.ArenaAllocator.init(allocator), + ._unique_id = _unique_store_id.fetchAdd(_store_id_range, .monotonic), }; + } + + /// Create an empty BufferStore. + /// Takes owneship of the given files. + pub fn initWithFiles(allocator: std.mem.Allocator, files: []const MemoryMappedFile) error{OutOfMemory}!BufferStore { + var self: BufferStore = .init(allocator); self.files = try self.arena.allocator().dupe(MemoryMappedFile, files); return self; } @@ -100,6 +111,61 @@ pub const BufferStore = struct { return self.buffers.get(key); } + pub fn loadBufferById(self: BufferStore, x: zml.Tensor, platform: zml.Platform) !zml.Buffer { + var host_buffer: zml.HostBuffer = switch (x._id) { + .buffer_id => |id| hb: { + if (id < self._unique_id or self._unique_id + _store_id_range <= id) { + @panic("`store.loadBufferById()` only works on Tensor created by `store.getTensor()`, using the same store object."); + } + break :hb self.buffers.values()[id - self._unique_id]; + }, + else => @panic("`store.loadBufferById()` only works on Tensor created by `store.getTensor()`"), + }; + + stdx.debug.assert(x.byteSize() == host_buffer.shape().byteSize(), "Can't load buffer {f} for tensor {f}: shape mismatch", .{ host_buffer, x }); + // Copy sharding info + host_buffer._shape = x._shape; + return try host_buffer.toDevice(platform); + } + + /// Creates a bufferized version of a model from the given BufferStore. + /// + /// This will represent the weights of the model, loaded on a specific platform. + /// It can be used with a `module.Exe` (a compiled version of the same Model), to make a + /// `module.ExeWithWeights` ready to be called. + pub fn loadModelById(self: BufferStore, Model: type, allocator: std.mem.Allocator, model: Model, platform: zml.Platform) !zml.Bufferized(Model) { + const Ctx = struct { + platform: *const zml.Platform, + store: *const BufferStore, + + pub fn cb(ctx: @This(), x: zml.Tensor) zml.Buffer { + return ctx.store.loadBufferById(x, ctx.platform.*) catch @panic("Failed to load buffer to device"); + } + }; + + var res: zml.Bufferized(Model) = undefined; + try zml.meta.mapAlloc(Ctx.cb, allocator, .{ .platform = &platform, .store = &self }, model, &res); + return res; + } + + pub fn getTensor(self: BufferStore, key: []const u8) zml.Tensor { + return self.getTensorOrNull(key) orelse { + self.findSimilarBufferKeys(std.heap.smp_allocator, key); + @panic("Tensor not found"); + }; + } + + pub fn getTensorOrNull(self: BufferStore, key: []const u8) ?zml.Tensor { + return if (self.buffers.getIndex(key)) |entry_idx| + .{ + ._shape = self.buffers.values()[entry_idx].shape(), + ._id = .{ .buffer_id = self._unique_id + entry_idx }, + ._donation = .input_buffer, + } + else + return null; + } + /// Count layers starting with the given prefix. pub fn countLayers(self: BufferStore, prefix: []const u8) usize { // Note: This is kinda inefficient @@ -140,6 +206,58 @@ pub const BufferStore = struct { return null; } + + /// Assists in debuggigng `BufferNotFound` error + /// This is useful when a buffer key is not found and you want to identify possible alternatives (or typos) + pub fn findSimilarBufferKeys(store: BufferStore, tmp_alloc: std.mem.Allocator, original_key: []const u8) void { + const suffixes = [_][]const u8{ "", ".weight", ".bias" }; + var shown_keys = std.StringHashMap(void).init(tmp_alloc); + defer shown_keys.deinit(); + + // remove suffix .weight and .bias + var base_key = original_key; + for (suffixes) |suffix| { + if (std.mem.endsWith(u8, original_key, suffix)) { + base_key = original_key[0 .. original_key.len - suffix.len]; + break; + } + } + + // first test: look for exact matches + var matches: usize = 0; + var it = store.buffers.iterator(); + while (it.next()) |entry| { + const key = entry.key_ptr.*; + if (std.mem.startsWith(u8, key, base_key)) { + if (matches == 0) log.warn("Similar buffers found:", .{}); + if (!shown_keys.contains(key)) { + log.warn(" - {s}: {f}", .{ key, entry.value_ptr.*.shape() }); + shown_keys.put(key, {}) catch continue; + matches += 1; + } + } + } + + // second test: progressive partial matches + if (matches == 0) { + var components = std.mem.splitScalar(u8, base_key, '.'); + while (components.next()) |component| { + matches = 0; + it = store.buffers.iterator(); + while (it.next()) |entry| { + const key = entry.key_ptr.*; + if (std.mem.indexOf(u8, key, component) != null and !shown_keys.contains(key)) { + if (matches == 0) log.warn("Partial matches for '{s}':", .{component}); + log.warn(" - {s}: {f}", .{ key, entry.value_ptr.*.shape() }); + shown_keys.put(key, {}) catch continue; + matches += 1; + if (matches >= 5) break; + } + } + if (matches > 0) break; + } + } + } }; pub const Metadata = union(enum) { @@ -260,17 +378,36 @@ pub const MemoryMappedFile = struct { /// Helper handling prefix building. /// /// This allows to easily push/pop prefixes and handles the generation of the string with the correct format. -const PrefixBuilder = struct { +pub const PrefixBuilder = struct { /// Stores the computed prefix. data: std.ArrayListUnmanaged(u8) = .{}, /// Stack storing the size of the intermediary prefix. subprefixes: std.ArrayListUnmanaged(u32) = .{}, + pub fn initCapacity(allocator: std.mem.Allocator, capacity: usize) !PrefixBuilder { + const data, const subprefixes = try stdx.mem.groupedAlloc(struct { []u8, []u32 }, allocator, .{ capacity, @divFloor(capacity, 4) }); + return .{ + .data = .initBuffer(data), + .subprefixes = .initBuffer(subprefixes), + }; + } + pub fn deinit(self: *PrefixBuilder, allocator: std.mem.Allocator) void { self.data.deinit(allocator); self.subprefixes.deinit(allocator); } + pub fn items(self: PrefixBuilder) []const u8 { + return self.data.items; + } + + pub fn concat(self: *PrefixBuilder, prefix: []const u8) []const u8 { + self.push(stdx.noalloc, prefix) catch unreachable; + const res = self.items(); + self.pop(); + return res; + } + pub fn push(self: *PrefixBuilder, allocator: std.mem.Allocator, prefix: []const u8) !void { const old_len: u32 = @intCast(self.data.items.len); try self.subprefixes.append(allocator, old_len); @@ -301,13 +438,20 @@ const PrefixBuilder = struct { const last_prefix_len = self.subprefixes.pop() orelse unreachable; self.data.shrinkRetainingCapacity(last_prefix_len); } + + pub fn checkpoint(self: PrefixBuilder) [2]usize { + return .{ self.data.items.len, self.subprefixes.items.len }; + } + + pub fn restore(self: *PrefixBuilder, ckpt: [2]usize) void { + self.data.items.len, self.subprefixes.items.len = ckpt; + } }; fn _populateStruct( allocator: std.mem.Allocator, prefix_builder: *PrefixBuilder, - unique_id: u64, - buffer_store: BufferStore, + store: BufferStore, obj: anytype, required: bool, ) !bool { @@ -322,35 +466,32 @@ fn _populateStruct( const prefix = prefix_builder.data.items; if (T == zml.Tensor) { - return if (buffer_store.buffers.getIndex(prefix)) |entry_idx| { - const buffer = buffer_store.get(prefix).?; - obj.* = zml.Tensor{ - ._shape = buffer.shape(), - ._id = .{ .buffer_id = unique_id + entry_idx }, - ._donation = .input_buffer, - }; + return if (store.getTensorOrNull(prefix)) |tensor| { + obj.* = tensor; return true; } else { if (required) { - log.err("Tensor not found: {s} ({d})", .{ prefix, buffer_store.buffers.count() }); + log.err("Tensor not found: {s} ({d})", .{ prefix, store.buffers.count() }); } return false; }; } + log.debug("Populating {s} ({s})", .{ prefix, @typeName(T) }); return switch (type_info) { .pointer => |ptr_info| { if (ptr_info.size == .slice) { obj.* = &.{}; - const len = buffer_store.countLayers(prefix); + if (!zml.meta.Contains(ptr_info.child, zml.Tensor)) return false; + const len = store.countLayers(prefix); if (len > 0) { obj.* = try allocator.alloc(ptr_info.child, len); for (obj.*, 0..) |*value, i| { try prefix_builder.pushDigit(allocator, i); defer prefix_builder.pop(); - const found = try _populateStruct(allocator, prefix_builder, unique_id, buffer_store, value, required); + const found = try _populateStruct(allocator, prefix_builder, store, value, required); if (!found) { log.err("Not able to load {s} as {s}", .{ prefix_builder.data.items, @typeName(ptr_info.child) }); return false; @@ -369,7 +510,7 @@ fn _populateStruct( for (obj, 0..) |*value, i| { try prefix_builder.pushDigit(allocator, i); defer prefix_builder.pop(); - const found = try _populateStruct(allocator, prefix_builder, unique_id, buffer_store, value, required); + const found = try _populateStruct(allocator, prefix_builder, store, value, required); if (!found) { log.err("Not able to load {s} as {s}", .{ prefix_builder.data.items, @typeName(arr_info.child) }); return false; @@ -386,19 +527,25 @@ fn _populateStruct( var has_default = false; if (field.default_value_ptr) |_| has_default = true; - const field_found = try _populateStruct(allocator, prefix_builder, unique_id, buffer_store, &@field(obj, field.name), required and !has_default); - partial_struct = partial_struct or field_found; - if (!field_found) { - if (field.default_value_ptr) |v| { - @field(obj, field.name) = @as(*const field.type, @ptrCast(@alignCast(v))).*; - } else { - if (partial_struct) { - log.warn("Incomplete metadata '{0s}': {1s}. Missing field: '{2s}'. '{0s}' will be ignored.", .{ prefix, @typeName(T), field.name }); - obj.* = undefined; + + if (zml.meta.Contains(field.type, zml.Tensor)) { + const field_found = try _populateStruct(allocator, prefix_builder, store, &@field(obj, field.name), required and !has_default); + partial_struct = partial_struct or field_found; + if (!field_found) { + if (field.default_value_ptr) |v| { + @field(obj, field.name) = @as(*const field.type, @ptrCast(@alignCast(v))).*; + } else { + if (partial_struct) { + log.warn("Incomplete struct '{0s}': {1s}. Missing field: '{2s}'. '{0s}' will be ignored.", .{ prefix, @typeName(T), field.name }); + // if (required) { + // findSimilarBufferKeys(prefix, store, allocator); + // } + obj.* = undefined; + return false; + } + return false; } - - return false; } } } @@ -406,7 +553,7 @@ fn _populateStruct( }, .optional => |opt_info| { obj.* = @as(opt_info.child, undefined); - const found = try _populateStruct(allocator, prefix_builder, unique_id, buffer_store, &(obj.*.?), false); + const found = try _populateStruct(allocator, prefix_builder, store, &(obj.*.?), false); if (!found) obj.* = null; return true; }, @@ -415,11 +562,22 @@ fn _populateStruct( return true; }, .float => { - obj.* = std.math.nan(@TypeOf(obj.*)); + // obj.* = std.math.nan(@TypeOf(obj.*)); return true; }, .void => true, - .@"union" => true, + .@"union" => |union_info| { + inline for (union_info.fields) |field| { + obj.* = @unionInit(T, field.name, undefined); + // @compileLog(T, field.name, field.type); + log.warn("Trying to init {s} as a {s}", .{ prefix, @typeName(field.type) }); + const field_found = try _populateStruct(allocator, prefix_builder, store, &@field(obj, field.name), true); + if (field_found) return true; + } + if (required) log.warn("Failed to init {s} as any of {s}", .{ prefix, @typeName(T) }); + obj.* = undefined; + return false; + }, .bool => { obj.* = undefined; return true; @@ -449,7 +607,7 @@ test populateModel { var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena_state.deinit(); - var store: BufferStore = .{ .arena = arena_state }; + var store: BufferStore = .init(arena_state.allocator()); try store.buffers.ensureUnusedCapacity(arena_state.allocator(), 16); store.buffers.putAssumeCapacity("a", Model._newHostBuffer(10)); store.buffers.putAssumeCapacity("b.a", Model._newHostBuffer(20)); @@ -494,11 +652,11 @@ test populateModel { pub fn loadBuffers( comptime Model: type, init_args: if (@hasDecl(Model, "init")) stdx.meta.Tail(stdx.meta.FnArgs(Model.init)) else void, - buffer_store: BufferStore, + store: BufferStore, allocator: std.mem.Allocator, platform: zml.Platform, ) !zml.Bufferized(Model) { - return loadBuffersWithPrefix(Model, init_args, buffer_store, allocator, platform, ""); + return loadBuffersWithPrefix(Model, init_args, store, allocator, platform, ""); } /// Creates a bufferized version of a Model from the given BufferStore with a specified prefix. @@ -512,7 +670,7 @@ pub fn loadBuffers( pub fn loadBuffersWithPrefix( comptime Model: type, init_args: if (@hasDecl(Model, "init")) stdx.meta.Tail(stdx.meta.FnArgs(Model.init)) else void, - buffer_store: BufferStore, + store: BufferStore, allocator: std.mem.Allocator, platform: zml.Platform, prefix: []const u8, @@ -522,14 +680,14 @@ pub fn loadBuffersWithPrefix( const arena = arena_state.allocator(); // Get model structure with tensor shapes from the buffer store with prefix - var model: Model = try zml.aio.populateModelWithPrefix(Model, arena, buffer_store, prefix); + var model: Model = try zml.aio.populateModelWithPrefix(Model, arena, store, prefix); // If the Model has a "init" function, call it with the given parameters. if (@hasDecl(Model, "init")) { @call(.auto, Model.init, .{&model} ++ init_args); } - return loadModelBuffersWithPrefix(Model, model, buffer_store, allocator, platform, prefix); + return loadModelBuffersWithPrefix(Model, model, store, allocator, platform, prefix); } /// Creates a bufferized version of a Model from the given BufferStore. For details about @@ -541,11 +699,11 @@ pub fn loadBuffersWithPrefix( pub fn loadModelBuffers( comptime Model: type, model: Model, - buffer_store: BufferStore, + store: BufferStore, allocator: std.mem.Allocator, platform: zml.Platform, ) !zml.Bufferized(Model) { - return try loadModelBuffersWithPrefix(Model, model, buffer_store, allocator, platform, ""); + return try loadModelBuffersWithPrefix(Model, model, store, allocator, platform, ""); } /// Creates a bufferized version of a Model from the given BufferStore and the given prefix. @@ -557,7 +715,7 @@ pub fn loadModelBuffers( pub fn loadModelBuffersWithPrefix( comptime Model: type, model: Model, - buffer_store: BufferStore, + store: BufferStore, allocator: std.mem.Allocator, platform: zml.Platform, prefix: []const u8, @@ -576,7 +734,7 @@ pub fn loadModelBuffersWithPrefix( try prefix_builder.push(allocator, prefix); defer prefix_builder.deinit(allocator); - try visitStructAndLoadBuffer(allocator, &prefix_builder, buffer_store, &res, platform); + try visitStructAndLoadBuffer(allocator, &prefix_builder, store, &res, platform); return res; } @@ -590,58 +748,6 @@ pub fn unloadBuffers(model: anytype) void { }).cb, {}, model); } -/// Assists in debuggigng `BufferNotFound` error -/// This is useful when a buffer key is not found and you want to identify possible alternatives (or typos) -fn findSimilarBufferKeys(original_key: []const u8, store: BufferStore, temp_allocator: std.mem.Allocator) void { - const suffixes = [_][]const u8{ "", ".weight", ".bias" }; - var shown_keys = std.StringHashMap(void).init(temp_allocator); - defer shown_keys.deinit(); - - // remove suffix .weight and .bias - var base_key = original_key; - for (suffixes) |suffix| { - if (std.mem.endsWith(u8, original_key, suffix)) { - base_key = original_key[0 .. original_key.len - suffix.len]; - break; - } - } - - // first test: look for exact matches - var matches: usize = 0; - var it = store.buffers.iterator(); - while (it.next()) |entry| { - const key = entry.key_ptr.*; - if (std.mem.startsWith(u8, key, base_key)) { - if (matches == 0) log.warn("Similar buffers found:", .{}); - if (!shown_keys.contains(key)) { - log.warn(" - {s}: {f}", .{ key, entry.value_ptr.*.shape() }); - shown_keys.put(key, {}) catch continue; - matches += 1; - } - } - } - - // second test: progressive partial matches - if (matches == 0) { - var components = std.mem.splitScalar(u8, base_key, '.'); - while (components.next()) |component| { - matches = 0; - it = store.buffers.iterator(); - while (it.next()) |entry| { - const key = entry.key_ptr.*; - if (std.mem.indexOf(u8, key, component) != null and !shown_keys.contains(key)) { - if (matches == 0) log.warn("Partial matches for '{s}':", .{component}); - log.warn(" - {s}: {f}", .{ key, entry.value_ptr.*.shape() }); - shown_keys.put(key, {}) catch continue; - matches += 1; - if (matches >= 5) break; - } - } - if (matches > 0) break; - } - } -} - /// deinit all buffers in the given struct pub fn awaitAll(buffers: anytype) !void { zml.meta.visit((struct { @@ -651,7 +757,7 @@ pub fn awaitAll(buffers: anytype) !void { }).cb, {}, buffers); } -fn visitStructAndLoadBuffer(allocator: std.mem.Allocator, prefix_builder: *PrefixBuilder, buffer_store: BufferStore, obj: anytype, platform: zml.Platform) !void { +fn visitStructAndLoadBuffer(allocator: std.mem.Allocator, prefix_builder: *PrefixBuilder, store: BufferStore, obj: anytype, platform: zml.Platform) !void { const err_msg = "visitStructAndLoadBuffer must be called with a pointer to type. Received "; const type_info, const T = switch (@typeInfo(@TypeOf(obj))) { .pointer => |ptr_info| switch (ptr_info.size) { @@ -663,7 +769,7 @@ fn visitStructAndLoadBuffer(allocator: std.mem.Allocator, prefix_builder: *Prefi const prefix = prefix_builder.data.items; if (T == zml.Buffer) { - return if (buffer_store.get(prefix)) |host_buffer| { + return if (store.get(prefix)) |host_buffer| { // obj._shape has been set inside `loadModelBuffersWithPrefix`, before calling us. var buf_with_metadata = host_buffer; log.debug("Loading buffer {s} ({f})", .{ prefix, obj._shape }); @@ -673,7 +779,7 @@ fn visitStructAndLoadBuffer(allocator: std.mem.Allocator, prefix_builder: *Prefi } else { log.err("Buffer not found: {s}", .{prefix}); - findSimilarBufferKeys(prefix, buffer_store, allocator); + store.findSimilarBufferKeys(allocator, prefix); return error.BufferNotFound; }; @@ -686,7 +792,7 @@ fn visitStructAndLoadBuffer(allocator: std.mem.Allocator, prefix_builder: *Prefi try prefix_builder.pushDigit(allocator, i); defer prefix_builder.pop(); - try visitStructAndLoadBuffer(allocator, prefix_builder, buffer_store, value, platform); + try visitStructAndLoadBuffer(allocator, prefix_builder, store, value, platform); } } else stdx.debug.compileError("type not supported by visitStructAndLoadBuffer: {}", .{T}); }, @@ -694,7 +800,7 @@ fn visitStructAndLoadBuffer(allocator: std.mem.Allocator, prefix_builder: *Prefi for (obj, 0..) |*value, i| { try prefix_builder.pushDigit(allocator, i); defer prefix_builder.pop(); - try visitStructAndLoadBuffer(allocator, prefix_builder, buffer_store, value, platform); + try visitStructAndLoadBuffer(allocator, prefix_builder, store, value, platform); } }, @@ -704,12 +810,12 @@ fn visitStructAndLoadBuffer(allocator: std.mem.Allocator, prefix_builder: *Prefi try prefix_builder.push(allocator, field.name); defer prefix_builder.pop(); - try visitStructAndLoadBuffer(allocator, prefix_builder, buffer_store, &@field(obj, field.name), platform); + try visitStructAndLoadBuffer(allocator, prefix_builder, store, &@field(obj, field.name), platform); } }, .optional => { if (obj.*) |*obj_val| { - try visitStructAndLoadBuffer(allocator, prefix_builder, buffer_store, obj_val, platform); + try visitStructAndLoadBuffer(allocator, prefix_builder, store, obj_val, platform); } }, else => {}, diff --git a/zml/aio/safetensors.zig b/zml/aio/safetensors.zig index 08952b8f..49ba5fca 100644 --- a/zml/aio/safetensors.zig +++ b/zml/aio/safetensors.zig @@ -13,9 +13,7 @@ const StringBuilder = std.ArrayListUnmanaged(u8); const log = std.log.scoped(.@"zml/io"); pub fn open(allocator: std.mem.Allocator, path: []const u8) !zml.aio.BufferStore { - var res: zml.aio.BufferStore = .{ - .arena = std.heap.ArenaAllocator.init(allocator), - }; + var res: zml.aio.BufferStore = .init(allocator); errdefer res.arena.deinit(); const arena = res.arena.allocator(); diff --git a/zml/aio/tinyllama.zig b/zml/aio/tinyllama.zig index 8114c90f..c51319d6 100644 --- a/zml/aio/tinyllama.zig +++ b/zml/aio/tinyllama.zig @@ -25,9 +25,7 @@ const TinyLlamaConfig = extern struct { /// For convenience we use the same layer names /// than the one used by the Llama-3.1 models. pub fn open(allocator: std.mem.Allocator, model_path: []const u8) !zml.aio.BufferStore { - var res: zml.aio.BufferStore = .{ - .arena = std.heap.ArenaAllocator.init(allocator), - }; + var res: zml.aio.BufferStore = .init(allocator); errdefer res.arena.deinit(); const arena = res.arena.allocator(); diff --git a/zml/aio/torch.zig b/zml/aio/torch.zig index 79487f64..3f1511d7 100644 --- a/zml/aio/torch.zig +++ b/zml/aio/torch.zig @@ -36,7 +36,7 @@ pub fn open(allocator: std.mem.Allocator, path: []const u8) !zml.aio.BufferStore const py_values = try eval.evaluate(tmp_alloc, ops, true); // file ownership is transferred to the BufferStore - var res = try zml.aio.BufferStore.init(allocator, &.{torch_file.mmap_file}); + var res = try zml.aio.BufferStore.initWithFiles(allocator, &.{mmap_file}); try torch_file.parseModel(py_values, &res); return res; } diff --git a/zml/aio/torch/file.zig b/zml/aio/torch/file.zig index 0307d51c..fe8811fd 100644 --- a/zml/aio/torch/file.zig +++ b/zml/aio/torch/file.zig @@ -480,7 +480,7 @@ test "Read pickle (zipped)" { // torch.save({ "model": model, "tensor": tensor}, "simple.pt") const file = try asynk.File.open("zml/aio/torch/simple.pt", .{ .mode = .read_only }); const mmap_file = try zml.aio.MemoryMappedFile.init(file); - var store = try zml.aio.BufferStore.init(testing.allocator, &.{mmap_file}); + var store = try zml.aio.BufferStore.initWithFiles(testing.allocator, &.{mmap_file}); defer store.deinit(); { diff --git a/zml/buffer.zig b/zml/buffer.zig index 3553bb94..8d202127 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -49,7 +49,7 @@ pub const Buffer = struct { pub const FromOptions = struct { wait: bool = true, - memory: ?pjrt.Memory.Kind = null, + memory: ?Memory = null, }; /// Copies the content of the given buffer from host memory to the accelerator memory. @@ -71,7 +71,7 @@ pub const Buffer = struct { break :cs @divExact(host_buffer.dim(ax), n_partitions); } else 0; - const buffer_type = bufferTypeFromDtype(host_buffer.shape().dtype()); + const buffer_type = pjrt.bufferTypeFromDtype(host_buffer.shape().dtype()); const byte_strides = host_buffer.strides(); const devices = platform.getDevices(); @@ -89,15 +89,20 @@ pub const Buffer = struct { .byte_strides = byte_strides, .host_buffer_semantics = .ImmutableUntilTransferCompletes, }; - if (opts.memory) |memory_kind| { - const memories = try devices[i].addressableMemories(platform.pjrt_api); - const memory = for (memories) |m| { - const kind = m.kind(platform.pjrt_api); - if (kind == memory_kind) break m; - } else return error.NotFound; - args.memory = memory; - } else { + if (platform.target == .cpu or opts.memory == null) { args.device = devices[i]; + } else { + const memory = opts.memory.?; + const device_memories = try devices[i].addressableMemories(platform.pjrt_api); + // TODO measure the cost of this and consider caching on Zig side inside the platform. + const selected_memory = for (device_memories) |m| { + const kind = m.kind(platform.pjrt_api); + if (kind == memory.toPjrtMemory()) break m; + } else { + log.warn("Platform {s} doesn't have memory {s}", .{ @tagName(platform.target), @tagName(memory) }); + return error.NotFound; + }; + args.memory = selected_memory; } const pjrt_buffer, const event = try platform.pjrt_client.bufferFromHostBuffer(platform.pjrt_api, args); @@ -152,6 +157,32 @@ pub const Buffer = struct { return from(platform, HostBuffer.fromBytes(sh, data), .{}); } + test fromBytes { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + { + const data: [8]u32 = .{ 0, 9, 2, 11, 4, 13, 6, 15 }; + const data_bytes: []const u8 = @ptrCast(&data); + try std.testing.expectEqual(@sizeOf(@TypeOf(data)), data_bytes.len); + const data_d = try zml.Buffer.fromBytes(platform, .init(.{8}, .u32), data_bytes); + const data_read = try data_d.toHostAlloc(std.testing.allocator); + defer data_read.deinit(std.testing.allocator); + try std.testing.expectEqualSlices(u32, &data, data_read.items(u32)); + } + + { + // We use u4 so that the bits are packed. + const data: [8]u4 = .{ 0, 9, 2, 11, 4, 13, 6, 15 }; + const data_bytes: []const u8 = @ptrCast(&data); + try std.testing.expectEqual(@sizeOf(@TypeOf(data)), data_bytes.len); + const expect: [8]u4 = .{ 0, 9, 2, 11, 4, 13, 6, 15 }; + const data_d = try zml.Buffer.fromBytes(platform, .init(.{8}, .u4), data_bytes); + const data_read = try data_d.toHostAlloc(std.testing.allocator); + defer data_read.deinit(std.testing.allocator); + try std.testing.expectEqualSlices(u8, std.mem.asBytes(&expect), data_read.bytes()); + } + } + /// Copies the given Zig array to the accelerator memory and /// return a Buffer using the array shape. pub fn fromArray(platform: Platform, arr: anytype) !Buffer { @@ -179,10 +210,10 @@ pub const Buffer = struct { return try from(platform, host_buffer, opts); } - pub fn asPinnedHostBuffer(self: Buffer) HostBuffer { - // TODO restore assert + pub fn asHostBuffer(self: Buffer) HostBuffer { + // TODO: skip this check on cpu // const memory = self.getMemory().kind(self._api); - // stdx.debug.assert(memory == .pinned_host, "asPinnedHostBuffer({}) expects a buffer allocated on host memory, got {}. see `toMemory`", .{ self, memory }); + // stdx.debug.assert((memory == .pinned_host) or (memory == .unpinned_host), "asHostBuffer({f}) expects a buffer allocated on host memory, got {t}. see `copyToMemory`", .{ self, memory }); const ptr: [*]u8 = @ptrCast(self._shards.get(0).getOpaqueDeviceMemoryDataPointer(self._api) catch unreachable); return HostBuffer.fromBytes(self._shape, ptr[0..self._shape.byteSize()]); } @@ -276,7 +307,7 @@ pub const Buffer = struct { const pjrt_buffer = platform.pjrt_client.createViewOfDeviceBuffer(platform.pjrt_api, .{ .data = device_data, - .element_type = bufferTypeFromDtype(shape_.dtype()), + .element_type = pjrt.bufferTypeFromDtype(shape_.dtype()), .dims = shape_.dims(), // TODO: exposes sharding in the API. .device = platform.getDevices()[0], @@ -299,6 +330,12 @@ pub const Buffer = struct { }; } + pub fn opaqueDeviceMemoryDataPointer(self: Buffer) [*]u8 { + stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer", .{}); + const opaque_ptr: *anyopaque = self._shards.get(0).getOpaqueDeviceMemoryDataPointer(self._api) catch unreachable; + return @ptrCast(opaque_ptr); + } + /// Fetches the content of the given buffer into a stack variable of the given type. pub fn getValue(self: Buffer, T: type) !T { stdx.debug.assert(self._shape.byteSize() == @sizeOf(T), "Buffer {f} has {d} bytes of data, can't load it to a {s} with {d} bytes", .{ self, self._shape.byteSize(), @typeName(T), @sizeOf(T) }); @@ -390,13 +427,31 @@ pub const Buffer = struct { return @reduce(.Or, self._shape._sharding_info); } - pub fn copyToMemory(self: Buffer, memory: *const pjrt.Memory) !Buffer { + pub const CopyToMemoryOpts = struct { + wait: bool = true, + }; + + pub fn copyToMemory(self: Buffer, platform: Platform, memory: Memory, opts: CopyToMemoryOpts) !Buffer { + const pjrt_memory = platform.pjrt_client.memoryByKind(self._api, memory.toPjrtMemory()); + if (pjrt_memory == null) { + stdx.debug.panic("Memory destination `{s}` for {f}", .{ memory.pjrtName(), self }); + } + var new_shards: Buffer.Shards = .{}; for (self._shards.slice()) |shard| { - const new_shard = try shard.copyToMemory(self._api, memory); + const new_shard = try shard.copyToMemory(self._api, pjrt_memory.?); new_shards.appendAssumeCapacity(new_shard); } + if (opts.wait) { + for (new_shards.constSlice()) |shard| { + const event = shard.getReadyEvent(self._api); + if (event) |e| { + try e.awaitBlocking(self._api); + } + } + } + return Buffer{ ._shape = self._shape, ._shards = new_shards, ._api = self._api }; } @@ -433,7 +488,7 @@ pub const Buffer = struct { break :s shard_shape; } else shape_; - const buffer_type = bufferTypeFromDtype(shape_.dtype()); + const buffer_type = pjrt.bufferTypeFromDtype(shape_.dtype()); const devices = platform.getDevices(); for (0..n_partitions) |i| { var args = pjrt.Client.CreateUninitializedBufferArgs{ @@ -465,29 +520,3 @@ pub const Buffer = struct { return res; } }; - -pub fn bufferTypeFromDtype(dt: DataType) pjrt.BufferType { - return switch (dt) { - inline else => |tag| @field(pjrt.BufferType, @tagName(tag)), - }; -} - -pub fn dtypeFromBufferType(pjrt_type: pjrt.BufferType) DataType { - return switch (pjrt_type) { - .invalid => @panic("Found an invalid pjrt buffer"), - inline else => |tag| @field(DataType, @tagName(tag)), - }; -} - -test bufferTypeFromDtype { - inline for (@typeInfo(DataType).@"enum".fields) |field| { - const dt: DataType = @enumFromInt(field.value); - try std.testing.expectEqual(dt, dtypeFromBufferType(bufferTypeFromDtype(dt))); - } - - inline for (@typeInfo(pjrt.BufferType).@"enum".fields) |field| { - const dt: pjrt.BufferType = @enumFromInt(field.value); - if (dt == .invalid) continue; - try std.testing.expectEqual(dt, bufferTypeFromDtype(dtypeFromBufferType(dt))); - } -} diff --git a/zml/callback.zig b/zml/callback.zig new file mode 100644 index 00000000..4d6ab90e --- /dev/null +++ b/zml/callback.zig @@ -0,0 +1,313 @@ +const std = @import("std"); + +const asynk = @import("async"); +const mlir = @import("mlir"); +const pjrt = @import("pjrt"); +const stablehlo = @import("mlir/dialects").stablehlo; +const stdx = @import("stdx"); + +const Buffer = @import("buffer.zig").Buffer; +const CompilationContext = @import("module.zig").CompilationContext; +const DataType = @import("dtype.zig").DataType; +const HostBuffer = @import("hostbuffer.zig").HostBuffer; +const mlirx = @import("mlirx.zig"); +const pjrtx = @import("pjrtx.zig"); +const Platform = @import("platform.zig").Platform; +const Shape = @import("shape.zig").Shape; +const Tensor = @import("tensor.zig").Tensor; + +const log = std.log.scoped(.@"zml/callback"); + +/// Inserts a user-defined callback into the computation graph. +/// The callback is defined with a struct, that store runtime information needed by the callback. +/// +/// ❗Experimental API❗ +/// +/// ```zig +/// pub const MyCallback = struct { +/// // a unique type_id will be set by the PJRT plugin during registration. +/// pub var type_id: pjrt.ffi.TypeId = undefined; +/// +/// pub const callback_config: zml.callback.Config = .{ +/// // assumption this custom call makes about the input / output buffers +/// }; +/// +/// // Required, this will tell the callback in which env it runs. +/// platform: zml.Platform, +/// // data needed by the callback +/// my_data: []const u8, +/// +/// // storage modified by the runtime to tell the callback where it should write its results. +/// // Normally the callback doesn't need to allocate as the input and output buffers are given. +/// results: [1]Buffer = undefined, +/// +/// pub fn init(my_data: []const u8) !MyCallback { +/// return .{ .my_data = my_data }; +/// } +/// +/// pub fn call(callback: *MyCallback, input: Buffer) !void { +/// // Do something with `input` and `callback.my_data`, write the results inside `callback.results[0]` +/// } +/// }; +/// ``` +/// +/// See eg the implementation of the `zml.callback.Print` callback, for a practical example. +/// +/// Note calling this during the compilation of a module, isn't enough: +/// +/// * backend need to be made aware of the callback, see `zml.Platform.registerCallback` +/// * executable need to know the specific data needed by `MyCallback`, see `zml.Exe.bind` +pub fn call( + comptime Callback: type, + inputs: TensorArgs(Callback), + output_shapes: []const Shape, +) []Tensor { + checkIsValidCallback(Callback); + + const ctx = CompilationContext.current(); + const allocator = ctx.allocator(); + const mlir_ctx = ctx.mlirCtx(); + const platform = ctx._platform; + const pjrt_api = platform.pjrt_api; + + if (pjrt_api.ffi() == null) { + stdx.debug.panic("Custom calls are not supported for target {s}", .{@tagName(platform.target)}); + } + + const output_tensors = allocator.alloc(Tensor, output_shapes.len) catch @panic("OOM"); + // Note: we don't always free output_tensor, because it's returned to the caller. + // It's also why we allocate it first so that it doesn't fragment the arena. + errdefer allocator.free(output_tensors); + + const output_types = allocator.alloc(mlir.Type, output_shapes.len) catch @panic("OOM"); + defer allocator.free(output_types); + for (output_types, output_shapes) |*output_type, output_shape| { + output_type.* = mlirx.tensorType(mlir_ctx, output_shape); + } + const input_values = allocator.alloc(mlir.Value, inputs.len) catch @panic("OOM"); + defer allocator.free(input_values); + for (input_values, inputs) |*input_value, input_tensor| { + input_value.* = input_tensor.value(); + } + + const target_name = "zml$" ++ @typeName(Callback); + const op = stablehlo.custom_call( + mlir_ctx, + input_values, + .{ + .call_target_name = target_name, + .api_version = .typed_ffi, + .backend_config = .dict(mlir_ctx, &.{}), + .additional_attributes = &.{.{ "mhlo.frontend_attributes", .dict(mlir_ctx, &.{}) }}, + .has_side_effect = true, + .output_operand_aliases = Callback.callback_config.output_operand_aliases, + }, + output_types, + mlir_ctx.location(@src()), + ); + + for (output_tensors, output_shapes, 0..) |*output_tensor, output_shape, i| { + output_tensor.* = Tensor._result(output_shape, op.result(i)); + } + return output_tensors; +} + +/// Describe properties of a callback +/// +/// * output_operand_aliases: the callback reuse input buffer to write the output +/// * copy_inputs_to_host_pinned: the callback need to work on host visible buffers +/// * traits: PJRT specified properties of the callback +pub const Config = struct { + output_operand_aliases: []const i64 = &.{}, + copy_inputs_to_host_pinned: bool = false, + // TODO: document precisely what `command_buffer_compatible` is doing and its limitations. + traits: pjrt.ffi.HandlerTraits = .{ .command_buffer_compatible = false }, + // TODO: handle sharded inputs +}; + +/// Compile-time check that a callback has all informations we require. +pub fn checkIsValidCallback(Callback: type) void { + stdx.debug.assertComptime(@hasDecl(Callback, "call"), "Expected callback {} to have a call method", .{Callback}); + const ArgsT = stdx.meta.FnArgs(Callback.call); + inline for (@typeInfo(ArgsT).@"struct".fields[1..]) |field| { + stdx.debug.assertComptime(field.type == Buffer, "Expected callback {}.call arguments to be of type zml.Buffer, got {}", .{ Callback, field.type }); + } + + stdx.debug.assertComptime(@hasDecl(Callback, "type_id") and @TypeOf(Callback.type_id) == pjrt.ffi.TypeId, "Expected callback {} to have a field `pub var type_id: pjrt.ffi.TypeId`", .{Callback}); + stdx.debug.assertComptime(@hasDecl(Callback, "callback_config") and @TypeOf(Callback.callback_config) == Config, "Expected callback {} to have a field `pub const callback_config: zml.CustomCallOptions`", .{Callback}); +} + +pub fn register(Callback: type, platform: Platform) pjrt.ApiError!void { + checkIsValidCallback(Callback); + + const ffi = platform.pjrt_api.ffi() orelse return error.Unavailable; + const target_name = "zml$" ++ @typeName(Callback); + + const proxy_cb = proxy(Callback); + Callback.type_id = try ffi.registerTypeId(platform.pjrt_api, @typeName(Callback)); + try ffi.register(platform.pjrt_api, target_name, @tagName(platform.target), &proxy_cb, Callback.callback_config.traits); + log.debug("Registered custom call {} with target name \"{s}\"", .{ Callback, target_name }); +} + +fn proxy(Callback: type) pjrt.ffi.Handler { + return struct { + pub fn cb(call_frame: *pjrt.ffi.CallFrame) callconv(.c) ?*pjrt.ffi.Error { + return CallbackImpl(Callback, call_frame); + } + }.cb; +} + +fn CallbackImpl(comptime Callback: type, call_frame: *pjrt.ffi.CallFrame) ?*pjrt.ffi.Error { + if (call_frame.registeringHook()) return null; + + const opts = Callback.callback_config; + + const execution_context = call_frame.ctx; + log.debug("Custom call {s} called !", .{@typeName(Callback)}); + const user_ctx_opaque = execution_context.getContext(Callback.type_id, call_frame.api) catch { + log.err("{} user data was never given for current executable", .{Callback}); + return .create(call_frame.api, .failed_precondition, "failed to fetch user context" ++ @typeName(Callback)); + }; + const user_ctx: *Callback = @ptrCast(@alignCast(user_ctx_opaque)); + // We actually have one more constraint here, we force the Callback to have a platform field, + // and to correctly set it. + // Is this good ? We could also simplify this by registering ourselves the `Platform` type id. + const platform: Platform = user_ctx.platform; + + // Hook to get a cuda stream in the callback. + if (@hasField(Callback, "stream") and platform.target != .cpu) { + const stream = call_frame.api.stream(execution_context); + user_ctx.stream = stream; + } + + var callback_args: std.meta.ArgsTuple(@TypeOf(Callback.call)) = undefined; + callback_args[0] = user_ctx; + + inline for (1..callback_args.len, call_frame.args.buffers()) |i, ffi_buffer| { + const shape = shapeFromFfi(ffi_buffer); + var zml_buffer: Buffer = if (platform.target == .cpu) + .asViewOfHostBuffer(platform, .fromBytes(shape, ffi_buffer.data[0..shape.byteSize()])) + else + .asViewOfDeviceBuffer(platform, shape, null, ffi_buffer.data); + if (opts.copy_inputs_to_host_pinned and platform.target != .cpu) { + log.debug("Copying argument {d} {f} {*} to host_pinned memory !", .{ i, zml_buffer, zml_buffer.opaqueDeviceMemoryDataPointer() }); + zml_buffer = zml_buffer.copyToMemory(platform, .host_pinned, .{ .wait = true }) catch |err| { + log.err("Failed to copy input buffer {d} {f} {*} to host_pinned: {}", .{ i, zml_buffer, zml_buffer.opaqueDeviceMemoryDataPointer(), err }); + return .create(call_frame.api, .resource_exhausted, "host pinned OOM"); + }; + log.debug("--> {f} {*} ({})", .{ zml_buffer, zml_buffer.opaqueDeviceMemoryDataPointer(), @as(*const f32, @ptrCast(@alignCast(zml_buffer.opaqueDeviceMemoryDataPointer()))).* }); + } + callback_args[i] = zml_buffer; + } + + defer { + if (opts.copy_inputs_to_host_pinned and platform.target != .cpu) { + inline for (1..callback_args.len) |i| callback_args[i].deinit(); + } + } + + for (0..call_frame.results.len) |i| { + const ffi_buffer = call_frame.results.buffers()[i]; + const ffi_buffer_shape = shapeFromFfi(ffi_buffer); + + if (platform.target == .cpu) { + user_ctx.results[i] = Buffer.asViewOfHostBuffer(platform, HostBuffer.fromBytes(ffi_buffer_shape, ffi_buffer.data[0..ffi_buffer_shape.byteSize()])); + } else { + user_ctx.results[i] = Buffer.asViewOfDeviceBuffer(platform, shapeFromFfi(ffi_buffer), null, ffi_buffer.data); + } + } + + @call(.auto, Callback.call, callback_args) catch |err| { + log.err("Callback {} failed with {}", .{ Callback, err }); + return .create(call_frame.api, .internal, "internal callback error"); + }; + + return .ok; +} + +/// Internal custom calls. +/// These are not meant to be used by users, but rather by the library itself. +pub const internal_callbacks = [_]type{ + Print, +}; + +pub fn registerInternalCallbacks(platform: Platform) !void { + inline for (internal_callbacks) |Callback| { + try register(Callback, platform); + // log.debug("Registered internal custom call {s} with type_id {d}", .{ @typeName(Callback), Callback.type_id.type_id }); + } +} + +/// Allocate user data data needed by the ZML provided custom calls. +pub fn bindInternalCallbacks( + arena: std.mem.Allocator, + platform: Platform, + ffi: pjrt.Ffi, + execute_context: *pjrt.ExecuteContext, +) (std.mem.Allocator.Error || pjrt.ApiError)!void { + // Atm we don't have a mechanism to detect which ZML callbacks the executable needs, + // so we always allocate. + { + // Print + const print_ptr = try arena.create(Print); + print_ptr.* = try .init(platform); + try addUserData(Print, platform.pjrt_api, ffi, execute_context, print_ptr); + } +} + +pub fn addUserData( + Callback: type, + api: *const pjrt.Api, + ffi: pjrt.Ffi, + execute_context: *pjrt.ExecuteContext, + user_data: *Callback, +) pjrt.ApiError!void { + try ffi.addUserData( + api, + execute_context, + .{ .type_id = Callback.type_id.type_id, .user_data = @ptrCast(user_data) }, + ); + log.debug("Bound {s}@{x} with type id {d} on {any}", .{ @typeName(Callback), @intFromPtr(user_data), Callback.type_id.type_id, execute_context }); +} + +/// The print callback +pub const Print = struct { + // a unique type_id will be set by the PJRT plugin during registration. + pub var type_id: pjrt.ffi.TypeId = undefined; + + pub const callback_config: Config = .{ + // Print callback pretends to modify the given input buffer, but just returns it unmodified. + .output_operand_aliases = &.{0}, + // It also needs PJRT to copy the data on the host first so it can print it. + .copy_inputs_to_host_pinned = true, + // Print is fairly predictable and can be captured in an execution graph. + .traits = .{ .command_buffer_compatible = false }, + }; + + platform: Platform, + results: [1]Buffer = undefined, + + pub fn init(platform: Platform) !Print { + return .{ .platform = platform }; + } + + pub fn call(_: *Print, input: Buffer) !void { + std.log.defaultLog(.info, .zml, "Device buffer: {f}: {d:20.3}", .{ input, input.asHostBuffer() }); + } +}; + +fn shapeFromFfi(ffi_buffer: *const pjrt.ffi.Buffer) Shape { + const dt: DataType = switch (ffi_buffer.dtype) { + .invalid => stdx.debug.panic("Invalid FFI dtype {any} used by {any}", .{ ffi_buffer.dtype, ffi_buffer }), + .token, .f8e4m3, .f8e3m4 => stdx.debug.panic("Unsupported FFI dtype {any} used by {any}", .{ ffi_buffer.dtype, ffi_buffer }), + inline else => |t| @field(DataType, @tagName(t)), + }; + return Shape.init(ffi_buffer.dims(), dt); +} + +fn TensorArgs(Callback: type) type { + const ArgsT = stdx.meta.FnArgs(Callback.call); + + const args = @typeInfo(ArgsT).@"struct".fields; + return [args.len - 1]Tensor; +} diff --git a/zml/context.zig b/zml/context.zig index 15acb93b..27d635a0 100644 --- a/zml/context.zig +++ b/zml/context.zig @@ -7,16 +7,19 @@ const runfiles = @import("runfiles"); const runtimes = @import("runtimes"); const stdx = @import("stdx"); -const DataType = @import("dtype.zig").DataType; -const HostBuffer = @import("hostbuffer.zig").HostBuffer; const pjrt = @import("pjrtx.zig"); -const Platform = @import("platform.zig").Platform; -const Shape = @import("shape.zig").Shape; -const Target = @import("platform.zig").Target; -const zml_platform = @import("platform.zig"); -const PjrtApiMap = std.EnumArray(Target, ?*const pjrt.Api); -const PlatformsMap = std.EnumArray(Target, ?Platform); +const zml = struct { + pub const callback = @import("callback.zig"); + pub const HostBuffer = @import("hostbuffer.zig").HostBuffer; + pub const Platform = @import("platform.zig").Platform; + pub const platform = @import("platform.zig"); + pub const Shape = @import("shape.zig").Shape; + pub const Target = @import("platform.zig").Target; +}; + +const PjrtApiMap = std.EnumArray(zml.Target, ?*const pjrt.Api); +const PlatformsMap = std.EnumArray(zml.Target, ?zml.Platform); const log = std.log.scoped(.@"zml/context"); test { @@ -94,7 +97,7 @@ pub const Context = struct { return .{ .platforms = PlatformsMap.initFill(null) }; } - fn platformToLibrary(comptime target: Target) []const u8 { + fn platformToLibrary(comptime target: zml.Target) []const u8 { const ext = switch (builtin.os.tag) { .windows => ".dll", .macos, .ios, .watchos => ".dylib", @@ -105,7 +108,7 @@ pub const Context = struct { }; } - pub fn pjrtApi(target: Target) *const pjrt.Api { + pub fn pjrtApi(target: zml.Target) *const pjrt.Api { return Context.apis.get(target).?; } @@ -119,12 +122,12 @@ pub const Context = struct { self.* = undefined; } - const prefered_targets = [_]Target{ .tpu, .neuron, .cuda, .rocm, .cpu }; + const prefered_targets = [_]zml.Target{ .tpu, .neuron, .cuda, .rocm, .cpu }; /// Automatically selects the best Platform loaded in the current Context. /// /// For example, if supported, this will select a platform corresponding to an accelerator (GPU, TPU, ...). - pub fn autoPlatform(self: *Context, opts: Platform.CreateOptions) Platform { + pub fn autoPlatform(self: *Context, opts: zml.Platform.CreateOptions) zml.Platform { stdx.debug.assert(prefered_targets.len == apis.values.len, "New target need to be inserted inside `zml.Context.preferred_targets`", .{}); return self.platformByPreferences(opts, &prefered_targets); @@ -133,7 +136,7 @@ pub const Context = struct { /// Given a list of preferred targets to select the best Platform /// /// For example, if supported, this will select a platform corresponding to an accelerator (GPU, TPU, ...). - pub fn platformByPreferences(self: *Context, opts: Platform.CreateOptions, prefered: []const Target) Platform { + pub fn platformByPreferences(self: *Context, opts: zml.Platform.CreateOptions, prefered: []const zml.Target) zml.Platform { // Try prefered targets. for (prefered) |target| { if (apis.get(target) == null) continue; @@ -150,7 +153,7 @@ pub const Context = struct { // CPU should only be use as fallback. if (target == .cpu) continue; if (entry.value.* == null) continue; - if (std.mem.indexOfScalar(Target, prefered, target) != null) continue; + if (std.mem.indexOfScalar(zml.Target, prefered, target) != null) continue; return self.platform(target, opts) catch |err| { log.err("Failed to load platform .{s}: {}", .{ @tagName(target), err }); continue; @@ -164,25 +167,25 @@ pub const Context = struct { }; } - pub fn platform(self: *Context, target: Target, opts: Platform.CreateOptions) !Platform { + pub fn platform(self: *Context, target: zml.Target, opts: zml.Platform.CreateOptions) !zml.Platform { if (self.platforms.get(target)) |p| { return p; } const api = Context.apis.get(target); if (api == null) return error.PlatformNotCompiled; - const p = try Platform.init(target, api.?, opts); + const p = try zml.Platform.init(target, api.?, opts); if (p.getDevices().len == 0) { log.err("No device found for platform {} !", .{target}); return error.NoDevicesFound; } - try CustomCall.registerZmlCustomCalls(p); - self.platforms.set(target, p); + try zml.callback.registerInternalCallbacks(p); + return p; } - pub fn printAvailablePlatforms(self: Context, selected: Platform) void { + pub fn printAvailablePlatforms(self: Context, selected: zml.Platform) void { // List available targets log.info("Available Platforms:", .{}); const selected_prefix = "✅"; @@ -190,7 +193,7 @@ pub const Context = struct { const selected_postfix = "(AUTO-SELECTED)"; const not_selected_postfix = ""; - for (zml_platform.available_targets) |target| { + for (zml.platform.available_targets) |target| { log.info(" {s} {s} {s}", .{ if (target == selected.target) selected_prefix else not_selected_prefix, @tagName(target), @@ -211,133 +214,4 @@ pub const Context = struct { } } } - - pub const HostCallback = fn (?*anyopaque, []const HostBuffer, []const HostBuffer) void; -}; - -const CustomCall = struct { - pub fn registerZmlCustomCalls(platform: Platform) !void { - const ffi = platform.pjrt_api.ffi() orelse { - log.warn("Registering custom calls failed: No FFI Extension found in {s} PJRT Plugin.", .{@tagName(platform.target)}); - return; - }; - try ffi.register(platform.pjrt_api, "zmlHostBufferCallback", @tagName(platform.target), &hostBufferCallback, .{}); - } - - fn hostBufferCallback(call_frame: *pjrt.ffi.CallFrame) callconv(.c) ?*pjrt.ffi.Error { - if (call_frame.registeringHook()) return null; - - const callback_attr = call_frame.attrs.getByName(.scalar, "callback") orelse unreachable; - std.debug.assert(callback_attr.dtype == .u64); - const callback: *const Context.HostCallback = @ptrFromInt(callback_attr.get(usize)); - - const user_ctx_ptr = call_frame.attrs.getByName(.scalar, "user_context") orelse unreachable; - std.debug.assert(user_ctx_ptr.dtype == .u64); - const user_ctx: ?*anyopaque = @ptrFromInt(user_ctx_ptr.get(usize)); - - const input_buffers = stdx.stackSlice(8, HostBuffer, call_frame.args.len); - for (input_buffers, 0..) |*b, i| { - b.* = hostBufferFromPinnedBuffer(call_frame.args.buffers()[i]); - } - - const output_buffers = stdx.stackSlice(8, HostBuffer, call_frame.results.len); - for (output_buffers, 0..) |*b, i| { - b.* = hostBufferFromPinnedBuffer(call_frame.results.buffers()[i]); - } - - callback(user_ctx, input_buffers, output_buffers); - return null; - } -}; - -fn getShape(buffer_desc: *const pjrt.ffi.Buffer) Shape { - // log.warn("received buffer {}", .{buffer_desc}); - const dt: DataType = switch (buffer_desc.dtype) { - .invalid => @panic("invalid ffi"), - .pred => .bool, - .i8 => .i8, - .i16 => .i16, - .i32 => .i32, - .i64 => .i64, - .token, .f8e4m3, .f8e3m4 => @panic("Unsupported ffi type"), - inline else => |t| @field(DataType, @tagName(t)), - }; - return Shape.init(buffer_desc.dims(), dt); -} - -/// Create a HostBuffer from a ffi description of a buffer. -/// Normally the ffi describe device buffer but we assume they are located in pinned memory, -/// and therefore the data pointer is readable both from host and from device. -fn hostBufferFromPinnedBuffer(buffer_desc: *const pjrt.ffi.Buffer) HostBuffer { - const buffer_shape = getShape(buffer_desc); - return HostBuffer.fromBytes( - buffer_shape, - buffer_desc.data[0..buffer_shape.byteSize()], - ); -} - -pub const cuda = struct { - pub var streamSynchronize: StreamSynchronize = @ptrFromInt(0xdeadc00da00); - pub var cuLaunchHostFunc: CuLaunchHostFunc = @ptrFromInt(0xdeadc00da00); - var _memcpyAsync: MemcpyAsync = @ptrFromInt(0xdeadc00da00); - var _memcpyBlocking: MemcpyBlocking = @ptrFromInt(0xdeadc00da00); - - pub const MemcpyKind = enum(c_int) { - host_to_host = 0, - host_to_device = 1, - device_to_host = 2, - device_to_device = 3, - inferred = 4, - }; - - const MemcpyAsync = *const fn (dst: *anyopaque, src: *const anyopaque, count: usize, kind: MemcpyKind, stream: ?*anyopaque) callconv(.c) c_int; - const MemcpyBlocking = *const fn (dst: *anyopaque, src: *const anyopaque, count: usize, kind: MemcpyKind) callconv(.c) c_int; - const StreamSynchronize = *const fn (stream: *anyopaque) callconv(.c) c_int; - const CuLaunchHostFunc = *const fn (stream: *anyopaque, host_func: *const fn (user_data: *const anyopaque) callconv(.c) void, user_data: *const anyopaque) callconv(.c) c_int; - - pub fn init() void { - var cudart = std.DynLib.open("libcudart.so.12") catch { - log.err("cudart not found, callback will segfault", .{}); - return; - }; - defer cudart.close(); - - _memcpyAsync = cudart.lookup(MemcpyAsync, "cudaMemcpyAsync") orelse { - @panic("cudaMemcpyAsync not found"); - }; - _memcpyBlocking = cudart.lookup(MemcpyBlocking, "cudaMemcpy") orelse { - @panic("cudaMemcpy not found"); - }; - streamSynchronize = cudart.lookup(StreamSynchronize, "cudaStreamSynchronize") orelse { - @panic("cudaStreamSynchronize not found"); - }; - cuLaunchHostFunc = cudart.lookup(CuLaunchHostFunc, "cudaLaunchHostFunc") orelse { - @panic("cudaLaunchHostFunc not found"); - }; - } - - pub fn memcpyToHostBlocking(dst: []u8, src: *const anyopaque) void { - const err = _memcpyBlocking(dst.ptr, src, dst.len, .device_to_host); - check(err); - } - - pub fn memcpyToDeviceBlocking(dst: *anyopaque, src: []const u8) void { - const err = _memcpyBlocking(dst, src.ptr, src.len, .host_to_device); - check(err); - } - - pub fn memcpyToDeviceAsync(dst: *anyopaque, src: []const u8, stream: ?*anyopaque) void { - const err = _memcpyAsync(dst, src.ptr, src.len, .host_to_device, stream); - check(err); - } - - pub fn memcpyToHostAsync(dst: []u8, src: *const anyopaque, stream: ?*anyopaque) void { - const err = _memcpyAsync(dst.ptr, src, dst.len, .device_to_host, stream); - check(err); - } - - pub fn check(err: c_int) void { - if (err == 0) return; - stdx.debug.panic("CUDA error: {d}", .{err}); - } }; diff --git a/zml/dtype.zig b/zml/dtype.zig index b0deb8f4..00fa8112 100644 --- a/zml/dtype.zig +++ b/zml/dtype.zig @@ -12,20 +12,26 @@ test { pub const DataType = enum(u8) { bool, // Note: the support of the float8 is a bit spotty, f8e4m3b11fnuz seems to be the most supported one on Cuda. + f4e2m1, + f8e3m4, + f8e4m3, f8e4m3b11fnuz, f8e4m3fn, f8e4m3fnuz, f8e5m2, f8e5m2fnuz, + f8e8m0, bf16, f16, f32, f64, + i2, i4, i8, i16, i32, i64, + u2, u4, u8, u16, @@ -50,8 +56,21 @@ pub const DataType = enum(u8) { pub fn class(self: DataType) Class { return switch (self) { .bool => .bool, - .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz, .bf16, .f16, .f32, .f64 => .float, - .i4, .i8, .i16, .i32, .i64, .u4, .u8, .u16, .u32, .u64 => .integer, + .f4e2m1, + .f8e3m4, + .f8e4m3, + .f8e4m3b11fnuz, + .f8e4m3fn, + .f8e4m3fnuz, + .f8e5m2, + .f8e5m2fnuz, + .f8e8m0, + .bf16, + .f16, + .f32, + .f64, + => .float, + .i2, .i4, .i8, .i16, .i32, .i64, .u2, .u4, .u8, .u16, .u32, .u64 => .integer, .c64, .c128 => .complex, }; } @@ -70,21 +89,27 @@ pub const DataType = enum(u8) { pub fn fromZigType(comptime T: type) DataType { return switch (T) { + floats.Float4E2M1 => .f4e2m1, + floats.Float8E3M4 => .f8e3m4, + floats.Float8E4M3 => .f8e4m3, floats.Float8E4M3B11FNUZ => .f8e4m3b11fnuz, floats.Float8E4M3FN => .f8e4m3fn, floats.Float8E4M3FNUZ => .f8e4m3fnuz, floats.Float8E5M2 => .f8e5m2, floats.Float8E5M2FNUZ => .f8e5m2fnuz, + floats.Float8E8M0 => .f8e8m0, floats.BFloat16 => .bf16, f16 => .f16, f32 => .f32, f64 => .f64, bool => .bool, + i2 => .i2, i4 => .i4, i8 => .i8, i16 => .i16, i32 => .i32, i64 => .i64, + u2 => .u2, u4 => .u4, u8 => .u8, u16 => .u16, @@ -192,10 +217,10 @@ pub const DataType = enum(u8) { pub fn maxValue(dtype: DataType) Data { return switch (dtype) { .bool => .{ .bool = true }, - inline .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2fnuz => |tag| @panic("DataType doesn't have a max value: " ++ @tagName(tag)), - inline .f8e5m2, .bf16 => |tag| @unionInit(Data, @tagName(tag), @FieldType(Data, @tagName(tag)).inf), + inline .f4e2m1, .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2fnuz, .f8e8m0 => |tag| @panic("DataType doesn't have a max value: " ++ @tagName(tag)), + inline .f8e3m4, .f8e4m3, .f8e5m2, .bf16 => |tag| @unionInit(Data, @tagName(tag), @FieldType(Data, @tagName(tag)).inf), inline .f16, .f32, .f64 => |tag| @unionInit(Data, @tagName(tag), std.math.inf(@FieldType(Data, @tagName(tag)))), - inline .i4, .i8, .i16, .i32, .i64, .u4, .u8, .u16, .u32, .u64 => |tag| @unionInit(Data, @tagName(tag), std.math.maxInt(@FieldType(Data, @tagName(tag)))), + inline .i2, .i4, .i8, .i16, .i32, .i64, .u2, .u4, .u8, .u16, .u32, .u64 => |tag| @unionInit(Data, @tagName(tag), std.math.maxInt(@FieldType(Data, @tagName(tag)))), inline .c64, .c128 => |tag| @panic("DataType doesn't have a max value: " ++ @tagName(tag)), }; } @@ -207,20 +232,26 @@ pub const DataType = enum(u8) { pub const Data = union(DataType) { bool: bool, + f4e2m1: floats.Float4E2M1, + f8e3m4: floats.Float8E3M4, + f8e4m3: floats.Float8E4M3, f8e4m3b11fnuz: floats.Float8E4M3B11FNUZ, f8e4m3fn: floats.Float8E4M3FN, f8e4m3fnuz: floats.Float8E4M3FNUZ, f8e5m2: floats.Float8E5M2, f8e5m2fnuz: floats.Float8E5M2FNUZ, + f8e8m0: floats.Float8E8M0, bf16: floats.BFloat16, f16: f16, f32: f32, f64: f64, + i2: i2, i4: i4, i8: i8, i16: i16, i32: i32, i64: i64, + u2: u2, u4: u4, u8: u8, u16: u16, @@ -244,7 +275,7 @@ pub const Data = union(DataType) { .comptime_int, .int, .comptime_float, .float => .{ .bool = value != 0 }, else => @panic("Could not create Data of type bool from value of type " ++ @typeName(T)), }, - inline .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz, .bf16 => |tag| switch (Ti) { + inline .f4e2m1, .f8e3m4, .f8e4m3, .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz, .f8e8m0, .bf16 => |tag| switch (Ti) { .comptime_int, .int => @unionInit(Data, @tagName(tag), @FieldType(Data, @tagName(tag)).fromF32(@floatFromInt(value))), .comptime_float, .float => @unionInit(Data, @tagName(tag), @FieldType(Data, @tagName(tag)).fromF32(@floatCast(value))), else => @panic("Could not create Data of type bf16 from value of type " ++ @typeName(T)), @@ -254,13 +285,13 @@ pub const Data = union(DataType) { .comptime_float, .float => @unionInit(Data, @tagName(tag), @floatCast(value)), else => @panic("Could not create Data of type " ++ @tagName(tag) ++ " from value of type " ++ @typeName(T)), }, - inline .i4, .i8, .i16, .i32, .i64, .u4, .u8, .u16, .u32, .u64 => |tag| switch (Ti) { + inline .i2, .i4, .i8, .i16, .i32, .i64, .u2, .u4, .u8, .u16, .u32, .u64 => |tag| switch (Ti) { .comptime_int => blk: { const OutT = @FieldType(Data, @tagName(tag)); if (value >= std.math.minInt(OutT) and value <= std.math.maxInt(OutT)) { break :blk @unionInit(Data, @tagName(tag), @intCast(value)); } else { - @panic("Could not create Data of type " ++ @tagName(tag) ++ " from value of type " ++ @typeName(T)); + std.debug.panic("Could not create Data of type {s} from value: {d}", .{ @tagName(tag), value }); } }, .int => @unionInit(Data, @tagName(tag), @intCast(value)), diff --git a/zml/exe.zig b/zml/exe.zig index 625fecf9..fd728fd8 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -5,6 +5,7 @@ const stdx = @import("stdx"); const aio = @import("aio.zig"); const Buffer = @import("buffer.zig").Buffer; const Bufferized = @import("tensor.zig").Bufferized; +const callback = @import("callback.zig"); const CompilationContext = @import("module.zig").CompilationContext; const meta = @import("meta.zig"); const pjrt = @import("pjrtx.zig"); @@ -154,7 +155,7 @@ pub const BaseExe = struct { exe: *pjrt.LoadedExecutable, /// The execution context for this executable. - context: ?*pjrt.ExecuteContext = null, + execute_context: ?*pjrt.ExecuteContext, /// Pre-allocated slice of buffers to use as inputs when the module is called. input_per_device: []const [*]*pjrt.Buffer, @@ -166,10 +167,10 @@ pub const BaseExe = struct { ready_buffer_count: u32, /// Total number of buffers needed by this executable. - input_buffer_count: u32, + num_inputs: u32, input_shapes: []Shape, - result_shapes: []Shape, + output_shapes: []Shape, /// Num devices used (>1 for sharded executable) num_devices: u8, @@ -181,53 +182,76 @@ pub const BaseExe = struct { parent_allocator: std.mem.Allocator, platform: Platform, exe: *pjrt.LoadedExecutable, - args: struct { input_shapes: []const Shape, result_shapes: []const Shape, n_devices: u8 }, + args: struct { input_shapes: []const Shape, output_shapes: []const Shape, n_devices: u8 }, ) !BaseExe { + // TODO: considere if we still need the arena since we only allocate once. var arena = std.heap.ArenaAllocator.init(parent_allocator); errdefer arena.deinit(); const allocator = arena.allocator(); + const n_in = args.input_shapes.len; - const n_out = args.result_shapes.len; + const n_out = args.output_shapes.len; const n_devices = args.n_devices; - // Allocate once for all the *pjrt.Buffer we need to store ... - const all_buffers = try allocator.alloc(*pjrt.Buffer, (n_in + n_out) * n_devices); - const all_input_buffers, const all_output_buffers = splitBuffer(*pjrt.Buffer, all_buffers, .{ n_in * n_devices, n_out * n_devices }); - // ... and once for all the [*]*pjrt.Buffer. - const all_per_device = try allocator.alloc([*]*pjrt.Buffer, 2 * n_devices); - const input_per_device, const output_per_device = splitBuffer([*]*pjrt.Buffer, all_per_device, .{ n_devices, n_devices }); + const Slices = struct { + all_input_buffers: []*pjrt.Buffer, + all_output_buffers: []*pjrt.Buffer, + input_per_device: [][*]*pjrt.Buffer, + output_per_device: [][*]*pjrt.Buffer, + input_shapes: []Shape, + output_shapes: []Shape, + }; + const slices = try stdx.mem.groupedAlloc(Slices, allocator, .{ + n_in * n_devices, + n_out * n_devices, + n_devices, + n_devices, + n_in, + n_out, + }); + + // Init the slices. + // all_input_buffers is left uninitialized, will be fed by `prepare` and `call` + // all_output_buffers will be written to by pjrt.execute for (0..n_devices) |i| { - input_per_device[i] = all_input_buffers[i * n_in ..].ptr; - output_per_device[i] = all_output_buffers[i * n_out ..].ptr; + slices.input_per_device[i] = slices.all_input_buffers[i * n_in ..].ptr; + slices.output_per_device[i] = slices.all_output_buffers[i * n_out ..].ptr; + } + @memcpy(slices.input_shapes, args.input_shapes); + @memcpy(slices.output_shapes, args.output_shapes); + + var execute_context: ?*pjrt.ExecuteContext = null; + if (platform.pjrt_api.ffi()) |ffi| { + log.info("Created context execution {*} for {*}", .{ execute_context, exe }); + execute_context = try platform.pjrt_api.createExecuteContext(); + try callback.bindInternalCallbacks(allocator, platform, ffi, execute_context.?); } - const all_shapes = try allocator.alloc(Shape, n_in + n_out); - @memcpy(all_shapes[0..n_in], args.input_shapes); - @memcpy(all_shapes[n_in..], args.result_shapes); return .{ .platform = platform, .exe = exe, + .execute_context = execute_context, .ready_buffer_count = 0, - .input_buffer_count = @intCast(n_in), + .num_inputs = @intCast(n_in), .num_devices = args.n_devices, - .input_per_device = input_per_device, - .output_per_device = output_per_device, - .input_shapes = all_shapes[0..n_in], - .result_shapes = all_shapes[n_in..], + .input_per_device = slices.input_per_device, + .output_per_device = slices.output_per_device, + .input_shapes = slices.input_shapes, + .output_shapes = slices.output_shapes, ._arena = arena, }; } pub fn deinit(self: BaseExe) void { - if (self.context) |ctx| { + if (self.execute_context) |ctx| { ctx.deinit(self.platform.pjrt_api); } self._arena.deinit(); } pub fn call(self: BaseExe) void { - stdx.debug.assert(self.input_buffer_count == self.ready_buffer_count, "BaseExe isn't ready to be called, expected {} buffer inputs got {}", .{ self.input_buffer_count, self.ready_buffer_count }); + stdx.debug.assert(self.num_inputs == self.ready_buffer_count, "BaseExe isn't ready to be called, expected {} buffer inputs got {}", .{ self.num_inputs, self.ready_buffer_count }); return self._unsafeCall(); } @@ -237,23 +261,23 @@ pub const BaseExe = struct { self.exe.execute(self.platform.pjrt_api, .{ .arguments = self.input_per_device, - .num_args = self.input_buffer_count, + .num_args = self.num_inputs, .results = self.output_per_device, .events = events[0..sharding.num_partitions], // this allows to tell a specific buffer shouldn't be donated, // even if it has been marked as "can be donated" during compilation. // TODO: expose it ? .non_donatable_input_indices = &.{}, - .context = self.context, + .context = self.execute_context, }) catch |err| { std.debug.panic("PJRT_LoadedExecutable_Execute failed with: {}", .{err}); }; - for (events[0..sharding.num_partitions]) |e| { - if (e) |ev| { - ev.await_(self.platform.pjrt_api) catch unreachable; - } - } + // for (events[0..sharding.num_partitions]) |e| { + // if (e) |ev| { + // ev.await_(self.platform.pjrt_api) catch unreachable; + // } + // } } pub fn _unsafeAssignResults(self: BaseExe, T: type, result: *T) void { @@ -267,7 +291,7 @@ pub const BaseExe = struct { .index = 0, .platform = self.platform, .outputs = self.output_per_device, - .output_shapes = self.result_shapes, + .output_shapes = self.output_shapes, }; meta.visit((struct { fn cb(ctx: *LocalContext, buffer: *Buffer) void { @@ -282,7 +306,18 @@ pub const BaseExe = struct { buffer.* = Buffer.fromPjrtBuffers(ctx.platform, ctx.output_shapes[i], shards.constSlice()); } }).cb, &local_ctx, result); - stdx.debug.internalAssert(local_ctx.index == self.result_shapes.len, "Pjrt call returned {} tensors, but the return type {s}, contains {} Buffers. Note that modules need to have a comptime know number of returned tensors.", .{ self.output_per_device.len, @typeName(T), local_ctx.index }); + stdx.debug.internalAssert(local_ctx.index == self.output_shapes.len, "Pjrt call returned {} tensors, but the return type {s}, contains {} Buffers. Note that modules need to have a comptime know number of returned tensors.", .{ self.output_per_device.len, @typeName(T), local_ctx.index }); + } + + pub fn bind(exe: BaseExe, Callback: type, op: *Callback) !void { + stdx.debug.assert(exe.execute_context != null, "Exe doesn't have an execution context", .{}); + const pjrt_api = exe.platform.pjrt_api; + + if (pjrt_api.ffi()) |ffi| { + try callback.addUserData(Callback, pjrt_api, ffi, exe.execute_context.?, op); + } else { + stdx.debug.panic("Callbacks are not supported for target {s}", .{@tagName(exe.platform.target)}); + } } pub fn serialize(self: BaseExe, writer: anytype) !void { @@ -309,16 +344,16 @@ pub const BaseExe = struct { shards.appendAssumeCapacity(dev_out[i]); } - return Buffer.fromPjrtBuffers(self.platform, self.result_shapes[i], shards.constSlice()); + return Buffer.fromPjrtBuffers(self.platform, self.output_shapes[i], shards.constSlice()); } - pub fn clone(self: BaseExe, parent_allocator: std.mem.Allocator) !BaseExe { + pub fn clone(self: BaseExe, parent_allocator: std.mem.Allocator) error{OutOfMemory}!BaseExe { var exe: BaseExe = try .init(parent_allocator, self.platform, self.exe, .{ - .n_in = self.input_buffer_count, - .result_shapes = self.result_shapes, + .input_shapes = self.input_shapes, + .output_shapes = self.output_shapes, .n_devices = self.num_devices, }); - exe.context = self.context; + exe.execute_context = self.execute_context; return exe; } }; @@ -342,12 +377,21 @@ pub fn Exe(ArgsT: type, ReturnT: type) type { /// /// **Warning:** the new Exe reuses the underlying memory of the previous one. /// The caller is responsible to come up with a strategy to call `deinit` exactly once. + /// If needed, new executable with independent memory can be created with `clone`. pub fn prepare(self: Self, first_arg: Bufferized(stdx.meta.Head(ArgsT))) Exe(stdx.meta.Tail(ArgsT), ReturnT) { var new: Exe(stdx.meta.Tail(ArgsT), ReturnT) = .{ .inner = self.inner }; new.inner.prepare(first_arg); return new; } + /// For a given customCall inside this executable, + /// provide a pointer to runtime data. + /// The caller keeps memory ownership and need to ensure that the value + /// stays alive as long as the executable. + pub fn bind(self: Self, comptime T: type, value: *T) !void { + try self.inner.bind(T, value); + } + pub fn serialize(self: Self, writer: anytype) !void { return try self.inner.serialize(writer); } @@ -358,12 +402,16 @@ pub fn Exe(ArgsT: type, ReturnT: type) type { pub fn call(self: Self, args: Bufferized(ArgsT)) Bufferized(ReturnT) { const total_ready = fillBuffers(&args, self.inner.input_shapes, self.inner.input_per_device, self.inner.ready_buffer_count); - std.debug.assert(total_ready == self.inner.input_buffer_count); + std.debug.assert(total_ready == self.inner.num_inputs); self.inner._unsafeCall(); var result: Bufferized(ReturnT) = undefined; self.inner._unsafeAssignResults(Bufferized(ReturnT), &result); return result; } + + pub fn clone(self: Self, allocator: std.mem.Allocator) error{OutOfMemory}!Self { + return .{ .inner = try self.inner.clone(allocator) }; + } }; } diff --git a/zml/floats.zig b/zml/floats.zig index 57bf5150..3e19dff8 100644 --- a/zml/floats.zig +++ b/zml/floats.zig @@ -48,6 +48,9 @@ fn FloatHelpers(Float: type) type { return if (vf32.sign == 0) Float.inf else Float.minus_inf; } else Float.nan; } + // TODO: This is too lossy. + // If exponent underflow, and the mantissa is zero we drop to 0, + // while we should set the mantissa to it's max value. return .{ .sign = vf32.sign, .exponent = @intCast(exponent), @@ -129,11 +132,7 @@ pub const Float8E4M3B11FNUZ = packed struct(u8) { exponent: u4, sign: u1, - pub const nan: Float8E4M3B11FNUZ = .{ - .sign = 1, - .exponent = 0, - .mantissa = 0, - }; + pub const nan: Float8E4M3B11FNUZ = .{ .sign = 1, .exponent = 0, .mantissa = 0 }; pub fn isNan(self: Float8E4M3B11FNUZ) bool { return self.sign == 1 and self.exponent == 0 and self.mantissa == 0; @@ -155,7 +154,7 @@ pub const Float8E4M3FN = packed struct(u8) { pub const nan: Float8E4M3FN = .{ .sign = 0, .exponent = std.math.maxInt(u4), .mantissa = std.math.maxInt(u3) }; pub fn isNan(self: Float8E4M3FN) bool { - return allBitsOne(self.exponent) and allBitsOne(self.mantissa); + return self.exponent == nan.exponent and self.mantissa == nan.mantissa; } const Helpers = FloatHelpers(@This()); pub const zero = Helpers.zero; @@ -170,11 +169,7 @@ pub const Float8E4M3FNUZ = packed struct(u8) { exponent: u4, sign: u1, - pub const nan: Float8E4M3FNUZ = .{ - .sign = 1, - .exponent = 0, - .mantissa = 0, - }; + pub const nan: Float8E4M3FNUZ = .{ .sign = 1, .exponent = 0, .mantissa = 0 }; pub fn isNan(self: Float8E4M3FNUZ) bool { return self.sign == 1 and self.exponent == 0 and self.mantissa == 0; @@ -209,14 +204,10 @@ pub const Float8E5M2 = packed struct(u8) { exponent: u5, sign: u1, - pub const nan: Float8E5M2 = .{ - .sign = 0, - .exponent = std.math.maxInt(u5), - .mantissa = 1, - }; + pub const nan: Float8E5M2 = .{ .sign = 0, .exponent = std.math.maxInt(u5), .mantissa = 1 }; pub fn isNan(self: Float8E5M2) bool { - return allBitsOne(self.exponent) and self.mantissa != 0; + return self.exponent == nan.exponent and self.mantissa != 0; } pub const minus_inf: Float8E5M2 = .{ @@ -232,7 +223,7 @@ pub const Float8E5M2 = packed struct(u8) { }; pub fn isInf(self: Float8E5M2) bool { - return allBitsOne(self.exponent) and self.mantissa == 0; + return self == inf or self == minus_inf; } const Helpers = FloatHelpers(@This()); @@ -333,10 +324,157 @@ test BFloat16 { }); } +pub const Float8E4M3 = packed struct(u8) { + mantissa: u3, + exponent: u4, + sign: u1, + + pub const nan: Float8E4M3 = @bitCast(0xFF); + + pub fn isNan(self: Float8E4M3) bool { + return self == nan or self == comptime nan.neg(); + } + + pub const minus_inf: Float8E4M3 = .{ + .sign = 1, + .exponent = std.math.maxInt(u4), + .mantissa = 0, + }; + + pub const inf: Float8E4M3 = .{ + .sign = 0, + .exponent = std.math.maxInt(u4), + .mantissa = 0, + }; + + const Helpers = FloatHelpers(@This()); + pub const zero = Helpers.zero; + pub const neg = Helpers.neg; + pub const fromF32 = Helpers.fromF32; + pub const toF32 = Helpers.toF32; + pub const formatNumber = Helpers.formatNumber; +}; + +pub const Float8E3M4 = packed struct(u8) { + mantissa: u4, + exponent: u3, + sign: u1, + + pub const nan: Float8E3M4 = @bitCast(0xFF); + + pub fn isNan(self: Float8E3M4) bool { + return self == nan or self == comptime nan.neg(); + } + + pub const minus_inf: Float8E3M4 = .{ + .sign = 1, + .exponent = std.math.maxInt(u3), + .mantissa = 0, + }; + + pub const inf: Float8E3M4 = .{ + .sign = 0, + .exponent = std.math.maxInt(u3), + .mantissa = 0, + }; + + const Helpers = FloatHelpers(@This()); + pub const zero = Helpers.zero; + pub const neg = Helpers.neg; + pub const fromF32 = Helpers.fromF32; + pub const toF32 = Helpers.toF32; + pub const formatNumber = Helpers.formatNumber; +}; + +pub const Float8E8M0 = packed struct(u8) { + mantissa: u0 = 0, + exponent: u8, + sign: u0 = 0, + + /// Lossy conversion from f32, similar to @floatCast + pub fn fromF32(f: f32) Float8E8M0 { + const vf32: Float32 = @bitCast(f); + return .{ .exponent = @intCast(vf32.exponent) }; + } + + /// Lossless conversion to f32. + pub fn toF32(x: Float8E8M0) f32 { + if (x.exponent == 0) return min_scale; + const vf32: Float32 = .{ + .sign = 0, + .exponent = x.exponent, + .mantissa = 0, + }; + return @bitCast(vf32); + } + + pub const min_scale: f32 = Float32.toF32(.{ .sign = 0, .exponent = 0, .mantissa = 0b1 << 22 }); + + const Helpers = FloatHelpers(@This()); + pub const formatNumber = Helpers.formatNumber; +}; + +test Float8E8M0 { + try std.testing.expectEqual(Float8E8M0{ .exponent = 127 }, Float8E8M0.fromF32(1.0)); + // try std.testing.expectEqual(5.877472e-39, Float8E8M0.toF32(.{ .exponent = 0})); + + try testCustomFloat(Float8E8M0, .{ + .lossless = &[_]f32{ Float8E8M0.min_scale, 1.0, 64.0, 1.0 / 128.0, std.math.pow(f32, 2.0, 127) }, + .lossy = &[_]f32{1.00001}, + }); +} + +pub const Float4E2M1 = packed struct(u4) { + mantissa: u1, + exponent: u2, + sign: u1, + + pub const nan: Float4E2M1 = @bitCast(@as(u4, 0xF)); + const Helpers = FloatHelpers(@This()); + pub const zero = Helpers.zero; + pub const neg = Helpers.neg; + pub const fromF32 = Helpers.fromF32; + pub const formatNumber = Helpers.formatNumber; + + pub const values = [_]f32{ 0.0, 0.5, 1, 1.5, 2, 3, 4, 6, -0.0, -0.5, -1, -1.5, -2, -3, -4, -6 }; + + pub fn toF32(x: Float4E2M1) f32 { + // the baseline toF32 doesn't work correctly: + // 0b0001 and 0b1001 shoud map to ±0.5, but are mapped to ±epsilon + return values[@as(u4, @bitCast(x))]; + } + + test toF32 { + var to_f32_res: [16]f32 = undefined; + for (&to_f32_res, 0..) |*r, i| { + const x_f4: Float4E2M1 = @bitCast(@as(u4, @intCast(i))); + r.* = x_f4.toF32(); + } + try std.testing.expectEqualSlices(f32, &Float4E2M1.values, &to_f32_res); + } + + test fromF32 { + // the baseline fromF32 doesn't work correctly: + // ±0.5 should map to 0b0001/0b1001 but are map to ±0.0 instead. + // TODO: it probably affects other types. + var from_f32_res: [16]Float4E2M1 = undefined; + for (&from_f32_res, 0..) |*r, i| { + r.* = .fromF32(Float4E2M1.values[i]); + } + try std.testing.expectEqualSlices(u4, &.{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }, @ptrCast(&from_f32_res)); + } +}; + pub fn floatCast(T: type, x: anytype) T { - return switch (@TypeOf(x)) { - f64, f32, f16 => @floatCast(x), - else => @floatCast(x.toF32()), + return switch (T) { + f64, f32, f16 => switch (@TypeOf(x)) { + f64, f32, f16 => @floatCast(x), + else => @floatCast(x.toF32()), + }, + else => switch (@TypeOf(x)) { + f64, f32, f16 => .fromF32(x), + else => .fromF32(x.toF32()), + }, }; } diff --git a/zml/hostbuffer.zig b/zml/hostbuffer.zig index c2099b0d..66b51328 100644 --- a/zml/hostbuffer.zig +++ b/zml/hostbuffer.zig @@ -40,7 +40,7 @@ pub const HostBuffer = struct { /// The returned HostBuffer doesn't take ownership of the slice /// that will still need to be deallocated. pub fn fromBytes(shape_: Shape, data_: []const u8) HostBuffer { - stdx.debug.assert(shape_.byteSize() == data_.len, "shape {} and data {} don't match", .{ shape_.byteSize(), data_.len }); + stdx.debug.assert(shape_.byteSize() == data_.len, "shape ({} bytes) and data ({} bytes) don't match", .{ shape_.byteSize(), data_.len }); return .{ ._shape = shape_, ._strides = shape_.computeStrides().buffer, @@ -96,7 +96,10 @@ pub const HostBuffer = struct { pub fn fromArray(arr_ptr: anytype) HostBuffer { const T = @TypeOf(arr_ptr.*); const sh = parseArrayInfo(T); - std.debug.assert(sh.byteSize() == @sizeOf(T)); + if (sh.dtype().bitSizeOf() < 8) { + // Zig isn't correctly packing the sub byte types in arrays. + stdx.debug.assert(sh.byteSize() == @sizeOf(T), "fromArray doesn't work on {}, cause the pjrt layout ({d} bytes) doesn't match the Zig layout ({d} bytes)", .{ T, sh.byteSize(), @sizeOf(T) }); + } return .{ ._shape = sh, ._strides = sh.computeStrides().buffer, @@ -325,37 +328,18 @@ pub const HostBuffer = struct { self: HostBuffer, writer: anytype, ) !void { - // TODO debug option - // try writer.print("HostBuffer(.{f})@0x{x}", .{ self._shape, @intFromPtr(self._data) }); try writer.print("HostBuffer(.{f})", .{self._shape}); } - /// Formatter for a HostBuffer that also print the values not just the shape. - /// Usage: `std.log.info("my buffer: {}", .{buffer.pretty()});` - pub fn pretty(self: HostBuffer) PrettyPrinter { - return .{ .x = self }; + pub fn formatNumber(self: HostBuffer, writer: *std.io.Writer, n: std.fmt.Number) std.io.Writer.Error!void { + return self.prettyPrintIndented(writer, 4, 0, n); } - pub const PrettyPrinter = struct { - x: HostBuffer, - - // TODO(0.15.0) revisit pretty printer - pub fn format(self: PrettyPrinter, writer: anytype) !void { - const fmt_: stdx.fmt.Fmt = switch (self.x.dtype().class()) { - .integer => .parse(i32, "d"), - .float => .parse(f32, "d"), - else => .parse(void, ""), - }; - const options: std.fmt.FormatOptions = .{}; - try prettyPrint(self.x, writer, .{ .fmt = fmt_, .options = options }); - } - }; - - pub fn prettyPrint(self: HostBuffer, writer: *std.Io.Writer, options: stdx.fmt.FullFormatOptions) !void { + pub fn prettyPrint(self: HostBuffer, writer: *std.Io.Writer, options: std.fmt.Number) !void { return self.prettyPrintIndented(writer, 4, 0, options); } - fn prettyPrintIndented(self: HostBuffer, writer: *std.Io.Writer, num_rows: u8, indent_level: u8, options: stdx.fmt.FullFormatOptions) !void { + fn prettyPrintIndented(self: HostBuffer, writer: *std.Io.Writer, num_rows: u8, indent_level: u8, options: std.fmt.Number) !void { if (self.rank() == 0) { // Special case input tensor is a scalar return switch (self.dtype()) { @@ -363,9 +347,10 @@ pub const HostBuffer = struct { const val: dt.toZigType() = self.items(dt.toZigType())[0]; return switch (comptime dt.class()) { // Since we have custom floats, we need to explicitly convert to float32 ourselves. - .float => stdx.fmt.formatFloatValue(floats.floatCast(f32, val), options, writer), - .integer => stdx.fmt.formatIntValue(val, options, writer), - .bool, .complex => stdx.fmt.formatAnyValue(val, options, writer), + .float => stdx.fmt.formatFloat(floats.floatCast(f32, val), options, writer), + .integer => stdx.fmt.formatInt(val, options, writer), + .bool => stdx.fmt.formatBool(val, options, writer), + .complex => stdx.fmt.formatComplex(val, options, writer), }; }, }; @@ -376,11 +361,17 @@ pub const HostBuffer = struct { try writer.splatByteAll(' ', indent_level); switch (self.dtype()) { inline else => |dt| { - const values = self.items(dt.toZigType()); + const T = dt.toZigType(); + const n: i64 = self._shape.dim(0); + // TODO: handle negative strides + const byte_stride: i64 = self._strides[0]; + const elem_strides: i64 = @divExact(byte_stride, @sizeOf(T)); + const values: []const T = @ptrCast(@alignCast(self._data[0..@intCast(n * byte_stride)])); switch (comptime dt.class()) { - .float => try stdx.fmt.formatFloatSlice(values, options, writer), - .integer => try stdx.fmt.formatIntSlice(values, options, writer), - .bool, .complex => try stdx.fmt.formatAnySlice(values, options, writer), + .float => try stdx.fmt.formatFloatSlice(values, options, elem_strides, writer), + .integer => try stdx.fmt.formatIntSlice(values, options, elem_strides, writer), + .complex => try stdx.fmt.formatComplexSlice(values, options, elem_strides, writer), + .bool => try stdx.fmt.formatBoolSlice(values, options, elem_strides, writer), } }, } diff --git a/zml/meta.zig b/zml/meta.zig index ee2fbc68..855318a5 100644 --- a/zml/meta.zig +++ b/zml/meta.zig @@ -141,6 +141,7 @@ test MapType { /// Any `To` struct inside `from` will be copied over to the target. pub fn mapAlloc(comptime cb: anytype, allocator: std.mem.Allocator, ctx: FnParam(cb, 0), from: anytype, to: anytype) !void { // TODO: handle tuple to slice conversion + // TODO: handle error bubbling up const From = FnParam(cb, 1); const To = stdx.meta.FnResult(cb); const FromStruct = @TypeOf(from); diff --git a/zml/mlirx.zig b/zml/mlirx.zig index 28f40383..7002a664 100644 --- a/zml/mlirx.zig +++ b/zml/mlirx.zig @@ -35,20 +35,26 @@ pub const Type = struct { pub fn fromDType(ctx: mlir.Context, dt: dtype.DataType) mlir.Type { return switch (dt) { .bool => .int(ctx, .i1), + .f4e2m1 => .float(ctx, .f4e2m1fn), + .f8e3m4 => .float(ctx, .f8e3m4), + .f8e4m3 => .float(ctx, .f8e4m3), .f8e4m3b11fnuz => .float(ctx, .f8e4m3b11fnuz), .f8e4m3fn => .float(ctx, .f8e4m3fn), .f8e4m3fnuz => .float(ctx, .f8e4m3fnuz), .f8e5m2 => .float(ctx, .f8e5m2), .f8e5m2fnuz => .float(ctx, .f8e5m2fnuz), + .f8e8m0 => .float(ctx, .f8e8m0fnu), .bf16 => .float(ctx, .bf16), .f16 => .float(ctx, .f16), .f32 => .float(ctx, .f32), .f64 => .float(ctx, .f64), + .i2 => .int(ctx, .i2), .i4 => .int(ctx, .i4), .i8 => .int(ctx, .i8), .i16 => .int(ctx, .i16), .i32 => .int(ctx, .i32), .i64 => .int(ctx, .i64), + .u2 => .int(ctx, .u2), .u4 => .int(ctx, .u4), .u8 => .int(ctx, .u8), .u16 => .int(ctx, .u16), @@ -62,23 +68,28 @@ pub const Type = struct { pub fn toDType(mlir_type: mlir.Type) dtype.DataType { const mapping = .{ .{ .bool, mlir.IntegerType(.i1) }, - + .{ .f4e2m1, mlir.FloatType(.f4e2m1fn) }, + .{ .f8e3m4, mlir.FloatType(.f8e3m4) }, + .{ .f8e4m3, mlir.FloatType(.f8e4m3) }, .{ .f8e4m3b11fnuz, mlir.FloatType(.f8e4m3b11fnuz) }, .{ .f8e4m3fn, mlir.FloatType(.f8e4m3fn) }, .{ .f8e4m3fnuz, mlir.FloatType(.f8e4m3fnuz) }, .{ .f8e5m2, mlir.FloatType(.f8e5m2) }, .{ .f8e5m2fnuz, mlir.FloatType(.f8e5m2fnuz) }, + .{ .f8e8m0, mlir.FloatType(.f8e8m0fnu) }, .{ .bf16, mlir.FloatType(.bf16) }, .{ .f16, mlir.FloatType(.f16) }, .{ .f32, mlir.FloatType(.f32) }, .{ .f64, mlir.FloatType(.f64) }, + .{ .i2, mlir.IntegerType(.i2) }, .{ .i4, mlir.IntegerType(.i4) }, .{ .i8, mlir.IntegerType(.i8) }, .{ .i16, mlir.IntegerType(.i16) }, .{ .i32, mlir.IntegerType(.i32) }, .{ .i64, mlir.IntegerType(.i64) }, + .{ .u2, mlir.IntegerType(.u2) }, .{ .u4, mlir.IntegerType(.u4) }, .{ .u8, mlir.IntegerType(.u8) }, .{ .u16, mlir.IntegerType(.u16) }, @@ -89,6 +100,8 @@ pub const Type = struct { .{ .c128, mlir.ComplexType(.c128) }, }; + // TODO: this seems quite slow to have all of those functions calls. + // Maybe we should memoize the ptr of a set of mlir types when creating the context. inline for (mapping) |entry| { const dt, const mlirT = entry; if (mlirT.is_a_fn(mlir_type._inner)) { diff --git a/zml/module.zig b/zml/module.zig index 33d22f78..30852ba8 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -234,7 +234,7 @@ pub const CompilationContext = struct { loaded_executable, .{ .input_shapes = f.args_shapes, - .result_shapes = f.res_shapes, + .output_shapes = f.res_shapes, .n_devices = sharding.num_replicas * sharding.num_partitions, }, ); @@ -1178,9 +1178,9 @@ pub fn hash(hasher: *std.hash.Wyhash, key: anytype, comptime strat: std.hash.Str .@"anyframe", .@"fn" => hash(hasher, @intFromPtr(key), strat), .pointer => |info| switch (info.size) { .one => switch (strat) { - .shallow => hash(hasher, @intFromPtr(key), .Shallow), - .deep => hash(hasher, key.*, .Shallow), - .deeprecursive => switch (@typeInfo(info.child)) { + .Shallow => hash(hasher, @intFromPtr(key), .Shallow), + .Deep => hash(hasher, key.*, .Shallow), + .DeepRecursive => switch (@typeInfo(info.child)) { .@"opaque", .@"fn" => hash(hasher, @intFromPtr(key), .Shallow), else => hash(hasher, key.*, .DeepRecursive), }, @@ -1196,7 +1196,7 @@ pub fn hash(hasher: *std.hash.Wyhash, key: anytype, comptime strat: std.hash.Str .many, .c, => switch (strat) { - .shallow => hash(hasher, @intFromPtr(key), .Shallow), + .Shallow => hash(hasher, @intFromPtr(key), .Shallow), else => @compileError( \\ unknown-length pointers and C pointers cannot be hashed deeply. \\ Consider providing your own hash function. diff --git a/zml/nn.zig b/zml/nn.zig index 631bce16..28958420 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -3,11 +3,13 @@ const std = @import("std"); const assert = std.debug.assert; const testing = std.testing; +const dialect = @import("mlir/dialects"); const stdx = @import("stdx"); const DataType = @import("dtype.zig").DataType; const helpers = @import("helpers.zig"); const meta = @import("meta.zig"); +const mlirx = @import("mlirx.zig"); const cuda = @import("nn/cuda.zig"); const ops = @import("ops.zig"); const Shape = @import("shape.zig").Shape; @@ -32,7 +34,71 @@ pub const Linear = struct { } // log.debug("Linear({*}): {d} -> {d} -> {d}", .{ self, x.dims(), y.dims(), if (self.bias) |bias| y.add(bias).dims() else y.dims() }); - return if (self.bias) |bias| y.add(bias.broadcastLeft(y.shape())) else y; + return if (self.bias) |bias| y.add(bias.broadcast(y.shape(), &.{y.axis(-1)})) else y; + } +}; + +pub const BlockScaledLinear = struct { + blocks: Tensor, + scale: Tensor, + bias: ?Tensor = null, + blocks_dtype: DataType, + + pub fn forward(self: BlockScaledLinear, x: Tensor) Tensor { + const ctx = x.getContext(); + const res_shape = x.shape().setDim(-1, self.blocks.dim(-3)); + log.warn("BlockScaledLinear({f} ({t}), {f}, {f}) -> {f}", .{ self.blocks, self.blocks_dtype, self.scale, x, res_shape }); + + // Bitcast to our actual type. This allows to load weights in a packed layout. + const blocks_0 = self.blocks.bitCast(self.blocks_dtype); + const blocks = blocks_0.merge(.{ .d_block = .{ .d_block, .bitcast } }); + + const scale = self.scale.bitCast(.f8e8m0); + + // log.warn("BlockScaledLinear({}): {f} -> {f}", .{ self, x, res_shape }); + const y = switch (ctx._platform.target) { + .rocm, .tpu => y: { + // Merge the .d and .d_block axes, the custom call takes care of dequant. + const weight = blocks.merge(.{ .d = .{ .d, .d_block } }); + log.warn("BlockScaledLinear: {f} -> {f} -> {f}", .{ self.blocks, blocks, weight }); + const scaled_dot_res_shape = weight.shape().dot(x.shape(), .{.d}).rename(.{ .out = .d }).withDtype(x.dtype()); + const mlir_ctx = ctx.mlirCtx(); + const loc = mlir_ctx.location(@src()); + const op = dialect.stablehlo.custom_call( + mlir_ctx, + &.{ + weight.value(), + x.value(), + scale.value(), + }, + .{ + .call_target_name = "__op$block_scaled_dot", + .backend_config = null, + .has_side_effect = false, + .api_version = .original, + }, + &.{mlirx.tensorType(mlir_ctx, scaled_dot_res_shape)}, + loc, + ); + break :y Tensor._result(scaled_dot_res_shape, op.result(0)).transpose(res_shape); + }, + else => y: { + var dequantized_weight: Tensor = .mul( + blocks.convert(x.dtype()), + scale.convert(x.dtype()).appendAxes(.{.d_block}), + ); + var y = x.dot(dequantized_weight.merge(.{ .d = .{ .d, .d_block } }), .{.d}); + std.log.warn("output shape: {f}", .{y}); + std.debug.assert(y.shape().eql(res_shape)); + y._shape = res_shape; + break :y y; + }, + }; + return if (self.bias) |bias| y.add(bias.broad(y.shape())) else y; + } + + pub fn format(self: BlockScaledLinear, writer: *std.Io.Writer) !void { + try writer.print("BlockScaledLinear(blocks={f}, scale={f}, bias={?f}, dt={t})", .{ self.blocks, self.scale, self.bias, self.blocks_dtype }); } }; @@ -42,10 +108,261 @@ pub const TokenEmbedding = struct { pub fn forward(self: TokenEmbedding, idx: Tensor) Tensor { stdx.debug.assert(idx.dtype().isInteger(), "TokenEmbedding expects an integer input, received: {f}", .{idx}); stdx.debug.assert(self.weight.rank() == 2, "TokenEmbedding expects it's weight Tensor to be a 2D matrix, got {f}", .{self.weight}); - return self.weight.gatherValues(0, idx, .{}); + return self.weight.withTags(.{ .voc, .d }).gather(.{ .voc = idx }, .{}); } }; +const MoeOpts = struct { + experts_per_token: u32, + tokens_per_expert_ratio: ?f32 = 0.0, +}; + +/// We have three algorithms, +/// * one for single-stream inference (naive), +/// * one for small batch sized with exact precision that sends all tokens to all experts. +/// this isn't too costly as long as the batch size is small and the experts are IO bound. +/// * one for big batch size that assign a fixed compute budget per expert and +/// experts chose the tokens they want to handle. This introduces noise since it's possible +/// a token doesn't get their requested expert. +/// The parameter `tokens_per_expert_ratio` control how much compute budget is granted: +/// expert_budget = ratio * (num_tokens * experts_per_token / num_experts). +/// Bigger values of ratio will ensure it's rare a token doesn't get it's top 2 tokens. +/// +/// The preferred algorithm is the batched one, +/// it is selected as soon there is enough tokens to guarantee that experts will be active most of the time. +/// +/// - input: .{ .s, .d } per-entry vector +/// - gating: .{ .s, .expert } per-entry expert-affinity +/// - experts: .{ .expert, .d_out, .d } expert layer (need to have a .forward method). +/// -> output: .{ .s, .d_out } +pub fn mixtureOfExperts(Expert: type, experts: Expert, input: Tensor, gating: Tensor, opts: MoeOpts) Tensor { + log.warn("mixtureOfExperts({s}, {f}, {f})", .{ @typeName(Expert), input, gating }); + const num_tokens = input.dim(.s); + const num_experts = gating.dim(.expert); + stdx.debug.assert(opts.experts_per_token > 0, "mixtureOfExperts expects opts.experts_per_token > 0, got {}", .{opts}); + + if (num_tokens == 1) { + // Only one tokens: most experts are unused, experts have at most one token. + // Select active experts and compute with that. + const routing = gating.topK(.{ .top_expert = .expert }, opts.experts_per_token, .{}); + const per_token_score = routing.values.div(routing.values.sum(.top_expert)); + + var output: ?Tensor = null; + for (0..opts.experts_per_token) |expert_rank| { + const expert_id = routing.indices.choose(.{ .top_expert = expert_rank }).asScalar(); + const expert_score = per_token_score.choose(.{ .top_expert = expert_rank }).asScalar(); + + var sliced_expert: Expert = undefined; + zml.meta.mapAlloc(struct { + pub fn cb(expert_id_: Tensor, expert_weight: Tensor) Tensor { + return expert_weight.gather(.{ .expert = expert_id_ }, .{}); + } + }.cb, stdx.noalloc, expert_id, experts, &sliced_expert) catch unreachable; + + // TODO how does this work when the two experts are on different gpus? + // does the compute overlap ? + const expert_output = sliced_expert.forward(input).mul(expert_score.convert(input.dtype())); + output = if (output) |o| o.add(expert_output) else expert_output; + } + + log.warn("mixtureOfExperts({s}, {f}, {f}) -> single-stream impl", .{ @typeName(Expert), input, gating }); + return output.?; + } + + const tokens_per_expert: u32 = if (opts.tokens_per_expert_ratio) |ratio| tpe: { + const compute_budget = ratio * @as(f32, @floatFromInt(num_tokens * opts.experts_per_token)); + var tpe: u32 = @intFromFloat(stdx.math.divFloat(f32, compute_budget, num_experts)); + // Round to next multiple of 8 to avoid weird shapes. + tpe = tpe + 8 - (tpe % 8); + break :tpe tpe; + } else num_tokens; + + if (3 * tokens_per_expert <= 2 * num_tokens) { + const routing, const tokens_ids_per_expert = moe.dispatchTokens(gating, .{ + .experts_per_token = opts.experts_per_token, + .tokens_per_expert = tokens_per_expert, + }); + const scores_per_expert = routing.transpose(.{ .expert, .s }).gather(.{ .s = tokens_ids_per_expert }, .{}); + const input_per_expert = input.gather(.{ .s = tokens_ids_per_expert }, .{}); + var output_per_expert = experts.forward(input_per_expert); + output_per_expert = output_per_expert.mul(scores_per_expert.convert(output_per_expert.dtype()).broad(output_per_expert.shape())); + + // Reverse engineer the normal output shape that one expert would have produced for all tokens. + // If this fall short, we could use the "sliced_expert" strategy and call forward ourselves. + const output_shape = output_per_expert.shape().drop(.expert).rename(.{ .top_token = .s }).setDim(.s, num_tokens); + const output = zml.Tensor.scatterSlices( + .zeros(output_shape), + .{ .s = tokens_ids_per_expert }, + output_per_expert, + .{ .update_fn = zml.Tensor.ScatterOpts.increment }, + ); + + log.warn("mixtureOfExperts({s}, {f}, {f}) -> fixed budget impl tpe: {d}, tokens: {d}", .{ @typeName(Expert), input, gating, tokens_per_expert, num_tokens }); + return output; + } else { + const routing = gating.topK(.{ .top_expert = .expert }, opts.experts_per_token, .{}); + const per_token_score = routing.values.div(routing.values.sum(.top_expert)).convert(gating.dtype()); + + // hard_gating is eql to gating where only the top-2 experts have a non-zero score. + const hard_gating = zml.Tensor.scatterSlices( + .zeros(gating.shape()), + .{ .expert = routing.indices }, + per_token_score, + .{ .indices_are_unique = true }, + ).withSharding(.{.expert}); + + return mixtureOfExpertsAllToAll(Expert, experts, input, hard_gating); + } +} + +/// Send all tokens to all experts, and apply gating. +pub fn mixtureOfExpertsAllToAll(Expert: type, experts: Expert, input: Tensor, gating: Tensor) Tensor { + log.warn("mixtureOfExperts({s}, {f}, {f}) -> all to all impl", .{ @typeName(Expert), input, gating }); + const num_experts = gating.dim(.expert); + // TODO: `input.insertAxes(0, .{.expert}).repeat1d(.expert, num_experts)` is too verbose for just broadcasting along a new axis` + const output_per_expert = experts.forward(input.insertAxes(0, .{.expert}).repeat1d(.expert, num_experts)); + return output_per_expert.dot(gating.convert(input.dtype()), .expert); +} + +pub const moe = struct { + /// Given `(token, expert) -> scores`, + /// keeps only the top-k expert per token, and normalize the scores accordingly. + /// Non selected experts will have a 0 score. + pub fn hardGating(gating: zml.Tensor, experts_per_token: u32) Tensor { + const routing = gating.topK(.{ .top_expert = .expert }, experts_per_token, .{}); + const per_token_score = routing.values.div(routing.values.sum(.top_expert)).convert(gating.dtype()); + return zml.Tensor.scatterSlices( + .zeros(gating.shape()), + .{ .expert = routing.indices }, + per_token_score, + .{ .indices_are_unique = true }, + ); + } + + /// Lot of tokens, each experts chose their tokens. + /// It means that some tokens may have only one expert assigned. + /// Each token will get assigned to at least one expert IIF the input gating is sums up to 1 (typically softmax output). + /// Returns the actual `(token, expert) -> scores` used. + pub fn dispatchTokens( + gating: zml.Tensor, + opts: struct { + // TODO take ratio + tokens_per_expert: u32, + experts_per_token: u32, + }, + ) [2]Tensor { + const num_experts = gating.dim(.expert); + + const token_pref = gating.argsort(.expert, .{ .descending = true }); + var expert_rank: zml.Tensor = .scatterSlices( + .zeros(gating.shape().withDtype(.i32)), + .{ .expert = token_pref }, + .addConstant(.iota(gating.shape(), .expert), 1), + .{ .indices_are_unique = true }, + ); + // The pow(expert_rank) here means that we strongly favor top 1 over top 2 and top 2 over top 3. + // expert_routing: (expert, top_token) -> token + const expert_routing = gating.pow(expert_rank.convert(gating.dtype())).topK(.{ .top_token = .s }, opts.tokens_per_expert, .{}); + const scores_per_expert = gating.gather(.{ .s = expert_routing.indices }, .{}); + + // Update the gating coefficient to account for the expert routing. + // Each (token, expert) which can't be computed within the given budget is left to 0. + const gating_v2: zml.Tensor = .scatterSlices( + .zeros(gating.shape()), + .{ .s = expert_routing.indices }, + scores_per_expert, + .{ .indices_are_unique = true, .update_fn = Tensor.ScatterOpts.override }, + ); + // Now set to zero the scores (token, expert) for tokens that have been assigned more than experts_per_token. + const lowest_experts = gating_v2.topK(.{ .top_expert = .expert }, num_experts - opts.experts_per_token, .{ .descending = false }); + var gating_v3: zml.Tensor = .scatterSlices( + gating_v2, + .{ .expert = lowest_experts.indices }, + .zeros(lowest_experts.values.shape()), + .{ .indices_are_unique = true, .update_fn = Tensor.ScatterOpts.override }, + ); + // Then normalize so the sum of experts scores for one token sums up to 1. + gating_v3 = gating_v3.div(gating_v3.sum(.expert)); + const tokens_ids_per_expert = expert_routing.indices.transpose(.{ .expert, .top_token }); + + return .{ gating_v3, tokens_ids_per_expert }; + } +}; + +test mixtureOfExperts { + const asynk = @import("async"); + + const Expert = struct { + weight: Tensor, + + pub fn forward(self: @This(), x: zml.Tensor) zml.Tensor { + // I'm always dumb founded by the weird output layout of `dot`. + return self.weight.dot(x, .{.d}).rename(.{ .d_out = .d }).contiguous(.{.d}); + } + }; + + const MoE = struct { + router: Linear, + experts: Expert, + + pub fn forward(self: @This(), x: zml.Tensor) zml.Tensor { + const routing = self.router.forward(x).convert(.f32).softmax(.expert); + return mixtureOfExperts(Expert, self.experts, x, routing, .{ .experts_per_token = 2, .tokens_per_expert_ratio = 2 }); + } + }; + + const platform = zml.testing.env(); + const store = zml.aio.detectFormatAndOpen(std.testing.allocator, "zml/test_data/moe.activations.pt") catch |err| switch (err) { + error.FileNotFound => return error.SkipZigTest, + else => return err, + }; + defer store.deinit(); + + var model_arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer model_arena.deinit(); + var moe_model = try zml.aio.populateModel(MoE, model_arena.allocator(), store); + moe_model.router.weight = moe_model.router.weight.withTags(.{ .expert, .d }); + moe_model.experts.weight = moe_model.experts.weight.withTags(.{ .expert, .d_out, .d }); + + var compilation_prefill = try asynk.asyncc(zml.compileModel, .{ + std.testing.allocator, + MoE.forward, + moe_model, + .{zml.Shape.init(.{ .s = 24, .d = 64 }, .bf16)}, + platform, + }); + var compilation_gen = try asynk.asyncc(zml.compileModel, .{ + std.testing.allocator, + MoE.forward, + moe_model, + .{zml.Shape.init(.{ .s = 1, .d = 64 }, .bf16)}, + platform, + }); + + var moe_weights = try zml.aio.loadModelBuffers(MoE, moe_model, store, std.testing.allocator, platform); + defer zml.aio.unloadBuffers(&moe_weights); + + { + const moe_prefill = (try compilation_prefill.awaitt()).prepare(moe_weights); + defer moe_prefill.deinit(); + + const x = try store.get("in").?.toDevice(platform); + const actual = moe_prefill.call(.{x}); + + try zml.testing.expectClose(store.get("out").?, actual, 5e-2); + } + + { + const moe_gen = (try compilation_gen.awaitt()).prepare(moe_weights); + defer moe_gen.deinit(); + + const x = try store.get("in").?.slice1d(0, .{ .start = 12, .end = 13 }).toDevice(platform); + const actual = moe_gen.call(.{x}); + + try zml.testing.expectClose(store.get("out").?.slice1d(0, .{ .start = 12, .end = 13 }), actual, 5e-2); + } +} + pub const Activation = union(enum) { sigmoid, tanh, @@ -110,9 +427,8 @@ pub const LayerNorm = struct { pub fn rmsNorm(x: Tensor, axis: anytype, eps: f32) Tensor { const ax = x.axis(axis); // upcast to improve precision - const xf32 = x.convert(.f32); - const mean = xf32.mul(xf32).mean(ax); - const rsqrt = Tensor.rsqrt(mean.addConstant(eps)).convert(x.dtype()); + const variance = x.convert(.f32).powByConst(2).mean(ax); + const rsqrt = Tensor.rsqrt(variance.addConstant(eps)).convert(x.dtype()); return x.mul(rsqrt.broad(x.shape())); } @@ -125,18 +441,18 @@ pub fn normalizeVariance(x: Tensor, eps: f32) Tensor { // Upcast to improve precision const xf32 = x.convert(.f32); const mean = xf32.sum(-1).scale(1.0 / N); - const mean_dev = xf32.sub(mean.broadcastRight(xf32.shape())); - const variance = mean_dev.mul(mean_dev).sum(-1).scale(1.0 / N); + const mean_dev = xf32.sub(mean); + const variance = mean_dev.mul(mean_dev).sum(-1).divByConst(N); const rsqrt = Tensor.rsqrt(variance.addConstant(eps)); - return mean_dev.mul(rsqrt.broadcastRight(mean_dev.shape())).convert(x.dtype()); + return mean_dev.mul(rsqrt).convert(x.dtype()); } // ref: https://pytorch.org/docs/stable/generated/torch.nn.functional.normalize.html // Implementation equivalent to `nn.functional.normalize(tensor, dim=-1)` call pub fn normalizeL2(input: Tensor, eps: f32) Tensor { - const inv_norm = input.pow(Tensor.scalar(2, input.dtype())).sum(-1).addConstant(eps).rsqrt(); - return input.mul(inv_norm.broad(input.shape())); + const inv_norm = input.powByConst(2).sum(-1).addConstant(eps).rsqrt(); + return input.mul(inv_norm); } test normalizeL2 { @@ -165,12 +481,22 @@ pub const RopeOpts = struct { default: void, custom: []const f32, llama3: Llama3, + yarn: Yarn, pub const Llama3 = struct { factor: f32, high_freq_factor: f32, low_freq_factor: f32, original_max_position_embeddings: u32, + truncate: bool = true, + }; + + pub const Yarn = struct { + beta_fast: f32 = 32.0, + beta_slow: f32 = 1.0, + factor: f32, + truncate: bool = true, + original_max_position_embeddings: u32, }; /// Read a Rope scaling config from HF config.json format. @@ -186,11 +512,20 @@ pub const RopeOpts = struct { if (std.mem.eql(u8, impl.string, "llama3")) { // Note: leaky is fine here cause Llama3 struct don't need to allocate memory. return .{ .llama3 = try std.json.parseFromValueLeaky(Llama3, undefined, content, .{ .ignore_unknown_fields = true }) }; + } else if (std.mem.eql(u8, impl.string, "yarn")) { + return .{ .yarn = try std.json.parseFromValueLeaky(Yarn, undefined, content, .{ .ignore_unknown_fields = true }) }; } else { log.warn("Unsupported Rope implementation: {s}, will use the default one which will produce altered results", .{impl.string}); return .{ .default = {} }; } } + + pub fn attentionScaling(scaling: Scaling) f32 { + return switch (scaling) { + .yarn => |yarn| 0.1 * @log(yarn.factor) + 1.0, + else => 1.0, + }; + } }; }; @@ -218,8 +553,9 @@ pub fn rope(x: Tensor, pos_idx: ?Tensor, opts: RopeOpts) Tensor { // compute sin and cos in f32 before downcasting to x type. const inv_freq = invFreq(x.dim(.hd), opts).withTags(.{.hd}); const inv_freq_pos = Tensor.outer(idx.convert(.f32), inv_freq); - const cos = inv_freq_pos.cos().convert(x.dtype()).broad(x_real.shape()); - const sin = inv_freq_pos.sin().convert(x.dtype()).broad(x_real.shape()); + const scaling = opts.scaling.attentionScaling(); + const cos = inv_freq_pos.cos().scale(scaling).convert(x.dtype()).broad(x_real.shape()); + const sin = inv_freq_pos.sin().scale(scaling).convert(x.dtype()).broad(x_real.shape()); // apply rotation const y_real = x_real.mul(cos).sub(x_imag.mul(sin)); @@ -250,7 +586,7 @@ pub fn mergeRealImg(x_real: Tensor, x_imag: Tensor, layout: RopeOpts.Layout) Ten .interleaved => Tensor.concatenate(&.{ x_real.appendAxes(.{.interleaved_real_img}), x_imag.appendAxes(.{.interleaved_real_img}), - }, -1).flatten(-2), + }, -1).reshape(x_imag.shape().setDim(-1, -1)), }; } @@ -259,6 +595,8 @@ pub fn invFreq(N: i64, opts: RopeOpts) Tensor { const allocator = zml.module.CompilationContext.current().allocator(); const N_half: usize = @intCast(@divExact(N, 2)); const inv_freq = allocator.alloc(f32, N_half) catch @panic("OOM"); + defer allocator.free(inv_freq); + _invFreq(opts, inv_freq); return zml.Tensor.constantTensor(.fromSlice(.{@divExact(N, 2)}, inv_freq)); } @@ -299,10 +637,38 @@ fn _invFreq(opts: RopeOpts, inv_freq: []f32) void { } } }, + .yarn => |s| { + const N_f: f64 = @floatFromInt(inv_freq.len); + const M: f64 = @floatFromInt(s.original_max_position_embeddings); + const f_high = s.beta_fast * (2 * std.math.pi) / M; + const f_low = s.beta_slow * (2 * std.math.pi) / M; + const downscaling = 1.0 / s.factor; + + // This isn't a typo: low n have a high frequency, high n have a low frequency. + var n_low: f64 = -@log(f_high) / @log(opts.freq_base) * N_f; + var n_high: f64 = -@log(f_low) / @log(opts.freq_base) * N_f; + if (s.truncate) { + n_high = std.math.ceil(n_high); + n_low = std.math.floor(n_low); + } + std.debug.assert(n_high > n_low); + for (0..N, inv_freq) |n, f| { + if (f > f_high) { + // High freq match default implem + } else if (f < f_low) { + // Downscaling for low freq + inv_freq[n] *= downscaling; + } else { + // Yarn use lerp too but not in the frequency space, in the time space. + const lerp: f64 = (n_high - @as(f64, @floatFromInt(n))) / (n_high - n_low); + inv_freq[n] *= @floatCast(lerp + (1 - lerp) * downscaling); + } + } + }, } } -test invFreq { +test "invFreq Llama3" { // Llama 3.2-1B config const llama_conf: RopeOpts = .{ .freq_base = 500_000, @@ -323,6 +689,28 @@ test invFreq { } } +test "invFreq Yarn" { + const yarn_conf: RopeOpts = .{ + .freq_base = 150_000, + .scaling = .{ .yarn = .{ + .factor = 32.0, + .beta_fast = 32.0, + .beta_slow = 1.0, + .original_max_position_embeddings = 4096, + .truncate = true, + } }, + }; + const yarn_freq = [_]f32{ 1.000000000000e+00, 6.890442967415e-01, 4.747820496559e-01, 3.271458745003e-01, 2.254180014133e-01, 1.553229838610e-01, 1.070244237781e-01, 7.374456524849e-02, 5.081327259541e-02, 3.162075206637e-02, 1.945096626878e-02, 1.179219130427e-02, 7.015713956207e-03, 4.069554619491e-03, 2.277272054926e-03, 1.206130953506e-03, 5.809474969283e-04, 2.279478358105e-04, 3.830881178146e-05, 2.639646845637e-05, 1.818833698053e-05, 1.253256959899e-05, 8.635495760245e-06, 5.950239483354e-06, 4.099978468730e-06, 2.825066758305e-06, 1.946596285052e-06, 1.341290953860e-06, 9.242089618056e-07, 6.368209142238e-07, 4.387978549403e-07, 3.023511396805e-07 }; + + var inv_freq: @TypeOf(yarn_freq) = undefined; + _invFreq(yarn_conf, &inv_freq); + for (yarn_freq, inv_freq, 0..) |expected, actual, i| { + errdefer log.err("Mismatch at position {d}.\nExpected: {d}\nActual: {d}", .{ i, stdx.fmt.slice(f32, &yarn_freq), stdx.fmt.slice(f32, &inv_freq) }); + try std.testing.expectApproxEqRel(expected, actual, 1e-5); + } + try std.testing.expectApproxEqRel(1.3465735902799727, yarn_conf.scaling.attentionScaling(), 1e-5); +} + test "real/img" { const platform = zml.testing.env(); @@ -332,8 +720,8 @@ test "real/img" { const real, const imag = splitRealImg(x, layout); const y = mergeRealImg(real, imag, layout); const real2, const imag2 = splitRealImg(y, layout); - return real.cmp(.EQ, real2).flatten(0).convert(.i32).sum(-1).add( - imag.cmp(.EQ, imag2).flatten(0).convert(.i32).sum(-1), + return real.cmp(.EQ, real2).flatten().convert(.i32).sum(-1).add( + imag.cmp(.EQ, imag2).flatten().convert(.i32).sum(-1), ); } @@ -349,8 +737,8 @@ test "real/img" { Tensor.arange(.{ .start = 3, .end = 20, .step = 4 }, .f32).reshape(.{ 5, 1 }), }, 1); - return real.cmp(.EQ, x_real).flatten(0).convert(.i32).sum(-1).add( - imag.cmp(.EQ, x_imag).flatten(0).convert(.i32).sum(-1), + return real.cmp(.EQ, x_real).flatten().convert(.i32).sum(-1).add( + imag.cmp(.EQ, x_imag).flatten().convert(.i32).sum(-1), ); } @@ -366,8 +754,8 @@ test "real/img" { Tensor.arange(.{ .start = 3, .end = 20, .step = 4 }, .f32).reshape(.{ 5, 1 }), }, 1); - return real.cmp(.EQ, x_real).flatten(0).convert(.i32).sum(-1).add( - imag.cmp(.EQ, x_imag).flatten(0).convert(.i32).sum(-1), + return real.cmp(.EQ, x_real).flatten().convert(.i32).sum(-1).add( + imag.cmp(.EQ, x_imag).flatten().convert(.i32).sum(-1), ); } @@ -377,8 +765,8 @@ test "real/img" { const x_real = Tensor.arange(.{ .start = 0, .end = 20, .step = 2 }, .f32).reshape(.{ 5, 2 }); const x_imag = Tensor.arange(.{ .start = 1, .end = 20, .step = 2 }, .f32).reshape(.{ 5, 2 }); - return real.cmp(.EQ, x_real).flatten(0).convert(.i32).sum(-1).add( - imag.cmp(.EQ, x_imag).flatten(0).convert(.i32).sum(-1), + return real.cmp(.EQ, x_real).flatten().convert(.i32).sum(-1).add( + imag.cmp(.EQ, x_imag).flatten().convert(.i32).sum(-1), ); } }; @@ -484,21 +872,21 @@ pub fn nearest(input: Tensor, scale_factor: []const f64) Tensor { out_shape._dims.set(i + 2, @intFromFloat(@floor(@as(f64, @floatFromInt(out_shape.dim(i + 2))) * sf))); } // TODO(james): remove this implicit two batching dims - var sd: [3]usize = undefined; + var sd: [3]u3 = undefined; var len_sd: usize = 0; for (2..input.rank()) |i| { if (input.dim(i) != out_shape.dim(i)) { - sd[len_sd] = i; + sd[len_sd] = @intCast(i); len_sd += 1; } } - const spatial_dims = sd[0..len_sd]; + const spatial_axes = sd[0..len_sd]; var res = input; - for (spatial_dims) |d| { - const n = out_shape.dim(d); - const ratio = stdx.math.divFloat(f32, input.dim(d), n); + for (spatial_axes) |ax| { + const n = out_shape.dim(ax); + const ratio = stdx.math.divFloat(f32, input.dim(ax), n); const offsets = Tensor.arange(.{ .end = n }, .f32).addConstant(0.5).scale(ratio).floor().convert(.i32); - res = res.gatherValues(d, offsets, .{ .indices_are_sorted = true }); + res = res.gather_(&.{ax}, &.{offsets}, .{ .indices_are_sorted = true }); } return res; } @@ -670,10 +1058,11 @@ test resizeBilinear { } pub fn resizeLinear1d(image: Tensor, axis: i8, new_len: u63, opt: ResizeOpts) Tensor { - const res_shape = image.shape().set(axis, new_len); + const ax = image.axis(axis); + const res_shape = image.shape().set(ax, new_len); const dtype = opt.precision orelse if (image.dtype().class() == .integer) .f32 else image.dtype(); - const og_len = opt.original_len orelse Tensor.scalar(image.dim(axis), dtype); + const og_len = opt.original_len orelse Tensor.scalar(image.dim(ax), dtype); const ratio = og_len.convert(dtype).scale(stdx.math.divFloat(f32, 1, new_len)); const scaled = Tensor.arange(.{ .end = new_len }, dtype).mul(ratio); const left = scaled.floor(); @@ -682,11 +1071,11 @@ pub fn resizeLinear1d(image: Tensor, axis: i8, new_len: u63, opt: ResizeOpts) Te // TODO: check that two gather isn't too bad perf wise. // Normally we should use gatherSlices to collect the values 2 by 2, // but gatherSlices messes up with the order of axes. - const left_val = image.gatherValues(axis, left.convert(.i32), .{ .indices_are_sorted = true }).convert(dtype); - const right_val = image.gatherValues(axis, right.convert(.i32), .{ .indices_are_sorted = true }).convert(dtype); + const left_val = image.gather_(&.{ax}, &.{left.convert(.i32)}, .{ .indices_are_sorted = true }).convert(dtype); + const right_val = image.gather_(&.{ax}, &.{right.convert(.i32)}, .{ .indices_are_sorted = true }).convert(dtype); - const left_weight = right.sub(scaled).broadcast(res_shape, &.{axis}); - const right_weight = scaled.sub(left).broadcast(res_shape, &.{axis}); + const left_weight = right.sub(scaled).broadcast(res_shape, &.{ax}); + const right_weight = scaled.sub(left).broadcast(res_shape, &.{ax}); const res = left_val.mul(left_weight).add(right_val.mul(right_weight)); return res.convert(image.dtype()).withTags(image.shape().tags()); @@ -743,7 +1132,7 @@ pub fn resizeCubic1d(image: Tensor, axis: i8, new_len: u63, opt: ResizeOpts) Ten Tensor.constant(t.shape(), dtype.one()), t, t.mul(t), - t.pow(Tensor.scalar(3, dtype)), + t.powByConst(3), }, .last, ._interpolated); std.debug.assert(pos.dim(0) == new_len); @@ -821,6 +1210,7 @@ pub fn causalAttnMask( pub const SdpaOpts = struct { attn_mask: ?Tensor = null, scale: ?Tensor = null, + softmax_bias: ?Tensor = null, allow_cudnn: bool = true, // TODO: put a callback instead of all this field, // so that @@ -849,17 +1239,14 @@ pub fn sdpa(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) Tensor { stdx.debug.assert(k.shape().hasTags(.{ .h, .k, .hd }), err_template ++ "k is missing tags {{.h, .k, .hd}}", err_args); stdx.debug.assert(v.shape().hasTags(.{ .h, .k, .hd }), err_template ++ "v is missing tags {{.h, .k, .hd}}", err_args); - if (opts.allow_cudnn and cuda.canUseCudnnSdpa(q.shape())) { + if (opts.allow_cudnn and cuda.canUseCudnnSdpa(q.shape()) and opts.softmax_bias == null) { return cuda.sdpa(q, k, v, opts); } - if (q.dim(.h) != k.dim(.h)) { - stdx.debug.assert(@mod(q.dim(.h), k.dim(.h)) == 0, err_template ++ "Different number of heads for keys and queries, but can't repeat keys.", err_args); - // Note: we don't try to repeat queries. - // Repeating keys is the interesting optimisation cause it reduces KV cache memory usage. - const num_rep: u63 = @intCast(@divExact(q.dim(.h), k.dim(.h))); - k, v = .{ k.repeat1d(.h, num_rep), v.repeat1d(.h, num_rep) }; - } + // Handle different numbers of head by splitting q heads. + // This is a bit error prone in the sense that it depends of the layout of q heads. + // This is the Llama convention though. + q = q.splitAxis(.h, .{ .h = k.dim(.h), .hq = .auto }); const attn_mask = if (opts.attn_mask) |m| m else null; const dims = helpers.collectDims(.{ .h, .q, .k, .hd }, &.{ q, k, v, attn_mask }, .strict) catch { @@ -870,344 +1257,18 @@ pub fn sdpa(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) Tensor { k = k.mul(head_scaling.convert(k.dtype())); var attn_weights = q.dot(k, .{.hd}); - // log.debug("attn_weights : {}, attn_mask : {?}", .{ attn_weights, attn_mask }); if (attn_mask) |mask| attn_weights = attn_weights.add(mask.broad(attn_weights.shape())); - attn_weights = attn_weights.convert(.f32).softmax(.k).convert(q.dtype()); + log.warn("attn_weights : {f}, attn_mask : {?f}", .{ attn_weights, attn_mask }); + attn_weights = attn_weights.convert(.f32); + attn_weights = if (opts.softmax_bias) |softmax_bias| attn: { + // The split is needed because we also split q ourselves. + // TODO: consider letting the user do that. + const bias = softmax_bias.splitAxis(.h, .{ .h = k.dim(.h), .hq = .auto }); + break :attn attn_weights.convert(.f32).softmaxBiased(.k, bias).convert(q.dtype()); + } else attn_weights.convert(.f32).softmax(.k).convert(q.dtype()); var attn = attn_weights.dot(v, .{.k}); - return attn.transpose(q.shape()); -} - -pub const SdpaChunks = struct { q_chunk_size: u32, k_chunk_size: u32 }; - -pub fn sdpaMemEfficient( - q: Tensor, - k: Tensor, - v: Tensor, - sdpa_opts: SdpaOpts, - chunking: SdpaChunks, -) Tensor { - const sdpa_mem_efficient: SdpaMemEfficient = .{ - .q = q, - .k = k, - .v = v, - .sdpa_opts = sdpa_opts, - .chunking = .{ - .q_chunk_size = @intCast(@min(q.dim(.q), chunking.q_chunk_size)), - .k_chunk_size = @intCast(@min(k.dim(.k), chunking.k_chunk_size)), - }, - }; - - return sdpa_mem_efficient.forward(); -} - -const SdpaMemEfficient = struct { - q: Tensor, - k: Tensor, - v: Tensor, - sdpa_opts: SdpaOpts, - chunking: SdpaChunks, - - fn forward(self: SdpaMemEfficient) Tensor { - stdx.debug.assert(@mod(self.q.dim(.q), self.chunking.q_chunk_size) == 0, "sdpaMemEfficient expects the chunk_size to exactly divise the seq_len, got: sdpaMemEfficient({f}, {})", .{ self.q, self.chunking }); - stdx.debug.assert(@mod(self.k.dim(.k), self.chunking.k_chunk_size) == 0, "sdpaMemEfficient expects the chunk_size to exactly divise the seq_len, got: sdpaMemEfficient({f}, {})", .{ self.k, self.chunking }); - const n_q_chunks: u32 = @intCast(@divExact(self.q.dim(.q), self.chunking.q_chunk_size)); - - const ctx = zml.module.CompilationContext.current(); - const q_chunks = ctx.allocator().alloc(zml.Tensor, n_q_chunks) catch unreachable; - defer ctx.allocator().free(q_chunks); - for (0..n_q_chunks) |i| { - const idx: u32 = @intCast(i); - const q_slice: zml.Tensor.DynSlice = .{ - .start = Tensor.scalar(idx * self.chunking.q_chunk_size, .i32), - .len = self.chunking.q_chunk_size, - }; - const q_chunk = self.q.dynamicSlice(.{ .q = q_slice }); - const attn_chunk = if (self.sdpa_opts.attn_mask) |attn_mask| attn_mask.dynamicSlice(.{ .q = q_slice }) else null; - - var chunk: SdpaMemEfficient = self; - chunk.q = q_chunk; - chunk.sdpa_opts.attn_mask = attn_chunk; - q_chunks[i] = chunk.scanKeyVal(); - } - - const res = zml.Tensor.concatenate(q_chunks, .q); - return res.transpose(self.q.shape()); - } - - fn nextQueriesChunk(self: SdpaMemEfficient, idx: Tensor) Tensor { - const q_slice: zml.Tensor.DynSlice = .{ - .start = idx.scale(self.chunking.q_chunk_size), - .len = self.chunking.q_chunk_size, - }; - const q_chunk = self.q.dynamicSlice(.{ .q = q_slice }); - const attn_chunk = if (self.sdpa_opts.attn_mask) |attn_mask| attn_mask.dynamicSlice(.{ .q = q_slice }) else null; - - var chunk: SdpaMemEfficient = self; - chunk.q = q_chunk; - chunk.sdpa_opts.attn_mask = attn_chunk; - return chunk.scanKeyVal(); - } - - fn scanKeyVal(self: SdpaMemEfficient) Tensor { - const n_chunks = @divExact(self.k.dim(.k), self.chunking.k_chunk_size); - return if (n_chunks <= 4) { - // Unrolled version - var partial_softmax: ?PartialSoftmax = null; - for (0..@intCast(n_chunks)) |idx| { - const next = self.nextKeyValChunk(Tensor.scalar(idx, .i32)); - partial_softmax = if (partial_softmax) |prev| prev.merge(next) else next; - } - return partial_softmax.?.finalize(); - } else { - // stablehlo.while version - const partial_softmax, _ = zml.ops.while_(hasNextKeyValChunk, nextKeyValChunkMerge, self, .{ PartialSoftmax.zeros(self.q.shape(), .f32), Tensor.scalar(0, .i32) }); - return partial_softmax.finalize(); - }; - } - - fn nextKeyValChunkMerge(self: SdpaMemEfficient, prev: PartialSoftmax, idx: Tensor) struct { PartialSoftmax, Tensor } { - const next = self.nextKeyValChunk(idx); - return .{ prev.merge(next), idx.addConstant(1) }; - } - - fn nextKeyValChunk(self: SdpaMemEfficient, idx: Tensor) PartialSoftmax { - const k_slice: zml.Tensor.DynSlice = .{ - .start = idx.scale(self.chunking.k_chunk_size), - .len = self.chunking.k_chunk_size, - }; - - const k_chunk = self.k.dynamicSlice(.{ .k = k_slice }); - const v_chunk = self.v.dynamicSlice(.{ .k = k_slice }); - const attn_chunk = if (self.sdpa_opts.attn_mask) |mask| mask.dynamicSlice(.{ .k = k_slice }) else null; - - return sdpaChunk(self.q, k_chunk, v_chunk, .{ .attn_mask = attn_chunk }); - } - - pub fn hasNextKeyValChunk(self: SdpaMemEfficient, _: PartialSoftmax, idx: Tensor) zml.Tensor { - const n_chunks = @divExact(self.k.dim(.k), self.chunking.k_chunk_size); - return idx.cmp(.LT, Tensor.scalar(n_chunks, idx.dtype())); - } -}; - -pub const PartialSoftmax = struct { - values: Tensor, - exp_sum: Tensor, - max_value: Tensor, - - pub fn zeros(q_shape: Shape, exp_sum_precision: DataType) PartialSoftmax { - return .{ - .values = Tensor.constant(q_shape, q_shape.dtype().zero()), - .exp_sum = Tensor.constant(q_shape.setDim(.hd, 1), exp_sum_precision.zero()), - .max_value = Tensor.constant(q_shape.setDim(.hd, 1), q_shape.dtype().minValue()), - }; - } - - pub fn merge(self: PartialSoftmax, other: PartialSoftmax) PartialSoftmax { - // Rescale self and other using the new global_max. - const global_max = self.max_value.maximum(other.max_value); - const new_self = self.rescale(global_max); - const new_other = other.rescale(global_max); - - // Now that self and other are using the same scale, we can just add them: - return .{ - .max_value = global_max, - .values = new_self.values.add(new_other.values), - .exp_sum = new_self.exp_sum.add(new_other.exp_sum), - }; - } - - /// Update max_value and rescale attn and exp_sum accordingly. - pub fn rescale(self: PartialSoftmax, max_value: Tensor) PartialSoftmax { - const max_diff_exp = self.max_value.sub(max_value).exp(); - const sum_dtype = self.exp_sum.dtype(); - return .{ - .max_value = max_value, - .values = self.values.mul(max_diff_exp.broad(self.values.shape())), - .exp_sum = self.exp_sum.mul(max_diff_exp.convert(sum_dtype)), - }; - } - - /// Divides the intermediary results by the exp_sum to get the proper attention values. - pub fn finalize(self: PartialSoftmax) Tensor { - return self.values.div(self.exp_sum.broad(self.values.shape()).convert(self.values.dtype())); - } -}; - -/// Compute softmax over a chunk. -/// Returns intermediary results to allow aggregating later. -pub fn partialSoftmax(self: Tensor, axis: anytype) PartialSoftmax { - const a = self.axis(axis); - const max_val = self.max(a).maximum(Tensor.scalar(-1e16, self.dtype())); - const out = self.sub(max_val.broad(self.shape())).exp(); - return .{ - .values = out, - .exp_sum = out.convert(.f32).sum(a), - .max_value = max_val, - }; -} - -/// Compute sdpa on a chunk, and computes a partial softmax. -/// q: (B, H, Sq, H_dim) ⊙ k: (B, H, Sk, H_dim) -> qk: (B, H, Sq, Sk) -pub fn sdpaChunk(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) PartialSoftmax { - // this is a dupe of sdpa, but return the PartialSoftmax instead of true Attn. - // Consider implementing sdpa from sdpaChunk. - var q, var k, var v = .{ q_, k_, v_ }; - - const err_template = "sdpa(q: {f}, k: {f}, v: {f}, attn: {?f}) is invalid ! "; - const err_args = .{ q, k, v, opts.attn_mask }; - stdx.debug.assert(q.shape().hasTags(.{ .h, .q, .hd }), err_template ++ "q is missing tags {{.h, .q, .hd}}", err_args); - stdx.debug.assert(k.shape().hasTags(.{ .h, .k, .hd }), err_template ++ "k is missing tags {{.h, .k, .hd}}", err_args); - stdx.debug.assert(v.shape().hasTags(.{ .h, .k, .hd }), err_template ++ "v is missing tags {{.h, .k, .hd}}", err_args); - - if (q.dim(.h) != k.dim(.h)) { - stdx.debug.assert(@mod(q.dim(.h), k.dim(.h)) == 0, err_template ++ "Different number of heads for keys and queries, but can't repeat keys.", err_args); - // Note: we don't try to repeat queries. - // Repeating keys is the interesting optimisation cause it reduces KV cache memory usage. - const num_rep: u63 = @intCast(@divExact(q.dim(.h), k.dim(.h))); - k, v = .{ k.repeat1d(.h, num_rep), v.repeat1d(.h, num_rep) }; - } - const attn_mask = if (opts.attn_mask) |m| m else null; - - const dims = helpers.collectDims(.{ .h, .q, .k, .hd }, &.{ q, k, v, attn_mask }, .strict) catch { - stdx.debug.panic(err_template ++ "Inputs have incompatible shapes.", err_args); - }; - const sqrtHeadDim: f32 = 1.0 / std.math.sqrt(@as(f32, @floatFromInt(dims.hd))); - const head_scaling = if (opts.scale) |s| s else Tensor.scalar(sqrtHeadDim, k.dtype()); - k = k.mul(head_scaling.convert(k.dtype())); - - var attn_weights = q.dot(k, .{.hd}); - // log.debug("attn_weights : {}", .{attn_weights}); - // log.debug("attn_mask : {?}", .{attn_mask}); - if (attn_mask) |mask| attn_weights = attn_weights.add(mask.broad(attn_weights.shape())); - - const partial = partialSoftmax(attn_weights, .k); - const attn = partial.values.dot(v, .{.k}).transpose(q.shape()); - - return .{ - .values = attn, - // The renaming is because the above dot projected values.k into .hd, - // do the same thing on the other tensors. - // This work because dot is a linear operation, and commutes with `PartialSoftmax.finalize` - .exp_sum = partial.exp_sum.rename(.{ .k = .hd }).transpose(attn.shape()), - .max_value = partial.max_value.rename(.{ .k = .hd }).transpose(attn.shape()), - }; -} - -test sdpaMemEfficient { - const platform = zml.testing.env(); - const allocator = std.testing.allocator; - - // Note we use small input vectors to have the tests run reasonably fast, - // but don't expect speed ups with this small sizes. - const rng = try zml.compileFn(allocator, Tensor.Rng.normal, .{ Shape.init(.{ 1, 10, 512, 64 }, .f32), .{ .mean = 0, .stddev = 1 } }, platform); - defer rng.deinit(); - - const rng_mask = try zml.compileFn(allocator, Tensor.Rng.normal, .{ Shape.init(.{ 512, 512 }, .f32), .{ .mean = 0, .stddev = 1 } }, platform); - defer rng_mask.deinit(); - - // Note: we pass void here, cause Rng.normal doesn't take any runtime inputs. - const q = rng.call({}).withTags(.{ .b, .h, .q, .hd }); - const k = rng.call({}).withTags(.{ .b, .h, .k, .hd }); - const v = rng.call({}).withTags(.{ .b, .h, .k, .hd }); - const mask = rng_mask.call({}).withTags(.{ .q, .k }); - - const ref_res = try zml.testing.compileAndCall( - platform, - sdpa, - .{ q, k, v, .{ .attn_mask = mask, .scale = null } }, - ); - try std.testing.expectEqualSlices(i64, q.shape().dims(), ref_res.shape().dims()); - { - // 4 k_chunks - const res = try zml.testing.compileAndCall( - platform, - sdpaMemEfficient, - .{ - q, - k, - v, - .{ .attn_mask = mask, .scale = null }, - .{ .q_chunk_size = 256, .k_chunk_size = @divExact(512, 4) }, - }, - ); - - try zml.testing.expectClose(ref_res, res, 2e-3); - } - { - // 16 k_chunks - const res = try zml.testing.compileAndCall( - platform, - sdpaMemEfficient, - .{ - q, - k, - v, - .{ .attn_mask = mask, .scale = null }, - .{ .q_chunk_size = 256, .k_chunk_size = @divExact(512, 16) }, - }, - ); - - try zml.testing.expectClose(ref_res, res, 2e-3); - } -} - -test "sdpaMemEfficient transposed" { - const platform = zml.testing.env(); - const allocator = std.testing.allocator; - - // Note we use small input vectors to have the tests run reasonably fast, - // but don't expect speed ups with this small sizes. - const rng = try zml.compileFn(allocator, Tensor.Rng.normal, .{ Shape.init(.{ 1, 512, 10, 64 }, .f32), .{ .mean = 0, .stddev = 1 } }, platform); - defer rng.deinit(); - - const rng_mask = try zml.compileFn(allocator, Tensor.Rng.normal, .{ Shape.init(.{ 512, 512 }, .f32), .{ .mean = 0, .stddev = 1 } }, platform); - defer rng_mask.deinit(); - - // Note: we pass void here, cause Rng.normal doesn't take any runtime inputs. - const q = rng.call({}).withTags(.{ .b, .q, .h, .hd }); - const k = rng.call({}).withTags(.{ .b, .k, .h, .hd }); - const v = rng.call({}).withTags(.{ .b, .k, .h, .hd }); - const mask = rng_mask.call({}).withTags(.{ .q, .k }); - - const ref_res = try zml.testing.compileAndCall( - platform, - sdpa, - .{ q, k, v, .{ .attn_mask = mask, .scale = null } }, - ); - try std.testing.expectEqualSlices(i64, q.shape().dims(), ref_res.shape().dims()); - - { - const res = try zml.testing.compileAndCall( - platform, - sdpaMemEfficient, - .{ - q, - k, - v, - .{ .attn_mask = mask, .scale = null }, - .{ .q_chunk_size = @divExact(512, 2), .k_chunk_size = @divExact(512, 4) }, - }, - ); - - try zml.testing.expectClose(ref_res, res, 1e-3); - } - - { - const res = try zml.testing.compileAndCall( - platform, - sdpaMemEfficient, - .{ - q, - k, - v, - .{ .attn_mask = mask, .scale = null }, - .{ .q_chunk_size = 512, .k_chunk_size = @divExact(512, 4) }, - }, - ); - - try zml.testing.expectClose(ref_res, res, 1e-3); - } + return attn.transpose(q.shape()).merge(.{ .h = .{ .h, .hq } }); } /// Options controlling generation. The default values correspond to greedy decoding. @@ -1225,9 +1286,9 @@ pub fn sampleTokens(activations: Tensor, opts: SamplingStrategy, rng: Tensor.Rng return .{ next_tokens, rng }; } - const topk = activations.topK(opts.topk, .voc, .{}); - // After the topk, we don't have .voc values, anymore, only topk. - var x = topk.values.rename(.{ .voc = .topk }); + const topk = activations.topK(.{ .topk = .voc }, opts.topk, .{}); + // After the topk, we don't have .voc values, anymore, only .topk. + var x = topk.values; if (opts.temperature != 1.0) { x = x.scale(1 / opts.temperature); } @@ -1242,7 +1303,7 @@ pub fn sampleTokens(activations: Tensor, opts: SamplingStrategy, rng: Tensor.Rng // topk_idx is indices into topk.values ! so in the range [0, topk] // Convert for the original indices from the full [0, voc] range. - const next_tokens = topk.indices.gatherValues(.voc, topk_idx.squeeze(.topk), .{}); + const next_tokens = topk.indices.gather(.{ .topk = topk_idx.squeeze(.topk) }, .{}); // log.debug("sampleTokens({}) -> {} -> {} -> {}", .{ activations, topk.indices, topk_idx, next_tokens }); return .{ next_tokens, next_rng }; } @@ -1330,7 +1391,7 @@ pub fn sampleTokensDynamic(logits: Tensor, opts: DynamicSamplingStrategy, rng: T x = x.add(gumbel_noise); const topk_idx = x.argMax(.topk).indices; - const next_tokens = topk_indices.gatherValues(.voc, topk_idx.squeeze(.topk), .{}); + const next_tokens = topk_indices.gather(.{ .voc = topk_idx.squeeze(.topk) }, .{}); return .{ next_tokens, next_rng }; } @@ -1339,7 +1400,7 @@ fn fixupLogits(logits: Tensor, opts: DynamicSamplingStrategy) [2]Tensor { // First reduce the vocab size to a reasonable sub set of candidate. const full_topk = if (opts.max_top_k > 0) - logits.topK(opts.max_top_k, .voc, .{ .descending = true }) + logits.topK(.{ .voc = .voc }, opts.max_top_k, .{ .descending = true }) else logits.sort(.voc, .{ .descending = true }); diff --git a/zml/ops.zig b/zml/ops.zig index c26eac3c..d475a86b 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -462,10 +462,9 @@ pub fn if_( } } - const scalar_pred = if (pred.rank() == 0) pred else pred.flattenAll().squeeze(0); const loc = ctx.mlirCtx().location(@src()); const op = mlir.Operation.make(ctx.mlirCtx(), "stablehlo.if", .{ - .operands = &.{scalar_pred.value()}, + .operands = &.{pred.asScalar().value()}, .result_type_inference = true, .blocks = &.{ true_branch_block, false_branch_block }, // We can't verify right away, cause the weights captured by the if haven't been added yet. @@ -764,33 +763,6 @@ pub fn fromMlirOperationWithTags(op: mlir.Operation, base: anytype) @TypeOf(base return res; } -pub const HostCallbackOpt = struct { - has_side_effect: bool = false, - output_operand_aliases: []const i64 = &.{}, -}; - -pub fn addHostCallback( - callback: *const Context.HostCallback, - blkctx: ?*anyopaque, - inputs: []const Tensor, - output_shapes: []const Shape, - opts: HostCallbackOpt, -) []Tensor { - return customCall( - "zmlHostBufferCallback", - inputs, - output_shapes, - .{ - .callback = @intFromPtr(callback), - .user_context = @intFromPtr(blkctx), - }, - .{ - .has_side_effect = opts.has_side_effect, - .output_operand_aliases = opts.output_operand_aliases, - }, - ); -} - pub const TritonOps = struct { debug: bool = false, name: [:0]const u8, @@ -937,7 +909,6 @@ pub fn scatter( // Note: I was a bit lazy here, and I only look at tags on the first tensor. // we probably should check all of them. const self = meta.first(Tensor, inputs); - const update = meta.first(Tensor, updates); var indices_per_axis, var indices_axes = Shape.parseStruct(Tensor, index_tensors); if (indices_per_axis.len == 0) return inputs; @@ -957,6 +928,7 @@ pub fn scatter( } break :blk higher_rank; }; + stdx.debug.assert(indices_shape.dtype().isInteger(), "zml.ops.scatter expects all indices tensor to have an integer dtype, got {f}", .{indices_shape}); for (indices_per_axis.slice()) |*idx| { stdx.debug.assert(idx.shape().canBroadcastTo(indices_shape), "zml.ops.scatter expects all indices tensor to have the same shape, got {any}", .{indices_per_axis.slice()}); stdx.debug.assert(idx.dtype() == indices_shape.dtype(), "zml.ops.scatter expects all indices tensor to have the same dtype, got {any}", .{indices_per_axis.slice()}); @@ -968,6 +940,9 @@ pub fn scatter( return self.dynamicUpdateSlice(index_tensors, updates); } + var update = meta.first(Tensor, updates); + if (update.rank() == 0) update = update.broadcast(indices_shape, &.{}); + // TODO: ideally we should catch all possible scatter errors and provide nice error messages. var config = scatterConfig(self.shape(), update.shape(), indices_per_axis, indices_axes); const indices = scatterPrepareIndices(&config, self.shape(), update.shape(), &indices_per_axis, &indices_axes); @@ -1049,6 +1024,7 @@ fn scatterConfig( var indices_batch_axes: Shape.DimsArray = .{}; var scatter_to_operand_axes: Shape.DimsArray = .{}; var updates_transpose: Shape.AxesArray = .{}; + // FIXME: updates_transpose is not correctly computed, so we can't transpose updates, and the user has to do it. const tagged_api = indices_axes.len > 0; const indices = indices_per_axis.get(0).shape(); diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index 84343a63..476bcf4a 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -5,25 +5,30 @@ const dialects = @import("mlir/dialects"); const mlir = @import("mlir"); const pjrt = @import("pjrt"); pub const ffi = pjrt.ffi; -pub const ApiError = pjrt.ApiError; -pub const ErrorCode = pjrt.ErrorCode; -pub const ExecuteContext = pjrt.ExecuteContext; -pub const BufferType = pjrt.BufferType; -pub const Device = pjrt.Device; -pub const MemoryStats = pjrt.MemoryStats; -pub const DeviceDescription = pjrt.DeviceDescription; pub const Api = pjrt.Api; -pub const NamedValue = pjrt.NamedValue; +pub const ApiError = pjrt.ApiError; +pub const BufferType = pjrt.BufferType; pub const ClientInitError = pjrt.ClientInitError; -pub const Error = pjrt.Error; -pub const GetCostAnalysisError = pjrt.GetCostAnalysisError; -pub const SerializeResult = pjrt.SerializeResult; -pub const Executable = pjrt.Executable; pub const CompiledMemoryStats = pjrt.CompiledMemoryStats; +pub const Device = pjrt.Device; +pub const DeviceDescription = pjrt.DeviceDescription; +pub const Error = pjrt.Error; +pub const ErrorCode = pjrt.ErrorCode; +pub const Executable = pjrt.Executable; +pub const ExecuteContext = pjrt.ExecuteContext; pub const ExecuteError = ApiError; +pub const GetCostAnalysisError = pjrt.GetCostAnalysisError; pub const Memory = pjrt.Memory; +pub const MemoryStats = pjrt.MemoryStats; +pub const NamedValue = pjrt.NamedValue; +pub const Profiler = pjrt.Profiler; +pub const SerializeResult = pjrt.SerializeResult; pub const Stream = pjrt.Stream; +const zml = struct { + pub const DataType = @import("dtype.zig").DataType; +}; + const log = std.log.scoped(.zml); pub const CompileError = std.mem.Allocator.Error || error{InvalidMlirBytecodeVersion} || ApiError; @@ -159,8 +164,8 @@ pub const Buffer = opaque { return @ptrCast(try self.inner().toHostBuffer(api, dst)); } - pub fn getElementType(self: *const Buffer, api: *const Api) BufferType { - return self.inner().getElementType(api); + pub fn getElementType(self: *const Buffer, api: *const Api) zml.DataType { + return dtypeFromBufferType(self.inner().getElementType(api)); } pub fn getDimensions(self: *const Buffer, api: *const Api) []const i64 { @@ -207,6 +212,13 @@ pub const Event = opaque { return self.inner().getEventError(api); } + pub fn awaitBlocking(self: *Event, api: *const Api) ApiError!void { + if (self.isReady(api)) { + return; + } + try self.inner().await_(api); + } + pub fn await_(self: *Event, api: *const Api) ApiError!void { defer self.deinit(api); @@ -264,14 +276,14 @@ pub const LoadedExecutable = opaque { }; pub fn execute(self: *const LoadedExecutable, api: *const Api, args: ExecuteArgs) ExecuteError!void { - try asynk.callBlocking(pjrt.LoadedExecutable.execute, .{ self.inner(), api, pjrt.LoadedExecutable.ExecuteArgs{ + try self.inner().execute(api, pjrt.LoadedExecutable.ExecuteArgs{ .num_args = args.num_args, .arguments = @ptrCast(args.arguments), .results = @ptrCast(args.results), .events = @ptrCast(args.events), .non_donatable_input_indices = args.non_donatable_input_indices, .context = args.context, - } }); + }); } pub fn getExecutable(self: *LoadedExecutable, api: *const Api) ApiError!*Executable { @@ -314,3 +326,29 @@ pub const AsyncHostToDeviceTransferManager = opaque { return self.inner().addMetadata(api, transfer_metadata); } }; + +pub fn bufferTypeFromDtype(dt: zml.DataType) pjrt.BufferType { + return switch (dt) { + inline else => |tag| @field(pjrt.BufferType, @tagName(tag)), + }; +} + +pub fn dtypeFromBufferType(pjrt_type: pjrt.BufferType) zml.DataType { + return switch (pjrt_type) { + .invalid => @panic("Found an invalid pjrt buffer"), + inline else => |tag| @field(zml.DataType, @tagName(tag)), + }; +} + +test bufferTypeFromDtype { + inline for (@typeInfo(zml.DataType).@"enum".fields) |field| { + const dt: zml.DataType = @enumFromInt(field.value); + try std.testing.expectEqual(dt, dtypeFromBufferType(bufferTypeFromDtype(dt))); + } + + inline for (@typeInfo(pjrt.BufferType).@"enum".fields) |field| { + const dt: pjrt.BufferType = @enumFromInt(field.value); + if (dt == .invalid) continue; + try std.testing.expectEqual(dt, bufferTypeFromDtype(dtypeFromBufferType(dt))); + } +} diff --git a/zml/platform.zig b/zml/platform.zig index 838cac48..6e13d7f0 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -22,6 +22,11 @@ pub const Platform = struct { target: Target, pjrt_api: *const pjrt.Api, pjrt_client: *pjrt.Client, + + // This make the pjrt struct quite fat, but is only used during compilation. + // TODO: Reconsider having it here, and maybe pass explicitly to compile, + // or create an intermediary struct: + // `const comp = platform.compiler(compile_opts); const exe = comp.compile(...);` compilation_options: CompilationOptions = .{}, pub const MAX_NUM_DEVICES: u8 = 32; @@ -71,26 +76,13 @@ pub const Platform = struct { return res; } - pub fn registerFFIType(self: Platform, comptime T: type) !void { - if (self.pjrt_api.ffi()) |ffi| { - if (!@hasDecl(T, "type_id")) { - stdx.debug.panic("registerFFIType requires type {s} to have a `type_id` i64 field ", .{@typeName(T)}); - } - try ffi.registerTypeId(self.pjrt_api, T); - } else { - stdx.debug.panic("registerFFIType is not available for target {s}", .{@tagName(self.target)}); - } - } - pub fn deinit(self: *Platform) void { self.pjrt_client.deinit(self.pjrt_api); } }; const _CreateOptions = struct { - // XLA CPU client doesn't read options - // https://github.com/openxla/xla/blob/42496a28c374bd35f493cc5dbde74805407245dc/xla/pjrt/c/pjrt_c_api_cpu_internal.cc#L33-L46 - cpu: struct {} = .{}, + cpu: Cpu = .{ .device_count = 4 }, // bump memory fraction from XLA defaults of 75% to 90%. // Even on a 8GB GPU it should leave enough space for the Cuda driver @@ -100,6 +92,14 @@ const _CreateOptions = struct { tpu: struct {} = .{}, neuron: struct {} = .{}, + pub const Cpu = struct { + device_count: u32, + + fn writeNamedValues(self: Cpu, values: *std.ArrayListUnmanaged(pjrt.NamedValue)) void { + values.appendAssumeCapacity(pjrt.NamedValue.from("cpu_device_count", @as(i64, self.device_count))); + } + }; + pub const Cuda = struct { allocator: Allocator = .{ .bfc = .{} }, // TODO support all of https://github.com/openxla/xla/blob/3d31c48c719d331d432132b3e0c2c5ce52650675/xla/pjrt/c/pjrt_c_api_gpu_internal.cc#L76-L86 @@ -124,7 +124,7 @@ const _CreateOptions = struct { }; }; - pub fn writeNamedValues(self: Cuda, values: *std.ArrayListUnmanaged(pjrt.NamedValue)) void { + fn writeNamedValues(self: Cuda, values: *std.ArrayListUnmanaged(pjrt.NamedValue)) void { switch (self.allocator) { .platform => { values.appendAssumeCapacity(pjrt.NamedValue.fromString("allocator", "platform")); @@ -148,6 +148,7 @@ const _CreateOptions = struct { var values = std.ArrayListUnmanaged(pjrt.NamedValue).fromOwnedSlice(out); values.shrinkRetainingCapacity(0); switch (target) { + .cpu => self.cpu.writeNamedValues(&values), .cuda => self.cuda.writeNamedValues(&values), inline else => |t| { stdx.debug.assertComptime(@hasField(_CreateOptions, @tagName(t)), "zml.platform.CreateOptions doesn't list target {s}", .{@tagName(t)}); diff --git a/zml/quantization.zig b/zml/quantization.zig index 2a850a4d..90a35bda 100644 --- a/zml/quantization.zig +++ b/zml/quantization.zig @@ -1,9 +1,11 @@ const std = @import("std"); -const zml = @import("zml.zig"); - const Allocator = std.mem.Allocator; + +const zml = @import("zml.zig"); const module = zml.module; +// TODO add tests, use modern zml + pub fn Q4_0(comptime dtype: zml.DataType) type { return struct { const Self = @This(); @@ -38,38 +40,12 @@ pub fn Q4_0(comptime dtype: zml.DataType) type { const scales = extractScales(block_count, input); const weights = extractWeights(block_count, input); - return scales.reshape(.{ block_count, 1 }) - .broadcastLeft(zml.Shape.init(.{ block_count, 32 }, .f32)) + return scales.broadcast(weights.shape(), &.{0}) .mul(weights) .convert(dtype) - .reshape(.{block_count * 32}) .reshape(shape); } - pub fn scaleIndices(block_count: u63) zml.Tensor { - // indices1 is the offsets of the scale bytes, repeated block_count times. - const indices1 = zml.Tensor.arange(.{ .start = 0, .end = 2 }, .i32).repeat1d(0, block_count); - - // indices2 is the offsets of the blocks, repeated for each scale byte, repeated block_count times. - const indices2 = zml.Tensor.arange(.{ .start = 0, .end = block_stride * block_count, .step = block_stride }, .i32) - .reshape(.{ block_count, 1 }).broadcastLeft(zml.Shape.init(.{ block_count, 2 }, .i32)).reshape(.{2 * block_count}); - - // indices is the sum of the two, which is the offsets to all the bytes we are interested in. - return indices1.add(indices2); - } - - pub fn weightIndices(block_count: u63) zml.Tensor { - // indices1 is the offsets of the data bytes, repeated block_count times. - const indices1 = zml.Tensor.arange(.{ .start = 2, .end = 18 }, .i32).repeat1d(0, block_count); - - // indices2 is the offsets of the blocks, repeated for each data byte, repeated block_count times. - const indices2 = zml.Tensor.arange(.{ .start = 0, .end = block_stride * block_count, .step = block_stride }, .i32) - .reshape(.{ block_count, 1 }).broadcastLeft(zml.Shape.init(.{ block_count, 16 }, .i32)).reshape(.{16 * block_count}); - - // indices is the sum of the two, which is the offsets to all the bytes we are interested in. - return indices1.add(indices2); - } - pub fn extractScales(block_count: u63, input: zml.Tensor) zml.Tensor { // The goal here is to get the first two bytes of every 18-bytes block in the input. For that, // we generate a list of indices that we will use to gather from the input. @@ -85,7 +61,7 @@ pub fn Q4_0(comptime dtype: zml.DataType) type { const indices = indices1.add(indices2); // We select the values we are interested in with the indices, group them by pair and bitcast them to f16, then convert them to f32. - const scales = input.gatherValues(0, indices, .{ .indices_are_sorted = true }).reshape(.{ block_count, 2 }).bitCast(.f16).convert(.f32); + const scales = input.gather_(&.{0}, &.{indices}, .{ .indices_are_sorted = true }).reshape(.{ block_count, 2 }).bitCast(.f16).convert(.f32); return scales; } @@ -107,7 +83,7 @@ pub fn Q4_0(comptime dtype: zml.DataType) type { // NOTE(Corendos): i4 is not supported by bitcast convert, so we need the following workaround. // We select the values we are interested in with the indices, these are our quantized_weights. - const quantized_weights = input.gatherValues(0, indices, .{ .indices_are_sorted = true }); + const quantized_weights = input.gather_(&.{0}, &.{indices}, .{ .indices_are_sorted = true }); const lb_weights = quantized_weights .logical(.And, zml.Tensor.constant(.{16 * block_count}, zml.Data.init(.u8, 0xf))) .bitCast(.i8); diff --git a/zml/shape.zig b/zml/shape.zig index bd5f2a22..b60cce8a 100644 --- a/zml/shape.zig +++ b/zml/shape.zig @@ -419,6 +419,13 @@ pub const Shape = struct { // Already the right shape if (std.mem.eql(i64, self.dims(), other.dims())) return true; + if (std.mem.eql(Tag, self.tags(), other.tags())) { + for (0..self.rank()) |i| { + if (self.dim(i) != 1 and self.dim(i) != other.dim(i)) return false; + } + return true; + } + // Non ambiguous broadcasting // TODO: broad is error prone because of this: // it will happily broadcast .{ .a = 10, .b = 1 } to .{ .b = 10, .a = 5 } @@ -430,12 +437,22 @@ pub const Shape = struct { } for (self.dims(), self.tags()) |d, t| { + // TODO this is also wrong when the axes are in different order in the two shapes. const other_ax = other.hasTag(t) orelse return false; if (d != 1 and d != other.dim(other_ax)) return false; } return true; } + test canBroadcastTo { + try std.testing.expect(canBroadcastTo(.scalar(.i32), .init(.{32}, .bf16))); + try std.testing.expect(canBroadcastTo(.scalar(.i32), .init(.{ 40, 16 }, .u16))); + try std.testing.expect(canBroadcastTo(.init(.{ 1, 16 }, .i32), .init(.{ 40, 16 }, .u16))); + try std.testing.expect(canBroadcastTo(.init(.{ .a = 16 }, .i32), .init(.{ .a = 16, .b = 32 }, .u16))); + try std.testing.expect(canBroadcastTo(.init(.{ .b = 32 }, .i32), .init(.{ .a = 16, .b = 32 }, .u16))); + try std.testing.expect(!canBroadcastTo(.init(.{ .b = 20 }, .i32), .init(.{ .a = 16, .b = 32 }, .u16))); + } + pub fn reshape(self: Shape, new_shape_: anytype) Shape { var new_shape: Shape = .{ ._dtype = self.dtype() }; new_shape._dims, new_shape._tags = parseDimensions(new_shape_); @@ -1092,4 +1109,94 @@ pub const Shape = struct { } return res_shape; } + + pub fn dot(lhs: Shape, rhs: Shape, comptime contracting: anytype) Shape { + var contracting_axes: stdx.BoundedArray([2]i8, MAX_RANK) = .{}; + if (@TypeOf(contracting) == EnumLiteral) { + contracting_axes.appendAssumeCapacity(.{ lhs.axis(contracting), rhs.axis(contracting) }); + } else { + inline for (contracting) |c| { + contracting_axes.appendAssumeCapacity(.{ lhs.axis(c), rhs.axis(c) }); + } + } + + var batching_axes: stdx.BoundedArray([2]i8, MAX_RANK) = .{}; + for (lhs.tags(), 0..) |l, li| { + stdx.debug.assert(l != Shape.TagUnknown, "Can't use `dot(..., {any})` on {any}, it need to be explictily tagged.", .{ contracting, lhs }); + + for (rhs.tags(), 0..) |r, ri| { + stdx.debug.assert(r != Shape.TagUnknown, "Can't use `dot(..., {any})` on {any}, it need to be explictily tagged.", .{ contracting, rhs }); + + if (l == r) { + for (contracting_axes.slice()) |ct| { + if (l == lhs.tag(ct[0])) { + break; + } + } else { + // tag is both in lhs and rhs but not in contracting -> it's a batching dim. + batching_axes.appendAssumeCapacity(.{ @intCast(li), @intCast(ri) }); + } + } + } + } + + return dot_(lhs, rhs, contracting_axes.slice(), batching_axes.slice()); + } + + pub fn dot_( + lhs: Shape, + rhs: Shape, + contracting_axes: []const [2]i8, + batching_axes: []const [2]i8, + ) Shape { + const Axes = stdx.BoundedArray(u8, MAX_RANK); + + var res_shape: Shape = .{ ._dtype = lhs.dtype() }; + // Validate batching axes + var lhs_batching_axes: Axes = .{}; + var rhs_batching_axes: Axes = .{}; + for (batching_axes) |b_axes| { + const l, const r = b_axes; + stdx.debug.assert(lhs.dim(l) == rhs.dim(r), "dotGeneral expects batching dimensions to be equal, got {d} and {d} in {f} and {f}", .{ l, r, lhs, rhs }); + var t = lhs.tag(l); + if (t == Shape.TagUnknown) t = rhs.tag(r); + res_shape = res_shape.appendDim(lhs.dim(l), t); + lhs_batching_axes.appendAssumeCapacity(lhs.axis(l)); + rhs_batching_axes.appendAssumeCapacity(rhs.axis(r)); + } + + // Validate contracting axes + var lhs_contracting_axes: Axes = .{}; + var rhs_contracting_axes: Axes = .{}; + for (contracting_axes) |c_axes| { + const l, const r = c_axes; + stdx.debug.assert(lhs.dim(l) == rhs.dim(r), "dotGeneral expects contracting dimensions to be equal, got {d} and {d} in {f} and {f}", .{ l, r, lhs, rhs }); + lhs_contracting_axes.appendAssumeCapacity(lhs.axis(l)); + rhs_contracting_axes.appendAssumeCapacity(rhs.axis(r)); + } + + // Result shape is obtained by concatenating batching dimensions, (already done) + // then dimensions from lhs axes that aren't contracting nor batching, + // then dimensions from rhs axes that aren't contracting nor batching. + for (0..lhs.rank()) |l| { + if (std.mem.indexOfScalar(u8, lhs_contracting_axes.constSlice(), @intCast(l))) |_| { + continue; + } + if (std.mem.indexOfScalar(u8, lhs_batching_axes.constSlice(), @intCast(l))) |_| { + continue; + } + res_shape = res_shape.appendDim(lhs.dim(l), lhs.tag(l)); + } + for (0..rhs.rank()) |r| { + if (std.mem.indexOfScalar(u8, rhs_contracting_axes.constSlice(), @intCast(r))) |_| { + continue; + } + if (std.mem.indexOfScalar(u8, rhs_batching_axes.constSlice(), @intCast(r))) |_| { + continue; + } + res_shape = res_shape.appendDim(rhs.dim(r), rhs.tag(r)); + } + + return res_shape; + } }; diff --git a/zml/tensor.zig b/zml/tensor.zig index 26f670d7..0b4f84fa 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -5,6 +5,7 @@ const mlir = @import("mlir"); const stdx = @import("stdx"); const Buffer = @import("buffer.zig").Buffer; +const callback = @import("callback.zig"); const CompilationContext = @import("module.zig").CompilationContext; const Data = @import("dtype.zig").Data; const DataType = @import("dtype.zig").DataType; @@ -125,8 +126,8 @@ pub const Tensor = struct { /// Returns the dimension of axis 'axis_'. /// /// 'axis_' can be an integer or a tag. - pub fn dim(self: Tensor, axis_: anytype) i64 { - return self._shape.dim(axis_); + pub fn dim(self: Tensor, axis_: anytype) u32 { + return @intCast(self._shape.dim(axis_)); } /// Returns the dimensions of a Tensor as a slice. @@ -268,18 +269,6 @@ pub const Tensor = struct { return res; } - var _global_tensor_counter: u64 = 0; - - /// Internal use - pub fn _reserveIdRange(len: u32) u64 { - return @atomicRmw(u64, &_global_tensor_counter, .Add, len, .seq_cst); - } - - /// Internal use - pub fn setUniqueId(self: *Tensor) void { - self._id = .{ .buffer_id = _reserveIdRange(1) }; - } - /// Returns a Tensor containing the absolute value of each element of the input Tensor. pub fn abs(self: Tensor) Tensor { const loc = self.getContext().mlirCtx().location(@src()); @@ -314,6 +303,14 @@ pub const Tensor = struct { res_shape = res_shape.withDtype(dt); + // TODO: it would be nice to do the bitcast when loading the model, so that inference code don't need to deal with missing metadata in safetensor. + // if (self._id == .buffer_id) { + // // Allow to use bitcast outside of compilation. + // var res = self; + // res._shape = res_shape; + // return res; + // } + const loc = self.getContext().location(@src(), "bitCast({s})", .{@tagName(dt)}); const op = dialect.stablehlo.bitcast_convert( self.getContext().mlirCtx(), @@ -711,7 +708,7 @@ pub const Tensor = struct { pub fn gumbelStats(rand: Rng, target_dist: Tensor) struct { Rng, Stats } { const s = Shape.init(.{ .n = 1024, .d = 4 }, .f32); const rng, const data = rand.gumbel(s); - const flat = data.flattenAll(); + const flat = data.flatten(); const mean_ = flat.mean(0); const variance = flat.sub(mean_.broad(flat.shape())).pow(Tensor.scalar(2, .f32)).mean(0); @@ -730,8 +727,8 @@ pub const Tensor = struct { break :blk powers; }; const values = Tensor.constantTensor(HostBuffer.fromArray(&powers)).withTags(.{.d}); - const counts = values.gatherValues(.d, samples, .{}).sum(.n).bitCast(.u16); - const actual_dist = counts.reshape(target_dist.shape()).convert(target_dist.dtype()).divByConst(s.dim(.n)); + const counts = values.gather(.{ .d = samples }, .{}).sum(.n).bitCast(.u16); + const actual_dist = counts.convert(target_dist.dtype()).divByConst(s.dim(.n)); return .{ rng, .{ .mean = mean_, .variance = variance, .actual_dist = actual_dist } }; } }; @@ -1013,10 +1010,15 @@ pub const Tensor = struct { } /// Returns a Tensor containing the element-wise multiplication of the input Tensor by a constant. - pub inline fn scale(self: Tensor, val: anytype) Tensor { + pub fn scale(self: Tensor, val: anytype) Tensor { return self.mul(Tensor.scalar(val, self.dtype())); } + /// Returns a Tensor containing the element-wise power of the input Tensor by a constant. + pub fn powByConst(self: Tensor, val: anytype) Tensor { + return self.pow(Tensor.scalar(val, self.dtype())); + } + pub const LogicalOp = enum { OR, XOR, AND }; /// Returns a Tensor containing the element-wise logical operation of the input Tensors. @@ -1053,6 +1055,21 @@ pub const Tensor = struct { return _result(self._shape.withDtype(to), op.result(0)); } + test convert { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + { + // We use u4 so that the bits of Float4E2M1 are correctly packed. + const input: [8]u4 = .{ 0, 9, 2, 11, 4, 13, 6, 15 }; + const expect: [8]f32 = .{ 0.0, -0.5, 1.0, -1.5, 2.0, -3.0, 4.0, -6.0 }; + const input_d = try zml.Buffer.fromBytes(platform, .init(.{8}, .f4e2m1), std.mem.asBytes(&input)); + const output_d = try zml.testing.compileAndCall(platform, Tensor.convert, .{ input_d, .f32 }); + const output = try output_d.getValue([8]f32); + try std.testing.expectEqualSlices(f32, &expect, &output); + } + } + /// Returns a Tensor containing the element-wise rounding operation of the input Tensor. pub fn round(self: Tensor) Tensor { const loc = self.getContext().mlirCtx().location(@src()); @@ -1077,13 +1094,16 @@ pub const Tensor = struct { /// Axes with the same tag on both sides, and which aren't contracting, /// are considered "batching axes". pub fn dot(lhs: Tensor, rhs: Tensor, comptime contracting: anytype) Tensor { - var contracting_axes: [contracting.len][2]i8 = undefined; - inline for (contracting, 0..) |c, i| { - contracting_axes[i] = .{ lhs.axis(c), rhs.axis(c) }; + var contracting_axes: stdx.BoundedArray([2]i8, MAX_RANK) = .{}; + if (@TypeOf(contracting) == EnumLiteral) { + contracting_axes.appendAssumeCapacity(.{ lhs.axis(contracting), rhs.axis(contracting) }); + } else { + inline for (contracting) |c| { + contracting_axes.appendAssumeCapacity(.{ lhs.axis(c), rhs.axis(c) }); + } } - var batching_axes: [MAX_RANK][2]i8 = undefined; - var n_batching: u8 = 0; + var batching_axes: stdx.BoundedArray([2]i8, MAX_RANK) = .{}; for (lhs._shape.tags(), 0..) |l, li| { stdx.debug.assert(l != Shape.TagUnknown, "Can't use `dot(..., {any})` on {any}, it need to be explictily tagged.", .{ contracting, lhs }); @@ -1091,20 +1111,19 @@ pub const Tensor = struct { stdx.debug.assert(r != Shape.TagUnknown, "Can't use `dot(..., {any})` on {any}, it need to be explictily tagged.", .{ contracting, rhs }); if (l == r) { - for (contracting_axes) |ct| { + for (contracting_axes.slice()) |ct| { if (l == lhs._shape.tag(ct[0])) { break; } } else { // tag is both in lhs and rhs but not in contracting -> it's a batching dim. - batching_axes[n_batching] = .{ @intCast(li), @intCast(ri) }; - n_batching += 1; + batching_axes.appendAssumeCapacity(.{ @intCast(li), @intCast(ri) }); } } } } - return dotGeneral(lhs, rhs, contracting_axes[0..], batching_axes[0..n_batching]); + return dotGeneral(lhs, rhs, contracting_axes.slice(), batching_axes.slice()); } test dot { @@ -1284,7 +1303,7 @@ pub const Tensor = struct { /// /// It's an even more crude approximation than gelu. pub fn quickGelu(x: Tensor) Tensor { - return x.scale(1.702).sigmoid().mul(x); + return x.mul(.sigmoid(x.scale(1.702))); } /// Returns a Tensor containing the Sigmoid Linear Unit (SiLU) activation function applied to each element of the input Tensor. @@ -1292,21 +1311,82 @@ pub const Tensor = struct { /// silu(x) = x σ(x) /// https://paperswithcode.com/method/silu pub fn silu(x: Tensor) Tensor { - return x.mul(x.sigmoid()); + return x.mul(.sigmoid(x)); } - /// Returns a Tensor containing the softmax function applied to each element of the input Tensor. - pub fn softmax(self: Tensor, axis_: anytype) Tensor { - const a = self.axis(axis_); - const max_val = self.max(a); - const row_mask = max_val.cmp(.GT, Tensor.scalar(-std.math.inf(f64), self.dtype())); + /// Computes softmax along the given axis. + /// y[i] = exp(x[i]) / ( Σ_k exp(x[k]) + bias ) + pub fn softmax(x: Tensor, axis_: anytype) Tensor { + const a = x.axis(axis_); + const max_val = x.max(a); - const exp_diff_max = self.sub(self.max(a).broad(self._shape)).exp(); - const res = exp_diff_max.div(exp_diff_max.sum(a).broad(self._shape)); + const exp_diff_max = x.sub(max_val).exp(); + const y = exp_diff_max.div(exp_diff_max.sum(a)); // If a row is full -inf return full 0 instead of full nan, // this fix attention when mask hides a full row. - return row_mask.broad(self.shape()).select(res, Tensor.scalar(0, self.dtype())); + const row_mask = max_val.cmp(.GT, .scalar(-std.math.inf(f64), x.dtype())); + return .select(row_mask.broad(y.shape()), y, .scalar(0, x.dtype())); + } + + test softmax { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + const Local = struct { + pub fn _softmax(x: Tensor) Tensor { + return x.softmax(-1); + } + }; + + const logits_d = try zml.Buffer.fromArray( + platform, + [4]f32{ @log(2.0), @log(1.0), @log(4.0), @log(3.0) }, + ); + const res = try zml.testing.compileAndCall(platform, Local._softmax, .{logits_d}); + try std.testing.expectEqual( + [4]f32{ 2.0 / 10.0, 1.0 / 10.0, 4.0 / 10.0, 3.0 / 10.0 }, + try res.getValue([4]f32), + ); + } + + /// Computes softmax, but adds a bias to the sum of exponentiel. + /// y[i] = exp(x[i]) / ( Σ_k exp(x[k]) + exp(bias) ) + /// + /// Note: the bias is passed pre-exponential, allowing to use precision tricks. + pub fn softmaxBiased(x: Tensor, axis_: anytype, bias: Tensor) Tensor { + const a = x.axis(axis_); + const b = bias.convert(x.dtype()).broad(x.shape().setDim(a, 1)); + const max_val: Tensor = maximum(x.max(a), b); + const exp_diff_max = x.sub(max_val).exp(); + const bias_diff_max = b.sub(max_val).exp(); + const y = exp_diff_max.div(exp_diff_max.sum(a).add(bias_diff_max)); + + const row_mask = max_val.cmp(.GT, .scalar(-std.math.inf(f64), x.dtype())); + return .select(row_mask.broad(y.shape()), y, .scalar(0, x.dtype())); + } + + test softmaxBiased { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + const Local = struct { + pub fn _softmaxBiased(x: Tensor, bias: Tensor) Tensor { + return x.softmaxBiased(-1, bias); + } + }; + + const logits_d = try zml.Buffer.fromArray( + platform, + [4]f32{ @log(2.0), @log(1.0), @log(4.0), @log(3.0) }, + ); + const bias_d = try zml.Buffer.fromArray(platform, [1]f32{@log(5.0)}); + + try zml.testing.expectClose( + zml.HostBuffer.fromArray(&[4]f32{ 2.0 / 15.0, 1.0 / 15.0, 4.0 / 15.0, 3.0 / 15.0 }), + try zml.testing.compileAndCall(platform, Local._softmaxBiased, .{ logits_d, bias_d }), + 1e-5, + ); } /// Returns a Tensor containing the log of the sum of exponential over the given axis. @@ -1496,27 +1576,7 @@ pub const Tensor = struct { return self.transpose(perm.constSlice()); } - /// Flattens the given axis and the next one, into one new axis. - pub fn flatten(self: Tensor, axis_: anytype) Tensor { - // TODO: move to torch.zig, this is equivalent to merge - const old_shape = self._shape; - const a = self.axis(axis_); - // stdx.debug.assert(a + 1 < self.rank(), "Can't flatten {} on the last axis {}.", .{ self, axis }); - const new_shape = old_shape.remove(a + 1).set(a, old_shape.dim(a) * old_shape.dim(a + 1)); - - const loc = self.getContext().location(@src(), "flatten({f},{})", .{ self, axis_ }); - const reshaped_val = dialect.stablehlo.reshape( - self.getContext().mlirCtx(), - self.value(), - mlirx.tensorType(self.getContext().mlirCtx(), new_shape), - loc, - ); - // log.debug("flatten({d}, {d}) -> {d}", .{ self.dims(), axis_, new_shape[0 .. self.rank() - 1] }); - return _result(new_shape, reshaped_val.result(0)); - } - - pub inline fn flattenAll(self: Tensor) Tensor { - // TODO: rename to just flatten, once flatten is moved to torch + pub inline fn flatten(self: Tensor) Tensor { return self.reshape(.{self.count()}); } @@ -1711,14 +1771,12 @@ pub const Tensor = struct { } const a = self.axis(axis_); - const broadshape = self._shape.insert(a + 1, .{n_rep}); - const repeat_dims = Shape.range(self.rank() + 1, self.dtype()).remove(a + 1); + const res_shape = self._shape.setDim(a, self.dim(a) * n_rep); - var res = self.broadcast(broadshape, repeat_dims.dims()).flatten(a); - // Restor the tag that has been lost by flatten. - res._shape._tags.set(a, self._shape.tag(a)); + const broadshape = self._shape.insert(a, .{n_rep}); + const repeat_dims = Shape.range(self.rank() + 1, self.dtype()).remove(a); - return res; + return self.broadcast(broadshape, repeat_dims.dims()).reshape(res_shape); } /// Repeats a Tensor several times along the given axes. @@ -1735,15 +1793,47 @@ pub const Tensor = struct { return res; } + test repeat1d { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + const Local = struct { + fn repeat1d(x: Tensor, axis_: u3, n_reps: u32) Tensor { + return x.repeat1d(axis_, n_reps); + } + }; + + { + const inputs: [3]u8 = .{ 1, 2, 3 }; + const expectations: [6]u8 = .{ 1, 2, 3, 1, 2, 3 }; + + const input = try zml.Buffer.fromArray(platform, inputs); + const output = try zml.testing.compileAndCall(platform, Local.repeat1d, .{ input, 0, 2 }); + + try std.testing.expectEqual(expectations, output.getValue(@TypeOf(expectations))); + } + { + const inputs: [2][3]u8 = .{ .{ 1, 2, 3 }, .{ 4, 5, 6 } }; + const expectations: [2][6]u8 = .{ .{ 1, 2, 3, 1, 2, 3 }, .{ 4, 5, 6, 4, 5, 6 } }; + + const input = try zml.Buffer.fromArray(platform, inputs); + const output = try zml.testing.compileAndCall(platform, Local.repeat1d, .{ input, 1, 2 }); + + try std.testing.expectEqual(expectations, output.getValue(@TypeOf(expectations))); + } + } + /// Repeats in line each value along the given axis. /// - /// * stutter1d([0, 1, 2, 3], 0, 2) = [0, 0, 1, 1, 2, 2, 3, 3] - pub fn stutter1d(self: Tensor, axis_: i64, n_rep: u63) Tensor { + /// * stutter1d([0, 1, 2, 3], -1, 2) = [0, 0, 1, 1, 2, 2, 3, 3] + /// This is equivalent to repeat(ax+1) unless ax is the last axis. + pub fn stutter1d(self: Tensor, axis_: i8, n_rep: u63) Tensor { const a = self.axis(axis_); const broadshape = self._shape.insert(a + 1, .{n_rep}); - const stutter_dims = Shape.range(self.rank() + 1, self.dtype()).remove(a + 1); + const res_shape = self._shape.setDim(a, self.dim(a) * n_rep); - return self.broadcast(broadshape, stutter_dims.dims()).flatten(a); + const stutter_dims = Shape.range(self.rank() + 1, self.dtype()).remove(a + 1); + return self.broadcast(broadshape, stutter_dims.dims()).reshape(res_shape); } /// Repeats in line each value along the given axes. @@ -1759,6 +1849,45 @@ pub const Tensor = struct { return res; } + test stutter1d { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + const Local = struct { + fn stutter1d(x: Tensor, axis_: u3, n_reps: u32) Tensor { + return x.stutter1d(axis_, n_reps); + } + }; + + { + const inputs: [3]u8 = .{ 1, 2, 3 }; + const expectations: [6]u8 = .{ 1, 1, 2, 2, 3, 3 }; + + const input = try zml.Buffer.fromArray(platform, inputs); + const output = try zml.testing.compileAndCall(platform, Local.stutter1d, .{ input, 0, 2 }); + + try std.testing.expectEqual(expectations, output.getValue(@TypeOf(expectations))); + } + { + const inputs: [2][3]u8 = .{ .{ 1, 2, 3 }, .{ 4, 5, 6 } }; + const expectations: [2][6]u8 = .{ .{ 1, 1, 2, 2, 3, 3 }, .{ 4, 4, 5, 5, 6, 6 } }; + + const input = try zml.Buffer.fromArray(platform, inputs); + const output = try zml.testing.compileAndCall(platform, Local.stutter1d, .{ input, 1, 2 }); + + try std.testing.expectEqual(expectations, output.getValue(@TypeOf(expectations))); + } + { + const inputs: [2][3]u8 = .{ .{ 1, 2, 3 }, .{ 4, 5, 6 } }; + const expectations: [2][6]u8 = .{ .{ 1, 2, 3, 1, 2, 3 }, .{ 4, 5, 6, 4, 5, 6 } }; + + const input = try zml.Buffer.fromArray(platform, inputs); + const output = try zml.testing.compileAndCall(platform, Local.stutter1d, .{ input, 0, 2 }); + + try std.testing.expectEqual(expectations, output.getValue(@TypeOf(expectations))); + } + } + /// Returns a Tensor containing the element-wise negation of the input Tensor. pub fn negate(self: Tensor) Tensor { const loc = self.getContext().mlirCtx().location(@src()); @@ -1946,6 +2075,10 @@ pub const Tensor = struct { return _result(sh, constant_op.result(0)).convert(val.dtype()); } + pub fn zeros(sh: Shape) Tensor { + return constant(sh, sh.dtype().zero()); + } + /// Embeds a buffer with concrete values into an Mlir program. pub fn constantTensor(val: HostBuffer) Tensor { const ctx = CompilationContext.current().mlirCtx(); @@ -2044,7 +2177,11 @@ pub const Tensor = struct { /// Reshapes the input Tensor with the given shape. pub fn reshape(self: Tensor, output_shape_: anytype) Tensor { - const output_shape = self._shape.reshape(output_shape_); + var output_shape = self._shape.reshape(output_shape_); + output_shape._dtype = self.dtype(); + // TODO: it would be nice to allow reshape when loading the model, + // so that inference code don't need to deal with missing metadata in safetensor. + const tensor_type = mlirx.tensorType(self.getContext().mlirCtx(), output_shape); const loc = self.getContext().location(@src(), "reshape({f})", .{output_shape}); const reshape_value = dialect.stablehlo.reshape(self.getContext().mlirCtx(), self.value(), tensor_type, loc); @@ -2144,88 +2281,99 @@ pub const Tensor = struct { pub const GatherOpts = struct { indices_are_sorted: bool = false }; - /// For each coordinate in `indices`, - /// `gatherValues` extracts a single value of the given tensor. + /// `gather` extracts slices from the given tensor at the specified offsets. + /// example: `values.gather(.{ .a = idx }, .{})` + /// + /// * indices is a named list of integer tensors: eg `.{ .a = idx }`. + /// Each names specify a gathering axis, it must refer to an axis of self, + /// and the corresponding idx Tensor must contains valid indices into axis .a. + /// All indices must have the same shape or broadcast to the same shape. /// - /// * axes_ is a single axis, or a tuple of axis: .b, or .{ .b, .c } - /// * indices is an integer tensor /// * result is a tensor whose shape is similar to the input shape /// where the gathered axes have been replaced by axes from 'indices'. /// /// Some example input for the base case where we work on one axis: - /// - gatherValues(f:[a]->float, .a, ind:[n]->int)[n] == f[ind[n]] - /// - gatherValues(f:[a, b], .a, ind:[n])[n, b] == f[ind[n], b] - /// - gatherValues(f: [a,b,c], .{.b}, ind: [n,m])[a, n, m, c] == f[a, ind[n, m], c] + /// - gather(f:[a], .{ .a = idx:[n]})[n] == f[idx[n]] + /// - gather(f:[a, b], .a, idx:[n])[n, b] == f[idx[n], b] + /// - gather(f:[a,b,c], .{.b = idx:[n,m]})[a, n, m, c] == f[a, idx[n, m], c] /// /// If an axis in common between `self` and `indices`, /// it is treated as a "batching" axis, meaning that semantically - /// the operator is doing a gatherValues one time per dimension of this axis: - /// - gatherValues(f: [a,b,c], .{.b}, ind: [a,n])[a, n] == f[a, ind[a, n]] + /// the operator is doing a gather one time per dimension of this axis: + /// - gather(f: [a,b,c], .{.b=idx: [a,n]})[a, n] == f[a, idx[a, n]] /// - /// It is an error to have an axis present in `self`, `axes_` and `indices`. + /// It's possible to pass several indices: + /// - gather(f: [a,b,c], .{.b=idx_b[n], .c=idx_c[n]})[a, n] == f[a, idx_b[n], idx_c[n]] + /// - gather(f: [a,b,c,d], .{.b=idx_b[a,n], .c=idx_c[a, n]})[a, n, d] == f[a, idx_b[a, n], idx_c[a, n], d] /// - /// If several axes are passed, then the last axis of indices is treated as coordinates: - /// - gatherValues(f: [a,b,c], .{.b, .c}, ind: [n,2])[a, n] == f[a, ind[n][0], ind[n][1]] - /// - gatherValues(f: [a,b,c,d], .{.b, .c}, ind: [a, n,2])[a, n, d] == f[a, ind[a, n][0], ind[a, n][1], d] + /// If `self` isn't tagged, you can use `gather_` to specify gathered axis by their position but batching won't be available. /// - /// It is possible to use gatherValues without tags, but batching won't be available. - pub fn gatherValues(self: Tensor, coord_axes: anytype, indices: Tensor, opts: GatherOpts) Tensor { - // scoped_log.debug("gatherValues({}, {any}, {})", .{ self, coord_axes, indices }); - const single_coord, const coord_axes_ = _parseGatherCoord(self, coord_axes); + /// For performance it's better to have batching and gathering axes of `self` be the first one, + /// so that gather can + pub fn gather(self: Tensor, _indices: anytype, opts: GatherOpts) Tensor { + const idx_per_axis, const idx_tags = Shape.parseStruct(Tensor, _indices); + var idx_axes: Shape.AxesArray = .{}; + for (idx_tags.slice()) |t| { + idx_axes.appendAssumeCapacity(self.axis(t)); + } - stdx.debug.assert(coord_axes_.len > 0, "gatherValues expects 1 or more axes to operate one, received none. Example: `x.gatherValues(.a, indices, .{{}})`", .{}); - for (coord_axes_.constSlice(), 0..) |a, i| { + // TODO: sort indices following self.shape instead of asking the user to do it. + return self.gather_(idx_axes.slice(), idx_per_axis.slice(), opts); + } + + pub fn gather_(self: Tensor, idx_axes: []const u3, idx_per_axis: []const Tensor, opts: GatherOpts) Tensor { + // scoped_log.warn("gather({f}, {any}, {any})", .{ self, idx_axes, idx_per_axis }); + stdx.debug.assert(idx_axes.len > 0, "gather expects 1 or more axes to operate one, received none. Example: `x.gather(.a, indices, .{{}})`", .{}); + for (idx_axes, 0..) |a, i| { if (i > 0) { - stdx.debug.assert(a == coord_axes_.get(i - 1) + 1, "gatherValues expects 'coord_axes' to be sequential. But {any} aren't sequential in {f}", .{ coord_axes, self }); + stdx.debug.assert(a == idx_axes[i - 1] + 1, "gather expects 'idx_axes' to be sequential. But {any} aren't sequential in {f}", .{ idx_axes, self }); } } + var indices_shape = idx_per_axis[0].shape(); + for (idx_per_axis[1..]) |idx| { + if (idx.rank() > indices_shape.rank()) { + indices_shape = idx.shape(); + } + } + for (idx_per_axis) |idx| { + stdx.debug.assert(idx.shape().canBroadcastTo(indices_shape), "gather indices can't be broadcasted together {any}", .{idx_per_axis}); + } + + var idx_batch_axes: Shape.DimsArray = .{}; const AxisKind = enum { batching, offset, collapsed, indices }; - var self_kind: stdx.BoundedArray(AxisKind, MAX_RANK) = .{}; - var indices_batch_axes: Shape.DimsArray = .{}; + var self_kind: stdx.BoundedArray(AxisKind, MAX_RANK) = .{ .buffer = @splat(.offset), .len = self.rank() }; + for (self._shape.tags(), 0..self.rank()) |t, self_ax| { - const maybe_coord_ax = std.mem.indexOfScalar(u3, coord_axes_.constSlice(), @intCast(self_ax)); - if (indices._shape.hasTag(t)) |id_ax| { + const is_gather_axis = std.mem.containsAtLeastScalar(u3, idx_axes, 1, @intCast(self_ax)); + if (indices_shape.hasTag(t)) |id_ax| { // tag is both in self and indices -> it's a batching dim // Note: tags are required for batching. - self_kind.appendAssumeCapacity(.batching); - indices_batch_axes.appendAssumeCapacity(id_ax); - stdx.debug.assert(maybe_coord_ax == null, "gatherValues expects axes to appear at most twice. Axis {s} has been found both in 'self={f}', in 'coord_axes_={any}' and in 'indices={f}'", .{ self._shape._tags.get(self_ax), self, coord_axes, indices }); - } else if (maybe_coord_ax) |_| { - // for gatherValues we collapsed all gathered axes - // (contrary to gatherSlices where we collapse none) - self_kind.appendAssumeCapacity(.collapsed); + self_kind.buffer[self_ax] = .batching; + idx_batch_axes.appendAssumeCapacity(id_ax); + stdx.debug.assert(!is_gather_axis, "gather expects axes to appear at most twice. Axis {s} has been found both in 'self={f}', in 'idx_axes={any}' and in 'indices={f}'", .{ t, self, idx_axes, indices_shape }); + } else if (is_gather_axis) { + // we collapsed all gathered axes + self_kind.buffer[self_ax] = .collapsed; + // idx_kind.buffer[id_ax] = .indices; } else { - self_kind.appendAssumeCapacity(.offset); + self_kind.buffer[self_ax] = .offset; } } - // When we receive several coord_axes we need an extra dimension to store - // one index per axis, which makes the coordinates of one value. - // Otherwi se stablehlo uses the "indices.rank()" default value. - const index_coord_axis = if (single_coord) - indices.rank() - else blk: { - const ax = indices._shape.hasTag(.coord) orelse indices._shape.axis(-1); - stdx.debug.assert(indices.dim(ax) == coord_axes_.len, "gatherValues with axes={any}, expects indices to be of shape [..., {}], got: {f}", .{ coord_axes, coord_axes_.len, indices }); - break :blk ax; - }; - // compute res shape var res_shape = Shape.init(.{}, self.dtype()); var res_kind: stdx.BoundedArray(AxisKind, MAX_RANK) = .{}; - for (self_kind.constSlice(), 0..) |kind, ax_usize| { + for (self_kind.slice(), 0..) |kind, ax_usize| { const ax: u3 = @intCast(ax_usize); - if (ax == coord_axes_.get(0)) { + if (ax == idx_axes[0]) { // The first val_ax is special cause this is the place where we insert indices axes. - for (indices._shape.tags(), 0..indices.rank()) |t, id_ax| { - if (id_ax == index_coord_axis) continue; - if (std.mem.indexOfScalar(i64, indices_batch_axes.constSlice(), @intCast(id_ax))) |_| { - // batching dim are already in res - continue; - } + for (0.., indices_shape.tags(), indices_shape.dims()) |id_axis_order, id_axis, id_inserted_dim| { + const is_batching_axis = std.mem.containsAtLeastScalar(i64, idx_batch_axes.constSlice(), 1, @intCast(id_axis_order)); + // Batching axis is already in self. + if (is_batching_axis) continue; - res_shape = res_shape.appendDim(indices.dim(id_ax), t); + res_shape = res_shape.appendDim(id_inserted_dim, id_axis); res_kind.appendAssumeCapacity(.indices); } } @@ -2241,12 +2389,12 @@ pub const Tensor = struct { // This is not a gather, but a dynamicSlice. // Sometimes the backend recognize this pattern, but not always. // So let us handle that. - if (indices.count() == 1) { - return self.dynamicSlice1d(coord_axes_.get(0), .{ .start = indices.flattenAll().squeeze(0), .len = 1 }).reshape(res_shape); + if (indices_shape.count() == 1 and idx_axes.len == 1) { + return self.dynamicSlice1d(idx_axes[0], .{ .start = idx_per_axis[0].asScalar(), .len = 1 }).reshape(res_shape); } var slice_dims: Shape.DimsArray = .{}; - for (self_kind.constSlice(), self.dims()) |k, d| { + for (self_kind.slice(), self.dims()) |k, d| { slice_dims.appendAssumeCapacity(switch (k) { .batching, .collapsed => 1, .offset => d, @@ -2254,34 +2402,48 @@ pub const Tensor = struct { }); } - // scoped_log.debug("gatherValues --> {} {any}", .{ res_shape, res_kind.constSlice() }); + // TODO: try changing .last by other axis and see the perf impact. + const indices = Tensor.stack(idx_per_axis, .last, .coord); + // scoped_log.debug("gather --> {} {any}", .{ res_shape, res_kind.constSlice() }); const loc = self.getContext().mlirCtx().location(@src()); + const args: dialect.stablehlo.GatherArgs = .{ + .offset_dims = _collectAxes(AxisKind, res_kind, .offset).constSlice(), + .collapsed_slice_dims = _collectAxes(AxisKind, self_kind, .collapsed).constSlice(), + .operand_batching_dims = _collectAxes(AxisKind, self_kind, .batching).constSlice(), + .start_indices_batching_dims = idx_batch_axes.constSlice(), + .start_index_map = _collectAxes(AxisKind, self_kind, .collapsed).constSlice(), + .index_vector_dim = indices.axis(.coord), + .indices_are_sorted = opts.indices_are_sorted, + }; + // scoped_log.warn(" --> stablehlo.gather({f}, {f}, {any}, {})", .{ self, indices, slice_dims.slice(), args }); + const gather_op = dialect.stablehlo.gather( self.getContext().mlirCtx(), self.value(), indices.value(), slice_dims.constSlice(), loc, - .{ - .offset_dims = _collectAxes(AxisKind, res_kind, .offset).constSlice(), - .collapsed_slice_dims = _collectAxes(AxisKind, self_kind, .collapsed).constSlice(), - .operand_batching_dims = _collectAxes(AxisKind, self_kind, .batching).constSlice(), - .start_indices_batching_dims = indices_batch_axes.constSlice(), - .start_index_map = _collectAxes(AxisKind, self_kind, .collapsed).constSlice(), - .index_vector_dim = index_coord_axis, - .indices_are_sorted = opts.indices_are_sorted, - }, + args, ); const mlir_shape = fromMlirValue(gather_op.result(0)).shape(); - stdx.debug.assert(mlir_shape.eql(res_shape), "gatherValues expects that batching indices appear in the same order in 'self' and 'indices', got: self={f}, indices={f}. You should transpose one or the other.", .{ self, indices }); + // Note: this is midly annoying, I'm inclined to automatically transpose the indices. + stdx.debug.assert(mlir_shape.eql(res_shape), "gather expects that batching indices appear in the same order in 'self' and 'indices', got: self={f}, indices={f}. You should transpose one or the other.", .{ self, indices }); return _result(res_shape, gather_op.result(0)); } - test gatherValues { + test gather { const zml = @import("zml.zig"); const platform = zml.testing.env(); + const Local = struct { + pub fn _idx(idx_shape: anytype) Tensor { + return Tensor.constant(idx_shape, .{ .i32 = 0 }); + } + }; + + const idx = Local._idx; + { // Only test shapes var comp = try zml.module.CompilationContext.init(std.testing.allocator, "test", platform); @@ -2290,32 +2452,37 @@ pub const Tensor = struct { defer comp.deactivate(); inline for (.{ - .{ .{ .a = 10 }, .a, .{}, .{} }, - .{ .{ .a = 10 }, .a, .{ .n = 8 }, .{ .n = 8 } }, - .{ .{ .a = 10, .b = 20 }, .a, .{}, .{ .b = 20 } }, - .{ .{ .a = 10, .b = 20 }, .a, .{ .n = 8 }, .{ .n = 8, .b = 20 } }, - .{ .{ .a = 10, .b = 20 }, 0, .{ .n = 8 }, .{ .n = 8, .b = 20 } }, + .{ .{ .a = 10 }, .{ .a = idx(.{}) }, .{} }, + .{ .{ .a = 10 }, .{ .a = idx(.{ .n = 8 }) }, .{ .n = 8 } }, + .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{}) }, .{ .b = 20 } }, + .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{ .n = 8 }) }, .{ .n = 8, .b = 20 } }, // Favor val shape, instead of indices shape. - .{ .{ .a = 10, .b = 20 }, .b, .{ .n = 8 }, .{ .a = 10, .n = 8 } }, - .{ .{ .a = 10, .b = 20, .c = 30 }, .b, .{ .n = 8 }, .{ .a = 10, .n = 8, .c = 30 } }, + .{ .{ .a = 10, .b = 20 }, .{ .b = idx(.{ .n = 8 }) }, .{ .a = 10, .n = 8 } }, + .{ .{ .a = 10, .b = 20, .c = 30 }, .{ .b = idx(.{ .n = 8 }) }, .{ .a = 10, .n = 8, .c = 30 } }, // batching axes are implicits. - .{ .{ .a = 10, .b = 20 }, .b, .{ .a = 10 }, .{ .a = 10 } }, - .{ .{ .a = 10, .b = 20 }, .a, .{ .b = 20 }, .{ .b = 20 } }, - .{ .{ .a = 10, .b = 20 }, .b, .{ .a = 10, .n = 8 }, .{ .a = 10, .n = 8 } }, + .{ .{ .a = 10, .b = 20 }, .{ .b = idx(.{ .a = 10 }) }, .{ .a = 10 } }, + .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{ .b = 20 }) }, .{ .b = 20 } }, + .{ .{ .a = 10, .b = 20 }, .{ .b = idx(.{ .a = 10, .n = 8 }) }, .{ .a = 10, .n = 8 } }, // stablehlo.gather is biased toward indices shape (like gatherSlice). // This make it awkward to use when you have both batching dimension and new indices dimensions. // For now we reject those, and let user explicitly transpose self or indices if needed. - // .{ .{ .a = 10, .b = 20 }, .b, .{ .n = 8, .a = 10 }, .{ .a = 10, .n = 8 } }, + // .{ .{ .a = 10, .b = 20 }, .{.b = idx(.{ .n = 8, .a = 10 })}, .{ .a = 10, .n = 8 } }, // Also handle tuples - .{ .{ .a = 10, .b = 20 }, .{ .a, .b }, .{ .n = 8, ._ = 2 }, .{ .n = 8 } }, - .{ .{ 10, 20 }, .{ -2, -1 }, .{ 8, 2 }, .{8} }, - // and 1-tuple - .{ .{ .a = 10, .b = 20 }, .{.b}, .{ .n = 8, ._ = 1 }, .{ .a = 10, .n = 8 } }, + .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{ .n = 8 }), .b = idx(.{ .n = 8 }) }, .{ .n = 8 } }, }) |testcase| { - const x_shape, const tag, const idx_shape, const res_shape = testcase; + const x_shape, const indices, const res_shape = testcase; const x = Tensor.constant(x_shape, .{ .f16 = 0 }); - const idx = Tensor.constant(idx_shape, .{ .i32 = 0 }); - const y = gatherValues(x, tag, idx, .{}); + const y = gather(x, indices, .{}); + try zml.testing.expectEqualShapes(Shape.init(res_shape, .f16), y.shape()); + try std.testing.expect(y.value().owner().verify()); + } + + inline for (.{ + .{ .{ 10, 20 }, &[_]u3{ 0, 1 }, &[_]Tensor{ idx(.{8}), idx(.{8}) }, .{8} }, + }) |testcase| { + const x_shape, const idx_axes, const idx_per_axis, const res_shape = testcase; + const x = Tensor.constant(x_shape, .{ .f16 = 0 }); + const y = gather_(x, idx_axes, idx_per_axis, .{}); try zml.testing.expectEqualShapes(Shape.init(res_shape, .f16), y.shape()); try std.testing.expect(y.value().owner().verify()); } @@ -2593,7 +2760,7 @@ pub const Tensor = struct { /// that requires host<->device synchronization. /// ZML tries to generate the easiest to optimize IR, and will warn you if it generates known problematic IR. pub fn scatterSlices(self: Tensor, indices: anytype, updates: Tensor, opts: ScatterOpts) Tensor { - scoped_log.debug("scatterSlices({}, {any}, {})", .{ self, indices, updates }); + scoped_log.debug("scatterSlices({f}, {any}, {f})", .{ self, indices, updates }); const UpdateType = @TypeOf(ScatterOpts.increment); @@ -2653,6 +2820,15 @@ pub const Tensor = struct { .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{ .n = 8 }) }, .{ .n = 8, .a = 2 } }, .{ .{ .a = 10, .b = 20 }, .{ .b = idx(.{ .n = 8 }), .a = idx(.{ .n = 8 }) }, .{ .n = 8, .a = 3, .b = 2 } }, .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{ .n = 8 }), .b = idx(.{ .n = 8 }) }, .{ .a = 3, .n = 8, .b = 2 } }, + // 2-indices axes, m and n + .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{ .n = 8, .m = 9 }) }, .{ .n = 8, .m = 9, .b = 20 } }, + // TODO: here update (m,n) doesn't match indices (n, m). + // We need to either handle this automatically or provide better errors. + // .{ .{. a = 10, .b=20 }, .{ .a = idx(.{.n=8,.m=9}) }, .{.m=9,.n=8,.b=20} }, + // Ideally this should be refused: + // .{.{ .expert = 8, .top_token=8}, .{ .top_token = idx(.{.s=24, .top_expert=2}) }, .{} } + // But it implies having an extra tag on indices tensor generated by topk and argmax. Here the problem is that the indices point into a [0..token] range, + // but are intepreted to point into a [0..top_token] range. }) |testcase| { const x_shape, const indices, const updates_shapes = testcase; const x = Tensor.constant(x_shape, .{ .f16 = 0 }); @@ -2949,13 +3125,26 @@ pub const Tensor = struct { } /// Returns a Tensor representing the result of Top-K over the given axis. - pub fn topK(self: Tensor, k: u32, axis_: anytype, opts: struct { descending: bool = true }) SortRes { - const a = self.axis(axis_); - const result = self.sort(a, .{ .descending = opts.descending }); - return .{ - .values = result.values.slice1d(a, .{ .end = k }), - .indices = result.indices.slice1d(a, .{ .end = k }), + pub fn topK(self: Tensor, named_axis_: anytype, k: u32, opts: struct { descending: bool = true }) SortRes { + // TODO: ask the user for a new name for the axis, this will simplify tagged code. + // See eg code in `sampleTokens` and `mixtureOfExpert` + const err_msg = "topK named axis should be an integer or a named axis, eg `x.topK(.{{ .best_token = .token }}, 16)` or `x.topK(-1, 16)`"; + const has_name: ?[:0]const u8, const a = switch (@typeInfo(@TypeOf(named_axis_))) { + .int, .comptime_int => .{ null, self.axis(@as(i64, @intCast(named_axis_))) }, + .@"struct" => |info| blk: { + stdx.debug.assertComptime(info.fields.len == 1, err_msg, .{}); + break :blk .{ info.fields[0].name, self.axis(@field(named_axis_, info.fields[0].name)) }; + }, + else => stdx.debug.compileError(err_msg, .{}), }; + var result = self.sort(a, .{ .descending = opts.descending }); + result.values = result.values.slice1d(a, .{ .end = k }); + result.indices = result.indices.slice1d(a, .{ .end = k }); + if (has_name) |new_name| { + result.values._shape._tags.set(a, new_name.ptr); + result.indices._shape._tags.set(a, new_name.ptr); + } + return result; } pub const MaxPoolRes = ArgMaxRes; @@ -3499,11 +3688,10 @@ pub const Tensor = struct { res_shape._tags.replaceRange(a, 1, &.{ @tagName(new_tags[0]), @tagName(new_tags[1]) }) catch unreachable; const values = self.insertAxes(a + 1, .{new_tags[1]}).broad(res_shape); - const zeros = Tensor.constant(res_shape, self.dtype().zero()); const x = Tensor.iota(res_shape, a); const y = Tensor.iota(res_shape, a + 1); - var res = x.cmp(.EQ, y).select(values, zeros); + var res = x.cmp(.EQ, y).select(values, .zeros(res_shape)); res._shape = res_shape; return res; } @@ -3554,8 +3742,7 @@ pub const Tensor = struct { const x = Tensor.iota(self.shape(), _axes.get(0)); const y = Tensor.iota(self.shape(), _axes.get(1)); - const zeros = Tensor.constant(self.shape(), self.dtype().zero()); - return x.addConstant(num_diagonals).cmp(.GE, y).select(self, zeros); + return x.addConstant(num_diagonals).cmp(.GE, y).select(self, .zeros(self._shape)); } test triangular { @@ -3810,11 +3997,20 @@ pub const Tensor = struct { return binaryOpHelper(self, other.broad(self._shape)); } - stdx.debug.assert(self._shape.eql(other._shape), "{s} expects tensor shapes to match, got {f} and {f}", .{ op_name, self._shape, other._shape }); + var other_ = other; + var same_shape = self._shape.eql(other._shape); + if (!same_shape and std.mem.eql(Shape.Tag, self._shape.tags(), other._shape.tags()) and other._shape.canBroadcastTo(self._shape)) { + // Only a restrictive version of broadcasting is allowed here, where all the tags matches already. + // Typical use case: `x.div(x.sum(.a))` + same_shape = true; + other_ = other.broad(self._shape); + } + + stdx.debug.assert(same_shape, "{s} expects tensor shapes to match, got {f} and {f}", .{ op_name, self._shape, other._shape }); const ctx = self.getContext(); - const location = ctx.location(src, "{s}({f}, {f})", .{ op_name, self, other }); - const ret = @call(.auto, op_fn, .{ ctx.mlirCtx(), self.value(), other.value(), location }); + const location = ctx.location(src, "{s}({f}, {f})", .{ op_name, self, other_ }); + const ret = @call(.auto, op_fn, .{ ctx.mlirCtx(), self.value(), other_.value(), location }); return _result(self._shape, ret.result(0)); } }.binaryOpHelper; @@ -3824,22 +4020,7 @@ pub const Tensor = struct { /// Only for debug purpose, it inserts device to host synchronization /// so it will slow down the program execution. pub fn print(input: Tensor) Tensor { - // TODO: find a way of doing print that doesn't involve a H2D copy. - return ops.addHostCallback( - &printCallback, - null, - &.{input}, - &.{input.shape()}, - .{ .output_operand_aliases = &.{0} }, - )[0]; - } - - fn printCallback(_: ?*anyopaque, inputs: []const HostBuffer, outputs: []const HostBuffer) void { - const host_buffer = inputs[0]; - std.log.defaultLog(.info, .zml, "Device buffer: {f}: {f}", .{ host_buffer.shape(), host_buffer.pretty() }); - // This is true because of the operand aliases. - // Since the result is already pointing to the input we don't need to modify the buffer. - std.debug.assert(host_buffer._data == outputs[0]._data); + return callback.call(callback.Print, .{input}, &.{input.shape()})[0]; } }; @@ -3876,11 +4057,11 @@ fn getPoolResDims(dt: DataType, in_dims: []const i64, base_dilations: @Vector(Te } fn getComparisonType(ctx: mlir.Context, dtype: DataType) dialect.stablehlo.CompareType { - return dialect.stablehlo.CompareType.init(ctx, switch (dtype) { - .i4, .i8, .i16, .i32, .i64 => .SIGNED, - .bool, .u4, .u8, .u16, .u32, .u64 => .UNSIGNED, - .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz, .bf16, .f16, .f32, .f64 => .FLOAT, - .c64, .c128 => @panic("Can't compare complex numbers"), + return dialect.stablehlo.CompareType.init(ctx, switch (dtype.class()) { + .bool => .UNSIGNED, + .integer => if (dtype.isSignedInt()) .SIGNED else .UNSIGNED, + .float => .FLOAT, + .complex => @panic("Can't compare complex numbers"), }); } @@ -3902,21 +4083,14 @@ test "Tensor.maxPool1d" { const x = try zml.Buffer.fromSlice(platform, .{ 2, 2, 5 }, &data); const result = try zml.testing.compileAndCall(platform, MaxPool._fwd, .{x}); - try zml.testing.expectEqualShapes(Shape.init(.{ 2, 2, 2 }, .f32), result.values.shape()); - try zml.testing.expectEqualShapes(Shape.init(.{ 2, 2, 2 }, .i32), result.indices.shape()); - const buffer = result.values.getValue([2][2][2]f32); + try zml.testing.expectEqualShapes(.init(.{ 2, 2, 2 }, .f32), result.values.shape()); + try zml.testing.expectEqualShapes(.init(.{ 2, 2, 2 }, .i32), result.indices.shape()); try std.testing.expectEqualDeep( [2][2][2]f32{ - [2][2]f32{ - [2]f32{ 2, 4 }, - [2]f32{ 7, 9 }, - }, - [2][2]f32{ - [2]f32{ 12, 14 }, - [2]f32{ 17, 19 }, - }, + .{ .{ 2, 4 }, .{ 7, 9 } }, + .{ .{ 12, 14 }, .{ 17, 19 } }, }, - buffer, + result.values.getValue([2][2][2]f32), ); } @@ -4081,16 +4255,6 @@ fn _parseGatherCoord(self: Tensor, axes_: anytype) struct { bool, stdx.BoundedAr return .{ axes_is_scalar, coord_axes }; } -fn parseArrayInfo(T: type) Shape { - return switch (@typeInfo(T)) { - .Array => |arr| { - const s = parseArrayInfo(arr.child); - return s.insert(0, .{arr.len}); - }, - else => .{ ._dtype = DataType.fromZigType(T) }, - }; -} - inline fn toI64(values: anytype) []i64 { var res: [Tensor.MAX_RANK]i64 = undefined; for (values, 0..) |val, i| res[i] = @intCast(val); diff --git a/zml/testing.zig b/zml/testing.zig index b236400b..1f5a521a 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -33,7 +33,7 @@ pub fn approxEq(comptime Float: type, l: Float, r: Float, tolerance: Float) bool return closeRel or closeAbs; } -/// Testing utility. Accepts both Tensor and HostBuffer but Tensor will be copied to the +/// Testing utility. Accepts both zml.Buffer and zml.HostBuffer but zml.Buffer will be copied to the /// host for comparison ! pub fn expectClose(left_: anytype, right_: anytype, tolerance: f32) !void { const allocator = if (builtin.is_test) std.testing.allocator else std.heap.smp_allocator; @@ -51,14 +51,13 @@ pub fn expectClose(left_: anytype, right_: anytype, tolerance: f32) !void { if (should_free_left) left.deinit(allocator); if (should_free_right) right.deinit(allocator); } - errdefer log.err("\n--> Left: {f}\n--> Right: {f}", .{ left.pretty(), right.pretty() }); - + errdefer log.err("\n--> Left: {0f}{0d:24.3}\n--> Right: {1f}{1d:24.3}", .{ left, right }); if (!std.mem.eql(i64, left.shape().dims(), right.shape().dims())) { log.err("left.shape() {f} != right.shape() {f}", .{ left.shape(), right.shape() }); return error.TestUnexpectedResult; } if (left.dtype() != right.dtype() and !(left.dtype() == .f16 and right.dtype() == .bf16)) { - log.err("left.dtype ({}) != right.dtype ({})", .{ left.dtype(), right.dtype() }); + log.err("left.dtype ({f}) != right.dtype ({f})", .{ left.shape(), right.shape() }); return error.TestUnexpectedResult; } switch (left.dtype()) { @@ -66,11 +65,15 @@ pub fn expectClose(left_: anytype, right_: anytype, tolerance: f32) !void { .f16, .f32, .f64, + .f4e2m1, + .f8e3m4, + .f8e4m3, .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz, + .f8e8m0, => |t| { const L = t.toZigType(); const left_data = left.items(L); @@ -89,7 +92,7 @@ pub fn expectClose(left_: anytype, right_: anytype, tolerance: f32) !void { const right_data = right.items(R); for (left_data, right_data, 0..) |l, r, i| { if (!approxEq(f32, zml.floats.floatCast(f32, l), zml.floats.floatCast(f32, r), tolerance)) { - log.err("left.data != right_data.\n < {any:.3} \n > {any:.3}\n error at idx {any}: {any:.3} != {any:.3}", .{ center(left_data, i), center(right_data, i), i, left_data[i], right_data[i] }); + log.err("left.data != right_data.\n < {d:40.3} \n > {d:40.3}\n error at idx {d}: {d:.3} != {d:.3}", .{ stdx.fmt.slice(L, center(left_data, i)), stdx.fmt.slice(R, center(right_data, i)), i, left_data[i], right_data[i] }); return error.TestUnexpectedResult; } } @@ -97,7 +100,7 @@ pub fn expectClose(left_: anytype, right_: anytype, tolerance: f32) !void { else => unreachable, } }, - inline .bool, .u4, .u8, .u16, .u32, .u64, .i4, .i8, .i16, .i32, .i64 => |t| { + inline .bool, .u2, .u4, .u8, .u16, .u32, .u64, .i2, .i4, .i8, .i16, .i32, .i64 => |t| { const T = t.toZigType(); return std.testing.expectEqualSlices(T, left.items(T), right.items(T)); }, @@ -222,8 +225,8 @@ pub fn testLayerOut( const exe = try zml.compileModel(alloc, fwd, layer, input_shapes, platform); const n_out_exp = activations.countLayers(out_name); - if (exe.inner.result_shapes.len != n_out_exp) { - log.warn("Reference models produces {d} outputs, but implementation produces {d}", .{ n_out_exp, exe.inner.result_shapes.len }); + if (exe.inner.output_shapes.len != n_out_exp) { + log.warn("Reference models produces {d} outputs, but implementation produces {d}", .{ n_out_exp, exe.inner.output_shapes.len }); } const mod = exe.prepare(layer_weights); @@ -264,7 +267,7 @@ pub fn testLayerOut( var buf: [1024]u8 = undefined; var failed: bool = false; - for (0..mod.inner.result_shapes.len) |i| { + for (0..mod.inner.output_shapes.len) |i| { const full_name = std.fmt.bufPrint(&buf, "{s}.{d}", .{ out_name, i }) catch unreachable; const expected_out = activations.get(full_name) orelse { log.warn("Output buffer not found: {s}", .{full_name}); @@ -299,7 +302,7 @@ test testLayer { }; // create a buffer store containing the activations: - var activations = try zml.aio.BufferStore.init(std.testing.allocator, &.{}); + var activations = zml.aio.BufferStore.init(std.testing.allocator); defer activations.deinit(); { const input = zml.HostBuffer.fromArray(&[2]f32{ 1, -1 }); diff --git a/zml/tokenizer/BUILD.bazel b/zml/tokenizer/BUILD.bazel index 9b2f9c71..b25f7070 100644 --- a/zml/tokenizer/BUILD.bazel +++ b/zml/tokenizer/BUILD.bazel @@ -1,4 +1,5 @@ load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library") +load("@zml//bazel:zig_srcs.bzl", "zig_srcs") zig_library( name = "tokenizer", @@ -7,7 +8,6 @@ zig_library( main = "tokenizer.zig", visibility = ["//visibility:public"], deps = [ - "//async", "//ffi:zig", "//stdx", "//zml/tokenizer/hftokenizers", @@ -25,3 +25,8 @@ zig_binary( "//stdx", ], ) + +zig_srcs( + name = "sources", + zig_lib = ":tokenizer", +) diff --git a/zml/tokenizer/sentencepiece/BUILD.bazel b/zml/tokenizer/sentencepiece/BUILD.bazel index dba86ddb..10ac080b 100644 --- a/zml/tokenizer/sentencepiece/BUILD.bazel +++ b/zml/tokenizer/sentencepiece/BUILD.bazel @@ -1,5 +1,6 @@ load("@rules_zig//zig:defs.bzl", "zig_library") load("//bazel:swig.bzl", "swig_cc_library") +load("//bazel:zig_srcs.bzl", "zig_srcs") swig_cc_library( name = "sentencepiece_swig", @@ -22,3 +23,13 @@ zig_library( "//stdx", ], ) + +zig_srcs( + name = "sources", + zig_lib = ":sentencepiece", +) + +cc_static_library( + name="sentencepiece_static", + deps = [":sentencepiece_swig"] +) diff --git a/zml/tokenizer/tokenizer.zig b/zml/tokenizer/tokenizer.zig index 443e2fb8..5109ce31 100644 --- a/zml/tokenizer/tokenizer.zig +++ b/zml/tokenizer/tokenizer.zig @@ -1,6 +1,5 @@ const std = @import("std"); -const asynk = @import("async"); const hftokenizers = @import("hftokenizers"); const sentencepiece = @import("sentencepiece"); @@ -98,15 +97,15 @@ pub const Tokenizer = union(Tokenizers) { pub fn fromFile(allocator: std.mem.Allocator, model: []const u8) !Tokenizer { if (std.mem.endsWith(u8, model, ".pb")) { - return .{ .sentencepiece = try asynk.callBlocking(sentencepiece.SentencePieceProcessor.fromFile, .{model}) }; + return .{ .sentencepiece = try sentencepiece.SentencePieceProcessor.fromFile(model) }; } if (std.mem.endsWith(u8, model, ".json")) { - return .{ .hftokenizers = try asynk.callBlocking(hftokenizers.HFTokenizer.fromFile, .{model}) }; + return .{ .hftokenizers = try hftokenizers.HFTokenizer.fromFile(model) }; } if (std.mem.endsWith(u8, model, ".tinyllama")) { const tokenizer = try allocator.create(homemade.Tokenizer); - tokenizer.* = try asynk.callBlocking(homemade.fromTinyLlamaFile, .{ allocator, model, 32000 }); + tokenizer.* = try homemade.fromTinyLlamaFile(allocator, model, 32000); return .{ .homemade = tokenizer }; } diff --git a/zml/torch.zig b/zml/torch.zig index e982aaec..09888aee 100644 --- a/zml/torch.zig +++ b/zml/torch.zig @@ -169,7 +169,7 @@ test pixelShuffle { /// /// Note: at the difference of Pytorch, shifts need to be explicitly repeated, even if they are the same for all axes. /// ref: https://pytorch.org/docs/stable/generated/torch.roll.html -pub fn roll(self: Tensor, shifts: []const i64, axes_: []const u8) Tensor { +pub fn roll(self: Tensor, shifts: []const i64, axes_: []const i8) Tensor { // TODO(hugo) accept following syntax: x.roll(.{ .a = 5, .b = 8 }) stdx.debug.assert(self.rank() > 0 and shifts.len == axes_.len, "Shifts length ({d}) and dims length ({d}) are not equal, we expect the same length.", .{ shifts.len, axes_.len }); @@ -180,11 +180,11 @@ pub fn roll(self: Tensor, shifts: []const i64, axes_: []const u8) Tensor { return roll(first_dim_rolled, tail_shifts, tail_dims); } - const a = axes_[0]; + const a = self.axis(axes_[0]); const start = @mod(self.dim(a) - shifts[0], self.dim(a)); const idx = Tensor.arange(.{ .start = start, .end = start + self.dim(a) }, .f32); const divisor: f32 = @floatFromInt(self.dim(a)); - return self.gatherValues(a, idx.fmod(divisor).convert(.i32), .{}); + return self.gather_(&.{a}, &.{idx.fmod(divisor).convert(.i32)}, .{}); } test roll { @@ -194,7 +194,7 @@ test roll { const res = try zml.testing.compileAndCall( platform, roll, - .{ input, &[_]i64{ 2, 1 }, &[_]u8{ 0, 1 } }, + .{ input, &[_]i64{ 2, 1 }, &[_]i8{ 0, 1 } }, ); const expectation = zml.HostBuffer.fromSlice(.{ 4, 2 }, &[_]f32{ 6, 5, 8, 7, 2, 1, 4, 3 }); @@ -274,3 +274,12 @@ test meshgrid { ); } } + +/// Flattens the given axis and the next one, into one new axis. +pub fn flatten(self: Tensor, axis_: anytype) Tensor { + const old_shape = self._shape; + const a = self.axis(axis_); + stdx.debug.assert(a + 1 < self.rank(), "Can't flatten {f} on the last axis {}.", .{ self, axis_ }); + const new_shape = old_shape.mergeAxis(a, .{ a, a + 1 }); + return self.reshape(new_shape); +} diff --git a/zml/zml.zig b/zml/zml.zig index b5cd8039..55559f5c 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -6,11 +6,13 @@ // Namespaces const std = @import("std"); +pub const platform_specific = @import("c"); pub const tokenizer = @import("zml/tokenizer"); pub const aio = @import("aio.zig"); pub const Buffer = @import("buffer.zig").Buffer; pub const Bufferized = @import("tensor.zig").Bufferized; +pub const callback = @import("callback.zig"); pub const CompilationOptions = @import("platform.zig").CompilationOptions; pub const context = @import("context.zig"); pub const Context = @import("context.zig").Context; @@ -43,7 +45,6 @@ pub const Tensor = @import("tensor.zig").Tensor; pub const testing = @import("testing.zig"); pub const torch = @import("torch.zig"); -// pub const tokenizer = @import("tokenizer.zig"); pub const tools = struct { pub const Tracer = @import("tools/tracer.zig").Tracer; };