diff --git a/examples/MODULE.bazel b/examples/MODULE.bazel index 14000765..6ae1b80e 100644 --- a/examples/MODULE.bazel +++ b/examples/MODULE.bazel @@ -185,3 +185,40 @@ copy_file( model = "openlm-research/open_llama_3b", ) use_repo(huggingface, "OpenLM-Research-OpenLLaMA-3B") + + +huggingface.model( + name = "StabilityAI-sd-turbo", + build_file_content = """\ +package(default_visibility = ["//visibility:public"]) + +filegroup( + name = "vae", + srcs = ["vae/diffusion_pytorch_model.fp16.safetensors"], +) + +filegroup( + name = "prompt_encoder", + srcs = ["text_encoder/model.fp16.safetensors",], +) + +filegroup( + name = "vocab", + srcs = ["tokenizer/vocab.json"], +) + +filegroup( + name = "unet", + srcs = ["unet/diffusion_pytorch_model.fp16.safetensors"], +) +""", + commit = "0096a99066b73650354282eeef18114cf0dab75c", + model = "stabilityai/sd-turbo", + includes = [ + "vae/diffusion_pytorch_model.fp16.safetensors", + "text_encoder/model.fp16.safetensors", + "tokenizer/vocab.json", + "unet/diffusion_pytorch_model.fp16.safetensors", + ], +) +use_repo(huggingface, "StabilityAI-sd-turbo") diff --git a/examples/sdxl/BUILD.bazel b/examples/sdxl/BUILD.bazel new file mode 100644 index 00000000..b08db89a --- /dev/null +++ b/examples/sdxl/BUILD.bazel @@ -0,0 +1,36 @@ +load("@bazel_skylib//rules:native_binary.bzl", "native_binary") +load("@zml//bazel:zig.bzl", "zig_cc_binary") + +zig_cc_binary( + name = "sdxl_bin", + main = "main.zig", + deps = [ + "//third_party/tigerbeetle:flags", + "@zml//async", + "@zml//zml", + ], + srcs = ["transformer.zig"], +) + +native_binary( + name = "sdxl", + src = ":sdxl_bin", + args = [ + # vae model + "$(location @StabilityAI-sd-turbo//:vae)", + # lightning model + "$(location @StabilityAI-sd-turbo//:unet)", + # prompt encoder, and vocab + "$(location @StabilityAI-sd-turbo//:prompt_encoder)", + "$(location @StabilityAI-sd-turbo//:vocab)", + "'A cinematic shot of a baby racoon wearing an intricate italian priest robe.'", + "$(location :sd-turbo.activations.pt)", + ], + data = [ + "@StabilityAI-sd-turbo//:vae", + "@StabilityAI-sd-turbo//:unet", + "@StabilityAI-sd-turbo//:prompt_encoder", + "@StabilityAI-sd-turbo//:vocab", + ":sd-turbo.activations.pt", + ], +) diff --git a/examples/sdxl/main.zig b/examples/sdxl/main.zig new file mode 100644 index 00000000..88e89cb5 --- /dev/null +++ b/examples/sdxl/main.zig @@ -0,0 +1,647 @@ +const std = @import("std"); + +const zml = @import("zml"); +const meta = zml.meta; +const asynk = @import("async"); +const flags = @import("tigerbeetle/flags"); + +const transformer = @import("transformer.zig"); +const ClipTextTransformer = transformer.ClipTextTransformer; + +const log = std.log.scoped(.sdxl); + +pub fn main() !void { + try asynk.AsyncThread.main(std.heap.c_allocator, asyncMain, .{}); +} + +pub fn asyncMain() !void { + var gpa = std.heap.GeneralPurposeAllocator(.{ .thread_safe = true }){}; + defer _ = gpa.deinit(); + const allocator = gpa.allocator(); + + var model_arena = std.heap.ArenaAllocator.init(allocator); + defer model_arena.deinit(); + const arena = model_arena.allocator(); + + const tmp = try std.fs.openDirAbsolute("/tmp", .{}); + try tmp.makePath("zml/sdxl/cache"); + var context = try zml.Context.init(); + defer context.deinit(); + + const platform = context.autoPlatform().withCompilationOptions(.{ + .cache_location = "/tmp/zml/sdxl/cache", + .xla_dump_to = "/tmp/zml/sdxl", + }); + + const args = try std.process.argsAlloc(allocator); + defer std.process.argsFree(allocator, args); + const vae_model_path = args[1]; + const unet_model_path = args[2]; + const prompt_encoder_model_path = args[3]; + const vocab_path = args[4]; + const prompt = args[5]; + const activation_path = args[6]; + var activations = try zml.aio.detectFormatAndOpen(allocator, activation_path); + defer activations.deinit(); + + log.info("Prompt: {s}", .{prompt}); + + const prompt_encoder_config: transformer.ClipTextConfig = .{ + .max_position_embeddings = 77, + .num_heads = 16, + .layer_norm_eps = 1e-5, + .hidden_act = .gelu, + }; + + const tokenizer = try loadSdTurboTokenizer(allocator, vocab_path); + defer tokenizer.deinit(); + const prompt_tokens = try tokenizer.encode(allocator, prompt, .{ + .add_bos = true, + .add_eos = true, + .pad_to = prompt_encoder_config.max_position_embeddings, + .debug = true, + }); + log.info("prompt tokens: {d}", .{prompt_tokens}); + defer allocator.free(prompt_tokens); + + const prompt_dev = try zml.Buffer.fromSlice( + platform, + .{ .s = prompt_encoder_config.max_position_embeddings }, + prompt_tokens, + ); + _ = prompt_dev; // autofix + + const prompt_encoder_exe = blk: { + var local_arena = std.heap.ArenaAllocator.init(allocator); + defer local_arena.deinit(); + const arena_alloc = local_arena.allocator(); + + var store = try zml.aio.detectFormatAndOpen(allocator, prompt_encoder_model_path); + defer store.deinit(); + + var prompt_encoder = try zml.aio.populateModelWithPrefix(ClipTextTransformer, arena_alloc, store, "text_model"); + prompt_encoder.init(prompt_encoder_config); + + const prompt_encoder_weights = try zml.aio.loadModelBuffersWithPrefix(ClipTextTransformer, prompt_encoder, store, arena_alloc, platform, "text_model"); + log.info("Loaded prompt encoder from {s}, found {} buffers: {}", .{ prompt_encoder_model_path, store.buffers.count(), prompt_encoder }); + + // try testPromptEncoder(platform, activations, prompt_encoder, prompt_encoder_weights); + + const exe = try zml.compileModel( + allocator, + prompt_encoder, + .forward, + .{zml.Shape.init(.{ .s = prompt_encoder.config.max_position_embeddings }, .i32)}, + platform, + ); + + break :blk try exe.prepare(allocator, prompt_encoder_weights); + }; + defer prompt_encoder_exe.deinit(); + + // const prompt_embed = prompt_encoder_exe.call(.{prompt_dev}); + // _ = prompt_embed; // autofix + + const vae_store = try zml.aio.detectFormatAndOpen(allocator, vae_model_path); + defer vae_store.deinit(); + var vae = try zml.aio.populateModel(AutoEncoderKl, arena, vae_store); + const vae_weights = try zml.aio.loadModelBuffers(AutoEncoderKl, vae, vae_store, arena, platform); + _ = vae_weights; // autofix + // defer zml.aio.unloadBuffers(vae_weights); + + log.info("Loaded VAE from {s}, found {} buffers.", .{ vae_model_path, vae_store.buffers.count() }); + const group_norm_conf: struct { u32, f32 } = .{ 32, 1e-5 }; + + log.info("vae.group_norm -> {}", .{vae.decoder.mid_block.attentions[0].group_norm}); + zml.meta.visit(struct { + fn cb(conf: struct { u32, f32 }, group_norm: *GroupNorm) void { + const group_size, const eps = conf; + group_norm.group_size = group_size; + group_norm.eps = eps; + } + }.cb, group_norm_conf, &vae); + log.info("vae.group_norm -> {}", .{vae.decoder.mid_block.attentions[0].group_norm}); + // try testVae(platform, activations, vae.decoder, vae_weights.decoder); + + var unet_store = try zml.aio.detectFormatAndOpen(allocator, unet_model_path); + defer unet_store.deinit(); + + log.info("Loaded unet from {s}, found {} buffers.", .{ unet_model_path, unet_store.buffers.count() }); + + var unet = try zml.aio.populateModel(Unet2DConditionModel, arena, unet_store); + zml.meta.visit(struct { + fn cb(conf: struct { u32, f32 }, group_norm: *GroupNorm) void { + const group_size, const eps = conf; + group_norm.group_size = group_size; + group_norm.eps = eps; + } + }.cb, group_norm_conf, &unet); + zml.meta.visit(struct { + fn cb(eps: f32, layer_norm: *zml.nn.LayerNorm) void { + layer_norm.eps = eps; + } + }.cb, 1e-5, &unet); + log.info("Unet: {}", .{unet}); + + const unet_weights = try zml.aio.loadModelBuffers(Unet2DConditionModel, unet, unet_store, arena, platform); + try testUnet(platform, activations, unet, unet_weights); +} + +fn testPromptEncoder(platform: zml.Platform, activations: zml.aio.BufferStore, encoder: ClipTextTransformer, encoder_weights: zml.Bufferized(ClipTextTransformer)) !void { + try zml.testing.testLayer(platform, activations, "text_encoder.text_model.encoder.layers.1.mlp", encoder.encoder.layers[1].mlp, encoder_weights.encoder.layers[1].mlp, 0.05); + try zml.testing.testLayer(platform, activations, "text_encoder.text_model.encoder.layers.1", encoder.encoder.layers[1], encoder_weights.encoder.layers[1], 0.05); + try zml.testing.testLayer(platform, activations, "text_encoder.text_model.embeddings", encoder.embeddings, encoder_weights.embeddings, 0.001); + try zml.testing.testLayer(platform, activations, "text_encoder.text_model", encoder, encoder_weights, 0.1); +} + +fn testUnet(platform: zml.Platform, activations: zml.aio.BufferStore, unet: Unet2DConditionModel, unet_weights: zml.Bufferized(Unet2DConditionModel)) !void { + try zml.testing.testLayer(platform, activations, "unet.conv_in", unet.conv_in, unet_weights.conv_in, 0.005); + try zml.testing.testLayer(platform, activations, "unet.conv_out", unet.conv_out, unet_weights.conv_out, 0.005); + try zml.testing.testLayer(platform, activations, "unet.conv_norm_out", unet.conv_norm_out, unet_weights.conv_norm_out, 0.005); + + // Down block + if (false) { + try zml.testing.testLayer(platform, activations, "unet.down_blocks.3.resnets.0", unet.down_blocks.@"3".resnets[0], unet_weights.down_blocks.@"3".resnets[0], 0.05); + + try zml.testing.testLayer(platform, activations, "unet.down_blocks.3", unet.down_blocks.@"3", unet_weights.down_blocks.@"3", 0.05); + + try zml.testing.testLayer(platform, activations, "unet.down_blocks.0.attentions.1.transformer_blocks.0.ff.net.0", unet.down_blocks.@"0".attentions[1].transformer_blocks[0].ff.net.@"0", unet_weights.down_blocks.@"0".attentions[1].transformer_blocks[0].ff.net.@"0", 0.05); + // try zml.testing.testLayer(platform, activations, "unet.down_blocks.0.attentions.1.transformer_blocks.0", unet.down_blocks.@"0".attentions[1].transformer_blocks[0], unet_weights.down_blocks.@"0".attentions[1].transformer_blocks[0], 0.05); + try zml.testing.testLayer(platform, activations, "unet.down_blocks.0.attentions.1", unet.down_blocks.@"0".attentions[1], unet_weights.down_blocks.@"0".attentions[1], 0.05); + try zml.testing.testLayer(platform, activations, "unet.down_blocks.0", unet.down_blocks.@"0", unet_weights.down_blocks.@"0", 0.05); + } + + // Middle block. + if (true) { + try zml.testing.testLayer(platform, activations, "unet.mid_block.resnets.0", unet.mid_block.resnets[0], unet_weights.mid_block.resnets[0], 0.05); + try zml.testing.testLayer(platform, activations, "unet.mid_block.attentions.0", unet.mid_block.attentions[0], unet_weights.mid_block.attentions[0], 0.05); + try zml.testing.testLayer(platform, activations, "unet.mid_block.resnets.1", unet.mid_block.resnets[1], unet_weights.mid_block.resnets[1], 0.05); + // try zml.testing.testLayer(platform, activations, "unet.mid_block.attentions.0", unet.mid_block.attentions[0], unet_weights.mid_block.attentions[0], 0.05); + try zml.testing.testLayer(platform, activations, "unet.mid_block", unet.mid_block, unet_weights.mid_block, 0.05); + } + + // Up blocks + { + try zml.testing.testLayer(platform, activations, "unet.up_blocks.0.upsamplers.0", unet.up_blocks.@"0".upsamplers.@"0", unet_weights.up_blocks.@"0".upsamplers.@"0", 0.05); + + // TODO: dig into the precision issues. + // try zml.testing.testLayer(platform, activations, "unet.up_blocks.0", unet.up_blocks.@"0", unet_weights.up_blocks.@"0", 0.05); + } +} + +fn testVae(platform: zml.Platform, activations: zml.aio.BufferStore, vae: AutoEncoderKl.Decoder, vae_weights: zml.Bufferized(AutoEncoderKl.Decoder)) !void { + // TODO this seems a bit high + try zml.testing.testLayer(platform, activations, "vae.decoder.conv_in", vae.conv_in, vae_weights.conv_in, 0.02); + try zml.testing.testLayer(platform, activations, "vae.decoder.conv_out", vae.conv_out, vae_weights.conv_out, 0.02); + try zml.testing.testLayer(platform, activations, "vae.decoder.mid_block.attentions.0.to_out.0", vae.mid_block.attentions[0].to_out.@"0", vae_weights.mid_block.attentions[0].to_out.@"0", 0.02); + // try zml.testing.testLayerOut(platform, activations, "vae.decoder.mid_block.attentions.0", "vae.decoder.mid_block.attentions.0.group_norm.in", vae.mid_block.attentions[0], vae_weights.mid_block.attentions[0], 0.02); + try zml.testing.testLayer(platform, activations, "vae.decoder.mid_block.attentions.0", vae.mid_block.attentions[0], vae_weights.mid_block.attentions[0], 0.02); + // try zml.testing.testLayer(platform, activations, "vae.decoder.mid_block", vae.mid_block, vae_weights.mid_block, 0.02); +} + +pub const AutoEncoderKl = struct { + decoder: Decoder, + post_quant_conv: Conv2dSame, + + const Decoder = struct { + conv_in: Conv2dSame, + // up_blocks: + mid_block: UNetMidBlock2D, + conv_norm_out: GroupNorm, + conv_act: zml.nn.Activation = .silu, + conv_out: Conv2dSame, + }; + + pub const UNetMidBlock2D = struct { + attentions: []Attention, + resnets: []ResnetBlock2D, + + pub fn forward(self: UNetMidBlock2D, images: zml.Tensor) zml.Tensor { + const x = images.withTags(.{ .b, .channels, .height, .width }); + var hidden = self.resnets[0].forward(x, null); + + for (self.attentions, self.resnets[1..]) |attn, resnet| { + hidden = attn.forward(hidden); + hidden = resnet.forward(hidden, null); + } + return hidden; + } + }; + + pub const Attention = struct { + group_norm: GroupNorm, + to_q: zml.nn.Linear, + to_k: zml.nn.Linear, + to_v: zml.nn.Linear, + to_out: struct { zml.nn.Linear }, + + pub fn forward(self: Attention, input: zml.Tensor) zml.Tensor { + return self.group_norm.forward(input).merge(.{ .hw = .{ -2, -1 } }); + // const x0 = if (input.shape().isFullyTagged()) input else input.withTags(.{ .b, .channels, .height, .width }); + // var x = x0.merge(.{ .hw = .{ .height, .width } }); + // x = groupNorm(x, .channels, .hw, self.group_norm); + // return x; + // x = x.contiguous(.{.channels}); + // log.info("x: {}", .{x}); + // const ctx = x.rename(.{ .hw = .k }); + // const nh = 1; + // const q = zml.call(self.to_q, .forward, .{x.rename(.{ .hw = .q })}).splitAxis(.channels, .{ .h = nh, .hd = .auto }); + // // log.info("q: {}", .{q}); + // const k = zml.call(self.to_k, .forward, .{ctx}).splitAxis(.channels, .{ .h = nh, .hd = .auto }); + // // log.info("k: {}", .{k}); + // const v = zml.call(self.to_v, .forward, .{ctx}).splitAxis(.channels, .{ .h = nh, .hd = .auto }); + // // log.info("v: {}", .{v}); + + // const attn_output = zml.nn.sdpa(q, k, v, .{}); + // const attn = attn_output.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .hw, .d = .channels }); + // var out = zml.call(self.to_out.@"0", .forward, .{attn}); + + // out = out.splitAxis(.hw, .{ .height = x0.dim(.height), .width = .auto }); + // out = out.transpose(x0.shape()); + // return out; + } + }; +}; + +pub const Unet2DConditionModel = struct { + conv_in: Conv2dSame, + + // time_proj + // time_embedding + down_blocks: DownBlocks, + mid_block: UNetMidBlock2DCrossAttn, + up_blocks: UpBlocks, + + conv_norm_out: GroupNorm, + conv_act: zml.nn.Activation = .silu, + conv_out: Conv2dSame, +}; + +pub const Conv2dSame = struct { + weight: zml.Tensor, + bias: zml.Tensor, + + pub fn forward(self: Conv2dSame, input: zml.Tensor) zml.Tensor { + const x = input.withPartialTags(.{ .channels, .height, .width }); + log.warn("Conv2dSame({}, {})", .{ self.weight, input }); + const y = zml.Tensor.conv2d(x, self.weight, .{ + .padding = &.{ 1, 1, 1, 1 }, + .input_feature_dimension = x.axis(.channels), + .input_spatial_dimensions = &.{ x.axis(.height), x.axis(.width) }, + }); + return y.add(self.bias.withTags(.{.channels}).broad(y.shape())); + } +}; + +pub const GroupNorm = struct { + weight: zml.Tensor, + bias: zml.Tensor, + + group_size: u32, + eps: f32, + + pub fn forward(self: GroupNorm, input: zml.Tensor) zml.Tensor { + const x = input.withPartialTags(.{ .channels, .height, .width }); + log.warn("GroupNorm({}, {})", .{ self.weight, input }); + const x_grouped = x.merge(.{ .hw = .{ .height, .width } }); + return zml.nn.groupNorm( + x_grouped, + .channels, + .hw, + .{ .weight = self.weight, .bias = self.bias, .group_size = self.group_size, .eps = self.eps }, + ).reshape(x.shape()); + } +}; + +pub fn groupNorm2D(self: GroupNorm, input: zml.Tensor) zml.Tensor { + // log.warn("GroupNorm({}, {})", .{ self.weight, input }); + // const x = input.withPartialTags(.{ .channels, .height, .width }).merge(.{ .hw = .{ .height, .width } }); + + // const x_norm = groupNorm(x, .channels, .hw, weights); + // return x_norm.reshape(input.shape()); + const x = input.withPartialTags(.{ .channels, .width, .height }); + log.warn("GroupNorm({}, {}, {})", .{ self.weight, self.group_size, input }); + var x_grouped = x.splitAxis(.channels, .{ .group = self.group_size, .c = .auto }); + x_grouped = x_grouped.merge(.{ .cwh = .{ .c, .width, .height } }); + + const normed = zml.nn.normalizeVariance(x_grouped, .cwh, self.eps).reshape(x.shape()); + + var out = normed.mul(self.weight.withTags(.{.channels}).broad(x.shape())); + out = out.add(self.bias.withTags(.{.channels}).broad(x.shape())); + return out; +} + +pub const DownBlocks = struct { + @"0": CrossAttnDownBlock2D, + @"1": CrossAttnDownBlock2D, + @"2": CrossAttnDownBlock2D, + @"3": DownBlock2D, + + pub fn forward(self: DownBlocks, images: zml.Tensor, time_embedding: zml.Tensor, encoder_hidden_states: zml.Tensor) zml.Tensor { + var hidden = images; + + hidden = self.@"0".forward(hidden, time_embedding, encoder_hidden_states); + hidden = self.@"1".forward(hidden, time_embedding, encoder_hidden_states); + hidden = self.@"2".forward(hidden, time_embedding, encoder_hidden_states); + hidden = self.@"3".forward(hidden, time_embedding); + + return hidden; + } + + pub const CrossAttnDownBlock2D = struct { + attentions: []Transformer2DModel, + resnets: []ResnetBlock2D, + downsamplers: struct { struct { conv: Conv2dHalf } }, + + pub fn forward( + self: CrossAttnDownBlock2D, + images: zml.Tensor, + time_embedding: zml.Tensor, + encoder_hidden_states: zml.Tensor, + ) zml.Tensor { + var hidden = images; + for (self.resnets, self.attentions) |resnet, attn| { + hidden = zml.call(resnet, .forward, .{ hidden, time_embedding }); + hidden = zml.call(attn, .forward, .{ hidden, encoder_hidden_states }); + } + hidden = zml.call(self.downsamplers[0].conv, .forward, .{hidden}); + return hidden; + } + }; + + pub const DownBlock2D = struct { + resnets: []ResnetBlock2D, + + pub fn forward(self: DownBlock2D, images: zml.Tensor, time_embedding: zml.Tensor) zml.Tensor { + var y = images; + for (self.resnets) |resnet| { + y = zml.call(resnet, .forward, .{ y, time_embedding }); + } + return y; + } + }; + + pub const Conv2dHalf = struct { + weight: zml.Tensor, + bias: zml.Tensor, + + pub fn forward(self: Conv2dHalf, input: zml.Tensor) zml.Tensor { + const x = input.withPartialTags(.{ .channels, .height, .width }); + const y = zml.Tensor.conv2d( + x, + self.weight, + .{ + .padding = &.{ 1, 1, 1, 1 }, + .window_strides = &.{ 2, 2 }, + }, + ); + return y.add(self.bias.withTags(.{.channels}).broad(y.shape())); + } + }; +}; + +pub const UNetMidBlock2DCrossAttn = struct { + attentions: []Transformer2DModel, + resnets: []ResnetBlock2D, + + pub fn forward(self: UNetMidBlock2DCrossAttn, images: zml.Tensor, time_embedding: zml.Tensor, encoder_hidden_states: zml.Tensor) zml.Tensor { + var hidden = self.resnets[0].forward(images, time_embedding); + + for (self.attentions, self.resnets[1..]) |attn, resnet| { + hidden = attn.forward(hidden, encoder_hidden_states); + hidden = resnet.forward(hidden, time_embedding); + } + return hidden; + } +}; + +pub const UpBlocks = struct { + @"0": UpBlock2D, + @"1": CrossAttnUpBlock2D, + @"2": CrossAttnUpBlock2D, + @"3": CrossAttnUpBlock2D, + + pub fn forward(self: UpBlocks, images: zml.Tensor, time_embedding: zml.Tensor, encoder_hidden_states: zml.Tensor) zml.Tensor { + var hidden = images; + + hidden = self.@"0".forward(hidden, time_embedding); + hidden = self.@"1".forward(hidden, time_embedding, encoder_hidden_states); + hidden = self.@"2".forward(hidden, time_embedding, encoder_hidden_states); + hidden = self.@"3".forward(hidden, time_embedding, encoder_hidden_states); + + return hidden; + } + + const UpBlock2D = struct { + resnets: []ResnetBlock2D, + upsamplers: struct { Upsampler }, + + // TODO: zml handle const slice as inputs + pub fn forward( + self: UpBlock2D, + images: zml.Tensor, + time_embedding: zml.Tensor, + downscaled_images: []zml.Tensor, + ) zml.Tensor { + var hidden = images; + const n = self.resnets.len; + zml.meta.assert(n == downscaled_images.len, "this UpBlock2D expects {} downscaled images, got: {}", .{ n, downscaled_images.len }); + for (self.resnets, 0..) |resnet, i| { + hidden = zml.Tensor.concatenate(&.{ hidden, downscaled_images[n - 1 - i] }, 1); + hidden = zml.call(resnet, .forward, .{ hidden, time_embedding }); + } + + hidden = zml.call(self.upsamplers[0], .forward, .{hidden}); + return hidden; + } + }; + + const CrossAttnUpBlock2D = struct {}; + + const Upsampler = struct { + conv: Conv2dSame, + pub fn forward(self: Upsampler, images: zml.Tensor) zml.Tensor { + var hidden = images; + hidden = zml.nn.nearest(hidden, &.{ 2, 2 }); + hidden = self.conv.forward(hidden); + return hidden; + } + }; +}; + +pub const ResnetBlock2D = struct { + norm1: GroupNorm, + conv1: Conv2dSame, + time_emb_proj: ?zml.nn.Linear, + norm2: GroupNorm, + conv2: Conv2dSame, + nonlinearity: zml.nn.Activation = .silu, + conv_shortcut: ?struct { weight: zml.Tensor }, + // TODO: output_scale_factor ? + + pub fn forward(self: ResnetBlock2D, images: zml.Tensor, time_embedding: ?zml.Tensor) zml.Tensor { + const x = images.withPartialTags(.{ .batch, .channels, .height, .width }); + var hidden = self.norm1.forward(x); + hidden = self.nonlinearity.forward(hidden); + hidden = self.conv1.forward(hidden); + + if (self.time_emb_proj) |time_emb_proj| { + var t_emb = time_embedding.?.withPartialTags(.{.channels}); + t_emb = self.nonlinearity.forward(t_emb); + t_emb = time_emb_proj.forward(t_emb); + hidden = hidden.add(t_emb.appendAxes(.{ .height, .width }).broad(hidden.shape())); + } else { + zml.meta.assert(time_embedding == null, "This model doesn' support time embeddings", .{}); + } + + hidden = self.norm2.forward(hidden); + hidden = self.nonlinearity.forward(hidden); + hidden = self.conv2.forward(hidden); + + return if (self.conv_shortcut) |conv_shortcut| { + const kernel = conv_shortcut.weight.withTags(.{ .channels_out, .channels, .w, .h }); + const shortcut = kernel.squeeze(.w).squeeze(.h); + const x2 = x.dot(shortcut, .{.channels}).rename(.{ .channels_out = .channels }).contiguous(.{ .channels, .height, .width }); + return x2.add(hidden); + } else x.add(hidden); + } +}; + +pub const Transformer2DModel = struct { + norm: GroupNorm, + proj_in: zml.nn.Linear, + transformer_blocks: []BasicTransformerBlock, + proj_out: zml.nn.Linear, + + pub fn forward(self: Transformer2DModel, images: zml.Tensor, encoder_hidden_states: zml.Tensor) zml.Tensor { + const x = images.withPartialTags(.{ .b, .channels, .height, .width }); + var hidden = x; + hidden = self.norm.forward(hidden); + hidden = hidden.merge(.{ .hw = .{ .height, .width } }).contiguous(.{.channels}); + hidden = self.proj_in.forward(hidden); + + for (self.transformer_blocks) |block| { + hidden = zml.call(block, .forward, .{ hidden, encoder_hidden_states }); + } + + hidden = self.proj_out.forward(hidden); + hidden = hidden.contiguous(.{.hw}).splitAxis(.hw, .{ .height = x.dim(.height), .width = .auto }); + return hidden.add(x); + } +}; + +pub const BasicTransformerBlock = struct { + norm1: zml.nn.LayerNorm, + attn1: Attention, + norm2: zml.nn.LayerNorm, + attn2: Attention, + norm3: zml.nn.LayerNorm, + ff: struct { net: struct { GEGLU, void, zml.nn.Linear } }, + + const Attention = struct { + to_q: zml.nn.Linear, + to_k: zml.nn.Linear, + to_v: zml.nn.Linear, + to_out: struct { zml.nn.Linear }, + + pub fn forward(self: Attention, input: zml.Tensor, context: zml.Tensor) zml.Tensor { + const x = if (input.shape().isFullyTagged()) input else input.withTags(.{ .b, .hw, .channels }); + const ctx = if (input.shape().isFullyTagged()) context else context.withTags(.{ .b, .k, .channels }); + const nh = 5; + const q = zml.call(self.to_q, .forward, .{x.rename(.{ .hw = .q })}).splitAxis(.channels, .{ .h = nh, .hd = .auto }); + const k = zml.call(self.to_k, .forward, .{ctx}).splitAxis(.channels, .{ .h = nh, .hd = .auto }); + const v = zml.call(self.to_v, .forward, .{ctx}).splitAxis(.channels, .{ .h = nh, .hd = .auto }); + + const attn_output = zml.nn.sdpa(q, k, v, .{}); + const attn = attn_output.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .hw, .d = .channels }); + const out = zml.call(self.to_out.@"0", .forward, .{attn}); + return out; + } + }; + + const GEGLU = struct { + proj: zml.nn.Linear, + + pub fn forward(self: GEGLU, x: zml.Tensor) zml.Tensor { + const hidden, const gate = self.proj.forward(x).chunkExact(-1, 2); + return hidden.mul(gate.gelu()); + } + }; + + pub fn forward(self: BasicTransformerBlock, images: zml.Tensor, encoder_hidden_state: zml.Tensor) zml.Tensor { + var hidden = images; + { + // First self-attention + var y = self.norm1.forward(hidden); + y = self.attn1.forward(y, y.withPartialTags(.{ .k, .channels })); + hidden = hidden.add(y); + } + + { + // Then cross-attention + var y = self.norm2.forward(hidden); + y = self.attn2.forward(y, encoder_hidden_state.withTags(.{ .b, .k, .channels })); + hidden = hidden.add(y); + } + + { + var y = self.norm3.forward(hidden); + y = self.ff.net.@"0".forward(y); // GEGLU + y = self.ff.net.@"2".forward(y); // Linear + hidden = hidden.add(y); + } + return hidden; + } +}; + +pub fn loadSdTurboTokenizer(allocator: std.mem.Allocator, vocab_json_path: []const u8) !zml.tokenizer.Tokenizer { + const file = try std.fs.cwd().openFile(vocab_json_path, .{}); + defer file.close(); + + var arena_state = std.heap.ArenaAllocator.init(allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + const file_content = try file.readToEndAlloc(arena, 32 * 1024 * 1024); + + const info = try std.json.parseFromSliceLeaky(std.json.Value, arena, file_content, .{ + .duplicate_field_behavior = .use_last, + }); + + const json_vocab = switch (info) { + .object => |obj| obj, + else => return error.InvalidFormat, + }; + + const n: u32 = @intCast(json_vocab.count() + 1); + const normalizer: zml.tokenizer.Normalizer = .{ .flags = .{ + .remove_extra_whitespaces = true, + .add_dummy_prefix = false, + .add_dummy_suffix = true, + .lower_case_ascii = true, + .split_on_punct_ascii = true, + } }; + var tokenizer = try zml.tokenizer.Tokenizer.init(allocator, n, 256, normalizer, undefined, true); + const tok_alloc = tokenizer.arena_state.allocator(); + for (json_vocab.keys(), json_vocab.values()) |key, value| { + const idx: u32 = switch (value) { + .integer => |i| @intCast(i), + else => return error.InvalidFormat, + }; + // Replaced "" suffix by " " + var word = try tok_alloc.dupe(u8, key); + if (std.mem.endsWith(u8, word, "")) { + word = word[0 .. word.len - "".len + 1]; + word[word.len - 1] = ' '; + } + tokenizer.addOwnedTokenByIndex(idx, @floatFromInt(n - idx), word); + } + tokenizer.addOwnedTokenByIndex(n - 1, 0, try tok_alloc.dupe(u8, " ")); + + tokenizer.special_tokens = .{ + .bos = tokenizer.token_lookup.get("<|startoftext|>").?, + .eos = tokenizer.token_lookup.get("<|endoftext|>").?, + .unk = tokenizer.token_lookup.get("<|endoftext|>").?, + .pad = tokenizer.token_lookup.get("!").?, + }; + return tokenizer; +} diff --git a/examples/sdxl/sdxl.py b/examples/sdxl/sdxl.py new file mode 100644 index 00000000..a3d533f5 --- /dev/null +++ b/examples/sdxl/sdxl.py @@ -0,0 +1,29 @@ +import torch +import diffusers + +import zml_utils + +model_path = "stabilityai/sd-turbo" + +device = "cpu" +precision = torch.float32 +prompt = "A cinematic shot of a baby racoon wearing an intricate italian priest robe." + +pipe = diffusers.AutoPipelineForText2Image.from_pretrained(model_path, torch_dtype=precision) +pipe = zml_utils.ActivationCollector(pipe, blacklist_regexes=[r"text_encoder.*"]) +output, activations = pipe(prompt=prompt, num_inference_steps=3, guidance_scale=0.0) + +image = output.images[0] +image.save("output.png") + +filename = model_path.split("/")[-1] + ".activations.pt" + +print(f"Found {len(activations)} activations") +for k in list(activations.keys()): + if (activations[k].dtype == torch.float32): + activations[k] = activations[k].to(torch.float16).contiguous(); + +breakpoint() +print(f"Saving {len(activations)} activations to {filename}") + +torch.save(activations, filename) diff --git a/examples/sdxl/transformer.zig b/examples/sdxl/transformer.zig new file mode 100644 index 00000000..d164da17 --- /dev/null +++ b/examples/sdxl/transformer.zig @@ -0,0 +1,138 @@ +const std = @import("std"); +const testing = std.testing; + +const zml = @import("zml"); +const meta = zml.meta; + +const log = std.log.scoped(.sdxl); +const gguf = zml.io.gguf; +const Buffer = zml.Buffer; +const Tensor = zml.Tensor; +const ShapeOf = zml.ShapeOf; +const expectClose = zml.testing.expectClose; + +pub const ClipTextConfig = struct { + num_heads: u32, + max_position_embeddings: u32, + hidden_act: zml.nn.Activation = .gelu, + layer_norm_eps: f32 = 1e-5, +}; + +pub const ClipTextTransformer = struct { + embeddings: ClipTextEmbeddings, + encoder: struct { layers: []TransformerLayer }, + final_layer_norm: zml.nn.LayerNorm, + + config: ClipTextConfig = undefined, + + pub fn init(self: *ClipTextTransformer, config: ClipTextConfig) void { + self.config = config; + for (self.encoder.layers) |*layer| { + layer.self_attn.num_heads = config.num_heads; + layer.layer_norm1.eps = config.layer_norm_eps; + layer.layer_norm2.eps = config.layer_norm_eps; + layer.mlp.activation = config.hidden_act; + } + self.embeddings.max_position_embeddings = config.max_position_embeddings; + } + + /// Forward one token, using KV cache for previous tokens. + /// Returns result and updated KV cache. + pub fn forward(self: ClipTextTransformer, tokens: Tensor) Tensor { + const embeds = zml.call(self.embeddings, .forward, .{tokens}).withPartialTags(.{ .s, .d }); + var hidden = embeds; + for (self.encoder.layers) |layer| { + hidden = zml.call(layer, .forward, .{hidden}); + // TODO: tags seem to be lost by `callFunc`. + hidden = hidden.withPartialTags(.{ .s, .d }); + } + return zml.call(self.final_layer_norm, .forward, .{hidden}); + } +}; + +pub const TransformerLayer = struct { + layer_norm1: zml.nn.LayerNorm, + self_attn: SelfAttn, + layer_norm2: zml.nn.LayerNorm, + mlp: ClipMlp, + + pub fn forward( + self: TransformerLayer, + input: Tensor, + ) Tensor { + // Self Attention + const x0 = if (input.shape().isFullyTagged()) input else input.withPartialTags(.{ .b, .s, .d }); + + const x0_normalized = zml.call(self.layer_norm1, .forward, .{x0}); + const delta0 = zml.call(self.self_attn, .forward, .{x0_normalized}); + const x1 = x0.add(delta0); + + // Fully Connected + const x1_normalized = zml.call(self.layer_norm2, .forward, .{x1}); + const x2 = zml.call(self.mlp, .forward, .{x1_normalized}).add(x1); + + return x2.reuseBuffer(x0); + } +}; + +const ClipTextEmbeddings = struct { + token_embedding: zml.nn.TokenEmbedding, + position_embedding: zml.nn.TokenEmbedding, + max_position_embeddings: u32, + + pub fn forward(self: ClipTextEmbeddings, token_ids: Tensor) Tensor { + const position_ids = Tensor.arange(.{ .end = self.max_position_embeddings }, .i32).broadcastLeft(token_ids.shape()); + + return self.token_embedding.forward(token_ids).add(self.position_embedding.forward(position_ids)); + } +}; + +const ClipMlp = struct { + fc1: zml.nn.Linear, + fc2: zml.nn.Linear, + activation: zml.nn.Activation = .gelu, + + pub fn forward(self: ClipMlp, x: Tensor) Tensor { + var y = x; + y = self.fc1.forward(y); + y = self.activation.forward(y); + y = self.fc2.forward(y); + return y; + } +}; + +pub const SelfAttn = struct { + q_proj: zml.nn.Linear, + k_proj: zml.nn.Linear, + v_proj: zml.nn.Linear, + + out_proj: zml.nn.Linear, + num_heads: u32, + + /// Self Attention. + /// - If token_index is set, x is assumed to be the representation of one new token, + /// and kv_cache will be read for the previous tokens. + /// - If token_index is not set, x is assumed to be the representation of all tokens + /// since the beginning of the sequence, and kv_cache won't be read. + /// In both case, kv_cache will be updated with the computed key and value. + /// x: {.b, .s, .d } -> .{.b, .s, .d} + pub fn forward(self: SelfAttn, x: Tensor) Tensor { + // log.debug("x.shape: {}", .{x.shape()}); + const nh = self.num_heads; + var q = zml.call(self.q_proj, .forward, .{x}).splitAxis(-1, .{ .h = nh, .hd = .auto }); + var k = zml.call(self.k_proj, .forward, .{x}).splitAxis(-1, .{ .h = nh, .hd = .auto }); + var v = zml.call(self.v_proj, .forward, .{x}).splitAxis(-1, .{ .h = nh, .hd = .auto }); + // Generate the attention mask. + const seq_len = k.dim(.s); + const attn_mask = zml.nn.causalAttnMask(.{ .q = seq_len, .k = seq_len }, x.dtype(), null); + + q = q.rename(.{ .s = .q }); + k = k.rename(.{ .s = .k }); + v = v.rename(.{ .s = .k }); + + log.info("q={}, k={}, v={}", .{ q, k, v }); + const attn_output = zml.nn.sdpa(q, k, v, .{ .attn_mask = attn_mask, .allow_cudnn = false }); + const attn = attn_output.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); + return zml.call(self.out_proj, .forward, .{attn}); + } +}; diff --git a/tools/zml_utils.py b/tools/zml_utils.py index a0053e77..ae2a7638 100644 --- a/tools/zml_utils.py +++ b/tools/zml_utils.py @@ -84,7 +84,7 @@ class ActivationCollector: tensors = {} - for name, outputs, inputs in self.outs.values(): + for name, inputs, outputs in self.outs.values(): # Only save first layer for a smaller file. for blacklist in self.blacklist_regexes: if re.match(blacklist, name): @@ -93,15 +93,16 @@ class ActivationCollector: if name == "": # Skip the softmax output continue - if (outputs, inputs) == (None, None): + if (inputs, outputs) == (None, None): # print(f"no inputs/outputs for {name}") continue - for idx, inp in enumerate(inputs): - tensors[f"{name}.in.{idx}"] = inp + for prefix, inp in inputs: + tensors[f"{name}.in{prefix}"] = inp - for idx, out in enumerate(outputs): - tensors[f"{name}.out.{idx}"] = out + for prefix, out in outputs: + tensors[f"{name}.out{prefix}"] = out + print(f"Collected activations for {len(self.outs)} layers, found {len(tensors)} activations") for k, v in tensors.items(): print(k, "->", v.shape) @@ -123,13 +124,12 @@ class ActivationCollector: return assert out is not None - outs = [o.detach().cpu() for o in _flatten(out)] - inputs = [i.detach().cpu() for i in _flatten(input)] + outs = _serialize((out,) if isinstance(out, torch.Tensor) else out) kwargs = inspect.stack()[1].frame.f_locals["kwargs"] - extra_inputs = [i.detach().cpu() for i in _flatten(kwargs)] + inputs = _serialize(list(input) + list(kwargs.values())) - self.outs[id(module)] = (name, outs, inputs + extra_inputs) + self.outs[id(module)] = (name, inputs, outs) if 0 < self.max_layers < self.count: print(f"stopping collection cause we got {self.count} activations already") raise ActivationCollector.CollectionOver() @@ -150,21 +150,24 @@ def save_with_confirmation(filename: str, tensors: dict): torch.save(tensors, filename) -def _flatten(out): +def _serialize(out, prefix = "") -> list: + """Extract tensor with their names from a given python object.""" + outs = [] if out is None: - return [] + return outs elif isinstance(out, torch.Tensor): - outs = [out] - elif isinstance(out, tuple): - outs = [] - for x in out: - outs.extend(_flatten(x)) + outs = [(prefix, out.detach().cpu())] + elif isinstance(out, tuple) or isinstance(out, list): + for i, x in enumerate(out): + outs.extend(_serialize(x, f"{prefix}.{i}" )) elif isinstance(out, dict): outs = [] - for x in out.values(): - outs.extend(_flatten(x)) + for k, x in out.items(): + outs.extend(_serialize(x, f"{prefix}.{k}")) + elif isinstance(out, float) or isinstance(out, int) or isinstance(out, bool): + pass else: - outs = [] + print("unsupported output type: {}", type(out)) return outs diff --git a/zml/aio.zig b/zml/aio.zig index 81631a0b..af50f30d 100644 --- a/zml/aio.zig +++ b/zml/aio.zig @@ -363,9 +363,8 @@ fn _populateStruct( const len = buffer_store.countLayers(prefix); if (len > 0) { - obj.* = try allocator.alloc(ptr_info.child, len); - - for (obj.*, 0..) |*value, i| { + const slice = try allocator.alloc(ptr_info.child, len); + for (slice, 0..) |*value, i| { try prefix_builder.pushDigit(allocator, i); defer prefix_builder.pop(); const found = try _populateStruct(allocator, prefix_builder, unique_id, buffer_store, value, required); @@ -374,6 +373,7 @@ fn _populateStruct( return false; } } + obj.* = slice; } else if (required) { log.warn("No layer found at {s}", .{prefix}); } @@ -383,7 +383,21 @@ fn _populateStruct( return false; } }, + .Array => |arr_info| { + for (obj, 0..) |*value, i| { + try prefix_builder.pushDigit(allocator, i); + defer prefix_builder.pop(); + const found = try _populateStruct(allocator, prefix_builder, unique_id, buffer_store, value, required); + if (!found) { + std.log.err("Not able to load {s} as {s}", .{ prefix, @typeName(arr_info.child) }); + return false; + } + } + + return true; + }, .Struct => |struct_info| { + // TODO support tuple var partial_struct = false; inline for (struct_info.fields) |field| { if (field.is_comptime or @sizeOf(field.type) == 0) continue; @@ -592,6 +606,7 @@ fn visitStructAndLoadBuffer(allocator: std.mem.Allocator, prefix_builder: *Prefi buf_with_metadata._shape = obj._shape; obj.* = try zml.Buffer.from(platform, buf_with_metadata); } else { + log.warn("Buffer not found: '{s}'", .{prefix}); return error.BufferNotFound; }; } @@ -621,6 +636,13 @@ fn visitStructAndLoadBuffer(allocator: std.mem.Allocator, prefix_builder: *Prefi try visitStructAndLoadBuffer(allocator, prefix_builder, buffer_store, obj_val, platform); } }, + .Array => { + for (obj, 0..) |*value, i| { + try prefix_builder.pushDigit(allocator, i); + defer prefix_builder.pop(); + try visitStructAndLoadBuffer(allocator, prefix_builder, buffer_store, value, platform); + } + }, else => {}, } } diff --git a/zml/meta.zig b/zml/meta.zig index 826e1598..88bde557 100644 --- a/zml/meta.zig +++ b/zml/meta.zig @@ -224,6 +224,8 @@ pub fn MapType(From: type, To: type) type { From => return To, *From => return *To, ?From => return ?To, + []From => return []To, + []const From => return []const To, else => {}, } @@ -292,7 +294,7 @@ pub fn MapType(From: type, To: type) type { /// `mapAlloc` can allocate new slices to write the result if the result struct requires it. /// The caller is owning said allocations, using an `ArenaAllocator` might help tracking them. // TODO: handle tuple to slice conversion -pub fn mapAlloc(comptime cb: anytype, allocator: std.mem.Allocator, ctx: FnParam(cb, 0), from: anytype, to: anytype) !void { +pub fn mapAlloc(comptime cb: anytype, allocator: std.mem.Allocator, ctx: FnParam(cb, 0), from: anytype, to: anytype) error{OutOfMemory}!void { // const Ctx = FnParam(cb, 0); const From = FnParam(cb, 1); const FromStruct = @TypeOf(from); diff --git a/zml/module.zig b/zml/module.zig index fb5e5142..bb83df00 100644 --- a/zml/module.zig +++ b/zml/module.zig @@ -540,7 +540,7 @@ pub const CompilationContext = struct { /// Create tensor from the given shapes. /// Each created tensor will receive a unique id, local to this CompilationContext. - pub fn tensorFromShapes(self: *CompilationContext, ArgsT: type, allocator: std.mem.Allocator, args_shapes: anytype) ArgsT { + pub fn tensorFromShapes(self: *CompilationContext, ArgsT: type, allocator: std.mem.Allocator, args_shapes: anytype) !ArgsT { const Local = struct { fn tensorFromShape(arg_id: *u64, shape: Shape) Tensor { defer arg_id.* += 1; @@ -899,7 +899,7 @@ fn compileInternal( const arena = arena_state.allocator(); var timer = std.time.Timer.start() catch null; - const tensor_args = context.tensorFromShapes(ModuleSignature(func).ArgsT, arena, args); + const tensor_args = try context.tensorFromShapes(ModuleSignature(func).ArgsT, arena, args); // Run in a dedicated thread because compilation relies on `threadlocal`. const f = try asynk.callGeneric(CompilationContext.generateBytecode, .{ context, arena, "main", func, &model, &tensor_args }); context._module.getBody().appendOperation(f.mlir_fn); diff --git a/zml/nn.zig b/zml/nn.zig index 8ba6903a..ae0dd821 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -84,52 +84,95 @@ pub fn chainModules(module_list: anytype, input: Tensor) Tensor { } /// Layer Normalization +/// https://paperswithcode.com/method/layer-normalization pub const LayerNorm = struct { weight: Tensor, bias: ?Tensor = null, eps: f32 = 1e-5, pub fn forward(self: LayerNorm, x: Tensor) Tensor { - const normed = normalizeVariance(x, self.eps); - const ax = x.axis(-1); - var out = normed.mul(self.weight.broadcast(x.shape(), &.{ax})); - if (self.bias) |bias| out = out.add(bias.broadcast(x.shape(), &.{ax})); - - return out; + return layerNorm(x, -1, self); } }; +/// Layer Normalization +/// https://paperswithcode.com/method/layer-normalization +pub fn layerNorm(input: Tensor, normalization_axis: anytype, opts: LayerNorm) Tensor { + const ax = input.axis(normalization_axis); + const normed = normalizeVariance(input, ax, opts.eps); + var out = normed.mul(opts.weight.broadcast(input.shape(), &.{ax})); + if (opts.bias) |bias| out = out.add(bias.broadcast(input.shape(), &.{ax})); + + return out; +} + +pub const GroupNorm = struct { + weight: zml.Tensor, + bias: zml.Tensor, + + group_size: u32, + eps: f32, + + pub fn forward(self: GroupNorm, grouping_axis: anytype, normalization_axis: anytype, input: zml.Tensor) zml.Tensor { + return groupNorm(input, grouping_axis, normalization_axis, self); + } +}; + +/// Group normalization +/// https://paperswithcode.com/method/group-normalization +/// ![Group normalization visualization](https://production-media.paperswithcode.com/methods/Screen_Shot_2020-05-23_at_11.26.56_PM_BQOdMKA.png) +pub fn groupNorm(input: zml.Tensor, grouping_axis: anytype, normalization_axis: anytype, opts: GroupNorm) zml.Tensor { + const group_ax = input.axis(grouping_axis); + + var grouped = input.merge(.{ ._group_ = .{ group_ax, normalization_axis } }); + grouped = grouped.splitAxis(._group_, .{ ._group_ = opts.group_size, ._group_items_ = .auto }); + + const normed = normalizeVariance(grouped, ._group_items_, opts.eps).reshape(input.shape()); + + var out = normed.mul(opts.weight.broadcast(input.shape(), &.{group_ax})); + out = out.add(opts.bias.broadcast(input.shape(), &.{group_ax})); + + return out; +} + /// Center and scale by the variance. -/// normalize(x, eps) = (x - mean(x)) / sqrt(var(x) + eps) +/// normalize(x, axis, eps) = (x - mean(x, axis)) / sqrt(var(x, axis) + eps) /// Work on the last axis. -pub fn normalizeVariance(x: Tensor, eps: f32) Tensor { - const N: f32 = @floatFromInt(x.dim(-1)); +pub fn normalizeVariance(x: Tensor, axis_: anytype, eps: f32) Tensor { + const a = x.axis(axis_); + const N: f32 = @floatFromInt(x.dim(a)); // Upcast to improve precision const xf32 = x.convert(.f32); - const mean = xf32.sum(-1).scale(1.0 / N); + const mean = xf32.sum(a).scale(1.0 / N); const mean_dev = xf32.sub(mean.broadcastRight(xf32.shape())); - const variance = mean_dev.mul(mean_dev).sum(-1).scale(1.0 / N); + const variance = mean_dev.mul(mean_dev).sum(a).scale(1.0 / N); const rsqrt = Tensor.rsqrt(variance.addConstant(eps)); return mean_dev.mul(rsqrt.broadcastRight(mean_dev.shape())).convert(x.dtype()); } -// ref: https://pytorch.org/docs/stable/generated/torch.nn.functional.normalize.html -// Implementation equivalent to `nn.functional.normalize(tensor, dim=-1)` call -pub fn normalizeL2(input: Tensor, eps: f32) Tensor { - const inv_norm = input.pow(Tensor.scalar(2, input.dtype())).sum(-1).addConstant(eps).rsqrt(); - return input.mul(inv_norm.broad(input.shape())); +/// L² normalization of each individual vector extracted along the given axis. +/// normalize(X, .b, ϵ)[a, b, c] = X(a, b, c) / max(‖X(a, :, c)‖², ϵ) +pub fn normalizeL2(input: Tensor, axis_: anytype, eps: f32) Tensor { + const norm = input.norm(axis_, 2); + // Replace norms that are too close to 0 by ϵ. + const max_norm_eps = norm.maximum(Tensor.scalar(eps, input.dtype())); + return input.div(max_norm_eps.broad(input.shape())); } test normalizeL2 { const platform = zml.testing.env(); const input = try zml.Buffer.fromSlice(platform, .{ 2, 2 }, &[_]f32{ -0.9686, -1.0058, -1.7808, 0.6698 }); - - const res = try zml.testing.compileAndCall(platform, zml.nn.normalizeL2, .{ input, 1e-12 }); - const expectation = zml.HostBuffer.fromSlice(.{ 2, 2 }, &[_]f32{ -0.6937, -0.7203, -0.9360, 0.3520 }); - try zml.testing.expectClose(expectation, res, 1e-4); + const Local = struct { + fn _normalizeL2(x: Tensor) Tensor { + return normalizeL2(x, -1, 1e-12); + } + }; + const actual = try zml.testing.compileAndCall(platform, Local._normalizeL2, .{input}); + const expected = [2][2]f32{ .{ -0.6937, -0.7203 }, .{ -0.9360, 0.3520 } }; + try zml.testing.expectClose(zml.HostBuffer.fromArray(&expected), actual, 1e-4); } pub const RopeOpts = struct { diff --git a/zml/ops.zig b/zml/ops.zig index 35765cec..737e34b7 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -332,7 +332,7 @@ pub fn for_(comptime func: anytype, blk_ctx: BlockSign(func).BlkCtx, num_steps_: const for_blk: ForBlk = .{ .blk_ctx = blk_ctx, .step_tag = step_tag, .num_steps = num_steps }; var result_buffers: @TypeOf(first_step) = undefined; - try meta.mapAlloc(ForBlk.prep, allocator, for_blk, first_step, &result_buffers); + meta.mapAlloc(ForBlk.prep, allocator, for_blk, first_step, &result_buffers) catch @panic("Out of memory"); return while_( ForBlk.done, diff --git a/zml/tensor.zig b/zml/tensor.zig index a8d71e8d..7dea0ee9 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -1292,6 +1292,13 @@ pub const Tensor = struct { return self.sum(axis_).divByConst(self.dim(axis_)); } + /// Returns a Tensor containing the norm of each vector along the given axis. + pub fn norm(self: Tensor, axis_: anytype, power: f32) Tensor { + const a = self.axis(axis_); + const norm_to_the_power = self.pow(Tensor.scalar(power, self.dtype())).sum(a); + return norm_to_the_power.pow(Tensor.scalar(1.0 / power, self.dtype())); + } + /// Returns a transposed Tensor computed using the given axes. pub fn transpose(self: Tensor, axes_: anytype) Tensor { const axes__ = self.axes(axes_).constSlice(); diff --git a/zml/testing.zig b/zml/testing.zig index 5a3a3bc9..89410e5d 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -69,6 +69,12 @@ pub fn expectClose(left_: anytype, right_: anytype, tolerance: f32) !void { log.err("left.dtype ({}) != right.dtype ({})", .{ left.dtype(), right.dtype() }); return error.TestUnexpectedResult; } + if (!left.isContiguous()) { + log.err("Left input isn't contiguous: {} ({d})", .{ left, left.strides().? }); + } + if (!right.isContiguous()) { + log.err("Right input isn't contiguous: {} ({d})", .{ right, right.strides().? }); + } switch (left.dtype()) { inline .f16, .f32, .f64 => |t| { const L = t.toZigType(); @@ -190,40 +196,9 @@ pub fn testLayerOut( } const mod = try exe.prepare(alloc, layer_weights); - const FetchCtx = struct { - store: zml.aio.BufferStore, - index: u32, - prefix: std.ArrayListUnmanaged(u8), - platform: zml.Platform, - - fn fetch(ctx: *@This(), x: zml.Tensor) zml.Buffer { - _ = x; - defer ctx.index += 1; - var full_prefix = ctx.*.prefix; - _ = full_prefix.writer(undefined).print("{d}", .{ctx.index}) catch unreachable; - log.info("prefix: {s}", .{full_prefix.items}); - const host = ctx.store.get(full_prefix.items) orelse { - log.err("Didn't find test input: {s}", .{full_prefix.items}); - @panic("Missing test input"); - }; - return host.toDevice(ctx.platform) catch unreachable; - } - }; - - // Note: zml.populateModelWithPrefix isn't enough, - // because it assumes we have the same structure in the activation file - // than in the function signature. - // But for sake of decoupling the reference implementation - // and ZML code that's not always the case. - { - var input_buffers: zml.Bufferized(FwdSign.ArgsT) = undefined; - var fetch_ctx: FetchCtx = .{ .store = activations, .index = 0, .prefix = .{}, .platform = platform }; - try fetch_ctx.prefix.ensureTotalCapacity(alloc, name.len + 32); - fetch_ctx.prefix.appendSliceAssumeCapacity(name ++ ".in."); - try zml.meta.mapAlloc(FetchCtx.fetch, alloc, &fetch_ctx, input_tensors, &input_buffers); - defer zml.aio.unloadBuffers(&input_buffers); - _ = mod.call(input_buffers); - } + var input_buffers = try zml.aio.loadModelBuffersWithPrefix(FwdSign.ArgsT, input_tensors, activations, alloc, platform, name ++ ".in"); + defer zml.aio.unloadBuffers(&input_buffers); + _ = mod.call(input_buffers); var buf: [1024]u8 = undefined; for (0..mod.inner.result_buffer_count) |i| { diff --git a/zml/tokenizer.zig b/zml/tokenizer.zig index 2e211eac..cc36616b 100644 --- a/zml/tokenizer.zig +++ b/zml/tokenizer.zig @@ -15,6 +15,12 @@ test { } /// Byte Pair Encoding tokenizer generally used for LLM. +/// The algorithm is as follow: +/// * Normalize the input string (eg lower-case) +/// * Split the normalized string into unicode codepoint, +/// retrieve the token id for each codepoint. +/// * For each token, look at its neighbor and try to merge the pair if possible. +/// * Repeat until no token can be merged with its neighbor. pub const Tokenizer = struct { tokens: [][]const u8, token_lookup: std.StringHashMapUnmanaged(u32),