diff --git a/BUILD.bazel b/BUILD.bazel index 5dbfe326..052f36a1 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -6,6 +6,7 @@ zls_completion( deps = [ "//examples/llama", "//examples/mnist", + "//jinja2", "//stdx", "//zml", ], diff --git a/MODULE.bazel b/MODULE.bazel index b0d54b40..8e917959 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -81,6 +81,7 @@ pip.parse( hub_name = "neuron_py_deps", python_version = "3.12", requirements_lock = "//platforms/neuron:requirements.lock.txt", + target_platforms = ["linux_x86_64"], ) use_repo(pip, "neuron_py_deps") @@ -93,6 +94,14 @@ pip.parse( ) use_repo(pip, "huggingface_hub") +pip.parse( + download_only = True, + hub_name = "jinja2_py_deps", + python_version = "3.12", + requirements_lock = "//jinja2:requirements.lock.txt", +) +use_repo(pip, "jinja2_py_deps") + cpu = use_extension("//platforms/cpu:cpu.bzl", "cpu_pjrt_plugin") use_repo(cpu, "libpjrt_cpu_darwin_amd64", "libpjrt_cpu_darwin_arm64", "libpjrt_cpu_linux_amd64") diff --git a/jinja2/BUILD.bazel b/jinja2/BUILD.bazel new file mode 100644 index 00000000..e7b94a2e --- /dev/null +++ b/jinja2/BUILD.bazel @@ -0,0 +1,58 @@ +load("@jinja2_py_deps//:requirements.bzl", "requirement") +load("@rules_python//python:defs.bzl", "py_binary") +load("@rules_python//python/uv:lock.bzl", uv_lock = "lock") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") +load(":generate_expected.bzl", "generate_expected_directory") + +FIXTURE_INPUTS = glob([ + "test_cases/basic/*.json", + "test_cases/chat/data/*.json", + "test_cases/chat/templates/*.jinja", + "test_cases/multimodal/data/*.json", + "test_cases/multimodal/templates/*.jinja", +]) + +zig_library( + name = "jinja2", + srcs = [ + "environment.zig", + "expr.zig", + "lib.zig", + "template.zig", + "value.zig", + ], + main = "lib.zig", + visibility = ["//visibility:public"], +) + +py_binary( + name = "generate_expected_py", + srcs = ["test_cases/generate_expected.py"], + main = "test_cases/generate_expected.py", + deps = [ + requirement("jinja2"), + requirement("transformers"), + ], +) + +generate_expected_directory( + name = "generate_expected", + srcs = FIXTURE_INPUTS, + input_root = "jinja2/test_cases", + tool = ":generate_expected_py", +) + +uv_lock( + name = "requirements", + srcs = ["requirements.in"], + out = "requirements.lock.txt", + generate_hashes = False, + args = ["--upgrade"], + tags = ["manual"], +) + +zig_test( + name = "test", + deps = [":jinja2"], + data = FIXTURE_INPUTS + [":generate_expected"], +) diff --git a/jinja2/README.md b/jinja2/README.md new file mode 100644 index 00000000..d406a28c --- /dev/null +++ b/jinja2/README.md @@ -0,0 +1,35 @@ +# Jinja2 in Zig + +```zig +const std = @import("std"); +const mj = @import("src/lib.zig"); + +pub fn main() !void { + const allocator = std.heap.smp_allocator; + + var env = mj.Environment.initMaxPerf(); + defer env.deinit(); + + try env.addTemplate("greet", "Hello {{ user.name }}! score={{ user.score }}"); + + const Ctx = struct { + user: struct { + name: []const u8, + score: i32, + }, + }; + + const ctx = Ctx{ + .user = .{ + .name = "Ada", + .score = 42, + }, + }; + + // Zero-alloc struct context adapter (nested structs supported) + const out = try env.renderTemplateStruct(allocator, "greet", ctx); + defer allocator.free(out); + + std.debug.print("{s}\n", .{out}); +} +``` diff --git a/jinja2/environment.zig b/jinja2/environment.zig new file mode 100644 index 00000000..fe5e3e2b --- /dev/null +++ b/jinja2/environment.zig @@ -0,0 +1,59 @@ +const std = @import("std"); +const template_mod = @import("template.zig"); +const value_mod = @import("value.zig"); +const Value = value_mod.Value; + +pub const Environment = struct { + allocator: std.mem.Allocator, + templates: std.StringHashMap(template_mod.Template), + + pub fn init(allocator: std.mem.Allocator) Environment { + return .{ + .allocator = allocator, + .templates = std.StringHashMap(template_mod.Template).init(allocator), + }; + } + + pub fn initMaxPerf() Environment { + return init(std.heap.smp_allocator); + } + + pub fn deinit(self: *Environment) void { + self.templates.deinit(); + } + + pub fn addTemplate(self: *Environment, name: []const u8, source: []const u8) !void { + const name_copy = try self.allocator.dupe(u8, name); + const src_copy = try self.allocator.dupe(u8, source); + const tmpl = try template_mod.Template.parse(self.allocator, name_copy, src_copy); + try self.templates.put(name_copy, tmpl); + } + + pub fn getTemplate(self: *Environment, name: []const u8) ?*const template_mod.Template { + return self.templates.getPtr(name); + } + + pub fn renderTemplate(self: *Environment, render_allocator: std.mem.Allocator, name: []const u8, ctx: []const Value.Entry) ![]u8 { + const tmpl = self.getTemplate(name) orelse return error.TemplateNotFound; + return tmpl.render(render_allocator, ctx); + } + + pub fn renderTemplateStruct(self: *Environment, render_allocator: std.mem.Allocator, name: []const u8, ctx: anytype) ![]u8 { + const CtxT = @TypeOf(ctx); + const T = switch (@typeInfo(CtxT)) { + .pointer => |p| p.child, + else => CtxT, + }; + var adapted: value_mod.StructContext(T) = .{}; + const entries = switch (@typeInfo(CtxT)) { + .pointer => adapted.from(ctx), + else => adapted.from(&ctx), + }; + return self.renderTemplate(render_allocator, name, entries); + } + + pub fn renderNamedStr(self: *Environment, render_allocator: std.mem.Allocator, name: []const u8, source: []const u8, ctx: []const Value.Entry) ![]u8 { + const tmpl = try template_mod.Template.parse(self.allocator, name, source); + return tmpl.render(render_allocator, ctx); + } +}; diff --git a/jinja2/expr.zig b/jinja2/expr.zig new file mode 100644 index 00000000..ad755154 --- /dev/null +++ b/jinja2/expr.zig @@ -0,0 +1,1414 @@ +const std = @import("std"); +const value_mod = @import("value.zig"); +const Value = value_mod.Value; + +pub const ExprError = error{ + UnexpectedToken, + UnterminatedString, + InvalidNumber, + OutOfMemory, + TypeError, + RaisedException, + NoSpaceLeft, +}; + +const TokenTag = enum { + ident, + int, + float, + string, + bool_true, + bool_false, + none_kw, + and_kw, + or_kw, + not_kw, + is_kw, + if_kw, + else_kw, + in_kw, + plus, + minus, + star, + slash, + percent, + lparen, + rparen, + lbracket, + rbracket, + comma, + colon, + dot, + pipe, + tilde, + assign, + eqeq, + noteq, + lt, + lte, + gt, + gte, + eof, +}; + +const Token = struct { + tag: TokenTag, + lexeme: []const u8, +}; + +const LexState = enum { start, ident, number, string, string_escape }; + +fn isAlpha(c: u8) bool { + return (c >= 'a' and c <= 'z') or (c >= 'A' and c <= 'Z') or c == '_'; +} + +fn isDigit(c: u8) bool { + return c >= '0' and c <= '9'; +} + +fn isAlphaNum(c: u8) bool { + return isAlpha(c) or isDigit(c); +} + +pub fn lex(allocator: std.mem.Allocator, input: []const u8) ExprError![]Token { + var tokens = std.ArrayList(Token).empty; + errdefer tokens.deinit(allocator); + + var i: usize = 0; + var state: LexState = .start; + var start: usize = 0; + var quote_char: u8 = 0; + + scan: while (true) { + switch (state) { + .start => { + if (i >= input.len) { + try tokens.append(allocator, .{ .tag = .eof, .lexeme = "" }); + break :scan; + } + const c = input[i]; + if (c == ' ' or c == '\t' or c == '\n' or c == '\r') { + i += 1; + continue :scan; + } + if (isAlpha(c)) { + start = i; + i += 1; + state = .ident; + continue :scan; + } + if (isDigit(c)) { + start = i; + i += 1; + state = .number; + continue :scan; + } + + switch (c) { + '\'', '"' => { + quote_char = c; + start = i + 1; + i += 1; + state = .string; + }, + '+' => { + try tokens.append(allocator, .{ .tag = .plus, .lexeme = "+" }); + i += 1; + }, + '-' => { + try tokens.append(allocator, .{ .tag = .minus, .lexeme = "-" }); + i += 1; + }, + '*' => { + try tokens.append(allocator, .{ .tag = .star, .lexeme = "*" }); + i += 1; + }, + '/' => { + try tokens.append(allocator, .{ .tag = .slash, .lexeme = "/" }); + i += 1; + }, + '%' => { + try tokens.append(allocator, .{ .tag = .percent, .lexeme = "%" }); + i += 1; + }, + '(' => { + try tokens.append(allocator, .{ .tag = .lparen, .lexeme = "(" }); + i += 1; + }, + ')' => { + try tokens.append(allocator, .{ .tag = .rparen, .lexeme = ")" }); + i += 1; + }, + '[' => { + try tokens.append(allocator, .{ .tag = .lbracket, .lexeme = "[" }); + i += 1; + }, + ']' => { + try tokens.append(allocator, .{ .tag = .rbracket, .lexeme = "]" }); + i += 1; + }, + ',' => { + try tokens.append(allocator, .{ .tag = .comma, .lexeme = "," }); + i += 1; + }, + ':' => { + try tokens.append(allocator, .{ .tag = .colon, .lexeme = ":" }); + i += 1; + }, + '.' => { + try tokens.append(allocator, .{ .tag = .dot, .lexeme = "." }); + i += 1; + }, + '|' => { + try tokens.append(allocator, .{ .tag = .pipe, .lexeme = "|" }); + i += 1; + }, + '~' => { + try tokens.append(allocator, .{ .tag = .tilde, .lexeme = "~" }); + i += 1; + }, + '=' => { + if (i + 1 < input.len and input[i + 1] == '=') { + try tokens.append(allocator, .{ .tag = .eqeq, .lexeme = "==" }); + i += 2; + } else { + try tokens.append(allocator, .{ .tag = .assign, .lexeme = "=" }); + i += 1; + } + }, + '!' => { + if (i + 1 < input.len and input[i + 1] == '=') { + try tokens.append(allocator, .{ .tag = .noteq, .lexeme = "!=" }); + i += 2; + } else return ExprError.UnexpectedToken; + }, + '<' => { + if (i + 1 < input.len and input[i + 1] == '=') { + try tokens.append(allocator, .{ .tag = .lte, .lexeme = "<=" }); + i += 2; + } else { + try tokens.append(allocator, .{ .tag = .lt, .lexeme = "<" }); + i += 1; + } + }, + '>' => { + if (i + 1 < input.len and input[i + 1] == '=') { + try tokens.append(allocator, .{ .tag = .gte, .lexeme = ">=" }); + i += 2; + } else { + try tokens.append(allocator, .{ .tag = .gt, .lexeme = ">" }); + i += 1; + } + }, + else => return ExprError.UnexpectedToken, + } + }, + .ident => { + while (i < input.len and isAlphaNum(input[i])) : (i += 1) {} + const lexeme = input[start..i]; + const tag: TokenTag = blk: { + if (std.mem.eql(u8, lexeme, "true")) break :blk .bool_true; + if (std.mem.eql(u8, lexeme, "false")) break :blk .bool_false; + if (std.mem.eql(u8, lexeme, "none") or std.mem.eql(u8, lexeme, "null")) break :blk .none_kw; + if (std.mem.eql(u8, lexeme, "and")) break :blk .and_kw; + if (std.mem.eql(u8, lexeme, "or")) break :blk .or_kw; + if (std.mem.eql(u8, lexeme, "not")) break :blk .not_kw; + if (std.mem.eql(u8, lexeme, "is")) break :blk .is_kw; + if (std.mem.eql(u8, lexeme, "if")) break :blk .if_kw; + if (std.mem.eql(u8, lexeme, "else")) break :blk .else_kw; + if (std.mem.eql(u8, lexeme, "in")) break :blk .in_kw; + break :blk .ident; + }; + try tokens.append(allocator, .{ .tag = tag, .lexeme = lexeme }); + state = .start; + }, + .number => { + var is_float = false; + while (i < input.len and isDigit(input[i])) : (i += 1) {} + if (i < input.len and input[i] == '.') { + if (i + 1 < input.len and isDigit(input[i + 1])) { + is_float = true; + i += 1; + while (i < input.len and isDigit(input[i])) : (i += 1) {} + } + } + try tokens.append(allocator, .{ .tag = if (is_float) .float else .int, .lexeme = input[start..i] }); + state = .start; + }, + .string => { + if (i >= input.len) return ExprError.UnterminatedString; + const c = input[i]; + if (c == '\\') { + i += 1; + state = .string_escape; + continue :scan; + } + if (c == quote_char) { + try tokens.append(allocator, .{ .tag = .string, .lexeme = input[start..i] }); + i += 1; + state = .start; + continue :scan; + } + i += 1; + }, + .string_escape => { + if (i >= input.len) return ExprError.UnterminatedString; + i += 1; + state = .string; + }, + } + } + + return tokens.toOwnedSlice(allocator); +} + +pub const BinaryOp = enum { + add, + sub, + mul, + div, + mod, + eq, + neq, + lt, + lte, + gt, + gte, + in, + not_in, + concat, + @"and", + @"or", +}; + +pub const CallArg = union(enum) { + positional: *Expr, + keyword: struct { name: []const u8, value: *Expr }, +}; + +const StringMethodId = enum { + unknown, + startswith, + endswith, + split, + strip, + rstrip, + lstrip, +}; + +const FilterId = enum { + unknown, + safe, + length, + string, + tojson, + trim, + items, + sort, +}; + +const TestId = enum { + unknown, + defined, + none, + string, + iterable, + mapping, + sequence, + undefined, +}; + +fn detectStringMethodId(name: []const u8) StringMethodId { + if (name.len == 0) return .unknown; + return switch (name[0]) { + 's' => if (std.mem.eql(u8, name, "startswith")) .startswith else if (std.mem.eql(u8, name, "split")) .split else if (std.mem.eql(u8, name, "strip")) .strip else .unknown, + 'e' => if (std.mem.eql(u8, name, "endswith")) .endswith else .unknown, + 'r' => if (std.mem.eql(u8, name, "rstrip")) .rstrip else .unknown, + 'l' => if (std.mem.eql(u8, name, "lstrip")) .lstrip else .unknown, + else => .unknown, + }; +} + +fn detectFilterId(name: []const u8) FilterId { + if (name.len == 0) return .unknown; + return switch (name[0]) { + 's' => if (std.mem.eql(u8, name, "safe")) .safe else if (std.mem.eql(u8, name, "string")) .string else if (std.mem.eql(u8, name, "sort")) .sort else .unknown, + 'l' => if (std.mem.eql(u8, name, "length")) .length else .unknown, + 't' => if (std.mem.eql(u8, name, "tojson")) .tojson else if (std.mem.eql(u8, name, "trim")) .trim else .unknown, + 'i' => if (std.mem.eql(u8, name, "items")) .items else .unknown, + else => .unknown, + }; +} + +fn detectTestId(name: []const u8) TestId { + if (name.len == 0) return .unknown; + return switch (name[0]) { + 'd' => if (std.mem.eql(u8, name, "defined")) .defined else .unknown, + 'n' => if (std.mem.eql(u8, name, "none")) .none else .unknown, + 's' => if (std.mem.eql(u8, name, "string")) .string else if (std.mem.eql(u8, name, "sequence")) .sequence else .unknown, + 'i' => if (std.mem.eql(u8, name, "iterable")) .iterable else .unknown, + 'm' => if (std.mem.eql(u8, name, "mapping")) .mapping else .unknown, + 'u' => if (std.mem.eql(u8, name, "undefined")) .undefined else .unknown, + else => .unknown, + }; +} + +pub const Expr = union(enum) { + ident: struct { name: []const u8, slot: u32 }, + int_lit: i64, + float_lit: f64, + str_lit: []const u8, + bool_lit: bool, + null_lit, + list_lit: []const *Expr, + getattr: struct { target: *Expr, attr: []const u8, method_id: StringMethodId }, + index: struct { target: *Expr, key: *Expr }, + slice: struct { target: *Expr, start: ?*Expr, end: ?*Expr, step: ?*Expr }, + call: struct { callee: *Expr, args: []const CallArg }, + filter: struct { target: *Expr, name: []const u8, filter_id: FilterId, args: []const CallArg }, + test_expr: struct { target: *Expr, name: []const u8, test_id: TestId, negated: bool, args: []const CallArg }, + cond_expr: struct { then_expr: *Expr, cond: *Expr, else_expr: *Expr }, + unary_not: *Expr, + unary_neg: *Expr, + binary: struct { op: BinaryOp, left: *Expr, right: *Expr }, +}; + +pub const unknown_slot: u32 = std.math.maxInt(u32); + +const Parser = struct { + allocator: std.mem.Allocator, + tokens: []const Token, + pos: usize, + + fn peek(self: *const Parser) Token { + return self.tokens[self.pos]; + } + + fn peekN(self: *const Parser, n: usize) Token { + const idx = @min(self.pos + n, self.tokens.len - 1); + return self.tokens[idx]; + } + + fn advance(self: *Parser) Token { + const t = self.tokens[self.pos]; + self.pos += 1; + return t; + } + + fn eat(self: *Parser, tag: TokenTag) bool { + if (self.peek().tag == tag) { + _ = self.advance(); + return true; + } + return false; + } + + fn expect(self: *Parser, tag: TokenTag) ExprError!Token { + if (self.peek().tag != tag) return ExprError.UnexpectedToken; + return self.advance(); + } + + fn allocExpr(self: *Parser, value: Expr) ExprError!*Expr { + const p = try self.allocator.create(Expr); + p.* = value; + return p; + } + + fn unescapeString(self: *Parser, raw: []const u8) ExprError![]const u8 { + if (std.mem.indexOfScalar(u8, raw, '\\') == null) return raw; + + var out = std.ArrayList(u8).empty; + errdefer out.deinit(self.allocator); + + var i: usize = 0; + while (i < raw.len) : (i += 1) { + const c = raw[i]; + if (c != '\\' or i + 1 >= raw.len) { + try out.append(self.allocator, c); + continue; + } + + i += 1; + const esc = raw[i]; + const decoded: u8 = switch (esc) { + 'n' => '\n', + 'r' => '\r', + 't' => '\t', + '\\' => '\\', + '"' => '"', + '\'' => '\'', + else => esc, + }; + try out.append(self.allocator, decoded); + } + + return out.toOwnedSlice(self.allocator); + } + + fn parse(self: *Parser) ExprError!*Expr { + const rv = try self.parseConditional(); + _ = try self.expect(.eof); + return rv; + } + + fn parseConditional(self: *Parser) ExprError!*Expr { + var then_expr = try self.parseOr(); + if (self.eat(.if_kw)) { + const cond = try self.parseOr(); + _ = try self.expect(.else_kw); + const else_expr = try self.parseConditional(); + then_expr = try self.allocExpr(.{ .cond_expr = .{ .then_expr = then_expr, .cond = cond, .else_expr = else_expr } }); + } + return then_expr; + } + + fn parseOr(self: *Parser) ExprError!*Expr { + var left = try self.parseAnd(); + while (self.eat(.or_kw)) { + const right = try self.parseAnd(); + left = try self.allocExpr(.{ .binary = .{ .op = .@"or", .left = left, .right = right } }); + } + return left; + } + + fn parseAnd(self: *Parser) ExprError!*Expr { + var left = try self.parseCompare(); + while (self.eat(.and_kw)) { + const right = try self.parseCompare(); + left = try self.allocExpr(.{ .binary = .{ .op = .@"and", .left = left, .right = right } }); + } + return left; + } + + fn parseCompare(self: *Parser) ExprError!*Expr { + var left = try self.parseAdd(); + while (true) { + if (self.eat(.is_kw)) { + const negated = self.eat(.not_kw); + const name_tok = self.peek(); + if (!isNameToken(name_tok.tag)) return ExprError.UnexpectedToken; + _ = self.advance(); + var args: []const CallArg = &[_]CallArg{}; + if (self.eat(.lparen)) { + args = try self.parseArgs(); + } + left = try self.allocExpr(.{ .test_expr = .{ + .target = left, + .name = name_tok.lexeme, + .test_id = detectTestId(name_tok.lexeme), + .negated = negated, + .args = args, + } }); + continue; + } + + var maybe_op: ?BinaryOp = null; + if (self.eat(.eqeq)) { + maybe_op = .eq; + } else if (self.eat(.noteq)) { + maybe_op = .neq; + } else if (self.eat(.lt)) { + maybe_op = .lt; + } else if (self.eat(.lte)) { + maybe_op = .lte; + } else if (self.eat(.gt)) { + maybe_op = .gt; + } else if (self.eat(.gte)) { + maybe_op = .gte; + } else if (self.eat(.in_kw)) { + maybe_op = .in; + } else if (self.eat(.not_kw)) { + if (self.eat(.in_kw)) { + maybe_op = .not_in; + } else return ExprError.UnexpectedToken; + } + const op = maybe_op orelse break; + + const right = try self.parseAdd(); + left = try self.allocExpr(.{ .binary = .{ .op = op, .left = left, .right = right } }); + } + return left; + } + + fn isNameToken(tag: TokenTag) bool { + return switch (tag) { + .ident, .none_kw, .bool_true, .bool_false, .and_kw, .or_kw, .not_kw, .is_kw, .if_kw, .else_kw, .in_kw => true, + else => false, + }; + } + + fn parseAdd(self: *Parser) ExprError!*Expr { + var left = try self.parseMul(); + while (true) { + const op: BinaryOp = switch (self.peek().tag) { + .plus => .add, + .minus => .sub, + .tilde => .concat, + else => break, + }; + _ = self.advance(); + const right = try self.parseMul(); + left = try self.allocExpr(.{ .binary = .{ .op = op, .left = left, .right = right } }); + } + return left; + } + + fn parseMul(self: *Parser) ExprError!*Expr { + var left = try self.parseUnary(); + while (true) { + const op: BinaryOp = switch (self.peek().tag) { + .star => .mul, + .slash => .div, + .percent => .mod, + else => break, + }; + _ = self.advance(); + const right = try self.parseUnary(); + left = try self.allocExpr(.{ .binary = .{ .op = op, .left = left, .right = right } }); + } + return left; + } + + fn parseUnary(self: *Parser) ExprError!*Expr { + if (self.eat(.not_kw)) { + const inner = try self.parseUnary(); + return self.allocExpr(.{ .unary_not = inner }); + } + if (self.eat(.minus)) { + const inner = try self.parseUnary(); + return self.allocExpr(.{ .unary_neg = inner }); + } + return self.parsePostfix(); + } + + fn parsePostfix(self: *Parser) ExprError!*Expr { + var base = try self.parsePrimary(); + while (true) { + if (self.eat(.dot)) { + const ident = try self.expect(.ident); + base = try self.allocExpr(.{ .getattr = .{ + .target = base, + .attr = ident.lexeme, + .method_id = detectStringMethodId(ident.lexeme), + } }); + continue; + } + if (self.eat(.lbracket)) { + if (self.eat(.colon)) { + var end_expr: ?*Expr = null; + var step_expr: ?*Expr = null; + if (self.peek().tag != .rbracket and self.peek().tag != .colon) end_expr = try self.parseConditional(); + if (self.eat(.colon)) { + if (self.peek().tag != .rbracket) step_expr = try self.parseConditional(); + } + _ = try self.expect(.rbracket); + base = try self.allocExpr(.{ .slice = .{ .target = base, .start = null, .end = end_expr, .step = step_expr } }); + continue; + } + + const first = try self.parseConditional(); + if (self.eat(.colon)) { + var end_expr: ?*Expr = null; + var step_expr: ?*Expr = null; + if (self.peek().tag != .rbracket and self.peek().tag != .colon) end_expr = try self.parseConditional(); + if (self.eat(.colon)) { + if (self.peek().tag != .rbracket) step_expr = try self.parseConditional(); + } + _ = try self.expect(.rbracket); + base = try self.allocExpr(.{ .slice = .{ .target = base, .start = first, .end = end_expr, .step = step_expr } }); + } else { + _ = try self.expect(.rbracket); + base = try self.allocExpr(.{ .index = .{ .target = base, .key = first } }); + } + continue; + } + if (self.eat(.lparen)) { + const args = try self.parseArgs(); + base = try self.allocExpr(.{ .call = .{ .callee = base, .args = args } }); + continue; + } + if (self.eat(.pipe)) { + const name = try self.expect(.ident); + var args: []const CallArg = &[_]CallArg{}; + if (self.eat(.lparen)) args = try self.parseArgs(); + base = try self.allocExpr(.{ .filter = .{ + .target = base, + .name = name.lexeme, + .filter_id = detectFilterId(name.lexeme), + .args = args, + } }); + continue; + } + break; + } + return base; + } + + fn parseArgs(self: *Parser) ExprError![]const CallArg { + var args = std.ArrayList(CallArg).empty; + errdefer args.deinit(self.allocator); + + if (self.eat(.rparen)) return args.toOwnedSlice(self.allocator); + + while (true) { + if (self.peek().tag == .ident and self.peekN(1).tag == .assign) { + const key = self.advance(); + _ = self.advance(); // '=' + const value = try self.parseConditional(); + try args.append(self.allocator, .{ .keyword = .{ .name = key.lexeme, .value = value } }); + } else { + const value = try self.parseConditional(); + try args.append(self.allocator, .{ .positional = value }); + } + if (self.eat(.comma)) continue; + _ = try self.expect(.rparen); + break; + } + + return args.toOwnedSlice(self.allocator); + } + + fn parsePrimary(self: *Parser) ExprError!*Expr { + const tok = self.peek(); + switch (tok.tag) { + .ident => { + _ = self.advance(); + return self.allocExpr(.{ .ident = .{ .name = tok.lexeme, .slot = unknown_slot } }); + }, + .int => { + _ = self.advance(); + const v = std.fmt.parseInt(i64, tok.lexeme, 10) catch return ExprError.InvalidNumber; + return self.allocExpr(.{ .int_lit = v }); + }, + .float => { + _ = self.advance(); + const v = std.fmt.parseFloat(f64, tok.lexeme) catch return ExprError.InvalidNumber; + return self.allocExpr(.{ .float_lit = v }); + }, + .string => { + _ = self.advance(); + const s = try self.unescapeString(tok.lexeme); + return self.allocExpr(.{ .str_lit = s }); + }, + .bool_true => { + _ = self.advance(); + return self.allocExpr(.{ .bool_lit = true }); + }, + .bool_false => { + _ = self.advance(); + return self.allocExpr(.{ .bool_lit = false }); + }, + .none_kw => { + _ = self.advance(); + return self.allocExpr(.null_lit); + }, + .lparen => { + _ = self.advance(); + const inner = try self.parseConditional(); + _ = try self.expect(.rparen); + return inner; + }, + .lbracket => { + _ = self.advance(); + var items = std.ArrayList(*Expr).empty; + errdefer items.deinit(self.allocator); + if (!self.eat(.rbracket)) { + while (true) { + try items.append(self.allocator, try self.parseConditional()); + if (self.eat(.comma)) continue; + _ = try self.expect(.rbracket); + break; + } + } + return self.allocExpr(.{ .list_lit = try items.toOwnedSlice(self.allocator) }); + }, + else => return ExprError.UnexpectedToken, + } + } +}; + +pub fn parseExpr(allocator: std.mem.Allocator, source: []const u8) ExprError!*Expr { + const tokens = try lex(allocator, source); + defer allocator.free(tokens); + var parser = Parser{ .allocator = allocator, .tokens = tokens, .pos = 0 }; + return parser.parse(); +} + +pub const Scope = std.StringHashMap(Value); + +pub const CallArgValue = union(enum) { + positional: Value, + keyword: struct { name: []const u8, value: Value }, +}; + +pub const Runtime = struct { + context: ?*anyopaque = null, + callFunction: ?*const fn (context: ?*anyopaque, allocator: std.mem.Allocator, name: []const u8, args: []const CallArgValue, scope: *Scope) ExprError!Value = null, + getIdentBySlot: ?*const fn (context: ?*anyopaque, slot: u32, name: []const u8, scope: *Scope) ?Value = null, +}; + +const StrWriter = struct { + list: *std.ArrayList(u8), + allocator: std.mem.Allocator, + + pub fn writeAll(self: *StrWriter, bytes: []const u8) !void { + try self.list.appendSlice(self.allocator, bytes); + } + + pub fn writeByte(self: *StrWriter, b: u8) !void { + try self.list.append(self.allocator, b); + } +}; + +fn valueToStringAlloc(allocator: std.mem.Allocator, value: Value) ![]u8 { + switch (value) { + .str => |s| return allocator.dupe(u8, s), + .undefined, .null => return allocator.dupe(u8, ""), + .bool => |b| return allocator.dupe(u8, if (b) "true" else "false"), + .int => |v| { + var buf: [32]u8 = undefined; + const rendered = try std.fmt.bufPrint(&buf, "{}", .{v}); + return allocator.dupe(u8, rendered); + }, + .float => |v| { + var buf: [64]u8 = undefined; + const rendered = try std.fmt.bufPrint(&buf, "{}", .{v}); + return allocator.dupe(u8, rendered); + }, + else => { + var list = std.ArrayList(u8).empty; + errdefer list.deinit(allocator); + var w = StrWriter{ .list = &list, .allocator = allocator }; + try value_mod.writeValue(&w, value); + return list.toOwnedSlice(allocator); + }, + } +} + +const StringView = struct { + bytes: []const u8, + owned: ?[]u8 = null, +}; + +fn valueToStringView(allocator: std.mem.Allocator, value: Value) !StringView { + return switch (value) { + .str => |s| .{ .bytes = s, .owned = null }, + else => blk: { + const owned = try valueToStringAlloc(allocator, value); + break :blk .{ .bytes = owned, .owned = owned }; + }, + }; +} + +fn freeStringView(allocator: std.mem.Allocator, view: StringView) void { + if (view.owned) |buf| allocator.free(buf); +} + +fn concatValues(allocator: std.mem.Allocator, left: Value, right: Value) !Value { + const left_view = try valueToStringView(allocator, left); + defer freeStringView(allocator, left_view); + const right_view = try valueToStringView(allocator, right); + defer freeStringView(allocator, right_view); + + const out = try allocator.alloc(u8, left_view.bytes.len + right_view.bytes.len); + @memcpy(out[0..left_view.bytes.len], left_view.bytes); + @memcpy(out[left_view.bytes.len..], right_view.bytes); + return .{ .str = out }; +} + +fn writeValueAsString(writer: anytype, value: Value) !void { + switch (value) { + .str => |s| try writer.writeAll(s), + else => try value_mod.writeValue(writer, value), + } +} + +fn writeExprAsString(allocator: std.mem.Allocator, ex: *const Expr, scope: *Scope, runtime: *const Runtime, writer: anytype) ExprError!void { + switch (ex.*) { + .binary => |b| { + switch (b.op) { + .concat => { + try writeExprAsString(allocator, b.left, scope, runtime, writer); + try writeExprAsString(allocator, b.right, scope, runtime, writer); + return; + }, + .add => { + const left = try evalExpr(allocator, b.left, scope, runtime); + const right = try evalExpr(allocator, b.right, scope, runtime); + if (left == .str or right == .str) { + try writeValueAsString(writer, left); + try writeValueAsString(writer, right); + return; + } + // Numeric add path. + const added = switch (left) { + .int => |lv| switch (right) { + .int => |rv| Value{ .int = lv + rv }, + .float => |rv| Value{ .float = @as(f64, @floatFromInt(lv)) + rv }, + else => return ExprError.TypeError, + }, + .float => |lv| switch (right) { + .int => |rv| Value{ .float = lv + @as(f64, @floatFromInt(rv)) }, + .float => |rv| Value{ .float = lv + rv }, + else => return ExprError.TypeError, + }, + else => return ExprError.TypeError, + }; + try value_mod.writeValue(writer, added); + return; + }, + else => {}, + } + }, + else => {}, + } + + const v = try evalExpr(allocator, ex, scope, runtime); + try value_mod.writeValue(writer, v); +} + +pub fn writeExprOutput(allocator: std.mem.Allocator, ex: *const Expr, scope: *Scope, runtime: *const Runtime, writer: anytype) ExprError!void { + return writeExprAsString(allocator, ex, scope, runtime, writer); +} + +fn asF64(v: Value) ExprError!f64 { + return switch (v) { + .int => |x| @floatFromInt(x), + .float => |x| x, + else => ExprError.TypeError, + }; +} + +fn normalizeIndex(len: usize, idx: i64) usize { + const l: i64 = @intCast(len); + var v = idx; + if (v < 0) v = l + v; + if (v < 0) return 0; + if (v > l) return len; + return @intCast(v); +} + +fn toIntIndex(v: Value) ?i64 { + return switch (v) { + .int => |i| i, + else => null, + }; +} + +fn writeJsonString(writer: *StrWriter, s: []const u8) !void { + try writer.writeByte('"'); + for (s) |c| { + switch (c) { + '"' => try writer.writeAll("\\\""), + '\\' => try writer.writeAll("\\\\"), + '\n' => try writer.writeAll("\\n"), + '\r' => try writer.writeAll("\\r"), + '\t' => try writer.writeAll("\\t"), + else => try writer.writeByte(c), + } + } + try writer.writeByte('"'); +} + +fn writeJsonValue(writer: *StrWriter, value: Value) !void { + switch (value) { + .undefined, .null => try writer.writeAll("null"), + .bool => |b| if (b) try writer.writeAll("true") else try writer.writeAll("false"), + .int => |v| { + var buf: [32]u8 = undefined; + const s = try std.fmt.bufPrint(&buf, "{}", .{v}); + try writer.writeAll(s); + }, + .float => |v| { + var buf: [64]u8 = undefined; + const s = try std.fmt.bufPrint(&buf, "{}", .{v}); + try writer.writeAll(s); + }, + .str => |s| try writeJsonString(writer, s), + .list => |items| { + try writer.writeByte('['); + for (items, 0..) |item, idx| { + if (idx > 0) try writer.writeByte(','); + try writeJsonValue(writer, item); + } + try writer.writeByte(']'); + }, + .map => |entries| { + try writer.writeByte('{'); + for (entries, 0..) |entry, idx| { + if (idx > 0) try writer.writeByte(','); + try writeJsonString(writer, entry.key); + try writer.writeByte(':'); + try writeJsonValue(writer, entry.value); + } + try writer.writeByte('}'); + }, + .namespace => |ns| { + try writer.writeByte('{'); + var it = ns.map.iterator(); + var idx: usize = 0; + while (it.next()) |entry| { + if (idx > 0) try writer.writeByte(','); + idx += 1; + try writeJsonString(writer, entry.key_ptr.*); + try writer.writeByte(':'); + try writeJsonValue(writer, entry.value_ptr.*); + } + try writer.writeByte('}'); + }, + .loopmeta => |lp| { + try writer.writeByte('{'); + try writeJsonString(writer, "index0"); + try writer.writeByte(':'); + try writeJsonValue(writer, .{ .int = lp.index0 }); + try writer.writeByte(','); + try writeJsonString(writer, "index"); + try writer.writeByte(':'); + try writeJsonValue(writer, .{ .int = lp.index }); + try writer.writeByte(','); + try writeJsonString(writer, "first"); + try writer.writeByte(':'); + try writeJsonValue(writer, .{ .bool = lp.first }); + try writer.writeByte(','); + try writeJsonString(writer, "last"); + try writer.writeByte(':'); + try writeJsonValue(writer, .{ .bool = lp.last }); + try writer.writeByte('}'); + }, + } +} + +fn argKeywordValue(args: []const CallArg, name: []const u8) ?*Expr { + for (args) |arg| { + switch (arg) { + .keyword => |kv| if (std.mem.eql(u8, kv.name, name)) return kv.value, + else => {}, + } + } + return null; +} + +fn valueInContainer(needle: Value, haystack: Value) bool { + return switch (haystack) { + .list => |items| for (items) |item| { + if (Value.eql(needle, item)) break true; + } else false, + .str => |rs| switch (needle) { + .str => |ls| std.mem.indexOf(u8, rs, ls) != null, + else => false, + }, + .map => |entries| switch (needle) { + .str => |ls| for (entries) |entry| { + if (std.mem.eql(u8, ls, entry.key)) break true; + } else false, + else => false, + }, + .namespace => |ns| switch (needle) { + .str => |ls| ns.map.contains(ls), + else => false, + }, + else => false, + }; +} + +fn evalStringMethod(allocator: std.mem.Allocator, method_id: StringMethodId, s: []const u8, arg_values: []const CallArgValue) ExprError!Value { + switch (method_id) { + .endswith => { + if (arg_values.len > 0 and arg_values[0] == .positional and arg_values[0].positional == .str) { + return .{ .bool = std.mem.endsWith(u8, s, arg_values[0].positional.str) }; + } + return .{ .bool = false }; + }, + .startswith => { + if (arg_values.len > 0 and arg_values[0] == .positional and arg_values[0].positional == .str) { + return .{ .bool = std.mem.startsWith(u8, s, arg_values[0].positional.str) }; + } + return .{ .bool = false }; + }, + .split => { + const delim: []const u8 = if (arg_values.len > 0 and arg_values[0] == .positional and arg_values[0].positional == .str) arg_values[0].positional.str else " "; + var it = std.mem.splitSequence(u8, s, delim); + var vals = std.ArrayList(Value).empty; + errdefer vals.deinit(allocator); + while (it.next()) |part| { + try vals.append(allocator, .{ .str = try allocator.dupe(u8, part) }); + } + return .{ .list = try vals.toOwnedSlice(allocator) }; + }, + .strip => { + const cut = if (arg_values.len > 0 and arg_values[0] == .positional and arg_values[0].positional == .str) arg_values[0].positional.str else " \t\n\r"; + return .{ .str = try allocator.dupe(u8, std.mem.trim(u8, s, cut)) }; + }, + .lstrip => { + const cut = if (arg_values.len > 0 and arg_values[0] == .positional and arg_values[0].positional == .str) arg_values[0].positional.str else " \t\n\r"; + return .{ .str = try allocator.dupe(u8, std.mem.trimStart(u8, s, cut)) }; + }, + .rstrip => { + const cut = if (arg_values.len > 0 and arg_values[0] == .positional and arg_values[0].positional == .str) arg_values[0].positional.str else " \t\n\r"; + return .{ .str = try allocator.dupe(u8, std.mem.trimEnd(u8, s, cut)) }; + }, + .unknown => {}, + } + return .undefined; +} + +fn applyFilter(allocator: std.mem.Allocator, filter_id: FilterId, target: Value, args: []const CallArg, scope: *Scope, runtime: *const Runtime) ExprError!Value { + switch (filter_id) { + .items => { + switch (target) { + .map => |entries| { + const out = try allocator.alloc(Value, entries.len); + for (entries, 0..) |entry, idx| { + const pair_vals = try allocator.alloc(Value, 2); + pair_vals[0] = .{ .str = entry.key }; + pair_vals[1] = entry.value; + out[idx] = .{ .list = pair_vals }; + } + return .{ .list = out }; + }, + .namespace => |ns| { + const count = ns.map.count(); + const out = try allocator.alloc(Value, count); + var it = ns.map.iterator(); + var idx: usize = 0; + while (it.next()) |entry| { + const pair_vals = try allocator.alloc(Value, 2); + pair_vals[0] = .{ .str = entry.key_ptr.* }; + pair_vals[1] = entry.value_ptr.*; + out[idx] = .{ .list = pair_vals }; + idx += 1; + } + return .{ .list = out }; + }, + else => return .{ .list = &[_]Value{} }, + } + }, + .length => { + const n: i64 = switch (target) { + .undefined, .null => 0, + .str => |s| @intCast(s.len), + .list => |items| @intCast(items.len), + .map => |entries| @intCast(entries.len), + .namespace => |ns| @intCast(ns.map.count()), + else => 0, + }; + return .{ .int = n }; + }, + .safe => return target, + .string => return .{ .str = try valueToStringAlloc(allocator, target) }, + .sort => { + if (target != .list) return target; + const items = target.list; + const out = try allocator.alloc(Value, items.len); + @memcpy(out, items); + + var attr_name: ?[]const u8 = null; + if (argKeywordValue(args, "attribute")) |expr_ptr| { + const v = try evalExpr(allocator, expr_ptr, scope, runtime); + if (v == .str) attr_name = v.str; + } + + const Ctx = struct { + attr: ?[]const u8, + fn less(ctx: @This(), a: Value, b: Value) bool { + const av = if (ctx.attr) |attr| (a.getAttr(attr) orelse .undefined) else a; + const bv = if (ctx.attr) |attr| (b.getAttr(attr) orelse .undefined) else b; + return switch (av) { + .str => |as| switch (bv) { + .str => |bs| std.mem.order(u8, as, bs) == .lt, + else => false, + }, + .int => |ai| switch (bv) { + .int => |bi| ai < bi, + else => false, + }, + else => false, + }; + } + }; + std.mem.sort(Value, out, Ctx{ .attr = attr_name }, Ctx.less); + return .{ .list = out }; + }, + .trim => { + if (target != .str) return target; + return .{ .str = try allocator.dupe(u8, std.mem.trim(u8, target.str, " \t\n\r")) }; + }, + .tojson => { + var list = std.ArrayList(u8).empty; + errdefer list.deinit(allocator); + var w = StrWriter{ .list = &list, .allocator = allocator }; + try writeJsonValue(&w, target); + return .{ .str = try list.toOwnedSlice(allocator) }; + }, + .unknown => {}, + } + return target; +} + +pub fn evalExpr(allocator: std.mem.Allocator, ex: *const Expr, scope: *Scope, runtime: *const Runtime) ExprError!Value { + return switch (ex.*) { + .ident => |ident| blk: { + if (ident.slot != unknown_slot) { + if (runtime.getIdentBySlot) |cb| { + if (cb(runtime.context, ident.slot, ident.name, scope)) |v| break :blk v; + } + } + break :blk scope.get(ident.name) orelse .undefined; + }, + .int_lit => |v| .{ .int = v }, + .float_lit => |v| .{ .float = v }, + .str_lit => |v| .{ .str = v }, + .bool_lit => |v| .{ .bool = v }, + .null_lit => .null, + .list_lit => |items| { + const out = try allocator.alloc(Value, items.len); + for (items, 0..) |item, idx| out[idx] = try evalExpr(allocator, item, scope, runtime); + return .{ .list = out }; + }, + .getattr => |item| { + const target = try evalExpr(allocator, item.target, scope, runtime); + return target.getAttr(item.attr) orelse .undefined; + }, + .index => |item| { + const target = try evalExpr(allocator, item.target, scope, runtime); + const key = try evalExpr(allocator, item.key, scope, runtime); + switch (target) { + .list => |items| { + if (toIntIndex(key)) |idx| { + const pos = normalizeIndex(items.len, idx); + if (pos < items.len) return items[pos]; + } + return .undefined; + }, + .str => |s| { + if (toIntIndex(key)) |idx| { + const pos = normalizeIndex(s.len, idx); + if (pos < s.len) return .{ .str = try allocator.dupe(u8, s[pos .. pos + 1]) }; + } + return .undefined; + }, + .map => |entries| { + if (key == .str) { + for (entries) |entry| { + if (entry.key.ptr == key.str.ptr and entry.key.len == key.str.len) return entry.value; + if (entry.key.len != key.str.len) continue; + if (entry.key.len > 0 and entry.key[0] != key.str[0]) continue; + if (std.mem.eql(u8, entry.key, key.str)) return entry.value; + } + } + return .undefined; + }, + .namespace => |ns| { + if (key == .str) return ns.map.get(key.str) orelse .undefined; + return .undefined; + }, + else => return .undefined, + } + }, + .slice => |s| { + const target = try evalExpr(allocator, s.target, scope, runtime); + const start_idx: i64 = if (s.start) |sp| (toIntIndex(try evalExpr(allocator, sp, scope, runtime)) orelse 0) else 0; + const end_idx: ?i64 = if (s.end) |ep| (toIntIndex(try evalExpr(allocator, ep, scope, runtime)) orelse 0) else null; + const step_idx: i64 = if (s.step) |ep| (toIntIndex(try evalExpr(allocator, ep, scope, runtime)) orelse 1) else 1; + if (step_idx == 0) return .undefined; + switch (target) { + .list => |items| { + const default_start: i64 = if (step_idx > 0) 0 else @intCast(items.len -| 1); + const default_end: i64 = if (step_idx > 0) @intCast(items.len) else -1; + var i: i64 = if (s.start != null) start_idx else default_start; + const end_bound: i64 = if (end_idx) |e| e else default_end; + var tmp = std.ArrayList(Value).empty; + errdefer tmp.deinit(allocator); + while ((step_idx > 0 and i < end_bound) or (step_idx < 0 and i > end_bound)) : (i += step_idx) { + const pos = normalizeIndex(items.len, i); + if (pos < items.len) try tmp.append(allocator, items[pos]); + } + const out = try tmp.toOwnedSlice(allocator); + return .{ .list = out }; + }, + .str => |str| { + const default_start: i64 = if (step_idx > 0) 0 else @intCast(str.len -| 1); + const default_end: i64 = if (step_idx > 0) @intCast(str.len) else -1; + var i: i64 = if (s.start != null) start_idx else default_start; + const end_bound: i64 = if (end_idx) |e| e else default_end; + var out = std.ArrayList(u8).empty; + errdefer out.deinit(allocator); + while ((step_idx > 0 and i < end_bound) or (step_idx < 0 and i > end_bound)) : (i += step_idx) { + const pos = normalizeIndex(str.len, i); + if (pos < str.len) try out.append(allocator, str[pos]); + } + return .{ .str = try out.toOwnedSlice(allocator) }; + }, + else => return .undefined, + } + }, + .call => |call| { + var small_args: [8]CallArgValue = undefined; + var heap_args: ?[]CallArgValue = null; + const arg_values: []CallArgValue = if (call.args.len <= small_args.len) + small_args[0..call.args.len] + else blk: { + const buf = try allocator.alloc(CallArgValue, call.args.len); + heap_args = buf; + break :blk buf; + }; + defer if (heap_args) |buf| allocator.free(buf); + + for (call.args, 0..) |arg, idx| { + arg_values[idx] = switch (arg) { + .positional => |p| .{ .positional = try evalExpr(allocator, p, scope, runtime) }, + .keyword => |kv| .{ .keyword = .{ .name = kv.name, .value = try evalExpr(allocator, kv.value, scope, runtime) } }, + }; + } + + if (call.callee.* == .ident) { + const callee_name = call.callee.ident.name; + if (std.mem.eql(u8, callee_name, "raise_exception")) return ExprError.RaisedException; + if (std.mem.eql(u8, callee_name, "namespace")) { + const ns_ptr = try allocator.create(value_mod.Namespace); + ns_ptr.* = value_mod.Namespace.init(allocator); + for (arg_values) |arg| { + switch (arg) { + .keyword => |kv| try ns_ptr.map.put(kv.name, kv.value), + else => {}, + } + } + return .{ .namespace = ns_ptr }; + } + if (runtime.callFunction) |cb| return cb(runtime.context, allocator, callee_name, arg_values, scope); + return .undefined; + } + if (call.callee.* == .getattr) { + const getter = call.callee.getattr; + const target = try evalExpr(allocator, getter.target, scope, runtime); + if (target == .str) { + return evalStringMethod(allocator, getter.method_id, target.str, arg_values); + } + if (std.mem.eql(u8, getter.attr, "items")) { + return applyFilter(allocator, .items, target, &[_]CallArg{}, scope, runtime); + } + } + return .undefined; + }, + .filter => |filter| { + const target = try evalExpr(allocator, filter.target, scope, runtime); + return applyFilter(allocator, filter.filter_id, target, filter.args, scope, runtime); + }, + .test_expr => |test_ex| { + const target = try evalExpr(allocator, test_ex.target, scope, runtime); + var ok = switch (test_ex.test_id) { + .defined => target != .undefined, + .none => target == .null, + .string => target == .str, + .iterable => switch (target) { .str, .list, .map, .namespace, .loopmeta => true, else => false }, + .mapping => switch (target) { .map, .namespace => true, else => false }, + .sequence => switch (target) { .str, .list => true, else => false }, + .undefined => target == .undefined, + .unknown => false, + }; + if (test_ex.negated) ok = !ok; + return .{ .bool = ok }; + }, + .cond_expr => |ce| { + const cond = try evalExpr(allocator, ce.cond, scope, runtime); + if (cond.isTruthy()) return evalExpr(allocator, ce.then_expr, scope, runtime); + return evalExpr(allocator, ce.else_expr, scope, runtime); + }, + .unary_not => |inner| { + const v = try evalExpr(allocator, inner, scope, runtime); + return .{ .bool = !v.isTruthy() }; + }, + .unary_neg => |inner| { + const v = try evalExpr(allocator, inner, scope, runtime); + return switch (v) { + .int => |i| .{ .int = -i }, + .float => |f| .{ .float = -f }, + else => ExprError.TypeError, + }; + }, + .binary => |b| { + if (b.op == .@"and") { + const left = try evalExpr(allocator, b.left, scope, runtime); + if (!left.isTruthy()) return .{ .bool = false }; + const right = try evalExpr(allocator, b.right, scope, runtime); + return .{ .bool = right.isTruthy() }; + } + if (b.op == .@"or") { + const left = try evalExpr(allocator, b.left, scope, runtime); + if (left.isTruthy()) return .{ .bool = true }; + const right = try evalExpr(allocator, b.right, scope, runtime); + return .{ .bool = right.isTruthy() }; + } + + const left = try evalExpr(allocator, b.left, scope, runtime); + const right = try evalExpr(allocator, b.right, scope, runtime); + + return switch (b.op) { + .add => { + if (left == .str or right == .str) { + return concatValues(allocator, left, right); + } + return switch (left) { + .int => |lv| switch (right) { + .int => |rv| .{ .int = lv + rv }, + .float => |rv| .{ .float = @as(f64, @floatFromInt(lv)) + rv }, + else => ExprError.TypeError, + }, + .float => |lv| switch (right) { + .int => |rv| .{ .float = lv + @as(f64, @floatFromInt(rv)) }, + .float => |rv| .{ .float = lv + rv }, + else => ExprError.TypeError, + }, + else => ExprError.TypeError, + }; + }, + .concat => { + return concatValues(allocator, left, right); + }, + .sub => switch (left) { + .int => |lv| switch (right) { + .int => |rv| .{ .int = lv - rv }, + .float => |rv| .{ .float = @as(f64, @floatFromInt(lv)) - rv }, + else => ExprError.TypeError, + }, + .float => |lv| switch (right) { + .int => |rv| .{ .float = lv - @as(f64, @floatFromInt(rv)) }, + .float => |rv| .{ .float = lv - rv }, + else => ExprError.TypeError, + }, + else => ExprError.TypeError, + }, + .mul => switch (left) { + .int => |lv| switch (right) { + .int => |rv| .{ .int = lv * rv }, + .float => |rv| .{ .float = @as(f64, @floatFromInt(lv)) * rv }, + else => ExprError.TypeError, + }, + .float => |lv| switch (right) { + .int => |rv| .{ .float = lv * @as(f64, @floatFromInt(rv)) }, + .float => |rv| .{ .float = lv * rv }, + else => ExprError.TypeError, + }, + else => ExprError.TypeError, + }, + .div => .{ .float = (try asF64(left)) / (try asF64(right)) }, + .mod => switch (left) { + .int => |lv| switch (right) { + .int => |rv| .{ .int = @mod(lv, rv) }, + else => ExprError.TypeError, + }, + else => ExprError.TypeError, + }, + .eq => .{ .bool = Value.eql(left, right) }, + .neq => .{ .bool = !Value.eql(left, right) }, + .lt => if (left == .str and right == .str) .{ .bool = std.mem.order(u8, left.str, right.str) == .lt } else .{ .bool = (try asF64(left)) < (try asF64(right)) }, + .lte => if (left == .str and right == .str) .{ .bool = std.mem.order(u8, left.str, right.str) != .gt } else .{ .bool = (try asF64(left)) <= (try asF64(right)) }, + .gt => if (left == .str and right == .str) .{ .bool = std.mem.order(u8, left.str, right.str) == .gt } else .{ .bool = (try asF64(left)) > (try asF64(right)) }, + .gte => if (left == .str and right == .str) .{ .bool = std.mem.order(u8, left.str, right.str) != .lt } else .{ .bool = (try asF64(left)) >= (try asF64(right)) }, + .in => .{ .bool = valueInContainer(left, right) }, + .not_in => .{ .bool = !valueInContainer(left, right) }, + .@"and", .@"or" => unreachable, + }; + }, + }; +} diff --git a/jinja2/generate_expected.bzl b/jinja2/generate_expected.bzl new file mode 100644 index 00000000..f34a6d1c --- /dev/null +++ b/jinja2/generate_expected.bzl @@ -0,0 +1,45 @@ +def _generate_expected_directory_impl(ctx): + out_dir = ctx.actions.declare_directory(ctx.attr.output_dir_name) + + ctx.actions.run( + executable = ctx.executable.tool, + arguments = [ + "--input-root", + ctx.attr.input_root, + "--output-root", + out_dir.path, + ], + inputs = depset(ctx.files.srcs), + outputs = [out_dir], + tools = [ctx.executable.tool], + mnemonic = "RenderExpectedFixtures", + progress_message = "Rendering generated Jinja fixture outputs", + ) + + return [ + DefaultInfo( + files = depset([out_dir]), + runfiles = ctx.runfiles(files = [out_dir]), + ), + ] + +generate_expected_directory = rule( + implementation = _generate_expected_directory_impl, + attrs = { + "srcs": attr.label_list( + allow_files = True, + mandatory = True, + ), + "tool": attr.label( + cfg = "exec", + executable = True, + mandatory = True, + ), + "input_root": attr.string( + mandatory = True, + ), + "output_dir_name": attr.string( + default = "test_cases_generated", + ), + }, +) diff --git a/jinja2/lib.zig b/jinja2/lib.zig new file mode 100644 index 00000000..b4b19a7c --- /dev/null +++ b/jinja2/lib.zig @@ -0,0 +1,556 @@ +const std = @import("std"); + +pub const Environment = @import("environment.zig").Environment; +const expr = @import("expr.zig"); +pub const Template = @import("template.zig").Template; +pub const Value = @import("value.zig").Value; + +fn testAllocator() std.mem.Allocator { + return std.testing.allocator; +} + +fn jsonToValue(allocator: std.mem.Allocator, json_value: std.json.Value) !Value { + return switch (json_value) { + .null => .null, + .bool => |v| Value.fromBool(v), + .integer => |v| Value.fromInt(v), + .float => |v| .{ .float = v }, + .string => |v| Value.fromString(v), + .number_string => |v| Value.fromString(v), + .array => |arr| blk: { + const out = try allocator.alloc(Value, arr.items.len); + for (arr.items, 0..) |item, i| { + out[i] = try jsonToValue(allocator, item); + } + break :blk Value.fromList(out); + }, + .object => |obj| blk: { + const out = try allocator.alloc(Value.Entry, obj.count()); + var it = obj.iterator(); + var i: usize = 0; + while (it.next()) |entry| { + out[i] = .{ + .key = entry.key_ptr.*, + .value = try jsonToValue(allocator, entry.value_ptr.*), + }; + i += 1; + } + break :blk Value.fromMap(out); + }, + }; +} + +fn freeNameList(allocator: std.mem.Allocator, list: *std.ArrayList([]const u8)) void { + for (list.items) |name| allocator.free(name); + list.deinit(allocator); +} + +fn listFixtureFiles( + io: std.Io, + allocator: std.mem.Allocator, + dir_path: []const u8, + extension: []const u8, +) !std.ArrayList([]const u8) { + var out = std.ArrayList([]const u8).empty; + errdefer freeNameList(allocator, &out); + + var dir = try std.Io.Dir.openDir(.cwd(), io, dir_path, .{ .iterate = true }); + defer dir.close(io); + + var iter = dir.iterate(); + while (try iter.next(io)) |entry| { + if (entry.kind != .file and entry.kind != .sym_link) continue; + if (!std.mem.endsWith(u8, entry.name, extension)) continue; + try out.append(allocator, try allocator.dupe(u8, entry.name)); + } + + std.mem.sort([]const u8, out.items, {}, struct { + fn lessThan(_: void, a: []const u8, b: []const u8) bool { + return std.mem.lessThan(u8, a, b); + } + }.lessThan); + + return out; +} + +fn filePrefixId(name: []const u8) []const u8 { + const dot = std.mem.lastIndexOfScalar(u8, name, '.') orelse name.len; + const stem = name[0..dot]; + const underscore = std.mem.indexOfScalar(u8, stem, '_') orelse stem.len; + return stem[0..underscore]; +} + +fn parseExpectedCaseIds(name: []const u8) !struct { template_id: []const u8, data_id: []const u8 } { + if (!std.mem.endsWith(u8, name, ".txt")) return error.InvalidExpectedFixture; + const stem = name[0 .. name.len - 4]; + const sep = std.mem.indexOfScalar(u8, stem, '_') orelse return error.InvalidExpectedFixture; + const template_id = stem[0..sep]; + const data_id = stem[sep + 1 ..]; + if (template_id.len == 0 or data_id.len == 0) return error.InvalidExpectedFixture; + return .{ .template_id = template_id, .data_id = data_id }; +} + +fn findFileById(files: []const []const u8, wanted_id: []const u8) ?[]const u8 { + for (files) |name| { + if (std.mem.eql(u8, filePrefixId(name), wanted_id)) return name; + } + return null; +} + +fn resolveSuiteRootPath( + io: std.Io, + allocator: std.mem.Allocator, + suite: []const u8, +) !struct { path: []const u8, owned: ?[]u8 } { + const p1 = try std.fmt.allocPrint(allocator, "test_cases/{s}", .{suite}); + errdefer allocator.free(p1); + if (std.Io.Dir.openDir(.cwd(), io, p1, .{})) |dir| { + dir.close(io); + return .{ .path = p1, .owned = p1 }; + } else |_| { + allocator.free(p1); + } + + const p2 = try std.fmt.allocPrint(allocator, "jinja2/test_cases/{s}", .{suite}); + errdefer allocator.free(p2); + if (std.Io.Dir.openDir(.cwd(), io, p2, .{})) |dir| { + dir.close(io); + return .{ .path = p2, .owned = p2 }; + } else |_| { + allocator.free(p2); + } + + const test_srcdir = std.c.getenv("TEST_SRCDIR") orelse return error.FileNotFound; + const test_workspace = std.c.getenv("TEST_WORKSPACE") orelse return error.FileNotFound; + const p3 = try std.fmt.allocPrint( + allocator, + "{s}/{s}/jinja2/test_cases/{s}", + .{ std.mem.span(test_srcdir), std.mem.span(test_workspace), suite }, + ); + errdefer allocator.free(p3); + if (std.Io.Dir.openDir(.cwd(), io, p3, .{})) |dir| { + dir.close(io); + return .{ .path = p3, .owned = p3 }; + } else |_| { + allocator.free(p3); + } + + return error.FileNotFound; +} + +fn resolveExpectedDirPath( + io: std.Io, + allocator: std.mem.Allocator, + suite: []const u8, + suite_root_path: []const u8, +) !struct { path: []const u8, owned: ?[]u8 } { + const p1 = try std.fmt.allocPrint(allocator, "test_cases_generated/{s}", .{suite}); + errdefer allocator.free(p1); + if (std.Io.Dir.openDir(.cwd(), io, p1, .{})) |dir| { + dir.close(io); + return .{ .path = p1, .owned = p1 }; + } else |_| { + allocator.free(p1); + } + + const p2 = try std.fmt.allocPrint(allocator, "jinja2/test_cases_generated/{s}", .{suite}); + errdefer allocator.free(p2); + if (std.Io.Dir.openDir(.cwd(), io, p2, .{})) |dir| { + dir.close(io); + return .{ .path = p2, .owned = p2 }; + } else |_| { + allocator.free(p2); + } + + if (std.c.getenv("TEST_SRCDIR")) |test_srcdir| { + if (std.c.getenv("TEST_WORKSPACE")) |test_workspace| { + const p3 = try std.fmt.allocPrint( + allocator, + "{s}/{s}/jinja2/test_cases_generated/{s}", + .{ std.mem.span(test_srcdir), std.mem.span(test_workspace), suite }, + ); + errdefer allocator.free(p3); + if (std.Io.Dir.openDir(.cwd(), io, p3, .{})) |dir| { + dir.close(io); + return .{ .path = p3, .owned = p3 }; + } else |_| { + allocator.free(p3); + } + } + } + + const fallback_with_expected = try std.fmt.allocPrint(allocator, "{s}/expected", .{suite_root_path}); + errdefer allocator.free(fallback_with_expected); + if (std.Io.Dir.openDir(.cwd(), io, fallback_with_expected, .{})) |dir| { + dir.close(io); + return .{ .path = fallback_with_expected, .owned = fallback_with_expected }; + } else |_| { + allocator.free(fallback_with_expected); + } + + const fallback = try allocator.dupe(u8, suite_root_path); + return .{ .path = fallback, .owned = fallback }; +} + +fn runChatLikeFixtureSuite(suite: []const u8) !void { + var threaded: std.Io.Threaded = .init(testAllocator(), .{}); + defer threaded.deinit(); + const io = threaded.io(); + + const suite_root = try resolveSuiteRootPath(io, testAllocator(), suite); + defer if (suite_root.owned) |p| testAllocator().free(p); + + const templates_dir = try std.fmt.allocPrint(testAllocator(), "{s}/templates", .{suite_root.path}); + defer testAllocator().free(templates_dir); + const data_dir = try std.fmt.allocPrint(testAllocator(), "{s}/data", .{suite_root.path}); + defer testAllocator().free(data_dir); + const expected_dir = try resolveExpectedDirPath(io, testAllocator(), suite, suite_root.path); + defer if (expected_dir.owned) |p| testAllocator().free(p); + + var template_names = try listFixtureFiles(io, testAllocator(), templates_dir, ".jinja"); + defer freeNameList(testAllocator(), &template_names); + var data_names = try listFixtureFiles(io, testAllocator(), data_dir, ".json"); + defer freeNameList(testAllocator(), &data_names); + var expected_names = try listFixtureFiles(io, testAllocator(), expected_dir.path, ".txt"); + defer freeNameList(testAllocator(), &expected_names); + + for (expected_names.items) |expected_name| { + const ids = parseExpectedCaseIds(expected_name) catch { + std.debug.print("{s} fixture {s}: expected filename must be _.txt\n", .{ suite, expected_name }); + return error.InvalidExpectedFixture; + }; + + const template_name = findFileById(template_names.items, ids.template_id) orelse { + std.debug.print("{s} fixture {s}: missing template id {s}\n", .{ suite, expected_name, ids.template_id }); + return error.InvalidExpectedFixture; + }; + const data_name = findFileById(data_names.items, ids.data_id) orelse { + std.debug.print("{s} fixture {s}: missing data id {s}\n", .{ suite, expected_name, ids.data_id }); + return error.InvalidExpectedFixture; + }; + + var arena = std.heap.ArenaAllocator.init(testAllocator()); + defer arena.deinit(); + const allocator = arena.allocator(); + + const template_path = try std.fmt.allocPrint(allocator, "{s}/{s}", .{ templates_dir, template_name }); + const data_path = try std.fmt.allocPrint(allocator, "{s}/{s}", .{ data_dir, data_name }); + const expected_path = try std.fmt.allocPrint(allocator, "{s}/{s}", .{ expected_dir.path, expected_name }); + + const template_src = try std.Io.Dir.cwd().readFileAlloc(io, template_path, allocator, .limited(1 << 20)); + const data_src = try std.Io.Dir.cwd().readFileAlloc(io, data_path, allocator, .limited(1 << 20)); + const expected_src = try std.Io.Dir.cwd().readFileAlloc(io, expected_path, allocator, .limited(1 << 20)); + + const parsed = try std.json.parseFromSliceLeaky(std.json.Value, allocator, data_src, .{}); + if (parsed != .object) { + std.debug.print("{s} fixture {s}: data JSON root must be object\n", .{ suite, data_name }); + return error.InvalidChatDataFixture; + } + + const ctx_value = try jsonToValue(allocator, parsed); + if (ctx_value != .map) { + std.debug.print("{s} fixture {s}: data must convert to map\n", .{ suite, data_name }); + return error.InvalidChatDataFixture; + } + + var env = Environment.init(allocator); + defer env.deinit(); + env.addTemplate("fixture", template_src) catch |err| { + std.debug.print("{s} fixture {s}: addTemplate failed ({s})\n", .{ suite, expected_name, @errorName(err) }); + return err; + }; + + const rendered = env.renderTemplate(testAllocator(), "fixture", ctx_value.map) catch |err| { + std.debug.print("{s} fixture {s}: renderTemplate failed ({s})\n", .{ suite, expected_name, @errorName(err) }); + return err; + }; + defer testAllocator().free(rendered); + + if (!std.mem.eql(u8, expected_src, rendered)) { + std.debug.print("{s} fixture mismatch: {s} (template={s} data={s})\n", .{ suite, expected_name, template_name, data_name }); + } + try std.testing.expectEqualStrings(expected_src, rendered); + } +} + +test "basic fixture corpus renders as expected" { + var threaded: std.Io.Threaded = .init(testAllocator(), .{}); + defer threaded.deinit(); + const io = threaded.io(); + + var basic_dir: std.Io.Dir = undefined; + var base_path: []const u8 = ""; + var owned_base_path: ?[]u8 = null; + defer if (owned_base_path) |p| testAllocator().free(p); + if (std.Io.Dir.openDir(.cwd(), io, "test_cases/basic", .{ .iterate = true })) |dir| { + basic_dir = dir; + base_path = "test_cases/basic"; + } else |_| if (std.Io.Dir.openDir(.cwd(), io, "jinja2/test_cases/basic", .{ .iterate = true })) |dir| { + basic_dir = dir; + base_path = "jinja2/test_cases/basic"; + } else |_| { + const test_srcdir = std.c.getenv("TEST_SRCDIR") orelse return error.FileNotFound; + const test_workspace = std.c.getenv("TEST_WORKSPACE") orelse return error.FileNotFound; + const runfiles_path = try std.fmt.allocPrint( + testAllocator(), + "{s}/{s}/jinja2/test_cases/basic", + .{ std.mem.span(test_srcdir), std.mem.span(test_workspace) }, + ); + errdefer testAllocator().free(runfiles_path); + basic_dir = try std.Io.Dir.openDir(.cwd(), io, runfiles_path, .{ .iterate = true }); + owned_base_path = runfiles_path; + base_path = runfiles_path; + } + defer basic_dir.close(io); + + const expected_dir = try resolveExpectedDirPath(io, testAllocator(), "basic", base_path); + defer if (expected_dir.owned) |p| testAllocator().free(p); + + var names = std.ArrayList([]const u8).empty; + defer names.deinit(testAllocator()); + + var iter = basic_dir.iterate(); + while (try iter.next(io)) |entry| { + if (entry.kind != .file and entry.kind != .sym_link) continue; + if (!std.mem.endsWith(u8, entry.name, ".json")) continue; + try names.append(testAllocator(), try testAllocator().dupe(u8, entry.name)); + } + defer { + for (names.items) |name| testAllocator().free(name); + } + + std.mem.sort([]const u8, names.items, {}, struct { + fn lessThan(_: void, a: []const u8, b: []const u8) bool { + return std.mem.lessThan(u8, a, b); + } + }.lessThan); + + for (names.items) |name| { + var arena = std.heap.ArenaAllocator.init(testAllocator()); + defer arena.deinit(); + const allocator = arena.allocator(); + + var path_buf: [1024]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buf, "{s}/{s}", .{ base_path, name }); + const case_src = try std.Io.Dir.cwd().readFileAlloc(io, path, allocator, .limited(1 << 20)); + + const expected_name = blk: { + const stem = name[0 .. name.len - ".json".len]; + break :blk try std.fmt.allocPrint(allocator, "{s}.txt", .{stem}); + }; + const expected_path = try std.fmt.allocPrint(allocator, "{s}/{s}", .{ expected_dir.path, expected_name }); + const expected_src = try std.Io.Dir.cwd().readFileAlloc(io, expected_path, allocator, .limited(1 << 20)); + + const parsed = try std.json.parseFromSliceLeaky(std.json.Value, allocator, case_src, .{}); + if (parsed != .object) { + std.debug.print("basic fixture {s}: expected root JSON object\n", .{name}); + return error.InvalidBasicFixture; + } + + const template_json = parsed.object.get("template") orelse { + std.debug.print("basic fixture {s}: missing template\n", .{name}); + return error.InvalidBasicFixture; + }; + const data_json = parsed.object.get("data") orelse { + std.debug.print("basic fixture {s}: missing data\n", .{name}); + return error.InvalidBasicFixture; + }; + if (template_json != .string) { + std.debug.print("basic fixture {s}: template must be string\n", .{name}); + return error.InvalidBasicFixture; + } + + const ctx_value = try jsonToValue(allocator, data_json); + if (ctx_value != .map) { + std.debug.print("basic fixture {s}: data must convert to map\n", .{name}); + return error.InvalidBasicFixture; + } + + var env = Environment.init(allocator); + defer env.deinit(); + env.addTemplate("fixture", template_json.string) catch |err| { + std.debug.print("basic fixture {s}: addTemplate failed: {s}\n", .{ name, @errorName(err) }); + return err; + }; + + const rendered = env.renderTemplate(testAllocator(), "fixture", ctx_value.map) catch |err| { + std.debug.print("basic fixture {s}: renderTemplate failed: {s}\n", .{ name, @errorName(err) }); + return err; + }; + defer testAllocator().free(rendered); + + if (!std.mem.eql(u8, expected_src, rendered)) { + std.debug.print("basic fixture mismatch: {s}\n", .{name}); + } + try std.testing.expectEqualStrings(expected_src, rendered); + } +} + +test "chat fixture corpus renders as expected" { + try runChatLikeFixtureSuite("chat"); +} + +test "multimodal fixture corpus renders as expected" { + try runChatLikeFixtureSuite("multimodal"); +} + +test "expression lexer and parser basic arithmetic" { + var arena = std.heap.ArenaAllocator.init(testAllocator()); + defer arena.deinit(); + const allocator = arena.allocator(); + const parsed = try expr.parseExpr(allocator, "1 + 2 * 3"); + + var scope = expr.Scope.init(allocator); + defer scope.deinit(); + const runtime = expr.Runtime{}; + + const result = try expr.evalExpr(allocator, parsed, &scope, &runtime); + try std.testing.expectEqual(@as(i64, 7), result.int); +} + +test "render text and variable" { + var arena = std.heap.ArenaAllocator.init(testAllocator()); + defer arena.deinit(); + const allocator = arena.allocator(); + var env = Environment.init(allocator); + defer env.deinit(); + + try env.addTemplate("hello", "Hello {{ name }}!"); + + const ctx = [_]Value.Entry{ + .{ .key = "name", .value = Value.fromString("World") }, + }; + + const output = try env.renderTemplate(allocator, "hello", &ctx); + defer allocator.free(output); + + try std.testing.expectEqualStrings("Hello World!", output); +} + +test "if else rendering" { + var arena = std.heap.ArenaAllocator.init(testAllocator()); + defer arena.deinit(); + const allocator = arena.allocator(); + var env = Environment.init(allocator); + defer env.deinit(); + + try env.addTemplate("if", "{% if enabled %}on{% else %}off{% endif %}"); + + const true_ctx = [_]Value.Entry{ + .{ .key = "enabled", .value = Value.fromBool(true) }, + }; + const false_ctx = [_]Value.Entry{ + .{ .key = "enabled", .value = Value.fromBool(false) }, + }; + + const out_true = try env.renderTemplate(allocator, "if", &true_ctx); + defer allocator.free(out_true); + const out_false = try env.renderTemplate(allocator, "if", &false_ctx); + defer allocator.free(out_false); + + try std.testing.expectEqualStrings("on", out_true); + try std.testing.expectEqualStrings("off", out_false); +} + +test "for loop over list" { + var arena = std.heap.ArenaAllocator.init(testAllocator()); + defer arena.deinit(); + const allocator = arena.allocator(); + var env = Environment.init(allocator); + defer env.deinit(); + + try env.addTemplate("loop", "{% for n in nums %}[{{ n }}]{% endfor %}"); + + const nums = [_]Value{ + Value.fromInt(1), + Value.fromInt(2), + Value.fromInt(3), + }; + const ctx = [_]Value.Entry{ + .{ .key = "nums", .value = Value.fromList(&nums) }, + }; + + const output = try env.renderTemplate(allocator, "loop", &ctx); + defer allocator.free(output); + + try std.testing.expectEqualStrings("[1][2][3]", output); +} + +test "attribute lookup via dot" { + var arena = std.heap.ArenaAllocator.init(testAllocator()); + defer arena.deinit(); + const allocator = arena.allocator(); + var env = Environment.init(allocator); + defer env.deinit(); + + try env.addTemplate("dot", "{{ user.name }}"); + + const user_map = [_]Value.Entry{ + .{ .key = "name", .value = Value.fromString("Ada") }, + }; + const ctx = [_]Value.Entry{ + .{ .key = "user", .value = Value.fromMap(&user_map) }, + }; + + const output = try env.renderTemplate(allocator, "dot", &ctx); + defer allocator.free(output); + + try std.testing.expectEqualStrings("Ada", output); +} + +test "expression in if condition" { + var arena = std.heap.ArenaAllocator.init(testAllocator()); + defer arena.deinit(); + const allocator = arena.allocator(); + var env = Environment.init(allocator); + defer env.deinit(); + + try env.addTemplate("cond", "{% if score >= 10 and score < 20 %}ok{% else %}bad{% endif %}"); + + const ctx = [_]Value.Entry{ + .{ .key = "score", .value = Value.fromInt(12) }, + }; + + const output = try env.renderTemplate(allocator, "cond", &ctx); + defer allocator.free(output); + + try std.testing.expectEqualStrings("ok", output); +} + +test "renderTemplateStruct zero-alloc nested context" { + var arena = std.heap.ArenaAllocator.init(testAllocator()); + defer arena.deinit(); + const allocator = arena.allocator(); + + var env = Environment.init(allocator); + defer env.deinit(); + + try env.addTemplate( + "struct_ctx", + "{{ user.name }}|{{ user.stats.score }}|{% for t in tags %}{{ t }}{% endfor %}|{{ active }}|{{ note is none }}", + ); + + const Stats = struct { score: i32 }; + const User = struct { + name: []const u8, + stats: Stats, + }; + const Ctx = struct { + user: User, + tags: [2][]const u8, + active: bool, + note: ?[]const u8, + }; + + const ctx = Ctx{ + .user = .{ .name = "Ada", .stats = .{ .score = 7 } }, + .tags = .{ "x", "y" }, + .active = true, + .note = null, + }; + + const output = try env.renderTemplateStruct(allocator, "struct_ctx", ctx); + defer allocator.free(output); + + try std.testing.expectEqualStrings("Ada|7|xy|true|true", output); +} diff --git a/jinja2/python_best_practice.md b/jinja2/python_best_practice.md new file mode 100644 index 00000000..f94919ff --- /dev/null +++ b/jinja2/python_best_practice.md @@ -0,0 +1,490 @@ +# Managing Python Scripts and Dependencies in Bazel 8 for Modern Projects + +## Executive summary + +Bazel 8 is the inflection point where “modern Bazel” effectively means **Bzlmod-first** dependency management: the legacy `WORKSPACE` system is **disabled by default** in Bazel 8 and is planned for removal in Bazel 9, so Python dependency management strategies should be designed around `MODULE.bazel`, module extensions, and lockfiles. citeturn21search16turn9search18turn9search11 + +Across project sizes, the most robust baseline is: + +- **Use `rules_python` as the core Python ruleset** (for `py_library`, `py_binary`, `py_test`, toolchains, PyPI integration, and packaging utilities). citeturn11search12turn18search14 +- **Pin Python toolchains explicitly in the root module** so your interpreter doesn’t drift as `rules_python` updates (a real risk in larger repos and long-lived branches). citeturn11search19turn7search1 +- **Treat Python dependencies as “build inputs,” not as a developer-local environment**: generate a fully resolved lock (`requirements_lock.txt`), use `pip.parse` (Bzlmod) or `pip_parse` (legacy), and commit the Bazel module lockfile(s). citeturn6view0turn17view0turn3search38 + +Clear, size-based recommendations: + +**Small projects (single app / a few libs, minimal native deps)** +Adopt **Bzlmod + `rules_python` toolchains + `pip.parse` from a single locked requirements file**. Use hermetic interpreter toolchains; prefer wheels-only where feasible (to avoid non-hermetic sdist builds). Add a lightweight developer loop via the `rules_python` REPL and an IDE plugin. citeturn7search1turn6view0turn18search0turn20view0turn15search5 + +**Medium projects (multiple services, some native wheels, CI caching)** +Add: **multi-platform dependency strategy** (requirements-by-platform and/or cross-download), **remote cache**, and packaging targets (wheels and/or OCI images) tied to Bazel outputs. Begin adopting `rules_license` and automated vulnerability scanning over lockfiles. citeturn9search30turn14search6turn12search0turn1search16turn2search26 + +**Large projects (monorepo, polyglot, native extensions, remote execution)** +Standardize: **one root `MODULE.bazel`**, a small number of **pip “hubs”** (separate dependency universes), strong reproducibility controls (hash-locked deps + Bazel lockfiles), and CI backed by **remote execution** plus hermetic toolchain containers. Add SBOM/license metadata in the build graph and verify module provenance/attestations where possible. citeturn6view0turn22search0turn21search1turn3search38turn21search7 + +## Bazel 8 Python rules and migration context + +Bazel 8’s ecosystem shift matters more than any single Python macro: + +Bzlmod replaces the legacy `WORKSPACE` mechanism for external dependencies; **`WORKSPACE` is disabled by default in Bazel 8**, and Bazel’s own guidance frames this as a core migration objective. citeturn21search16turn9search18turn9search20 + +Bazel’s roadmap explicitly states that with Bazel 8, support for several language rules—including **Python**—has been migrated out of the Bazel codebase into corresponding Starlark rule repositories (where they can evolve independently). citeturn21search12turn8search19 + +For Python specifically, the durable implication is: **toolchains are not optional plumbing**. Bazel has long been migrating Python execution away from global “pick a runtime” flags (like `--python_top`) toward **toolchain resolution**, and Bazel’s CLI docs in the Bazel 8 era emphasize the toolchain-driven model. citeturn7search2turn7search23turn8search1 + +For teams moving from Bazel ≤7 or WORKSPACE-centric repos, it’s important to also track command churn: the Bazel 8.5 changelog notes that **`bazel sync` has been removed** (use `bazel fetch --all`). This intersects directly with older Python-deps workflows that advised using `bazel sync` to “refresh” pip repos. citeturn10view0 + +## Community rulesets and tool comparison + +### Comparison table of Bazel Python dependency and execution tooling + +| Tool / rule | What it does | Pros | Cons / risks | Best fit | +| ------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------- | +| `rules_python` core rules (`py_library`, `py_binary`, `py_test`) | First-class Python build/run/test rules, now largely Starlark-driven in modern Bazel citeturn18search14turn11search12 | Aligns with Bazel 8 migration direction; integrates with toolchains, runfiles, packaging & PyPI rules citeturn18search14turn18search2 | Requires learning Bazel idioms (deps, runfiles, sandboxes) | All Bazel 8 Python projects | +| `rules_python` toolchains (`python.toolchain` via Bzlmod extension) | Pins Python runtimes and makes them available via toolchain resolution citeturn7search1turn7search4 | Stronger reproducibility; avoids “works on my machine” interpreter drift; supports multi-version setups citeturn11search19turn12search26 | Extra moving parts; multi-Python graphs have real overhead and need careful isolation citeturn12search26turn7search5 | Any project that cares about consistent Python versions | +| `pip.parse` (Bzlmod module extension) | Imports locked PyPI dependencies into a “hub” repo; supports per-Python-version grouping citeturn6view0turn9search9 | Bazel 8-native; can group multiple Python versions under one name; supports wheel patching & overrides citeturn6view0turn5view0 | Requires fully resolved lock(s); hub design: don’t mix hubs in one program; advanced options can be tricky citeturn6view0 | Bazel 8 + Bzlmod projects | +| `pip_parse` (legacy repo rule) | WORKSPACE-era analogue to `pip.parse` using `requirements_lock` citeturn9search8turn9search0 | Works in WORKSPACE mode; mature migration path | WORKSPACE is off by default in Bazel 8 and removed in Bazel 9; long-term dead-end citeturn21search16turn9search11 | Transitional repos still on WORKSPACE | +| `pip_install` | Older WORKSPACE-era rule for PyPI deps | Familiar to older Bazel Python users | Deprecated and removed in later `rules_python` lines; migration target is `pip_parse`/`pip.parse` citeturn3search0turn3search12 | Avoid in new Bazel 8 projects | +| `pip_import` | Very old rules_python external deps rule | Historical only | Known to build from source during workspace load and had Python-version pitfalls; replaced by newer rules citeturn3search1turn3search9 | Avoid | +| `compile_pip_requirements` (`rules_python`) | Generates and validates hash-pinned lockfiles via pip-compile semantics citeturn17view0turn11search11 | Gives a Bazel-native “update” and “check up-to-date” workflow; works with `requirements.in` and `pyproject.toml` metadata citeturn17view0 | If reading `pyproject.toml`, unpinned build-system deps can cause non-hermetic behavior citeturn17view0turn2search1 | Teams wanting Bazel-enforced lock maintenance | +| `aspect_rules_py` | Alternate implementations of Python rules on top of `rules_python`; optionally integrates `uv` citeturn11search23turn3search10 | Can improve Python ergonomics (pytest-focused flows, venv targets, `uv` options); stays compatible with `rules_python` foundations citeturn11search23turn19search5 | Additional abstraction layer; introduces its own compatibility surface (and its own bugs) | Medium/large teams needing stronger Python dev UX | +| `rules_oci` (and emerging `rules_img`) | Build OCI images from Bazel outputs citeturn12search0turn12search20 | Hermetic-ish packaging; integrates well with CI and deployment; avoids handwritten Dockerfiles for many use cases citeturn12search0turn12search27 | Runfiles layout + container layering can be subtle; remote-exec performance tradeoffs exist citeturn12search3turn12search20 | Production deployments & hermetic runtime environments | +| `rules_license` | Captures license metadata in Bazel graphs; supports compliance reporting workflows citeturn1search16turn1search27 | Bazel-native license attribution; integrates with broader Bazel compliance tooling | Requires consistent metadata discipline | Orgs with compliance needs | +| `bazel-contrib/supply-chain` | Inject/collect supply-chain metadata (SBOM/provenance scaffolding) citeturn21search2 | Helps systematize supply-chain metadata in Bazel builds | Still evolving ecosystem | Security-conscious orgs, regulated environments | + +### A qualitative chart of execution options + +```mermaid +quadrantChart + title Python execution models in Bazel 8 (qualitative) + x-axis Developer convenience --> Hermeticity & reproducibility + y-axis Faster local iteration --> Stronger CI parity + quadrant-1 "Best of both (rare)" + quadrant-2 "CI-first" + quadrant-3 "Ad-hoc dev" + quadrant-4 "Bazel-native dev" + "Local venv + pip install": [0.25, 0.75] + "Bazel run (runfiles, system_python bootstrap)": [0.55, 0.55] + "Bazel run (rules_python toolchain + script bootstrap)": [0.75, 0.65] + "Bazel run (venvs_site_packages=yes)": [0.8, 0.7] + "OCI image (rules_oci) execution": [0.9, 0.4] +``` + +(Interpretation: the further right, the more the runtime resembles a reproducible build artifact; the further up, the more it favors tight edit-run loops.) + +## Reproducible dependency resolution in Bazel 8 + +### Lockfiles and the “two-layer” reproducibility story + +Modern Bazel 8 Python reproducibility typically requires **two lock layers**: + +1. **Bazel module resolution lock**: `MODULE.bazel.lock` records module resolution and extension evaluation results, and its purpose is explicitly to promote reproducible builds and skip repeated resolution work. citeturn3search38 +2. **Python package resolution lock**: a fully pinned `requirements_lock.txt` (plus optional platform-specific lockfiles) which `pip.parse` / `pip_parse` consumes via `requirements_lock`. citeturn6view0turn9search8 + +This separation is not redundant: Bazel’s lockfile pins **Bazel modules** and extension outputs, whereas the requirements lock pins **Python distribution selection** (versions, transitive closure, and—if you choose—hashes). + +### Rules_python’s lockfile guidance and why PEP 517/518 matters + +`rules_python`’s “Lock” documentation is explicit that it currently supports **`requirements.txt` format** for lock inputs. citeturn17view0 That has practical consequences for modern packaging-first Python projects that normally live in `pyproject.toml`. + +`rules_python` recommends generating a pinned transitive lock (e.g., `requirements_lock.txt`) from higher-level inputs (like `requirements.in` or `pyproject.toml`) and shows a Bazel-native path using `compile_pip_requirements` (with `bazel run …update` and `bazel test …_test` to enforce freshness). citeturn17view0turn11search11 + +The key reproducibility warning is also explicit: if dependency resolution consults `pyproject.toml`, you should ensure `[build-system]` is present **with pinned dependencies**, otherwise resolution can become non-hermetic due to unpinned build-system defaults. citeturn17view0turn2search1 This aligns with PEP 518’s role: declaring build-system requirements in `pyproject.toml`. citeturn2search1turn2search21 + +### Hashes, wheels, sdists, and constraints + +**Hash checking**: Pip supports a strict “hash-checking mode” (`--require-hashes`) to ensure artifacts match recorded hashes—useful for deployment scripts and for defending against tampering when consuming third-party artifacts. citeturn2search2 In a Bazel context, hashes help align “downloaded artifacts as inputs” with Bazel’s model of content-addressable correctness. + +**Wheel-first strategy**: Bazel-friendly reproducibility is generally better when you can avoid building sdists during repository fetching. In the `pip.parse` API, `download_only=True` switches from “wheel building” to “pip download,” disabling building wheels from source and enabling cross-platform wheel downloads via extra pip args. citeturn6view0 + +At the configuration level, `rules_python` exposes flags to choose wheel vs sdist behavior: e.g. `pip_whl=only` (wheels only) vs `pip_whl=auto` (prefer wheel, fallback to sdist) vs `pip_whl=no` (sdist only, building wheels non-hermetically inside repository rules). citeturn20view0 + +**Platform-specific locks and cross-platform wheels**: rules_python documents multi-platform support where you can provide multiple requirements files and use `download_only` and platform selectors to obtain the right wheels. citeturn9search30turn6view0 + +**Why “platform wheels” are hard**: Python packaging relies on platform compatibility tags such as manylinux; the manylinux project and PEP 600 define the “future-proof” manylinux tag scheme and note the existence of musllinux tags (PEP 656). citeturn2search3turn2search7 This is the underlying reason your Bazel Python deps may differ across Linux distros/architectures even with identical version pins. + +## Execution models and local development ergonomics + +### Virtualenv vs runfiles vs container execution + +**Runfiles-based execution (classic Bazel)** +Bazel’s Python rule reference describes a `py_binary` as including `.py` sources, a `*.runfiles` tree of runtime files, and a stub script that sets up the environment correctly. citeturn18search2 +This model is Bazel-native and works well when Python deps behave as importable files with predictable paths. + +**Build-time venv / site-packages layout (rules_python modern option)** +Recent `rules_python` introduces configuration flags around startup implementation (`--bootstrap_impl`) and “site-packages layout” behavior (`venvs_site_packages`). The docs describe `venvs_site_packages` as an _experimental_ API that makes libraries importable by creating paths in a binary’s site-packages directory (rather than only manipulating `sys.path`), and it applies to `--bootstrap_impl=script` binaries. citeturn20view0 + +The same config docs emphasize that `bootstrap_impl=script` requires a `rules_python` toolchain providing `PyRuntimeInfo`; in particular, **WORKSPACE builds may default to Bazel’s legacy toolchain**, which doesn’t support script bootstrap. citeturn20view0 + +This is directly relevant to problematic packages relying on `site` initialization, `*.pth` handling, or dist-info discovery: `rules_python`’s venv/site-packages work is motivated by such issues and documents known-problem classes (e.g., `*.pth` loading). citeturn19search3 + +**Containerized execution (deployment parity)** +For production parity and hermetic runtime closure, OCI images built from Bazel outputs are common. `rules_oci` explicitly positions itself as a Bazel ruleset for building OCI containers and notes it is not a complete replacement for older container rules in every niche, but covers most use cases. citeturn12search0 +Performance and remote-execution considerations are driving new work in container rulesets (e.g., `rules_img` emphasizing faster, less blob-heavy builds), which matters if Python deployables are built frequently in CI. citeturn12search20 + +### Debugging, REPL, and IDE integration + +**REPL**: `rules_python` provides an interactive REPL whose goal is to present an environment identical to what a `py_binary` creates (useful for debugging import paths and dependency issues). citeturn18search0 + +**Precompiling `.pyc`**: rules_python documents build-time Python bytecode compilation as a performance optimization; it is disabled by default and must be enabled via flags/attributes. citeturn18search3turn20view0 + +**Avoid “import everything” surprises**: a config flag exists to add top-level runfiles directories (presumed repo directories) to Python import search path, and the docs recommend setting it to `false` to avoid external dependencies interfering with import behavior. citeturn20view0 + +**IDE**: Bazel’s official docs cover IDE integration pathways. citeturn15search5 For JetBrains IDEs, the vendor provides Bazel plugin documentation, and JetBrains announced GA availability of the revamped Bazel plugin across IDEs including Python workflows. citeturn15search0turn15search4 + +**Editor tooling**: the `vscode-bazel` extension documents configuring Bazel and buildifier executables in VS Code. citeturn15search1 + +**Watch mode**: `bazel-watcher` (`ibazel`) offers file watching to rerun build/test/run on changes—useful for Python’s tight iteration loops. citeturn15search7 + +**Sandbox debugging**: Bazel’s sandboxing docs recommend `--verbose_failures` and `--sandbox_debug` to surface the underlying action command and sandbox structure when debugging. citeturn14search2turn14search12 + +## Packaging, distribution, and native extensions + +### Building wheels and publishing artifacts + +`rules_python` documents a “How to build a wheel” guide built around `py_wheel` and a source `py_library`. citeturn16search17 + +For platform tags, `packaging.bzl` explicitly states that pure-Python wheels use `platform="any"`, but you must specify an appropriate platform tag if your wheel includes platform-specific data like `.so`/`.pyd` extension modules; it also shows using constraints + `select()` for multi-platform tagging. citeturn13search33 + +A representative community reference implementation is Buildkite’s step-by-step guide and example project, which walks through `py_library`, `py_test`, `py_wheel`, and dependency workflows under Bazel. citeturn11search18 + +### Native C extensions and platform-specific wheels + +**Prefer binary wheels when possible** +For Linux, “portable” wheels hinge on manylinux tagging. PEP 600 defines the perennial manylinux tag scheme, and the manylinux project notes musllinux tagging (PEP 656) as well. citeturn2search3turn2search7 +In practice, this is why “works on one Linux image” doesn’t always translate across glibc/musl variants unless you constrain wheel selection or standardize execution environments. + +**When you must build extensions** +`rules_python` has introduced a `py_cc_toolchain` rule to carry Python runtime C/C++ info (headers, libs) and explicitly labels it beta-quality. citeturn13search14 A related change history describes exposing Python C headers through toolchain resolution to allow consistently retrieving `Python.h` and related linking info. citeturn13search10 + +A common pattern for extension modules is to build them as Bazel C/C++ targets and then include them in Python targets (and eventually in wheels). For pybind11 specifically, `pybind11_bazel` provides rules such as `pybind_extension` to build a Python extension module and notes it can be included as a `data` dependency of `py_*` targets. citeturn13search0turn13search16 + +If your extension depends on third-party C/C++ projects that don’t have Bazel rules, `rules_foreign_cc` provides mechanisms to build external projects using systems like CMake/configure/make inside Bazel’s sandbox. citeturn13search35turn13search1 + +## Monorepos, incremental builds, CI/CD, and security + +### Monorepo workspace layout patterns + +A Bazel monorepo benefits from a small number of “control planes”: + +- Root module and lockfiles (`MODULE.bazel`, `MODULE.bazel.lock`) +- A single location for Python dependency policy (one or a few pip hubs + lockfiles) +- Package-local BUILD files defining `py_library` / `py_binary` / `py_test` targets + +```mermaid +flowchart TB + Root["repo root
MODULE.bazel
MODULE.bazel.lock
.bazelrc"] --> ThirdParty["third_party/python/
requirements.in
requirements_lock.txt
BUILD.bazel"] + Root --> LibA["libs/lib_a/
BUILD.bazel
src/..."] + Root --> Svc1["services/svc1/
BUILD.bazel
app/..."] + Root --> Tools["tools/
linters, generators
py_binary targets"] + ThirdParty --> PipHub["@pip (hub repo)
requirements.bzl macros
wheel repos"] + PipHub --> Svc1 + PipHub --> LibA +``` + +The build performance motivation is canonical: Bazel caches its dependency graph and only reanalyzes what it must, making incremental builds fast when packages haven’t changed. citeturn14search0 + +### Dependency flow in a Bazel 8 Python setup + +```mermaid +flowchart LR + A["pyproject.toml or requirements.in"] --> B["compile_pip_requirements
(pins transitive closure)"] + B --> C["requirements_lock.txt"] + C --> D["pip.parse (Bzlmod)
hub + wheel repos"] + D --> E["@pip//:requirements.bzl
requirement('pkg')"] + E --> F["py_library / py_binary / py_test"] + F --> G["Runfiles env
or venv/site-packages
or OCI image"] +``` + +This matches rules_python guidance: generate a fully pinned transitive lockfile, then install/use those requirements in Bazel. citeturn17view0turn6view0 + +### CI/CD: remote cache, remote execution, and Bazel 8 operational notes + +**Remote caching**: Bazel’s remote caching docs explain that a remote cache lets teams reuse action outputs computed elsewhere, identified by content hashes, reducing redundant compilation and test work. citeturn14search6 + +**Remote execution**: Bazel’s remote execution overview defines remote execution as distributing build and test actions across multiple machines, with benefits including parallel scaling, consistent execution environments, and reuse of outputs across a team; it also notes Bazel uses an open-source gRPC protocol for remote execution and caching. citeturn22search0 + +**Bazel 8+ command reality**: avoid workflows that depend on `bazel sync`; Bazel’s changelog states it has been removed (use `bazel fetch --all`). citeturn10view0 + +### Security, provenance, and license scanning + +**Supply-chain framing (SLSA)**: SLSA is a supply-chain security framework intended to prevent tampering and improve integrity across the build and release chain. citeturn21search0turn21search4 + +**Bazel Central Registry provenance support**: Bazel’s official blog announces attestation support in the Bazel Central Registry (BCR), including updated GitHub Actions for releasing rulesets and presubmit verification of security metadata. citeturn21search1 A related BCR discussion describes requiring SLSA attestations over security-critical files for a module to be considered verified. citeturn21search5 + +**License scanning**: `rules_license` provides Bazel rules and tooling for gathering and managing license information. citeturn1search16turn1search27 + +**SBOM generation**: Bazel has internal tooling for SBOM generation (e.g., `write_sbom.py` consuming “packages used” inputs and locks), citeturn21search10 and Bazel community discussions describe evolving SBOM tooling to parse `MODULE.bazel.lock` and include ecosystems such as PyPI in SPDX output. citeturn21search7 + +**Python vulnerability scanning**: `pip-audit` scans Python environments for dependencies with known vulnerabilities using the Python Packaging Advisory Database. citeturn2search26 + +## Reference configurations, migration checklist, CI snippets, and troubleshooting + +### Example snippets for common scenarios + +The snippets below are intentionally “pattern-first” rather than tied to a single repo; you should adapt versions and repository names to your organization’s pinning/security requirements. + +#### Simple app with third-party deps + +`MODULE.bazel` (Bzlmod-first; one Python version, one pip hub): + +```starlark +module(name = "my_app", version = "0.1.0") + +bazel_dep(name = "rules_python", version = "1.5.0") # example version + +# Python toolchain +python = use_extension("@rules_python//python/extensions:python.bzl", "python") +python.toolchain( + is_default = True, + python_version = "3.11", +) +use_repo(python, "python_3_11") +# Some setups also register toolchains from the generated repo: +# register_toolchains("@python_3_11//:all") + +# PyPI dependencies +pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") +pip.parse( + hub_name = "pip", + python_version = "3.11", + requirements_lock = "//third_party/python:requirements_lock.txt", +) +use_repo(pip, "pip") +``` + +`third_party/python/BUILD.bazel`: + +```starlark +load("@rules_python//python:pip.bzl", "compile_pip_requirements") + +compile_pip_requirements( + name = "requirements", + src = "requirements.in", + requirements_txt = "requirements_lock.txt", +) +``` + +This matches `rules_python` guidance for producing a pinned lockfile with update/test targets. citeturn17view0 + +`app/BUILD.bazel`: + +```starlark +load("@rules_python//python:defs.bzl", "py_binary") +load("@pip//:requirements.bzl", "requirement") + +py_binary( + name = "main", + srcs = ["main.py"], + deps = [ + requirement("requests"), + ], +) +``` + +Notes: + +- `pip.parse` groups dependencies by hub and Python version; it’s designed so the hub name remains stable while the underlying deps can be version-specific. citeturn6view0 + +#### App that depends on a native extension (pybind11 example) + +`MODULE.bazel` add-ons: + +```starlark +bazel_dep(name = "rules_cc", version = "0.0.9") # example +bazel_dep(name = "pybind11_bazel", version = "3.0.0") # example +``` + +`native_ext/BUILD.bazel`: + +```starlark +load("@rules_cc//cc:defs.bzl", "cc_library") +load("@pybind11_bazel//:build_defs.bzl", "pybind_extension") +load("@rules_python//python:defs.bzl", "py_library") + +cc_library( + name = "core", + srcs = ["core.cc"], + hdrs = ["core.h"], +) + +pybind_extension( + name = "core_py", + srcs = ["bindings.cc"], + deps = [":core"], +) + +py_library( + name = "core_pkg", + srcs = ["__init__.py"], + data = [":core_py"], # extension module (.so/.pyd) as runtime data +) +``` + +This aligns with `pybind11_bazel`’s described intent: build a Python extension module and include it as runtime data for `py_*` rules. citeturn13search0turn13search16 + +#### Monorepo with multiple packages and multiple dependency “hubs” + +When different services require incompatible versions of some library, `pip.parse` allows creating separate hubs, but its docs caution that targets from different hubs should not be used together. citeturn6view0 + +`MODULE.bazel` sketch: + +```starlark +# ... python.toolchain config ... + +pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") + +pip.parse( + hub_name = "pip_prod", + python_version = "3.11", + requirements_lock = "//third_party/python:requirements_lock_prod.txt", +) + +pip.parse( + hub_name = "pip_devtools", + python_version = "3.11", + requirements_lock = "//third_party/python:requirements_lock_devtools.txt", +) + +use_repo(pip, "pip_prod", "pip_devtools") +``` + +Service BUILD example: + +```starlark +load("@rules_python//python:defs.bzl", "py_binary") +load("@pip_prod//:requirements.bzl", "requirement") + +py_binary( + name = "svc", + srcs = ["main.py"], + deps = [requirement("fastapi")], +) +``` + +### Migration checklist from older Bazel versions or pip-based workflows + +This checklist focuses on “pip/venv mindset → Bazel inputs mindset” and the Bazel 8 Bzlmod reality. + +- Align on external deps model: migrate from legacy `WORKSPACE` to Bzlmod (`MODULE.bazel`) because WORKSPACE is disabled by default in Bazel 8 and removed in Bazel 9. citeturn21search16turn9search11 +- Pin Python runtime via toolchains (root module should name the Python version explicitly to prevent drift). citeturn11search19turn7search1 +- Replace legacy PyPI integration: + - If using `pip_install`, migrate to `pip_parse` / `pip.parse`; `pip_install` is deprecated/removed in newer rules_python versions. citeturn3search0turn3search12 + - If using `pip_import`, migrate (it’s legacy and had Python-version pitfalls). citeturn3search1turn3search9 +- Introduce a proper lockfile workflow: + - Convert unconstrained `requirements.txt` to `requirements.in` + generated `requirements_lock.txt` via `compile_pip_requirements` (or a compatible external tool), and enforce freshness in CI via the generated test target. citeturn17view0turn11search11 + - If lock generation consults `pyproject.toml`, pin `[build-system]` dependencies to avoid non-hermetic behavior. citeturn17view0turn2search1 + - Consider adding hashes (pip `--require-hashes`) for stronger integrity guarantees. citeturn2search2 +- Commit Bazel lockfiles: + - Commit `MODULE.bazel.lock` (and keep module extension evaluation stable) for reproducibility and faster resolution. citeturn3search38 +- Normalize new-fetch workflows: + - If you used `bazel sync`, switch to `bazel fetch --all` (Bazel removed `bazel sync`). citeturn10view0 +- Decide on native extensions strategy early: + - Prefer wheels when available; otherwise plan for toolchains/headers (`py_cc_toolchain`) or a Bazel-native extension build path (e.g., pybind11). citeturn2search3turn13search14turn13search0 +- Update dev ergonomics deliberately: + - Choose whether to keep a developer-local `.venv` (pure dev tool) or adopt Bazel-created venv/site-packages behavior (`venvs_site_packages`) for closer parity. citeturn20view0turn19search3 + +### CI snippets for GitHub Actions and GitLab + +These are templates; remote cache/execution endpoints, auth, and cache scoping should be customized to your infra/security posture. + +#### GitHub Actions + +(Uses Bazelisk, which Bazel docs recommend for installing/updating Bazel across OSes. citeturn15search14turn15search2) + +```yaml +name: ci +on: + push: + pull_request: + +jobs: + bazel: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Bazelisk + uses: bazelbuild/setup-bazelisk@v3 + # setup-bazelisk exists for pinning/configuring Bazelisk in Actions environments. citeturn15search6 + + - name: Bazel info + run: | + bazelisk version + bazelisk info release + + - name: Build + run: | + bazelisk build //... + + - name: Test + run: | + bazelisk test //... +``` + +To add remote cache / remote execution, incorporate Bazel flags such as `--remote_cache=…` and/or `--remote_executor=…`, which are core parts of Bazel’s remote execution model. citeturn22search0turn14search6 + +#### GitLab CI + +```yaml +stages: [build, test] + +bazel_build: + stage: build + image: ubuntu:22.04 + script: + - ./bazelisk build //... + cache: + key: bazel-cache + paths: + - .cache/bazel + - .bazelisk + +bazel_test: + stage: test + image: ubuntu:22.04 + script: + - ./bazelisk test //... + cache: + key: bazel-cache + paths: + - .cache/bazel + - .bazelisk +``` + +(For remote execution, Bazel’s docs emphasize that remote execution provides parallel scaling, consistent environments, and shared outputs—exactly the reasons many large teams pair Bazel monorepos with RBE. citeturn22search0) + +### Pitfalls and troubleshooting tips + +The points below reflect recurrent failure modes in Bazel 8 Python setups and how the primary sources suggest avoiding them. + +**WORKSPACE “works locally but CI fails”** +In Bazel 8, the legacy WORKSPACE mechanism is disabled by default; builds that implicitly depended on WORKSPACE evaluation can fail suddenly when upgrading or when CI uses newer defaults. citeturn21search16turn9search18 + +**Using deprecated pip rules (`pip_install`, `pip_import`)** +`pip_install` has been deprecated/removed in `rules_python`, and `pip_import` is long superseded; staying on them increases breakage risk when upgrading rulesets. citeturn3search12turn3search1turn3search9 + +**Lock generation non-hermetic due to missing pinned build backend** +If you generate locks from `pyproject.toml`, `rules_python` warns that failing to pin `[build-system]` dependencies can introduce non-hermetic behavior. citeturn17view0turn2search1 + +**Unexpected source builds of sdists (C toolchain required, flaky builds, slow fetches)** +Mitigations include: + +- Prefer wheels-only selection (`pip_whl=only`) when feasible. citeturn20view0 +- Use `download_only=True` in `pip.parse` to avoid building wheels from source and to support cross-platform wheel download. citeturn6view0 + +**Import-path weirdness (namespace packages, auto `__init__.py`)** +`rules_python` exposes a Bazel-compatible flag (`incompatible_default_to_explicit_init_py`) and recommends disabling auto-generation of missing `__init__.py` files to avoid surprising import effects and interference with implicit namespace packages. citeturn20view0 +If you enable implicit namespace package support via pip integration, the pip extension docs note you must set `legacy_create_init=False` (or equivalent) to prevent auto `__init__.py` generation. citeturn18search15turn18search1 + +**“Why does my Python app behave differently under Bazel than in a venv?”** +This can happen with packages relying on `site-packages` layout or `*.pth` behavior; the `rules_python` venv/site-packages effort exists specifically to address these categories. citeturn20view0turn19search3 + +**Debugging in sandboxes** +Use `--verbose_failures` and `--sandbox_debug` to capture the failing command and inspect the sandbox. citeturn14search2turn14search12 +When diagnosing dependency/import issues, use the `rules_python` REPL to reproduce the `py_binary` environment interactively. citeturn18search0 + +**Conflating “dependency hubs”** +`pip.parse` supports multiple hubs, but its docs explicitly warn that hubs are separate dependency resolutions and **targets from different hubs should not be used together**. citeturn6view0 + +**Operational drift: “bazel sync” no longer exists** +Some older guidance for refreshing repository rules uses `bazel sync`; Bazel’s changelog states the command is removed and recommends `bazel fetch --all`. citeturn10view0 diff --git a/jinja2/requirements.in b/jinja2/requirements.in new file mode 100644 index 00000000..4b595220 --- /dev/null +++ b/jinja2/requirements.in @@ -0,0 +1,2 @@ +jinja2==3.1.6 +transformers==5.2.0 diff --git a/jinja2/requirements.lock.txt b/jinja2/requirements.lock.txt new file mode 100644 index 00000000..2f63cf56 --- /dev/null +++ b/jinja2/requirements.lock.txt @@ -0,0 +1,80 @@ +# This file was autogenerated by uv via the following command: +# bazel run //jinja2:requirements.update +annotated-doc==0.0.4 + # via typer +anyio==4.12.1 + # via httpx +certifi==2026.1.4 + # via + # httpcore + # httpx +click==8.3.1 + # via typer +filelock==3.24.3 + # via huggingface-hub +fsspec==2026.2.0 + # via huggingface-hub +h11==0.16.0 + # via httpcore +hf-xet==1.2.0 + # via huggingface-hub +httpcore==1.0.9 + # via httpx +httpx==0.28.1 + # via huggingface-hub +huggingface-hub==1.4.1 + # via + # tokenizers + # transformers +idna==3.11 + # via + # anyio + # httpx +jinja2==3.1.6 + # via -r jinja2/requirements.in +markdown-it-py==4.0.0 + # via rich +markupsafe==3.0.3 + # via jinja2 +mdurl==0.1.2 + # via markdown-it-py +numpy==2.4.2 + # via transformers +packaging==26.0 + # via + # huggingface-hub + # transformers +pygments==2.19.2 + # via rich +pyyaml==6.0.3 + # via + # huggingface-hub + # transformers +regex==2026.1.15 + # via transformers +rich==14.3.2 + # via typer +safetensors==0.7.0 + # via transformers +shellingham==1.5.4 + # via + # huggingface-hub + # typer +tokenizers==0.22.2 + # via transformers +tqdm==4.67.3 + # via + # huggingface-hub + # transformers +transformers==5.2.0 + # via -r jinja2/requirements.in +typer==0.24.0 + # via typer-slim +typer-slim==0.24.0 + # via + # huggingface-hub + # transformers +typing-extensions==4.15.0 + # via + # anyio + # huggingface-hub diff --git a/jinja2/template.zig b/jinja2/template.zig new file mode 100644 index 00000000..0cbcc640 --- /dev/null +++ b/jinja2/template.zig @@ -0,0 +1,783 @@ +const std = @import("std"); +const expr = @import("expr.zig"); +const value_mod = @import("value.zig"); +const Value = value_mod.Value; + +pub const ParseError = error{ + OutOfMemory, + UnexpectedEnd, + UnterminatedTag, + InvalidStatement, + MissingEndTag, +} || expr.ExprError; + +pub const Node = union(enum) { + text: []const u8, + output: *expr.Expr, + if_block: IfBlock, + for_block: ForBlock, + set_var: SetVar, + set_attr: SetAttr, + macro_def: MacroDef, + + pub const IfBlock = struct { + condition: *expr.Expr, + then_nodes: []const Node, + else_nodes: []const Node, + }; + + pub const ForBlock = struct { + var_names: []const []const u8, + iterable: *expr.Expr, + body: []const Node, + }; + + pub const SetVar = struct { + name: []const u8, + value: *expr.Expr, + }; + + pub const SetAttr = struct { + object_name: []const u8, + attr_name: []const u8, + value: *expr.Expr, + }; + + pub const MacroParam = struct { + name: []const u8, + default_expr: ?*expr.Expr, + }; + + pub const MacroDef = struct { + name: []const u8, + params: []const MacroParam, + body: []const Node, + }; +}; + +pub const Template = struct { + name: []const u8, + source: []const u8, + nodes: []const Node, + slot_indices: std.StringHashMap(u32), + + pub fn parse(allocator: std.mem.Allocator, name: []const u8, source: []const u8) ParseError!Template { + var parser = Parser{ + .allocator = allocator, + .source = source, + .index = 0, + .slot_indices = std.StringHashMap(u32).init(allocator), + }; + const parsed = try parser.parseNodes(&[_][]const u8{}); + if (parsed.matched_end != null) return ParseError.InvalidStatement; + return .{ + .name = name, + .source = source, + .nodes = parsed.nodes, + .slot_indices = parser.slot_indices, + }; + } + + pub fn render(self: *const Template, allocator: std.mem.Allocator, ctx_entries: []const Value.Entry) ![]u8 { + var arena = std.heap.ArenaAllocator.init(allocator); + defer arena.deinit(); + const temp_allocator = arena.allocator(); + + var scope = expr.Scope.init(temp_allocator); + defer scope.deinit(); + + for (ctx_entries) |entry| try scope.put(entry.key, entry.value); + + var out = std.ArrayList(u8).empty; + errdefer out.deinit(temp_allocator); + var writer = ArrayListWriter{ .list = &out, .allocator = temp_allocator }; + + var state = RenderState{ + .allocator = temp_allocator, + .scope = &scope, + .macros = std.StringHashMap(Node.MacroDef).init(temp_allocator), + .slot_indices = &self.slot_indices, + .slot_values = try temp_allocator.alloc(Value, self.slot_indices.count()), + }; + defer state.macros.deinit(); + for (state.slot_values) |*v| v.* = .undefined; + + for (ctx_entries) |entry| { + if (self.slot_indices.get(entry.key)) |slot| { + state.slot_values[slot] = entry.value; + } + } + + const runtime = expr.Runtime{ .context = &state, .callFunction = callFunction, .getIdentBySlot = getIdentBySlot }; + try renderNodes(&state, &runtime, self.nodes, &writer); + // Match MiniJinja default: strip a single trailing newline from rendered output. + if (out.items.len > 0 and out.items[out.items.len - 1] == '\n') { + out.items.len -= 1; + } + const temp_output = try out.toOwnedSlice(temp_allocator); + return allocator.dupe(u8, temp_output); + } +}; + +const ArrayListWriter = struct { + list: *std.ArrayList(u8), + allocator: std.mem.Allocator, + + pub fn writeAll(self: *ArrayListWriter, bytes: []const u8) !void { + try self.list.appendSlice(self.allocator, bytes); + } + + pub fn writeByte(self: *ArrayListWriter, b: u8) !void { + try self.list.append(self.allocator, b); + } +}; + +const ParseResult = struct { + nodes: []const Node, + matched_end: ?[]const u8, +}; + +const TagKind = enum { output_tag, stmt_tag, comment_tag }; +const NextTag = struct { pos: usize, kind: TagKind }; + +const Parser = struct { + allocator: std.mem.Allocator, + source: []const u8, + index: usize, + slot_indices: std.StringHashMap(u32), + + fn slotForName(self: *Parser, name: []const u8) ParseError!u32 { + const gop = try self.slot_indices.getOrPut(name); + if (!gop.found_existing) { + gop.value_ptr.* = @intCast(self.slot_indices.count() - 1); + } + return gop.value_ptr.*; + } + + fn bindExprSlots(self: *Parser, ex: *expr.Expr) ParseError!void { + switch (ex.*) { + .ident => |*id| id.slot = try self.slotForName(id.name), + .list_lit => |items| for (items) |item| try self.bindExprSlots(item), + .getattr => |g| try self.bindExprSlots(g.target), + .index => |i| { + try self.bindExprSlots(i.target); + try self.bindExprSlots(i.key); + }, + .slice => |s| { + try self.bindExprSlots(s.target); + if (s.start) |p| try self.bindExprSlots(p); + if (s.end) |p| try self.bindExprSlots(p); + if (s.step) |p| try self.bindExprSlots(p); + }, + .call => |c| { + try self.bindExprSlots(c.callee); + for (c.args) |arg| switch (arg) { + .positional => |p| try self.bindExprSlots(p), + .keyword => |kv| try self.bindExprSlots(kv.value), + }; + }, + .filter => |f| { + try self.bindExprSlots(f.target); + for (f.args) |arg| switch (arg) { + .positional => |p| try self.bindExprSlots(p), + .keyword => |kv| try self.bindExprSlots(kv.value), + }; + }, + .test_expr => |t| { + try self.bindExprSlots(t.target); + for (t.args) |arg| switch (arg) { + .positional => |p| try self.bindExprSlots(p), + .keyword => |kv| try self.bindExprSlots(kv.value), + }; + }, + .cond_expr => |c| { + try self.bindExprSlots(c.then_expr); + try self.bindExprSlots(c.cond); + try self.bindExprSlots(c.else_expr); + }, + .unary_not => |p| try self.bindExprSlots(p), + .unary_neg => |p| try self.bindExprSlots(p), + .binary => |b| { + try self.bindExprSlots(b.left); + try self.bindExprSlots(b.right); + }, + .int_lit, .float_lit, .str_lit, .bool_lit, .null_lit => {}, + } + } + + fn isWs(c: u8) bool { + return c == ' ' or c == '\t' or c == '\n' or c == '\r'; + } + + fn skipWsAfterTag(self: *Parser) void { + while (self.index < self.source.len and isWs(self.source[self.index])) : (self.index += 1) {} + } + + fn findNextTag(self: *const Parser) ?NextTag { + if (self.source.len < 2 or self.index + 1 >= self.source.len) return null; + var i = self.index; + while (i + 1 < self.source.len) : (i += 1) { + if (self.source[i] != '{') continue; + switch (self.source[i + 1]) { + '{' => return .{ .pos = i, .kind = .output_tag }, + '%' => return .{ .pos = i, .kind = .stmt_tag }, + '#' => return .{ .pos = i, .kind = .comment_tag }, + else => continue, + } + } + return null; + } + + fn parseMacroParams(self: *Parser, source: []const u8) ParseError![]const Node.MacroParam { + var parts = std.ArrayList(Node.MacroParam).empty; + errdefer parts.deinit(self.allocator); + + const trimmed = std.mem.trim(u8, source, " \t\n\r"); + if (trimmed.len == 0) return parts.toOwnedSlice(self.allocator); + + var it = std.mem.splitScalar(u8, trimmed, ','); + while (it.next()) |raw_part| { + const part = std.mem.trim(u8, raw_part, " \t\n\r"); + if (part.len == 0) continue; + + if (std.mem.indexOfScalar(u8, part, '=')) |eq_pos| { + const name = std.mem.trim(u8, part[0..eq_pos], " \t\n\r"); + const expr_src = std.mem.trim(u8, part[eq_pos + 1 ..], " \t\n\r"); + if (name.len == 0) return ParseError.InvalidStatement; + const ex = try expr.parseExpr(self.allocator, expr_src); + try self.bindExprSlots(ex); + try parts.append(self.allocator, .{ .name = name, .default_expr = ex }); + } else { + try parts.append(self.allocator, .{ .name = part, .default_expr = null }); + } + } + + return parts.toOwnedSlice(self.allocator); + } + + fn parseIfBlock(self: *Parser, condition: *expr.Expr) ParseError!Node.IfBlock { + const then_part = try self.parseNodes(&[_][]const u8{ "else", "elif", "endif" }); + + var else_nodes: []const Node = &[_]Node{}; + if (then_part.matched_end) |tag| { + if (std.mem.eql(u8, tag, "else")) { + const else_part = try self.parseNodes(&[_][]const u8{"endif"}); + if (else_part.matched_end == null or !std.mem.eql(u8, else_part.matched_end.?, "endif")) { + return ParseError.MissingEndTag; + } + else_nodes = else_part.nodes; + } else if (std.mem.startsWith(u8, tag, "elif ")) { + const elif_cond_src = std.mem.trim(u8, tag[5..], " \t\n\r"); + const elif_cond = try expr.parseExpr(self.allocator, elif_cond_src); + try self.bindExprSlots(elif_cond); + const elif_block = try self.parseIfBlock(elif_cond); + const one = try self.allocator.alloc(Node, 1); + one[0] = .{ .if_block = elif_block }; + else_nodes = one; + } else if (!std.mem.eql(u8, tag, "endif")) { + return ParseError.InvalidStatement; + } + } else return ParseError.MissingEndTag; + + return .{ + .condition = condition, + .then_nodes = then_part.nodes, + .else_nodes = else_nodes, + }; + } + + fn parseNodes(self: *Parser, end_tags: []const []const u8) ParseError!ParseResult { + var nodes = std.ArrayList(Node).empty; + errdefer nodes.deinit(self.allocator); + + while (self.index < self.source.len) { + const next = self.findNextTag(); + if (next == null) { + if (self.index < self.source.len) { + try nodes.append(self.allocator, .{ .text = self.source[self.index..] }); + } + self.index = self.source.len; + break; + } + + const marker_pos = next.?.pos; + if (marker_pos > self.index) { + var text_end = marker_pos; + switch (next.?.kind) { + .output_tag => { + if (marker_pos + 2 < self.source.len and self.source[marker_pos + 2] == '-') { + while (text_end > self.index and isWs(self.source[text_end - 1])) : (text_end -= 1) {} + } + }, + .stmt_tag => { + if (marker_pos + 2 < self.source.len and self.source[marker_pos + 2] == '-') { + while (text_end > self.index and isWs(self.source[text_end - 1])) : (text_end -= 1) {} + } + }, + .comment_tag => { + if (marker_pos + 2 < self.source.len and self.source[marker_pos + 2] == '-') { + while (text_end > self.index and isWs(self.source[text_end - 1])) : (text_end -= 1) {} + } + }, + } + if (text_end > self.index) { + try nodes.append(self.allocator, .{ .text = self.source[self.index..text_end] }); + } + } + + switch (next.?.kind) { + .comment_tag => { + const end_pos = std.mem.indexOfPos(u8, self.source, marker_pos + 2, "#}") orelse return ParseError.UnterminatedTag; + const right_trim = end_pos > marker_pos + 2 and self.source[end_pos - 1] == '-'; + self.index = end_pos + 2; + if (right_trim) self.skipWsAfterTag(); + continue; + }, + .output_tag => { + const end_pos = std.mem.indexOfPos(u8, self.source, marker_pos + 2, "}}") orelse return ParseError.UnterminatedTag; + var start_pos = marker_pos + 2; + if (start_pos < end_pos and self.source[start_pos] == '-') start_pos += 1; + var content_end = end_pos; + const right_trim = content_end > start_pos and self.source[content_end - 1] == '-'; + if (right_trim) content_end -= 1; + const raw = self.source[start_pos..content_end]; + const trimmed = std.mem.trim(u8, raw, " \t\n\r"); + const ex = try expr.parseExpr(self.allocator, trimmed); + try self.bindExprSlots(ex); + try nodes.append(self.allocator, .{ .output = ex }); + self.index = end_pos + 2; + if (right_trim) self.skipWsAfterTag(); + continue; + }, + .stmt_tag => { + const end_pos = std.mem.indexOfPos(u8, self.source, marker_pos + 2, "%}") orelse return ParseError.UnterminatedTag; + var start_pos = marker_pos + 2; + if (start_pos < end_pos and self.source[start_pos] == '-') start_pos += 1; + var content_end = end_pos; + const right_trim = content_end > start_pos and self.source[content_end - 1] == '-'; + if (right_trim) content_end -= 1; + const raw_stmt = self.source[start_pos..content_end]; + const stmt = std.mem.trim(u8, raw_stmt, " \t\n\r"); + self.index = end_pos + 2; + if (right_trim) self.skipWsAfterTag(); + + for (end_tags) |tag| { + if (std.mem.eql(u8, stmt, tag) or + (std.mem.eql(u8, tag, "elif") and std.mem.startsWith(u8, stmt, "elif "))) + { + return .{ .nodes = try nodes.toOwnedSlice(self.allocator), .matched_end = stmt }; + } + } + + if (std.mem.startsWith(u8, stmt, "if ")) { + const cond_src = std.mem.trim(u8, stmt[3..], " \t\n\r"); + const condition = try expr.parseExpr(self.allocator, cond_src); + try self.bindExprSlots(condition); + const if_block = try self.parseIfBlock(condition); + try nodes.append(self.allocator, .{ .if_block = if_block }); + continue; + } + + if (std.mem.startsWith(u8, stmt, "for ")) { + const rhs = std.mem.trim(u8, stmt[4..], " \t\n\r"); + const in_pos = std.mem.indexOf(u8, rhs, " in ") orelse return ParseError.InvalidStatement; + const lhs = std.mem.trim(u8, rhs[0..in_pos], " \t\n\r"); + const iter_src = std.mem.trim(u8, rhs[in_pos + 4 ..], " \t\n\r"); + + var names = std.ArrayList([]const u8).empty; + errdefer names.deinit(self.allocator); + var name_it = std.mem.splitScalar(u8, lhs, ','); + while (name_it.next()) |raw_name| { + const n = std.mem.trim(u8, raw_name, " \t\n\r"); + if (n.len > 0) try names.append(self.allocator, n); + } + if (names.items.len == 0) return ParseError.InvalidStatement; + + const iterable = try expr.parseExpr(self.allocator, iter_src); + try self.bindExprSlots(iterable); + const body = try self.parseNodes(&[_][]const u8{"endfor"}); + if (body.matched_end == null or !std.mem.eql(u8, body.matched_end.?, "endfor")) return ParseError.MissingEndTag; + try nodes.append(self.allocator, .{ .for_block = .{ + .var_names = try names.toOwnedSlice(self.allocator), + .iterable = iterable, + .body = body.nodes, + } }); + continue; + } + + if (std.mem.startsWith(u8, stmt, "set ")) { + const rhs = std.mem.trim(u8, stmt[4..], " \t\n\r"); + const eq_pos = std.mem.indexOfScalar(u8, rhs, '=') orelse return ParseError.InvalidStatement; + const lhs = std.mem.trim(u8, rhs[0..eq_pos], " \t\n\r"); + const value_src = std.mem.trim(u8, rhs[eq_pos + 1 ..], " \t\n\r"); + const value_expr = try expr.parseExpr(self.allocator, value_src); + try self.bindExprSlots(value_expr); + + if (std.mem.indexOfScalar(u8, lhs, '.')) |dot| { + const object_name = std.mem.trim(u8, lhs[0..dot], " \t\n\r"); + const attr_name = std.mem.trim(u8, lhs[dot + 1 ..], " \t\n\r"); + if (object_name.len == 0 or attr_name.len == 0) return ParseError.InvalidStatement; + try nodes.append(self.allocator, .{ .set_attr = .{ .object_name = object_name, .attr_name = attr_name, .value = value_expr } }); + } else { + if (lhs.len == 0) return ParseError.InvalidStatement; + try nodes.append(self.allocator, .{ .set_var = .{ .name = lhs, .value = value_expr } }); + } + continue; + } + + if (std.mem.startsWith(u8, stmt, "macro ")) { + const rest = std.mem.trim(u8, stmt[6..], " \t\n\r"); + const open_idx = std.mem.indexOfScalar(u8, rest, '(') orelse return ParseError.InvalidStatement; + const close_idx = std.mem.lastIndexOfScalar(u8, rest, ')') orelse return ParseError.InvalidStatement; + if (close_idx <= open_idx) return ParseError.InvalidStatement; + const name = std.mem.trim(u8, rest[0..open_idx], " \t\n\r"); + const params_src = rest[open_idx + 1 .. close_idx]; + const params = try self.parseMacroParams(params_src); + const body = try self.parseNodes(&[_][]const u8{"endmacro"}); + if (body.matched_end == null) return ParseError.MissingEndTag; + try nodes.append(self.allocator, .{ .macro_def = .{ .name = name, .params = params, .body = body.nodes } }); + continue; + } + + return ParseError.InvalidStatement; + }, + } + } + + if (end_tags.len > 0) return ParseError.MissingEndTag; + return .{ .nodes = try nodes.toOwnedSlice(self.allocator), .matched_end = null }; + } +}; + +const RenderState = struct { + allocator: std.mem.Allocator, + scope: *expr.Scope, + macros: std.StringHashMap(Node.MacroDef), + slot_indices: *const std.StringHashMap(u32), + slot_values: []Value, + + fn getVar(self: *const RenderState, name: []const u8) ?Value { + if (self.slot_indices.get(name)) |slot| { + const v = self.slot_values[slot]; + if (v == .undefined) return null; + return v; + } + return self.scope.get(name); + } + + fn setVar(self: *RenderState, name: []const u8, value: Value) !void { + if (self.slot_indices.get(name)) |slot| { + self.slot_values[slot] = value; + return; + } + try self.scope.put(name, value); + } + + fn removeVar(self: *RenderState, name: []const u8) bool { + if (self.slot_indices.get(name)) |slot| { + self.slot_values[slot] = .undefined; + return true; + } + return self.scope.remove(name); + } +}; + +fn writeLoopMeta(state: *RenderState, loop_meta_ptr: *value_mod.LoopMeta, idx: usize, len: usize, prev: ?Value, next: ?Value) !void { + loop_meta_ptr.* = .{ + .index0 = @intCast(idx), + .index = @intCast(idx + 1), + .first = idx == 0, + .last = idx + 1 == len, + .previtem = prev orelse .undefined, + .nextitem = next orelse .undefined, + }; + try state.setVar("loop", .{ .loopmeta = loop_meta_ptr }); +} + +fn getIdentBySlot(context: ?*anyopaque, slot: u32, _: []const u8, _: *expr.Scope) ?Value { + const state: *RenderState = @ptrCast(@alignCast(context)); + if (slot >= state.slot_values.len) return null; + return state.slot_values[slot]; +} + +fn callFunction(context: ?*anyopaque, allocator: std.mem.Allocator, name: []const u8, args: []const expr.CallArgValue, scope: *expr.Scope) expr.ExprError!Value { + const state: *RenderState = @ptrCast(@alignCast(context)); + _ = scope; + const macro_def = state.macros.get(name) orelse return .undefined; + + var old_present_stack: [8]bool = undefined; + var old_vals_stack: [8]Value = undefined; + var old_present_heap: ?[]bool = null; + var old_vals_heap: ?[]Value = null; + + const old_present: []bool = if (macro_def.params.len <= old_present_stack.len) + old_present_stack[0..macro_def.params.len] + else blk: { + const buf = try allocator.alloc(bool, macro_def.params.len); + old_present_heap = buf; + break :blk buf; + }; + defer if (old_present_heap) |buf| allocator.free(buf); + + const old_vals: []Value = if (macro_def.params.len <= old_vals_stack.len) + old_vals_stack[0..macro_def.params.len] + else blk: { + const buf = try allocator.alloc(Value, macro_def.params.len); + old_vals_heap = buf; + break :blk buf; + }; + defer if (old_vals_heap) |buf| allocator.free(buf); + + for (macro_def.params, 0..) |param, idx| { + if (state.getVar(param.name)) |v| { + old_present[idx] = true; + old_vals[idx] = v; + } else { + old_present[idx] = false; + old_vals[idx] = .undefined; + } + } + + var has_keywords = false; + for (args) |arg| { + if (arg == .keyword) { + has_keywords = true; + break; + } + } + + var positional_index: usize = 0; + for (macro_def.params) |param| { + var bound: ?Value = null; + + if (has_keywords) { + for (args) |arg| { + switch (arg) { + .keyword => |kv| { + if (std.mem.eql(u8, kv.name, param.name)) { + bound = kv.value; + break; + } + }, + else => {}, + } + } + } + + if (bound == null) { + while (positional_index < args.len) : (positional_index += 1) { + switch (args[positional_index]) { + .positional => |pv| { + bound = pv; + positional_index += 1; + break; + }, + else => {}, + } + } + } + + if (bound == null) { + if (param.default_expr) |def_ex| { + const runtime = expr.Runtime{ + .context = state, + .callFunction = callFunction, + .getIdentBySlot = getIdentBySlot, + }; + bound = try expr.evalExpr(allocator, def_ex, state.scope, &runtime); + } + } + + try state.setVar(param.name, bound orelse .undefined); + } + + var out = std.ArrayList(u8).empty; + errdefer out.deinit(allocator); + var writer = ArrayListWriter{ .list = &out, .allocator = allocator }; + const runtime = expr.Runtime{ + .context = state, + .callFunction = callFunction, + .getIdentBySlot = getIdentBySlot, + }; + try renderNodes(state, &runtime, macro_def.body, &writer); + + for (macro_def.params, 0..) |param, idx| { + if (old_present[idx]) { + try state.setVar(param.name, old_vals[idx]); + } else { + _ = state.removeVar(param.name); + } + } + + return .{ .str = try out.toOwnedSlice(allocator) }; +} + +const NodeFrame = struct { + nodes: []const Node, + index: usize, +}; + +const LoopFrame = struct { + var_names: []const []const u8, + items: []const Value, + idx: usize, + body: []const Node, + old_loop: ?Value, + old_vals: []Value, + old_present: []bool, + loop_meta_ptr: *value_mod.LoopMeta, +}; + +const ExecFrame = union(enum) { + nodes: NodeFrame, + loop: LoopFrame, +}; + +fn freeLoopFrame(state: *RenderState, loop: *const LoopFrame) void { + state.allocator.free(loop.old_vals); + state.allocator.free(loop.old_present); + state.allocator.destroy(loop.loop_meta_ptr); +} + +fn renderNodes(state: *RenderState, runtime: *const expr.Runtime, nodes: []const Node, writer: anytype) !void { + var stack = std.ArrayList(ExecFrame).empty; + defer { + while (stack.pop()) |frame| { + switch (frame) { + .loop => |loop| freeLoopFrame(state, &loop), + .nodes => {}, + } + } + stack.deinit(state.allocator); + } + + try stack.append(state.allocator, .{ .nodes = .{ .nodes = nodes, .index = 0 } }); + + exec: while (stack.items.len > 0) { + const frame = &stack.items[stack.items.len - 1]; + switch (frame.*) { + .nodes => |*nf| { + if (nf.index >= nf.nodes.len) { + _ = stack.pop(); + continue :exec; + } + + const node = nf.nodes[nf.index]; + nf.index += 1; + + switch (node) { + .text => |txt| try writer.writeAll(txt), + .output => |ex| { + try expr.writeExprOutput(state.allocator, ex, state.scope, runtime, writer); + }, + .macro_def => |m| { + try state.macros.put(m.name, m); + }, + .set_var => |sv| { + const v = try expr.evalExpr(state.allocator, sv.value, state.scope, runtime); + try state.setVar(sv.name, v); + }, + .set_attr => |sa| { + const base = state.getVar(sa.object_name) orelse return error.TypeError; + const v = try expr.evalExpr(state.allocator, sa.value, state.scope, runtime); + try base.setAttr(sa.attr_name, v); + }, + .if_block => |ifb| { + const cond = try expr.evalExpr(state.allocator, ifb.condition, state.scope, runtime); + const branch = if (cond.isTruthy()) ifb.then_nodes else ifb.else_nodes; + if (branch.len > 0) { + try stack.append(state.allocator, .{ .nodes = .{ .nodes = branch, .index = 0 } }); + } + }, + .for_block => |loop| { + const iter_val = try expr.evalExpr(state.allocator, loop.iterable, state.scope, runtime); + switch (iter_val) { + .list => |items| { + const old_vals = try state.allocator.alloc(Value, loop.var_names.len); + errdefer state.allocator.free(old_vals); + const old_present = try state.allocator.alloc(bool, loop.var_names.len); + errdefer state.allocator.free(old_present); + + for (loop.var_names, 0..) |vn, idx| { + if (state.getVar(vn)) |v| { + old_present[idx] = true; + old_vals[idx] = v; + } else { + old_present[idx] = false; + old_vals[idx] = .undefined; + } + } + + const loop_meta_ptr = try state.allocator.create(value_mod.LoopMeta); + errdefer state.allocator.destroy(loop_meta_ptr); + + try stack.append(state.allocator, .{ .loop = .{ + .var_names = loop.var_names, + .items = items, + .idx = 0, + .body = loop.body, + .old_loop = state.getVar("loop"), + .old_vals = old_vals, + .old_present = old_present, + .loop_meta_ptr = loop_meta_ptr, + } }); + }, + else => {}, + } + }, + } + continue :exec; + }, + .loop => |*lf| { + if (lf.idx >= lf.items.len) { + if (lf.old_loop) |v| { + try state.setVar("loop", v); + } else { + _ = state.removeVar("loop"); + } + + for (lf.var_names, 0..) |vn, idx| { + if (lf.old_present[idx]) { + try state.setVar(vn, lf.old_vals[idx]); + } else { + _ = state.removeVar(vn); + } + } + + const done = stack.pop().?; + switch (done) { + .loop => |loop_done| freeLoopFrame(state, &loop_done), + .nodes => unreachable, + } + continue :exec; + } + + const idx = lf.idx; + const item = lf.items[idx]; + lf.idx += 1; + + if (lf.var_names.len == 1) { + try state.setVar(lf.var_names[0], item); + } else switch (item) { + .list => |pair| { + var n: usize = 0; + while (n < lf.var_names.len and n < pair.len) : (n += 1) { + try state.setVar(lf.var_names[n], pair[n]); + } + }, + else => { + try state.setVar(lf.var_names[0], item); + }, + } + + const prev = if (idx > 0) lf.items[idx - 1] else null; + const next = if (idx + 1 < lf.items.len) lf.items[idx + 1] else null; + try writeLoopMeta(state, lf.loop_meta_ptr, idx, lf.items.len, prev, next); + if (lf.body.len > 0) { + try stack.append(state.allocator, .{ .nodes = .{ .nodes = lf.body, .index = 0 } }); + } + continue :exec; + }, + } + } +} diff --git a/jinja2/test_cases/.gitignore b/jinja2/test_cases/.gitignore new file mode 100644 index 00000000..ef08bddd --- /dev/null +++ b/jinja2/test_cases/.gitignore @@ -0,0 +1,2 @@ +/chat/expected +/multimodal/expected diff --git a/jinja2/test_cases/__pycache__/generate_expected.cpython-313.pyc b/jinja2/test_cases/__pycache__/generate_expected.cpython-313.pyc new file mode 100644 index 00000000..24e7bb38 Binary files /dev/null and b/jinja2/test_cases/__pycache__/generate_expected.cpython-313.pyc differ diff --git a/jinja2/test_cases/__pycache__/render_expected.cpython-313.pyc b/jinja2/test_cases/__pycache__/render_expected.cpython-313.pyc new file mode 100644 index 00000000..6588bfb7 Binary files /dev/null and b/jinja2/test_cases/__pycache__/render_expected.cpython-313.pyc differ diff --git a/jinja2/test_cases/basic/01_for_simple.json b/jinja2/test_cases/basic/01_for_simple.json new file mode 100644 index 00000000..2dbade1b --- /dev/null +++ b/jinja2/test_cases/basic/01_for_simple.json @@ -0,0 +1,13 @@ +{ + "template": "{% for item in seq %}{{ item }}{% endfor %}", + "data": { + "seq": [ + 0, + 1, + 2, + 3, + 4 + ] + }, + "expected": "01234" +} diff --git a/jinja2/test_cases/basic/02_for_else.json b/jinja2/test_cases/basic/02_for_else.json new file mode 100644 index 00000000..ea063567 --- /dev/null +++ b/jinja2/test_cases/basic/02_for_else.json @@ -0,0 +1,7 @@ +{ + "template": "{% for item in seq %}XXX{% else %}...{% endfor %}", + "data": { + "seq": [] + }, + "expected": "..." +} diff --git a/jinja2/test_cases/basic/03_for_loop_context_vars.json b/jinja2/test_cases/basic/03_for_loop_context_vars.json new file mode 100644 index 00000000..69de37f3 --- /dev/null +++ b/jinja2/test_cases/basic/03_for_loop_context_vars.json @@ -0,0 +1,10 @@ +{ + "template": "{% for item in seq %}[{{ loop.index }}|{{ loop.index0 }}|{{ loop.revindex }}|{{ loop.revindex0 }}|{{ loop.first }}|{{ loop.last }}|{{ loop.length }}]{% endfor %}", + "data": { + "seq": [ + 42, + 24 + ] + }, + "expected": "[1|0|2|1|True|False|2][2|1|1|0|False|True|2]" +} diff --git a/jinja2/test_cases/basic/04_if_else.json b/jinja2/test_cases/basic/04_if_else.json new file mode 100644 index 00000000..3a15e99d --- /dev/null +++ b/jinja2/test_cases/basic/04_if_else.json @@ -0,0 +1,5 @@ +{ + "template": "{% if false %}XXX{% else %}...{% endif %}", + "data": {}, + "expected": "..." +} diff --git a/jinja2/test_cases/basic/05_if_elif_complete.json b/jinja2/test_cases/basic/05_if_elif_complete.json new file mode 100644 index 00000000..6e1c08fc --- /dev/null +++ b/jinja2/test_cases/basic/05_if_elif_complete.json @@ -0,0 +1,10 @@ +{ + "template": "{% if a %}A{% elif b %}B{% elif c == d %}C{% else %}D{% endif %}", + "data": { + "a": 0, + "b": false, + "c": 42, + "d": 42.0 + }, + "expected": "C" +} diff --git a/jinja2/test_cases/basic/06_set_simple.json b/jinja2/test_cases/basic/06_set_simple.json new file mode 100644 index 00000000..a3792d65 --- /dev/null +++ b/jinja2/test_cases/basic/06_set_simple.json @@ -0,0 +1,5 @@ +{ + "template": "{% set foo = 1 %}{{ foo }}", + "data": {}, + "expected": "1" +} diff --git a/jinja2/test_cases/basic/07_set_loop_scoping.json b/jinja2/test_cases/basic/07_set_loop_scoping.json new file mode 100644 index 00000000..7794821a --- /dev/null +++ b/jinja2/test_cases/basic/07_set_loop_scoping.json @@ -0,0 +1,12 @@ +{ + "template": "{% for item in seq %}{{ x }}{% set x = item %}{{ x }}{% endfor %}", + "data": { + "x": 0, + "seq": [ + 1, + 2, + 3 + ] + }, + "expected": "010203" +} diff --git a/jinja2/test_cases/basic/08_filter_default.json b/jinja2/test_cases/basic/08_filter_default.json new file mode 100644 index 00000000..80d14946 --- /dev/null +++ b/jinja2/test_cases/basic/08_filter_default.json @@ -0,0 +1,8 @@ +{ + "template": "{{ missing|default('no') }}|{{ false|default('no') }}|{{ false|default('no', true) }}|{{ given|default('no') }}", + "data": { + "given": "yes", + "false": false + }, + "expected": "no|False|no|yes" +} diff --git a/jinja2/test_cases/basic/09_filter_join.json b/jinja2/test_cases/basic/09_filter_join.json new file mode 100644 index 00000000..95f7cfa8 --- /dev/null +++ b/jinja2/test_cases/basic/09_filter_join.json @@ -0,0 +1,5 @@ +{ + "template": "{{ [1, 2, 3]|join('|') }}", + "data": {}, + "expected": "1|2|3" +} diff --git a/jinja2/test_cases/basic/10_filter_length_lower_upper.json b/jinja2/test_cases/basic/10_filter_length_lower_upper.json new file mode 100644 index 00000000..3909c263 --- /dev/null +++ b/jinja2/test_cases/basic/10_filter_length_lower_upper.json @@ -0,0 +1,10 @@ +{ + "source_tests": [ + "tests/test_filters.py::TestFilter::test_length", + "tests/test_filters.py::TestFilter::test_lower", + "tests/test_filters.py::TestFilter::test_upper" + ], + "template": "{{ 'hello world'|length }}|{{ 'FOO'|lower }}|{{ 'foo'|upper }}", + "data": {}, + "expected": "11|foo|FOO" +} diff --git a/jinja2/test_cases/basic/11_filter_escape.json b/jinja2/test_cases/basic/11_filter_escape.json new file mode 100644 index 00000000..e1218c5f --- /dev/null +++ b/jinja2/test_cases/basic/11_filter_escape.json @@ -0,0 +1,5 @@ +{ + "template": "{{ '<\">&'|escape }}", + "data": {}, + "expected": "<">&" +} diff --git a/jinja2/test_cases/basic/12_test_defined.json b/jinja2/test_cases/basic/12_test_defined.json new file mode 100644 index 00000000..6ecf5d0f --- /dev/null +++ b/jinja2/test_cases/basic/12_test_defined.json @@ -0,0 +1,5 @@ +{ + "template": "{{ missing is defined }}|{{ true is defined }}", + "data": {}, + "expected": "False|True" +} diff --git a/jinja2/test_cases/basic/13_test_even_odd.json b/jinja2/test_cases/basic/13_test_even_odd.json new file mode 100644 index 00000000..068875a4 --- /dev/null +++ b/jinja2/test_cases/basic/13_test_even_odd.json @@ -0,0 +1,9 @@ +{ + "source_tests": [ + "tests/test_tests.py::TestTestsCase::test_even", + "tests/test_tests.py::TestTestsCase::test_odd" + ], + "template": "{{ 1 is even }}|{{ 2 is even }}|{{ 1 is odd }}|{{ 2 is odd }}", + "data": {}, + "expected": "False|True|True|False" +} diff --git a/jinja2/test_cases/basic/14_test_in.json b/jinja2/test_cases/basic/14_test_in.json new file mode 100644 index 00000000..92bc50c3 --- /dev/null +++ b/jinja2/test_cases/basic/14_test_in.json @@ -0,0 +1,5 @@ +{ + "template": "{{ 'o' is in 'foo' }}|{{ 3 is in [1, 2] }}|{{ 'foo' is in {'foo': 1} }}", + "data": {}, + "expected": "True|False|True" +} diff --git a/jinja2/test_cases/basic/15_test_eq.json b/jinja2/test_cases/basic/15_test_eq.json new file mode 100644 index 00000000..24b2fc61 --- /dev/null +++ b/jinja2/test_cases/basic/15_test_eq.json @@ -0,0 +1,8 @@ +{ + "template": "{{ foo is eq 12 }}|{{ bar is eq 'baz' }}|{{ bar is eq foo }}", + "data": { + "foo": 12, + "bar": "baz" + }, + "expected": "True|True|False" +} diff --git a/jinja2/test_cases/basic/16_item_lookup.json b/jinja2/test_cases/basic/16_item_lookup.json new file mode 100644 index 00000000..f0a163ca --- /dev/null +++ b/jinja2/test_cases/basic/16_item_lookup.json @@ -0,0 +1,9 @@ +{ + "template": "{{ foo['items'] }}", + "data": { + "foo": { + "items": 42 + } + }, + "expected": "42" +} diff --git a/jinja2/test_cases/basic/17_expression_arithmetic.json b/jinja2/test_cases/basic/17_expression_arithmetic.json new file mode 100644 index 00000000..d653098b --- /dev/null +++ b/jinja2/test_cases/basic/17_expression_arithmetic.json @@ -0,0 +1,7 @@ +{ + "template": "{{ 42 + foo }}", + "data": { + "foo": 42 + }, + "expected": "84" +} diff --git a/jinja2/test_cases/basic/18_raw_block.json b/jinja2/test_cases/basic/18_raw_block.json new file mode 100644 index 00000000..a5153ed6 --- /dev/null +++ b/jinja2/test_cases/basic/18_raw_block.json @@ -0,0 +1,7 @@ +{ + "template": "{% raw %}foo{% endraw %}|{%raw%}{{ bar }}|{% baz %}{% endraw %}", + "data": { + "bar": "SHOULD_NOT_RENDER" + }, + "expected": "foo|{{ bar }}|{% baz %}" +} diff --git a/jinja2/test_cases/basic/19_raw_whitespace_trim.json b/jinja2/test_cases/basic/19_raw_whitespace_trim.json new file mode 100644 index 00000000..67569e5c --- /dev/null +++ b/jinja2/test_cases/basic/19_raw_whitespace_trim.json @@ -0,0 +1,5 @@ +{ + "template": "1 {%- raw -%} 2 {%- endraw -%} 3", + "data": {}, + "expected": "123" +} diff --git a/jinja2/test_cases/basic/20_comment_and_output.json b/jinja2/test_cases/basic/20_comment_and_output.json new file mode 100644 index 00000000..16b36999 --- /dev/null +++ b/jinja2/test_cases/basic/20_comment_and_output.json @@ -0,0 +1,7 @@ +{ + "template": "{# hidden comment #}Hello {{ name }}", + "data": { + "name": "Jinja" + }, + "expected": "Hello Jinja" +} diff --git a/jinja2/test_cases/chat/data/01_text_minimal.json b/jinja2/test_cases/chat/data/01_text_minimal.json new file mode 100644 index 00000000..707750f6 --- /dev/null +++ b/jinja2/test_cases/chat/data/01_text_minimal.json @@ -0,0 +1,15 @@ +{ + "messages": [ + { + "role": "user", + "content": "Hello! Give me a one-line summary of Jinja templates.", + "from": "human", + "value": "Hello! Give me a one-line summary of Jinja templates." + } + ], + "tools": [], + "documents": [], + "add_generation_prompt": true, + "bos_token": "", + "eos_token": "" +} diff --git a/jinja2/test_cases/chat/data/02_system_user_generation_prompt.json b/jinja2/test_cases/chat/data/02_system_user_generation_prompt.json new file mode 100644 index 00000000..8c60af1f --- /dev/null +++ b/jinja2/test_cases/chat/data/02_system_user_generation_prompt.json @@ -0,0 +1,21 @@ +{ + "messages": [ + { + "role": "system", + "content": "You are a concise assistant.", + "from": "system", + "value": "You are a concise assistant." + }, + { + "role": "user", + "content": "List 3 template engines.", + "from": "human", + "value": "List 3 template engines." + } + ], + "tools": [], + "documents": [], + "add_generation_prompt": true, + "bos_token": "", + "eos_token": "" +} diff --git a/jinja2/test_cases/chat/data/03_multiturn_roles.json b/jinja2/test_cases/chat/data/03_multiturn_roles.json new file mode 100644 index 00000000..4aa5f3ba --- /dev/null +++ b/jinja2/test_cases/chat/data/03_multiturn_roles.json @@ -0,0 +1,33 @@ +{ + "messages": [ + { + "role": "system", + "content": "Answer with short bullets.", + "from": "system", + "value": "Answer with short bullets." + }, + { + "role": "user", + "content": "What is Jinja?", + "from": "human", + "value": "What is Jinja?" + }, + { + "role": "assistant", + "content": "Jinja is a text templating engine used in Python ecosystems.", + "from": "gpt", + "value": "Jinja is a text templating engine used in Python ecosystems." + }, + { + "role": "user", + "content": "Name one advantage.", + "from": "human", + "value": "Name one advantage." + } + ], + "tools": [], + "documents": [], + "add_generation_prompt": true, + "bos_token": "", + "eos_token": "" +} diff --git a/jinja2/test_cases/chat/data/04_tool_call_single.json b/jinja2/test_cases/chat/data/04_tool_call_single.json new file mode 100644 index 00000000..29a9cab0 --- /dev/null +++ b/jinja2/test_cases/chat/data/04_tool_call_single.json @@ -0,0 +1,62 @@ +{ + "messages": [ + { + "role": "system", + "content": "Use tools when needed.", + "from": "system", + "value": "Use tools when needed." + }, + { + "role": "user", + "content": "What is the weather in Paris?", + "from": "human", + "value": "What is the weather in Paris?" + }, + { + "role": "assistant", + "content": "I will check that.", + "tool_calls": [ + { + "id": "wxparis01", + "type": "function", + "function": { + "name": "get_weather", + "arguments": { + "city": "Paris", + "units": "c" + } + } + } + ], + "from": "gpt", + "value": "I will check that." + } + ], + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather by city", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string" + }, + "units": { + "type": "string" + } + }, + "required": [ + "city" + ] + } + } + } + ], + "documents": [], + "add_generation_prompt": false, + "bos_token": "", + "eos_token": "" +} diff --git a/jinja2/test_cases/chat/data/05_tool_call_multi.json b/jinja2/test_cases/chat/data/05_tool_call_multi.json new file mode 100644 index 00000000..1784e710 --- /dev/null +++ b/jinja2/test_cases/chat/data/05_tool_call_multi.json @@ -0,0 +1,68 @@ +{ + "messages": [ + { + "role": "system", + "content": "You may call multiple tools in sequence.", + "from": "system", + "value": "You may call multiple tools in sequence." + }, + { + "role": "user", + "content": "Compare weather in Paris and Tokyo.", + "from": "human", + "value": "Compare weather in Paris and Tokyo." + }, + { + "role": "assistant", + "content": "Checking both cities.", + "tool_calls": [ + { + "id": "paris123a", + "type": "function", + "function": { + "name": "get_weather", + "arguments": { + "city": "Paris" + } + } + }, + { + "id": "tokyo456b", + "type": "function", + "function": { + "name": "get_weather", + "arguments": { + "city": "Tokyo" + } + } + } + ], + "from": "gpt", + "value": "Checking both cities." + } + ], + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather by city", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ] + } + } + } + ], + "documents": [], + "add_generation_prompt": false, + "bos_token": "", + "eos_token": "" +} diff --git a/jinja2/test_cases/chat/data/06_tool_response_roundtrip.json b/jinja2/test_cases/chat/data/06_tool_response_roundtrip.json new file mode 100644 index 00000000..a889fba6 --- /dev/null +++ b/jinja2/test_cases/chat/data/06_tool_response_roundtrip.json @@ -0,0 +1,63 @@ +{ + "messages": [ + { + "role": "system", + "content": "Use tools and then provide the final answer.", + "from": "system", + "value": "Use tools and then provide the final answer." + }, + { + "role": "user", + "content": "What is 23 * 19?", + "from": "human", + "value": "What is 23 * 19?" + }, + { + "role": "assistant", + "content": "Calling calculator.", + "tool_calls": [ + { + "id": "mulcall01", + "type": "function", + "function": { + "name": "multiply", + "arguments": { + "a": 23, + "b": 19 + } + } + } + ], + "from": "gpt", + "value": "Calling calculator." + } + ], + "tools": [ + { + "type": "function", + "function": { + "name": "multiply", + "description": "Multiply two numbers", + "parameters": { + "type": "object", + "properties": { + "a": { + "type": "number" + }, + "b": { + "type": "number" + } + }, + "required": [ + "a", + "b" + ] + } + } + } + ], + "documents": [], + "add_generation_prompt": true, + "bos_token": "", + "eos_token": "" +} diff --git a/jinja2/test_cases/chat/data/07_reasoning_content.json b/jinja2/test_cases/chat/data/07_reasoning_content.json new file mode 100644 index 00000000..9e6061e9 --- /dev/null +++ b/jinja2/test_cases/chat/data/07_reasoning_content.json @@ -0,0 +1,30 @@ +{ + "messages": [ + { + "role": "system", + "content": "Think step-by-step, then answer.", + "from": "system", + "value": "Think step-by-step, then answer." + }, + { + "role": "user", + "content": "What is the next prime after 17?", + "from": "human", + "value": "What is the next prime after 17?" + }, + { + "role": "assistant", + "content": "The next prime after 17 is 19.", + "reasoning_content": "18 is divisible by 2, 19 is prime.", + "from": "gpt", + "value": "The next prime after 17 is 19." + } + ], + "tools": [], + "documents": [], + "enable_thinking": true, + "clear_thinking": false, + "add_generation_prompt": true, + "bos_token": "", + "eos_token": "" +} diff --git a/jinja2/test_cases/chat/data/10_prefill_assistant_continue.json b/jinja2/test_cases/chat/data/10_prefill_assistant_continue.json new file mode 100644 index 00000000..e2464bf7 --- /dev/null +++ b/jinja2/test_cases/chat/data/10_prefill_assistant_continue.json @@ -0,0 +1,28 @@ +{ + "messages": [ + { + "role": "system", + "content": "Return strict JSON.", + "from": "system", + "value": "Return strict JSON." + }, + { + "role": "user", + "content": "Give me a JSON object with status and value.", + "from": "human", + "value": "Give me a JSON object with status and value." + }, + { + "role": "assistant", + "content": "{\"status\": \"ok\", \"value\": ", + "from": "gpt", + "value": "{\"status\": \"ok\", \"value\": " + } + ], + "tools": [], + "documents": [], + "continue_final_message": true, + "add_generation_prompt": false, + "bos_token": "", + "eos_token": "" +} diff --git a/jinja2/test_cases/chat/data/11_rag_documents.json b/jinja2/test_cases/chat/data/11_rag_documents.json new file mode 100644 index 00000000..eba9da60 --- /dev/null +++ b/jinja2/test_cases/chat/data/11_rag_documents.json @@ -0,0 +1,30 @@ +{ + "messages": [ + { + "role": "system", + "content": "Use provided documents when relevant.", + "from": "system", + "value": "Use provided documents when relevant." + }, + { + "role": "user", + "content": "Summarize retrieval-augmented generation in one line.", + "from": "human", + "value": "Summarize retrieval-augmented generation in one line." + } + ], + "tools": [], + "documents": [ + { + "title": "RAG", + "text": "RAG combines retrieval of external knowledge with generation." + }, + { + "title": "Jinja", + "text": "Jinja templates can format prompts before tokenization." + } + ], + "add_generation_prompt": true, + "bos_token": "", + "eos_token": "" +} diff --git a/jinja2/test_cases/chat/data/12_custom_controls.json b/jinja2/test_cases/chat/data/12_custom_controls.json new file mode 100644 index 00000000..176a0668 --- /dev/null +++ b/jinja2/test_cases/chat/data/12_custom_controls.json @@ -0,0 +1,70 @@ +{ + "messages": [ + { + "role": "system", + "content": "Follow tool and date metadata.", + "from": "system", + "value": "Follow tool and date metadata." + }, + { + "role": "user", + "content": "What can you do today?", + "from": "human", + "value": "What can you do today?" + } + ], + "tools": [ + { + "type": "function", + "function": { + "name": "search_docs", + "description": "Search docs by query", + "parameters": { + "type": "object", + "properties": { + "query": { + "type": "string" + } + }, + "required": [ + "query" + ] + } + } + } + ], + "custom_tools": [ + { + "type": "function", + "function": { + "name": "search_docs", + "description": "Search docs by query", + "parameters": { + "type": "object", + "properties": { + "query": { + "type": "string" + } + }, + "required": [ + "query" + ] + } + } + } + ], + "documents": [], + "builtin_tools": [ + "browser", + "python" + ], + "model_identity": "Helpful assistant v1", + "reasoning_effort": "medium", + "date_string": "18 Feb 2026", + "tools_in_user_message": false, + "enable_thinking": true, + "clear_thinking": false, + "add_generation_prompt": true, + "bos_token": "", + "eos_token": "" +} diff --git a/jinja2/test_cases/chat/data/13_legacy_function_role.json b/jinja2/test_cases/chat/data/13_legacy_function_role.json new file mode 100644 index 00000000..4783f50b --- /dev/null +++ b/jinja2/test_cases/chat/data/13_legacy_function_role.json @@ -0,0 +1,58 @@ +{ + "messages": [ + { + "role": "system", + "content": "Support legacy function-role traces.", + "from": "system", + "value": "Support legacy function-role traces." + }, + { + "role": "user", + "content": "Get stock price for ACME.", + "from": "human", + "value": "Get stock price for ACME." + }, + { + "role": "assistant", + "content": "Calling a function.", + "tool_calls": [ + { + "id": "stock001a", + "type": "function", + "function": { + "name": "get_stock", + "arguments": { + "ticker": "ACME" + } + } + } + ], + "from": "gpt", + "value": "Calling a function." + } + ], + "tools": [ + { + "type": "function", + "function": { + "name": "get_stock", + "description": "Get latest stock price", + "parameters": { + "type": "object", + "properties": { + "ticker": { + "type": "string" + } + }, + "required": [ + "ticker" + ] + } + } + } + ], + "documents": [], + "add_generation_prompt": true, + "bos_token": "", + "eos_token": "" +} diff --git a/jinja2/test_cases/chat/data/14_tool_call_arguments_as_json_string.json b/jinja2/test_cases/chat/data/14_tool_call_arguments_as_json_string.json new file mode 100644 index 00000000..2d11c9c0 --- /dev/null +++ b/jinja2/test_cases/chat/data/14_tool_call_arguments_as_json_string.json @@ -0,0 +1,63 @@ +{ + "messages": [ + { + "role": "system", + "content": "Tool arguments may be objects or JSON strings.", + "from": "system", + "value": "Tool arguments may be objects or JSON strings." + }, + { + "role": "user", + "content": "Book a flight to SFO for Friday.", + "from": "human", + "value": "Book a flight to SFO for Friday." + }, + { + "role": "assistant", + "content": "Attempting booking.", + "tool_calls": [ + { + "id": "flight01a", + "type": "function", + "function": { + "name": "book_flight", + "arguments": { + "destination": "SFO", + "day": "Friday" + } + } + } + ], + "from": "gpt", + "value": "Attempting booking." + } + ], + "tools": [ + { + "type": "function", + "function": { + "name": "book_flight", + "description": "Book a flight", + "parameters": { + "type": "object", + "properties": { + "destination": { + "type": "string" + }, + "day": { + "type": "string" + } + }, + "required": [ + "destination", + "day" + ] + } + } + } + ], + "documents": [], + "add_generation_prompt": false, + "bos_token": "", + "eos_token": "" +} diff --git a/jinja2/test_cases/chat/data/15_messages_with_developer_role.json b/jinja2/test_cases/chat/data/15_messages_with_developer_role.json new file mode 100644 index 00000000..e00feb32 --- /dev/null +++ b/jinja2/test_cases/chat/data/15_messages_with_developer_role.json @@ -0,0 +1,33 @@ +{ + "messages": [ + { + "role": "system", + "content": "You are a general-purpose assistant. Always answer in markdown with one heading.", + "from": "system", + "value": "You are a general-purpose assistant. Always answer in markdown with one heading." + }, + { + "role": "user", + "content": "Explain template inheritance in one short paragraph.", + "from": "human", + "value": "Explain template inheritance in one short paragraph." + }, + { + "role": "assistant", + "content": "## Template Inheritance\nTemplate inheritance lets child templates reuse and override parts of a base layout, which keeps shared structure centralized and page-specific content isolated.", + "from": "gpt", + "value": "## Template Inheritance\nTemplate inheritance lets child templates reuse and override parts of a base layout, which keeps shared structure centralized and page-specific content isolated." + }, + { + "role": "user", + "content": "Now give a tiny example with a base and child block.", + "from": "human", + "value": "Now give a tiny example with a base and child block." + } + ], + "tools": [], + "documents": [], + "add_generation_prompt": true, + "bos_token": "", + "eos_token": "" +} diff --git a/jinja2/test_cases/chat/data/README.md b/jinja2/test_cases/chat/data/README.md new file mode 100644 index 00000000..7b8ddcee --- /dev/null +++ b/jinja2/test_cases/chat/data/README.md @@ -0,0 +1,19 @@ +# Chat Data Fixtures + +These files are reusable **input data fixtures** for chat-template rendering. + +- Each file is a JSON object that can be passed as the template context. +- The intent is to avoid repeating large `messages`/`tools` payloads in every generated test case. +- Later, generate concrete test cases as a Cartesian product of: + - `chat/templates/*.jinja` + - `chat/data/*.json` + +Suggested generated test-case shape (filename references only): + +```json +{ + "name": "__", + "template_file": "chat/templates/", + "data_file": "chat/data/" +} +``` diff --git a/jinja2/test_cases/chat/templates/ab_LiquidAI__LFM2-700M__chat_template.jinja b/jinja2/test_cases/chat/templates/ab_LiquidAI__LFM2-700M__chat_template.jinja new file mode 100644 index 00000000..ed66b6de --- /dev/null +++ b/jinja2/test_cases/chat/templates/ab_LiquidAI__LFM2-700M__chat_template.jinja @@ -0,0 +1,37 @@ +{{- bos_token -}} +{%- set system_prompt = "" -%} +{%- set ns = namespace(system_prompt="") -%} +{%- if messages[0]["role"] == "system" -%} + {%- set ns.system_prompt = messages[0]["content"] -%} + {%- set messages = messages[1:] -%} +{%- endif -%} +{%- if tools -%} + {%- set ns.system_prompt = ns.system_prompt + ("\n" if ns.system_prompt else "") + "List of tools: <|tool_list_start|>[" -%} + {%- for tool in tools -%} + {%- if tool is not string -%} + {%- set tool = tool | tojson -%} + {%- endif -%} + {%- set ns.system_prompt = ns.system_prompt + tool -%} + {%- if not loop.last -%} + {%- set ns.system_prompt = ns.system_prompt + ", " -%} + {%- endif -%} + {%- endfor -%} + {%- set ns.system_prompt = ns.system_prompt + "]<|tool_list_end|>" -%} +{%- endif -%} +{%- if ns.system_prompt -%} + {{- "<|im_start|>system\n" + ns.system_prompt + "<|im_end|>\n" -}} +{%- endif -%} +{%- for message in messages -%} + {{- "<|im_start|>" + message["role"] + "\n" -}} + {%- set content = message["content"] -%} + {%- if content is not string -%} + {%- set content = content | tojson -%} + {%- endif -%} + {%- if message["role"] == "tool" -%} + {%- set content = "<|tool_response_start|>" + content + "<|tool_response_end|>" -%} + {%- endif -%} + {{- content + "<|im_end|>\n" -}} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{- "<|im_start|>assistant\n" -}} +{%- endif -%} \ No newline at end of file diff --git a/jinja2/test_cases/chat/templates/ac_Qwen__Qwen-Image-Edit__tokenizer__chat_template.jinja b/jinja2/test_cases/chat/templates/ac_Qwen__Qwen-Image-Edit__tokenizer__chat_template.jinja new file mode 100644 index 00000000..28028c05 --- /dev/null +++ b/jinja2/test_cases/chat/templates/ac_Qwen__Qwen-Image-Edit__tokenizer__chat_template.jinja @@ -0,0 +1,54 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] }} + {%- else %} + {{- 'You are a helpful assistant.' }} + {%- endif %} + {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role }} + {%- if message.content %} + {{- '\n' + message.content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/jinja2/test_cases/chat/templates/ad_Trelis__Qwen1.5-function-calling-chat-template__tokenizer_config.json.jinja b/jinja2/test_cases/chat/templates/ad_Trelis__Qwen1.5-function-calling-chat-template__tokenizer_config.json.jinja new file mode 100644 index 00000000..8381ff0c --- /dev/null +++ b/jinja2/test_cases/chat/templates/ad_Trelis__Qwen1.5-function-calling-chat-template__tokenizer_config.json.jinja @@ -0,0 +1,21 @@ +{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}<|im_start|>system +You are a helpful assistant<|im_end|> +{% endif %}{% if message['role'] == 'function_metadata' %}<|im_start|>tools +You have access to the following functions. Use them only if required: + +{{ message['content'] }} + +To call a function, respond with a JSON object in this format: +{ + "name": "function_name", + "arguments": { + "argument1": "value1", + "argument2": "value2" + } +} +Respond with a JSON object only if you wish to make a function call. Any other response will be treated as a regular query.When making a function call, provide only the JSON object, nothing else. Make one function call at a time. After the function call, wait for the response. +Only make use of the functions if they assist in providing the user with an answer. Otherwise, answer without making a function call.{% elif message['role'] == 'function_response' %}<|im_start|>function_response +Here is the response to the function call. If helpful, use it to respond to the user's question:{{ message['content'] }}{% else %}{{'<|im_start|>' + message['role'] + ' +' + message['content']}}{% endif %}{% if (loop.last and add_generation_prompt) or not loop.last %}{{ '<|im_end|>' + ' +'}}{% endif %}{% endfor %}{% if add_generation_prompt and messages[-1]['role'] != 'assistant' %}{{ '<|im_start|>assistant +' }}{% endif %} \ No newline at end of file diff --git a/jinja2/test_cases/chat/templates/ae_abdulmannan-01__llama3.1-8b-lora-finetuned-openscholar-dataset-llama-chat-template__tokenizer_config.json.jinja b/jinja2/test_cases/chat/templates/ae_abdulmannan-01__llama3.1-8b-lora-finetuned-openscholar-dataset-llama-chat-template__tokenizer_config.json.jinja new file mode 100644 index 00000000..33089ace --- /dev/null +++ b/jinja2/test_cases/chat/templates/ae_abdulmannan-01__llama3.1-8b-lora-finetuned-openscholar-dataset-llama-chat-template__tokenizer_config.json.jinja @@ -0,0 +1,109 @@ +{{- bos_token }} +{%- if custom_tools is defined %} + {%- set tools = custom_tools %} +{%- endif %} +{%- if not tools_in_user_message is defined %} + {%- set tools_in_user_message = true %} +{%- endif %} +{%- if not date_string is defined %} + {%- set date_string = "26 Jul 2024" %} +{%- endif %} +{%- if not tools is defined %} + {%- set tools = none %} +{%- endif %} + +{#- This block extracts the system message, so we can slot it into the right place. #} +{%- if messages[0]['role'] == 'system' %} + {%- set system_message = messages[0]['content']|trim %} + {%- set messages = messages[1:] %} +{%- else %} + {%- set system_message = "" %} +{%- endif %} + +{#- System message + builtin tools #} +{{- "<|start_header_id|>system<|end_header_id|>\n\n" }} +{%- if builtin_tools is defined or tools is not none %} + {{- "Environment: ipython\n" }} +{%- endif %} +{%- if builtin_tools is defined %} + {{- "Tools: " + builtin_tools | reject('equalto', 'code_interpreter') | join(", ") + "\n\n"}} +{%- endif %} +{{- "Cutting Knowledge Date: December 2023\n" }} +{{- "Today Date: " + date_string + "\n\n" }} +{%- if tools is not none and not tools_in_user_message %} + {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }} + {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }} + {{- "Do not use variables.\n\n" }} + {%- for t in tools %} + {{- t | tojson(indent=4) }} + {{- "\n\n" }} + {%- endfor %} +{%- endif %} +{{- system_message }} +{{- "<|eot_id|>" }} + +{#- Custom tools are passed in a user message with some extra guidance #} +{%- if tools_in_user_message and not tools is none %} + {#- Extract the first user message so we can plug it in here #} + {%- if messages | length != 0 %} + {%- set first_user_message = messages[0]['content']|trim %} + {%- set messages = messages[1:] %} + {%- else %} + {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }} +{%- endif %} + {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}} + {{- "Given the following functions, please respond with a JSON for a function call " }} + {{- "with its proper arguments that best answers the given prompt.\n\n" }} + {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }} + {{- "Do not use variables.\n\n" }} + {%- for t in tools %} + {{- t | tojson(indent=4) }} + {{- "\n\n" }} + {%- endfor %} + {{- first_user_message + "<|eot_id|>"}} +{%- endif %} + +{%- for message in messages %} + {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %} + {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }} + {%- elif 'tool_calls' in message %} + {%- if not message.tool_calls|length == 1 %} + {{- raise_exception("This model only supports single tool-calls at once!") }} + {%- endif %} + {%- set tool_call = message.tool_calls[0].function %} + {%- if builtin_tools is defined and tool_call.name in builtin_tools %} + {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}} + {{- "<|python_tag|>" + tool_call.name + ".call(" }} + {%- for arg_name, arg_val in tool_call.arguments | items %} + {{- arg_name + '="' + arg_val + '"' }} + {%- if not loop.last %} + {{- ", " }} + {%- endif %} + {%- endfor %} + {{- ")" }} + {%- else %} + {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}} + {{- '{"name": "' + tool_call.name + '", ' }} + {{- '"parameters": ' }} + {{- tool_call.arguments | tojson }} + {{- "}" }} + {%- endif %} + {%- if builtin_tools is defined %} + {#- This means we're in ipython mode #} + {{- "<|eom_id|>" }} + {%- else %} + {{- "<|eot_id|>" }} + {%- endif %} + {%- elif message.role == "tool" or message.role == "ipython" %} + {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }} + {%- if message.content is mapping or message.content is iterable %} + {{- message.content | tojson }} + {%- else %} + {{- message.content }} + {%- endif %} + {{- "<|eot_id|>" }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }} +{%- endif %} diff --git a/jinja2/test_cases/chat/templates/af_blackhole33__llama-chat_template_10000-sample__tokenizer_config.json.jinja b/jinja2/test_cases/chat/templates/af_blackhole33__llama-chat_template_10000-sample__tokenizer_config.json.jinja new file mode 100644 index 00000000..8d549174 --- /dev/null +++ b/jinja2/test_cases/chat/templates/af_blackhole33__llama-chat_template_10000-sample__tokenizer_config.json.jinja @@ -0,0 +1,8 @@ +{% for message in messages %}{% if message['from'] == 'human' %}{{'<|im_start|>user +' + message['value'] + '<|im_end|> +'}}{% elif message['from'] == 'gpt' %}{{'<|im_start|>assistant +' + message['value'] + '<|im_end|> +' }}{% else %}{{ '<|im_start|>system +' + message['value'] + '<|im_end|> +' }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant +' }}{% endif %} \ No newline at end of file diff --git a/jinja2/test_cases/chat/templates/ag_deepseek-ai__DeepSeek-R1__tokenizer_config.json.jinja b/jinja2/test_cases/chat/templates/ag_deepseek-ai__DeepSeek-R1__tokenizer_config.json.jinja new file mode 100644 index 00000000..1f77c0e7 --- /dev/null +++ b/jinja2/test_cases/chat/templates/ag_deepseek-ai__DeepSeek-R1__tokenizer_config.json.jinja @@ -0,0 +1 @@ +{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set ns = namespace(is_first=false, is_tool=false, is_output_first=true, system_prompt='', is_first_sp=true) %}{%- for message in messages %}{%- if message['role'] == 'system' %}{%- if ns.is_first_sp %}{% set ns.system_prompt = ns.system_prompt + message['content'] %}{% set ns.is_first_sp = false %}{%- else %}{% set ns.system_prompt = ns.system_prompt + '\n\n' + message['content'] %}{%- endif %}{%- endif %}{%- endfor %}{{ bos_token }}{{ ns.system_prompt }}{%- for message in messages %}{%- if message['role'] == 'user' %}{%- set ns.is_tool = false -%}{{'<|User|>' + message['content']}}{%- endif %}{%- if message['role'] == 'assistant' and 'tool_calls' in message %}{%- set ns.is_tool = false -%}{%- for tool in message['tool_calls'] %}{%- if not ns.is_first %}{%- if message['content'] is none %}{{'<|Assistant|><|tool▁calls▁begin|><|tool▁call▁begin|>' + tool['type'] + '<|tool▁sep|>' + tool['function']['name'] + '\n' + '```json' + '\n' + tool['function']['arguments'] + '\n' + '```' + '<|tool▁call▁end|>'}}{%- else %}{{'<|Assistant|>' + message['content'] + '<|tool▁calls▁begin|><|tool▁call▁begin|>' + tool['type'] + '<|tool▁sep|>' + tool['function']['name'] + '\n' + '```json' + '\n' + tool['function']['arguments'] + '\n' + '```' + '<|tool▁call▁end|>'}}{%- endif %}{%- set ns.is_first = true -%}{%- else %}{{'\n' + '<|tool▁call▁begin|>' + tool['type'] + '<|tool▁sep|>' + tool['function']['name'] + '\n' + '```json' + '\n' + tool['function']['arguments'] + '\n' + '```' + '<|tool▁call▁end|>'}}{%- endif %}{%- endfor %}{{'<|tool▁calls▁end|><|end▁of▁sentence|>'}}{%- endif %}{%- if message['role'] == 'assistant' and 'tool_calls' not in message %}{%- if ns.is_tool %}{{'<|tool▁outputs▁end|>' + message['content'] + '<|end▁of▁sentence|>'}}{%- set ns.is_tool = false -%}{%- else %}{% set content = message['content'] %}{% if '' in content %}{% set content = content.split('')[-1] %}{% endif %}{{'<|Assistant|>' + content + '<|end▁of▁sentence|>'}}{%- endif %}{%- endif %}{%- if message['role'] == 'tool' %}{%- set ns.is_tool = true -%}{%- if ns.is_output_first %}{{'<|tool▁outputs▁begin|><|tool▁output▁begin|>' + message['content'] + '<|tool▁output▁end|>'}}{%- set ns.is_output_first = false %}{%- else %}{{'<|tool▁output▁begin|>' + message['content'] + '<|tool▁output▁end|>'}}{%- endif %}{%- endif %}{%- endfor -%}{% if ns.is_tool %}{{'<|tool▁outputs▁end|>'}}{% endif %}{% if add_generation_prompt and not ns.is_tool %}{{'<|Assistant|>\n'}}{% endif %} \ No newline at end of file diff --git a/jinja2/test_cases/chat/templates/ah_janhq__Jan-v1-4B__chat_template.jinja b/jinja2/test_cases/chat/templates/ah_janhq__Jan-v1-4B__chat_template.jinja new file mode 100644 index 00000000..05bf3627 --- /dev/null +++ b/jinja2/test_cases/chat/templates/ah_janhq__Jan-v1-4B__chat_template.jinja @@ -0,0 +1,87 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "In this environment you have access to a set of tools you can use to answer the user's question. You can use one tool per message, and will receive the result of that tool use in the user's response. You use tools step-by-step to accomplish a given task, with each tool use informed by the result of the previous tool use.\n\nTool Use Rules\nHere are the rules you should always follow to solve your task:\n1. Always use the right arguments for the tools. Never use variable names as the action arguments, use the value instead.\n2. Call a tool only when needed: do not call the search agent if you do not need information, try to solve the task yourself.\n3. If no tool call is needed, just answer the question directly.\n4. Never re-do a tool call that you previously did with the exact same parameters.\n5. For tool use, MARK SURE use XML tag format as shown in the examples above. Do not use any other format.\nNow Begin! If you solve the task correctly, you will receive a reward of $1,000,000.\n\n" }} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} \ No newline at end of file diff --git a/jinja2/test_cases/chat/templates/ai_jiaop24__Mixtral-8x22B-Instruct-v0.1-better_chat_template__tokenizer_config.json.jinja b/jinja2/test_cases/chat/templates/ai_jiaop24__Mixtral-8x22B-Instruct-v0.1-better_chat_template__tokenizer_config.json.jinja new file mode 100644 index 00000000..2f6e0a72 --- /dev/null +++ b/jinja2/test_cases/chat/templates/ai_jiaop24__Mixtral-8x22B-Instruct-v0.1-better_chat_template__tokenizer_config.json.jinja @@ -0,0 +1,89 @@ +{%- if messages[0]["role"] == "system" %} + {%- set system_message = messages[0]["content"] %} + {%- set loop_messages = messages[1:] %} +{%- else %} + {%- set loop_messages = messages %} +{%- endif %} +{%- if not tools is defined %} + {%- set tools = none %} +{%- endif %} +{%- set user_messages = loop_messages | selectattr("role", "equalto", "user") | list %} + +{#- This block checks for alternating user/assistant messages, skipping tool calling messages #} +{%- set ns = namespace() %} +{%- set ns.index = 0 %} +{%- for message in loop_messages %} + {%- if not (message.role == "tool" or message.role == "tool_results" or (message.tool_calls is defined and message.tool_calls is not none)) %} + {%- if (message["role"] == "user") != (ns.index % 2 == 0) %} + {{- raise_exception("After the optional system message, conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif %} + {%- set ns.index = ns.index + 1 %} + {%- endif %} +{%- endfor %} + +{{- bos_token }} +{%- for message in loop_messages %} + {%- if message["role"] == "user" %} + {%- if tools is not none and (message == user_messages[-1]) %} + {{- "[AVAILABLE_TOOLS] [" }} + {%- for tool in tools %} + {%- set tool = tool.function %} + {{- '{"type": "function", "function": {' }} + {%- for key, val in tool.items() if key != "return" %} + {%- if val is string %} + {{- '"' + key + '": "' + val + '"' }} + {%- else %} + {{- '"' + key + '": ' + val|tojson }} + {%- endif %} + {%- if not loop.last %} + {{- ", " }} + {%- endif %} + {%- endfor %} + {{- "}}" }} + {%- if not loop.last %} + {{- ", " }} + {%- else %} + {{- "]" }} + {%- endif %} + {%- endfor %} + {{- "[/AVAILABLE_TOOLS]" }} + {%- endif %} + {%- if (loop.last or loop.index == loop.length - 1) and system_message is defined %} + {{- "[INST] " + system_message + " + +" + message["content"] + "[/INST]" }} + {%- else %} + {{- "[INST] " + message["content"] + "[/INST]" }} + {%- endif %} + {%- elif message.tool_calls is defined and message.tool_calls is not none %} + {{- "[TOOL_CALLS] [" }} + {%- for tool_call in message.tool_calls %} + {%- set out = tool_call.function|tojson %} + {{- out[:-1] }} + {%- if not tool_call.id is defined or tool_call.id|length != 9 %} + {{- raise_exception("Tool call IDs should be alphanumeric strings with length 9!") }} + {%- endif %} + {{- ', "id": "' + tool_call.id + '"}' }} + {%- if not loop.last %} + {{- ", " }} + {%- else %} + {{- "]" + eos_token }} + {%- endif %} + {%- endfor %} + {%- elif message["role"] == "assistant" %} + {{- " " + message["content"]|trim + ("" if loop.last else eos_token)}} + {%- elif message["role"] == "tool_results" or message["role"] == "tool" %} + {%- if message.content is defined and message.content.content is defined %} + {%- set content = message.content.content %} + {%- else %} + {%- set content = message.content %} + {%- endif %} + {{- '[TOOL_RESULTS] {"content": ' + content|string + ", " }} + {%- if not message.tool_call_id is defined or message.tool_call_id|length != 9 %} + {{- raise_exception("Tool call IDs should be alphanumeric strings with length 9!") }} + {%- endif %} + {{- '"call_id": "' + message.tool_call_id + '"}[/TOOL_RESULTS]' }} + {%- else %} + {{- raise_exception("Only user and assistant roles are supported, with the exception of an initial optional system message!") }} + {%- endif %} +{%- endfor %} \ No newline at end of file diff --git a/jinja2/test_cases/chat/templates/aj_keithdrexel__unsloth-llama-3.2-3b-tldr-unsloth-chat-template__tokenizer_config.json.jinja b/jinja2/test_cases/chat/templates/aj_keithdrexel__unsloth-llama-3.2-3b-tldr-unsloth-chat-template__tokenizer_config.json.jinja new file mode 100644 index 00000000..9fdc70c5 --- /dev/null +++ b/jinja2/test_cases/chat/templates/aj_keithdrexel__unsloth-llama-3.2-3b-tldr-unsloth-chat-template__tokenizer_config.json.jinja @@ -0,0 +1 @@ +{{ bos_token }}{% for message in messages %}{{' ' if not loop.first else ''}}{{message['content']}}{% if loop.last and not add_generation_prompt %}{{ eos_token }}{% endif %}{% endfor %} \ No newline at end of file diff --git a/jinja2/test_cases/chat/templates/ak_lewtun__qwen2.5-1.5b-distill-default-chat-template__tokenizer_config.json.jinja b/jinja2/test_cases/chat/templates/ak_lewtun__qwen2.5-1.5b-distill-default-chat-template__tokenizer_config.json.jinja new file mode 100644 index 00000000..bdf7919a --- /dev/null +++ b/jinja2/test_cases/chat/templates/ak_lewtun__qwen2.5-1.5b-distill-default-chat-template__tokenizer_config.json.jinja @@ -0,0 +1,54 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] }} + {%- else %} + {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }} + {%- endif %} + {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role }} + {%- if message.content %} + {{- '\n' + message.content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/jinja2/test_cases/chat/templates/al_lewtun__smollm2-distill-default-chat-template__tokenizer_config.json.jinja b/jinja2/test_cases/chat/templates/al_lewtun__smollm2-distill-default-chat-template__tokenizer_config.json.jinja new file mode 100644 index 00000000..a3c992f0 --- /dev/null +++ b/jinja2/test_cases/chat/templates/al_lewtun__smollm2-distill-default-chat-template__tokenizer_config.json.jinja @@ -0,0 +1,6 @@ +{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system +You are a helpful AI assistant named SmolLM, trained by Hugging Face<|im_end|> +' }}{% endif %}{{'<|im_start|>' + message['role'] + ' +' + message['content'] + '<|im_end|>' + ' +'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant +' }}{% endif %} \ No newline at end of file diff --git a/jinja2/test_cases/chat/templates/am_mkshing__opt-tokenizer-with-chat-template__tokenizer_config.json.jinja b/jinja2/test_cases/chat/templates/am_mkshing__opt-tokenizer-with-chat-template__tokenizer_config.json.jinja new file mode 100644 index 00000000..057a3952 --- /dev/null +++ b/jinja2/test_cases/chat/templates/am_mkshing__opt-tokenizer-with-chat-template__tokenizer_config.json.jinja @@ -0,0 +1,4 @@ +{% for message in messages %}{{'<|im_start|>' + message['role'] + ' +' + message['content'] + '<|im_end|>' + ' +'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant +' }}{% endif %} \ No newline at end of file diff --git a/jinja2/test_cases/chat/templates/an_mssfj__gemma-2-9b-bnb-4bit-chat-template__tokenizer_config.json.jinja b/jinja2/test_cases/chat/templates/an_mssfj__gemma-2-9b-bnb-4bit-chat-template__tokenizer_config.json.jinja new file mode 100644 index 00000000..f4bee869 --- /dev/null +++ b/jinja2/test_cases/chat/templates/an_mssfj__gemma-2-9b-bnb-4bit-chat-template__tokenizer_config.json.jinja @@ -0,0 +1,4 @@ +{%- for message in messages %} +{{ message.role }}: {{ message.content }} +{%- endfor %}{% if add_generation_prompt %} +assistant: {% endif %} \ No newline at end of file diff --git a/jinja2/test_cases/chat/templates/ao_newmindai__Llama-3.1-8B-Instruct-capital-chat-template__tokenizer_config.json.jinja b/jinja2/test_cases/chat/templates/ao_newmindai__Llama-3.1-8B-Instruct-capital-chat-template__tokenizer_config.json.jinja new file mode 100644 index 00000000..386cbb0e --- /dev/null +++ b/jinja2/test_cases/chat/templates/ao_newmindai__Llama-3.1-8B-Instruct-capital-chat-template__tokenizer_config.json.jinja @@ -0,0 +1,5 @@ +{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|> + +'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|> + +' }}{% endif %} \ No newline at end of file diff --git a/jinja2/test_cases/chat/templates/ap_openai__gpt-oss-20b__chat_template.jinja b/jinja2/test_cases/chat/templates/ap_openai__gpt-oss-20b__chat_template.jinja new file mode 100644 index 00000000..dc7bb119 --- /dev/null +++ b/jinja2/test_cases/chat/templates/ap_openai__gpt-oss-20b__chat_template.jinja @@ -0,0 +1,331 @@ +{#- + In addition to the normal inputs of `messages` and `tools`, this template also accepts the + following kwargs: + - "builtin_tools": A list, can contain "browser" and/or "python". + - "model_identity": A string that optionally describes the model identity. + - "reasoning_effort": A string that describes the reasoning effort, defaults to "medium". + #} + +{#- Tool Definition Rendering ============================================== #} +{%- macro render_typescript_type(param_spec, required_params, is_nullable=false) -%} + {%- if param_spec.type == "array" -%} + {%- if param_spec['items'] -%} + {%- if param_spec['items']['type'] == "string" -%} + {{- "string[]" }} + {%- elif param_spec['items']['type'] == "number" -%} + {{- "number[]" }} + {%- elif param_spec['items']['type'] == "integer" -%} + {{- "number[]" }} + {%- elif param_spec['items']['type'] == "boolean" -%} + {{- "boolean[]" }} + {%- else -%} + {%- set inner_type = render_typescript_type(param_spec['items'], required_params) -%} + {%- if inner_type == "object | object" or inner_type|length > 50 -%} + {{- "any[]" }} + {%- else -%} + {{- inner_type + "[]" }} + {%- endif -%} + {%- endif -%} + {%- if param_spec.nullable -%} + {{- " | null" }} + {%- endif -%} + {%- else -%} + {{- "any[]" }} + {%- if param_spec.nullable -%} + {{- " | null" }} + {%- endif -%} + {%- endif -%} + {%- elif param_spec.type is defined and param_spec.type is iterable and param_spec.type is not string and param_spec.type is not mapping and param_spec.type[0] is defined -%} + {#- Handle array of types like ["object", "object"] from Union[dict, list] #} + {%- if param_spec.type | length > 1 -%} + {{- param_spec.type | join(" | ") }} + {%- else -%} + {{- param_spec.type[0] }} + {%- endif -%} + {%- elif param_spec.oneOf -%} + {#- Handle oneOf schemas - check for complex unions and fallback to any #} + {%- set has_object_variants = false -%} + {%- for variant in param_spec.oneOf -%} + {%- if variant.type == "object" -%} + {%- set has_object_variants = true -%} + {%- endif -%} + {%- endfor -%} + {%- if has_object_variants and param_spec.oneOf|length > 1 -%} + {{- "any" }} + {%- else -%} + {%- for variant in param_spec.oneOf -%} + {{- render_typescript_type(variant, required_params) -}} + {%- if variant.description %} + {{- "// " + variant.description }} + {%- endif -%} + {%- if variant.default is defined %} + {{ "// default: " + variant.default|tojson }} + {%- endif -%} + {%- if not loop.last %} + {{- " | " }} + {% endif -%} + {%- endfor -%} + {%- endif -%} + {%- elif param_spec.type == "string" -%} + {%- if param_spec.enum -%} + {{- '"' + param_spec.enum|join('" | "') + '"' -}} + {%- else -%} + {{- "string" }} + {%- if param_spec.nullable %} + {{- " | null" }} + {%- endif -%} + {%- endif -%} + {%- elif param_spec.type == "number" -%} + {{- "number" }} + {%- elif param_spec.type == "integer" -%} + {{- "number" }} + {%- elif param_spec.type == "boolean" -%} + {{- "boolean" }} + + {%- elif param_spec.type == "object" -%} + {%- if param_spec.properties -%} + {{- "{\n" }} + {%- for prop_name, prop_spec in param_spec.properties.items() -%} + {{- prop_name -}} + {%- if prop_name not in (param_spec.required or []) -%} + {{- "?" }} + {%- endif -%} + {{- ": " }} + {{ render_typescript_type(prop_spec, param_spec.required or []) }} + {%- if not loop.last -%} + {{-", " }} + {%- endif -%} + {%- endfor -%} + {{- "}" }} + {%- else -%} + {{- "object" }} + {%- endif -%} + {%- else -%} + {{- "any" }} + {%- endif -%} +{%- endmacro -%} + +{%- macro render_tool_namespace(namespace_name, tools) -%} + {{- "## " + namespace_name + "\n\n" }} + {{- "namespace " + namespace_name + " {\n\n" }} + {%- for tool in tools %} + {%- set tool = tool.function %} + {{- "// " + tool.description + "\n" }} + {{- "type "+ tool.name + " = " }} + {%- if tool.parameters and tool.parameters.properties %} + {{- "(_: {\n" }} + {%- for param_name, param_spec in tool.parameters.properties.items() %} + {%- if param_spec.description %} + {{- "// " + param_spec.description + "\n" }} + {%- endif %} + {{- param_name }} + {%- if param_name not in (tool.parameters.required or []) -%} + {{- "?" }} + {%- endif -%} + {{- ": " }} + {{- render_typescript_type(param_spec, tool.parameters.required or []) }} + {%- if param_spec.default is defined -%} + {%- if param_spec.enum %} + {{- ", // default: " + param_spec.default }} + {%- elif param_spec.oneOf %} + {{- "// default: " + param_spec.default }} + {%- else %} + {{- ", // default: " + param_spec.default|tojson }} + {%- endif -%} + {%- endif -%} + {%- if not loop.last %} + {{- ",\n" }} + {%- else %} + {{- ",\n" }} + {%- endif -%} + {%- endfor %} + {{- "}) => any;\n\n" }} + {%- else -%} + {{- "() => any;\n\n" }} + {%- endif -%} + {%- endfor %} + {{- "} // namespace " + namespace_name }} +{%- endmacro -%} + +{%- macro render_builtin_tools(browser_tool, python_tool) -%} + {%- if browser_tool %} + {{- "## browser\n\n" }} + {{- "// Tool for browsing.\n" }} + {{- "// The `cursor` appears in brackets before each browsing display: `[{cursor}]`.\n" }} + {{- "// Cite information from the tool using the following format:\n" }} + {{- "// `【{cursor}†L{line_start}(-L{line_end})?】`, for example: `【6†L9-L11】` or `【8†L3】`.\n" }} + {{- "// Do not quote more than 10 words directly from the tool output.\n" }} + {{- "// sources=web (default: web)\n" }} + {{- "namespace browser {\n\n" }} + {{- "// Searches for information related to `query` and displays `topn` results.\n" }} + {{- "type search = (_: {\n" }} + {{- "query: string,\n" }} + {{- "topn?: number, // default: 10\n" }} + {{- "source?: string,\n" }} + {{- "}) => any;\n\n" }} + {{- "// Opens the link `id` from the page indicated by `cursor` starting at line number `loc`, showing `num_lines` lines.\n" }} + {{- "// Valid link ids are displayed with the formatting: `【{id}†.*】`.\n" }} + {{- "// If `cursor` is not provided, the most recent page is implied.\n" }} + {{- "// If `id` is a string, it is treated as a fully qualified URL associated with `source`.\n" }} + {{- "// If `loc` is not provided, the viewport will be positioned at the beginning of the document or centered on the most relevant passage, if available.\n" }} + {{- "// Use this function without `id` to scroll to a new location of an opened page.\n" }} + {{- "type open = (_: {\n" }} + {{- "id?: number | string, // default: -1\n" }} + {{- "cursor?: number, // default: -1\n" }} + {{- "loc?: number, // default: -1\n" }} + {{- "num_lines?: number, // default: -1\n" }} + {{- "view_source?: boolean, // default: false\n" }} + {{- "source?: string,\n" }} + {{- "}) => any;\n\n" }} + {{- "// Finds exact matches of `pattern` in the current page, or the page given by `cursor`.\n" }} + {{- "type find = (_: {\n" }} + {{- "pattern: string,\n" }} + {{- "cursor?: number, // default: -1\n" }} + {{- "}) => any;\n\n" }} + {{- "} // namespace browser\n\n" }} + {%- endif -%} + + {%- if python_tool %} + {{- "## python\n\n" }} + {{- "Use this tool to execute Python code in your chain of thought. The code will not be shown to the user. This tool should be used for internal reasoning, but not for code that is intended to be visible to the user (e.g. when creating plots, tables, or files).\n\n" }} + {{- "When you send a message containing Python code to python, it will be executed in a stateful Jupyter notebook environment. python will respond with the output of the execution or time out after 120.0 seconds. The drive at '/mnt/data' can be used to save and persist user files. Internet access for this session is UNKNOWN. Depends on the cluster.\n\n" }} + {%- endif -%} +{%- endmacro -%} + +{#- System Message Construction ============================================ #} +{%- macro build_system_message() -%} + {%- if model_identity is not defined %} + {%- set model_identity = "You are ChatGPT, a large language model trained by OpenAI." %} + {%- endif %} + {{- model_identity + "\n" }} + {{- "Knowledge cutoff: 2024-06\n" }} + {{- "Current date: " + strftime_now("%Y-%m-%d") + "\n\n" }} + {%- if reasoning_effort is not defined %} + {%- set reasoning_effort = "medium" %} + {%- endif %} + {{- "Reasoning: " + reasoning_effort + "\n\n" }} + {%- if builtin_tools %} + {{- "# Tools\n\n" }} + {%- set available_builtin_tools = namespace(browser=false, python=false) %} + {%- for tool in builtin_tools %} + {%- if tool == "browser" %} + {%- set available_builtin_tools.browser = true %} + {%- elif tool == "python" %} + {%- set available_builtin_tools.python = true %} + {%- endif %} + {%- endfor %} + {{- render_builtin_tools(available_builtin_tools.browser, available_builtin_tools.python) }} + {%- endif -%} + {{- "# Valid channels: analysis, commentary, final. Channel must be included for every message." }} + {%- if tools -%} + {{- "\nCalls to these tools must go to the commentary channel: 'functions'." }} + {%- endif -%} +{%- endmacro -%} + +{#- Main Template Logic ================================================= #} +{#- Set defaults #} + +{#- Render system message #} +{{- "<|start|>system<|message|>" }} +{{- build_system_message() }} +{{- "<|end|>" }} + +{#- Extract developer message #} +{%- if messages[0].role == "developer" or messages[0].role == "system" %} + {%- set developer_message = messages[0].content %} + {%- set loop_messages = messages[1:] %} +{%- else %} + {%- set developer_message = "" %} + {%- set loop_messages = messages %} +{%- endif %} + +{#- Render developer message #} +{%- if developer_message or tools %} + {{- "<|start|>developer<|message|>" }} + {%- if developer_message %} + {{- "# Instructions\n\n" }} + {{- developer_message }} + {{- "\n\n" }} + {%- endif %} + {%- if tools -%} + {{- "# Tools\n\n" }} + {{- render_tool_namespace("functions", tools) }} + {%- endif -%} + {{- "<|end|>" }} +{%- endif %} + +{#- Render messages #} +{%- set last_tool_call = namespace(name=none) %} +{%- for message in loop_messages -%} + {#- At this point only assistant/user/tool messages should remain #} + {%- if message.role == 'assistant' -%} + {#- Checks to ensure the messages are being passed in the format we expect #} + {%- if "content" in message %} + {%- if "<|channel|>analysis<|message|>" in message.content or "<|channel|>final<|message|>" in message.content %} + {{- raise_exception("You have passed a message containing <|channel|> tags in the content field. Instead of doing this, you should pass analysis messages (the string between '<|message|>' and '<|end|>') in the 'thinking' field, and final messages (the string between '<|message|>' and '<|end|>') in the 'content' field.") }} + {%- endif %} + {%- endif %} + {%- if "thinking" in message %} + {%- if "<|channel|>analysis<|message|>" in message.thinking or "<|channel|>final<|message|>" in message.thinking %} + {{- raise_exception("You have passed a message containing <|channel|> tags in the thinking field. Instead of doing this, you should pass analysis messages (the string between '<|message|>' and '<|end|>') in the 'thinking' field, and final messages (the string between '<|message|>' and '<|end|>') in the 'content' field.") }} + {%- endif %} + {%- endif %} + {%- if "tool_calls" in message %} + {#- We need very careful handling here - we want to drop the tool call analysis message if the model #} + {#- has output a later <|final|> message, but otherwise we want to retain it. This is the only case #} + {#- when we render CoT/analysis messages in inference. #} + {%- set future_final_message = namespace(found=false) %} + {%- for future_message in loop_messages[loop.index:] %} + {%- if future_message.role == 'assistant' and "tool_calls" not in future_message %} + {%- set future_final_message.found = true %} + {%- endif %} + {%- endfor %} + {#- We assume max 1 tool call per message, and so we infer the tool call name #} + {#- in "tool" messages from the most recent assistant tool call name #} + {%- set tool_call = message.tool_calls[0] %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if message.content and message.thinking %} + {{- raise_exception("Cannot pass both content and thinking in an assistant message with tool calls! Put the analysis message in one or the other, but not both.") }} + {%- elif message.content and not future_final_message.found %} + {{- "<|start|>assistant<|channel|>analysis<|message|>" + message.content + "<|end|>" }} + {%- elif message.thinking and not future_final_message.found %} + {{- "<|start|>assistant<|channel|>analysis<|message|>" + message.thinking + "<|end|>" }} + {%- endif %} + {{- "<|start|>assistant to=" }} + {{- "functions." + tool_call.name + "<|channel|>commentary " }} + {{- (tool_call.content_type if tool_call.content_type is defined else "json") + "<|message|>" }} + {{- tool_call.arguments|tojson }} + {{- "<|call|>" }} + {%- set last_tool_call.name = tool_call.name %} + {%- elif loop.last and not add_generation_prompt %} + {#- Only render the CoT if the final turn is an assistant turn and add_generation_prompt is false #} + {#- This is a situation that should only occur in training, never in inference. #} + {%- if "thinking" in message %} + {{- "<|start|>assistant<|channel|>analysis<|message|>" + message.thinking + "<|end|>" }} + {%- endif %} + {#- <|return|> indicates the end of generation, but <|end|> does not #} + {#- <|return|> should never be an input to the model, but we include it as the final token #} + {#- when training, so the model learns to emit it. #} + {{- "<|start|>assistant<|channel|>final<|message|>" + message.content + "<|return|>" }} + {%- else %} + {#- CoT is dropped during all previous turns, so we never render it for inference #} + {{- "<|start|>assistant<|channel|>final<|message|>" + message.content + "<|end|>" }} + {%- set last_tool_call.name = none %} + {%- endif %} + {%- elif message.role == 'tool' -%} + {%- if last_tool_call.name is none %} + {{- raise_exception("Message has tool role, but there was no previous assistant message with a tool call!") }} + {%- endif %} + {{- "<|start|>functions." + last_tool_call.name }} + {{- " to=assistant<|channel|>commentary<|message|>" + message.content|tojson + "<|end|>" }} + {%- elif message.role == 'user' -%} + {{- "<|start|>user<|message|>" + message.content + "<|end|>" }} + {%- endif -%} +{%- endfor -%} + +{#- Generation prompt #} +{%- if add_generation_prompt -%} +<|start|>assistant +{%- endif -%} \ No newline at end of file diff --git a/jinja2/test_cases/chat/templates/aq_p208p2002__chatglm3-6b-chat-template__tokenizer_config.json.jinja b/jinja2/test_cases/chat/templates/aq_p208p2002__chatglm3-6b-chat-template__tokenizer_config.json.jinja new file mode 100644 index 00000000..b2e614b6 --- /dev/null +++ b/jinja2/test_cases/chat/templates/aq_p208p2002__chatglm3-6b-chat-template__tokenizer_config.json.jinja @@ -0,0 +1,3 @@ +{% for message in messages %}{% if loop.first %}[gMASK]sop<|{{ message['role'] }}|> + {{ message['content'] }}{% else %}<|{{ message['role'] }}|> + {{ message['content'] }}{% endif %}{% endfor %}{% if add_generation_prompt %}<|assistant|>{% endif %} \ No newline at end of file diff --git a/jinja2/test_cases/chat/templates/ar_phuonglk__openchat_3.5_tokenizer_chat_template__tokenizer_config.json.jinja b/jinja2/test_cases/chat/templates/ar_phuonglk__openchat_3.5_tokenizer_chat_template__tokenizer_config.json.jinja new file mode 100644 index 00000000..0feceebf --- /dev/null +++ b/jinja2/test_cases/chat/templates/ar_phuonglk__openchat_3.5_tokenizer_chat_template__tokenizer_config.json.jinja @@ -0,0 +1 @@ +{% for message in messages %}{% if message['role'] == 'user' %}{{ 'GPT4 Correct User: ' + message['content'] + eos_token }}{% elif message['role'] == 'system' %}{{ message['content'] + eos_token }}{% elif message['role'] == 'assistant' %}{{ 'GPT4 Correct Assistant: ' + message['content'] + eos_token }}{% endif %}{% if loop.last %}{{ 'GPT4 Correct Assistant: ' }}{% endif %}{% endfor %} \ No newline at end of file diff --git a/jinja2/test_cases/chat/templates/as_prince-canuma__c4ai-command-r-v01-tokenizer-chat-template__tokenizer_config.json.jinja b/jinja2/test_cases/chat/templates/as_prince-canuma__c4ai-command-r-v01-tokenizer-chat-template__tokenizer_config.json.jinja new file mode 100644 index 00000000..d3a85e39 --- /dev/null +++ b/jinja2/test_cases/chat/templates/as_prince-canuma__c4ai-command-r-v01-tokenizer-chat-template__tokenizer_config.json.jinja @@ -0,0 +1,2 @@ +{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{{ '<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>' + system_message + '<|END_OF_TURN_TOKEN|>' }}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if message.role == 'user' %}<|START_OF_TURN_TOKEN|><|USER_TOKEN|>{{ message.content }}<|END_OF_TURN_TOKEN|> +{% elif message.role == 'assistant' %}<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>{{ message.content }}<|END_OF_TURN_TOKEN|>{% endif %}{% endfor %}{% if add_generation_prompt %}<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>{% endif %} \ No newline at end of file diff --git a/jinja2/test_cases/chat/templates/at_pxyyy__llama3-8B-with-chat-template__tokenizer_config.json.jinja b/jinja2/test_cases/chat/templates/at_pxyyy__llama3-8B-with-chat-template__tokenizer_config.json.jinja new file mode 100644 index 00000000..39bd0c9f --- /dev/null +++ b/jinja2/test_cases/chat/templates/at_pxyyy__llama3-8B-with-chat-template__tokenizer_config.json.jinja @@ -0,0 +1,5 @@ +{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|> + +'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|> + +' }}{% endif %} \ No newline at end of file diff --git a/jinja2/test_cases/chat/templates/au_t-tech__T-lite-it-2.1-FP8__chat_template.jinja b/jinja2/test_cases/chat/templates/au_t-tech__T-lite-it-2.1-FP8__chat_template.jinja new file mode 100644 index 00000000..0b04cc1e --- /dev/null +++ b/jinja2/test_cases/chat/templates/au_t-tech__T-lite-it-2.1-FP8__chat_template.jinja @@ -0,0 +1,86 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n\n\n\n\n' }} +{%- endif %} \ No newline at end of file diff --git a/jinja2/test_cases/chat/templates/av_tjtanaa__chat-template-test-vicuna-v1.1__tokenizer_config.json.jinja b/jinja2/test_cases/chat/templates/av_tjtanaa__chat-template-test-vicuna-v1.1__tokenizer_config.json.jinja new file mode 100644 index 00000000..45148ffb --- /dev/null +++ b/jinja2/test_cases/chat/templates/av_tjtanaa__chat-template-test-vicuna-v1.1__tokenizer_config.json.jinja @@ -0,0 +1 @@ +{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %} \ No newline at end of file diff --git a/jinja2/test_cases/chat/templates/aw_unsloth__Qwen3-32B__chat_template.jinja b/jinja2/test_cases/chat/templates/aw_unsloth__Qwen3-32B__chat_template.jinja new file mode 100644 index 00000000..e079919f --- /dev/null +++ b/jinja2/test_cases/chat/templates/aw_unsloth__Qwen3-32B__chat_template.jinja @@ -0,0 +1,98 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for forward_message in messages %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- set message = messages[index] %} + {%- set current_content = message.content if message.content is not none else '' %} + {%- set tool_start = '' %} + {%- set tool_start_length = tool_start|length %} + {%- set start_of_message = current_content[:tool_start_length] %} + {%- set tool_end = '' %} + {%- set tool_end_length = tool_end|length %} + {%- set start_pos = (current_content|length) - tool_end_length %} + {%- if start_pos < 0 %} + {%- set start_pos = 0 %} + {%- endif %} + {%- set end_of_message = current_content[start_pos:] %} + {%- if ns.multi_step_tool and message.role == "user" and not(start_of_message == tool_start and end_of_message == tool_end) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = (message.content.split('')|last).lstrip('\n') %} + {%- set reasoning_content = (message.content.split('')|first).rstrip('\n') %} + {%- set reasoning_content = (reasoning_content.split('')|last).lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/jinja2/test_cases/generate_expected.py b/jinja2/test_cases/generate_expected.py new file mode 100644 index 00000000..9f3c7c11 --- /dev/null +++ b/jinja2/test_cases/generate_expected.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +import jinja2 # pyright: ignore[reportMissingImports] +from transformers.utils.chat_template_utils import render_jinja_template # pyright: ignore[reportMissingImports] + + +KNOWN_IGNORED_CASES: set[tuple[str, str]] = { + # DeepSeek-R1 template concatenates tool.function.arguments as a raw string, + # while our fixtures intentionally use OpenAI-like dict arguments. + ("ag", "04"), + ("ag", "05"), + ("ag", "06"), + ("ag", "13"), + ("ag", "14"), + # This template only supports a single tool call in one assistant turn. + ("ae", "05"), + # This template renders from/value instead of role/content, which breaks + # continue_final_message compatibility. + ("af", "10"), +} + + +def extract_id(path: Path) -> str: + stem = path.stem + if "_" in stem: + return stem.split("_", 1)[0] + return stem + + +def render_basic_suite(input_root: Path, output_root: Path) -> int: + basic_dir = input_root / "basic" + output_basic_dir = output_root / "basic" + output_basic_dir.mkdir(parents=True, exist_ok=True) + env = jinja2.Environment() # pyright: ignore[reportAttributeAccessIssue] + rendered_count = 0 + + for path in output_basic_dir.glob("*.txt"): + path.unlink() + + for path in sorted(basic_dir.glob("*.json")): + payload = json.loads(path.read_text(encoding="utf-8")) + template_src = payload["template"] + data = payload.get("data", {}) + expected = env.from_string(template_src).render(**data) + output_name = f"{path.stem}.txt" + (output_basic_dir / output_name).write_text(expected, encoding="utf-8") + rendered_count += 1 + + return rendered_count + + +def validate_unique_prefix_ids(files: list[Path], label: str) -> None: + ids = [extract_id(path) for path in files] + if len(set(ids)) != len(ids): + raise ValueError( + f"Duplicate {label} ids found; ensure unique filename prefixes." + ) + + +def render_chat_like_suite( + input_root: Path, + output_root: Path, + suite: str, + render_jinja_template, +) -> tuple[int, list[tuple[str, str, str, str]]]: + suite_dir = input_root / suite + data_dir = suite_dir / "data" + templates_dir = suite_dir / "templates" + expected_dir = output_root / suite + + if not data_dir.exists() or not templates_dir.exists(): + return 0, [] + + expected_dir.mkdir(parents=True, exist_ok=True) + for path in expected_dir.glob("*.txt"): + path.unlink() + + data_files = sorted(data_dir.glob("*.json")) + template_files = sorted(templates_dir.glob("*.jinja")) + validate_unique_prefix_ids(data_files, f"{suite}/data") + validate_unique_prefix_ids(template_files, f"{suite}/templates") + + rendered_count = 0 + failures: list[tuple[str, str, str, str]] = [] + + for template_path in template_files: + template_id = extract_id(template_path) + chat_template = template_path.read_text(encoding="utf-8") + + for data_path in data_files: + data_id = extract_id(data_path) + payload = json.loads(data_path.read_text(encoding="utf-8")) + + messages = payload.get("messages", []) + tools = payload.get("tools") + documents = payload.get("documents") + add_generation_prompt = bool(payload.get("add_generation_prompt", False)) + continue_final_message = bool(payload.get("continue_final_message", False)) + + kwargs = dict(payload) + for key in ( + "messages", + "tools", + "documents", + "add_generation_prompt", + "continue_final_message", + ): + kwargs.pop(key, None) + + output_path = expected_dir / f"{template_id}_{data_id}.txt" + try: + rendered, _ = render_jinja_template( + conversations=[messages], + tools=tools, + documents=documents, + chat_template=chat_template, + continue_final_message=continue_final_message, + add_generation_prompt=add_generation_prompt, + **kwargs, + ) + output_path.write_text(rendered[0], encoding="utf-8") + rendered_count += 1 + except Exception as exc: # noqa: BLE001 + if (template_id, data_id) in KNOWN_IGNORED_CASES: + continue + failures.append( + ( + f"{suite}/{template_path.name}", + f"{suite}/{data_path.name}", + type(exc).__name__, + str(exc), + ) + ) + + return rendered_count, failures + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument( + "--input-root", + type=Path, + default=Path(__file__).resolve().parent, + ) + parser.add_argument( + "--output-root", + type=Path, + default=None, + ) + parser.add_argument( + "--skip-basic", + action="store_true", + ) + args = parser.parse_args() + + input_root = args.input_root + output_root = args.output_root if args.output_root is not None else input_root + + basic_rendered = 0 + if not args.skip_basic: + basic_rendered = render_basic_suite(input_root, output_root) + + chat_rendered, chat_failures = render_chat_like_suite( + input_root, + output_root, + "chat", + render_jinja_template, + ) + multimodal_rendered, multimodal_failures = render_chat_like_suite( + input_root, + output_root, + "multimodal", + render_jinja_template, + ) + + failures = [*chat_failures, *multimodal_failures] + + print(f"basic_rendered={basic_rendered}") + print(f"chat_rendered={chat_rendered}") + print(f"multimodal_rendered={multimodal_rendered}") + print(f"rendered={basic_rendered + chat_rendered + multimodal_rendered}") + print(f"errors={len(failures)}") + + if failures: + print("failure_report_start") + for template_name, data_name, exc_type, message in failures: + print( + f"- template={template_name} data={data_name} error={exc_type}: {message}" + ) + print("failure_report_end") + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/jinja2/test_cases/multimodal/data/08_multimodal_image_text.json b/jinja2/test_cases/multimodal/data/08_multimodal_image_text.json new file mode 100644 index 00000000..078a218c --- /dev/null +++ b/jinja2/test_cases/multimodal/data/08_multimodal_image_text.json @@ -0,0 +1,36 @@ +{ + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You can reason about images." + } + ], + "from": "system", + "value": "You can reason about images." + }, + { + "role": "user", + "content": [ + { + "type": "image", + "url": "https://example.com/cat.png" + }, + { + "type": "text", + "text": "What is in this image?" + } + ], + "from": "human", + "value": "What is in this image?" + } + ], + "tools": [], + "documents": [], + "add_vision_id": true, + "add_generation_prompt": true, + "bos_token": "", + "eos_token": "" +} diff --git a/jinja2/test_cases/multimodal/data/09_multimodal_video_text.json b/jinja2/test_cases/multimodal/data/09_multimodal_video_text.json new file mode 100644 index 00000000..8114f174 --- /dev/null +++ b/jinja2/test_cases/multimodal/data/09_multimodal_video_text.json @@ -0,0 +1,35 @@ +{ + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You can reason about short videos." + } + ], + "from": "system", + "value": "You can reason about short videos." + }, + { + "role": "user", + "content": [ + { + "type": "video", + "url": "https://example.com/clip.mp4" + }, + { + "type": "text", + "text": "Describe the motion in one sentence." + } + ], + "from": "human", + "value": "Describe the motion in one sentence." + } + ], + "tools": [], + "documents": [], + "add_generation_prompt": true, + "bos_token": "", + "eos_token": "" +} diff --git a/jinja2/test_cases/multimodal/templates/aa_Lightricks__LTX-2__tokenizer__chat_template.jinja b/jinja2/test_cases/multimodal/templates/aa_Lightricks__LTX-2__tokenizer__chat_template.jinja new file mode 100644 index 00000000..1117055a --- /dev/null +++ b/jinja2/test_cases/multimodal/templates/aa_Lightricks__LTX-2__tokenizer__chat_template.jinja @@ -0,0 +1,47 @@ +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} diff --git a/jinja2/test_cases/multimodal/templates/ax_zai-org__GLM-4.5__chat_template.jinja b/jinja2/test_cases/multimodal/templates/ax_zai-org__GLM-4.5__chat_template.jinja new file mode 100644 index 00000000..51ecb5cc --- /dev/null +++ b/jinja2/test_cases/multimodal/templates/ax_zai-org__GLM-4.5__chat_template.jinja @@ -0,0 +1,103 @@ +[gMASK] +{%- if tools -%} +<|system|> +# Tools + +You may call one or more functions to assist with the user query. + +You are provided with function signatures within XML tags: + +{% for tool in tools %} +{{ tool | tojson(ensure_ascii=False) }} +{% endfor %} + + +For each function call, output the function name and arguments within the following XML format: +{function-name} +{arg-key-1} +{arg-value-1} +{arg-key-2} +{arg-value-2} +... +{%- endif -%} +{%- macro visible_text(content) -%} + {%- if content is string -%} + {{- content }} + {%- elif content is iterable and content is not mapping -%} + {%- for item in content -%} + {%- if item is mapping and item.type == 'text' -%} + {{- item.text }} + {%- elif item is string -%} + {{- item }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{- content }} + {%- endif -%} +{%- endmacro -%} +{%- set ns = namespace(last_user_index=-1) %} +{%- for m in messages %} + {%- if m.role == 'user' %} + {% set ns.last_user_index = loop.index0 -%} + {%- endif %} +{%- endfor %} +{% for m in messages %} +{%- if m.role == 'user' -%}<|user|> +{{ visible_text(m.content) }} +{{- '/nothink' if (enable_thinking is defined and not enable_thinking and not visible_text(m.content).endswith("/nothink")) else '' -}} +{%- elif m.role == 'assistant' -%} +<|assistant|> +{%- set reasoning_content = '' %} +{%- set content = visible_text(m.content) %} +{%- if m.reasoning_content is string %} + {%- set reasoning_content = m.reasoning_content %} +{%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} +{%- endif %} +{%- if loop.index0 > ns.last_user_index and reasoning_content -%} +{{ '\n' + reasoning_content.strip() + ''}} +{%- else -%} +{{ '\n' }} +{%- endif -%} +{%- if content.strip() -%} +{{ '\n' + content.strip() }} +{%- endif -%} +{% if m.tool_calls %} +{% for tc in m.tool_calls %} +{%- if tc.function %} + {%- set tc = tc.function %} +{%- endif %} +{{ '\n' + tc.name }} +{% set _args = tc.arguments %} +{% for k, v in _args.items() %} +{{ k }} +{{ v | tojson(ensure_ascii=False) if v is not string else v }} +{% endfor %} +{% endfor %} +{% endif %} +{%- elif m.role == 'tool' -%} +{%- if m.content is string -%} +{%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|observation|>' }} +{%- endif %} +{{- '\n\n' }} +{{- m.content }} +{{- '\n' }} +{%- else -%} +<|observation|>{% for tr in m.content %} + + +{{ tr.output if tr.output is defined else tr }} +{% endfor -%} +{% endif -%} +{%- elif m.role == 'system' -%} +<|system|> +{{ visible_text(m.content) }} +{%- endif -%} +{%- endfor -%} +{%- if add_generation_prompt -%} + <|assistant|>{{- '\n' if (enable_thinking is defined and not enable_thinking) else '' -}} +{%- endif -%} diff --git a/jinja2/test_cases/multimodal/templates/ay_zai-org__GLM-4.7-Flash__chat_template.jinja b/jinja2/test_cases/multimodal/templates/ay_zai-org__GLM-4.7-Flash__chat_template.jinja new file mode 100644 index 00000000..1ee05c17 --- /dev/null +++ b/jinja2/test_cases/multimodal/templates/ay_zai-org__GLM-4.7-Flash__chat_template.jinja @@ -0,0 +1,86 @@ +[gMASK] +{%- if tools -%} +<|system|> +# Tools + +You may call one or more functions to assist with the user query. + +You are provided with function signatures within XML tags: + +{% for tool in tools %} +{{ tool | tojson(ensure_ascii=False) }} +{% endfor %} + + +For each function call, output the function name and arguments within the following XML format: +{function-name}{arg-key-1}{arg-value-1}{arg-key-2}{arg-value-2}...{%- endif -%} +{%- macro visible_text(content) -%} + {%- if content is string -%} + {{- content }} + {%- elif content is iterable and content is not mapping -%} + {%- for item in content -%} + {%- if item is mapping and item.type == 'text' -%} + {{- item.text }} + {%- elif item is string -%} + {{- item }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{- content }} + {%- endif -%} +{%- endmacro -%} +{%- set ns = namespace(last_user_index=-1) %} +{%- for m in messages %} + {%- if m.role == 'user' %} + {% set ns.last_user_index = loop.index0 -%} + {%- endif %} +{%- endfor %} +{% for m in messages %} +{%- if m.role == 'user' -%}<|user|>{{ visible_text(m.content) }} +{%- elif m.role == 'assistant' -%} +<|assistant|> +{%- set reasoning_content = '' %} +{%- set content = visible_text(m.content) %} +{%- if m.reasoning_content is string %} + {%- set reasoning_content = m.reasoning_content %} +{%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} +{%- endif %} +{%- if ((clear_thinking is defined and not clear_thinking) or loop.index0 > ns.last_user_index) and reasoning_content -%} +{{ '' + reasoning_content.strip() + ''}} +{%- else -%} +{{ '' }} +{%- endif -%} +{%- if content.strip() -%} +{{ content.strip() }} +{%- endif -%} +{% if m.tool_calls %} +{% for tc in m.tool_calls %} +{%- if tc.function %} + {%- set tc = tc.function %} +{%- endif %} +{{- '' + tc.name -}} +{% set _args = tc.arguments %}{% for k, v in _args.items() %}{{ k }}{{ v | tojson(ensure_ascii=False) if v is not string else v }}{% endfor %}{% endfor %} +{% endif %} +{%- elif m.role == 'tool' -%} +{%- if m.content is string -%} +{%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|observation|>' }} +{%- endif %} +{{- '' }} +{{- m.content }} +{{- '' }} +{%- else -%} +<|observation|>{% for tr in m.content %} +{{ tr.output if tr.output is defined else tr }}{% endfor -%} +{% endif -%} +{%- elif m.role == 'system' -%} +<|system|>{{ visible_text(m.content) }} +{%- endif -%} +{%- endfor -%} +{%- if add_generation_prompt -%} + <|assistant|>{{- '' if (enable_thinking is defined and not enable_thinking) else '' -}} +{%- endif -%} diff --git a/jinja2/value.zig b/jinja2/value.zig new file mode 100644 index 00000000..adc5ad47 --- /dev/null +++ b/jinja2/value.zig @@ -0,0 +1,308 @@ +const std = @import("std"); + +pub const Namespace = struct { + map: std.StringHashMap(Value), + + pub fn init(allocator: std.mem.Allocator) Namespace { + return .{ .map = std.StringHashMap(Value).init(allocator) }; + } +}; + +pub const LoopMeta = struct { + index0: i64, + index: i64, + first: bool, + last: bool, + previtem: Value, + nextitem: Value, +}; + +pub const Value = union(enum) { + undefined, + null, + bool: bool, + int: i64, + float: f64, + str: []const u8, + list: []const Value, + map: []const Entry, + namespace: *Namespace, + loopmeta: *LoopMeta, + + pub const Entry = struct { + key: []const u8, + value: Value, + }; + + pub fn fromBool(v: bool) Value { + return .{ .bool = v }; + } + + pub fn fromInt(v: i64) Value { + return .{ .int = v }; + } + + pub fn fromString(v: []const u8) Value { + return .{ .str = v }; + } + + pub fn fromList(v: []const Value) Value { + return .{ .list = v }; + } + + pub fn fromMap(v: []const Entry) Value { + return .{ .map = v }; + } + + pub fn isTruthy(self: Value) bool { + return switch (self) { + .undefined, .null => false, + .bool => |v| v, + .int => |v| v != 0, + .float => |v| v != 0.0, + .str => |v| v.len > 0, + .list => |v| v.len > 0, + .map => |v| v.len > 0, + .namespace => |ns| ns.map.count() > 0, + .loopmeta => true, + }; + } + + pub fn getAttr(self: Value, key: []const u8) ?Value { + return switch (self) { + .map => |entries| mapGet(entries, key), + .namespace => |ns| ns.map.get(key), + .loopmeta => |lp| loopMetaGet(lp, key), + else => null, + }; + } + + pub fn setAttr(self: Value, key: []const u8, value: Value) !void { + switch (self) { + .namespace => |ns| try ns.map.put(key, value), + else => return error.TypeError, + } + } + + pub fn eql(a: Value, b: Value) bool { + return switch (a) { + .undefined => switch (b) { + .undefined => true, + else => false, + }, + .null => switch (b) { + .null => true, + else => false, + }, + .bool => |av| switch (b) { + .bool => |bv| av == bv, + else => false, + }, + .int => |av| switch (b) { + .int => |bv| av == bv, + .float => |bv| @as(f64, @floatFromInt(av)) == bv, + else => false, + }, + .float => |av| switch (b) { + .int => |bv| av == @as(f64, @floatFromInt(bv)), + .float => |bv| av == bv, + else => false, + }, + .str => |av| switch (b) { + .str => |bv| std.mem.eql(u8, av, bv), + else => false, + }, + .list => false, + .map => false, + .namespace => false, + .loopmeta => false, + }; + } +}; + +const Counts = struct { + values: usize = 0, + entries: usize = 0, +}; + +fn typeCounts(comptime T: type) Counts { + if (T == Value or T == Value.Entry) return .{}; + return switch (@typeInfo(T)) { + .bool, .int, .comptime_int, .float, .comptime_float => .{}, + .@"enum" => .{}, + .optional => |opt| typeCounts(opt.child), + .pointer => |ptr| switch (ptr.size) { + .one => typeCounts(ptr.child), + .slice => if (ptr.child == u8) .{} else @compileError("zero-alloc struct context does not support non-string slices"), + else => @compileError("unsupported pointer type for zero-alloc struct context"), + }, + .array => |arr| blk: { + if (arr.child == u8) break :blk .{}; + const child_counts = typeCounts(arr.child); + break :blk .{ + .values = arr.len + child_counts.values * arr.len, + .entries = child_counts.entries * arr.len, + }; + }, + .@"struct" => |s| blk: { + var out = Counts{ .entries = s.fields.len }; + inline for (s.fields) |f| { + const fc = typeCounts(f.type); + out.values += fc.values; + out.entries += fc.entries; + } + break :blk out; + }, + else => @compileError("unsupported type for zero-alloc struct context: " ++ @typeName(T)), + }; +} + +pub fn StructContext(comptime T: type) type { + const counts = typeCounts(T); + + return struct { + const Self = @This(); + + values: [counts.values]Value = undefined, + entries: [counts.entries]Value.Entry = undefined, + values_len: usize = 0, + entries_len: usize = 0, + + pub fn from(self: *Self, src: *const T) []const Value.Entry { + self.values_len = 0; + self.entries_len = 0; + const root = self.toValue(T, src); + return root.map; + } + + fn toValue(self: *Self, comptime U: type, src: *const U) Value { + if (U == Value) return src.*; + return switch (@typeInfo(U)) { + .bool => .{ .bool = src.* }, + .int, .comptime_int => .{ .int = @intCast(src.*) }, + .float, .comptime_float => .{ .float = @floatCast(src.*) }, + .@"enum" => .{ .str = @tagName(src.*) }, + .optional => |opt| if (src.*) |v| self.toValue(opt.child, &v) else .null, + .pointer => |ptr| switch (ptr.size) { + .one => self.toValue(ptr.child, src.*), + .slice => if (ptr.child == u8) .{ .str = src.* } else @compileError("zero-alloc struct context does not support non-string slices"), + else => @compileError("unsupported pointer type for zero-alloc struct context"), + }, + .array => |arr| blk: { + if (arr.child == u8) break :blk .{ .str = src.*[0..] }; + const start = self.values_len; + self.values_len += arr.len; + inline for (0..arr.len) |i| { + self.values[start + i] = self.toValue(arr.child, &src.*[i]); + } + break :blk .{ .list = self.values[start .. start + arr.len] }; + }, + .@"struct" => |s| blk: { + const start = self.entries_len; + self.entries_len += s.fields.len; + inline for (s.fields, 0..) |f, i| { + self.entries[start + i] = .{ + .key = f.name, + .value = self.toValue(f.type, &@field(src.*, f.name)), + }; + } + break :blk .{ .map = self.entries[start .. start + s.fields.len] }; + }, + else => @compileError("unsupported type for zero-alloc struct context: " ++ @typeName(U)), + }; + } + }; +} + +fn mapGet(entries: []const Value.Entry, key: []const u8) ?Value { + for (entries) |entry| { + if (entry.key.ptr == key.ptr and entry.key.len == key.len) return entry.value; + if (entry.key.len != key.len) continue; + if (entry.key.len > 0 and entry.key[0] != key[0]) continue; + if (std.mem.eql(u8, entry.key, key)) return entry.value; + } + return null; +} + +fn loopMetaGet(lp: *const LoopMeta, key: []const u8) ?Value { + if (key.len == 0) return null; + switch (key[0]) { + 'f' => if (std.mem.eql(u8, key, "first")) return .{ .bool = lp.first }, + 'i' => { + if (std.mem.eql(u8, key, "index")) return .{ .int = lp.index }; + if (std.mem.eql(u8, key, "index0")) return .{ .int = lp.index0 }; + }, + 'l' => if (std.mem.eql(u8, key, "last")) return .{ .bool = lp.last }, + 'n' => if (std.mem.eql(u8, key, "nextitem")) return lp.nextitem, + 'p' => if (std.mem.eql(u8, key, "previtem")) return lp.previtem, + else => {}, + } + return null; +} + +pub fn writeValue(writer: anytype, value: Value) !void { + switch (value) { + .undefined, .null => try writer.writeAll(""), + .bool => |v| { + if (v) { + try writer.writeAll("true"); + } else { + try writer.writeAll("false"); + } + }, + .int => |v| { + var buf: [32]u8 = undefined; + const rendered = try std.fmt.bufPrint(&buf, "{}", .{v}); + try writer.writeAll(rendered); + }, + .float => |v| { + var buf: [64]u8 = undefined; + const rendered = try std.fmt.bufPrint(&buf, "{}", .{v}); + try writer.writeAll(rendered); + }, + .str => |v| try writer.writeAll(v), + .list => |v| { + try writer.writeByte('['); + for (v, 0..) |item, idx| { + if (idx > 0) try writer.writeAll(", "); + try writeValue(writer, item); + } + try writer.writeByte(']'); + }, + .map => |entries| { + try writer.writeByte('{'); + for (entries, 0..) |entry, idx| { + if (idx > 0) try writer.writeAll(", "); + try writer.writeAll(entry.key); + try writer.writeAll(": "); + try writeValue(writer, entry.value); + } + try writer.writeByte('}'); + }, + .namespace => |ns| { + try writer.writeByte('{'); + var it = ns.map.iterator(); + var idx: usize = 0; + while (it.next()) |entry| { + if (idx > 0) try writer.writeAll(", "); + idx += 1; + try writer.writeAll(entry.key_ptr.*); + try writer.writeAll(": "); + try writeValue(writer, entry.value_ptr.*); + } + try writer.writeByte('}'); + }, + .loopmeta => |lp| { + try writer.writeByte('{'); + try writer.writeAll("index0: "); + try writeValue(writer, .{ .int = lp.index0 }); + try writer.writeAll(", index: "); + try writeValue(writer, .{ .int = lp.index }); + try writer.writeAll(", first: "); + try writeValue(writer, .{ .bool = lp.first }); + try writer.writeAll(", last: "); + try writeValue(writer, .{ .bool = lp.last }); + try writer.writeByte('}'); + }, + } +}