From a18c230189b9739f78bbbe6c16190e56b4c9db6d Mon Sep 17 00:00:00 2001 From: Daniel Hiltgen Date: Tue, 21 Jul 2026 16:06:29 -0700 Subject: [PATCH] model: add Laguna v8 chat support and fix Metal inference (#17291) Add a laguna-v8 renderer/parser matching the Laguna XS 2.1 template, and fix v2 handling of embedded thinking and structured tool arguments. Prevent FP16 overflow in Metal's quantized routed-MoE prefill path by scaling the linear branch and folding the inverse into the routing scale. Other backends and token-generation paths are unchanged. Add comprehensive v2/v8 Jinja parity and parser tests. --- llama/compat/models/laguna.cpp | 25 +- model/parsers/laguna.go | 11 + model/parsers/laguna_test.go | 124 ++++ model/parsers/parsers.go | 2 + model/renderers/laguna.go | 140 +++- model/renderers/laguna_test.go | 699 ++++++++++++++++-- model/renderers/renderer.go | 2 + model/renderers/renderer_test.go | 1 + .../testdata/laguna_v2_chat_template.jinja2 | 132 ++++ .../testdata/laguna_v8_chat_template.jinja2 | 93 +++ parser/parser.go | 8 + parser/parser_test.go | 2 + 12 files changed, 1180 insertions(+), 59 deletions(-) create mode 100644 model/renderers/testdata/laguna_v2_chat_template.jinja2 create mode 100644 model/renderers/testdata/laguna_v8_chat_template.jinja2 diff --git a/llama/compat/models/laguna.cpp b/llama/compat/models/laguna.cpp index 5a41142ae..76c8f6c2e 100644 --- a/llama/compat/models/laguna.cpp +++ b/llama/compat/models/laguna.cpp @@ -186,6 +186,25 @@ llama_model_laguna::graph::graph(const llama_model & model, const llm_graph_para NULL, LLM_FFN_SILU, LLM_FFN_PAR, il); cb(cur, "ffn_out", il); } else { + ggml_tensor * up_scale = nullptr; + float expert_weights_scale = hparams.expert_weights_scale; + +#if defined(GGML_USE_METAL) + if (n_tokens >= 32 && ggml_is_quantized(model.layers[il].ffn_down_exps->type)) { + // ggml-metal switches MUL_MAT_ID from its range-safe + // matrix-vector kernel to FP16 matrix tiles at 32 tokens + // (ne21_mm_id_min in ggml_metal_op_mul_mat_id). Laguna's routed + // SwiGLU activations can overflow those tiles. Scale the linear + // up branch and fold the inverse power-of-two factor into the + // existing routing-weight scale. Revisit this guard if the + // Metal dispatch threshold changes. + constexpr float down_input_scale = 1.0f / 256.0f; + up_scale = ggml_fill(ctx0, + model.layers[il].ffn_exp_probs_b, down_input_scale); + expert_weights_scale /= down_input_scale; + } +#endif + ggml_tensor * moe_out = build_moe_ffn(cur, model.layers[il].ffn_gate_inp, model.layers[il].ffn_up_exps, @@ -194,9 +213,11 @@ llama_model_laguna::graph::graph(const llama_model & model, const llm_graph_para model.layers[il].ffn_exp_probs_b, n_expert, n_expert_used, LLM_FFN_SILU, hparams.expert_weights_norm, - hparams.expert_weights_scale, + expert_weights_scale, (llama_expert_gating_func_type) hparams.expert_gating_func, - il); + il, + nullptr, nullptr, + up_scale); cb(moe_out, "ffn_moe_out", il); ggml_tensor * ffn_shexp = build_ffn(cur, diff --git a/model/parsers/laguna.go b/model/parsers/laguna.go index 29ce5c398..1bdca088d 100644 --- a/model/parsers/laguna.go +++ b/model/parsers/laguna.go @@ -94,6 +94,17 @@ func (p *LagunaParser) Init(tools []api.Tool, lastMessage *api.Message, thinkVal return tools } +// LagunaV8Parser matches the v8 renderer, which closes any assistant history +// turn and emits a fresh assistant generation prompt instead of continuing the +// final assistant message in place. +type LagunaV8Parser struct { + LagunaParser +} + +func (p *LagunaV8Parser) Init(tools []api.Tool, _ *api.Message, thinkValue *api.ThinkValue) []api.Tool { + return p.LagunaParser.Init(tools, nil, thinkValue) +} + func (p *LagunaParser) Add(s string, done bool) (content string, thinking string, calls []api.ToolCall, err error) { p.buffer.WriteString(s) var contentSB, thinkingSB strings.Builder diff --git a/model/parsers/laguna_test.go b/model/parsers/laguna_test.go index 8112b1641..e0dd055ab 100644 --- a/model/parsers/laguna_test.go +++ b/model/parsers/laguna_test.go @@ -20,6 +20,23 @@ func lagunaTestTools() []api.Tool { }} } +func lagunaParseChunks(t *testing.T, parser Parser, chunks ...string) (string, string, []api.ToolCall) { + t.Helper() + + var content, thinking string + var calls []api.ToolCall + for i, chunk := range chunks { + chunkContent, chunkThinking, chunkCalls, err := parser.Add(chunk, i == len(chunks)-1) + if err != nil { + t.Fatalf("Add(%q, done=%t): %v", chunk, i == len(chunks)-1, err) + } + content += chunkContent + thinking += chunkThinking + calls = append(calls, chunkCalls...) + } + return content, thinking, calls +} + func TestLagunaParserToolCall(t *testing.T) { parser := ParserForName("laguna") if parser == nil { @@ -515,6 +532,27 @@ func TestLagunaParserNonAssistantLastMessageStillPrimesThinking(t *testing.T) { } } +func TestLagunaV8ParserAssistantHistoryStillPrimesThinking(t *testing.T) { + // Laguna v8 closes assistant history and emits a fresh generation prompt, + // so an assistant tail message must not switch the parser into prefill mode. + parser := ParserForName("laguna-v8") + if parser == nil { + t.Fatal("expected laguna-v8 parser") + } + if !parser.HasToolSupport() || !parser.HasThinkingSupport() { + t.Fatal("laguna-v8 parser should advertise tools and thinking") + } + + parser.Init(nil, &api.Message{Role: "assistant", Content: "Previous."}, &api.ThinkValue{Value: true}) + content, thinking, calls, err := parser.Add("Reasoning.Answer.", true) + if err != nil { + t.Fatal(err) + } + if content != "Answer." || thinking != "Reasoning." || len(calls) != 0 { + t.Fatalf("content=%q thinking=%q calls=%d", content, thinking, len(calls)) + } +} + func TestLagunaParserStripsLeadingContentWhitespace(t *testing.T) { // No-think prompts prime , so the model emits a leading newline // before content; the parser drops it. @@ -575,3 +613,89 @@ func TestLagunaParserSplitToolTag(t *testing.T) { t.Fatalf("second chunk content=%q thinking=%q calls=%d", content, thinking, len(calls)) } } + +func TestLagunaParserPartialToolCallFakeoutInContent(t *testing.T) { + parser := ParserForName("laguna") + parser.Init(lagunaTestTools(), nil, nil) + + content, thinking, calls := lagunaParseChunks(t, parser, "Document literal Document literal \n") case "assistant": + content, reasoning := lagunaV2AssistantContent(message.Content, message.Thinking) lastMessage := i == len(messages)-1 - prefill := lastMessage && (strings.TrimSpace(content) != "" || strings.TrimSpace(message.Thinking) != "" || len(message.ToolCalls) > 0) + prefill := lastMessage && (strings.TrimSpace(content) != "" || strings.TrimSpace(reasoning) != "" || len(message.ToolCalls) > 0) sb.WriteString("\n") // Every assistant turn opens with the reasoning block: a full // … when there is reasoning, otherwise a bare // marking the turn as direct. - if reasoning := strings.TrimSpace(message.Thinking); reasoning != "" { + if reasoning := strings.TrimSpace(reasoning); reasoning != "" { sb.WriteString("\n") sb.WriteString(reasoning) sb.WriteString("\n\n") @@ -112,7 +113,7 @@ func (r *LagunaRenderer) Render(messages []api.Message, tools []api.Tool, think sb.WriteString(name) sb.WriteString("\n") sb.WriteString("") - sb.WriteString(formatToolCallArgument(value)) + sb.WriteString(formatLagunaToolCallArgument(value)) sb.WriteString("\n") } sb.WriteString("\n") @@ -146,3 +147,136 @@ func (r *LagunaRenderer) Render(messages []api.Message, tools []api.Tool, think return sb.String(), nil } + +func lagunaV2AssistantContent(content, reasoning string) (string, string) { + parts := strings.Split(content, lagunaThoughtClose) + if len(parts) == 1 { + return content, reasoning + } + + if reasoning == "" { + before := strings.TrimRight(parts[0], "\n") + if i := strings.LastIndex(before, lagunaThoughtOpen); i >= 0 { + before = before[i+len(lagunaThoughtOpen):] + } + reasoning = strings.TrimLeft(before, "\n") + } + + content = strings.TrimLeft(parts[len(parts)-1], "\n") + return content, reasoning +} + +type LagunaV8Renderer struct{} + +func (r *LagunaV8Renderer) LeadingBOS() string { + return lagunaBOS +} + +func (r *LagunaV8Renderer) Render(messages []api.Message, tools []api.Tool, think *api.ThinkValue) (string, error) { + var sb strings.Builder + sb.WriteString(lagunaBOS) + + thinkingEnabled := think != nil && think.Bool() + + systemMessage := lagunaDefaultSystem + firstMessageIsSystem := len(messages) > 0 && messages[0].Role == "system" + if firstMessageIsSystem { + systemMessage = messages[0].Content + } + + hasSystem := strings.TrimSpace(systemMessage) != "" + if hasSystem || len(tools) > 0 || thinkingEnabled { + sb.WriteString("") + if hasSystem { + sb.WriteString(strings.TrimRightFunc(systemMessage, unicode.IsSpace)) + if len(tools) > 0 { + sb.WriteString("\n\n") + } + } + if len(tools) > 0 { + sb.WriteString("### Tools\n\n") + sb.WriteString("You may call functions to assist with the user query.\n") + sb.WriteString("All available function signatures are listed below:\n") + sb.WriteString("\n") + for _, tool := range tools { + if b, err := marshalWithSpaces(tool); err == nil { + sb.Write(b) + sb.WriteByte('\n') + } + } + sb.WriteString("") + } + sb.WriteString("\n") + } + + for i, message := range messages { + if i == 0 && firstMessageIsSystem { + continue + } + content := message.Content + switch message.Role { + case "user": + sb.WriteString("") + sb.WriteString(content) + sb.WriteString("\n") + case "assistant": + sb.WriteString("") + if thinkingEnabled { + sb.WriteString(lagunaThoughtOpen) + sb.WriteString(message.Thinking) + sb.WriteString(lagunaThoughtClose) + } else { + sb.WriteString(lagunaThoughtClose) + } + if content != "" { + sb.WriteString(content) + } + for _, toolCall := range message.ToolCalls { + sb.WriteString("") + sb.WriteString(toolCall.Function.Name) + for name, value := range toolCall.Function.Arguments.All() { + sb.WriteString("") + sb.WriteString(name) + sb.WriteString("") + sb.WriteString("") + sb.WriteString(formatLagunaToolCallArgument(value)) + sb.WriteString("") + } + sb.WriteString("") + } + sb.WriteString("\n") + case "tool": + sb.WriteString("") + sb.WriteString(content) + sb.WriteString("\n") + case "system": + sb.WriteString("") + sb.WriteString(content) + sb.WriteString("\n") + } + } + + sb.WriteString("") + if thinkingEnabled { + sb.WriteString(lagunaThoughtOpen) + } else { + sb.WriteString(lagunaThoughtClose) + } + + return sb.String(), nil +} + +func formatLagunaToolCallArgument(value any) string { + switch v := value.(type) { + case string: + return v + case []byte: + return string(v) + } + + if b, err := marshalWithSpaces(value); err == nil { + return string(b) + } + + return formatToolCallArgument(value) +} diff --git a/model/renderers/laguna_test.go b/model/renderers/laguna_test.go index 44ad03a83..495e7cb94 100644 --- a/model/renderers/laguna_test.go +++ b/model/renderers/laguna_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "os" "os/exec" + "path/filepath" "strings" "testing" @@ -11,17 +12,23 @@ import ( "github.com/ollama/ollama/api" ) +const ( + lagunaV2Template = "testdata/laguna_v2_chat_template.jinja2" + lagunaV8Template = "testdata/laguna_v8_chat_template.jinja2" +) + // lagunaToolJSON is the get_weather tool as serialized into , // matching lagunaWeatherTool(). const lagunaToolJSON = `{"type": "function", "function": {"name": "get_weather", "description": "Get weather", "parameters": {"type": "object", "required": ["location"], "properties": {"location": {"type": "string", "description": "City"}}}}}` +const lagunaMathToolJSON = `{"type": "function", "function": {"name": "add", "description": "Add numbers", "parameters": {"type": "object", "required": ["a", "b"], "properties": {"a": {"type": "number", "description": "First number"}, "b": {"type": "number", "description": "Second number"}}}}}` -// TestLagunaRendererReferenceFlowCoverage checks the renderer against the Laguna -// chat template. Each want is byte-for-byte template output (verified by -// rendering chat_template.jinja), except that history tool-calls use the clean -// form — the template leaks Jinja indentation there. +// TestLagunaRendererReferenceFlowCoverage checks the renderer against byte-for-byte +// expected output from the Laguna v2 chat template. VERIFY_JINJA2=1 also verifies +// these expected values against the checked-in Jinja fixture. func TestLagunaRendererReferenceFlowCoverage(t *testing.T) { weather := lagunaWeatherTool() think := func(v bool) *api.ThinkValue { return &api.ThinkValue{Value: v} } + verifyJinja2 := lagunaVerifyJinja2(t) // system header is always emitted; with no system message the default is used defaultHeader := "〈|EOS|〉\n\n" + lagunaDefaultSystem + "\n\n" @@ -33,6 +40,10 @@ func TestLagunaRendererReferenceFlowCoverage(t *testing.T) { think *api.ThinkValue want string }{ + { + name: "empty_messages", + want: defaultHeader + "\n", + }, { name: "user_only_default", messages: []api.Message{{Role: "user", Content: "Hello"}}, @@ -59,6 +70,14 @@ func TestLagunaRendererReferenceFlowCoverage(t *testing.T) { want: "〈|EOS|〉\n\nStay concise.\n\n" + "\nHi\n\n\n", }, + { + name: "empty_first_system_opts_out_of_header", + messages: []api.Message{ + {Role: "system", Content: ""}, + {Role: "user", Content: "Hi"}, + }, + want: "〈|EOS|〉\nHi\n\n\n", + }, { name: "additional_system", messages: []api.Message{ @@ -71,6 +90,22 @@ func TestLagunaRendererReferenceFlowCoverage(t *testing.T) { "\nSecondary.\n\n" + "\n", }, + { + name: "empty_first_system_with_tools", + messages: []api.Message{ + {Role: "system", Content: ""}, + {Role: "user", Content: "Weather?"}, + }, + tools: weather, + want: "〈|EOS|〉\n\n\n### Tools\n\n" + + "You may call functions to assist with the user query.\n" + + "All available function signatures are listed below:\n" + + "\n" + lagunaToolJSON + "\n\n\n" + + "For each function call, return an unescaped XML-like object with function name and arguments within '' and '' tags, like here:\n" + + "function-name\nargument-key\nvalue-of-argument-key\n" + + "\n\n" + + "\nWeather?\n\n\n", + }, { name: "tools_in_header", messages: []api.Message{ @@ -102,6 +137,19 @@ func TestLagunaRendererReferenceFlowCoverage(t *testing.T) { "\n\n" + "\nWeather?\n\n\n", }, + { + name: "multiple_tools_in_header", + messages: []api.Message{{Role: "user", Content: "Add then report weather"}}, + tools: append(weather, lagunaMathTool()...), + want: "〈|EOS|〉\n\n" + lagunaDefaultSystem + "\n\n### Tools\n\n" + + "You may call functions to assist with the user query.\n" + + "All available function signatures are listed below:\n" + + "\n" + lagunaToolJSON + "\n" + lagunaMathToolJSON + "\n\n\n" + + "For each function call, return an unescaped XML-like object with function name and arguments within '' and '' tags, like here:\n" + + "function-name\nargument-key\nvalue-of-argument-key\n" + + "\n\n" + + "\nAdd then report weather\n\n\n", + }, { name: "assistant_history", messages: []api.Message{ @@ -138,12 +186,80 @@ func TestLagunaRendererReferenceFlowCoverage(t *testing.T) { "\nThanks\n\n\n", }, { - name: "final_assistant_prefill", + name: "assistant_extracts_thinking_from_content", messages: []api.Message{ - {Role: "user", Content: "Complete this"}, - {Role: "assistant", Content: "Partial"}, + {Role: "user", Content: "Explain"}, + {Role: "assistant", Content: "\nPlan\n\nAnswer\n\n"}, + {Role: "user", Content: "Next"}, }, - want: defaultHeader + "\nComplete this\n\n\n\nPartial\n", + think: think(true), + want: defaultHeader + + "\nExplain\n\n" + + "\n\nPlan\n\nAnswer\n\n" + + "\nNext\n\n\n", + }, + { + name: "assistant_thinking_metadata_overrides_content_tags", + messages: []api.Message{ + {Role: "user", Content: "Explain"}, + {Role: "assistant", Thinking: "Use metadata.", Content: "Ignore this\nAnswer"}, + {Role: "user", Content: "Next"}, + }, + want: defaultHeader + + "\nExplain\n\n" + + "\n\nUse metadata.\n\nAnswer\n\n" + + "\nNext\n\n\n", + }, + { + name: "assistant_whitespace_content_only", + messages: []api.Message{ + {Role: "user", Content: "Continue"}, + {Role: "assistant", Content: " \n\t "}, + {Role: "user", Content: "Next"}, + }, + want: defaultHeader + + "\nContinue\n\n" + + "\n\n\n" + + "\nNext\n\n\n", + }, + { + name: "assistant_multiple_tool_calls_mixed_args", + messages: []api.Message{ + {Role: "user", Content: "Do calls"}, + { + Role: "assistant", + ToolCalls: []api.ToolCall{ + {Function: api.ToolCallFunction{ + Name: "echo", + Arguments: testArgsOrdered([]orderedArg{ + {Key: "text", Value: "hello"}, + {Key: "count", Value: 2}, + }), + }}, + {Function: api.ToolCallFunction{ + Name: "configure", + Arguments: testArgsOrdered([]orderedArg{ + {Key: "flag", Value: true}, + {Key: "options", Value: map[string]any{"mode": "fast"}}, + }), + }}, + }, + }, + {Role: "user", Content: "Done?"}, + }, + want: defaultHeader + + "\nDo calls\n\n" + + "\n\n" + + "echo\n" + + "text\nhello\n" + + "count\n2\n" + + "\n" + + "configure\n" + + "flag\ntrue\n" + + "options\n{\"mode\": \"fast\"}\n" + + "\n" + + "\n" + + "\nDone?\n\n\n", }, } @@ -157,22 +273,346 @@ func TestLagunaRendererReferenceFlowCoverage(t *testing.T) { if diff := cmp.Diff(tt.want, got); diff != "" { t.Fatalf("renderer output mismatch vs template (-want +got):\n%s", diff) } + if verifyJinja2 { + jinja := renderLagunaJinja2Template(t, lagunaV2Template, tt.messages, tt.tools, tt.think) + if diff := cmp.Diff(jinja, tt.want); diff != "" { + t.Fatalf("hardcoded expected mismatch vs Jinja2 template (-jinja +want):\n%s", diff) + } + if diff := cmp.Diff(jinja, got); diff != "" { + t.Fatalf("renderer output mismatch vs Jinja2 template (-jinja +got):\n%s", diff) + } + } }) } } -func TestLagunaRendererMatchesLocalJinjaControlFlow(t *testing.T) { - if os.Getenv("VERIFY_LAGUNA_JINJA2") == "" { - t.Skip("set VERIFY_LAGUNA_JINJA2=1 to compare against the local Laguna chat_template.jinja") +func TestLagunaRendererAssistantPrefill(t *testing.T) { + got, err := (&LagunaRenderer{}).Render([]api.Message{ + {Role: "user", Content: "Complete this"}, + {Role: "assistant", Content: "Partial"}, + }, nil, nil) + if err != nil { + t.Fatal(err) } - python := "/Users/daniel/.codex/worktrees/7038/ollama/.venv/bin/python3" - if _, err := os.Stat(python); err != nil { - t.Fatalf("VERIFY_LAGUNA_JINJA2 requires %s with jinja2 installed", python) + + want := "〈|EOS|〉\n\n" + lagunaDefaultSystem + "\n\n" + + "\nComplete this\n\n\n\nPartial\n" + if diff := cmp.Diff(want, got); diff != "" { + t.Fatalf("renderer prefill mismatch (-want +got):\n%s", diff) } +} + +func TestLagunaRendererKnownJinja2Differences(t *testing.T) { + if !lagunaVerifyJinja2(t) { + t.Skip("set VERIFY_JINJA2=1 to run Jinja2 difference checks") + } + + messages := []api.Message{ + {Role: "user", Content: "Complete this"}, + {Role: "assistant", Content: "Partial"}, + } + got, err := (&LagunaRenderer{}).Render(messages, nil, nil) + if err != nil { + t.Fatal(err) + } + + jinja := renderLagunaJinja2Template(t, lagunaV2Template, messages, nil, nil) + if got == jinja { + t.Fatal("v2 assistant prefill no longer differs from Jinja2 output") + } + + wantJinja := "〈|EOS|〉\n\n" + lagunaDefaultSystem + "\n\n" + + "\nComplete this\n\n\n\nPartial\n\n\n" + if diff := cmp.Diff(wantJinja, jinja); diff != "" { + t.Fatalf("v2 assistant prefill Jinja2 reference mismatch (-want +jinja):\n%s", diff) + } +} + +func TestLagunaV8RendererReferenceFlowCoverage(t *testing.T) { + weather := lagunaWeatherTool() + think := func(v bool) *api.ThinkValue { return &api.ThinkValue{Value: v} } + verifyJinja2 := lagunaVerifyJinja2(t) + + defaultHeader := "〈|EOS|〉" + lagunaDefaultSystem + "\n" tests := []struct { name string messages []api.Message + tools []api.Tool + think *api.ThinkValue + want string + }{ + { + name: "empty_messages", + want: defaultHeader + "", + }, + { + name: "user_only_default", + messages: []api.Message{{Role: "user", Content: "Hello"}}, + want: defaultHeader + "Hello\n", + }, + { + name: "user_only_think", + messages: []api.Message{{Role: "user", Content: "Hello"}}, + think: think(true), + want: defaultHeader + "Hello\n", + }, + { + name: "user_only_nothink", + messages: []api.Message{{Role: "user", Content: "Hello"}}, + think: think(false), + want: defaultHeader + "Hello\n", + }, + { + name: "first_system_is_header", + messages: []api.Message{ + {Role: "system", Content: "Stay concise.\n\n"}, + {Role: "user", Content: "Hi"}, + }, + want: "〈|EOS|〉Stay concise.\n" + + "Hi\n", + }, + { + name: "empty_first_system_opts_out_of_header", + messages: []api.Message{ + {Role: "system", Content: ""}, + {Role: "user", Content: "Hi"}, + }, + want: "〈|EOS|〉Hi\n", + }, + { + name: "empty_first_system_with_tools", + messages: []api.Message{ + {Role: "system", Content: ""}, + {Role: "user", Content: "Weather?"}, + }, + tools: weather, + want: "〈|EOS|〉" + + "### Tools\n\n" + + "You may call functions to assist with the user query.\n" + + "All available function signatures are listed below:\n" + + "\n" + lagunaToolJSON + "\n" + + "\n" + + "Weather?\n", + }, + { + name: "empty_first_system_thinking_enabled", + messages: []api.Message{ + {Role: "system", Content: ""}, + {Role: "user", Content: "Hi"}, + }, + think: think(true), + want: "〈|EOS|〉\nHi\n", + }, + { + name: "additional_system", + messages: []api.Message{ + {Role: "system", Content: "Primary."}, + {Role: "user", Content: "Hi"}, + {Role: "system", Content: "Secondary."}, + }, + want: "〈|EOS|〉Primary.\n" + + "Hi\n" + + "Secondary.\n" + + "", + }, + { + name: "tools_in_header", + messages: []api.Message{ + {Role: "system", Content: "Stay concise."}, + {Role: "user", Content: "Weather?"}, + }, + tools: weather, + think: think(true), + want: "〈|EOS|〉Stay concise.\n\n" + + "### Tools\n\n" + + "You may call functions to assist with the user query.\n" + + "All available function signatures are listed below:\n" + + "\n" + lagunaToolJSON + "\n" + + "\n" + + "Weather?\n", + }, + { + name: "tools_default", + messages: []api.Message{{Role: "user", Content: "Weather?"}}, + tools: weather, + want: "〈|EOS|〉" + lagunaDefaultSystem + "\n\n" + + "### Tools\n\n" + + "You may call functions to assist with the user query.\n" + + "All available function signatures are listed below:\n" + + "\n" + lagunaToolJSON + "\n" + + "\n" + + "Weather?\n", + }, + { + name: "multiple_tools_in_header", + messages: []api.Message{{Role: "user", Content: "Add then report weather"}}, + tools: append(weather, lagunaMathTool()...), + want: "〈|EOS|〉" + lagunaDefaultSystem + "\n\n" + + "### Tools\n\n" + + "You may call functions to assist with the user query.\n" + + "All available function signatures are listed below:\n" + + "\n" + lagunaToolJSON + "\n" + lagunaMathToolJSON + "\n" + + "\n" + + "Add then report weather\n", + }, + { + name: "assistant_history", + messages: []api.Message{ + {Role: "user", Content: "Add these."}, + { + Role: "assistant", + Content: "\nCalling the tool.\n", + Thinking: "Need addition.", + ToolCalls: []api.ToolCall{{ + Function: api.ToolCallFunction{ + Name: "add", + Arguments: testArgsOrdered([]orderedArg{ + {Key: "a", Value: 2}, + {Key: "b", Value: 3}, + }), + }, + }}, + }, + {Role: "tool", Content: "5"}, + {Role: "user", Content: "Thanks"}, + }, + think: think(true), + want: defaultHeader + + "Add these.\n" + + "" + + "Need addition." + + "\nCalling the tool.\n" + + "add" + + "a2" + + "b3" + + "" + + "\n" + + "5\n" + + "Thanks\n", + }, + { + name: "assistant_reasoning_ignored_when_thinking_disabled", + messages: []api.Message{ + {Role: "user", Content: "Explain"}, + {Role: "assistant", Thinking: "Hidden plan.", Content: "Answer"}, + {Role: "user", Content: "Next"}, + }, + want: defaultHeader + + "Explain\n" + + "Answer\n" + + "Next\n", + }, + { + name: "assistant_empty_reasoning_when_thinking_enabled", + messages: []api.Message{ + {Role: "user", Content: "Explain"}, + {Role: "assistant", Content: "Answer"}, + {Role: "user", Content: "Next"}, + }, + think: think(true), + want: defaultHeader + + "Explain\n" + + "Answer\n" + + "Next\n", + }, + { + name: "assistant_preserves_content_whitespace", + messages: []api.Message{ + {Role: "user", Content: "Explain"}, + {Role: "assistant", Content: "\nAnswer\n"}, + {Role: "user", Content: "Next"}, + }, + want: defaultHeader + + "Explain\n" + + "\nAnswer\n\n" + + "Next\n", + }, + { + name: "assistant_multiple_tool_calls_mixed_args", + messages: []api.Message{ + {Role: "user", Content: "Do calls"}, + { + Role: "assistant", + ToolCalls: []api.ToolCall{ + {Function: api.ToolCallFunction{ + Name: "echo", + Arguments: testArgsOrdered([]orderedArg{ + {Key: "text", Value: "hello"}, + {Key: "count", Value: 2}, + }), + }}, + {Function: api.ToolCallFunction{ + Name: "configure", + Arguments: testArgsOrdered([]orderedArg{ + {Key: "flag", Value: true}, + {Key: "options", Value: map[string]any{"mode": "fast"}}, + }), + }}, + }, + }, + {Role: "user", Content: "Done?"}, + }, + want: defaultHeader + + "Do calls\n" + + "" + + "echo" + + "texthello" + + "count2" + + "" + + "configure" + + "flagtrue" + + "options{\"mode\": \"fast\"}" + + "" + + "\n" + + "Done?\n", + }, + { + name: "final_assistant_closes_then_generation_prompt", + messages: []api.Message{ + {Role: "user", Content: "Complete this"}, + {Role: "assistant", Content: "Partial"}, + }, + want: defaultHeader + + "Complete this\n" + + "Partial\n" + + "", + }, + } + + renderer := &LagunaV8Renderer{} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := renderer.Render(tt.messages, tt.tools, tt.think) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Fatalf("renderer output mismatch vs template (-want +got):\n%s", diff) + } + if verifyJinja2 { + jinja := renderLagunaJinja2Template(t, lagunaV8Template, tt.messages, tt.tools, tt.think) + if diff := cmp.Diff(jinja, tt.want); diff != "" { + t.Fatalf("hardcoded expected mismatch vs Jinja2 template (-jinja +want):\n%s", diff) + } + if diff := cmp.Diff(jinja, got); diff != "" { + t.Fatalf("renderer output mismatch vs Jinja2 template (-jinja +got):\n%s", diff) + } + } + }) + } +} + +func TestLagunaRendererMatchesJinja2ExpandedParity(t *testing.T) { + if os.Getenv("VERIFY_JINJA2") == "" { + t.Skip("set VERIFY_JINJA2=1 to run expanded Jinja2 parity checks") + } + lagunaVerifyJinja2(t) + + tests := []struct { + name string + messages []api.Message + tools []api.Tool think *api.ThinkValue }{ { @@ -206,77 +646,198 @@ func TestLagunaRendererMatchesLocalJinjaControlFlow(t *testing.T) { messages: []api.Message{{Role: "user", Content: "Answer directly."}}, think: &api.ThinkValue{Value: false}, }, + { + name: "tools_and_assistant_history", + messages: []api.Message{ + {Role: "system", Content: "Stay concise."}, + {Role: "user", Content: "Weather?"}, + {Role: "assistant", Content: "Calling.", Thinking: "Need weather.", ToolCalls: []api.ToolCall{{ + Function: api.ToolCallFunction{ + Name: "get_weather", + Arguments: testArgsOrdered([]orderedArg{{Key: "location", Value: "Paris"}}), + }, + }}}, + {Role: "tool", Content: "Sunny"}, + {Role: "user", Content: "Thanks"}, + }, + tools: lagunaWeatherTool(), + think: &api.ThinkValue{Value: true}, + }, } - renderer := &LagunaRenderer{} - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := renderer.Render(tt.messages, nil, tt.think) - if err != nil { - t.Fatal(err) - } - for _, modelDir := range []string{ - "/Users/daniel/Models/poolside/laguna-xs-23-04-2026", - } { - want := renderLagunaChatTemplate(t, python, modelDir, tt.messages, tt.think) - if diff := cmp.Diff(want, got); diff != "" { - t.Fatalf("%s mismatch (-chat_template +renderer):\n%s", modelDir, diff) - } + variants := []struct { + name string + renderer Renderer + template string + }{ + {name: "v2", renderer: &LagunaRenderer{}, template: lagunaV2Template}, + {name: "v8", renderer: &LagunaV8Renderer{}, template: lagunaV8Template}, + } + + for _, variant := range variants { + t.Run(variant.name, func(t *testing.T) { + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := variant.renderer.Render(tt.messages, tt.tools, tt.think) + if err != nil { + t.Fatal(err) + } + want := renderLagunaJinja2Template(t, variant.template, tt.messages, tt.tools, tt.think) + if diff := cmp.Diff(want, got); diff != "" { + t.Fatalf("renderer output mismatch vs Jinja2 template (-jinja +got):\n%s", diff) + } + }) } }) } } -func renderLagunaChatTemplate(t *testing.T, python, modelDir string, messages []api.Message, think *api.ThinkValue) string { +func lagunaVerifyJinja2(t *testing.T) bool { + t.Helper() + if os.Getenv("VERIFY_JINJA2") == "" { + return false + } + python := lagunaJinjaPython(t) + cmd := exec.Command(python, "-c", "from transformers.utils.chat_template_utils import _compile_jinja_template") + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("VERIFY_JINJA2=1 requires transformers chat template support in %s: %v\n%s", python, err, out) + } + return true +} + +func lagunaJinjaPython(t *testing.T) string { + t.Helper() + python, err := exec.LookPath("python3") + if err != nil { + t.Fatal("VERIFY_JINJA2=1 requires python3 on PATH") + } + return python +} + +func renderLagunaJinja2Template(t *testing.T, templateRelPath string, messages []api.Message, tools []api.Tool, think *api.ThinkValue) string { t.Helper() - type templateMessage struct { - Role string `json:"role"` - Content string `json:"content"` + templatePath, err := filepath.Abs(templateRelPath) + if err != nil { + t.Fatalf("failed to get template path: %v", err) } - templateMessages := make([]templateMessage, 0, len(messages)) + + type jinjaToolCall struct { + Function struct { + Name string `json:"name"` + Arguments json.RawMessage `json:"arguments"` + } `json:"function"` + } + type jinjaMessage struct { + Role string `json:"role"` + Content string `json:"content"` + Reasoning string `json:"reasoning,omitempty"` + ReasoningContent string `json:"reasoning_content,omitempty"` + ToolCalls []jinjaToolCall `json:"tool_calls,omitempty"` + } + + jinjaMessages := make([]jinjaMessage, 0, len(messages)) for _, msg := range messages { - templateMessages = append(templateMessages, templateMessage{ - Role: msg.Role, - Content: msg.Content, - }) + jm := jinjaMessage{ + Role: msg.Role, + Content: msg.Content, + Reasoning: msg.Thinking, + ReasoningContent: msg.Thinking, + } + for _, call := range msg.ToolCalls { + jc := jinjaToolCall{} + jc.Function.Name = call.Function.Name + raw, err := call.Function.Arguments.MarshalJSON() + if err != nil { + t.Fatalf("failed to marshal tool args: %v", err) + } + jc.Function.Arguments = json.RawMessage(raw) + jm.ToolCalls = append(jm.ToolCalls, jc) + } + jinjaMessages = append(jinjaMessages, jm) } - messagesJSON, err := json.Marshal(templateMessages) + + messagesJSON, err := json.Marshal(jinjaMessages) if err != nil { t.Fatalf("failed to marshal messages: %v", err) } - enableThinking := "False" - if think != nil && think.Bool() { - enableThinking = "True" + toolsJSON := "None" + if len(tools) > 0 { + b, err := json.Marshal(tools) + if err != nil { + t.Fatalf("failed to marshal tools: %v", err) + } + toolsJSON = string(b) + } + + enableThinking := "unset" + if think != nil { + if think.Bool() { + enableThinking = "true" + } else { + enableThinking = "false" + } } script := ` import json import sys -from transformers import AutoTokenizer +from pathlib import Path +from transformers.utils.chat_template_utils import _compile_jinja_template -model_dir = sys.argv[1] -messages = json.loads(sys.argv[2]) -enable_thinking = sys.argv[3] == "True" -tok = AutoTokenizer.from_pretrained(model_dir, trust_remote_code=True) -print(tok.apply_chat_template( - messages, - tokenize=False, - add_generation_prompt=True, - enable_thinking=enable_thinking, -), end="") +template_path, messages_json, tools_json, enable_thinking = sys.argv[1:5] +tmpl = _compile_jinja_template(Path(template_path).read_text()) +kwargs = { + "messages": json.loads(messages_json), + "add_generation_prompt": True, +} +if tools_json != "None": + kwargs["tools"] = json.loads(tools_json) +if enable_thinking == "true": + kwargs["enable_thinking"] = True +elif enable_thinking == "false": + kwargs["enable_thinking"] = False +print(tmpl.render(**kwargs), end="") ` - cmd := exec.Command(python, "-c", script, modelDir, string(messagesJSON), enableThinking) + cmd := exec.Command(lagunaJinjaPython(t), "-c", script, templatePath, string(messagesJSON), toolsJSON, enableThinking) var stdout, stderr strings.Builder cmd.Stdout = &stdout cmd.Stderr = &stderr if err := cmd.Run(); err != nil { - t.Fatalf("chat_template render failed: %v\nstderr: %s", err, stderr.String()) + t.Fatalf("python render failed: %v\nstderr: %s", err, stderr.String()) } return stdout.String() } +func TestLagunaTemplateFixturesMatchExpectedVersions(t *testing.T) { + v2, err := os.ReadFile(lagunaV2Template) + if err != nil { + t.Fatalf("failed to read %s: %v", lagunaV2Template, err) + } + + v8, err := os.ReadFile(lagunaV8Template) + if err != nil { + t.Fatalf("failed to read %s: %v", lagunaV8Template, err) + } + + if !strings.Contains(string(v2), "laguna_glm_thinking_v5/chat_template.jinja") { + t.Fatalf("%s does not look like the v2 Laguna template fixture", lagunaV2Template) + } + if !strings.Contains(string(v8), "laguna_glm_thinking_v8/chat_template.jinja") { + t.Fatalf("%s does not look like the v8 Laguna template fixture", lagunaV8Template) + } + if !strings.Contains(string(v2), "render_assistant_messages_raw") { + t.Fatalf("%s should retain the v2 raw assistant branch", lagunaV2Template) + } + if strings.Contains(string(v8), "render_assistant_messages_raw") { + t.Fatalf("%s unexpectedly contains the v2 raw assistant branch", lagunaV8Template) + } + if diff := cmp.Diff(string(v2), string(v8)); diff == "" { + t.Fatal("Laguna v2 and v8 template fixtures unexpectedly match") + } +} + func lagunaWeatherTool() []api.Tool { return []api.Tool{{ Type: "function", @@ -297,3 +858,33 @@ func lagunaWeatherTool() []api.Tool { }, }} } + +func lagunaMathTool() []api.Tool { + return []api.Tool{{ + Type: "function", + Function: api.ToolFunction{ + Name: "add", + Description: "Add numbers", + Parameters: api.ToolFunctionParameters{ + Type: "object", + Required: []string{"a", "b"}, + Properties: testPropsOrdered([]orderedProp{ + { + Key: "a", + Value: api.ToolProperty{ + Type: api.PropertyType{"number"}, + Description: "First number", + }, + }, + { + Key: "b", + Value: api.ToolProperty{ + Type: api.PropertyType{"number"}, + Description: "Second number", + }, + }, + }), + }, + }, + }} +} diff --git a/model/renderers/renderer.go b/model/renderers/renderer.go index 53520c794..4223806c8 100644 --- a/model/renderers/renderer.go +++ b/model/renderers/renderer.go @@ -109,6 +109,8 @@ func rendererForName(name string) Renderer { return &LFM2Renderer{IsThinking: true, useImgTags: RenderImgTags} case "laguna": return &LagunaRenderer{} + case "laguna-v8": + return &LagunaV8Renderer{} case "cohere": return &CohereRenderer{} default: diff --git a/model/renderers/renderer_test.go b/model/renderers/renderer_test.go index 5f9db37ee..134d1b45f 100644 --- a/model/renderers/renderer_test.go +++ b/model/renderers/renderer_test.go @@ -69,6 +69,7 @@ func TestLeadingBOSForRenderer(t *testing.T) { {name: "lfm2", want: "<|startoftext|>"}, {name: "lfm2-thinking", want: "<|startoftext|>"}, {name: "laguna", want: "〈|EOS|〉"}, + {name: "laguna-v8", want: "〈|EOS|〉"}, {name: "deepseek3.1", want: "<｜begin▁of▁sentence｜>"}, {name: "cogito", want: "<｜begin▁of▁sentence｜>"}, {name: "qwen3-coder", want: ""}, diff --git a/model/renderers/testdata/laguna_v2_chat_template.jinja2 b/model/renderers/testdata/laguna_v2_chat_template.jinja2 new file mode 100644 index 000000000..4baa3fded --- /dev/null +++ b/model/renderers/testdata/laguna_v2_chat_template.jinja2 @@ -0,0 +1,132 @@ +{#- Iteration on laguna_glm_thinking_v5/chat_template.jinja -#} +{#- Adds a default system message (used when no system message is provided in `messages`). -#} +{{- "〈|EOS|〉" -}} +{%- set enable_thinking = enable_thinking | default(false) -%} +{%- set render_assistant_messages_raw = render_assistant_messages_raw | default(false) -%} +{%- set add_generation_prompt = add_generation_prompt | default(false) -%} + +{#- ───── header (system message) ───── -#} +{%- set system_message = "You are a helpful, conversationally-fluent assistant made by Poolside. You are here to be helpful to users through natural language conversations." -%} +{%- if messages and messages[0].role == "system" -%} + {%- set system_message = messages[0].content -%} +{%- endif -%} + +{%- if (system_message and system_message.strip()) or tools -%} + {{- "\n" -}} + + {%- if system_message and system_message.strip() -%} + {{- "\n" -}} + {{- system_message.rstrip() -}} + {%- endif -%} + + {%- if tools -%} + {{- "\n\n### Tools\n\n" -}} + {%- set ns = namespace(tool_string="You may call functions to assist with the user query.\n" + ~ "All available function signatures are listed below:\n" + ~ "\n") -%} + {%- for tool in tools -%} + {%- set ns.tool_string = ns.tool_string ~ (tool | tojson) ~ "\n" -%} + {%- endfor -%} + {%- if enable_thinking -%} + {%- set tool_string = ns.tool_string + "\n\n" ~ + "Wrap your thinking in '', '' tags, followed by a function call. For each function call, return an unescaped XML-like object with function name and arguments within '' and '' tags, like here:\n" ~ + " your thoughts here \n" ~ + "function-name\nargument-key\nvalue-of-argument-key\n" ~ + "" -%} + {%- else -%} + {%- set tool_string = ns.tool_string + "\n\n" ~ + "For each function call, return an unescaped XML-like object " ~ + "with function name and arguments within '' and '' tags, like here:\n" ~ + "function-name\nargument-key\nvalue-of-argument-key\n" ~ + "" -%} + {%- endif -%} + {{- tool_string -}} + {%- endif -%} + + {{- "\n\n" -}} +{%- endif -%} + +{#- ───── main loop ───── -#} +{%- for message in messages -%} + {%- set content = message.content if message.content is string else "" -%} + {%- if message.role == "user" -%} + {{- "\n" + content + "\n\n" -}} + {%- elif message.role == "assistant" -%} + {%- generation -%} + {{- "\n" -}} + {%- if render_assistant_messages_raw -%} + {#- Raw mode: prepend the generation prompt token, then dump content verbatim. -#} + {#- The generation prompt is when enable_thinking, otherwise. -#} + {#- Only prepend if content doesn't already start with it. -#} + {%- if enable_thinking -%} + {%- if not content.startswith('') -%} + {{- '' -}} + {%- endif -%} + {%- else -%} + {%- if not content.startswith('') -%} + {{- '' -}} + {%- endif -%} + {%- endif -%} + {{- content -}} + {#- Append closing tag if content doesn't already end with it. -#} + {%- if not content.endswith('\n') and not content.endswith('') -%} + {{- '\n' -}} + {%- endif -%} + {{- "\n" -}} + {%- else -%} + {#- Extract reasoning content from message.reasoning (vLLM field name) or message.reasoning_content, or from tags -#} + {%- set reasoning_content = '' %} + {%- if message.reasoning is string %} + {%- set reasoning_content = message.reasoning %} + {%- elif message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- endif %} + {#- Always strip tags from content if present to avoid duplication -#} + {%- if '' in content %} + {%- if not reasoning_content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {#- Display reasoning content for all messages -#} + {%- if reasoning_content -%} + {{- '\n' + reasoning_content.strip() + '\n\n' -}} + {%- else -%} + {{- '\n' -}} + {%- endif -%} + {#- Display main content -#} + {%- if content.strip() -%} + {{- content.strip() ~ "\n" -}} + {%- endif -%} + {%- if message.tool_calls -%} + {%- for tool_call in message.tool_calls -%} + {%- set function_data = tool_call.function -%} + {{- '' + function_data.name }} + {% set _args = function_data.arguments %} + {%- for k, v in _args.items() -%} + {{- "" ~ k ~ "\n" -}} + {{- ""}}{{ v | tojson(ensure_ascii=False) if v is not string else v }}{{ "\n" -}} + {%- endfor -%} + {{- "\n" -}} + {%- endfor -%} + {%- endif -%} + {{- "\n" -}} + {%- endif -%} + {%- endgeneration -%} + {%- elif message.role == "tool" -%} + {{- "\n" + content + "\n\n" -}} + {%- elif message.role == "system" and loop.index0 != 0 -%} + {#- Render additional system messages (skip the first one which is handled separately in the header) -#} + {{- "\n" + content + "\n\n" -}} + {%- endif -%} +{%- endfor -%} +{#- ───── generation prompt ───── -#} +{%- if add_generation_prompt -%} + {{- "\n" -}} + {#- ───── Include reasoning mode directive ───── -#} + {%- if not enable_thinking %} + {{- '' -}} + {%- else %} + {{- '' -}} + {%- endif %} +{%- endif -%} diff --git a/model/renderers/testdata/laguna_v8_chat_template.jinja2 b/model/renderers/testdata/laguna_v8_chat_template.jinja2 new file mode 100644 index 000000000..d1a3db589 --- /dev/null +++ b/model/renderers/testdata/laguna_v8_chat_template.jinja2 @@ -0,0 +1,93 @@ +{#- Iteration on laguna_glm_thinking_v8/chat_template.jinja -#} +{#- No formatting instructions -#} +{{- "〈|EOS|〉" -}} +{%- set enable_thinking = enable_thinking | default(false) -%} +{%- set add_generation_prompt = add_generation_prompt | default(false) -%} + +{#- ───── header (system message) ───── -#} +{#- A caller-supplied system message with empty content opts out of the default below, producing no block — used to train without a system message. -#} +{%- set system_message = "You are a helpful, conversationally-fluent assistant made by Poolside. You are here to be helpful to users through natural language conversations." -%} +{%- if messages and messages[0].role == "system" -%} + {%- set system_message = messages[0].content -%} + {%- set messages = messages[1:] -%} +{%- endif -%} + +{%- set has_sys = system_message and system_message.strip() -%} +{%- if has_sys or tools or enable_thinking -%} + {{- "" -}} + + {%- if has_sys -%} + {{- system_message.rstrip() -}} + {%- if tools -%}{{- "\n\n" -}}{%- endif -%} + {%- endif -%} + + {%- if tools -%} + {{- "### Tools\n\n" -}} + {{- "You may call functions to assist with the user query.\n" -}} + {{- "All available function signatures are listed below:\n" -}} + {{- "\n" -}} + {%- for tool in tools -%} + {{- (tool | tojson) ~ "\n" -}} + {%- endfor -%} + {{- "" -}} + {%- endif -%} + + {{- "\n" -}} +{%- endif -%} + +{#- ───── main loop ───── -#} +{%- for message in messages -%} + {%- set content = message.content if message.content is string else "" -%} + {%- if message.role == "user" -%} + {{- "" + content + "\n" -}} + {%- elif message.role == "assistant" -%} + {%- generation -%} + {{- "" -}} + {#- Extract reasoning content from message.reasoning (vLLM field name) or message.reasoning_content -#} + {%- set reasoning_content = '' -%} + {%- if message.reasoning is string -%} + {%- set reasoning_content = message.reasoning -%} + {%- elif message.reasoning_content is string -%} + {%- set reasoning_content = message.reasoning_content -%} + {%- endif -%} + {#- Display reasoning content for all messages if enable_thinking -#} + {%- if enable_thinking -%} + {{- '' + reasoning_content + '' -}} + {%- else -%} + {{- '' -}} + {%- endif -%} + {#- Display main content (trailing newline only when no tool_calls follow) -#} + {%- if content -%} + {{- content -}} + {%- endif -%} + {%- if message.tool_calls -%} + {%- for tool_call in message.tool_calls -%} + {%- set function_data = tool_call.function -%} + {{- '' + function_data.name -}} + {%- set _args = function_data.arguments -%} + {%- for k, v in _args.items() -%} + {{- "" ~ k ~ "" -}} + {{- "" -}}{{- v | tojson(ensure_ascii=False) if v is not string else v -}}{{- "" -}} + {%- endfor -%} + {{- "" -}} + {%- endfor -%} + {%- endif -%} + {{- "\n" -}} + {%- endgeneration -%} + {%- elif message.role == "tool" -%} + {{- "" + content + "\n" -}} + {%- elif message.role == "system" -%} + {#- Render additional system messages (the first one, if any, is handled separately in the header and was sliced off above) -#} + {{- "" + content + "\n" -}} + {%- endif -%} +{%- endfor -%} +{#- ───── generation prompt ───── -#} +{%- if add_generation_prompt -%} + {{- "" -}} + {#- ───── Include reasoning mode directive ───── -#} + {%- if enable_thinking -%} + {{- '' -}} + {%- else -%} + {{- '' -}} + {%- endif -%} +{%- endif -%} diff --git a/parser/parser.go b/parser/parser.go index b9ff36cf4..8940c38f5 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -380,6 +380,14 @@ func filesForModel(path string) ([]string, error) { } files = append(files, js...) + // Transformers stores a tokenizer's default template in this standalone + // file when it is not embedded in tokenizer_config.json. + chatTemplates, err := glob(filepath.Join(path, "chat_template.jinja"), "text/plain") + if err != nil { + return nil, err + } + files = append(files, chatTemplates...) + // add tokenizer.model if it exists (tokenizer.json is automatically picked up by the previous glob) // tokenizer.model might be a unresolved git lfs reference; error if it is if tks, _ := glob(filepath.Join(path, "tokenizer.model"), "application/octet-stream"); len(tks) > 0 { diff --git a/parser/parser_test.go b/parser/parser_test.go index aa892d603..75cfdd054 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -936,6 +936,7 @@ func TestFilesForModel(t *testing.T) { "model-00002-of-00002.safetensors", "config.json", "tokenizer.json", + "chat_template.jinja", } for _, file := range files { if err := os.WriteFile(filepath.Join(dir, file), []byte("test content"), 0o644); err != nil { @@ -949,6 +950,7 @@ func TestFilesForModel(t *testing.T) { "model-00002-of-00002.safetensors", "config.json", "tokenizer.json", + "chat_template.jinja", }, }, {