anthropic: Preserve Claude local image-path tool results in renderer-owned prompt formatting (#16047)

model/renderers/gemma4.go

@@ -98,7 +98,8 @@ func (r *Gemma4Renderer) Render(messages []api.Message, tools []api.Tool, thinkV
 		toolResponsesEmitted := false
 		if len(message.ToolCalls) > 0 {
 			for k := i + 1; k < len(loopMessages) && loopMessages[k].Role == "tool"; k++ {
-				sb.WriteString(r.formatToolResponseBlock(r.toolResponseName(loopMessages[k], message.ToolCalls), loopMessages[k].Content))
+				response := r.renderToolResponseContent(loopMessages[k], &imageOffset)
+				sb.WriteString(r.formatToolResponseBlock(r.toolResponseName(loopMessages[k], message.ToolCalls), response))
 				toolResponsesEmitted = true
 				prevMessageType = "tool_response"
 			}
@@ -160,19 +161,22 @@ func stripThinking(text string) string {
 // When trim is true, leading/trailing whitespace is stripped (matching the Jinja2
 // template's | trim filter applied to non-model content).
 func (r *Gemma4Renderer) renderContent(sb *strings.Builder, msg api.Message, imageOffset *int, trim bool) {
-	if len(msg.Images) > 0 && r.useImgTags {
-		for range msg.Images {
-			sb.WriteString(fmt.Sprintf("[img-%d]", *imageOffset))
-			*imageOffset++
-		}
-	}
 	content := msg.Content
 	if trim {
 		content = strings.TrimSpace(content)
 	}
+	if len(msg.Images) > 0 && r.useImgTags {
+		content, *imageOffset = renderContentWithImageTags(content, len(msg.Images), *imageOffset)
+	}
 	sb.WriteString(content)
 }
 
+func (r *Gemma4Renderer) renderToolResponseContent(msg api.Message, imageOffset *int) string {
+	var sb strings.Builder
+	r.renderContent(&sb, msg, imageOffset, false)
+	return sb.String()
+}
+
 func (r *Gemma4Renderer) previousNonToolRole(messages []api.Message, idx int) string {
 	for i := idx - 1; i >= 0; i-- {
 		if messages[i].Role != "tool" {

model/renderers/glmocr.go

@@ -13,15 +13,11 @@ type GlmOcrRenderer struct {
 }
 
 func (r *GlmOcrRenderer) renderContent(message api.Message, imageOffset int) (string, int) {
-	var sb strings.Builder
-	for range message.Images {
-		if r.useImgTags {
-			sb.WriteString(fmt.Sprintf("[img-%d]", imageOffset))
-			imageOffset++
-		}
+	if r.useImgTags {
+		return renderContentWithImageTags(message.Content, len(message.Images), imageOffset)
 	}
-	sb.WriteString(message.Content)
-	return sb.String(), imageOffset
+
+	return message.Content, imageOffset
 }
 
 func (r *GlmOcrRenderer) Render(messages []api.Message, tools []api.Tool, thinkValue *api.ThinkValue) (string, error) {
@@ -85,8 +81,10 @@ func (r *GlmOcrRenderer) Render(messages []api.Message, tools []api.Tool, thinkV
 			if i == 0 || messages[i-1].Role != "tool" {
 				sb.WriteString("<|observation|>")
 			}
+			content, nextOffset := r.renderContent(message, imageOffset)
+			imageOffset = nextOffset
 			sb.WriteString("\n<tool_response>\n")
-			sb.WriteString(message.Content)
+			sb.WriteString(content)
 			sb.WriteString("\n</tool_response>\n")
 		case "system":
 			sb.WriteString("<|system|>\n")

model/renderers/glmocr_test.go

@@ -25,7 +25,7 @@ func TestGlmOcrRenderer_Images(t *testing.T) {
 					Images:  []api.ImageData{api.ImageData("img1")},
 				},
 			},
-			expected: "[gMASK]<sop><|user|>\n[img-0]Describe this image.<|assistant|>\n",
+			expected: "[gMASK]<sop><|user|>\n[img-0] Describe this image.<|assistant|>\n",
 		},
 		{
 			name:     "use_img_tags_multiple_images",
@@ -37,7 +37,7 @@ func TestGlmOcrRenderer_Images(t *testing.T) {
 					Images:  []api.ImageData{api.ImageData("img1"), api.ImageData("img2")},
 				},
 			},
-			expected: "[gMASK]<sop><|user|>\n[img-0][img-1]Describe these images.<|assistant|>\n",
+			expected: "[gMASK]<sop><|user|>\n[img-0][img-1] Describe these images.<|assistant|>\n",
 		},
 		{
 			name:     "multi_turn_increments_image_offset",
@@ -58,7 +58,7 @@ func TestGlmOcrRenderer_Images(t *testing.T) {
 					Images:  []api.ImageData{api.ImageData("img2")},
 				},
 			},
-			expected: "[gMASK]<sop><|user|>\n[img-0]First image<|assistant|>\n<think></think>\nProcessed.\n<|user|>\n[img-1]Second image<|assistant|>\n",
+			expected: "[gMASK]<sop><|user|>\n[img-0] First image<|assistant|>\n<think></think>\nProcessed.\n<|user|>\n[img-1] Second image<|assistant|>\n",
 		},
 		{
 			name:     "default_no_img_tags",

model/renderers/image_tags.go

@@ -0,0 +1,39 @@
+package renderers
+
+import (
+	"fmt"
+	"strings"
+	"unicode"
+	"unicode/utf8"
+)
+
+// renderContentWithImageTags preserves the legacy server-side placeholder
+// semantics for explicit [img] tokens: replace placeholders in order, and
+// only prepend tags for any remaining images without placeholders.
+func renderContentWithImageTags(content string, imageCount int, imageOffset int) (string, int) {
+	if imageCount == 0 {
+		return content, imageOffset
+	}
+
+	if strings.Contains(content, "[img-") {
+		return content, imageOffset + imageCount
+	}
+
+	var prefix strings.Builder
+	for i := range imageCount {
+		imgTag := fmt.Sprintf("[img-%d]", imageOffset+i)
+		if strings.Contains(content, "[img]") {
+			content = strings.Replace(content, "[img]", imgTag, 1)
+		} else {
+			prefix.WriteString(imgTag)
+		}
+	}
+
+	if prefix.Len() > 0 && content != "" {
+		if r, _ := utf8.DecodeRuneInString(content); r != utf8.RuneError && !unicode.IsSpace(r) {
+			prefix.WriteByte(' ')
+		}
+	}
+
+	return prefix.String() + content, imageOffset + imageCount
+}

model/renderers/image_tags_test.go

@@ -0,0 +1,67 @@
+package renderers
+
+import "testing"
+
+func TestRenderContentWithImageTags(t *testing.T) {
+	tests := []struct {
+		name        string
+		content     string
+		imageCount  int
+		imageOffset int
+		want        string
+		wantOffset  int
+	}{
+		{
+			name:        "prefixes when there are no placeholders",
+			content:     "describe this image",
+			imageCount:  2,
+			imageOffset: 0,
+			want:        "[img-0][img-1] describe this image",
+			wantOffset:  2,
+		},
+		{
+			name:        "replaces explicit placeholders in order",
+			content:     "compare [img] and [img]",
+			imageCount:  2,
+			imageOffset: 3,
+			want:        "compare [img-3] and [img-4]",
+			wantOffset:  5,
+		},
+		{
+			name:        "prefixes extra images after placeholders are exhausted",
+			content:     "compare [img]",
+			imageCount:  2,
+			imageOffset: 0,
+			want:        "[img-1] compare [img-0]",
+			wantOffset:  2,
+		},
+		{
+			name:        "leaves leftover placeholders when there are fewer images",
+			content:     "compare [img] and [img]",
+			imageCount:  1,
+			imageOffset: 0,
+			want:        "compare [img-0] and [img]",
+			wantOffset:  1,
+		},
+		{
+			name:        "preserves already-numbered placeholders",
+			content:     "compare [img-0] and [img-1]",
+			imageCount:  2,
+			imageOffset: 0,
+			want:        "compare [img-0] and [img-1]",
+			wantOffset:  2,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, gotOffset := renderContentWithImageTags(tt.content, tt.imageCount, tt.imageOffset)
+			if got != tt.want {
+				t.Fatalf("content = %q, want %q", got, tt.want)
+			}
+			if gotOffset != tt.wantOffset {
+				t.Fatalf("offset = %d, want %d", gotOffset, tt.wantOffset)
+			}
+		})
+	}
+}

model/renderers/lfm2.go

@@ -3,7 +3,6 @@ package renderers
 import (
 	"bytes"
 	"encoding/json"
-	"fmt"
 	"sort"
 	"strings"
 
@@ -199,19 +198,18 @@ func (r *LFM2Renderer) renderMessageContent(message api.Message, imageOffset int
 		return content
 	}
 
-	var sb strings.Builder
 	if r.useImgTags {
-		for i := range message.Images {
-			sb.WriteString(fmt.Sprintf("[img-%d]", imageOffset+i))
-		}
-	} else {
-		placeholder := lfm2ImagePlaceholder(false)
-		if strings.Contains(content, placeholder) {
-			return content
-		}
-		for range message.Images {
-			sb.WriteString(placeholder)
-		}
+		content, _ = renderContentWithImageTags(content, len(message.Images), imageOffset)
+		return content
+	}
+
+	var sb strings.Builder
+	placeholder := lfm2ImagePlaceholder(false)
+	if strings.Contains(content, placeholder) {
+		return content
+	}
+	for range message.Images {
+		sb.WriteString(placeholder)
 	}
 	sb.WriteString(content)
 	return sb.String()

model/renderers/lfm2_test.go

@@ -236,7 +236,7 @@ func TestLFM2Renderer_Images(t *testing.T) {
 				Content: "Describe this image.",
 				Images:  []api.ImageData{api.ImageData("img1")},
 			},
-			expected: "<|startoftext|><|im_start|>user\n[img-0]Describe this image.<|im_end|>\n<|im_start|>assistant\n",
+			expected: "<|startoftext|><|im_start|>user\n[img-0] Describe this image.<|im_end|>\n<|im_start|>assistant\n",
 		},
 		{
 			name:     "existing_template_image_placeholder_not_duplicated",

model/renderers/nemotron3nano.go

@@ -79,12 +79,14 @@ func (r *Nemotron3NanoRenderer) Render(messages []api.Message, tools []api.Tool,
 			// Check if previous message was also a tool message
 			prevWasTool := i > 0 && loopMessages[i-1].Role == "tool"
 			nextIsTool := i+1 < len(loopMessages) && loopMessages[i+1].Role == "tool"
+			content := r.renderMessageContent(message, imageOffset)
+			imageOffset += len(message.Images)
 
 			if !prevWasTool {
 				sb.WriteString("<|im_start|>user\n")
 			}
 			sb.WriteString("<tool_response>\n")
-			sb.WriteString(message.Content)
+			sb.WriteString(content)
 			sb.WriteString("\n</tool_response>\n")
 
 			if !nextIsTool {
@@ -237,23 +239,8 @@ func (r *Nemotron3NanoRenderer) renderMessageContent(message api.Message, imageO
 		return content
 	}
 
-	if strings.Contains(content, "[img-") {
-		return content
-	}
-
-	if strings.Contains(content, "[img]") {
-		for i := range message.Images {
-			content = strings.Replace(content, "[img]", fmt.Sprintf("[img-%d]", imageOffset+i), 1)
-		}
-		return content
-	}
-
-	var sb strings.Builder
-	for i := range message.Images {
-		sb.WriteString(fmt.Sprintf("[img-%d]", imageOffset+i))
-	}
-	sb.WriteString(content)
-	return sb.String()
+	content, _ = renderContentWithImageTags(content, len(message.Images), imageOffset)
+	return content
 }
 
 func nemotron3NanoRenderContent(content any) string {

model/renderers/nemotron3nano_test.go

@@ -19,7 +19,7 @@ func TestNemotron3NanoRenderer_Images(t *testing.T) {
 			msgs: []api.Message{
 				{Role: "user", Content: "Describe this image.", Images: []api.ImageData{api.ImageData("img1")}},
 			},
-			expected: "\n\n\n<|im_start|>system\n<|im_end|>\n\n<|im_start|>user\n[img-0]Describe this image.<|im_end|>\n\n<|im_start|>assistant\n<think>\n",
+			expected: "\n\n\n<|im_start|>system\n<|im_end|>\n\n<|im_start|>user\n[img-0] Describe this image.<|im_end|>\n\n<|im_start|>assistant\n<think>\n",
 		},
 		{
 			name: "generic image placeholder is rewritten",
@@ -35,7 +35,7 @@ func TestNemotron3NanoRenderer_Images(t *testing.T) {
 				{Role: "assistant", Content: "It shows something."},
 				{Role: "user", Content: "Compare these.", Images: []api.ImageData{api.ImageData("img2"), api.ImageData("img3")}},
 			},
-			expected: "\n\n\n<|im_start|>system\n<|im_end|>\n\n<|im_start|>user\n[img-0]Describe the first image.<|im_end|>\n<|im_start|>assistant\n<think></think>It shows something.<|im_end|>\n<|im_start|>user\n[img-1][img-2]Compare these.<|im_end|>\n\n<|im_start|>assistant\n<think>\n",
+			expected: "\n\n\n<|im_start|>system\n<|im_end|>\n\n<|im_start|>user\n[img-0] Describe the first image.<|im_end|>\n<|im_start|>assistant\n<think></think>It shows something.<|im_end|>\n<|im_start|>user\n[img-1][img-2] Compare these.<|im_end|>\n\n<|im_start|>assistant\n<think>\n",
 		},
 	}
 

model/renderers/qwen35.go

@@ -1,7 +1,6 @@
 package renderers
 
 import (
-	"fmt"
 	"strings"
 
 	"github.com/ollama/ollama/api"
@@ -45,15 +44,14 @@ type Qwen35Renderer struct {
 }
 
 func (r *Qwen35Renderer) renderContent(content api.Message, imageOffset int) (string, int) {
+	if r.useImgTags {
+		return renderContentWithImageTags(content.Content, len(content.Images), imageOffset)
+	}
+
 	// This assumes all images are at the front of the message - same assumption as ollama/ollama/runner.go
 	var subSb strings.Builder
 	for range content.Images {
-		if r.useImgTags {
-			subSb.WriteString(fmt.Sprintf("[img-%d]", imageOffset))
-			imageOffset++
-		} else {
-			subSb.WriteString("<|vision_start|><|image_pad|><|vision_end|>")
-		}
+		subSb.WriteString("<|vision_start|><|image_pad|><|vision_end|>")
 	}
 	// TODO: support videos
 

model/renderers/qwen3vl.go

@@ -1,7 +1,6 @@
 package renderers
 
 import (
-	"fmt"
 	"strings"
 
 	"github.com/ollama/ollama/api"
@@ -15,18 +14,17 @@ type Qwen3VLRenderer struct {
 }
 
 func (r *Qwen3VLRenderer) renderContent(content api.Message, imageOffset int) (string, int) {
+	if r.useImgTags {
+		return renderContentWithImageTags(content.Content, len(content.Images), imageOffset)
+	}
+
 	// This assumes all images are at the front of the message - same assumption as ollama/ollama/runner.go
 	var subSb strings.Builder
 	for range content.Images {
 		// TODO: (jmorganca): how to render this is different for different
 		// model backends, and so we should eventually parameterize this or
 		// only output a placeholder such as [img]
-		if r.useImgTags {
-			subSb.WriteString(fmt.Sprintf("[img-%d]", imageOffset))
-			imageOffset++
-		} else {
-			subSb.WriteString("<|vision_start|><|image_pad|><|vision_end|>")
-		}
+		subSb.WriteString("<|vision_start|><|image_pad|><|vision_end|>")
 	}
 	// TODO: support videos
 
@@ -126,7 +124,7 @@ func (r *Qwen3VLRenderer) Render(messages []api.Message, tools []api.Tool, think
 			if i == 0 || messages[i-1].Role != "tool" {
 				sb.WriteString("<|im_start|>user")
 			}
-			sb.WriteString("\n<tool_response>\n" + message.Content + "\n</tool_response>")
+			sb.WriteString("\n<tool_response>\n" + content + "\n</tool_response>")
 			if i == len(messages)-1 || messages[i+1].Role != "tool" {
 				sb.WriteString("<|im_end|>\n")
 			}

model/renderers/qwen3vl_nonthinking_test.go

@@ -101,7 +101,7 @@ Let me analyze this image.`,
 			},
 			useImgTags: true,
 			expected: `<|im_start|>user
-[img-0]Describe this image.<|im_end|>
+[img-0] Describe this image.<|im_end|>
 <|im_start|>assistant
 Let me analyze this image.`,
 		},
@@ -123,7 +123,7 @@ Let me analyze this image.`,
 			},
 			useImgTags: true,
 			expected: `<|im_start|>user
-[img-0][img-1]Describe these images.<|im_end|>
+[img-0][img-1] Describe these images.<|im_end|>
 <|im_start|>assistant
 Let me analyze this image.`,
 		},