diff --git a/packages/ai/src/providers/anthropic.ts b/packages/ai/src/providers/anthropic.ts index 8b9b99bce..bd1f75f27 100644 --- a/packages/ai/src/providers/anthropic.ts +++ b/packages/ai/src/providers/anthropic.ts @@ -2911,11 +2911,22 @@ function ensureErrorToolResultWireContent( : [{ type: "text", text: EMPTY_ERROR_TOOL_RESULT_TEXT }]; } -function buildToolResultBlock(model: Model<"anthropic-messages">, msg: ToolResultMessage): ContentBlockParam { - const content = ensureErrorToolResultWireContent( - convertContentBlocks(msg.content, model.input.includes("image")), - msg.isError, - ); +function buildToolResultBlock( + model: Model<"anthropic-messages">, + msg: ToolResultMessage, + hoistedImages: ContentBlockParam[], +): ContentBlockParam { + let content = convertContentBlocks(msg.content, model.input.includes("image")); + // Anthropic rejects images inside error tool results ("all content must be + // type `text` if `is_error` is true") — keep the text in the block and + // hoist the images after the message's tool_result run. + if (msg.isError && typeof content !== "string" && content.some(block => block.type === "image")) { + for (const block of content) { + if (block.type === "image") hoistedImages.push(block); + } + content = content.filter(block => block.type === "text"); + } + content = ensureErrorToolResultWireContent(content, msg.isError); const block: ContentBlockParam = { type: "tool_result", tool_use_id: msg.toolCallId, @@ -3082,21 +3093,30 @@ export function convertAnthropicMessages( } else if (msg.role === "toolResult") { // Collect all consecutive toolResult messages, needed for z.ai Anthropic endpoint const toolResults: ContentBlockParam[] = []; + // Images stripped out of error tool results, re-attached after the run. + const hoistedImages: ContentBlockParam[] = []; // Add the current tool result - toolResults.push(buildToolResultBlock(model, msg)); + toolResults.push(buildToolResultBlock(model, msg, hoistedImages)); // Look ahead for consecutive toolResult messages let j = i + 1; while (j < transformedMessages.length && transformedMessages[j].role === "toolResult") { const nextMsg = transformedMessages[j] as ToolResultMessage; // We know it's a toolResult - toolResults.push(buildToolResultBlock(model, nextMsg)); + toolResults.push(buildToolResultBlock(model, nextMsg, hoistedImages)); j++; } // Skip the messages we've already processed i = j - 1; + if (hoistedImages.length > 0) { + toolResults.push( + { type: "text", text: "Attached image(s) from the tool result(s) above:" }, + ...hoistedImages, + ); + } + // Add a single user message with all tool results params.push({ role: "user", diff --git a/packages/ai/test/anthropic-error-tool-result-image.test.ts b/packages/ai/test/anthropic-error-tool-result-image.test.ts new file mode 100644 index 000000000..5fee9346f --- /dev/null +++ b/packages/ai/test/anthropic-error-tool-result-image.test.ts @@ -0,0 +1,148 @@ +import { describe, expect, it } from "bun:test"; +import { convertAnthropicMessages } from "@oh-my-pi/pi-ai/providers/anthropic"; +import type { AssistantMessage, Model, ModelSpec, ToolResultMessage, UserMessage } from "@oh-my-pi/pi-ai/types"; +import { buildModel } from "@oh-my-pi/pi-catalog/build"; + +// Anthropic rejects images inside error tool results: +// "messages.N.content.0.tool_result: all content must be type `text` if `is_error` is true" +// The converter must keep error tool_result content text-only and re-attach the +// images after the tool_result run in the same user message. + +const baseModel: Omit, "provider" | "baseUrl"> = { + api: "anthropic-messages", + id: "claude-sonnet-4-6", + name: "Claude Sonnet 4.6", + input: ["text", "image"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + maxTokens: 8192, + contextWindow: 200000, + reasoning: false, +}; + +const visionModel: Model<"anthropic-messages"> = buildModel({ + ...baseModel, + provider: "anthropic", + baseUrl: "https://api.anthropic.com", +}); + +const user: UserMessage = { + role: "user", + content: "run the tool", + timestamp: Date.now(), +}; + +function assistantWithCalls(ids: string[]): AssistantMessage { + return { + role: "assistant", + content: ids.map(id => ({ + type: "toolCall", + id, + name: "browser", + arguments: {}, + })), + api: "anthropic-messages", + provider: "anthropic", + model: "claude-sonnet-4-6", + usage: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + stopReason: "toolUse", + timestamp: Date.now(), + }; +} + +const PNG_DATA = "iVBORw0KGgoAAAANSUhEUg=="; + +function toolResult(id: string, opts: { isError: boolean; text?: string; image?: boolean }): ToolResultMessage { + const content: ToolResultMessage["content"] = []; + if (opts.text !== undefined) content.push({ type: "text", text: opts.text }); + if (opts.image) content.push({ type: "image", data: PNG_DATA, mimeType: "image/png" }); + return { + role: "toolResult", + toolCallId: id, + toolName: "browser", + content, + isError: opts.isError, + timestamp: Date.now(), + }; +} + +function lastUserBlocks(messages: Parameters[0]): Array> { + const params = convertAnthropicMessages(messages, visionModel, false); + const last = params.at(-1); + expect(last?.role).toBe("user"); + const blocks = last?.content as unknown as Array>; + expect(Array.isArray(blocks)).toBe(true); + return blocks; +} + +describe("anthropic error tool_result image hoisting", () => { + it("keeps error tool_result content text-only and hoists the image after it", () => { + const blocks = lastUserBlocks([ + user, + assistantWithCalls(["toolu_err"]), + toolResult("toolu_err", { isError: true, text: "assertion failed", image: true }), + ]); + + const result = blocks.find(b => b.type === "tool_result"); + expect(result?.is_error).toBe(true); + const content = result?.content as Array>; + expect(content.every(b => b.type === "text")).toBe(true); + expect(content.some(b => (b.text as string).includes("assertion failed"))).toBe(true); + + // Image re-attached in the same user message, after the tool_result. + const imageIndex = blocks.findIndex(b => b.type === "image"); + expect(imageIndex).toBeGreaterThan(blocks.findIndex(b => b.type === "tool_result")); + const source = blocks[imageIndex]?.source as Record; + expect(source.data).toBe(PNG_DATA); + }); + + it("keeps a non-empty text body when the error result was image-only", () => { + const blocks = lastUserBlocks([ + user, + assistantWithCalls(["toolu_err"]), + toolResult("toolu_err", { isError: true, image: true }), + ]); + + const result = blocks.find(b => b.type === "tool_result"); + expect(result?.is_error).toBe(true); + const content = result?.content as Array>; + expect(content.length).toBeGreaterThan(0); + expect(content.every(b => b.type === "text")).toBe(true); + expect(blocks.some(b => b.type === "image")).toBe(true); + }); + + it("hoists images after the whole tool_result run for consecutive results", () => { + const blocks = lastUserBlocks([ + user, + assistantWithCalls(["toolu_a", "toolu_b"]), + toolResult("toolu_a", { isError: true, text: "boom", image: true }), + toolResult("toolu_b", { isError: false, text: "ok" }), + ]); + + // All tool_result blocks come first — Anthropic requires the run at the + // beginning of the message — then the hoisted image content. + const types = blocks.map(b => b.type); + const lastResult = types.lastIndexOf("tool_result"); + expect(types.slice(0, lastResult + 1).every(t => t === "tool_result")).toBe(true); + expect(types.indexOf("image")).toBeGreaterThan(lastResult); + }); + + it("leaves images inside successful tool_results untouched", () => { + const blocks = lastUserBlocks([ + user, + assistantWithCalls(["toolu_ok"]), + toolResult("toolu_ok", { isError: false, text: "screenshot", image: true }), + ]); + + const result = blocks.find(b => b.type === "tool_result"); + const content = result?.content as Array>; + expect(content.some(b => b.type === "image")).toBe(true); + expect(blocks.filter(b => b.type === "image").length).toBe(0); + }); +});