fix(ai): hoisted images out of error Anthropic tool results
- Adjusted Anthropic tool-result conversion to keep error results text-only while collecting any image blocks separately. - Reattached collected images after the tool-result run in the same user message with a separator note for Anthropic compatibility. - Added tests covering image-only error tool_results, image hoisting order, and successful tool_result image preservation.
This commit is contained in:
can1357
@@ -2911,11 +2911,22 @@ function ensureErrorToolResultWireContent(
|
||||
: [{ type: "text", text: EMPTY_ERROR_TOOL_RESULT_TEXT }];
|
||||
}
|
||||
|
||||
function buildToolResultBlock(model: Model<"anthropic-messages">, msg: ToolResultMessage): ContentBlockParam {
|
||||
const content = ensureErrorToolResultWireContent(
|
||||
convertContentBlocks(msg.content, model.input.includes("image")),
|
||||
msg.isError,
|
||||
);
|
||||
function buildToolResultBlock(
|
||||
model: Model<"anthropic-messages">,
|
||||
msg: ToolResultMessage,
|
||||
hoistedImages: ContentBlockParam[],
|
||||
): ContentBlockParam {
|
||||
let content = convertContentBlocks(msg.content, model.input.includes("image"));
|
||||
// Anthropic rejects images inside error tool results ("all content must be
|
||||
// type `text` if `is_error` is true") — keep the text in the block and
|
||||
// hoist the images after the message's tool_result run.
|
||||
if (msg.isError && typeof content !== "string" && content.some(block => block.type === "image")) {
|
||||
for (const block of content) {
|
||||
if (block.type === "image") hoistedImages.push(block);
|
||||
}
|
||||
content = content.filter(block => block.type === "text");
|
||||
}
|
||||
content = ensureErrorToolResultWireContent(content, msg.isError);
|
||||
const block: ContentBlockParam = {
|
||||
type: "tool_result",
|
||||
tool_use_id: msg.toolCallId,
|
||||
@@ -3082,21 +3093,30 @@ export function convertAnthropicMessages(
|
||||
} else if (msg.role === "toolResult") {
|
||||
// Collect all consecutive toolResult messages, needed for z.ai Anthropic endpoint
|
||||
const toolResults: ContentBlockParam[] = [];
|
||||
// Images stripped out of error tool results, re-attached after the run.
|
||||
const hoistedImages: ContentBlockParam[] = [];
|
||||
|
||||
// Add the current tool result
|
||||
toolResults.push(buildToolResultBlock(model, msg));
|
||||
toolResults.push(buildToolResultBlock(model, msg, hoistedImages));
|
||||
|
||||
// Look ahead for consecutive toolResult messages
|
||||
let j = i + 1;
|
||||
while (j < transformedMessages.length && transformedMessages[j].role === "toolResult") {
|
||||
const nextMsg = transformedMessages[j] as ToolResultMessage; // We know it's a toolResult
|
||||
toolResults.push(buildToolResultBlock(model, nextMsg));
|
||||
toolResults.push(buildToolResultBlock(model, nextMsg, hoistedImages));
|
||||
j++;
|
||||
}
|
||||
|
||||
// Skip the messages we've already processed
|
||||
i = j - 1;
|
||||
|
||||
if (hoistedImages.length > 0) {
|
||||
toolResults.push(
|
||||
{ type: "text", text: "Attached image(s) from the tool result(s) above:" },
|
||||
...hoistedImages,
|
||||
);
|
||||
}
|
||||
|
||||
// Add a single user message with all tool results
|
||||
params.push({
|
||||
role: "user",
|
||||
|
||||
@@ -0,0 +1,148 @@
|
||||
import { describe, expect, it } from "bun:test";
|
||||
import { convertAnthropicMessages } from "@oh-my-pi/pi-ai/providers/anthropic";
|
||||
import type { AssistantMessage, Model, ModelSpec, ToolResultMessage, UserMessage } from "@oh-my-pi/pi-ai/types";
|
||||
import { buildModel } from "@oh-my-pi/pi-catalog/build";
|
||||
|
||||
// Anthropic rejects images inside error tool results:
|
||||
// "messages.N.content.0.tool_result: all content must be type `text` if `is_error` is true"
|
||||
// The converter must keep error tool_result content text-only and re-attach the
|
||||
// images after the tool_result run in the same user message.
|
||||
|
||||
const baseModel: Omit<ModelSpec<"anthropic-messages">, "provider" | "baseUrl"> = {
|
||||
api: "anthropic-messages",
|
||||
id: "claude-sonnet-4-6",
|
||||
name: "Claude Sonnet 4.6",
|
||||
input: ["text", "image"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
maxTokens: 8192,
|
||||
contextWindow: 200000,
|
||||
reasoning: false,
|
||||
};
|
||||
|
||||
const visionModel: Model<"anthropic-messages"> = buildModel({
|
||||
...baseModel,
|
||||
provider: "anthropic",
|
||||
baseUrl: "https://api.anthropic.com",
|
||||
});
|
||||
|
||||
const user: UserMessage = {
|
||||
role: "user",
|
||||
content: "run the tool",
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
|
||||
function assistantWithCalls(ids: string[]): AssistantMessage {
|
||||
return {
|
||||
role: "assistant",
|
||||
content: ids.map(id => ({
|
||||
type: "toolCall",
|
||||
id,
|
||||
name: "browser",
|
||||
arguments: {},
|
||||
})),
|
||||
api: "anthropic-messages",
|
||||
provider: "anthropic",
|
||||
model: "claude-sonnet-4-6",
|
||||
usage: {
|
||||
input: 0,
|
||||
output: 0,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
totalTokens: 0,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
||||
},
|
||||
stopReason: "toolUse",
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
}
|
||||
|
||||
const PNG_DATA = "iVBORw0KGgoAAAANSUhEUg==";
|
||||
|
||||
function toolResult(id: string, opts: { isError: boolean; text?: string; image?: boolean }): ToolResultMessage {
|
||||
const content: ToolResultMessage["content"] = [];
|
||||
if (opts.text !== undefined) content.push({ type: "text", text: opts.text });
|
||||
if (opts.image) content.push({ type: "image", data: PNG_DATA, mimeType: "image/png" });
|
||||
return {
|
||||
role: "toolResult",
|
||||
toolCallId: id,
|
||||
toolName: "browser",
|
||||
content,
|
||||
isError: opts.isError,
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
}
|
||||
|
||||
function lastUserBlocks(messages: Parameters<typeof convertAnthropicMessages>[0]): Array<Record<string, unknown>> {
|
||||
const params = convertAnthropicMessages(messages, visionModel, false);
|
||||
const last = params.at(-1);
|
||||
expect(last?.role).toBe("user");
|
||||
const blocks = last?.content as unknown as Array<Record<string, unknown>>;
|
||||
expect(Array.isArray(blocks)).toBe(true);
|
||||
return blocks;
|
||||
}
|
||||
|
||||
describe("anthropic error tool_result image hoisting", () => {
|
||||
it("keeps error tool_result content text-only and hoists the image after it", () => {
|
||||
const blocks = lastUserBlocks([
|
||||
user,
|
||||
assistantWithCalls(["toolu_err"]),
|
||||
toolResult("toolu_err", { isError: true, text: "assertion failed", image: true }),
|
||||
]);
|
||||
|
||||
const result = blocks.find(b => b.type === "tool_result");
|
||||
expect(result?.is_error).toBe(true);
|
||||
const content = result?.content as Array<Record<string, unknown>>;
|
||||
expect(content.every(b => b.type === "text")).toBe(true);
|
||||
expect(content.some(b => (b.text as string).includes("assertion failed"))).toBe(true);
|
||||
|
||||
// Image re-attached in the same user message, after the tool_result.
|
||||
const imageIndex = blocks.findIndex(b => b.type === "image");
|
||||
expect(imageIndex).toBeGreaterThan(blocks.findIndex(b => b.type === "tool_result"));
|
||||
const source = blocks[imageIndex]?.source as Record<string, unknown>;
|
||||
expect(source.data).toBe(PNG_DATA);
|
||||
});
|
||||
|
||||
it("keeps a non-empty text body when the error result was image-only", () => {
|
||||
const blocks = lastUserBlocks([
|
||||
user,
|
||||
assistantWithCalls(["toolu_err"]),
|
||||
toolResult("toolu_err", { isError: true, image: true }),
|
||||
]);
|
||||
|
||||
const result = blocks.find(b => b.type === "tool_result");
|
||||
expect(result?.is_error).toBe(true);
|
||||
const content = result?.content as Array<Record<string, unknown>>;
|
||||
expect(content.length).toBeGreaterThan(0);
|
||||
expect(content.every(b => b.type === "text")).toBe(true);
|
||||
expect(blocks.some(b => b.type === "image")).toBe(true);
|
||||
});
|
||||
|
||||
it("hoists images after the whole tool_result run for consecutive results", () => {
|
||||
const blocks = lastUserBlocks([
|
||||
user,
|
||||
assistantWithCalls(["toolu_a", "toolu_b"]),
|
||||
toolResult("toolu_a", { isError: true, text: "boom", image: true }),
|
||||
toolResult("toolu_b", { isError: false, text: "ok" }),
|
||||
]);
|
||||
|
||||
// All tool_result blocks come first — Anthropic requires the run at the
|
||||
// beginning of the message — then the hoisted image content.
|
||||
const types = blocks.map(b => b.type);
|
||||
const lastResult = types.lastIndexOf("tool_result");
|
||||
expect(types.slice(0, lastResult + 1).every(t => t === "tool_result")).toBe(true);
|
||||
expect(types.indexOf("image")).toBeGreaterThan(lastResult);
|
||||
});
|
||||
|
||||
it("leaves images inside successful tool_results untouched", () => {
|
||||
const blocks = lastUserBlocks([
|
||||
user,
|
||||
assistantWithCalls(["toolu_ok"]),
|
||||
toolResult("toolu_ok", { isError: false, text: "screenshot", image: true }),
|
||||
]);
|
||||
|
||||
const result = blocks.find(b => b.type === "tool_result");
|
||||
const content = result?.content as Array<Record<string, unknown>>;
|
||||
expect(content.some(b => b.type === "image")).toBe(true);
|
||||
expect(blocks.filter(b => b.type === "image").length).toBe(0);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user