Files
oh-my-pi/packages/coding-agent/src/session/messages.test.ts
T
roboomp 00c8e921f6 fix(agent): strip images for non-vision models mid-session
Switching from a vision model to a text-only model kept replaying
historical image content blocks to the new provider, which rejected them
with invalid_argument. The convertToLlm wrapper in sdk.ts only filtered
images when images.blockImages was set, never by model capability, so
the outbound request carried image blocks the active model could not
accept.

Add replaceLlmImagesWithText() to scrub image blocks out of the
already-converted LLM message view, and call it from the sdk wrapper
when the active model's input lacks "image". History on disk keeps its
images; only the provider request is scrubbed, and the check reads the
active model dynamically so a /model switch takes effect next turn.

Fixes #5400
2026-07-14 20:11:25 +00:00

192 lines
5.9 KiB
TypeScript

import { describe, expect, it } from "bun:test";
import type { AgentMessage } from "@oh-my-pi/pi-agent-core";
import type { AssistantMessage } from "@oh-my-pi/pi-ai";
import {
type CustomMessage,
convertToLlm,
INTERRUPTED_THINKING_MESSAGE_TYPE,
replaceLlmImagesWithText,
SKILL_PROMPT_MESSAGE_TYPE,
type SkillPromptDetails,
} from "./messages";
function customMessage(customType: string, attribution: "agent" | "user"): CustomMessage<SkillPromptDetails> {
return {
role: "custom",
customType,
content: "Use this skill.",
display: true,
details: { name: "atomic-commit", path: "/tmp/SKILL.md", lineCount: 1 },
attribution,
timestamp: 1,
};
}
const interruptedUsage: AssistantMessage["usage"] = {
input: 0,
output: 0,
cacheRead: 0,
cacheWrite: 0,
totalTokens: 0,
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
};
function abortedAssistant(content: AssistantMessage["content"]): AssistantMessage {
return {
role: "assistant",
content,
api: "anthropic-messages",
provider: "anthropic",
model: "claude-sonnet-4-5",
usage: interruptedUsage,
stopReason: "aborted",
timestamp: 1,
};
}
function interruptedThinkingContinuity(): CustomMessage {
return {
role: "custom",
customType: INTERRUPTED_THINKING_MESSAGE_TYPE,
content: "preserved reasoning",
display: false,
attribution: "agent",
timestamp: 2,
};
}
describe("convertToLlm", () => {
it("presents user-invoked skill prompts as user turns", () => {
const [message] = convertToLlm([customMessage(SKILL_PROMPT_MESSAGE_TYPE, "user")]);
expect(message?.role).toBe("user");
if (message?.role !== "user") {
throw new Error(`Expected user role, received ${message?.role ?? "none"}`);
}
expect(message.attribution).toBe("user");
});
it("keeps auto-applied skill prompts and other custom messages as developer turns", () => {
const [autoSkill, otherCustom] = convertToLlm([
customMessage(SKILL_PROMPT_MESSAGE_TYPE, "agent"),
customMessage("extension-note", "user"),
]);
expect(autoSkill?.role).toBe("developer");
expect(otherCustom?.role).toBe("developer");
});
it("strips the demoted trailing thinking run from the assistant LLM view when its continuity message follows", () => {
const messages: AgentMessage[] = [
abortedAssistant([
{ type: "text", text: "partial answer" },
{ type: "thinking", thinking: "interrupted reasoning" },
]),
interruptedThinkingContinuity(),
];
const llm = convertToLlm(messages);
const assistant = llm.find(entry => entry.role === "assistant");
expect(Array.isArray(assistant?.content) && assistant.content.map(block => block.type)).toEqual(["text"]);
expect(llm.some(entry => entry.role === "developer")).toBe(true);
});
it("keeps trailing thinking on the assistant LLM view when no continuity message follows", () => {
const messages: AgentMessage[] = [
abortedAssistant([
{ type: "text", text: "partial answer" },
{ type: "thinking", thinking: "interrupted reasoning" },
]),
];
const llm = convertToLlm(messages);
const assistant = llm.find(entry => entry.role === "assistant");
expect(Array.isArray(assistant?.content) && assistant.content.map(block => block.type)).toEqual([
"text",
"thinking",
]);
});
it("keeps a signed (complete) trailing thinking block in the assistant LLM view even with a continuity message", () => {
const messages: AgentMessage[] = [
abortedAssistant([
{ type: "text", text: "partial answer" },
{ type: "thinking", thinking: "complete reasoning", thinkingSignature: "sig" },
]),
interruptedThinkingContinuity(),
];
const llm = convertToLlm(messages);
const assistant = llm.find(entry => entry.role === "assistant");
expect(Array.isArray(assistant?.content) && assistant.content.map(block => block.type)).toEqual([
"text",
"thinking",
]);
});
});
describe("replaceLlmImagesWithText", () => {
it("replaces image blocks in user, developer, and tool-result messages with the placeholder", () => {
const converted = convertToLlm([
{
role: "user",
content: [
{ type: "text", text: "look" },
{ type: "image", data: "aaaa", mimeType: "image/png" },
],
attribution: "user",
timestamp: 1,
},
{
role: "toolResult",
toolCallId: "c1",
toolName: "inspect_image",
content: [{ type: "image", data: "bbbb", mimeType: "image/png" }],
isError: false,
timestamp: 2,
},
]);
const scrubbed = replaceLlmImagesWithText(converted, "[image omitted]");
expect(scrubbed).not.toBe(converted);
const types = scrubbed.flatMap(m => (Array.isArray(m.content) ? m.content.map(b => b.type) : []));
expect(types).not.toContain("image");
const user = scrubbed.find(m => m.role === "user");
expect(Array.isArray(user?.content) && user.content.map(b => (b.type === "text" ? b.text : b.type))).toEqual([
"look",
"[image omitted]",
]);
const toolResult = scrubbed.find(m => m.role === "toolResult");
expect(Array.isArray(toolResult?.content) && toolResult.content).toEqual([
{ type: "text", text: "[image omitted]" },
]);
});
it("collapses consecutive image blocks into a single placeholder", () => {
const converted = convertToLlm([
{
role: "user",
content: [
{ type: "image", data: "aaaa", mimeType: "image/png" },
{ type: "image", data: "bbbb", mimeType: "image/png" },
],
attribution: "user",
timestamp: 1,
},
]);
const scrubbed = replaceLlmImagesWithText(converted, "[image omitted]");
const user = scrubbed.find(m => m.role === "user");
expect(Array.isArray(user?.content) && user.content).toEqual([{ type: "text", text: "[image omitted]" }]);
});
it("returns the same array reference when there are no image blocks", () => {
const converted = convertToLlm([
{ role: "user", content: [{ type: "text", text: "hi" }], attribution: "user", timestamp: 1 },
]);
expect(replaceLlmImagesWithText(converted, "[image omitted]")).toBe(converted);
});
});