00c8e921f6
Switching from a vision model to a text-only model kept replaying historical image content blocks to the new provider, which rejected them with invalid_argument. The convertToLlm wrapper in sdk.ts only filtered images when images.blockImages was set, never by model capability, so the outbound request carried image blocks the active model could not accept. Add replaceLlmImagesWithText() to scrub image blocks out of the already-converted LLM message view, and call it from the sdk wrapper when the active model's input lacks "image". History on disk keeps its images; only the provider request is scrubbed, and the check reads the active model dynamically so a /model switch takes effect next turn. Fixes #5400
192 lines
5.9 KiB
TypeScript
192 lines
5.9 KiB
TypeScript
import { describe, expect, it } from "bun:test";
|
|
import type { AgentMessage } from "@oh-my-pi/pi-agent-core";
|
|
import type { AssistantMessage } from "@oh-my-pi/pi-ai";
|
|
import {
|
|
type CustomMessage,
|
|
convertToLlm,
|
|
INTERRUPTED_THINKING_MESSAGE_TYPE,
|
|
replaceLlmImagesWithText,
|
|
SKILL_PROMPT_MESSAGE_TYPE,
|
|
type SkillPromptDetails,
|
|
} from "./messages";
|
|
|
|
function customMessage(customType: string, attribution: "agent" | "user"): CustomMessage<SkillPromptDetails> {
|
|
return {
|
|
role: "custom",
|
|
customType,
|
|
content: "Use this skill.",
|
|
display: true,
|
|
details: { name: "atomic-commit", path: "/tmp/SKILL.md", lineCount: 1 },
|
|
attribution,
|
|
timestamp: 1,
|
|
};
|
|
}
|
|
|
|
const interruptedUsage: AssistantMessage["usage"] = {
|
|
input: 0,
|
|
output: 0,
|
|
cacheRead: 0,
|
|
cacheWrite: 0,
|
|
totalTokens: 0,
|
|
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
|
};
|
|
|
|
function abortedAssistant(content: AssistantMessage["content"]): AssistantMessage {
|
|
return {
|
|
role: "assistant",
|
|
content,
|
|
api: "anthropic-messages",
|
|
provider: "anthropic",
|
|
model: "claude-sonnet-4-5",
|
|
usage: interruptedUsage,
|
|
stopReason: "aborted",
|
|
timestamp: 1,
|
|
};
|
|
}
|
|
|
|
function interruptedThinkingContinuity(): CustomMessage {
|
|
return {
|
|
role: "custom",
|
|
customType: INTERRUPTED_THINKING_MESSAGE_TYPE,
|
|
content: "preserved reasoning",
|
|
display: false,
|
|
attribution: "agent",
|
|
timestamp: 2,
|
|
};
|
|
}
|
|
|
|
describe("convertToLlm", () => {
|
|
it("presents user-invoked skill prompts as user turns", () => {
|
|
const [message] = convertToLlm([customMessage(SKILL_PROMPT_MESSAGE_TYPE, "user")]);
|
|
|
|
expect(message?.role).toBe("user");
|
|
if (message?.role !== "user") {
|
|
throw new Error(`Expected user role, received ${message?.role ?? "none"}`);
|
|
}
|
|
expect(message.attribution).toBe("user");
|
|
});
|
|
|
|
it("keeps auto-applied skill prompts and other custom messages as developer turns", () => {
|
|
const [autoSkill, otherCustom] = convertToLlm([
|
|
customMessage(SKILL_PROMPT_MESSAGE_TYPE, "agent"),
|
|
customMessage("extension-note", "user"),
|
|
]);
|
|
|
|
expect(autoSkill?.role).toBe("developer");
|
|
expect(otherCustom?.role).toBe("developer");
|
|
});
|
|
|
|
it("strips the demoted trailing thinking run from the assistant LLM view when its continuity message follows", () => {
|
|
const messages: AgentMessage[] = [
|
|
abortedAssistant([
|
|
{ type: "text", text: "partial answer" },
|
|
{ type: "thinking", thinking: "interrupted reasoning" },
|
|
]),
|
|
interruptedThinkingContinuity(),
|
|
];
|
|
|
|
const llm = convertToLlm(messages);
|
|
const assistant = llm.find(entry => entry.role === "assistant");
|
|
expect(Array.isArray(assistant?.content) && assistant.content.map(block => block.type)).toEqual(["text"]);
|
|
expect(llm.some(entry => entry.role === "developer")).toBe(true);
|
|
});
|
|
|
|
it("keeps trailing thinking on the assistant LLM view when no continuity message follows", () => {
|
|
const messages: AgentMessage[] = [
|
|
abortedAssistant([
|
|
{ type: "text", text: "partial answer" },
|
|
{ type: "thinking", thinking: "interrupted reasoning" },
|
|
]),
|
|
];
|
|
|
|
const llm = convertToLlm(messages);
|
|
const assistant = llm.find(entry => entry.role === "assistant");
|
|
expect(Array.isArray(assistant?.content) && assistant.content.map(block => block.type)).toEqual([
|
|
"text",
|
|
"thinking",
|
|
]);
|
|
});
|
|
|
|
it("keeps a signed (complete) trailing thinking block in the assistant LLM view even with a continuity message", () => {
|
|
const messages: AgentMessage[] = [
|
|
abortedAssistant([
|
|
{ type: "text", text: "partial answer" },
|
|
{ type: "thinking", thinking: "complete reasoning", thinkingSignature: "sig" },
|
|
]),
|
|
interruptedThinkingContinuity(),
|
|
];
|
|
|
|
const llm = convertToLlm(messages);
|
|
const assistant = llm.find(entry => entry.role === "assistant");
|
|
expect(Array.isArray(assistant?.content) && assistant.content.map(block => block.type)).toEqual([
|
|
"text",
|
|
"thinking",
|
|
]);
|
|
});
|
|
});
|
|
|
|
describe("replaceLlmImagesWithText", () => {
|
|
it("replaces image blocks in user, developer, and tool-result messages with the placeholder", () => {
|
|
const converted = convertToLlm([
|
|
{
|
|
role: "user",
|
|
content: [
|
|
{ type: "text", text: "look" },
|
|
{ type: "image", data: "aaaa", mimeType: "image/png" },
|
|
],
|
|
attribution: "user",
|
|
timestamp: 1,
|
|
},
|
|
{
|
|
role: "toolResult",
|
|
toolCallId: "c1",
|
|
toolName: "inspect_image",
|
|
content: [{ type: "image", data: "bbbb", mimeType: "image/png" }],
|
|
isError: false,
|
|
timestamp: 2,
|
|
},
|
|
]);
|
|
|
|
const scrubbed = replaceLlmImagesWithText(converted, "[image omitted]");
|
|
|
|
expect(scrubbed).not.toBe(converted);
|
|
const types = scrubbed.flatMap(m => (Array.isArray(m.content) ? m.content.map(b => b.type) : []));
|
|
expect(types).not.toContain("image");
|
|
const user = scrubbed.find(m => m.role === "user");
|
|
expect(Array.isArray(user?.content) && user.content.map(b => (b.type === "text" ? b.text : b.type))).toEqual([
|
|
"look",
|
|
"[image omitted]",
|
|
]);
|
|
const toolResult = scrubbed.find(m => m.role === "toolResult");
|
|
expect(Array.isArray(toolResult?.content) && toolResult.content).toEqual([
|
|
{ type: "text", text: "[image omitted]" },
|
|
]);
|
|
});
|
|
|
|
it("collapses consecutive image blocks into a single placeholder", () => {
|
|
const converted = convertToLlm([
|
|
{
|
|
role: "user",
|
|
content: [
|
|
{ type: "image", data: "aaaa", mimeType: "image/png" },
|
|
{ type: "image", data: "bbbb", mimeType: "image/png" },
|
|
],
|
|
attribution: "user",
|
|
timestamp: 1,
|
|
},
|
|
]);
|
|
|
|
const scrubbed = replaceLlmImagesWithText(converted, "[image omitted]");
|
|
const user = scrubbed.find(m => m.role === "user");
|
|
expect(Array.isArray(user?.content) && user.content).toEqual([{ type: "text", text: "[image omitted]" }]);
|
|
});
|
|
|
|
it("returns the same array reference when there are no image blocks", () => {
|
|
const converted = convertToLlm([
|
|
{ role: "user", content: [{ type: "text", text: "hi" }], attribution: "user", timestamp: 1 },
|
|
]);
|
|
|
|
expect(replaceLlmImagesWithText(converted, "[image omitted]")).toBe(converted);
|
|
});
|
|
});
|