fix(coding-agent): tolerate legacy image metadata

This commit is contained in:
Ethan Cawse
2026-08-12 09:13:03 -04:00
parent f43467f048
commit 4c71e92e3e
4 changed files with 42 additions and 11 deletions
+1 -1
View File
@@ -4,7 +4,7 @@
### Fixed
- Fixed custom STB-backed vision providers retaining decoder metadata and sending WebP bytes that llama.cpp cannot decode. Image format is now detected from bytes, attached and historical WebP blocks are normalized on the actual provider-request path, and corrupt WebP is omitted instead of making the turn unrecoverable.
- Fixed custom STB-backed vision providers retaining decoder metadata and sending WebP bytes that llama.cpp cannot decode. Image format is now detected from bytes, attached and historical WebP blocks are normalized on the actual provider-request path, and corrupt WebP is omitted instead of making the turn unrecoverable ([#8339](https://github.com/can1357/oh-my-pi/pull/8339) by [@ethancawse](https://github.com/ethancawse)).
## [17.2.15] - 2026-08-12
@@ -16,10 +16,11 @@ const modelBoundaryImageCache = new LRUCache<string, NormalizedImagePayload | nu
sizeCalculation: payload => Math.max(1, payload?.data.length ?? 1),
});
const modelBoundaryImageNormalizations = new Map<string, Promise<NormalizedImagePayload | null>>();
const UNDECODABLE_STB_IMAGE_OMISSION: TextContent = {
type: "text",
text: "[image omitted: WebP could not be decoded for this model]",
};
const UNDECODABLE_STB_IMAGE_OMISSION_TEXT = "[image omitted: WebP could not be decoded for this model]";
function createUndecodableStbImageOmission(): TextContent {
return { type: "text", text: UNDECODABLE_STB_IMAGE_OMISSION_TEXT };
}
function hasWebPMagic(data: string): boolean {
const header = Buffer.from(data.slice(0, 16), "base64");
@@ -29,7 +30,8 @@ function hasWebPMagic(data: string): boolean {
}
function isWebPImage(image: ImageContent): boolean {
return image.mimeType.toLowerCase() === "image/webp" || hasWebPMagic(image.data);
const mimeType = typeof image.mimeType === "string" ? image.mimeType.toLowerCase() : undefined;
return mimeType === "image/webp" || (typeof image.data === "string" && hasWebPMagic(image.data));
}
function modelBoundaryImageCacheKey(image: ImageContent, resize: ImageResizeOptions | undefined): string {
@@ -230,7 +232,7 @@ export async function normalizeModelContextMessages(messages: Message[], model:
}
content ??= message.content.slice(0, partIndex);
const normalized = await memoizedStbImageNormalization(part, undefined);
content.push(normalized ?? UNDECODABLE_STB_IMAGE_OMISSION);
content.push(normalized ?? createUndecodableStbImageOmission());
}
if (!content) continue;
output ??= messages.slice();
@@ -322,11 +322,12 @@ describe("AgentSession message pipeline", () => {
agentId: "SubAgent",
});
try {
const corrupt: ImageContent = {
// Session persistence accepts historical image blocks without MIME
// metadata, so exercise that runtime shape through the real provider path.
const corrupt = {
type: "image",
data: Buffer.from("RIFF0000WEBPbroken-attachment").toBase64(),
mimeType: "image/webp",
};
} as unknown as ImageContent;
await session.sendUserMessage([{ type: "text", text: "inspect this" }, corrupt]);
@@ -1,5 +1,5 @@
import { afterEach, beforeEach, describe, expect, test } from "bun:test";
import type { Api, Model } from "@oh-my-pi/pi-ai";
import type { Api, Message, Model } from "@oh-my-pi/pi-ai";
import { buildModel } from "@oh-my-pi/pi-catalog/build";
import { getBundledModels } from "@oh-my-pi/pi-catalog/models";
import type { CustomMessage } from "@oh-my-pi/pi-coding-agent/session/messages";
@@ -274,4 +274,32 @@ describe("normalizeModelContextImages model-aware WebP exclusion", () => {
]);
expect(messages[0]!.content[0]).toBe(corrupt);
});
test("normalizes persisted WebP blocks with malformed MIME metadata", async () => {
for (const mimeType of [undefined, null, 42]) {
const malformedImage = {
type: "image",
data: Buffer.from("RIFF0000WEBPbad-persisted-image").toBase64(),
...(mimeType === undefined ? {} : { mimeType }),
};
const messages = [
{
role: "toolResult",
toolCallId: "read-malformed",
toolName: "read",
content: [malformedImage],
isError: false,
timestamp: 1,
},
] as unknown as Message[];
const result = await normalizeModelContextMessages(messages, buildStbVisionModel("managed-primary"));
const resultMessage = result[0]!;
expect(resultMessage.role).toBe("toolResult");
if (resultMessage.role !== "toolResult") throw new Error("Expected tool result message");
expect(resultMessage.content).toEqual([
{ type: "text", text: "[image omitted: WebP could not be decoded for this model]" },
]);
}
});
});