fix(coding-agent): tolerate legacy image metadata
This commit is contained in:
@@ -4,7 +4,7 @@
|
||||
|
||||
### Fixed
|
||||
|
||||
- Fixed custom STB-backed vision providers retaining decoder metadata and sending WebP bytes that llama.cpp cannot decode. Image format is now detected from bytes, attached and historical WebP blocks are normalized on the actual provider-request path, and corrupt WebP is omitted instead of making the turn unrecoverable.
|
||||
- Fixed custom STB-backed vision providers retaining decoder metadata and sending WebP bytes that llama.cpp cannot decode. Image format is now detected from bytes, attached and historical WebP blocks are normalized on the actual provider-request path, and corrupt WebP is omitted instead of making the turn unrecoverable ([#8339](https://github.com/can1357/oh-my-pi/pull/8339) by [@ethancawse](https://github.com/ethancawse)).
|
||||
|
||||
## [17.2.15] - 2026-08-12
|
||||
|
||||
|
||||
@@ -16,10 +16,11 @@ const modelBoundaryImageCache = new LRUCache<string, NormalizedImagePayload | nu
|
||||
sizeCalculation: payload => Math.max(1, payload?.data.length ?? 1),
|
||||
});
|
||||
const modelBoundaryImageNormalizations = new Map<string, Promise<NormalizedImagePayload | null>>();
|
||||
const UNDECODABLE_STB_IMAGE_OMISSION: TextContent = {
|
||||
type: "text",
|
||||
text: "[image omitted: WebP could not be decoded for this model]",
|
||||
};
|
||||
const UNDECODABLE_STB_IMAGE_OMISSION_TEXT = "[image omitted: WebP could not be decoded for this model]";
|
||||
|
||||
function createUndecodableStbImageOmission(): TextContent {
|
||||
return { type: "text", text: UNDECODABLE_STB_IMAGE_OMISSION_TEXT };
|
||||
}
|
||||
|
||||
function hasWebPMagic(data: string): boolean {
|
||||
const header = Buffer.from(data.slice(0, 16), "base64");
|
||||
@@ -29,7 +30,8 @@ function hasWebPMagic(data: string): boolean {
|
||||
}
|
||||
|
||||
function isWebPImage(image: ImageContent): boolean {
|
||||
return image.mimeType.toLowerCase() === "image/webp" || hasWebPMagic(image.data);
|
||||
const mimeType = typeof image.mimeType === "string" ? image.mimeType.toLowerCase() : undefined;
|
||||
return mimeType === "image/webp" || (typeof image.data === "string" && hasWebPMagic(image.data));
|
||||
}
|
||||
|
||||
function modelBoundaryImageCacheKey(image: ImageContent, resize: ImageResizeOptions | undefined): string {
|
||||
@@ -230,7 +232,7 @@ export async function normalizeModelContextMessages(messages: Message[], model:
|
||||
}
|
||||
content ??= message.content.slice(0, partIndex);
|
||||
const normalized = await memoizedStbImageNormalization(part, undefined);
|
||||
content.push(normalized ?? UNDECODABLE_STB_IMAGE_OMISSION);
|
||||
content.push(normalized ?? createUndecodableStbImageOmission());
|
||||
}
|
||||
if (!content) continue;
|
||||
output ??= messages.slice();
|
||||
|
||||
@@ -322,11 +322,12 @@ describe("AgentSession message pipeline", () => {
|
||||
agentId: "SubAgent",
|
||||
});
|
||||
try {
|
||||
const corrupt: ImageContent = {
|
||||
// Session persistence accepts historical image blocks without MIME
|
||||
// metadata, so exercise that runtime shape through the real provider path.
|
||||
const corrupt = {
|
||||
type: "image",
|
||||
data: Buffer.from("RIFF0000WEBPbroken-attachment").toBase64(),
|
||||
mimeType: "image/webp",
|
||||
};
|
||||
} as unknown as ImageContent;
|
||||
|
||||
await session.sendUserMessage([{ type: "text", text: "inspect this" }, corrupt]);
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { afterEach, beforeEach, describe, expect, test } from "bun:test";
|
||||
import type { Api, Model } from "@oh-my-pi/pi-ai";
|
||||
import type { Api, Message, Model } from "@oh-my-pi/pi-ai";
|
||||
import { buildModel } from "@oh-my-pi/pi-catalog/build";
|
||||
import { getBundledModels } from "@oh-my-pi/pi-catalog/models";
|
||||
import type { CustomMessage } from "@oh-my-pi/pi-coding-agent/session/messages";
|
||||
@@ -274,4 +274,32 @@ describe("normalizeModelContextImages model-aware WebP exclusion", () => {
|
||||
]);
|
||||
expect(messages[0]!.content[0]).toBe(corrupt);
|
||||
});
|
||||
|
||||
test("normalizes persisted WebP blocks with malformed MIME metadata", async () => {
|
||||
for (const mimeType of [undefined, null, 42]) {
|
||||
const malformedImage = {
|
||||
type: "image",
|
||||
data: Buffer.from("RIFF0000WEBPbad-persisted-image").toBase64(),
|
||||
...(mimeType === undefined ? {} : { mimeType }),
|
||||
};
|
||||
const messages = [
|
||||
{
|
||||
role: "toolResult",
|
||||
toolCallId: "read-malformed",
|
||||
toolName: "read",
|
||||
content: [malformedImage],
|
||||
isError: false,
|
||||
timestamp: 1,
|
||||
},
|
||||
] as unknown as Message[];
|
||||
|
||||
const result = await normalizeModelContextMessages(messages, buildStbVisionModel("managed-primary"));
|
||||
const resultMessage = result[0]!;
|
||||
expect(resultMessage.role).toBe("toolResult");
|
||||
if (resultMessage.role !== "toolResult") throw new Error("Expected tool result message");
|
||||
expect(resultMessage.content).toEqual([
|
||||
{ type: "text", text: "[image omitted: WebP could not be decoded for this model]" },
|
||||
]);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user