From 4c71e92e3eaf6365c5451ed3d586536d538c4f06 Mon Sep 17 00:00:00 2001 From: Ethan Cawse Date: Wed, 12 Aug 2026 09:13:03 -0400 Subject: [PATCH] fix(coding-agent): tolerate legacy image metadata --- packages/coding-agent/CHANGELOG.md | 2 +- .../coding-agent/src/utils/image-loading.ts | 14 +++++---- .../agent-session-message-pipeline.test.ts | 7 +++-- .../test/image-webp-exclusion.test.ts | 30 ++++++++++++++++++- 4 files changed, 42 insertions(+), 11 deletions(-) diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index fa7f39bb6..d76861c5c 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -4,7 +4,7 @@ ### Fixed -- Fixed custom STB-backed vision providers retaining decoder metadata and sending WebP bytes that llama.cpp cannot decode. Image format is now detected from bytes, attached and historical WebP blocks are normalized on the actual provider-request path, and corrupt WebP is omitted instead of making the turn unrecoverable. +- Fixed custom STB-backed vision providers retaining decoder metadata and sending WebP bytes that llama.cpp cannot decode. Image format is now detected from bytes, attached and historical WebP blocks are normalized on the actual provider-request path, and corrupt WebP is omitted instead of making the turn unrecoverable ([#8339](https://github.com/can1357/oh-my-pi/pull/8339) by [@ethancawse](https://github.com/ethancawse)). ## [17.2.15] - 2026-08-12 diff --git a/packages/coding-agent/src/utils/image-loading.ts b/packages/coding-agent/src/utils/image-loading.ts index a9f8ae036..0bad86f6a 100644 --- a/packages/coding-agent/src/utils/image-loading.ts +++ b/packages/coding-agent/src/utils/image-loading.ts @@ -16,10 +16,11 @@ const modelBoundaryImageCache = new LRUCache Math.max(1, payload?.data.length ?? 1), }); const modelBoundaryImageNormalizations = new Map>(); -const UNDECODABLE_STB_IMAGE_OMISSION: TextContent = { - type: "text", - text: "[image omitted: WebP could not be decoded for this model]", -}; +const UNDECODABLE_STB_IMAGE_OMISSION_TEXT = "[image omitted: WebP could not be decoded for this model]"; + +function createUndecodableStbImageOmission(): TextContent { + return { type: "text", text: UNDECODABLE_STB_IMAGE_OMISSION_TEXT }; +} function hasWebPMagic(data: string): boolean { const header = Buffer.from(data.slice(0, 16), "base64"); @@ -29,7 +30,8 @@ function hasWebPMagic(data: string): boolean { } function isWebPImage(image: ImageContent): boolean { - return image.mimeType.toLowerCase() === "image/webp" || hasWebPMagic(image.data); + const mimeType = typeof image.mimeType === "string" ? image.mimeType.toLowerCase() : undefined; + return mimeType === "image/webp" || (typeof image.data === "string" && hasWebPMagic(image.data)); } function modelBoundaryImageCacheKey(image: ImageContent, resize: ImageResizeOptions | undefined): string { @@ -230,7 +232,7 @@ export async function normalizeModelContextMessages(messages: Message[], model: } content ??= message.content.slice(0, partIndex); const normalized = await memoizedStbImageNormalization(part, undefined); - content.push(normalized ?? UNDECODABLE_STB_IMAGE_OMISSION); + content.push(normalized ?? createUndecodableStbImageOmission()); } if (!content) continue; output ??= messages.slice(); diff --git a/packages/coding-agent/test/agent-session-message-pipeline.test.ts b/packages/coding-agent/test/agent-session-message-pipeline.test.ts index d4dcf4c4d..fe7d378a9 100644 --- a/packages/coding-agent/test/agent-session-message-pipeline.test.ts +++ b/packages/coding-agent/test/agent-session-message-pipeline.test.ts @@ -322,11 +322,12 @@ describe("AgentSession message pipeline", () => { agentId: "SubAgent", }); try { - const corrupt: ImageContent = { + // Session persistence accepts historical image blocks without MIME + // metadata, so exercise that runtime shape through the real provider path. + const corrupt = { type: "image", data: Buffer.from("RIFF0000WEBPbroken-attachment").toBase64(), - mimeType: "image/webp", - }; + } as unknown as ImageContent; await session.sendUserMessage([{ type: "text", text: "inspect this" }, corrupt]); diff --git a/packages/coding-agent/test/image-webp-exclusion.test.ts b/packages/coding-agent/test/image-webp-exclusion.test.ts index 94d09370f..143d3d157 100644 --- a/packages/coding-agent/test/image-webp-exclusion.test.ts +++ b/packages/coding-agent/test/image-webp-exclusion.test.ts @@ -1,5 +1,5 @@ import { afterEach, beforeEach, describe, expect, test } from "bun:test"; -import type { Api, Model } from "@oh-my-pi/pi-ai"; +import type { Api, Message, Model } from "@oh-my-pi/pi-ai"; import { buildModel } from "@oh-my-pi/pi-catalog/build"; import { getBundledModels } from "@oh-my-pi/pi-catalog/models"; import type { CustomMessage } from "@oh-my-pi/pi-coding-agent/session/messages"; @@ -274,4 +274,32 @@ describe("normalizeModelContextImages model-aware WebP exclusion", () => { ]); expect(messages[0]!.content[0]).toBe(corrupt); }); + + test("normalizes persisted WebP blocks with malformed MIME metadata", async () => { + for (const mimeType of [undefined, null, 42]) { + const malformedImage = { + type: "image", + data: Buffer.from("RIFF0000WEBPbad-persisted-image").toBase64(), + ...(mimeType === undefined ? {} : { mimeType }), + }; + const messages = [ + { + role: "toolResult", + toolCallId: "read-malformed", + toolName: "read", + content: [malformedImage], + isError: false, + timestamp: 1, + }, + ] as unknown as Message[]; + + const result = await normalizeModelContextMessages(messages, buildStbVisionModel("managed-primary")); + const resultMessage = result[0]!; + expect(resultMessage.role).toBe("toolResult"); + if (resultMessage.role !== "toolResult") throw new Error("Expected tool result message"); + expect(resultMessage.content).toEqual([ + { type: "text", text: "[image omitted: WebP could not be decoded for this model]" }, + ]); + } + }); });