diff --git a/packages/coding-agent/src/session/provider-image-budget.ts b/packages/coding-agent/src/session/provider-image-budget.ts index ade57c48c..972309486 100644 --- a/packages/coding-agent/src/session/provider-image-budget.ts +++ b/packages/coding-agent/src/session/provider-image-budget.ts @@ -45,13 +45,13 @@ function clampContent( function clampUserMessage(message: UserMessage, state: { remainingDrops: number }): UserMessage { if (!Array.isArray(message.content) || state.remainingDrops <= 0) return message; const content = clampContent(message.content, state); - return content ? { ...message, content } : message; + return content ? { ...message, content, providerPayload: undefined } : message; } function clampDeveloperMessage(message: DeveloperMessage, state: { remainingDrops: number }): DeveloperMessage { if (!Array.isArray(message.content) || state.remainingDrops <= 0) return message; const content = clampContent(message.content, state); - return content ? { ...message, content } : message; + return content ? { ...message, content, providerPayload: undefined } : message; } function clampToolResultMessage(message: ToolResultMessage, state: { remainingDrops: number }): ToolResultMessage { diff --git a/packages/coding-agent/src/utils/image-loading.ts b/packages/coding-agent/src/utils/image-loading.ts index 8216fe6c5..8267a0db2 100644 --- a/packages/coding-agent/src/utils/image-loading.ts +++ b/packages/coding-agent/src/utils/image-loading.ts @@ -1,6 +1,13 @@ import * as fs from "node:fs/promises"; -import type { Context, ImageContent, Message, Model, TextContent } from "@oh-my-pi/pi-ai"; -import { formatBytes, logger, readImageMetadata, SUPPORTED_IMAGE_MIME_TYPES } from "@oh-my-pi/pi-utils"; +import type { + Context, + ImageContent, + Message, + Model, + OpenAIResponsesHistoryPayload, + TextContent, +} from "@oh-my-pi/pi-ai"; +import { formatBytes, isRecord, logger, readImageMetadata, SUPPORTED_IMAGE_MIME_TYPES } from "@oh-my-pi/pi-utils"; import { LRUCache } from "@oh-my-pi/pi-utils/lru"; import { resolveReadPath } from "../tools/path-utils"; import { formatDimensionNote, type ImageResizeOptions, resizeImage } from "./image-resize"; @@ -22,6 +29,10 @@ function createUndecodableStbImageOmission(): TextContent { return { type: "text", text: UNDECODABLE_STB_IMAGE_OMISSION_TEXT }; } +function createNativeUndecodableStbImageOmission(): Record { + return { type: "input_text", text: UNDECODABLE_STB_IMAGE_OMISSION_TEXT }; +} + function hasWebPMagic(data: string): boolean { const header = Buffer.from(data.slice(0, 16), "base64"); return ( @@ -35,6 +46,16 @@ function isWebPImage(image: ImageContent): boolean { return mimeType === "image/webp" || hasWebPMagic(image.data); } +function imageFromBase64DataUrl(imageUrl: unknown): ImageContent | undefined { + if (typeof imageUrl !== "string" || !imageUrl.toLowerCase().startsWith("data:")) return undefined; + const separator = ";base64,"; + const separatorIndex = imageUrl.toLowerCase().indexOf(separator); + if (separatorIndex < 5) return undefined; + const mimeType = imageUrl.slice(5, separatorIndex); + if (!mimeType.toLowerCase().startsWith("image/")) return undefined; + return { type: "image", mimeType, data: imageUrl.slice(separatorIndex + separator.length) }; +} + function modelBoundaryImageCacheKey(image: ImageContent, resize: ImageResizeOptions | undefined): string { const resizeKey = JSON.stringify([ resize?.maxWidth, @@ -78,6 +99,44 @@ async function memoizedStbImageNormalization( return normalized ? { ...image, ...normalized } : null; } +async function normalizeNativeResponsesImagePart(part: unknown): Promise { + if (!isRecord(part) || part.type !== "input_image") return part; + const image = imageFromBase64DataUrl(part.image_url); + if (!image || !isWebPImage(image)) return part; + const normalized = await memoizedStbImageNormalization(image, undefined); + if (!normalized) return createNativeUndecodableStbImageOmission(); + return { ...part, image_url: `data:${normalized.mimeType};base64,${normalized.data}` }; +} + +async function normalizeNativeResponsesItem(item: Record): Promise> { + const normalizedItem = await normalizeNativeResponsesImagePart(item); + if (normalizedItem !== item) return normalizedItem as Record; + if (!Array.isArray(item.content)) return item; + + let content: unknown[] | undefined; + for (let index = 0; index < item.content.length; index++) { + const part = item.content[index]; + const normalizedPart = await normalizeNativeResponsesImagePart(part); + if (normalizedPart !== part) content ??= item.content.slice(0, index); + content?.push(normalizedPart); + } + return content ? { ...item, content } : item; +} + +async function normalizeNativeResponsesHistoryPayload( + payload: OpenAIResponsesHistoryPayload | undefined, +): Promise { + if (payload?.type !== "openaiResponsesHistory" || !Array.isArray(payload.items)) return payload; + let items: Array> | undefined; + for (let index = 0; index < payload.items.length; index++) { + const item = payload.items[index]!; + const normalizedItem = await normalizeNativeResponsesItem(item); + if (normalizedItem !== item) items ??= payload.items.slice(0, index); + items?.push(normalizedItem); + } + return items ? { ...payload, items } : payload; +} + /** * Ollama and its local-backend family decode image input through llama.cpp / * `stb_image`, which is compiled without WebP support, so a WebP upload fails @@ -223,26 +282,36 @@ export async function normalizeModelContextMessages(messages: Message[], model: let output: Message[] | undefined; for (let messageIndex = 0; messageIndex < messages.length; messageIndex++) { const message = messages[messageIndex]!; - if (typeof message.content === "string") continue; + const hasNativePayload = message.role === "user" || message.role === "developer"; + const normalizedProviderPayload = hasNativePayload + ? await normalizeNativeResponsesHistoryPayload(message.providerPayload) + : undefined; + const providerPayloadChanged = hasNativePayload && normalizedProviderPayload !== message.providerPayload; let content: Array<(typeof message.content)[number]> | undefined; - for (let partIndex = 0; partIndex < message.content.length; partIndex++) { - const part = message.content[partIndex]!; - if (part.type !== "image" || !isWebPImage(part)) { - content?.push(part); - continue; + if (typeof message.content !== "string") { + for (let partIndex = 0; partIndex < message.content.length; partIndex++) { + const part = message.content[partIndex]!; + if (part.type !== "image" || !isWebPImage(part)) { + content?.push(part); + continue; + } + content ??= message.content.slice(0, partIndex); + const normalized = await memoizedStbImageNormalization(part, undefined); + content.push(normalized ?? createUndecodableStbImageOmission()); } - content ??= message.content.slice(0, partIndex); - const normalized = await memoizedStbImageNormalization(part, undefined); - content.push(normalized ?? createUndecodableStbImageOmission()); } - if (!content) continue; + if (!content && !providerPayloadChanged) continue; output ??= messages.slice(); - const normalizedMessage = { ...message, content } as Message; + const normalizedMessage = { ...message, ...(content ? { content } : {}) } as Message; if (normalizedMessage.role === "user" || normalizedMessage.role === "developer") { - // Native Responses history takes precedence over message content. Once an - // image changes, that opaque replay payload is stale and could resend the - // original WebP bytes instead of this normalized transport copy. - delete normalizedMessage.providerPayload; + if (providerPayloadChanged) { + normalizedMessage.providerPayload = normalizedProviderPayload; + } else if (content) { + // Native Responses history takes precedence over message content. If an + // image changed but no matching native image was found, discard the opaque + // replay payload rather than risk resending stale bytes. + delete normalizedMessage.providerPayload; + } } output[messageIndex] = normalizedMessage; } diff --git a/packages/coding-agent/test/image-webp-exclusion.test.ts b/packages/coding-agent/test/image-webp-exclusion.test.ts index 991adf475..0d0a33bc7 100644 --- a/packages/coding-agent/test/image-webp-exclusion.test.ts +++ b/packages/coding-agent/test/image-webp-exclusion.test.ts @@ -248,7 +248,7 @@ describe("normalizeModelContextImages model-aware WebP exclusion", () => { expect(messages[0]!.content[1]).toBe(original); }); - test("drops stale native Responses history after rewriting an image", async () => { + test("rewrites native Responses history alongside generic image content", async () => { const model = buildStbVisionModel("managed-primary", "openai-responses"); const original = { type: "image" as const, @@ -278,7 +278,9 @@ describe("normalizeModelContextImages model-aware WebP exclusion", () => { const normalizedMessage = messages[0]!; expect(normalizedMessage.role).toBe("user"); if (normalizedMessage.role !== "user") throw new Error("Expected user message"); - expect(normalizedMessage.providerPayload).toBeUndefined(); + expect(normalizedMessage.providerPayload).not.toBe(providerPayload); + expect(JSON.stringify(normalizedMessage.providerPayload)).not.toContain("image/webp"); + expect(JSON.stringify(normalizedMessage.providerPayload)).not.toContain(original.data); expect(message.providerPayload).toBe(providerPayload); const wire = buildResponsesInput({ @@ -294,6 +296,44 @@ describe("normalizeModelContextImages model-aware WebP exclusion", () => { expect(serializedWire).not.toContain(original.data); }); + test("rewrites WebP retained only in native Responses history", async () => { + const model = buildStbVisionModel("managed-primary", "openai-responses"); + const webp = await makeRedWebP(200, 200); + const providerPayload = { + type: "openaiResponsesHistory" as const, + provider: model.provider, + dt: true, + items: [ + { + type: "message", + role: "user", + content: [{ type: "input_image", image_url: `data:image/webp;base64,${webp}` }], + }, + ], + }; + const message: Message = { role: "user", content: "inspect native image", providerPayload, timestamp: 1 }; + + const messages = await normalizeModelContextMessages([message], model); + const normalizedMessage = messages[0]!; + expect(normalizedMessage.role).toBe("user"); + if (normalizedMessage.role !== "user") throw new Error("Expected user message"); + expect(normalizedMessage.content).toBe("inspect native image"); + expect(normalizedMessage.providerPayload).not.toBe(providerPayload); + expect(message.providerPayload).toBe(providerPayload); + + const wire = buildResponsesInput({ + model, + context: { messages }, + strictResponsesPairing: false, + supportsImageDetailOriginal: true, + nativeHistory: { replay: true, filterReasoning: false }, + }); + const serializedWire = JSON.stringify(wire); + expect(serializedWire).toContain("input_image"); + expect(serializedWire).not.toContain("image/webp"); + expect(serializedWire).not.toContain(webp); + }); + test("replaces an undecodable historical WebP with an omission note", async () => { const corrupt = { type: "image" as const, diff --git a/packages/coding-agent/test/session/provider-image-budget.test.ts b/packages/coding-agent/test/session/provider-image-budget.test.ts index eefafd2b6..a164eca2f 100644 --- a/packages/coding-agent/test/session/provider-image-budget.test.ts +++ b/packages/coding-agent/test/session/provider-image-budget.test.ts @@ -91,6 +91,52 @@ describe("provider context image budgets", () => { expect(firstMessage?.content).toEqual([text("[image omitted: provider image limit]")]); }); + it("invalidates native replay payloads when user or developer images are clamped", () => { + const userPayload = { + type: "openaiResponsesHistory" as const, + items: [{ type: "message", role: "user", content: [{ type: "input_image", image_url: "user-native" }] }], + }; + const developerPayload = { + type: "openaiResponsesHistory" as const, + items: [{ type: "message", role: "developer", content: [{ type: "input_image", image_url: "dev-native" }] }], + }; + const context: Context = { + systemPrompt: [], + tools: [], + messages: [ + { role: "user", content: [image("user-image")], providerPayload: userPayload, timestamp: 0 }, + { role: "developer", content: [image("developer-image")], providerPayload: developerPayload, timestamp: 1 }, + ...Array.from({ length: 10 }, (_, index) => ({ + role: "user" as const, + content: [image(`kept-image-${index}`)], + timestamp: index + 2, + })), + ], + }; + + const clamped = clampProviderContextImages(context, UMANS_MODEL); + const clampedUser = clamped.messages[0]; + const clampedDeveloper = clamped.messages[1]; + const originalUser = context.messages[0]; + const originalDeveloper = context.messages[1]; + + expect(clampedUser?.role).toBe("user"); + expect(clampedDeveloper?.role).toBe("developer"); + if ( + clampedUser?.role !== "user" || + clampedDeveloper?.role !== "developer" || + originalUser?.role !== "user" || + originalDeveloper?.role !== "developer" + ) { + throw new Error("Expected clamped user and developer messages"); + } + expect(clampedUser.providerPayload).toBeUndefined(); + expect(clampedDeveloper.providerPayload).toBeUndefined(); + expect(originalUser.providerPayload).toBe(userPayload); + expect(originalDeveloper.providerPayload).toBe(developerPayload); + expect(imageData(clamped)).toEqual(Array.from({ length: 10 }, (_, index) => `kept-image-${index}`)); + }); + it("preserves context identity when the provider cap is not exceeded", () => { const context: Context = { systemPrompt: [],