smashimo/oh-my-pi
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(coding-agent): normalize native WebP replay

Browse Source
This commit is contained in:
Ethan Cawse
2026-08-12 11:27:27 -04:00
parent 54965de624
commit 8be57a0ce6
4 changed files with 176 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/coding-agent/src/session/provider-image-budget.ts
+2 -2
View File
@@ -45,13 +45,13 @@ function clampContent(
function clampUserMessage(message: UserMessage, state: { remainingDrops: number }): UserMessage {
if (!Array.isArray(message.content) || state.remainingDrops <= 0) return message;
const content = clampContent(message.content, state);
return content ? { ...message, content } : message;
return content ? { ...message, content, providerPayload: undefined } : message;
}
function clampDeveloperMessage(message: DeveloperMessage, state: { remainingDrops: number }): DeveloperMessage {
if (!Array.isArray(message.content) || state.remainingDrops <= 0) return message;
const content = clampContent(message.content, state);
return content ? { ...message, content } : message;
return content ? { ...message, content, providerPayload: undefined } : message;
}
function clampToolResultMessage(message: ToolResultMessage, state: { remainingDrops: number }): ToolResultMessage {
packages/coding-agent/src/utils/image-loading.ts
+86 -17
View File
@@ -1,6 +1,13 @@
import * as fs from "node:fs/promises";
import type { Context, ImageContent, Message, Model, TextContent } from "@oh-my-pi/pi-ai";
import { formatBytes, logger, readImageMetadata, SUPPORTED_IMAGE_MIME_TYPES } from "@oh-my-pi/pi-utils";
import type {
Context,
ImageContent,
Message,
Model,
OpenAIResponsesHistoryPayload,
TextContent,
} from "@oh-my-pi/pi-ai";
import { formatBytes, isRecord, logger, readImageMetadata, SUPPORTED_IMAGE_MIME_TYPES } from "@oh-my-pi/pi-utils";
import { LRUCache } from "@oh-my-pi/pi-utils/lru";
import { resolveReadPath } from "../tools/path-utils";
import { formatDimensionNote, type ImageResizeOptions, resizeImage } from "./image-resize";
@@ -22,6 +29,10 @@ function createUndecodableStbImageOmission(): TextContent {
return { type: "text", text: UNDECODABLE_STB_IMAGE_OMISSION_TEXT };
}
function createNativeUndecodableStbImageOmission(): Record<string, unknown> {
return { type: "input_text", text: UNDECODABLE_STB_IMAGE_OMISSION_TEXT };
}
function hasWebPMagic(data: string): boolean {
const header = Buffer.from(data.slice(0, 16), "base64");
return (
@@ -35,6 +46,16 @@ function isWebPImage(image: ImageContent): boolean {
return mimeType === "image/webp" || hasWebPMagic(image.data);
}
function imageFromBase64DataUrl(imageUrl: unknown): ImageContent | undefined {
if (typeof imageUrl !== "string" || !imageUrl.toLowerCase().startsWith("data:")) return undefined;
const separator = ";base64,";
const separatorIndex = imageUrl.toLowerCase().indexOf(separator);
if (separatorIndex < 5) return undefined;
const mimeType = imageUrl.slice(5, separatorIndex);
if (!mimeType.toLowerCase().startsWith("image/")) return undefined;
return { type: "image", mimeType, data: imageUrl.slice(separatorIndex + separator.length) };
}
function modelBoundaryImageCacheKey(image: ImageContent, resize: ImageResizeOptions | undefined): string {
const resizeKey = JSON.stringify([
resize?.maxWidth,
@@ -78,6 +99,44 @@ async function memoizedStbImageNormalization(
return normalized ? { ...image, ...normalized } : null;
}
async function normalizeNativeResponsesImagePart(part: unknown): Promise<unknown> {
if (!isRecord(part) || part.type !== "input_image") return part;
const image = imageFromBase64DataUrl(part.image_url);
if (!image || !isWebPImage(image)) return part;
const normalized = await memoizedStbImageNormalization(image, undefined);
if (!normalized) return createNativeUndecodableStbImageOmission();
return { ...part, image_url: `data:${normalized.mimeType};base64,${normalized.data}` };
}
async function normalizeNativeResponsesItem(item: Record<string, unknown>): Promise<Record<string, unknown>> {
const normalizedItem = await normalizeNativeResponsesImagePart(item);
if (normalizedItem !== item) return normalizedItem as Record<string, unknown>;
if (!Array.isArray(item.content)) return item;
let content: unknown[] | undefined;
for (let index = 0; index < item.content.length; index++) {
const part = item.content[index];
const normalizedPart = await normalizeNativeResponsesImagePart(part);
if (normalizedPart !== part) content ??= item.content.slice(0, index);
content?.push(normalizedPart);
}
return content ? { ...item, content } : item;
}
async function normalizeNativeResponsesHistoryPayload(
payload: OpenAIResponsesHistoryPayload | undefined,
): Promise<OpenAIResponsesHistoryPayload | undefined> {
if (payload?.type !== "openaiResponsesHistory" || !Array.isArray(payload.items)) return payload;
let items: Array<Record<string, unknown>> | undefined;
for (let index = 0; index < payload.items.length; index++) {
const item = payload.items[index]!;
const normalizedItem = await normalizeNativeResponsesItem(item);
if (normalizedItem !== item) items ??= payload.items.slice(0, index);
items?.push(normalizedItem);
}
return items ? { ...payload, items } : payload;
}
/**
* Ollama and its local-backend family decode image input through llama.cpp /
* `stb_image`, which is compiled without WebP support, so a WebP upload fails
@@ -223,26 +282,36 @@ export async function normalizeModelContextMessages(messages: Message[], model:
let output: Message[] | undefined;
for (let messageIndex = 0; messageIndex < messages.length; messageIndex++) {
const message = messages[messageIndex]!;
if (typeof message.content === "string") continue;
const hasNativePayload = message.role === "user" || message.role === "developer";
const normalizedProviderPayload = hasNativePayload
? await normalizeNativeResponsesHistoryPayload(message.providerPayload)
: undefined;
const providerPayloadChanged = hasNativePayload && normalizedProviderPayload !== message.providerPayload;
let content: Array<(typeof message.content)[number]> | undefined;
for (let partIndex = 0; partIndex < message.content.length; partIndex++) {
const part = message.content[partIndex]!;
if (part.type !== "image" || !isWebPImage(part)) {
content?.push(part);
continue;
if (typeof message.content !== "string") {
for (let partIndex = 0; partIndex < message.content.length; partIndex++) {
const part = message.content[partIndex]!;
if (part.type !== "image" || !isWebPImage(part)) {
content?.push(part);
continue;
}
content ??= message.content.slice(0, partIndex);
const normalized = await memoizedStbImageNormalization(part, undefined);
content.push(normalized ?? createUndecodableStbImageOmission());
}
content ??= message.content.slice(0, partIndex);
const normalized = await memoizedStbImageNormalization(part, undefined);
content.push(normalized ?? createUndecodableStbImageOmission());
}
if (!content) continue;
if (!content && !providerPayloadChanged) continue;
output ??= messages.slice();
const normalizedMessage = { ...message, content } as Message;
const normalizedMessage = { ...message, ...(content ? { content } : {}) } as Message;
if (normalizedMessage.role === "user" || normalizedMessage.role === "developer") {
// Native Responses history takes precedence over message content. Once an
// image changes, that opaque replay payload is stale and could resend the
// original WebP bytes instead of this normalized transport copy.
delete normalizedMessage.providerPayload;
if (providerPayloadChanged) {
normalizedMessage.providerPayload = normalizedProviderPayload;
} else if (content) {
// Native Responses history takes precedence over message content. If an
// image changed but no matching native image was found, discard the opaque
// replay payload rather than risk resending stale bytes.
delete normalizedMessage.providerPayload;
}
}
output[messageIndex] = normalizedMessage;
}
packages/coding-agent/test/image-webp-exclusion.test.ts
+42 -2
View File
@@ -248,7 +248,7 @@ describe("normalizeModelContextImages model-aware WebP exclusion", () => {
expect(messages[0]!.content[1]).toBe(original);
});
test("drops stale native Responses history after rewriting an image", async () => {
test("rewrites native Responses history alongside generic image content", async () => {
const model = buildStbVisionModel("managed-primary", "openai-responses");
const original = {
type: "image" as const,
@@ -278,7 +278,9 @@ describe("normalizeModelContextImages model-aware WebP exclusion", () => {
const normalizedMessage = messages[0]!;
expect(normalizedMessage.role).toBe("user");
if (normalizedMessage.role !== "user") throw new Error("Expected user message");
expect(normalizedMessage.providerPayload).toBeUndefined();
expect(normalizedMessage.providerPayload).not.toBe(providerPayload);
expect(JSON.stringify(normalizedMessage.providerPayload)).not.toContain("image/webp");
expect(JSON.stringify(normalizedMessage.providerPayload)).not.toContain(original.data);
expect(message.providerPayload).toBe(providerPayload);
const wire = buildResponsesInput({
@@ -294,6 +296,44 @@ describe("normalizeModelContextImages model-aware WebP exclusion", () => {
expect(serializedWire).not.toContain(original.data);
});
test("rewrites WebP retained only in native Responses history", async () => {
const model = buildStbVisionModel("managed-primary", "openai-responses");
const webp = await makeRedWebP(200, 200);
const providerPayload = {
type: "openaiResponsesHistory" as const,
provider: model.provider,
dt: true,
items: [
{
type: "message",
role: "user",
content: [{ type: "input_image", image_url: `data:image/webp;base64,${webp}` }],
},
],
};
const message: Message = { role: "user", content: "inspect native image", providerPayload, timestamp: 1 };
const messages = await normalizeModelContextMessages([message], model);
const normalizedMessage = messages[0]!;
expect(normalizedMessage.role).toBe("user");
if (normalizedMessage.role !== "user") throw new Error("Expected user message");
expect(normalizedMessage.content).toBe("inspect native image");
expect(normalizedMessage.providerPayload).not.toBe(providerPayload);
expect(message.providerPayload).toBe(providerPayload);
const wire = buildResponsesInput({
model,
context: { messages },
strictResponsesPairing: false,
supportsImageDetailOriginal: true,
nativeHistory: { replay: true, filterReasoning: false },
});
const serializedWire = JSON.stringify(wire);
expect(serializedWire).toContain("input_image");
expect(serializedWire).not.toContain("image/webp");
expect(serializedWire).not.toContain(webp);
});
test("replaces an undecodable historical WebP with an omission note", async () => {
const corrupt = {
type: "image" as const,
packages/coding-agent/test/session/provider-image-budget.test.ts
+46
View File
@@ -91,6 +91,52 @@ describe("provider context image budgets", () => {
expect(firstMessage?.content).toEqual([text("[image omitted: provider image limit]")]);
});
it("invalidates native replay payloads when user or developer images are clamped", () => {
const userPayload = {
type: "openaiResponsesHistory" as const,
items: [{ type: "message", role: "user", content: [{ type: "input_image", image_url: "user-native" }] }],
};
const developerPayload = {
type: "openaiResponsesHistory" as const,
items: [{ type: "message", role: "developer", content: [{ type: "input_image", image_url: "dev-native" }] }],
};
const context: Context = {
systemPrompt: [],
tools: [],
messages: [
{ role: "user", content: [image("user-image")], providerPayload: userPayload, timestamp: 0 },
{ role: "developer", content: [image("developer-image")], providerPayload: developerPayload, timestamp: 1 },
...Array.from({ length: 10 }, (_, index) => ({
role: "user" as const,
content: [image(`kept-image-${index}`)],
timestamp: index + 2,
})),
],
};
const clamped = clampProviderContextImages(context, UMANS_MODEL);
const clampedUser = clamped.messages[0];
const clampedDeveloper = clamped.messages[1];
const originalUser = context.messages[0];
const originalDeveloper = context.messages[1];
expect(clampedUser?.role).toBe("user");
expect(clampedDeveloper?.role).toBe("developer");
if (
clampedUser?.role !== "user" ||
clampedDeveloper?.role !== "developer" ||
originalUser?.role !== "user" ||
originalDeveloper?.role !== "developer"
) {
throw new Error("Expected clamped user and developer messages");
}
expect(clampedUser.providerPayload).toBeUndefined();
expect(clampedDeveloper.providerPayload).toBeUndefined();
expect(originalUser.providerPayload).toBe(userPayload);
expect(originalDeveloper.providerPayload).toBe(developerPayload);
expect(imageData(clamped)).toEqual(Array.from({ length: 10 }, (_, index) => `kept-image-${index}`));
});
it("preserves context identity when the provider cap is not exceeded", () => {
const context: Context = {
systemPrompt: [],