diff --git a/docs/models.md b/docs/models.md index c837cf293..dd709ed7b 100644 --- a/docs/models.md +++ b/docs/models.md @@ -61,6 +61,7 @@ providers: api: openai-completions reasoning: false input: [text] + imageInputDecoder: stb # local STB decoder; OMP converts WebP before dispatch cost: input: 0 output: 0 @@ -101,6 +102,7 @@ providers: - `auth`: `apiKey` (default), `none`, or `oauth`; for `models.yml` custom models, `oauth` is accepted by schema but does not waive the `apiKey` requirement - `discovery.type`: `ollama`, `llama.cpp`, `lm-studio`, `openai-models-list`, `proxy`, or `litellm` - `transport`: `pi-native` only. When set, every model under that provider is sent to an `omp auth-gateway` compatible `baseUrl` via `POST /v1/pi/stream`; `apiKey` is the gateway bearer. +- `imageInputDecoder`: `stb` only. Set this on a custom model or `modelOverrides` entry when the serving backend uses an STB-compatible image decoder that cannot accept WebP; OMP converts attached and historical WebP images before provider dispatch. ## Validation rules (current) @@ -190,7 +192,7 @@ Provider defaults vs per-model overrides: - Provider `headers`, `compat`, and `remoteCompaction` are baselines. - Model `headers` override provider header keys. -- `modelOverrides` can override model metadata (`name`, `reasoning`, `thinking`, `input`, +- `modelOverrides` can override model metadata (`name`, `reasoning`, `thinking`, `input`, `imageInputDecoder`, `supportsTools`, `cost`, `premiumMultiplier`, `contextWindow`, `maxTokens`, `omitMaxOutputTokens`, `headers`, `compat`, `contextPromotionTarget`, `compactionModel`, and `remoteCompaction`). diff --git a/packages/coding-agent/src/utils/image-loading.ts b/packages/coding-agent/src/utils/image-loading.ts index 0bad86f6a..6b321d6c0 100644 --- a/packages/coding-agent/src/utils/image-loading.ts +++ b/packages/coding-agent/src/utils/image-loading.ts @@ -236,7 +236,14 @@ export async function normalizeModelContextMessages(messages: Message[], model: } if (!content) continue; output ??= messages.slice(); - output[messageIndex] = { ...message, content } as Message; + const normalizedMessage = { ...message, content } as Message; + if (normalizedMessage.role === "user" || normalizedMessage.role === "developer") { + // Native Responses history takes precedence over message content. Once an + // image changes, that opaque replay payload is stale and could resend the + // original WebP bytes instead of this normalized transport copy. + delete normalizedMessage.providerPayload; + } + output[messageIndex] = normalizedMessage; } return output ?? messages; } diff --git a/packages/coding-agent/test/image-webp-exclusion.test.ts b/packages/coding-agent/test/image-webp-exclusion.test.ts index 143d3d157..952df6731 100644 --- a/packages/coding-agent/test/image-webp-exclusion.test.ts +++ b/packages/coding-agent/test/image-webp-exclusion.test.ts @@ -1,5 +1,6 @@ import { afterEach, beforeEach, describe, expect, test } from "bun:test"; import type { Api, Message, Model } from "@oh-my-pi/pi-ai"; +import { buildResponsesInput } from "@oh-my-pi/pi-ai/providers/openai-shared"; import { buildModel } from "@oh-my-pi/pi-catalog/build"; import { getBundledModels } from "@oh-my-pi/pi-catalog/models"; import type { CustomMessage } from "@oh-my-pi/pi-coding-agent/session/messages"; @@ -247,6 +248,52 @@ describe("normalizeModelContextImages model-aware WebP exclusion", () => { expect(messages[0]!.content[1]).toBe(original); }); + test("drops stale native Responses history after rewriting an image", async () => { + const model = buildStbVisionModel("managed-primary", "openai-responses"); + const original = { + type: "image" as const, + data: await makeRedWebP(200, 200), + mimeType: "image/webp", + }; + const providerPayload = { + type: "openaiResponsesHistory" as const, + provider: model.provider, + dt: true, + items: [ + { + type: "message", + role: "user", + content: [{ type: "input_image", image_url: `data:image/webp;base64,${original.data}` }], + }, + ], + }; + const message: Message = { + role: "user", + content: [{ type: "text", text: "inspect" }, original], + providerPayload, + timestamp: 1, + }; + + const messages = await normalizeModelContextMessages([message], model); + const normalizedMessage = messages[0]!; + expect(normalizedMessage.role).toBe("user"); + if (normalizedMessage.role !== "user") throw new Error("Expected user message"); + expect(normalizedMessage.providerPayload).toBeUndefined(); + expect(message.providerPayload).toBe(providerPayload); + + const wire = buildResponsesInput({ + model, + context: { messages }, + strictResponsesPairing: false, + supportsImageDetailOriginal: true, + nativeHistory: { replay: true, filterReasoning: false }, + }); + const serializedWire = JSON.stringify(wire); + expect(serializedWire).toContain("input_image"); + expect(serializedWire).not.toContain("image/webp"); + expect(serializedWire).not.toContain(original.data); + }); + test("replaces an undecodable historical WebP with an omission note", async () => { const corrupt = { type: "image" as const,