diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index b387e76fd..a10240708 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -25,6 +25,9 @@ - Fixed `/handoff` losing local artifacts (plans, scratch files, research notes) by copying them across the handoff session boundary. - Replaced libarchive-based tar parsing with a hardened, in-process tar reader to prevent crashes and safely handle complex archive structures, symlinks, and sparse metadata. - Fixed `Ctrl+O` tool-output expansion failing to reach launch-completion messages wrapped in the hidden tool activity container. +### Fixed + +- Fixed custom STB-backed vision providers retaining decoder metadata and re-encoding resumed WebP images before provider requests. ## [17.2.14] - 2026-08-11 diff --git a/packages/coding-agent/src/config/custom-models.ts b/packages/coding-agent/src/config/custom-models.ts index d692dc8a8..ce4310fbb 100644 --- a/packages/coding-agent/src/config/custom-models.ts +++ b/packages/coding-agent/src/config/custom-models.ts @@ -85,6 +85,7 @@ export function buildCustomModelOverlay( reasoning: modelDef.reasoning, thinking: modelDef.thinking, input: modelDef.input, + imageInputDecoder: modelDef.imageInputDecoder, supportsTools: modelDef.supportsTools, cost: modelDef.cost, contextWindow: modelDef.contextWindow, @@ -127,6 +128,7 @@ export function finalizeCustomModel(model: CustomModelOverlay, options: CustomMo reasoning: resolvedModel.reasoning ?? reference?.reasoning ?? (options.useDefaults ? false : undefined), thinking: inheritReferenceThinking(resolvedModel.thinking, reference, resolvedModel.provider), input: input as ("text" | "image")[], + imageInputDecoder: resolvedModel.imageInputDecoder, ...(supportsTools !== undefined ? { supportsTools } : {}), cost, contextWindow: resolvedModel.contextWindow ?? reference?.contextWindow ?? (options.useDefaults ? 128000 : null), diff --git a/packages/coding-agent/src/config/model-patch.ts b/packages/coding-agent/src/config/model-patch.ts index 5cc8d2415..38e9920d0 100644 --- a/packages/coding-agent/src/config/model-patch.ts +++ b/packages/coding-agent/src/config/model-patch.ts @@ -183,6 +183,7 @@ export interface ModelPatch { reasoning?: boolean; thinking?: ThinkingConfig; input?: ("text" | "image")[]; + imageInputDecoder?: Model["imageInputDecoder"]; supportsTools?: boolean; cost?: Partial["cost"]>; contextWindow?: number; @@ -210,6 +211,7 @@ export function applyModelPatch(base: Model, patch: ModelPatch, transport: if (patch.reasoning !== undefined) result.reasoning = patch.reasoning; if (patch.thinking !== undefined) result.thinking = patch.thinking; if (patch.input !== undefined) result.input = patch.input; + if (patch.imageInputDecoder !== undefined) result.imageInputDecoder = patch.imageInputDecoder; if (patch.supportsTools !== undefined) result.supportsTools = patch.supportsTools; if (patch.contextWindow !== undefined) result.contextWindow = patch.contextWindow; if (patch.maxTokens !== undefined) result.maxTokens = patch.maxTokens; diff --git a/packages/coding-agent/src/config/models-config-schema-bundle.ts b/packages/coding-agent/src/config/models-config-schema-bundle.ts index 2620360ff..458eeaca1 100644 --- a/packages/coding-agent/src/config/models-config-schema-bundle.ts +++ b/packages/coding-agent/src/config/models-config-schema-bundle.ts @@ -167,6 +167,7 @@ export const getModelsConfigSchemaBundle = once(() => { "reasoning?": "boolean", "thinking?": ModelThinkingSchema, "input?": '("text" | "image")[]', + "imageInputDecoder?": '"stb"', "supportsTools?": "boolean", "cost?": { input: "number", @@ -216,6 +217,7 @@ export const getModelsConfigSchemaBundle = once(() => { "reasoning?": "boolean", "thinking?": ModelThinkingSchema, "input?": '("text" | "image")[]', + "imageInputDecoder?": '"stb"', "supportsTools?": "boolean", "cost?": { "input?": "number", diff --git a/packages/coding-agent/src/session/session-provider-boundary.ts b/packages/coding-agent/src/session/session-provider-boundary.ts index 66754cd89..bd87cb52e 100644 --- a/packages/coding-agent/src/session/session-provider-boundary.ts +++ b/packages/coding-agent/src/session/session-provider-boundary.ts @@ -13,7 +13,7 @@ import type { LocalProtocolOptions } from "../internal-urls"; import { deobfuscateSessionContext, obfuscateMessages } from "../secrets/message-transform"; import type { SecretObfuscator } from "../secrets/obfuscator"; import { stripPendingSecretPlaceholderSuffix } from "../secrets/placeholder"; -import { normalizeModelContextImages } from "../utils/image-loading"; +import { normalizeModelContextImages, normalizeModelContextMessages } from "../utils/image-loading"; import { describeAttachedImagesForTextModel } from "../utils/image-vision-fallback"; import { type CustomMessage, convertToLlm } from "./messages"; import { IMAGE_ATTACHMENT_DESCRIPTION_TYPE } from "./queued-messages"; @@ -126,7 +126,8 @@ export class SessionProviderBoundary { /** Converts session messages using the configured pre-LLM pipeline. */ async convertMessagesToLlm(messages: AgentMessage[], signal?: AbortSignal): Promise { const transformedMessages = await this.#host.transformContext(messages, signal); - return await this.#host.convertToLlm(transformedMessages); + const converted = await this.#host.convertToLlm(transformedMessages); + return await normalizeModelContextMessages(converted, this.#host.model()); } /** Applies session-level stream hooks and provider defaults to a side request. */ diff --git a/packages/coding-agent/src/utils/image-loading.ts b/packages/coding-agent/src/utils/image-loading.ts index 408406870..d0fb7a525 100644 --- a/packages/coding-agent/src/utils/image-loading.ts +++ b/packages/coding-agent/src/utils/image-loading.ts @@ -1,11 +1,50 @@ import * as fs from "node:fs/promises"; -import type { ImageContent, Model } from "@oh-my-pi/pi-ai"; +import type { ImageContent, Message, Model } from "@oh-my-pi/pi-ai"; import { formatBytes, readImageMetadata, SUPPORTED_IMAGE_MIME_TYPES } from "@oh-my-pi/pi-utils"; +import { LRUCache } from "@oh-my-pi/pi-utils/lru"; import { resolveReadPath } from "../tools/path-utils"; import { formatDimensionNote, type ImageResizeOptions, resizeImage } from "./image-resize"; export const MAX_IMAGE_INPUT_BYTES = 20 * 1024 * 1024; export const SUPPORTED_INPUT_IMAGE_MIME_TYPES = SUPPORTED_IMAGE_MIME_TYPES; +const MODEL_BOUNDARY_IMAGE_CACHE_LIMIT = 128; +type NormalizedImagePayload = Pick; +const modelBoundaryImageCache = new LRUCache>({ + max: MODEL_BOUNDARY_IMAGE_CACHE_LIMIT, +}); + +function hasWebPMagic(data: string): boolean { + const header = Buffer.from(data.slice(0, 16), "base64"); + return ( + header.length >= 12 && header.toString("ascii", 0, 4) === "RIFF" && header.toString("ascii", 8, 12) === "WEBP" + ); +} + +function isWebPImage(image: ImageContent): boolean { + return image.mimeType.toLowerCase() === "image/webp" || hasWebPMagic(image.data); +} + +function modelBoundaryImageCacheKey(image: ImageContent): string { + return `${image.data.length}:${image.data.slice(0, 32)}:${image.data.slice(-32)}:${String(Bun.hash(image.data))}`; +} + +async function memoizedStbImageNormalization(image: ImageContent): Promise { + const key = modelBoundaryImageCacheKey(image); + let pending = modelBoundaryImageCache.get(key); + if (!pending) { + pending = resizeImage(image, { excludeWebP: true }).then(resized => { + if (resized.mimeType === "image/webp" || hasWebPMagic(resized.data)) { + throw new Error("Image normalization retained WebP for an STB-backed model"); + } + return { data: resized.data, mimeType: resized.mimeType }; + }); + modelBoundaryImageCache.set(key, pending); + void pending.catch(() => { + if (modelBoundaryImageCache.peek(key) === pending) modelBoundaryImageCache.delete(key); + }); + } + return { ...image, ...(await pending) }; +} /** * Ollama and its local-backend family decode image input through llama.cpp / @@ -116,15 +155,23 @@ export async function normalizeModelContextImages( options?: NormalizeModelContextImagesOptions, ): Promise { if (!images || images.length === 0) return undefined; - const resize: ImageResizeOptions | undefined = modelLacksWebpSupport(options?.model) + const excludesWebP = modelLacksWebpSupport(options?.model); + const resize: ImageResizeOptions | undefined = excludesWebP ? { ...options?.resize, excludeWebP: true } : options?.resize; const normalized: ImageContent[] = []; for (const image of images) { try { + if (excludesWebP && isWebPImage(image)) { + normalized.push(await memoizedStbImageNormalization(image)); + continue; + } const resized = await resizeImage(image, resize); - normalized.push({ type: "image", data: resized.data, mimeType: resized.mimeType }); - } catch { + normalized.push({ ...image, data: resized.data, mimeType: resized.mimeType }); + } catch (error) { + if (excludesWebP && isWebPImage(image)) { + throw new Error("Failed to convert WebP image for an STB-backed model", { cause: error }); + } // Preserve existing caller behavior for decode/resize failures: keep the // user's image block rather than dropping it from the turn. normalized.push(image); @@ -133,6 +180,34 @@ export async function normalizeModelContextImages( return normalized; } +/** + * Rewrites historical/resumed WebP blocks in the ephemeral provider request. + * Persisted session messages remain untouched, while STB-backed local servers + * never receive a format they cannot decode. + */ +export async function normalizeModelContextMessages(messages: Message[], model: Model | undefined): Promise { + if (!modelLacksWebpSupport(model)) return messages; + let output: Message[] | undefined; + for (let messageIndex = 0; messageIndex < messages.length; messageIndex++) { + const message = messages[messageIndex]!; + if (typeof message.content === "string") continue; + const webpImages: ImageContent[] = []; + for (const part of message.content) { + if (part.type === "image" && isWebPImage(part)) webpImages.push(part); + } + if (webpImages.length === 0) continue; + const normalized = await normalizeModelContextImages(webpImages, { model }); + if (!normalized) continue; + let imageIndex = 0; + const content = message.content.map(part => + part.type === "image" && isWebPImage(part) ? normalized[imageIndex++]! : part, + ); + output ??= messages.slice(); + output[messageIndex] = { ...message, content } as Message; + } + return output ?? messages; +} + export async function loadImageInput(options: LoadImageInputOptions): Promise { const maxBytes = options.maxBytes ?? MAX_IMAGE_INPUT_BYTES; const resolvedPath = options.resolvedPath ?? resolveReadPath(options.path, options.cwd); diff --git a/packages/coding-agent/src/utils/image-resize.ts b/packages/coding-agent/src/utils/image-resize.ts index 1d61586b7..00071b691 100644 --- a/packages/coding-agent/src/utils/image-resize.ts +++ b/packages/coding-agent/src/utils/image-resize.ts @@ -167,7 +167,9 @@ export async function resizeImage(img: ImageContent, options?: ImageResizeOption try { const { width: originalWidth, height: originalHeight, format } = await new Bun.Image(inputBuffer).metadata(); - const sourceMime = img.mimeType ?? `image/${format}`; + // Trust decoded bytes over caller metadata. A mislabeled WebP must not take + // the fast path when the target decoder explicitly excludes WebP. + const sourceMime = format ? `image/${format}` : img.mimeType; // Fast path: already within dimensions AND well under budget. // Threshold is 1/4 of budget — if already that compact, don't re-encode. diff --git a/packages/coding-agent/test/image-webp-exclusion.test.ts b/packages/coding-agent/test/image-webp-exclusion.test.ts index 926887ec6..dacf4d0ad 100644 --- a/packages/coding-agent/test/image-webp-exclusion.test.ts +++ b/packages/coding-agent/test/image-webp-exclusion.test.ts @@ -5,6 +5,7 @@ import { getBundledModels } from "@oh-my-pi/pi-catalog/models"; import { modelLacksWebpSupport, normalizeModelContextImages, + normalizeModelContextMessages, webpExclusionForModel, } from "@oh-my-pi/pi-coding-agent/utils/image-loading"; @@ -145,4 +146,39 @@ describe("normalizeModelContextImages model-aware WebP exclusion", () => { expect(result?.[0]?.mimeType).toBe("image/webp"); }); + + test("rewrites resumed tool-result WebP blocks at the STB provider boundary", async () => { + const original = { + type: "image" as const, + data: await makeRedWebP(200, 200), + // Exercise byte sniffing as well as declared-MIME handling. + mimeType: "image/png", + detail: "original" as const, + }; + const messages = [ + { + role: "toolResult" as const, + toolCallId: "read-1", + toolName: "read", + content: [{ type: "text" as const, text: "screenshot" }, original], + isError: false, + timestamp: 1, + }, + ]; + + const result = await normalizeModelContextMessages(messages, buildStbVisionModel("managed-primary")); + const resultMessage = result[0]!; + expect(resultMessage.role).toBe("toolResult"); + if (resultMessage.role !== "toolResult") throw new Error("Expected tool result message"); + const image = resultMessage.content[1]!; + + expect(image.type).toBe("image"); + if (image.type !== "image") throw new Error("Expected normalized image block"); + expect(image.mimeType).not.toBe("image/webp"); + expect(["image/png", "image/jpeg"]).toContain(image.mimeType); + expect(Buffer.from(image.data.slice(0, 16), "base64").toString("ascii", 8, 12)).not.toBe("WEBP"); + expect(image.detail).toBe("original"); + // Provider-boundary normalization is ephemeral; persisted history is not mutated. + expect(messages[0]!.content[1]).toBe(original); + }); }); diff --git a/packages/coding-agent/test/model-registry-default-config.test.ts b/packages/coding-agent/test/model-registry-default-config.test.ts index 61f686fd0..bb06df38f 100644 --- a/packages/coding-agent/test/model-registry-default-config.test.ts +++ b/packages/coding-agent/test/model-registry-default-config.test.ts @@ -33,6 +33,20 @@ describe("ModelRegistry default custom models config", () => { expect(model?.baseUrl).toBe("https://yaml-default.example.com/v1"); }); + test("retains STB decoder metadata on a renamed custom provider", () => { + writeModelsYaml("models.yml", { + provider: "managed-primary", + modelId: "local-vision", + modelName: "Local vision", + baseUrl: "http://127.0.0.1:8080/v1", + imageInputDecoder: "stb", + }); + + const model = loadDefaultRegistryModel({ provider: "managed-primary", modelId: "local-vision" }); + + expect(model?.imageInputDecoder).toBe("stb"); + }); + test("loads Bedrock cache capabilities from a model override", () => { writeBedrockCacheOverride(); @@ -112,6 +126,7 @@ interface ProviderFixture { modelId: string; modelName: string; baseUrl: string; + imageInputDecoder?: "stb"; } interface ModelLookup { @@ -124,6 +139,7 @@ interface ModelSnapshot { id: string; name: string; baseUrl: string | undefined; + imageInputDecoder?: "stb"; compat: { promptCacheMode: string; supportsLongPromptCacheRetention: boolean; @@ -134,6 +150,9 @@ interface ModelSnapshot { } function writeModelsYaml(file: "models.yml" | "models.yaml", fixture: ProviderFixture): void { + const decoderLine = fixture.imageInputDecoder + ? ` imageInputDecoder: ${fixture.imageInputDecoder}` + : undefined; fs.writeFileSync( path.join(tempDir.path(), file), [ @@ -146,7 +165,8 @@ function writeModelsYaml(file: "models.yml" | "models.yaml", fixture: ProviderFi ` - id: ${fixture.modelId}`, ` name: ${fixture.modelName}`, " reasoning: false", - " input: [text]", + fixture.imageInputDecoder ? " input: [text, image]" : " input: [text]", + ...(decoderLine ? [decoderLine] : []), " cost:", " input: 0", " output: 0", @@ -217,6 +237,7 @@ function loadDefaultRegistryModel(lookup: ModelLookup): ModelSnapshot | undefine id: model.id, name: model.name, baseUrl: model.baseUrl, + imageInputDecoder: model.imageInputDecoder, compat: model.compat, } : null)); } finally {