smashimo/oh-my-pi
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(coding-agent): normalize WebP for custom STB models

Browse Source
This commit is contained in:
Ethan Cawse
2026-08-11 22:11:37 -04:00
parent 06aecdd51f
commit 2cd25fb4ee
9 changed files with 152 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/coding-agent/CHANGELOG.md
+3
View File
@@ -25,6 +25,9 @@
- Fixed `/handoff` losing local artifacts (plans, scratch files, research notes) by copying them across the handoff session boundary.
- Replaced libarchive-based tar parsing with a hardened, in-process tar reader to prevent crashes and safely handle complex archive structures, symlinks, and sparse metadata.
- Fixed `Ctrl+O` tool-output expansion failing to reach launch-completion messages wrapped in the hidden tool activity container.
### Fixed
- Fixed custom STB-backed vision providers retaining decoder metadata and re-encoding resumed WebP images before provider requests.
## [17.2.14] - 2026-08-11
packages/coding-agent/src/config/custom-models.ts
+2
View File
@@ -85,6 +85,7 @@ export function buildCustomModelOverlay(
reasoning: modelDef.reasoning,
thinking: modelDef.thinking,
input: modelDef.input,
imageInputDecoder: modelDef.imageInputDecoder,
supportsTools: modelDef.supportsTools,
cost: modelDef.cost,
contextWindow: modelDef.contextWindow,
@@ -127,6 +128,7 @@ export function finalizeCustomModel(model: CustomModelOverlay, options: CustomMo
reasoning: resolvedModel.reasoning ?? reference?.reasoning ?? (options.useDefaults ? false : undefined),
thinking: inheritReferenceThinking(resolvedModel.thinking, reference, resolvedModel.provider),
input: input as ("text" | "image")[],
imageInputDecoder: resolvedModel.imageInputDecoder,
...(supportsTools !== undefined ? { supportsTools } : {}),
cost,
contextWindow: resolvedModel.contextWindow ?? reference?.contextWindow ?? (options.useDefaults ? 128000 : null),
packages/coding-agent/src/config/model-patch.ts
+2
View File
@@ -183,6 +183,7 @@ export interface ModelPatch {
reasoning?: boolean;
thinking?: ThinkingConfig;
input?: ("text" | "image")[];
imageInputDecoder?: Model<Api>["imageInputDecoder"];
supportsTools?: boolean;
cost?: Partial<Model<Api>["cost"]>;
contextWindow?: number;
@@ -210,6 +211,7 @@ export function applyModelPatch(base: Model<Api>, patch: ModelPatch, transport:
if (patch.reasoning !== undefined) result.reasoning = patch.reasoning;
if (patch.thinking !== undefined) result.thinking = patch.thinking;
if (patch.input !== undefined) result.input = patch.input;
if (patch.imageInputDecoder !== undefined) result.imageInputDecoder = patch.imageInputDecoder;
if (patch.supportsTools !== undefined) result.supportsTools = patch.supportsTools;
if (patch.contextWindow !== undefined) result.contextWindow = patch.contextWindow;
if (patch.maxTokens !== undefined) result.maxTokens = patch.maxTokens;
packages/coding-agent/src/config/models-config-schema-bundle.ts
+2
View File
@@ -167,6 +167,7 @@ export const getModelsConfigSchemaBundle = once(() => {
"reasoning?": "boolean",
"thinking?": ModelThinkingSchema,
"input?": '("text" | "image")[]',
"imageInputDecoder?": '"stb"',
"supportsTools?": "boolean",
"cost?": {
input: "number",
@@ -216,6 +217,7 @@ export const getModelsConfigSchemaBundle = once(() => {
"reasoning?": "boolean",
"thinking?": ModelThinkingSchema,
"input?": '("text" | "image")[]',
"imageInputDecoder?": '"stb"',
"supportsTools?": "boolean",
"cost?": {
"input?": "number",
packages/coding-agent/src/session/session-provider-boundary.ts
+3 -2
View File
@@ -13,7 +13,7 @@ import type { LocalProtocolOptions } from "../internal-urls";
import { deobfuscateSessionContext, obfuscateMessages } from "../secrets/message-transform";
import type { SecretObfuscator } from "../secrets/obfuscator";
import { stripPendingSecretPlaceholderSuffix } from "../secrets/placeholder";
import { normalizeModelContextImages } from "../utils/image-loading";
import { normalizeModelContextImages, normalizeModelContextMessages } from "../utils/image-loading";
import { describeAttachedImagesForTextModel } from "../utils/image-vision-fallback";
import { type CustomMessage, convertToLlm } from "./messages";
import { IMAGE_ATTACHMENT_DESCRIPTION_TYPE } from "./queued-messages";
@@ -126,7 +126,8 @@ export class SessionProviderBoundary {
/** Converts session messages using the configured pre-LLM pipeline. */
async convertMessagesToLlm(messages: AgentMessage[], signal?: AbortSignal): Promise<Message[]> {
const transformedMessages = await this.#host.transformContext(messages, signal);
return await this.#host.convertToLlm(transformedMessages);
const converted = await this.#host.convertToLlm(transformedMessages);
return await normalizeModelContextMessages(converted, this.#host.model());
}
/** Applies session-level stream hooks and provider defaults to a side request. */
packages/coding-agent/src/utils/image-loading.ts
+79 -4
View File
@@ -1,11 +1,50 @@
import * as fs from "node:fs/promises";
import type { ImageContent, Model } from "@oh-my-pi/pi-ai";
import type { ImageContent, Message, Model } from "@oh-my-pi/pi-ai";
import { formatBytes, readImageMetadata, SUPPORTED_IMAGE_MIME_TYPES } from "@oh-my-pi/pi-utils";
import { LRUCache } from "@oh-my-pi/pi-utils/lru";
import { resolveReadPath } from "../tools/path-utils";
import { formatDimensionNote, type ImageResizeOptions, resizeImage } from "./image-resize";
export const MAX_IMAGE_INPUT_BYTES = 20 * 1024 * 1024;
export const SUPPORTED_INPUT_IMAGE_MIME_TYPES = SUPPORTED_IMAGE_MIME_TYPES;
const MODEL_BOUNDARY_IMAGE_CACHE_LIMIT = 128;
type NormalizedImagePayload = Pick<ImageContent, "data" | "mimeType">;
const modelBoundaryImageCache = new LRUCache<string, Promise<NormalizedImagePayload>>({
max: MODEL_BOUNDARY_IMAGE_CACHE_LIMIT,
});
function hasWebPMagic(data: string): boolean {
const header = Buffer.from(data.slice(0, 16), "base64");
return (
header.length >= 12 && header.toString("ascii", 0, 4) === "RIFF" && header.toString("ascii", 8, 12) === "WEBP"
);
}
function isWebPImage(image: ImageContent): boolean {
return image.mimeType.toLowerCase() === "image/webp" || hasWebPMagic(image.data);
}
function modelBoundaryImageCacheKey(image: ImageContent): string {
return `${image.data.length}:${image.data.slice(0, 32)}:${image.data.slice(-32)}:${String(Bun.hash(image.data))}`;
}
async function memoizedStbImageNormalization(image: ImageContent): Promise<ImageContent> {
const key = modelBoundaryImageCacheKey(image);
let pending = modelBoundaryImageCache.get(key);
if (!pending) {
pending = resizeImage(image, { excludeWebP: true }).then(resized => {
if (resized.mimeType === "image/webp" || hasWebPMagic(resized.data)) {
throw new Error("Image normalization retained WebP for an STB-backed model");
}
return { data: resized.data, mimeType: resized.mimeType };
});
modelBoundaryImageCache.set(key, pending);
void pending.catch(() => {
if (modelBoundaryImageCache.peek(key) === pending) modelBoundaryImageCache.delete(key);
});
}
return { ...image, ...(await pending) };
}
/**
* Ollama and its local-backend family decode image input through llama.cpp /
@@ -116,15 +155,23 @@ export async function normalizeModelContextImages(
options?: NormalizeModelContextImagesOptions,
): Promise<ImageContent[] | undefined> {
if (!images || images.length === 0) return undefined;
const resize: ImageResizeOptions | undefined = modelLacksWebpSupport(options?.model)
const excludesWebP = modelLacksWebpSupport(options?.model);
const resize: ImageResizeOptions | undefined = excludesWebP
? { ...options?.resize, excludeWebP: true }
: options?.resize;
const normalized: ImageContent[] = [];
for (const image of images) {
try {
if (excludesWebP && isWebPImage(image)) {
normalized.push(await memoizedStbImageNormalization(image));
continue;
}
const resized = await resizeImage(image, resize);
normalized.push({ type: "image", data: resized.data, mimeType: resized.mimeType });
} catch {
normalized.push({ ...image, data: resized.data, mimeType: resized.mimeType });
} catch (error) {
if (excludesWebP && isWebPImage(image)) {
throw new Error("Failed to convert WebP image for an STB-backed model", { cause: error });
}
// Preserve existing caller behavior for decode/resize failures: keep the
// user's image block rather than dropping it from the turn.
normalized.push(image);
@@ -133,6 +180,34 @@ export async function normalizeModelContextImages(
return normalized;
}
/**
* Rewrites historical/resumed WebP blocks in the ephemeral provider request.
* Persisted session messages remain untouched, while STB-backed local servers
* never receive a format they cannot decode.
*/
export async function normalizeModelContextMessages(messages: Message[], model: Model | undefined): Promise<Message[]> {
if (!modelLacksWebpSupport(model)) return messages;
let output: Message[] | undefined;
for (let messageIndex = 0; messageIndex < messages.length; messageIndex++) {
const message = messages[messageIndex]!;
if (typeof message.content === "string") continue;
const webpImages: ImageContent[] = [];
for (const part of message.content) {
if (part.type === "image" && isWebPImage(part)) webpImages.push(part);
}
if (webpImages.length === 0) continue;
const normalized = await normalizeModelContextImages(webpImages, { model });
if (!normalized) continue;
let imageIndex = 0;
const content = message.content.map(part =>
part.type === "image" && isWebPImage(part) ? normalized[imageIndex++]! : part,
);
output ??= messages.slice();
output[messageIndex] = { ...message, content } as Message;
}
return output ?? messages;
}
export async function loadImageInput(options: LoadImageInputOptions): Promise<LoadedImageInput | null> {
const maxBytes = options.maxBytes ?? MAX_IMAGE_INPUT_BYTES;
const resolvedPath = options.resolvedPath ?? resolveReadPath(options.path, options.cwd);
packages/coding-agent/src/utils/image-resize.ts
+3 -1
View File
@@ -167,7 +167,9 @@ export async function resizeImage(img: ImageContent, options?: ImageResizeOption
try {
const { width: originalWidth, height: originalHeight, format } = await new Bun.Image(inputBuffer).metadata();
const sourceMime = img.mimeType ?? `image/${format}`;
// Trust decoded bytes over caller metadata. A mislabeled WebP must not take
// the fast path when the target decoder explicitly excludes WebP.
const sourceMime = format ? `image/${format}` : img.mimeType;
// Fast path: already within dimensions AND well under budget.
// Threshold is 1/4 of budget — if already that compact, don't re-encode.
packages/coding-agent/test/image-webp-exclusion.test.ts
+36
View File
@@ -5,6 +5,7 @@ import { getBundledModels } from "@oh-my-pi/pi-catalog/models";
import {
modelLacksWebpSupport,
normalizeModelContextImages,
normalizeModelContextMessages,
webpExclusionForModel,
} from "@oh-my-pi/pi-coding-agent/utils/image-loading";
@@ -145,4 +146,39 @@ describe("normalizeModelContextImages model-aware WebP exclusion", () => {
expect(result?.[0]?.mimeType).toBe("image/webp");
});
test("rewrites resumed tool-result WebP blocks at the STB provider boundary", async () => {
const original = {
type: "image" as const,
data: await makeRedWebP(200, 200),
// Exercise byte sniffing as well as declared-MIME handling.
mimeType: "image/png",
detail: "original" as const,
};
const messages = [
{
role: "toolResult" as const,
toolCallId: "read-1",
toolName: "read",
content: [{ type: "text" as const, text: "screenshot" }, original],
isError: false,
timestamp: 1,
},
];
const result = await normalizeModelContextMessages(messages, buildStbVisionModel("managed-primary"));
const resultMessage = result[0]!;
expect(resultMessage.role).toBe("toolResult");
if (resultMessage.role !== "toolResult") throw new Error("Expected tool result message");
const image = resultMessage.content[1]!;
expect(image.type).toBe("image");
if (image.type !== "image") throw new Error("Expected normalized image block");
expect(image.mimeType).not.toBe("image/webp");
expect(["image/png", "image/jpeg"]).toContain(image.mimeType);
expect(Buffer.from(image.data.slice(0, 16), "base64").toString("ascii", 8, 12)).not.toBe("WEBP");
expect(image.detail).toBe("original");
// Provider-boundary normalization is ephemeral; persisted history is not mutated.
expect(messages[0]!.content[1]).toBe(original);
});
});
packages/coding-agent/test/model-registry-default-config.test.ts
+22 -1
View File
@@ -33,6 +33,20 @@ describe("ModelRegistry default custom models config", () => {
expect(model?.baseUrl).toBe("https://yaml-default.example.com/v1");
});
test("retains STB decoder metadata on a renamed custom provider", () => {
writeModelsYaml("models.yml", {
provider: "managed-primary",
modelId: "local-vision",
modelName: "Local vision",
baseUrl: "http://127.0.0.1:8080/v1",
imageInputDecoder: "stb",
});
const model = loadDefaultRegistryModel({ provider: "managed-primary", modelId: "local-vision" });
expect(model?.imageInputDecoder).toBe("stb");
});
test("loads Bedrock cache capabilities from a model override", () => {
writeBedrockCacheOverride();
@@ -112,6 +126,7 @@ interface ProviderFixture {
modelId: string;
modelName: string;
baseUrl: string;
imageInputDecoder?: "stb";
}
interface ModelLookup {
@@ -124,6 +139,7 @@ interface ModelSnapshot {
id: string;
name: string;
baseUrl: string | undefined;
imageInputDecoder?: "stb";
compat: {
promptCacheMode: string;
supportsLongPromptCacheRetention: boolean;
@@ -134,6 +150,9 @@ interface ModelSnapshot {
}
function writeModelsYaml(file: "models.yml" | "models.yaml", fixture: ProviderFixture): void {
const decoderLine = fixture.imageInputDecoder
? ` imageInputDecoder: ${fixture.imageInputDecoder}`
: undefined;
fs.writeFileSync(
path.join(tempDir.path(), file),
[
@@ -146,7 +165,8 @@ function writeModelsYaml(file: "models.yml" | "models.yaml", fixture: ProviderFi
` - id: ${fixture.modelId}`,
` name: ${fixture.modelName}`,
" reasoning: false",
" input: [text]",
fixture.imageInputDecoder ? " input: [text, image]" : " input: [text]",
...(decoderLine ? [decoderLine] : []),
" cost:",
" input: 0",
" output: 0",
@@ -217,6 +237,7 @@ function loadDefaultRegistryModel(lookup: ModelLookup): ModelSnapshot | undefine
id: model.id,
name: model.name,
baseUrl: model.baseUrl,
imageInputDecoder: model.imageInputDecoder,
compat: model.compat,
} : null));
} finally {