fix(images): preserve replay and transcript invariants

This commit is contained in:
can1357
2026-07-16 03:29:02 +02:00
parent 93cc1fed1c
commit 3a75368b92
5 changed files with 60 additions and 21 deletions
@@ -554,6 +554,13 @@ export function transformMessages<TApi extends Api>(
return [];
}
if (block.type === "image") {
// Assistant images are display artifacts. No provider accepts them
// in an assistant replay turn; the native Responses result remains
// in providerPayload for OpenAI replay.
return [];
}
if (block.type === "text") {
if (isSameModel) return block;
return {
@@ -272,3 +272,25 @@ describe("transformMessages drops malformed (empty-name) tool calls", () => {
expect(toolResults[0]?.toolName).toBe("read");
});
});
describe("transformMessages drops assistant images from provider replay", () => {
it("preserves replayable text while removing native image artifacts", () => {
const messages: Message[] = [
{ role: "user", content: "Draw a dot", timestamp: 1 },
assistant(
[
{ type: "text", text: "Here it is." },
{ type: "image", data: "aW1hZ2U=", mimeType: "image/png" },
],
2,
),
];
const transformed = transformMessages(messages, model);
expect(transformed[1]).toMatchObject({
role: "assistant",
content: [{ type: "text", text: "Here it is." }],
});
});
});
@@ -558,22 +558,12 @@ export class AssistantMessageComponent extends Container {
}
}
#renderImages(message: AssistantMessage): void {
if (!this.#showImages) return;
const nativeEntries = message.content.flatMap((content, index) =>
content.type === "image" && content.data && content.mimeType
? [{ image: content, key: `native:${index}` }]
: [],
);
const toolEntries = Array.from(this.#toolImagesByCallId.entries()).flatMap(([toolCallId, images]) =>
images.map((image, index) => ({ image, key: `${toolCallId}:${index}` })),
);
const imageEntries = [...nativeEntries, ...toolEntries];
if (imageEntries.length === 0) return;
this.#convertImagesForKitty(imageEntries);
#renderImageEntries(entries: Array<{ image: ImageContent; key: string }>, withLeadingSpacer: boolean): void {
if (!this.#showImages || entries.length === 0) return;
this.#convertImagesForKitty(entries);
this.#contentContainer.addChild(new Spacer(1));
for (const { image, key } of imageEntries) {
if (withLeadingSpacer) this.#contentContainer.addChild(new Spacer(1));
for (const { image, key } of entries) {
const displayImage =
TERMINAL.imageProtocol === ImageProtocol.Kitty && image.mimeType !== "image/png"
? this.#convertedKittyImages.get(key)
@@ -593,6 +583,13 @@ export class AssistantMessageComponent extends Container {
}
}
#renderToolImages(): void {
const entries = Array.from(this.#toolImagesByCallId.entries()).flatMap(([toolCallId, images]) =>
images.map((image, index) => ({ image, key: `${toolCallId}:${index}` })),
);
this.#renderImageEntries(entries, true);
}
#appendThinkingExtensions(contentIndex: number, thinkingIndex: number, text: string): void {
for (const renderer of this.thinkingRenderers) {
try {
@@ -785,6 +782,7 @@ export class AssistantMessageComponent extends Container {
const hasVisibleContent = message.content.some(
c =>
(c.type === "text" && canonicalizeMessage(c.text)) ||
(c.type === "image" && c.data && c.mimeType) ||
(!this.hideThinkingBlock &&
c.type === "thinking" &&
resolveThinkingDisplay(c, this.proseOnlyThinking).visible),
@@ -792,6 +790,7 @@ export class AssistantMessageComponent extends Container {
// Render content in order
let thinkingIndex = 0;
let hasRenderedContent = false;
for (let i = 0; i < message.content.length; i++) {
const content = message.content[i];
if (content.type === "text" && canonicalizeMessage(content.text)) {
@@ -801,6 +800,7 @@ export class AssistantMessageComponent extends Container {
md.transientRenderCache = this.#lastUpdateTransient;
this.#contentContainer.addChild(md);
captureItems?.push({ md, contentIndex: i, blockType: "text", lastText: trimmed });
hasRenderedContent = true;
} else if (content.type === "thinking" && resolveThinkingDisplay(content, this.proseOnlyThinking).visible) {
const thinkingText = resolveThinkingDisplay(content, this.proseOnlyThinking).text;
if (this.hideThinkingBlock) {
@@ -814,6 +814,7 @@ export class AssistantMessageComponent extends Container {
.some(
c =>
(c.type === "text" && canonicalizeMessage(c.text)) ||
(c.type === "image" && c.data && c.mimeType) ||
(c.type === "thinking" && resolveThinkingDisplay(c, this.proseOnlyThinking).visible),
);
@@ -826,10 +827,14 @@ export class AssistantMessageComponent extends Container {
this.#contentContainer.addChild(md);
captureItems?.push({ md, contentIndex: i, blockType: "thinking", lastText: thinkingText });
this.#appendThinkingExtensions(i, thinkingIndex, thinkingText);
hasRenderedContent = true;
thinkingIndex += 1;
if (hasVisibleContentAfter) {
this.#contentContainer.addChild(new Spacer(1));
}
} else if (content.type === "image" && content.data && content.mimeType) {
this.#renderImageEntries([{ image: content, key: `native:${i}` }], hasRenderedContent);
hasRenderedContent ||= this.#showImages;
}
}
@@ -842,7 +847,7 @@ export class AssistantMessageComponent extends Container {
this.#stopThinkingAnimation();
}
this.#renderImages(message);
this.#renderToolImages();
const errorPresentation = resolveAssistantErrorPresentation(message);
const hasToolCalls = message.content.some(c => c.type === "toolCall");
if (errorPresentation.kind === "compact-recovered") {
@@ -17,7 +17,7 @@ export function createAssistantMessageComponent(
message?: AssistantMessage,
): AssistantMessageComponent {
const component = new AssistantMessageComponent(
undefined,
message,
ctx.effectiveHideThinkingBlock,
() => ctx.ui.requestRender(),
ctx.viewSession.extensionRunner?.getAssistantThinkingRenderers(),
@@ -25,6 +25,5 @@ export function createAssistantMessageComponent(
ctx.proseOnlyThinking,
);
component.setImagesVisible(ctx.settings.get("terminal.showImages"));
if (message) component.updateContent(message);
return component;
}
@@ -260,14 +260,20 @@ describe("AssistantMessageComponent thinking renderers", () => {
});
describe("AssistantMessageComponent images", () => {
it("renders native assistant images and honors image visibility", () => {
it("renders native assistant images in content order and honors image visibility", () => {
const message: AssistantMessage = {
...createAssistantMessage(""),
content: [{ type: "image", data: "aW1hZ2U=", mimeType: "image/png" }],
content: [
{ type: "text", text: "Before image" },
{ type: "image", data: "aW1hZ2U=", mimeType: "image/png" },
{ type: "text", text: "After image" },
],
};
const component = new AssistantMessageComponent(message);
expect(Bun.stripANSI(component.render(80).join("\n"))).toContain("[Image: image/png]");
const rendered = Bun.stripANSI(component.render(80).join("\n"));
expect(rendered.indexOf("Before image")).toBeLessThan(rendered.indexOf("[Image: image/png]"));
expect(rendered.indexOf("[Image: image/png]")).toBeLessThan(rendered.indexOf("After image"));
component.setImagesVisible(false);
expect(Bun.stripANSI(component.render(80).join("\n"))).not.toContain("[Image: image/png]");
});