fix(images): preserve replay and transcript invariants
This commit is contained in:
@@ -554,6 +554,13 @@ export function transformMessages<TApi extends Api>(
|
||||
return [];
|
||||
}
|
||||
|
||||
if (block.type === "image") {
|
||||
// Assistant images are display artifacts. No provider accepts them
|
||||
// in an assistant replay turn; the native Responses result remains
|
||||
// in providerPayload for OpenAI replay.
|
||||
return [];
|
||||
}
|
||||
|
||||
if (block.type === "text") {
|
||||
if (isSameModel) return block;
|
||||
return {
|
||||
|
||||
@@ -272,3 +272,25 @@ describe("transformMessages drops malformed (empty-name) tool calls", () => {
|
||||
expect(toolResults[0]?.toolName).toBe("read");
|
||||
});
|
||||
});
|
||||
|
||||
describe("transformMessages drops assistant images from provider replay", () => {
|
||||
it("preserves replayable text while removing native image artifacts", () => {
|
||||
const messages: Message[] = [
|
||||
{ role: "user", content: "Draw a dot", timestamp: 1 },
|
||||
assistant(
|
||||
[
|
||||
{ type: "text", text: "Here it is." },
|
||||
{ type: "image", data: "aW1hZ2U=", mimeType: "image/png" },
|
||||
],
|
||||
2,
|
||||
),
|
||||
];
|
||||
|
||||
const transformed = transformMessages(messages, model);
|
||||
|
||||
expect(transformed[1]).toMatchObject({
|
||||
role: "assistant",
|
||||
content: [{ type: "text", text: "Here it is." }],
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -558,22 +558,12 @@ export class AssistantMessageComponent extends Container {
|
||||
}
|
||||
}
|
||||
|
||||
#renderImages(message: AssistantMessage): void {
|
||||
if (!this.#showImages) return;
|
||||
const nativeEntries = message.content.flatMap((content, index) =>
|
||||
content.type === "image" && content.data && content.mimeType
|
||||
? [{ image: content, key: `native:${index}` }]
|
||||
: [],
|
||||
);
|
||||
const toolEntries = Array.from(this.#toolImagesByCallId.entries()).flatMap(([toolCallId, images]) =>
|
||||
images.map((image, index) => ({ image, key: `${toolCallId}:${index}` })),
|
||||
);
|
||||
const imageEntries = [...nativeEntries, ...toolEntries];
|
||||
if (imageEntries.length === 0) return;
|
||||
this.#convertImagesForKitty(imageEntries);
|
||||
#renderImageEntries(entries: Array<{ image: ImageContent; key: string }>, withLeadingSpacer: boolean): void {
|
||||
if (!this.#showImages || entries.length === 0) return;
|
||||
this.#convertImagesForKitty(entries);
|
||||
|
||||
this.#contentContainer.addChild(new Spacer(1));
|
||||
for (const { image, key } of imageEntries) {
|
||||
if (withLeadingSpacer) this.#contentContainer.addChild(new Spacer(1));
|
||||
for (const { image, key } of entries) {
|
||||
const displayImage =
|
||||
TERMINAL.imageProtocol === ImageProtocol.Kitty && image.mimeType !== "image/png"
|
||||
? this.#convertedKittyImages.get(key)
|
||||
@@ -593,6 +583,13 @@ export class AssistantMessageComponent extends Container {
|
||||
}
|
||||
}
|
||||
|
||||
#renderToolImages(): void {
|
||||
const entries = Array.from(this.#toolImagesByCallId.entries()).flatMap(([toolCallId, images]) =>
|
||||
images.map((image, index) => ({ image, key: `${toolCallId}:${index}` })),
|
||||
);
|
||||
this.#renderImageEntries(entries, true);
|
||||
}
|
||||
|
||||
#appendThinkingExtensions(contentIndex: number, thinkingIndex: number, text: string): void {
|
||||
for (const renderer of this.thinkingRenderers) {
|
||||
try {
|
||||
@@ -785,6 +782,7 @@ export class AssistantMessageComponent extends Container {
|
||||
const hasVisibleContent = message.content.some(
|
||||
c =>
|
||||
(c.type === "text" && canonicalizeMessage(c.text)) ||
|
||||
(c.type === "image" && c.data && c.mimeType) ||
|
||||
(!this.hideThinkingBlock &&
|
||||
c.type === "thinking" &&
|
||||
resolveThinkingDisplay(c, this.proseOnlyThinking).visible),
|
||||
@@ -792,6 +790,7 @@ export class AssistantMessageComponent extends Container {
|
||||
|
||||
// Render content in order
|
||||
let thinkingIndex = 0;
|
||||
let hasRenderedContent = false;
|
||||
for (let i = 0; i < message.content.length; i++) {
|
||||
const content = message.content[i];
|
||||
if (content.type === "text" && canonicalizeMessage(content.text)) {
|
||||
@@ -801,6 +800,7 @@ export class AssistantMessageComponent extends Container {
|
||||
md.transientRenderCache = this.#lastUpdateTransient;
|
||||
this.#contentContainer.addChild(md);
|
||||
captureItems?.push({ md, contentIndex: i, blockType: "text", lastText: trimmed });
|
||||
hasRenderedContent = true;
|
||||
} else if (content.type === "thinking" && resolveThinkingDisplay(content, this.proseOnlyThinking).visible) {
|
||||
const thinkingText = resolveThinkingDisplay(content, this.proseOnlyThinking).text;
|
||||
if (this.hideThinkingBlock) {
|
||||
@@ -814,6 +814,7 @@ export class AssistantMessageComponent extends Container {
|
||||
.some(
|
||||
c =>
|
||||
(c.type === "text" && canonicalizeMessage(c.text)) ||
|
||||
(c.type === "image" && c.data && c.mimeType) ||
|
||||
(c.type === "thinking" && resolveThinkingDisplay(c, this.proseOnlyThinking).visible),
|
||||
);
|
||||
|
||||
@@ -826,10 +827,14 @@ export class AssistantMessageComponent extends Container {
|
||||
this.#contentContainer.addChild(md);
|
||||
captureItems?.push({ md, contentIndex: i, blockType: "thinking", lastText: thinkingText });
|
||||
this.#appendThinkingExtensions(i, thinkingIndex, thinkingText);
|
||||
hasRenderedContent = true;
|
||||
thinkingIndex += 1;
|
||||
if (hasVisibleContentAfter) {
|
||||
this.#contentContainer.addChild(new Spacer(1));
|
||||
}
|
||||
} else if (content.type === "image" && content.data && content.mimeType) {
|
||||
this.#renderImageEntries([{ image: content, key: `native:${i}` }], hasRenderedContent);
|
||||
hasRenderedContent ||= this.#showImages;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -842,7 +847,7 @@ export class AssistantMessageComponent extends Container {
|
||||
this.#stopThinkingAnimation();
|
||||
}
|
||||
|
||||
this.#renderImages(message);
|
||||
this.#renderToolImages();
|
||||
const errorPresentation = resolveAssistantErrorPresentation(message);
|
||||
const hasToolCalls = message.content.some(c => c.type === "toolCall");
|
||||
if (errorPresentation.kind === "compact-recovered") {
|
||||
|
||||
@@ -17,7 +17,7 @@ export function createAssistantMessageComponent(
|
||||
message?: AssistantMessage,
|
||||
): AssistantMessageComponent {
|
||||
const component = new AssistantMessageComponent(
|
||||
undefined,
|
||||
message,
|
||||
ctx.effectiveHideThinkingBlock,
|
||||
() => ctx.ui.requestRender(),
|
||||
ctx.viewSession.extensionRunner?.getAssistantThinkingRenderers(),
|
||||
@@ -25,6 +25,5 @@ export function createAssistantMessageComponent(
|
||||
ctx.proseOnlyThinking,
|
||||
);
|
||||
component.setImagesVisible(ctx.settings.get("terminal.showImages"));
|
||||
if (message) component.updateContent(message);
|
||||
return component;
|
||||
}
|
||||
|
||||
@@ -260,14 +260,20 @@ describe("AssistantMessageComponent thinking renderers", () => {
|
||||
});
|
||||
|
||||
describe("AssistantMessageComponent images", () => {
|
||||
it("renders native assistant images and honors image visibility", () => {
|
||||
it("renders native assistant images in content order and honors image visibility", () => {
|
||||
const message: AssistantMessage = {
|
||||
...createAssistantMessage(""),
|
||||
content: [{ type: "image", data: "aW1hZ2U=", mimeType: "image/png" }],
|
||||
content: [
|
||||
{ type: "text", text: "Before image" },
|
||||
{ type: "image", data: "aW1hZ2U=", mimeType: "image/png" },
|
||||
{ type: "text", text: "After image" },
|
||||
],
|
||||
};
|
||||
const component = new AssistantMessageComponent(message);
|
||||
|
||||
expect(Bun.stripANSI(component.render(80).join("\n"))).toContain("[Image: image/png]");
|
||||
const rendered = Bun.stripANSI(component.render(80).join("\n"));
|
||||
expect(rendered.indexOf("Before image")).toBeLessThan(rendered.indexOf("[Image: image/png]"));
|
||||
expect(rendered.indexOf("[Image: image/png]")).toBeLessThan(rendered.indexOf("After image"));
|
||||
component.setImagesVisible(false);
|
||||
expect(Bun.stripANSI(component.render(80).join("\n"))).not.toContain("[Image: image/png]");
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user