diff --git a/packages/agent/src/agent-loop.ts b/packages/agent/src/agent-loop.ts index ded2e4afe..8550d0be3 100644 --- a/packages/agent/src/agent-loop.ts +++ b/packages/agent/src/agent-loop.ts @@ -183,6 +183,7 @@ type AssistantToolCallBlock = Extract( } closeOpenItem(event.output_index, item.id, entry, item.call_id, prefixedFunctionCallItemKey(item.call_id)); stream.push({ type: "toolcall_end", contentIndex, toolCall, partial: output }); + } else if (item.type === "image_generation_call" && item.status === "completed" && item.result) { + const image: ImageContent = { + type: "image", + data: item.result, + mimeType: parseImageMetadata(Buffer.from(item.result, "base64"))?.mimeType ?? "image/png", + }; + output.content.push(image); + stream.push({ + type: "image_end", + contentIndex: output.content.length - 1, + content: image, + partial: output, + }); } } else if (terminalEvent) { const response = terminalEvent.response; diff --git a/packages/ai/src/types.ts b/packages/ai/src/types.ts index d5903e5dd..fd0a140b9 100644 --- a/packages/ai/src/types.ts +++ b/packages/ai/src/types.ts @@ -706,7 +706,14 @@ export interface ContextSnapshot { export interface AssistantMessage { role: "assistant"; - content: (TextContent | ThinkingContent | RedactedThinkingContent | AnthropicFallbackContent | ToolCall)[]; + content: ( + | TextContent + | ThinkingContent + | RedactedThinkingContent + | AnthropicFallbackContent + | ImageContent + | ToolCall + )[]; api: Api; provider: Provider; model: string; @@ -893,6 +900,7 @@ export type AssistantMessageEvent = | { type: "thinking_start"; contentIndex: number; partial: AssistantMessage } | { type: "thinking_delta"; contentIndex: number; delta: string; partial: AssistantMessage } | { type: "thinking_end"; contentIndex: number; content: string; partial: AssistantMessage } + | { type: "image_end"; contentIndex: number; content: ImageContent; partial: AssistantMessage } | { type: "toolcall_start"; contentIndex: number; partial: AssistantMessage } | { type: "toolcall_delta"; contentIndex: number; delta: string; partial: AssistantMessage } | { type: "toolcall_end"; contentIndex: number; toolCall: ToolCall; partial: AssistantMessage } diff --git a/packages/ai/src/utils/empty-completion-retry.ts b/packages/ai/src/utils/empty-completion-retry.ts index 6ace44c1f..3af9337e5 100644 --- a/packages/ai/src/utils/empty-completion-retry.ts +++ b/packages/ai/src/utils/empty-completion-retry.ts @@ -27,12 +27,13 @@ export const EMPTY_COMPLETION_BASE_DELAY_MS = 500; const NON_WHITESPACE_RE = /\S/; /** - * Whether a completed assistant message carries content worth delivering: a tool - * call or any non-whitespace text. An empty/whitespace-only message — or one - * that only ever produced thinking — is the "empty response" failure. + * Whether a completed assistant message carries content worth delivering: an + * image, tool call, or any non-whitespace text. An empty/whitespace-only message + * — or one that only ever produced thinking — is the "empty response" failure. */ export function hasVisibleAssistantContent(message: AssistantMessage): boolean { for (const block of message.content) { + if (block.type === "image") return true; if (block.type === "toolCall") return true; if (block.type === "text" && NON_WHITESPACE_RE.test(block.text)) return true; } @@ -49,6 +50,8 @@ function isMeaningfulCompletionEvent(event: AssistantMessageEvent): boolean { case "text_end": case "thinking_end": return event.content.length > 0; + case "image_end": + return true; case "toolcall_start": case "toolcall_end": return true; diff --git a/packages/ai/src/utils/leaked-thinking-stream.ts b/packages/ai/src/utils/leaked-thinking-stream.ts index 074a4c507..6acd5e961 100644 --- a/packages/ai/src/utils/leaked-thinking-stream.ts +++ b/packages/ai/src/utils/leaked-thinking-stream.ts @@ -25,7 +25,7 @@ * events are forwarded verbatim. */ -import type { AssistantMessage, TextContent, ThinkingContent, ToolCall } from "../types"; +import type { AssistantMessage, ImageContent, TextContent, ThinkingContent, ToolCall } from "../types"; import { clearStreamingPartialJson, getStreamingPartialJson, @@ -78,6 +78,10 @@ export function wrapLeakedThinkingStream(inner: AssistantMessageEventStream): As projector.thinking(event.delta, block?.type === "thinking" ? block.thinkingSignature : undefined); break; } + case "image_end": + projector ??= new LeakedThinkingProjector(out, event.partial); + projector.image(event.content); + break; case "toolcall_start": { projector ??= new LeakedThinkingProjector(out, event.partial); const block = event.partial.content[event.contentIndex]; @@ -163,6 +167,20 @@ class LeakedThinkingProjector { this.#out.push({ type: "thinking_delta", contentIndex: index, delta, partial: this.#partial }); } + /** Forward a completed native image after releasing held text. */ + image(content: ImageContent): void { + this.#apply(this.#healer.flushEvents(), this.#lastTextSignature); + this.#closeText(); + this.#closeThinking(); + this.#partial.content.push(content); + this.#out.push({ + type: "image_end", + contentIndex: this.#partial.content.length - 1, + content, + partial: this.#partial, + }); + } + /** Forward a native tool call's start, releasing any held-back text first. */ toolStart(srcIndex: number, source: StreamingToolCall | undefined): void { if (!source) return; diff --git a/packages/ai/test/openai-responses-stream-terminal.test.ts b/packages/ai/test/openai-responses-stream-terminal.test.ts index 4272195a6..d7f80ffe5 100644 --- a/packages/ai/test/openai-responses-stream-terminal.test.ts +++ b/packages/ai/test/openai-responses-stream-terminal.test.ts @@ -400,6 +400,41 @@ describe("processResponsesStream: lost output_item.added recovery", () => { expect(end?.content).toBe("Recovered text"); }); + test("normalizes a completed native image generation call into visible assistant content", async () => { + const output = makeOutput(); + const emitted: EmittedEvent[] = []; + const stream = { push: (event: unknown) => emitted.push(event as EmittedEvent), end: () => {} } as never; + const data = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII="; + + await processResponsesStream( + makeStream([ + { + type: "response.output_item.done", + output_index: 0, + item: { + type: "image_generation_call", + id: "ig_1", + status: "completed", + result: data, + }, + }, + { type: "response.completed", response: { id: "resp_image", status: "completed" } }, + ]), + output, + stream, + makeModel(), + ); + + expect(output.content).toEqual([{ type: "image", data, mimeType: "image/png" }]); + const end = emitted.find(event => event.type === "image_end"); + expect(end).toEqual({ + type: "image_end", + contentIndex: 0, + content: { type: "image", data, mimeType: "image/png" }, + partial: output, + }); + }); + test("routes reasoning finalization by output_index when item ids are absent", async () => { const output = makeOutput(); const stream = { push: () => {}, end: () => {} } as never; diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 1501c9d5d..acc4c9d5d 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -517,6 +517,7 @@ - Fixed the `nerd` status-line preset's session icon using a removed Nerd Fonts v2 codepoint instead of the current Nerd Fonts v3 mapping ([#4795](https://github.com/can1357/oh-my-pi/issues/4795)). - Fixed omp crashing at startup (`TypeError: undefined is not an object (evaluating 'this.#theme.symbols.boxRound')`) after installing a plugin whose custom editor subclasses `CustomEditor`/`Editor` and forwards the upstream-pi `super(tui, theme, keybindings)` constructor — the arg order that `setEditorComponent`'s factory contract advertises. `CustomEditor` now resolves the real `EditorTheme` by shape rather than position and captures a leading `TUI` for plugin overrides ([#4766](https://github.com/can1357/oh-my-pi/issues/4766)). - Fixed `Other` response editors leaving Windows Terminal IME candidate windows at the terminal edge by forwarding dialog focus to the nested editor ([#4760](https://github.com/can1357/oh-my-pi/issues/4760)). +- Rendered and persisted native OpenAI Responses `image_generation_call` results as session images ([#4768](https://github.com/can1357/oh-my-pi/issues/4768)). ## [16.3.11] - 2026-07-06 diff --git a/packages/coding-agent/src/cli/bench-cli.ts b/packages/coding-agent/src/cli/bench-cli.ts index f324aad76..74c86f52e 100644 --- a/packages/coding-agent/src/cli/bench-cli.ts +++ b/packages/coding-agent/src/cli/bench-cli.ts @@ -159,12 +159,14 @@ function isFirstTokenEvent(event: AssistantMessageEvent): boolean { case "text_end": case "thinking_end": return event.content.length > 0; + case "image_end": + return true; default: return false; } } -/** Final message carries visible output — non-empty text/thinking or a tool call. */ +/** Final message carries visible output — non-empty text/thinking, an image, or a tool call. */ function hasVisibleFinalContent(message: AssistantMessage): boolean { return message.content.some(block => { switch (block.type) { @@ -172,6 +174,7 @@ function hasVisibleFinalContent(message: AssistantMessage): boolean { return block.text.length > 0; case "thinking": return block.thinking.length > 0; + case "image": case "redactedThinking": case "toolCall": return true; diff --git a/packages/coding-agent/src/export/html/template.js b/packages/coding-agent/src/export/html/template.js index 5fe4d9be2..77319294f 100644 --- a/packages/coding-agent/src/export/html/template.js +++ b/packages/coding-agent/src/export/html/template.js @@ -1082,6 +1082,8 @@
${escapeHtml(thinking)}
Thinking ...
`; + } else if (block.type === 'image') { + html += `
`; } } for (const block of msg.content) { diff --git a/packages/coding-agent/src/modes/acp/acp-agent.ts b/packages/coding-agent/src/modes/acp/acp-agent.ts index 970c2edf6..4c8aabdc1 100644 --- a/packages/coding-agent/src/modes/acp/acp-agent.ts +++ b/packages/coding-agent/src/modes/acp/acp-agent.ts @@ -2110,6 +2110,23 @@ export class AcpAgent implements Agent { }); continue; } + if ( + item.type === "image" && + "data" in item && + typeof item.data === "string" && + "mimeType" in item && + typeof item.mimeType === "string" + ) { + notifications.push({ + sessionId, + update: { + sessionUpdate: "agent_message_chunk", + content: { type: "image", data: item.data, mimeType: item.mimeType }, + messageId, + }, + }); + continue; + } if (item.type === "thinking" && "thinking" in item && typeof item.thinking === "string") { const thinking = canonicalizeMessage(item.thinking); if (thinking.length === 0) continue; diff --git a/packages/coding-agent/src/modes/acp/acp-event-mapper.ts b/packages/coding-agent/src/modes/acp/acp-event-mapper.ts index fac1807f0..553fbcde0 100644 --- a/packages/coding-agent/src/modes/acp/acp-event-mapper.ts +++ b/packages/coding-agent/src/modes/acp/acp-event-mapper.ts @@ -272,6 +272,14 @@ function mapAssistantMessageUpdate( let text: string; const progress = options.getMessageProgress?.(event.message); switch (event.assistantMessageEvent.type) { + case "image_end": + return [ + toSessionNotification(sessionId, { + sessionUpdate: "agent_message_chunk", + content: event.assistantMessageEvent.content, + messageId: options.getMessageId?.(event.message), + }), + ]; case "text_delta": sessionUpdate = "agent_message_chunk"; text = event.assistantMessageEvent.delta; diff --git a/packages/coding-agent/src/modes/components/assistant-message.ts b/packages/coding-agent/src/modes/components/assistant-message.ts index e09fc8c15..75391dceb 100644 --- a/packages/coding-agent/src/modes/components/assistant-message.ts +++ b/packages/coding-agent/src/modes/components/assistant-message.ts @@ -173,6 +173,7 @@ export class AssistantMessageComponent extends Container { #lastMessage?: AssistantMessage; #toolImagesByCallId = new Map(); #convertedKittyImages = new Map(); + #showImages = true; #kittyConversionsInFlight = new Set(); #transcriptBlockFinalized: boolean; /** @@ -497,6 +498,15 @@ export class AssistantMessageComponent extends Container { } } + /** Toggle rendering for assistant-native and tool-result images. */ + setImagesVisible(visible: boolean): void { + if (this.#showImages === visible) return; + this.#showImages = visible; + if (this.#lastMessage) { + this.updateContent(this.#lastMessage, { transient: this.#lastUpdateTransient }); + } + } + setToolResultImages(toolCallId: string, images: ImageContent[]): void { if (!toolCallId) return; const validImages = images.filter(img => img.type === "image" && img.data && img.mimeType); @@ -514,19 +524,17 @@ export class AssistantMessageComponent extends Container { this.#toolImagesByCallId.delete(toolCallId); } else { this.#toolImagesByCallId.set(toolCallId, validImages); - this.#convertToolImagesForKitty(toolCallId, validImages); + this.#convertImagesForKitty(validImages.map((image, index) => ({ image, key: `${toolCallId}:${index}` }))); } if (this.#lastMessage) { this.updateContent(this.#lastMessage, { transient: this.#lastUpdateTransient }); } } - #convertToolImagesForKitty(toolCallId: string, images: ImageContent[]): void { + #convertImagesForKitty(entries: Array<{ image: ImageContent; key: string }>): void { if (TERMINAL.imageProtocol !== ImageProtocol.Kitty) return; - for (let index = 0; index < images.length; index++) { - const image = images[index]; - if (!image || image.mimeType === "image/png") continue; - const key = `${toolCallId}:${index}`; + for (const { image, key } of entries) { + if (image.mimeType === "image/png") continue; if (this.#convertedKittyImages.has(key) || this.#kittyConversionsInFlight.has(key)) continue; this.#kittyConversionsInFlight.add(key); new Bun.Image(Buffer.from(image.data, "base64")) @@ -550,11 +558,19 @@ export class AssistantMessageComponent extends Container { } } - #renderToolImages(): void { - const imageEntries = Array.from(this.#toolImagesByCallId.entries()).flatMap(([toolCallId, images]) => + #renderImages(message: AssistantMessage): void { + if (!this.#showImages) return; + const nativeEntries = message.content.flatMap((content, index) => + content.type === "image" && content.data && content.mimeType + ? [{ image: content, key: `native:${index}` }] + : [], + ); + const toolEntries = Array.from(this.#toolImagesByCallId.entries()).flatMap(([toolCallId, images]) => images.map((image, index) => ({ image, key: `${toolCallId}:${index}` })), ); + const imageEntries = [...nativeEntries, ...toolEntries]; if (imageEntries.length === 0) return; + this.#convertImagesForKitty(imageEntries); this.#contentContainer.addChild(new Spacer(1)); for (const { image, key } of imageEntries) { @@ -620,7 +636,7 @@ export class AssistantMessageComponent extends Container { #canFastPath(message: AssistantMessage): boolean { for (const content of message.content) { - if (content.type === "toolCall") return false; + if (content.type === "toolCall" || content.type === "image") return false; } if (this.#toolImagesByCallId.size > 0) return false; const errorPresentation = resolveAssistantErrorPresentation(message); @@ -826,7 +842,7 @@ export class AssistantMessageComponent extends Container { this.#stopThinkingAnimation(); } - this.#renderToolImages(); + this.#renderImages(message); const errorPresentation = resolveAssistantErrorPresentation(message); const hasToolCalls = message.content.some(c => c.type === "toolCall"); if (errorPresentation.kind === "compact-recovered") { diff --git a/packages/coding-agent/src/modes/components/chat-transcript-builder.ts b/packages/coding-agent/src/modes/components/chat-transcript-builder.ts index ace89611c..4203f602a 100644 --- a/packages/coding-agent/src/modes/components/chat-transcript-builder.ts +++ b/packages/coding-agent/src/modes/components/chat-transcript-builder.ts @@ -283,6 +283,7 @@ export class ChatTranscriptBuilder { this.deps.ui.imageBudget, proseOnlyThinking, ); + assistantComponent.setImagesVisible(settings.get("terminal.showImages")); this.container.addChild(assistantComponent); if (settings.get("display.cacheMissMarker")) { @@ -313,6 +314,7 @@ export class ChatTranscriptBuilder { undefined, proseOnlyThinking, ); + component.setImagesVisible(settings.get("terminal.showImages")); this.container.addChild(component); }; diff --git a/packages/coding-agent/src/modes/utils/interactive-context-helpers.ts b/packages/coding-agent/src/modes/utils/interactive-context-helpers.ts index e6c3f4ae2..525daa8ca 100644 --- a/packages/coding-agent/src/modes/utils/interactive-context-helpers.ts +++ b/packages/coding-agent/src/modes/utils/interactive-context-helpers.ts @@ -16,12 +16,15 @@ export function createAssistantMessageComponent( ctx: InteractiveModeContext, message?: AssistantMessage, ): AssistantMessageComponent { - return new AssistantMessageComponent( - message, + const component = new AssistantMessageComponent( + undefined, ctx.effectiveHideThinkingBlock, () => ctx.ui.requestRender(), ctx.viewSession.extensionRunner?.getAssistantThinkingRenderers(), ctx.ui.imageBudget, ctx.proseOnlyThinking, ); + component.setImagesVisible(ctx.settings.get("terminal.showImages")); + if (message) component.updateContent(message); + return component; } diff --git a/packages/coding-agent/src/modes/utils/transcript-render-helpers.ts b/packages/coding-agent/src/modes/utils/transcript-render-helpers.ts index 5d90d2454..7664471ce 100644 --- a/packages/coding-agent/src/modes/utils/transcript-render-helpers.ts +++ b/packages/coding-agent/src/modes/utils/transcript-render-helpers.ts @@ -125,12 +125,13 @@ export function buildFileMentionBlock(files: FileMentionMessage["files"], indent } /** - * Whether an assistant turn has visible text or thinking content (after - * canonicalization) — i.e. content that closes the current read-tool run. + * Whether an assistant turn has visible text, thinking, or image content — i.e. + * content that closes the current read-tool run. */ export function assistantHasVisibleContent(message: AssistantAgentMessage): boolean { return message.content.some( content => + content.type === "image" || (content.type === "text" && canonicalizeMessage(content.text)) || (content.type === "thinking" && canonicalizeMessage(content.thinking)), ); diff --git a/packages/coding-agent/src/session/agent-session.ts b/packages/coding-agent/src/session/agent-session.ts index 5aa17e500..494fb1f79 100644 --- a/packages/coding-agent/src/session/agent-session.ts +++ b/packages/coding-agent/src/session/agent-session.ts @@ -1538,15 +1538,20 @@ function queuedTextContent(message: AgentMessage): string | undefined { if (!("content" in message)) return undefined; const content = message.content; if (typeof content === "string") return content; - return content.find((part): part is TextContent => part.type === "text")?.text; + for (const part of content) { + if (part.type === "text") return part.text; + } + return undefined; } function queuedImageContent(message: AgentMessage): ImageContent[] | undefined { if (!("content" in message) || typeof message.content === "string") return undefined; - const images = message.content.filter( - (part): part is ImageContent => - part.type === "image" && typeof part.data === "string" && typeof part.mimeType === "string", - ); + const images: ImageContent[] = []; + for (const part of message.content) { + if (part.type === "image" && typeof part.data === "string" && typeof part.mimeType === "string") { + images.push(part); + } + } return images.length > 0 ? images : undefined; } diff --git a/packages/coding-agent/src/session/session-listing.ts b/packages/coding-agent/src/session/session-listing.ts index 4b554fddb..586cbd825 100644 --- a/packages/coding-agent/src/session/session-listing.ts +++ b/packages/coding-agent/src/session/session-listing.ts @@ -1,6 +1,6 @@ import * as os from "node:os"; import * as path from "node:path"; -import type { Message, TextContent } from "@oh-my-pi/pi-ai"; +import type { Message } from "@oh-my-pi/pi-ai"; import { getAgentDir as getDefaultAgentDir, logger, parseJsonlLenient, toError } from "@oh-my-pi/pi-utils"; import { computeDefaultSessionDir } from "./session-paths"; import { FileSessionStorage, type SessionStorage } from "./session-storage"; @@ -108,10 +108,11 @@ function sessionDisplayName(info: SessionInfo): string { function extractTextFromContent(content: Message["content"]): string { if (typeof content === "string") return content; - return content - .filter((block): block is TextContent => block.type === "text") - .map(block => block.text) - .join(" "); + const text: string[] = []; + for (const block of content) { + if (block.type === "text") text.push(block.text); + } + return text.join(" "); } /** diff --git a/packages/coding-agent/src/session/session-loader.ts b/packages/coding-agent/src/session/session-loader.ts index 0915db7b5..9950568ac 100644 --- a/packages/coding-agent/src/session/session-loader.ts +++ b/packages/coding-agent/src/session/session-loader.ts @@ -252,6 +252,15 @@ async function resolvePersistedBlobRefs(value: unknown, blobStore: BlobStore, ke } if (typeof value !== "object" || value === null) return; + if ( + "type" in value && + value.type === "image_generation_call" && + "result" in value && + typeof value.result === "string" && + isBlobRef(value.result) + ) { + value.result = await resolveImageData(blobStore, value.result); + } if (hasImageUrl(value) && isBlobRef(value.image_url)) { value.image_url = await resolveImageDataUrl(blobStore, value.image_url); diff --git a/packages/coding-agent/src/session/session-persistence.ts b/packages/coding-agent/src/session/session-persistence.ts index 68a2e3455..cc2d6fdfb 100644 --- a/packages/coding-agent/src/session/session-persistence.ts +++ b/packages/coding-agent/src/session/session-persistence.ts @@ -79,6 +79,17 @@ function isNonEmptyString(value: unknown): value is string { */ function truncateForPersistence(obj: unknown, blobStore: BlobStore, key?: string): unknown { if (obj === null || obj === undefined) return obj; + if ( + typeof obj === "object" && + "type" in obj && + obj.type === "image_generation_call" && + "result" in obj && + typeof obj.result === "string" && + !isBlobRef(obj.result) && + obj.result.length >= BLOB_EXTERNALIZE_THRESHOLD + ) { + return { ...obj, result: externalizeImageDataSync(blobStore, obj.result) }; + } if (shouldExternalizeImagePayload(obj, key)) { return { ...obj, data: externalizeImageDataSync(blobStore, obj.data, obj.mimeType) }; } diff --git a/packages/coding-agent/test/agent-session-eager-compaction.test.ts b/packages/coding-agent/test/agent-session-eager-compaction.test.ts index cd8528ac9..11226994c 100644 --- a/packages/coding-agent/test/agent-session-eager-compaction.test.ts +++ b/packages/coding-agent/test/agent-session-eager-compaction.test.ts @@ -58,10 +58,11 @@ function getMessageText(message: AgentMessage): string { if (!("content" in message)) return ""; if (typeof message.content === "string") return message.content; if (!Array.isArray(message.content)) return ""; - return message.content - .filter(isTextContentBlock) - .map(content => content.text) - .join("\n"); + const text: string[] = []; + for (const content of message.content) { + if (isTextContentBlock(content)) text.push(content.text); + } + return text.join("\n"); } function createAssistantResponse(text: string) { diff --git a/packages/coding-agent/test/agent-session-eager-task.test.ts b/packages/coding-agent/test/agent-session-eager-task.test.ts index 566e75306..4cbab7a1e 100644 --- a/packages/coding-agent/test/agent-session-eager-task.test.ts +++ b/packages/coding-agent/test/agent-session-eager-task.test.ts @@ -55,10 +55,11 @@ function getMessageText(message: AgentMessage): string { if (!Array.isArray(message.content)) { return ""; } - return message.content - .filter(isTextContentBlock) - .map(content => content.text) - .join("\n"); + const text: string[] = []; + for (const content of message.content) { + if (isTextContentBlock(content)) text.push(content.text); + } + return text.join("\n"); } describe("AgentSession eager task prelude", () => { diff --git a/packages/coding-agent/test/agent-session-eager-todo.test.ts b/packages/coding-agent/test/agent-session-eager-todo.test.ts index baa6d83f2..5e1c42f7b 100644 --- a/packages/coding-agent/test/agent-session-eager-todo.test.ts +++ b/packages/coding-agent/test/agent-session-eager-todo.test.ts @@ -88,10 +88,11 @@ function getMessageText(message: AgentMessage): string { if (!Array.isArray(message.content)) { return ""; } - return message.content - .filter(isTextContentBlock) - .map(content => content.text) - .join("\n"); + const text: string[] = []; + for (const content of message.content) { + if (isTextContentBlock(content)) text.push(content.text); + } + return text.join("\n"); } describe("AgentSession eager todo enforcement", () => { diff --git a/packages/coding-agent/test/agent-session-openai-responses-replay.test.ts b/packages/coding-agent/test/agent-session-openai-responses-replay.test.ts index 0837c416a..78257a81a 100644 --- a/packages/coding-agent/test/agent-session-openai-responses-replay.test.ts +++ b/packages/coding-agent/test/agent-session-openai-responses-replay.test.ts @@ -130,7 +130,10 @@ function getMessageEntries(sessionManager: SessionManager): SessionMessageEntry[ function getTextContent(message: Message): string | undefined { if (typeof message.content === "string") return message.content; - return message.content.find(block => block.type === "text")?.text; + for (const block of message.content) { + if (block.type === "text") return block.text; + } + return undefined; } function findPersistedMessageEntry( diff --git a/packages/coding-agent/test/agent-session-plan-mode-convergence.test.ts b/packages/coding-agent/test/agent-session-plan-mode-convergence.test.ts index 6343fe01d..99fce55cb 100644 --- a/packages/coding-agent/test/agent-session-plan-mode-convergence.test.ts +++ b/packages/coding-agent/test/agent-session-plan-mode-convergence.test.ts @@ -55,10 +55,11 @@ function messageText(message: AgentMessage): string { const content = message.content; if (typeof content === "string") return content; if (!Array.isArray(content)) return ""; - return content - .filter(block => block.type === "text") - .map(block => block.text) - .join("\n"); + const text: string[] = []; + for (const block of content) { + if (block.type === "text") text.push(block.text); + } + return text.join("\n"); } function countReminders(messages: readonly AgentMessage[]): number { diff --git a/packages/coding-agent/test/agent-session-plan-reference-compaction.test.ts b/packages/coding-agent/test/agent-session-plan-reference-compaction.test.ts index 2885fc6e5..67b942741 100644 --- a/packages/coding-agent/test/agent-session-plan-reference-compaction.test.ts +++ b/packages/coding-agent/test/agent-session-plan-reference-compaction.test.ts @@ -50,10 +50,11 @@ function getMessageText(message: AgentMessage): string { if (!("content" in message)) return ""; if (typeof message.content === "string") return message.content; if (!Array.isArray(message.content)) return ""; - return message.content - .filter(isTextContentBlock) - .map(content => content.text) - .join("\n"); + const text: string[] = []; + for (const content of message.content) { + if (isTextContentBlock(content)) text.push(content.text); + } + return text.join("\n"); } function createAssistantResponse(text: string) { diff --git a/packages/coding-agent/test/agent-session-skill-keywords.test.ts b/packages/coding-agent/test/agent-session-skill-keywords.test.ts index 25c8273ac..f9e12a59d 100644 --- a/packages/coding-agent/test/agent-session-skill-keywords.test.ts +++ b/packages/coding-agent/test/agent-session-skill-keywords.test.ts @@ -1,7 +1,6 @@ import { afterEach, beforeEach, describe, expect, it } from "bun:test"; import * as path from "node:path"; import { Agent, type AgentTool } from "@oh-my-pi/pi-agent-core"; -import type { TextContent } from "@oh-my-pi/pi-ai"; import { AssistantMessageEventStream } from "@oh-my-pi/pi-ai/utils/event-stream"; import { getBundledModel } from "@oh-my-pi/pi-catalog/models"; import { ModelRegistry } from "@oh-my-pi/pi-coding-agent/config/model-registry"; @@ -64,10 +63,11 @@ describe("AgentSession skill prompt keyword steering", () => { const content = message.content; if (typeof content === "string") return content; if (!Array.isArray(content)) return ""; - return content - .filter((block): block is TextContent => block.type === "text") - .map(block => block.text) - .join("\n"); + const text: string[] = []; + for (const block of content) { + if (block.type === "text") text.push(block.text); + } + return text.join("\n"); }), }); const stream = new AssistantMessageEventStream(); diff --git a/packages/coding-agent/test/modes/components/assistant-message-mermaid.test.ts b/packages/coding-agent/test/modes/components/assistant-message-mermaid.test.ts index 33582e8bd..e375ec0ac 100644 --- a/packages/coding-agent/test/modes/components/assistant-message-mermaid.test.ts +++ b/packages/coding-agent/test/modes/components/assistant-message-mermaid.test.ts @@ -259,7 +259,19 @@ describe("AssistantMessageComponent thinking renderers", () => { }); }); -describe("AssistantMessageComponent tool images", () => { +describe("AssistantMessageComponent images", () => { + it("renders native assistant images and honors image visibility", () => { + const message: AssistantMessage = { + ...createAssistantMessage(""), + content: [{ type: "image", data: "aW1hZ2U=", mimeType: "image/png" }], + }; + const component = new AssistantMessageComponent(message); + + expect(Bun.stripANSI(component.render(80).join("\n"))).toContain("[Image: image/png]"); + component.setImagesVisible(false); + expect(Bun.stripANSI(component.render(80).join("\n"))).not.toContain("[Image: image/png]"); + }); + it("converts WebP tool images for Kitty terminal rendering", async () => { const webpBase64 = Buffer.from( await Bun.file(path.join(import.meta.dir, "../../../../../assets/python.webp")).arrayBuffer(), diff --git a/packages/coding-agent/test/session-persistence-images.test.ts b/packages/coding-agent/test/session-persistence-images.test.ts index 60b123b89..4b1c39441 100644 --- a/packages/coding-agent/test/session-persistence-images.test.ts +++ b/packages/coding-agent/test/session-persistence-images.test.ts @@ -68,4 +68,61 @@ describe("session image persistence", () => { expect(resolvedDetails.images[0]?.data).toBe(generatedImageData); expect(resolvedDetails.images[1]?.data).toBe(typedDetailImageData); }); + + it("externalizes and restores native Responses images in assistant content and provider history", async () => { + using tempDir = TempDir.createSync("@session-native-image-persistence-"); + const blobStore = new BlobStore(tempDir.path()); + const data = Buffer.alloc(1500, 4).toString("base64"); + const original: SessionMessageEntry = { + type: "message", + id: "entry-native-image", + parentId: null, + timestamp: new Date(0).toISOString(), + message: { + role: "assistant", + content: [png(data)], + api: "openai-responses", + provider: "openai", + model: "gpt-image-test", + usage: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + stopReason: "stop", + providerPayload: { + type: "openaiResponsesHistory", + provider: "openai", + items: [{ type: "image_generation_call", id: "ig_1", status: "completed", result: data }], + }, + timestamp: Date.now(), + }, + }; + + const persisted = prepareEntryForPersistence(original, blobStore); + if (persisted.type !== "message" || persisted.message.role !== "assistant") { + throw new Error("expected persisted assistant message"); + } + const persistedImage = persisted.message.content.find(block => block.type === "image"); + const persistedItem = persisted.message.providerPayload?.items[0]; + if (!persistedItem || typeof persistedItem.result !== "string") { + throw new Error("expected persisted image generation item"); + } + expect(isBlobRef(persistedImage?.data ?? "")).toBe(true); + expect(isBlobRef(persistedItem.result)).toBe(true); + + const loaded: FileEntry[] = [structuredClone(persisted)]; + await resolveBlobRefsInEntries(loaded, blobStore); + const resolved = loaded[0]; + if (resolved?.type !== "message" || resolved.message.role !== "assistant") { + throw new Error("expected resolved assistant message"); + } + const resolvedImage = resolved.message.content.find(block => block.type === "image"); + const resolvedItem = resolved.message.providerPayload?.items[0]; + expect(resolvedImage?.data).toBe(data); + expect(resolvedItem?.result).toBe(data); + }); });