Merge pull request #1577 from shoucandanghehe/feat/assistant-thinking-renderer

feat(coding-agent): add assistant thinking renderers
This commit is contained in:
Can Bölük
2026-06-01 18:16:50 +03:00
committed by GitHub
13 changed files with 266 additions and 49 deletions
+15 -1
View File
@@ -112,7 +112,7 @@ Core methods:
- `on(event, handler)`
- `registerTool`, `registerCommand`, `registerShortcut`, `registerFlag`
- `registerMessageRenderer`
- `registerMessageRenderer`, `registerAssistantThinkingRenderer`
- `setLabel`, `getFlag`
- `sendMessage`, `sendUserMessage`, `appendEntry`, `exec`
- `getActiveTools`, `getAllTools`, `setActiveTools`
@@ -359,6 +359,20 @@ pi.registerMessageRenderer("my-type", (message, { expanded }, theme) => {
Used by interactive rendering when custom messages are displayed.
## Assistant thinking renderer
```ts
import { Container, Text } from "@oh-my-pi/pi-tui";
pi.registerAssistantThinkingRenderer((context, theme) => {
const container = new Container();
container.addChild(new Text(theme.fg("dim", `thinking chars: ${context.text.length}`), 1, 0));
return container;
});
```
Used by interactive rendering to add display-only supplemental UI below each visible assistant thinking block. The renderer receives the already-visible thinking text, content/thinking indexes, theme, and a `requestRender()` callback for async renderers. All registered renderers that return a component are appended in registration order. Renderers must not mutate messages; the original thinking block remains the provider/session source of truth.
## Tool call/result renderer
Provide `renderCall` / `renderResult` on `registerTool` definitions for custom tool visualization in TUI.
+2 -1
View File
@@ -4,7 +4,8 @@
### Added
- Added `ask` option descriptions so agents can keep short labels and render explanatory text as separate muted rows in the selector.
- Added an extension API for rendering supplemental UI below visible assistant thinking blocks.
### Fixed
- Fixed the `eval` tool aborting in-flight `agent()`/`parallel()` subagents and `llm()` requests by mistaking them for a stalled cell. The per-cell `timeout` is an *inactivity* budget that only re-arms on status events, but a host-side bridge call can legitimately run long stretches with no intermediate status (a subagent's time-to-first-token on a reasoning model, a long quiet nested tool, or an entire oneshot `llm()` request). Those calls now pump a lightweight heartbeat while they await, re-arming the idle watchdog through the existing status channel; the heartbeat is a pure keepalive and is never persisted or rendered, so a genuinely stalled cell is still interrupted once the call settles.
@@ -41,6 +41,7 @@ cp permission-gate.ts ~/.omp/agent/extensions/
| `handoff.ts` | Transfer context to a new focused session via `/handoff <goal>` |
| `qna.ts` | Extracts questions from last response into editor via `ctx.ui.setEditorText()` |
| `status-line.ts` | Shows turn progress in footer via `ctx.ui.setStatus()` with themed colors |
| `thinking-note.ts` | Adds display-only supplemental UI below assistant thinking blocks |
| `snake.ts` | Snake game with custom UI, keyboard handling, and session persistence |
### Git Integration
@@ -0,0 +1,13 @@
import type { ExtensionFactory } from "@oh-my-pi/pi-coding-agent";
import { Container, Text } from "@oh-my-pi/pi-tui";
const extension: ExtensionFactory = pi => {
pi.setLabel("Thinking note");
pi.registerAssistantThinkingRenderer((context, theme) => {
const container = new Container();
container.addChild(new Text(theme.fg("dim", `thinking chars: ${context.text.length}`), 1, 0));
return container;
});
};
export default extension;
@@ -5,7 +5,8 @@ import type * as fs1 from "node:fs";
import * as fs from "node:fs/promises";
import * as path from "node:path";
import type { ThinkingLevel } from "@oh-my-pi/pi-agent-core";
import type { ImageContent, Model, TextContent } from "@oh-my-pi/pi-ai";
import type { ImageContent, Model, TextContent, TSchema } from "@oh-my-pi/pi-ai";
import * as PiCodingAgent from "@oh-my-pi/pi-coding-agent";
import type { KeyId } from "@oh-my-pi/pi-tui";
import { hasFsCode, isEacces, isEnoent, logger } from "@oh-my-pi/pi-utils";
import * as Zod from "zod/v4";
@@ -22,6 +23,7 @@ import * as TypeBox from "../typebox";
import { resolvePath } from "../utils";
import type {
AssistantThinkingRenderer,
Extension,
ExtensionAPI,
ExtensionContext,
@@ -29,6 +31,7 @@ import type {
ExtensionRuntime as IExtensionRuntime,
LoadExtensionsResult,
MessageRenderer,
ProviderConfig,
RegisteredCommand,
ToolDefinition,
} from "./types";
@@ -55,8 +58,7 @@ export class ExtensionRuntimeNotInitializedError extends Error {
*/
export class ExtensionRuntime implements IExtensionRuntime {
flagValues = new Map<string, boolean | string>();
pendingProviderRegistrations: Array<{ name: string; config: import("./types").ProviderConfig; sourceId: string }> =
[];
pendingProviderRegistrations: Array<{ name: string; config: ProviderConfig; sourceId: string }> = [];
sendMessage(): void {
throw new ExtensionRuntimeNotInitializedError();
@@ -123,12 +125,12 @@ class ConcreteExtensionAPI implements ExtensionAPI, IExtensionRuntime {
readonly flagValues = new Map<string, boolean | string>();
readonly pendingProviderRegistrations: Array<{
name: string;
config: import("./types").ProviderConfig;
config: ProviderConfig;
sourceId: string;
}> = [];
constructor(
public readonly pi: typeof import("@oh-my-pi/pi-coding-agent"),
public readonly pi: typeof PiCodingAgent,
private readonly extension: Extension,
private readonly runtime: IExtensionRuntime,
private readonly cwd: string,
@@ -141,10 +143,7 @@ class ConcreteExtensionAPI implements ExtensionAPI, IExtensionRuntime {
this.extension.handlers.set(event, list);
}
registerTool<
TParams extends import("@oh-my-pi/pi-ai").TSchema = import("@oh-my-pi/pi-ai").TSchema,
TDetails = unknown,
>(tool: ToolDefinition<TParams, TDetails>): void {
registerTool<TParams extends TSchema = TSchema, TDetails = unknown>(tool: ToolDefinition<TParams, TDetails>): void {
this.extension.tools.set(tool.name, {
definition: tool,
extensionPath: this.extension.path,
@@ -190,6 +189,10 @@ class ConcreteExtensionAPI implements ExtensionAPI, IExtensionRuntime {
this.extension.messageRenderers.set(customType, renderer as MessageRenderer);
}
registerAssistantThinkingRenderer(renderer: AssistantThinkingRenderer): void {
this.extension.assistantThinkingRenderers.push(renderer);
}
getFlag(name: string): boolean | string | undefined {
if (!this.extension.flags.has(name)) return undefined;
return this.runtime.flagValues.get(name);
@@ -253,7 +256,7 @@ class ConcreteExtensionAPI implements ExtensionAPI, IExtensionRuntime {
return this.runtime.setSessionName(name);
}
registerProvider(name: string, config: import("./types").ProviderConfig): void {
registerProvider(name: string, config: ProviderConfig): void {
this.runtime.pendingProviderRegistrations.push({ name, config, sourceId: this.extension.path });
}
}
@@ -267,6 +270,7 @@ function createExtension(extensionPath: string, resolvedPath: string): Extension
resolvedPath,
handlers: new Map(),
tools: new Map(),
assistantThinkingRenderers: [],
messageRenderers: new Map(),
commands: new Map(),
flags: new Map(),
@@ -293,13 +297,7 @@ async function loadExtension(
}
const extension = createExtension(extensionPath, resolvedPath);
const api = new ConcreteExtensionAPI(
await import("@oh-my-pi/pi-coding-agent"),
extension,
runtime,
cwd,
eventBus,
);
const api = new ConcreteExtensionAPI(PiCodingAgent, extension, runtime, cwd, eventBus);
await factory(api);
return { extension, error: null };
@@ -320,7 +318,7 @@ export async function loadExtensionFromFactory(
name = "<inline>",
): Promise<Extension> {
const extension = createExtension(name, name);
const api = new ConcreteExtensionAPI(await import("@oh-my-pi/pi-coding-agent"), extension, runtime, cwd, eventBus);
const api = new ConcreteExtensionAPI(PiCodingAgent, extension, runtime, cwd, eventBus);
await factory(api);
return extension;
}
@@ -10,6 +10,7 @@ import { type Theme, theme } from "../../modes/theme/theme";
import type { SessionManager } from "../../session/session-manager";
import type {
AfterProviderResponseEvent,
AssistantThinkingRenderer,
BeforeAgentStartEvent,
BeforeAgentStartEventResult,
BeforeProviderRequestEvent,
@@ -343,22 +344,22 @@ export class ExtensionRunner {
this.runtime.flagValues.set(name, value);
}
static readonly #RESERVED_SHORTCUTS = new Set([
"ctrl+c",
"ctrl+d",
"ctrl+z",
"ctrl+k",
"ctrl+p",
"ctrl+l",
"ctrl+o",
"ctrl+t",
"ctrl+g",
"shift+tab",
"shift+ctrl+p",
"alt+enter",
"escape",
"enter",
]);
static readonly #RESERVED_SHORTCUTS: Record<string, true> = {
"ctrl+c": true,
"ctrl+d": true,
"ctrl+z": true,
"ctrl+k": true,
"ctrl+p": true,
"ctrl+l": true,
"ctrl+o": true,
"ctrl+t": true,
"ctrl+g": true,
"shift+tab": true,
"shift+ctrl+p": true,
"alt+enter": true,
escape: true,
enter: true,
};
getShortcuts(): Map<KeyId, ExtensionShortcut> {
const allShortcuts = new Map<KeyId, ExtensionShortcut>();
@@ -366,7 +367,7 @@ export class ExtensionRunner {
for (const [key, shortcut] of ext.shortcuts) {
const normalizedKey = key.toLowerCase() as KeyId;
if (ExtensionRunner.#RESERVED_SHORTCUTS.has(normalizedKey)) {
if (ExtensionRunner.#RESERVED_SHORTCUTS[normalizedKey]) {
logger.warn("Extension shortcut conflicts with built-in shortcut", {
key,
extensionPath: shortcut.extensionPath,
@@ -419,6 +420,10 @@ export class ExtensionRunner {
return undefined;
}
getAssistantThinkingRenderers(): AssistantThinkingRenderer[] {
return this.extensions.flatMap(ext => ext.assistantThinkingRenderers);
}
getRegisteredCommands(reserved?: Set<string>): RegisteredCommand[] {
this.#commandDiagnostics = [];
@@ -25,6 +25,8 @@ import type {
import type { OAuthCredentials, OAuthLoginCallbacks } from "@oh-my-pi/pi-ai/utils/oauth/types";
import type * as piCodingAgent from "@oh-my-pi/pi-coding-agent";
import type { AutocompleteItem, Component, EditorTheme, KeyId, TUI } from "@oh-my-pi/pi-tui";
import type { logger as PiLogger } from "@oh-my-pi/pi-utils";
import type * as Zod from "zod/v4";
import type { KeybindingsManager } from "../../config/keybindings";
import type { ModelRegistry } from "../../config/model-registry";
import type { EditToolDetails } from "../../edit";
@@ -81,6 +83,7 @@ import type {
TurnStartEvent,
} from "../shared-events";
import type { SlashCommandInfo } from "../slash-commands";
import type * as TypeBox from "../typebox";
export type { AppKeybinding, KeybindingsManager } from "../../config/keybindings";
export type { ExecOptions, ExecResult } from "../../exec/exec";
@@ -814,6 +817,18 @@ export type MessageRenderer<T = unknown> = (
theme: Theme,
) => Component | undefined;
export interface AssistantThinkingRenderContext {
contentIndex: number;
thinkingIndex: number;
text: string;
requestRender(): void;
}
export type AssistantThinkingRenderer = (
context: AssistantThinkingRenderContext,
theme: Theme,
) => Component | undefined;
// ============================================================================
// Command Registration
// ============================================================================
@@ -845,13 +860,13 @@ export interface ExtensionAPI {
// =========================================================================
/** File logger for error/warning/debug messages */
logger: typeof import("@oh-my-pi/pi-utils").logger;
logger: typeof PiLogger;
/** Injected zod-backed typebox shim for legacy `Type.Object(...)` parameter authoring. */
typebox: typeof import("../typebox");
typebox: typeof TypeBox;
/** Injected zod module for Zod-authored extension tools (canonical going forward). */
zod: typeof import("zod/v4");
zod: typeof Zod;
/** Injected pi-coding-agent exports for accessing SDK utilities */
pi: typeof piCodingAgent;
@@ -965,6 +980,9 @@ export interface ExtensionAPI {
/** Register a custom renderer for CustomMessageEntry. */
registerMessageRenderer<T = unknown>(customType: string, renderer: MessageRenderer<T>): void;
/** Register a renderer for assistant thinking blocks. Rendered after the original thinking text. */
registerAssistantThinkingRenderer(renderer: AssistantThinkingRenderer): void;
// =========================================================================
// Actions
// =========================================================================
@@ -1247,6 +1265,7 @@ export interface Extension {
label?: string;
handlers: Map<string, HandlerFn[]>;
tools: Map<string, RegisteredTool<any, any>>;
assistantThinkingRenderers: AssistantThinkingRenderer[];
messageRenderers: Map<string, MessageRenderer>;
commands: Map<string, RegisteredCommand>;
flags: Map<string, ExtensionFlag>;
@@ -2,6 +2,7 @@ import type { AssistantMessage, ImageContent, Usage } from "@oh-my-pi/pi-ai";
import { Container, Image, ImageProtocol, Markdown, Spacer, TERMINAL, Text } from "@oh-my-pi/pi-tui";
import { formatNumber } from "@oh-my-pi/pi-utils";
import { settings } from "../../config/settings";
import type { AssistantThinkingRenderer } from "../../extensibility/extensions/types";
import { getMarkdownTheme, theme } from "../../modes/theme/theme";
import { isSilentAbort } from "../../session/messages";
import { resolveImageOptions } from "../../tools/render-utils";
@@ -21,6 +22,7 @@ export class AssistantMessageComponent extends Container {
message?: AssistantMessage,
private hideThinkingBlock = false,
private readonly onImageUpdate?: () => void,
private readonly thinkingRenderers: readonly AssistantThinkingRenderer[] = [],
) {
super();
@@ -131,6 +133,27 @@ export class AssistantMessageComponent extends Container {
}
}
#appendThinkingExtensions(contentIndex: number, thinkingIndex: number, text: string): void {
for (const renderer of this.thinkingRenderers) {
try {
const component = renderer(
{
contentIndex,
thinkingIndex,
text,
requestRender: () => this.onImageUpdate?.(),
},
theme,
);
if (component) {
this.#contentContainer.addChild(component);
}
} catch {
// Ignore extension renderer failures and keep the original thinking block visible.
}
}
}
updateContent(message: AssistantMessage): void {
this.#lastMessage = message;
@@ -146,6 +169,7 @@ export class AssistantMessageComponent extends Container {
}
// Render content in order
let thinkingIndex = 0;
for (let i = 0; i < message.content.length; i++) {
const content = message.content[i];
if (content.type === "text" && content.text.trim()) {
@@ -166,13 +190,16 @@ export class AssistantMessageComponent extends Container {
this.#contentContainer.addChild(new Spacer(1));
}
} else {
const thinkingText = content.thinking.trim();
// Thinking traces in thinkingText color, italic
this.#contentContainer.addChild(
new Markdown(content.thinking.trim(), 1, 0, getMarkdownTheme(), {
new Markdown(thinkingText, 1, 0, getMarkdownTheme(), {
color: (text: string) => theme.fg("thinkingText", text),
italic: true,
}),
);
this.#appendThinkingExtensions(i, thinkingIndex, thinkingText);
thinkingIndex += 1;
if (hasVisibleContentAfter) {
this.#contentContainer.addChild(new Spacer(1));
}
@@ -277,8 +277,11 @@ export class EventController {
this.#lastThinkingCount = 0;
this.#assistantMessageStreaming = true;
this.#resetReadGroup();
this.ctx.streamingComponent = new AssistantMessageComponent(undefined, this.ctx.hideThinkingBlock, () =>
this.ctx.ui.requestRender(),
this.ctx.streamingComponent = new AssistantMessageComponent(
undefined,
this.ctx.hideThinkingBlock,
() => this.ctx.ui.requestRender(),
this.ctx.session.extensionRunner?.getAssistantThinkingRenderers(),
);
this.ctx.streamingMessage = event.message;
this.ctx.chatContainer.addChild(this.ctx.streamingComponent);
@@ -262,8 +262,11 @@ export class UiHelpers {
break;
}
case "assistant": {
const assistantComponent = new AssistantMessageComponent(message, this.ctx.hideThinkingBlock, () =>
this.ctx.ui.requestRender(),
const assistantComponent = new AssistantMessageComponent(
message,
this.ctx.hideThinkingBlock,
() => this.ctx.ui.requestRender(),
this.ctx.session.extensionRunner?.getAssistantThinkingRenderers(),
);
this.ctx.chatContainer.addChild(assistantComponent);
break;
@@ -305,6 +305,26 @@ describe("ExtensionRunner", () => {
const missing = runner.getMessageRenderer("not-exists");
expect(missing).toBeUndefined();
});
it("collects assistant thinking renderers", async () => {
const extCode = `
export default function(pi) {
pi.registerAssistantThinkingRenderer((context, theme) => null);
}
`;
fs.writeFileSync(path.join(extensionsDir, "thinking-renderer.ts"), extCode);
const result = await loadTestExtensions();
const runner = new ExtensionRunner(
result.extensions,
result.runtime,
tempDir.path(),
sessionManager,
modelRegistry,
);
expect(runner.getAssistantThinkingRenderers().length).toBe(1);
});
});
describe("flags", () => {
@@ -621,7 +641,7 @@ describe("ExtensionRunner", () => {
`
export default function(pi) {
pi.on("session_start", async () => {
await new Promise(() => {});
await Promise.withResolvers().promise;
});
}
`,
@@ -2,10 +2,11 @@ import { afterEach, beforeAll, beforeEach, describe, expect, it } from "bun:test
import * as path from "node:path";
import type { AssistantMessage } from "@oh-my-pi/pi-ai";
import { resetSettingsForTest, Settings } from "@oh-my-pi/pi-coding-agent/config/settings";
import type { AssistantThinkingRenderer } from "@oh-my-pi/pi-coding-agent/extensibility/extensions";
import { AssistantMessageComponent } from "@oh-my-pi/pi-coding-agent/modes/components/assistant-message";
import { clearMermaidCache } from "@oh-my-pi/pi-coding-agent/modes/theme/mermaid-cache";
import { initTheme } from "@oh-my-pi/pi-coding-agent/modes/theme/theme";
import { ImageProtocol, setTerminalImageProtocol, TERMINAL } from "@oh-my-pi/pi-tui";
import { ImageProtocol, setTerminalImageProtocol, TERMINAL, Text } from "@oh-my-pi/pi-tui";
const originalImageProtocol = TERMINAL.imageProtocol;
@@ -29,8 +30,8 @@ function createAssistantMessage(markdown: string): AssistantMessage {
};
}
function renderAssistantMessage(markdown: string): string {
const component = new AssistantMessageComponent(createAssistantMessage(markdown));
function renderAssistantMessage(markdown: string, renderers: readonly AssistantThinkingRenderer[] = []): string {
const component = new AssistantMessageComponent(createAssistantMessage(markdown), false, undefined, renderers);
return Bun.stripANSI(component.render(120).join("\n"))
.split("\n")
.map(line => line.trimEnd())
@@ -74,6 +75,117 @@ describe("AssistantMessageComponent mermaid markdown", () => {
});
});
describe("AssistantMessageComponent thinking renderers", () => {
it("renders all extension outputs below visible thinking blocks in registration order", () => {
const contexts: Array<{ contentIndex: number; thinkingIndex: number; text: string }> = [];
const component = new AssistantMessageComponent(
{
...createAssistantMessage(""),
content: [{ type: "thinking", thinking: "I should inspect the input." }],
},
false,
undefined,
[
context => {
contexts.push({
contentIndex: context.contentIndex,
thinkingIndex: context.thinkingIndex,
text: context.text,
});
return new Text("first note", 1, 0);
},
() => new Text("second note", 1, 0),
],
);
const rendered = Bun.stripANSI(component.render(120).join("\n"));
expect(rendered).toContain("I should inspect the input.");
expect(rendered.indexOf("I should inspect the input.")).toBeLessThan(rendered.indexOf("first note"));
expect(rendered.indexOf("first note")).toBeLessThan(rendered.indexOf("second note"));
expect(contexts).toEqual([{ contentIndex: 0, thinkingIndex: 0, text: "I should inspect the input." }]);
});
it("keeps original thinking visible when an extension renderer throws", () => {
const component = new AssistantMessageComponent(
{
...createAssistantMessage(""),
content: [{ type: "thinking", thinking: "I should inspect the input." }],
},
false,
undefined,
[
() => {
throw new Error("renderer failed");
},
],
);
const rendered = Bun.stripANSI(component.render(120).join("\n"));
expect(rendered).toContain("I should inspect the input.");
expect(rendered).not.toContain("renderer failed");
});
it("keeps async renderer components mounted when they request a render", () => {
let renderRequests = 0;
let rendererCalls = 0;
let mountedNote: Text | undefined;
let requestRender: (() => void) | undefined;
const component = new AssistantMessageComponent(
{
...createAssistantMessage(""),
content: [{ type: "thinking", thinking: "I should inspect the input." }],
},
false,
() => {
renderRequests += 1;
},
[
context => {
rendererCalls += 1;
requestRender = context.requestRender;
const note = new Text("translation loading", 1, 0);
mountedNote ??= note;
return note;
},
],
);
expect(Bun.stripANSI(component.render(120).join("\n"))).toContain("translation loading");
mountedNote?.setText("translation ready");
requestRender?.();
const rendered = Bun.stripANSI(component.render(120).join("\n"));
expect(renderRequests).toBe(1);
expect(rendererCalls).toBe(1);
expect(rendered).toContain("translation ready");
expect(rendered).not.toContain("translation loading");
});
it("does not invoke extension renderers when thinking is hidden", () => {
let rendererCalled = false;
const component = new AssistantMessageComponent(
{
...createAssistantMessage(""),
content: [{ type: "thinking", thinking: "I should inspect the input." }],
},
true,
undefined,
[
() => {
rendererCalled = true;
return new Text("hidden note", 1, 0);
},
],
);
const rendered = Bun.stripANSI(component.render(120).join("\n"));
expect(rendered).toContain("Thinking...");
expect(rendered).not.toContain("I should inspect the input.");
expect(rendered).not.toContain("hidden note");
expect(rendererCalled).toBe(false);
});
});
describe("AssistantMessageComponent tool images", () => {
it("converts WebP tool images for Kitty terminal rendering", async () => {
const webpBase64 = Buffer.from(
@@ -493,6 +493,7 @@ describe("createAgentSession credential_disabled subscription", () => {
],
]),
tools: new Map(),
assistantThinkingRenderers: [],
messageRenderers: new Map(),
commands: new Map(),
flags: new Map(),