smashimo/oh-my-pi
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(coding-agent): added live session delegation with turn-based transcript display

Browse Source 
- Added `LIVE_DELEGATION_MESSAGE_TYPE` constant and delegation message handling for voice sessions.
- Implemented turn-based transcript coalescing with user and assistant turn counters.
- Added transcript display row with normalized rendering in the live visualizer.
- Refactored controller to send delegation messages via `sendCustomMessage` with configurable frame styling.
- Removed microphone permission error reporting from silence detection logic.
This commit is contained in:
can1357
2026-07-24 09:16:50 +02:00
parent c9c0882724
commit c1d4e38aa6
11 changed files with 164 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/pi-natives/src/audio.rs
-20
View File
@@ -276,7 +276,6 @@ impl AudioCapture {
on_audio: CaptureCallback,
) -> Result<Self> {
let sample_rate = audio_sample_rate(sample_rate).map_err(napi::Error::from_reason)?;
let sample_rate_hz = u32::from(sample_rate);
let mut builder = DeviceBuilder::capture().f32();
builder
.sample_rate(sample_rate)
@@ -284,30 +283,11 @@ impl AudioCapture {
.period_size_millis(AUDIO_PERIOD_MS)
.performance_profile(PerformanceProfile::LowLatency)
.backends(AUDIO_BACKENDS);
let mut inspected_samples = 0usize;
let mut received_signal = false;
let mut silence_reported = false;
let mut device = builder
.with_callback(move |_device, samples| {
if samples.is_empty() {
return;
}
if !received_signal {
received_signal = samples.iter().any(|sample| *sample != 0.0);
inspected_samples = inspected_samples.saturating_add(samples.len());
}
if !received_signal && !silence_reported && inspected_samples >= sample_rate_hz as usize
{
silence_reported = true;
on_audio.call(
Err(napi::Error::from_reason(
"Default microphone produced only silence; check microphone permission and \
the default input device",
)),
ThreadsafeFunctionCallMode::NonBlocking,
);
return;
}
on_audio.call(
Ok(Float32Array::new(samples.to_vec())),
ThreadsafeFunctionCallMode::NonBlocking,
packages/coding-agent/src/live/attestation.ts
+2 -2
View File
@@ -78,8 +78,8 @@ function buildClientAttestation(result: DeviceCheckResult): string {
return `v1.${cborMap(entries).toString("base64url")}`;
}
/** Generates the Codex Desktop DeviceCheck header accepted by the private live-call endpoints. */
export async function generateLiveAttestation(): Promise<string | undefined> {
/** Generates the Codex DeviceCheck attestation envelope sent as `x-oai-attestation` on ChatGPT-OAuth requests. */
export async function generateCodexAttestation(): Promise<string | undefined> {
if (process.platform !== "darwin" || process.arch !== "arm64") return undefined;
let result: DeviceCheckResult;
try {
packages/coding-agent/src/live/controller.ts
+42 -4
View File
@@ -5,6 +5,7 @@ import { AudioCapture } from "@oh-my-pi/pi-natives";
import { prompt } from "@oh-my-pi/pi-utils";
import type { AgentSession } from "../session/agent-session";
import type { AgentSessionEvent } from "../session/agent-session-events";
import { LIVE_DELEGATION_MESSAGE_TYPE } from "../session/messages";
import agentFinalMessageTemplate from "./prompts/agent-final-message.md" with { type: "text" };
import liveInstructionsTemplate from "./prompts/live-instructions.md" with { type: "text" };
import {
@@ -26,6 +27,8 @@ const OUTPUT_ECHO_RATIO = 0.65;
export interface LiveTranscript {
role: "user" | "assistant";
text: string;
/** Monotonic role-local turn number used to coalesce streaming updates. */
turn: number;
final: boolean;
}
@@ -109,6 +112,8 @@ export class LiveSessionController {
#assistantTranscript = "";
#userTranscriptFinal = false;
#assistantTranscriptFinal = false;
#userTranscriptTurn = 0;
#assistantTranscriptTurn = 0;
#lastTranscript: LiveTranscript | undefined;
constructor(options: LiveSessionControllerOptions) {
@@ -296,7 +301,17 @@ export class LiveSessionController {
if (!request) return;
this.#activeDelegationId = event.item.id;
this.#emitPhase("working");
void this.#session.sendUserMessage(request).catch(cause => this.#reportFailure(errorFrom(cause)));
void this.#session
.sendCustomMessage(
{
customType: LIVE_DELEGATION_MESSAGE_TYPE,
content: request,
display: true,
attribution: "agent",
},
{ triggerTurn: true },
)
.catch(cause => this.#reportFailure(errorFrom(cause)));
}
#handleSessionEvent(event: AgentSessionEvent): void {
@@ -362,7 +377,12 @@ export class LiveSessionController {
const current = role === "user" ? this.#userTranscript : this.#assistantTranscript;
const wasFinal = role === "user" ? this.#userTranscriptFinal : this.#assistantTranscriptFinal;
let next: string;
if (wasFinal && current !== text && !text.startsWith(current) && !current.endsWith(text)) {
if (!current) {
this.#startTranscriptTurn(role);
next = text;
} else if (wasFinal) {
if (text === current || current.endsWith(text)) return;
this.#startTranscriptTurn(role);
next = text;
} else if (text.startsWith(current)) {
next = text;
@@ -377,12 +397,29 @@ export class LiveSessionController {
#finishTranscript(role: LiveTranscript["role"], text: string): void {
if (!text) return;
const current = role === "user" ? this.#userTranscript : this.#assistantTranscript;
const next = current.startsWith(text) && current.length > text.length ? current : text;
const wasFinal = role === "user" ? this.#userTranscriptFinal : this.#assistantTranscriptFinal;
if (!current) {
this.#startTranscriptTurn(role);
} else if (wasFinal) {
if (text === current) return;
this.#startTranscriptTurn(role);
}
const next = !wasFinal && current.startsWith(text) && current.length > text.length ? current : text;
this.#storeTranscript(role, next, true);
}
#startTranscriptTurn(role: LiveTranscript["role"]): void {
if (role === "user") {
this.#userTranscriptTurn += 1;
} else {
this.#assistantTranscriptTurn += 1;
}
}
#storeTranscript(role: LiveTranscript["role"], text: string, final: boolean): void {
const normalized = text.trim();
if (!normalized) return;
const turn = role === "user" ? this.#userTranscriptTurn : this.#assistantTranscriptTurn;
if (role === "user") {
this.#userTranscript = normalized;
this.#userTranscriptFinal = final;
@@ -392,12 +429,13 @@ export class LiveSessionController {
}
if (
this.#lastTranscript?.role === role &&
this.#lastTranscript.turn === turn &&
this.#lastTranscript.text === normalized &&
this.#lastTranscript.final === final
) {
return;
}
this.#emitTranscript({ role, text: normalized, final });
this.#emitTranscript({ role, turn, text: normalized, final });
}
#queueSend(message: LiveClientMessage): void {
packages/coding-agent/src/live/transport.ts
+2 -2
View File
@@ -7,7 +7,7 @@ import {
OPENAI_HEADERS,
} from "@oh-my-pi/pi-catalog/wire/codex";
import { LiveWebRtcPeer } from "@oh-my-pi/pi-natives";
import { generateLiveAttestation } from "./attestation";
import { generateCodexAttestation } from "./attestation";
import {
buildLiveSessionPayload,
type LiveClientMessage,
@@ -184,7 +184,7 @@ export class CodexLiveTransport {
}
async #signal(offer: string): Promise<LiveSignalingResult> {
const attestation = await generateLiveAttestation();
const attestation = await generateCodexAttestation();
return await withOAuthAccess(
this.#options.authStorage,
LIVE_PROVIDER,
packages/coding-agent/src/live/visualizer.ts
+49 -3
View File
@@ -1,4 +1,12 @@
import { type Component, matchesKey, truncateToWidth, visibleWidth } from "@oh-my-pi/pi-tui";
import {
type Component,
matchesKey,
replaceTabs,
sliceWithWidth,
truncateToWidth,
visibleWidth,
} from "@oh-my-pi/pi-tui";
import { sanitizeText } from "@oh-my-pi/pi-utils";
import { type ThemeColor, theme } from "../modes/theme/theme";
/** Distinct states of a realtime call connection. */
@@ -9,6 +17,18 @@ export interface LiveVisualizerOptions {
onToggleMute(): void;
}
function normalizeTranscript(text: string): string {
return replaceTabs(sanitizeText(text)).replace(/\s+/g, " ").trim();
}
function truncateFromStart(text: string, width: number): string {
if (width <= 0) return "";
const textWidth = visibleWidth(text);
if (textWidth <= width) return text;
if (width === 1) return "…";
return `…${sliceWithWidth(text, textWidth - width + 1, width - 1, true).text}`;
}
/** A compact, fixed-height terminal component for displaying a realtime call. */
export class LiveVisualizer implements Component {
readonly wantsKeyRelease = false;
@@ -19,6 +39,7 @@ export class LiveVisualizer implements Component {
#inputLevel = 0;
#displayLevel = 0;
#frame = 0;
#userTranscript = "";
#cache:
| {
@@ -26,6 +47,7 @@ export class LiveVisualizer implements Component {
phase: LivePhase;
displayLevel: number;
frame: number;
userTranscript: string;
lines: readonly string[];
}
| undefined;
@@ -61,6 +83,21 @@ export class LiveVisualizer implements Component {
}
}
/** Updates the user's streaming voice transcript. */
setTranscript(text: string): void {
const normalized = normalizeTranscript(text);
if (this.#userTranscript === normalized) return;
this.#userTranscript = normalized;
this.invalidate();
}
/** Clears the user's voice transcript row. */
clearTranscript(): void {
if (!this.#userTranscript) return;
this.#userTranscript = "";
this.invalidate();
}
/** Processes user keypresses. */
handleInput(data: string): void {
if (matchesKey(data, "escape") || matchesKey(data, "ctrl+c")) {
@@ -82,7 +119,8 @@ export class LiveVisualizer implements Component {
this.#cache.width === width &&
this.#cache.phase === this.#phase &&
this.#cache.displayLevel === this.#displayLevel &&
this.#cache.frame === this.#frame
this.#cache.frame === this.#frame &&
this.#cache.userTranscript === this.#userTranscript
) {
return this.#cache.lines;
}
@@ -93,6 +131,7 @@ export class LiveVisualizer implements Component {
phase: this.#phase,
displayLevel: this.#displayLevel,
frame: this.#frame,
userTranscript: this.#userTranscript,
lines,
};
return lines;
@@ -107,7 +146,14 @@ export class LiveVisualizer implements Component {
const spectrumColor: ThemeColor = this.#phase === "muted" ? "dim" : this.#phase === "error" ? "error" : "success";
const spectrum = this.#generateSpectrum(innerWidth, 2);
const spectrumRows = spectrum.map(row => border(theme.fg(spectrumColor, row)));
return [top, ...spectrumRows, this.#renderFooter(width, innerWidth)];
const transcript = this.#renderTranscript(this.#userTranscript, innerWidth, border);
return [top, ...spectrumRows, transcript, this.#renderFooter(width, innerWidth)];
}
#renderTranscript(transcript: string, innerWidth: number, border: (content: string) => string): string {
const content = truncateFromStart(transcript, innerWidth);
const padding = " ".repeat(Math.max(0, innerWidth - visibleWidth(content)));
return border(theme.fg("accent", content) + padding);
}
#renderFooter(width: number, innerWidth: number): string {
packages/coding-agent/src/modes/components/assistant-message.ts
+7 -2
View File
@@ -235,6 +235,11 @@ export class AssistantMessageComponent extends Container {
* on a fresh block that has no live token throughput of its own. */
#thinkingRateLive = false;
#textColorTransform?: (text: string) => string;
setTextColorTransform(transform?: (text: string) => string): void {
this.#textColorTransform = transform;
}
constructor(
message?: AssistantMessage,
private hideThinkingBlock = false,
@@ -796,8 +801,8 @@ export class AssistantMessageComponent extends Container {
if (content.type === "text" && canonicalizeMessage(content.text)) {
// Set paddingY=0 to avoid extra spacing before tool executions
const trimmed = content.text.trim();
const md = new Markdown(trimmed, 1, 0, getMarkdownTheme());
md.transientRenderCache = this.#lastUpdateTransient;
const mdOptions = this.#textColorTransform ? { color: this.#textColorTransform } : undefined;
const md = new Markdown(trimmed, 1, 0, getMarkdownTheme(), mdOptions);
this.#contentContainer.addChild(md);
captureItems?.push({ md, contentIndex: i, blockType: "text", lastText: trimmed });
hasRenderedContent = true;
packages/coding-agent/src/modes/components/custom-message.ts
+4 -1
View File
@@ -2,7 +2,7 @@ import type { Component } from "@oh-my-pi/pi-tui";
import { Box, Container } from "@oh-my-pi/pi-tui";
import type { MessageRenderer } from "../../extensibility/extensions/types";
import { theme } from "../../modes/theme/theme";
import type { CustomMessage } from "../../session/messages";
import { LIVE_DELEGATION_MESSAGE_TYPE, type CustomMessage } from "../../session/messages";
import { renderFramedMessage } from "./message-frame";
/**
@@ -49,12 +49,15 @@ export class CustomMessageComponent extends Container {
// The transcript dispatch routes both `custom` and legacy `hookMessage` roles here:
// tag hooks with the hook glyph, other injected messages with a neutral package.
const isHook = (this.message.role as string) === "hookMessage";
const isLiveDelegation = this.message.customType === LIVE_DELEGATION_MESSAGE_TYPE;
const custom = renderFramedMessage({
message: this.message,
box: this.#box,
expanded: this.#expanded,
customRenderer: this.customRenderer,
icon: isHook ? theme.icon.extensionHook : theme.icon.package,
hideHeader: isLiveDelegation,
borderColor: isLiveDelegation ? "borderAccent" : undefined,
});
if (custom) {
packages/coding-agent/src/modes/components/message-frame.ts
+14 -8
View File
@@ -11,7 +11,7 @@
import type { TextContent } from "@oh-my-pi/pi-ai";
import type { Box, Component } from "@oh-my-pi/pi-tui";
import { Markdown, Spacer, Text } from "@oh-my-pi/pi-tui";
import { getMarkdownTheme, type Theme, theme } from "../../modes/theme/theme";
import { getMarkdownTheme, type Theme, type ThemeColor, theme } from "../../modes/theme/theme";
/** Message shape consumed by the shared frame. */
export interface FramedMessage {
@@ -36,6 +36,10 @@ export interface RebuildFrameOptions<M extends FramedMessage> {
expanded: boolean;
/** Icon glyph shown before the customType in the default header (e.g. a hook/extension icon). */
icon?: string;
/** Hide the default type header while retaining the message body. */
hideHeader?: boolean;
/** Semantic color for the outline, defaulting to the muted border. */
borderColor?: ThemeColor;
/** Collapse the markdown body to this many lines when `expanded` is false. Omit to never collapse. */
collapseAfterLines?: number;
customRenderer?: FramedRenderer<M>;
@@ -43,9 +47,9 @@ export interface RebuildFrameOptions<M extends FramedMessage> {
/**
* Attempt the custom renderer; on failure or undefined return, populate `box`
* with the default outlined card — an `icon customType` header + markdown body —
* and return undefined. When the custom renderer succeeds, return its Component
* so the caller can mount it and skip the default box.
* with the configured outline, optional type header, and markdown body. When
* the custom renderer succeeds, return its Component so the caller can mount
* it and skip the default box.
*/
export function renderFramedMessage<M extends FramedMessage>(opts: RebuildFrameOptions<M>): Component | undefined {
if (opts.customRenderer) {
@@ -59,11 +63,13 @@ export function renderFramedMessage<M extends FramedMessage>(opts: RebuildFrameO
opts.box.clear();
// Match the skill card: a subtle rounded outline so injected messages read as cards.
opts.box.setBorder({ chars: theme.boxRound, color: t => theme.fg("borderMuted", t) });
opts.box.setBorder({ chars: theme.boxRound, color: t => theme.fg(opts.borderColor ?? "borderMuted", t) });
const tag = opts.icon ? `${opts.icon} ${opts.message.customType}` : opts.message.customType;
opts.box.addChild(new Text(theme.fg("customMessageLabel", theme.bold(tag)), 0, 0));
opts.box.addChild(new Spacer(1));
if (!opts.hideHeader) {
const tag = opts.icon ? `${opts.icon} ${opts.message.customType}` : opts.message.customType;
opts.box.addChild(new Text(theme.fg("customMessageLabel", theme.bold(tag)), 0, 0));
opts.box.addChild(new Spacer(1));
}
let text: string;
if (typeof opts.message.content === "string") {
packages/coding-agent/src/modes/controllers/live-command-controller.ts
+24 -15
View File
@@ -1,4 +1,5 @@
import type { AssistantMessage } from "@oh-my-pi/pi-ai";
import chalk from "chalk";
import { logger } from "@oh-my-pi/pi-utils";
import { LiveSessionController, type LiveTranscript } from "../../live/controller";
import { LIVE_MODEL } from "../../live/protocol";
@@ -6,6 +7,7 @@ import { LiveVisualizer } from "../../live/visualizer";
import { vocalizer } from "../../tts/vocalizer";
import type { AssistantMessageComponent } from "../components/assistant-message";
import type { CustomEditor } from "../components/custom-editor";
import { theme } from "../theme/theme";
import type { InteractiveModeContext } from "../types";
import { createAssistantMessageComponent } from "../utils/interactive-context-helpers";
@@ -19,7 +21,6 @@ const LIVE_MESSAGE_USAGE: AssistantMessage["usage"] = {
totalTokens: 0,
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
};
function errorFrom(cause: unknown): Error {
return cause instanceof Error ? cause : new Error(String(cause));
}
@@ -37,6 +38,7 @@ export class LiveCommandController {
#previousUseTerminalCursor: boolean | undefined;
#resumeVocalizer: (() => void) | undefined;
#assistantTranscriptComponent: AssistantMessageComponent | undefined;
#assistantTranscriptTurn = 0;
#assistantTranscriptStartedAt = 0;
constructor(ctx: InteractiveModeContext) {
@@ -88,6 +90,8 @@ export class LiveCommandController {
}
async #start(): Promise<void> {
this.#assistantTranscriptTurn = 0;
this.#assistantTranscriptStartedAt = 0;
const visualizer = new LiveVisualizer({
onStop: () => {
void this.stop().catch(cause => this.#ctx.showError(errorFrom(cause).message));
@@ -113,7 +117,15 @@ export class LiveCommandController {
},
onTranscript: transcript => {
if (this.#visualizer !== visualizer) return;
this.#presentTranscript(transcript);
if (!transcript) {
visualizer.clearTranscript();
this.#ctx.ui.requestComponentRender(visualizer);
} else if (transcript.role === "user") {
visualizer.setTranscript(transcript.text);
this.#ctx.ui.requestComponentRender(visualizer);
} else {
this.#presentAssistantTranscript(transcript);
}
},
onTerminal: error => this.#finish(session, error),
},
@@ -130,24 +142,22 @@ export class LiveCommandController {
}
}
#presentTranscript(transcript: LiveTranscript | undefined): void {
if (!transcript) return;
if (transcript.role === "user") {
if (!transcript.final) return;
this.#finalizeAssistantTranscript();
this.#ctx.addMessageToChat({
role: "user",
content: [{ type: "text", text: transcript.text }],
attribution: "user",
timestamp: Date.now(),
});
this.#ctx.ui.requestRender();
#presentAssistantTranscript(transcript: LiveTranscript): void {
if (
transcript.turn < this.#assistantTranscriptTurn ||
(transcript.turn === this.#assistantTranscriptTurn && !this.#assistantTranscriptComponent)
) {
return;
}
if (transcript.turn > this.#assistantTranscriptTurn) {
this.#finalizeAssistantTranscript();
this.#assistantTranscriptTurn = transcript.turn;
}
let component = this.#assistantTranscriptComponent;
if (!component) {
component = createAssistantMessageComponent(this.#ctx);
component.setTextColorTransform(text => theme.fg("borderAccent", text));
this.#assistantTranscriptComponent = component;
this.#assistantTranscriptStartedAt = Date.now();
}
@@ -162,7 +172,6 @@ export class LiveCommandController {
timestamp: this.#assistantTranscriptStartedAt,
};
component.updateContent(message, { transient: !transcript.final });
if (component !== this.#assistantTranscriptComponent) return;
if (transcript.final) {
component.markTranscriptBlockFinalized();
this.#assistantTranscriptComponent = undefined;
packages/coding-agent/src/session/messages.ts
+3
View File
@@ -306,6 +306,9 @@ function normalizeSessionMessageForProviderReplay(message: AgentMessage): unknow
/** Fallback type for extension-injected messages that omit a custom type. */
export const DEFAULT_CUSTOM_MESSAGE_TYPE = "custom-message";
/** Custom message carrying a coding request delegated by the live voice model. */
export const LIVE_DELEGATION_MESSAGE_TYPE = "live-delegation";
/** Content shape accepted for extension-injected messages. */
export type CustomMessageContent = string | (TextContent | ImageContent)[];
packages/coding-agent/src/tts/streaming-player.ts
+17 -12
View File
@@ -22,17 +22,28 @@ export class StreamingAudioPlayer {
#ending: Promise<void> | null = null;
#inputClosed = false;
#stopped = false;
#failNative(native: AudioPlayback, cause: unknown): void {
this.#error = errorFrom(cause);
if (this.#native === native) this.#native = null;
try {
native.stop();
} catch {
// Preserve the original playback failure.
}
}
/** Opens the default speaker at the stream's logical sample rate. */
start(sampleRate = DEFAULT_SAMPLE_RATE): void {
if (this.#native || this.#error || this.#inputClosed || this.#stopped) return;
this.#sampleRate = sampleRate > 0 ? sampleRate : DEFAULT_SAMPLE_RATE;
let native: AudioPlayback | undefined;
try {
this.#native = new AudioPlayback(this.#sampleRate);
this.#native.setGain(this.#gain);
native = new AudioPlayback(this.#sampleRate);
native.setGain(this.#gain);
this.#native = native;
} catch (cause) {
this.#error = errorFrom(cause);
this.#native = null;
if (native) this.#failNative(native, cause);
else this.#error = errorFrom(cause);
}
}
@@ -45,13 +56,7 @@ export class StreamingAudioPlayer {
try {
native.write(pcm);
} catch (cause) {
this.#error = errorFrom(cause);
this.#native = null;
try {
native.stop();
} catch {
// Preserve the write failure surfaced by end().
}
this.#failNative(native, cause);
}
}
@@ -64,7 +69,7 @@ export class StreamingAudioPlayer {
try {
native.setGain(gain);
} catch (cause) {
this.#error = errorFrom(cause);
this.#failNative(native, cause);
}
}