feat(coding-agent): wrapped mid-turn steers in wire-only envelope

- Marked steering user messages and wrapped them pre-LLM so the model sees a `` envelope.
- Kept transcripts and persisted history with the user's original raw text.
- Restored `AssistantMessageComponent` stable-prefix completion API.
This commit is contained in:
can1357
2026-06-04 17:41:43 +02:00
parent c8b4bf09c7
commit 4a9ee14e33
8 changed files with 263 additions and 5 deletions
+2
View File
@@ -579,6 +579,8 @@ export interface UserMessage {
content: string | (TextContent | ImageContent)[];
/** True if the message was injected by the system (e.g., auto-continue). */
synthetic?: boolean;
/** True when injected mid-turn as a steer; consumed by the agent's pre-LLM transform to wrap it for emphasis. Never rendered. */
steering?: boolean;
/** Who initiated this message for billing/attribution semantics. */
attribution?: MessageAttribution;
/** Provider-specific opaque payload used to reconstruct transport-native history. */
+2 -1
View File
@@ -8,6 +8,7 @@
### Changed
- Changed mid-turn user steers to reach the model inside a wire-only interjection envelope, while transcripts and persisted session history keep the user's original text.
- Changed the system prompt to treat user requests for parallel work as `task` subagent fan-out rather than parallel tool calls.
- Changed the Agent Control Center's new-agent description field to use the multiline TUI editor, with Enter inserting lines and Ctrl+Enter generating the spec.
- Changed the Agent Control Center and Extension Control Center to accept Left/Right arrow keys for switching tabs (source / provider), in addition to Tab / Shift+Tab — matching the model and settings selectors, whose `TabBar` already supported arrow navigation.
@@ -16,9 +17,9 @@
### Fixed
- Fixed `AssistantMessageComponent` exposing its stable-prefix completion API again so streamed assistant messages remain unstable until explicitly completed.
- Fixed session restoration to ignore transient fallback model switches (such as automatic context-promotion or retry fallback) so resumed or resumed-switch sessions revert to the configured default model unless the last change was a user-selected temporary model
- Fixed in-session `/resume` to restore both the last user-selected temporary model and persisted plan/goal mode state instead of falling back to the default model with plan mode off.
- Fixed transcript scrollback stability on terminals with eager erase risk so completed assistant messages remain stable while new streaming lines are rendering
- Fixed the `/resume` session picker overflowing short viewports: the visible window was hardcoded to 5 entries (and assumed 3 lines each), but titled sessions render 4 lines, so on a typical-height terminal the picker's header and search box scrolled off the top and the first entry was hidden until you scrolled the terminal up. The visible-entry count is now derived from the live terminal height (budgeting the worst-case 4-line titled entry plus the picker's chrome), so the whole picker fits the viewport and grows on taller terminals.
- Fixed the Agent Control Center and Extension Control Center dashboards overflowing the terminal: they were mounted inline below the chat transcript, so the combined height exceeded the viewport — the tab bar and controls scrolled off the top into native scrollback, and every state change yanked the view back to the bottom. Both dashboards now render as full-screen overlays sized to the live terminal height (`process.stdout.rows`), re-fit on resize, fill the viewport, and reserve space for the footer keyhints so the controls stay visible.
- Fixed Ctrl+R history search results to remain globally sorted by prompt recency after merging FTS prefix matches with substring fallback matches.
@@ -17,6 +17,7 @@ export class AssistantMessageComponent extends Container {
#usageInfo?: Usage;
#convertedKittyImages = new Map<string, ImageContent>();
#kittyConversionsInFlight = new Set<string>();
#complete = false;
constructor(
message?: AssistantMessage,
@@ -47,6 +48,14 @@ export class AssistantMessageComponent extends Container {
this.hideThinkingBlock = hide;
}
setComplete(): void {
this.#complete = true;
}
getStableLineCount(width: number): number {
return this.#complete ? this.render(width).length : 0;
}
setToolResultImages(toolCallId: string, images: ImageContent[]): void {
if (!toolCallId) return;
const validImages = images.filter(img => img.type === "image" && img.data && img.mimeType);
+3 -2
View File
@@ -102,7 +102,7 @@ import {
import { AgentSession } from "./session/agent-session";
import { resolveAuthBrokerConfig } from "./session/auth-broker-config";
import { AuthBrokerClient, AuthStorage, RemoteAuthCredentialStore } from "./session/auth-storage";
import { type CustomMessage, convertToLlm } from "./session/messages";
import { type CustomMessage, convertToLlm, wrapSteeringForModel } from "./session/messages";
import { getRestorableSessionModels, SessionManager } from "./session/session-manager";
import { closeAllConnections } from "./ssh/connection-manager";
import { unmountAll } from "./ssh/sshfs-mount";
@@ -1888,7 +1888,8 @@ export async function createAgentSession(options: CreateAgentSessionOptions = {}
return obfuscateMessages(obfuscator, converted);
};
const transformContext = async (messages: AgentMessage[], _signal?: AbortSignal) => {
return await extensionRunner.emitContext(messages);
const withContext = await extensionRunner.emitContext(messages);
return wrapSteeringForModel(withContext);
};
const onPayload = async (payload: unknown, _model?: Model) => {
return await extensionRunner.emitBeforeProviderRequest(payload);
@@ -4646,6 +4646,7 @@ export class AgentSession {
this.agent.steer({
role: "user",
content,
steering: true,
attribution: "user",
timestamp: Date.now(),
});
@@ -18,7 +18,10 @@ import type {
MessageAttribution,
TextContent,
ToolResultMessage,
UserMessage,
} from "@oh-my-pi/pi-ai";
import { prompt } from "@oh-my-pi/pi-utils";
import userInterjectionTemplate from "../prompts/steering/user-interjection.md" with { type: "text" };
export {
type BranchSummaryMessage,
@@ -105,6 +108,83 @@ export function stripInternalDetailsFields<T>(details: T | undefined): T | undef
return cleaned as T;
}
function isSteeringUserMessage(message: AgentMessage | undefined): message is UserMessage & { steering: true } {
return message?.role === "user" && message.steering === true;
}
function userMessageWithoutSteering(message: UserMessage): UserMessage {
const { steering, ...rest } = message;
void steering;
return rest;
}
function renderSteeringEnvelope(message: string): string {
return prompt.render(userInterjectionTemplate, { message });
}
function getArrayContentText(content: (TextContent | ImageContent)[]): string {
let firstText: string | undefined;
let textParts: string[] | undefined;
for (const part of content) {
if (part.type !== "text") continue;
if (firstText === undefined) {
firstText = part.text;
continue;
}
if (textParts === undefined) {
textParts = [firstText];
}
textParts.push(part.text);
}
return textParts === undefined ? (firstText ?? "") : textParts.join("\n");
}
function getArrayContentImages(content: (TextContent | ImageContent)[]): ImageContent[] {
let images: ImageContent[] | undefined;
for (const part of content) {
if (part.type !== "image") continue;
if (images === undefined) images = [];
images.push(part);
}
return images ?? [];
}
function wrapSteeringUserMessage(message: UserMessage): UserMessage {
if (typeof message.content === "string") {
if (message.content.length === 0) return message;
return { ...userMessageWithoutSteering(message), content: renderSteeringEnvelope(message.content) };
}
const text = getArrayContentText(message.content);
if (text.length === 0) return message;
const content: (TextContent | ImageContent)[] = [{ type: "text", text: renderSteeringEnvelope(text) }];
content.push(...getArrayContentImages(message.content));
return { ...userMessageWithoutSteering(message), content };
}
export function wrapSteeringForModel(messages: AgentMessage[]): AgentMessage[] {
const last = messages[messages.length - 1];
if (!isSteeringUserMessage(last)) return messages;
let firstSteer = messages.length - 1;
while (firstSteer > 0 && isSteeringUserMessage(messages[firstSteer - 1])) {
firstSteer--;
}
let wrappedMessages: AgentMessage[] | undefined;
for (let i = firstSteer; i < messages.length; i++) {
const message = messages[i];
if (!isSteeringUserMessage(message)) continue;
const wrappedMessage = wrapSteeringUserMessage(message);
if (wrappedMessage === message) continue;
if (wrappedMessages === undefined) {
wrappedMessages = messages.slice();
}
wrappedMessages[i] = wrappedMessage;
}
return wrappedMessages ?? messages;
}
function getPrunedToolResultContent(message: ToolResultMessage): (TextContent | ImageContent)[] {
if (message.prunedAt === undefined) {
return message.content;
@@ -6,6 +6,7 @@ import {
type Model,
registerCustomApi,
type SimpleStreamOptions,
type TextContent,
} from "@oh-my-pi/pi-ai";
import { AssistantMessageEventStream } from "@oh-my-pi/pi-ai/utils/event-stream";
import { Settings } from "@oh-my-pi/pi-coding-agent/config/settings";
@@ -15,6 +16,7 @@ import {
ANTHROPIC_TOOL_CALL_BATCH_CAP,
resolveToolCallBatchCapForModel,
} from "@oh-my-pi/pi-coding-agent/session/agent-session";
import { convertToLlm, wrapSteeringForModel } from "@oh-my-pi/pi-coding-agent/session/messages";
import { SessionManager } from "@oh-my-pi/pi-coding-agent/session/session-manager";
import { createAssistantMessage } from "./helpers/agent-session-setup";
@@ -28,6 +30,20 @@ function createAgent(): Agent {
});
}
function getConvertedUserText(message: Message | undefined): string {
if (message?.role !== "user") {
throw new Error("Expected converted user message");
}
if (typeof message.content === "string") {
return message.content;
}
const text = message.content.find((content): content is TextContent => content.type === "text");
if (!text) {
throw new Error("Expected converted text content");
}
return text.text;
}
describe("AgentSession message pipeline", () => {
const sessions: AgentSession[] = [];
@@ -115,6 +131,56 @@ describe("AgentSession message pipeline", () => {
expect(result).toEqual(convertedMessages);
});
it("marks queued user steers without changing the public queue text", async () => {
const session = new AgentSession({
agent: createAgent(),
sessionManager: SessionManager.inMemory(),
settings: Settings.isolated({ "compaction.enabled": false }),
modelRegistry: {} as never,
});
sessions.push(session);
await session.sendUserMessage("raw <steer> &", { deliverAs: "steer" });
expect(session.getQueuedMessages().steering).toEqual(["raw <steer> &"]);
const queued = session.agent.popLastSteer();
if (queued?.role !== "user") {
throw new Error("Expected queued user steer");
}
expect(queued.steering).toBe(true);
expect(queued.content).toEqual([{ type: "text", text: "raw <steer> &" }]);
session.clearQueue();
});
it("keeps stored steering text raw while pre-LLM conversion wraps it", async () => {
const session = new AgentSession({
agent: createAgent(),
sessionManager: SessionManager.inMemory(),
settings: Settings.isolated({ "compaction.enabled": false }),
modelRegistry: {} as never,
transformContext: wrapSteeringForModel,
convertToLlm,
});
sessions.push(session);
const raw: AgentMessage = {
role: "user",
content: [{ type: "text", text: "steer with <xml> & ampersand" }],
steering: true,
timestamp: 1,
};
session.agent.appendMessage(raw);
const converted = await session.convertMessagesToLlm(session.messages);
expect(session.messages[0]).toBe(raw);
expect(raw.content).toEqual([{ type: "text", text: "steer with <xml> & ampersand" }]);
const convertedText = getConvertedUserText(converted[0]);
expect(convertedText).toContain("<user_interjection>");
expect(convertedText).toContain("<message>\nsteer with <xml> & ampersand\n</message>");
expect(convertedText).not.toContain("&lt;xml&gt;");
expect(convertedText).not.toContain("&amp;");
});
it("composes session payload hooks into direct side-request options", async () => {
const sessionOnPayload = vi.fn(async (payload: unknown) => ({
...(payload as Record<string, unknown>),
@@ -1,8 +1,8 @@
import { describe, expect, it } from "bun:test";
import type { AgentMessage } from "@oh-my-pi/pi-agent-core";
import type { Message } from "@oh-my-pi/pi-ai";
import type { ImageContent, Message } from "@oh-my-pi/pi-ai";
import { inferCopilotInitiator } from "@oh-my-pi/pi-ai/providers/github-copilot-headers";
import { convertToLlm } from "@oh-my-pi/pi-coding-agent/session/messages";
import { convertToLlm, wrapSteeringForModel } from "@oh-my-pi/pi-coding-agent/session/messages";
function expectAttribution(message: Message | undefined, expected: "user" | "agent" | undefined): void {
expect(message).toBeDefined();
@@ -93,3 +93,101 @@ describe("convertToLlm custom message mapping", () => {
expect(inferCopilotInitiator(converted)).toBe("user");
});
});
function getUserText(message: AgentMessage | undefined): string {
expect(message).toBeDefined();
if (message?.role !== "user") {
throw new Error("Expected user message");
}
if (typeof message.content === "string") {
return message.content;
}
const text = message.content.find(content => content.type === "text");
if (!text) {
throw new Error("Expected text content");
}
return text.text;
}
describe("wrapSteeringForModel", () => {
it("wraps trailing steering text for the model without escaping user code", () => {
const rawText = "Use <tag> & keep it literal";
const message: AgentMessage = {
role: "user",
content: [{ type: "text", text: rawText }],
steering: true,
timestamp: 1,
};
const messages = [message];
const wrapped = wrapSteeringForModel(messages);
expect(wrapped).not.toBe(messages);
expect(wrapped[0]).not.toBe(message);
expect(message.content).toEqual([{ type: "text", text: rawText }]);
const wrappedText = getUserText(wrapped[0]);
expect(wrappedText).toContain("<user_interjection>");
expect(wrappedText).toContain("<message>\nUse <tag> & keep it literal\n</message>");
expect(wrappedText).not.toContain("&lt;tag&gt;");
expect(wrappedText).not.toContain("&amp;");
});
it("leaves buried steering messages unchanged", () => {
const buried: AgentMessage = {
role: "user",
content: "old steer",
steering: true,
timestamp: 1,
};
const later: AgentMessage = { role: "user", content: "later", timestamp: 2 };
const messages = [buried, later];
const wrapped = wrapSteeringForModel(messages);
expect(wrapped).toBe(messages);
expect(wrapped[0]).toBe(buried);
});
it("leaves trailing user messages without the steering marker unchanged", () => {
const message: AgentMessage = { role: "user", content: "plain user", timestamp: 1 };
const messages = [message];
const wrapped = wrapSteeringForModel(messages);
expect(wrapped).toBe(messages);
expect(wrapped[0]).toBe(message);
});
it("preserves images after the wrapped steering text", () => {
const image: ImageContent = { type: "image", data: "abc123", mimeType: "image/png" };
const message: AgentMessage = {
role: "user",
content: [{ type: "text", text: "look at this" }, image],
steering: true,
timestamp: 1,
};
const wrapped = wrapSteeringForModel([message]);
const wrappedMessage = wrapped[0];
if (wrappedMessage?.role !== "user" || typeof wrappedMessage.content === "string") {
throw new Error("Expected user array content");
}
expect(wrappedMessage.content[0]?.type).toBe("text");
expect(wrappedMessage.content[1]).toBe(image);
});
it("wraps every message in the trailing steering run", () => {
const first: AgentMessage = { role: "user", content: "first steer", steering: true, timestamp: 1 };
const second: AgentMessage = { role: "user", content: "second steer", steering: true, timestamp: 2 };
const messages = [first, second];
const wrapped = wrapSteeringForModel(messages);
expect(wrapped).not.toBe(messages);
expect(wrapped[0]).not.toBe(first);
expect(wrapped[1]).not.toBe(second);
expect(getUserText(wrapped[0])).toContain("<message>\nfirst steer\n</message>");
expect(getUserText(wrapped[1])).toContain("<message>\nsecond steer\n</message>");
});
});