import { afterEach, describe, expect, it, vi } from "bun:test"; import { type AssistantMessage, Effort } from "@oh-my-pi/pi-ai"; import { Settings } from "../../src/config/settings"; import type { LoadExtensionsResult } from "../../src/extensibility/extensions/types"; import type { CreateAgentSessionResult } from "../../src/sdk"; import * as sdkModule from "../../src/sdk"; import type { AgentSession, AgentSessionEvent, PromptOptions } from "../../src/session/agent-session"; import type { AuthStorage } from "../../src/session/auth-storage"; import { runSubprocess, SUBAGENT_WARNING_MISSING_SUBMIT_RESULT } from "../../src/task/executor"; import type { AgentDefinition } from "../../src/task/types"; import { EventBus } from "../../src/utils/event-bus"; function createAssistantStopMessage(text: string): AssistantMessage { return { role: "assistant", content: text ? [{ type: "text", text }] : [], api: "openai-responses", provider: "openai", model: "mock", usage: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, totalTokens: 0, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, }, stopReason: "stop", timestamp: Date.now(), }; } function createMockSession( onPrompt: (params: { text: string; options?: PromptOptions; promptIndex: number; emit: (event: AgentSessionEvent) => void; state: { messages: AssistantMessage[] }; }) => void, ): AgentSession { const listeners: Array<(event: AgentSessionEvent) => void> = []; const state = { messages: [] as AssistantMessage[] }; let promptIndex = 0; const emit = (event: AgentSessionEvent) => { for (const listener of listeners) listener(event); }; const session = { state, agent: { state: { systemPrompt: "test" } }, model: undefined, extensionRunner: undefined, sessionManager: { appendSessionInit: () => {}, }, getActiveToolNames: () => ["read", "submit_result"], setActiveToolsByName: async (_toolNames: string[]) => {}, subscribe: (listener: (event: AgentSessionEvent) => void) => { listeners.push(listener); return () => { const index = listeners.indexOf(listener); if (index >= 0) listeners.splice(index, 1); }; }, prompt: async (text: string, options?: PromptOptions) => { promptIndex += 1; onPrompt({ text, options, promptIndex, emit, state }); }, waitForIdle: async () => {}, getLastAssistantMessage: () => state.messages[state.messages.length - 1], abort: async () => {}, dispose: async () => {}, }; return session as unknown as AgentSession; } function createSessionResult(session: AgentSession): CreateAgentSessionResult { return { session, extensionsResult: {} as unknown as LoadExtensionsResult, setToolUIContext: () => {}, eventBus: new EventBus(), }; } function mockCreateAgentSession(session: AgentSession) { return vi.spyOn(sdkModule, "createAgentSession").mockResolvedValue(createSessionResult(session)); } describe("runSubprocess submit_result reminders", () => { afterEach(() => { vi.restoreAllMocks(); }); const baseAgent: AgentDefinition = { name: "task", description: "test", systemPrompt: "test", source: "bundled", }; const baseOptions = { cwd: "/tmp", agent: baseAgent, task: "do work", index: 0, id: "subagent-1", settings: Settings.isolated(), authStorage: {} as unknown as AuthStorage, modelRegistry: { refresh: async () => {} } as unknown as import("../../src/config/model-registry").ModelRegistry, enableLsp: false, }; it("sends reminder prompt when subagent stops without submit_result", async () => { const prompts: string[] = []; const promptOptions: Array = []; const session = createMockSession(({ text, options, promptIndex, emit, state }) => { prompts.push(text); promptOptions.push(options); if (promptIndex === 1) { const assistant = createAssistantStopMessage("did some work"); state.messages.push(assistant); emit({ type: "message_end", message: assistant }); return; } emit({ type: "tool_execution_end", toolCallId: "tool-1", toolName: "submit_result", result: { content: [{ type: "text", text: "Result submitted." }], details: { status: "success", data: { done: true } }, }, isError: false, }); }); mockCreateAgentSession(session); const result = await runSubprocess(baseOptions); expect(prompts.length).toBe(2); expect(promptOptions).toHaveLength(2); expect(promptOptions[0]?.attribution).toBe("agent"); expect(promptOptions[1]?.attribution).toBe("agent"); expect(prompts[1]).toContain("You stopped without calling submit_result"); expect(result.output).toContain('"done": true'); expect(result.output.includes("SYSTEM WARNING")).toBe(false); }); it("keeps null submit_result warning when subagent submits success without data", async () => { const session = createMockSession(({ promptIndex, emit, state }) => { if (promptIndex === 1) { const assistant = createAssistantStopMessage("partial output"); state.messages.push(assistant); emit({ type: "message_end", message: assistant }); return; } emit({ type: "tool_execution_end", toolCallId: "tool-2", toolName: "submit_result", result: { content: [{ type: "text", text: "Result submitted." }], details: { status: "success" }, }, isError: false, }); }); mockCreateAgentSession(session); const result = await runSubprocess({ ...baseOptions, id: "subagent-2" }); expect(result.output).toContain("SYSTEM WARNING: Subagent called submit_result with null data."); }); it("retries when submit_result tool returns an error before succeeding", async () => { const prompts: string[] = []; const session = createMockSession(({ text, promptIndex, emit, state }) => { prompts.push(text); if (promptIndex === 1) { const assistant = createAssistantStopMessage("attempted submit_result"); state.messages.push(assistant); emit({ type: "message_end", message: assistant }); emit({ type: "tool_execution_end", toolCallId: "tool-error", toolName: "submit_result", result: { content: [{ type: "text", text: "Output does not match schema" }], details: { status: "error", error: "Output does not match schema" }, }, isError: true, }); return; } emit({ type: "tool_execution_end", toolCallId: "tool-success", toolName: "submit_result", result: { content: [{ type: "text", text: "Result submitted." }], details: { status: "success", data: { ok: true } }, }, isError: false, }); }); mockCreateAgentSession(session); const result = await runSubprocess({ ...baseOptions, id: "subagent-err-then-success" }); expect(prompts).toHaveLength(2); expect(result.exitCode).toBe(0); expect(result.output).toContain('"ok": true'); }); it("uses provided thinking level when model override has no explicit suffix", async () => { vi.clearAllMocks(); const session = createMockSession(({ emit }) => { emit({ type: "tool_execution_end", toolCallId: "tool-thinking-fallback", toolName: "submit_result", result: { content: [{ type: "text", text: "Result submitted." }], details: { status: "success", data: { ok: true } }, }, isError: false, }); }); const createAgentSessionSpy = mockCreateAgentSession(session); const modelRegistry = { refresh: async () => {}, getAvailable: () => [{ provider: "openai", id: "gpt-4o", name: "GPT-4o" }], } as unknown as import("../../src/config/model-registry").ModelRegistry; await runSubprocess({ ...baseOptions, id: "subagent-thinking-fallback", modelOverride: "openai/gpt-4o", thinkingLevel: Effort.High, modelRegistry, }); expect(createAgentSessionSpy).toHaveBeenCalledTimes(1); expect(createAgentSessionSpy.mock.calls[0]?.[0]?.thinkingLevel).toBe(Effort.High); }); it("prefers explicit modelOverride thinking suffix over provided thinking level, including off", async () => { vi.clearAllMocks(); const modelRegistry = { refresh: async () => {}, getAvailable: () => [{ provider: "openai", id: "gpt-4o", name: "GPT-4o" }], } as unknown as import("../../src/config/model-registry").ModelRegistry; const cases = [ { modelOverride: "openai/gpt-4o:low", expectedThinkingLevel: Effort.Low }, { modelOverride: "openai/gpt-4o:off", expectedThinkingLevel: "off" }, ] as const; const createAgentSessionSpy = vi.spyOn(sdkModule, "createAgentSession"); for (const [index, testCase] of cases.entries()) { const session = createMockSession(({ emit }) => { emit({ type: "tool_execution_end", toolCallId: `tool-thinking-override-${index}`, toolName: "submit_result", result: { content: [{ type: "text", text: "Result submitted." }], details: { status: "success", data: { ok: true } }, }, isError: false, }); }); createAgentSessionSpy.mockResolvedValue(createSessionResult(session)); await runSubprocess({ ...baseOptions, id: `subagent-thinking-override-${index}`, modelOverride: testCase.modelOverride, thinkingLevel: Effort.High, modelRegistry, }); } expect(createAgentSessionSpy).toHaveBeenCalledTimes(2); expect(createAgentSessionSpy.mock.calls[0]?.[0]?.thinkingLevel).toBe(cases[0].expectedThinkingLevel); expect(createAgentSessionSpy.mock.calls[1]?.[0]?.thinkingLevel).toBe(cases[1].expectedThinkingLevel); }); it("aborts after 3 reminders when submit_result is never called", async () => { const prompts: string[] = []; const session = createMockSession(({ text, promptIndex, emit, state }) => { prompts.push(text); const assistant = createAssistantStopMessage(promptIndex === 1 ? "did work" : "still no submit_result"); state.messages.push(assistant); emit({ type: "message_end", message: assistant }); }); mockCreateAgentSession(session); const result = await runSubprocess({ ...baseOptions, id: "subagent-3", outputSchema: { type: "object", properties: { ok: { type: "boolean" } }, required: ["ok"] }, }); expect(prompts).toHaveLength(4); expect(result.exitCode).toBe(1); expect(result.aborted).toBe(true); expect(result.stderr).toBe(SUBAGENT_WARNING_MISSING_SUBMIT_RESULT); expect(result.abortReason).toBe(SUBAGENT_WARNING_MISSING_SUBMIT_RESULT); }); it("surfaces abort reason when submit_result reports aborted status", async () => { const session = createMockSession(({ promptIndex, emit, state }) => { if (promptIndex === 1) { const assistant = createAssistantStopMessage("cannot proceed"); state.messages.push(assistant); emit({ type: "message_end", message: assistant }); } emit({ type: "tool_execution_end", toolCallId: "tool-abort", toolName: "submit_result", result: { content: [{ type: "text", text: "Task aborted: blocked by permissions" }], details: { status: "aborted", error: "blocked by permissions" }, }, isError: false, }); }); mockCreateAgentSession(session); const result = await runSubprocess({ ...baseOptions, id: "subagent-aborted-submit-result" }); expect(result.aborted).toBe(true); expect(result.abortReason).toBe("blocked by permissions"); }); it("marks pre-aborted subprocess with a concrete reason", async () => { const abortController = new AbortController(); abortController.abort("caller cancelled task"); const result = await runSubprocess({ ...baseOptions, id: "subagent-cancelled-before-start", signal: abortController.signal, }); expect(result.aborted).toBe(true); expect(result.abortReason).toBe("Cancelled before start"); expect(result.stderr).toBe("Cancelled before start"); }); });