From a52ed682c79a482784a238bd86ac104953793275 Mon Sep 17 00:00:00 2001 From: roboomp Date: Fri, 3 Jul 2026 16:44:12 +0000 Subject: [PATCH 1/2] fix(ai): separated codex orchestration usage - Added a Usage.orchestration sidecar for provider-side service tokens so Responses/Codex totals and costs stay accurate without inflating visible prompt input/cache buckets. - Updated Codex/WebSocket usage, session/status aggregates, and usage reporting to preserve orchestration-aware totals. - Added regressions for OpenAI Responses accounting, Codex WebSocket terminal usage, cost calculation, and session aggregation. Fixes #4469 --- packages/ai/CHANGELOG.md | 4 ++ packages/ai/src/providers/mock.ts | 13 +++-- .../src/providers/openai-codex-responses.ts | 11 +++- packages/ai/src/providers/openai-shared.ts | 31 +++++++++-- packages/ai/test/models-cost.test.ts | 32 ++++++++++++ packages/ai/test/openai-codex-stream.test.ts | 52 +++++++++++++++++++ packages/ai/test/usage-attribution.test.ts | 39 ++++++++++++-- packages/catalog/CHANGELOG.md | 4 ++ packages/catalog/src/models.ts | 7 +-- packages/catalog/src/types.ts | 19 +++++-- packages/coding-agent/CHANGELOG.md | 4 ++ .../src/eval/__tests__/budget-bridge.test.ts | 13 ++++- .../coding-agent/src/modes/acp/acp-agent.ts | 6 ++- .../modes/components/status-line/component.ts | 4 ++ .../modes/components/status-line/segments.ts | 9 ++-- .../src/modes/components/status-line/types.ts | 4 ++ .../coding-agent/src/session/agent-session.ts | 9 +++- .../src/session/session-entries.ts | 4 ++ .../src/session/session-manager.ts | 17 +++++- .../slash-commands/helpers/usage-report.ts | 3 ++ .../test/collab/guest-idle-reconciler.test.ts | 4 ++ .../test/core/js-workflow-helpers.test.ts | 4 ++ .../session-manager/usage-statistics.test.ts | 31 +++++++++++ .../test/status-line-cache-hit.test.ts | 4 ++ .../test/status-line-context-cache.test.ts | 4 ++ .../status-line-dispose-async-leak.test.ts | 4 ++ .../test/status-line-model.test.ts | 4 ++ .../test/status-line-overflow.test.ts | 8 +++ .../test/status-line-path.test.ts | 4 ++ .../status-line-pr-lookup-timeout.test.ts | 4 ++ .../test/status-line-settings-cache.test.ts | 4 ++ .../test/status-line-time-spent.test.ts | 8 +++ .../test/status-line-transparent.test.ts | 4 ++ .../test/status-line-usage-refresh.test.ts | 4 ++ .../test/status-line-usage.test.ts | 8 +++ 35 files changed, 355 insertions(+), 29 deletions(-) diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index 20ef357ef..3582cdbcd 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Fixed + +- Fixed OpenAI Responses/Codex orchestration token accounting so provider-side orchestration tokens stay billable and included in totals without appearing as ordinary uncached prompt input. ([#4469](https://github.com/can1357/oh-my-pi/issues/4469)) + ## [16.3.4] - 2026-07-03 ### Added diff --git a/packages/ai/src/providers/mock.ts b/packages/ai/src/providers/mock.ts index f79bb8d03..1e81292d2 100644 --- a/packages/ai/src/providers/mock.ts +++ b/packages/ai/src/providers/mock.ts @@ -446,10 +446,17 @@ function mergeUsage(partial?: Partial> & { cost?: Partial 0 && + Math.abs(reportedTotalTokens - reportedPrimaryTokens) <= + Math.abs(reportedTotalTokens - reportedWithSeparateOrchestration); + const orchestrationInputCached = Math.min(orchestrationInputTokens, orchestrationInputCachedTokens); + const orchestrationInput = Math.max(0, orchestrationInputTokens - orchestrationInputCached); const accounting = calculateOpenAIUsageAccounting({ - promptTokens: (usage.input_tokens ?? 0) + orchestrationInputTokens, - outputTokens: (usage.output_tokens ?? 0) + orchestrationOutputTokens, - cachedTokens: (details?.cached_tokens ?? usage.prompt_cache_hit_tokens ?? 0) + orchestrationInputCachedTokens, + promptTokens: Math.max(0, reportedInputTokens - (primaryIncludesOrchestration ? orchestrationInputTokens : 0)), + outputTokens: Math.max(0, reportedOutputTokens - (primaryIncludesOrchestration ? orchestrationOutputTokens : 0)), + cachedTokens: Math.max(0, reportedCachedTokens - (primaryIncludesOrchestration ? orchestrationInputCached : 0)), reasoningTokens: outputDetails?.reasoning_tokens ?? 0, cacheWriteOpenRouter: details?.cache_write_tokens ?? undefined, cacheWriteDeepSeek: usage.prompt_cache_miss_tokens ?? undefined, hasDeepSeekCacheHitAndMiss: usage.prompt_cache_hit_tokens !== undefined && usage.prompt_cache_miss_tokens !== undefined, }); + const orchestrationTotal = orchestrationInput + orchestrationInputCached + orchestrationOutputTokens; + if (orchestrationTotal > 0) { + accounting.orchestration = { + ...(orchestrationInput > 0 ? { input: orchestrationInput } : {}), + ...(orchestrationInputCached > 0 ? { cacheRead: orchestrationInputCached } : {}), + ...(orchestrationOutputTokens > 0 ? { output: orchestrationOutputTokens } : {}), + }; + accounting.totalTokens = reportedTotalTokens ?? accounting.totalTokens + orchestrationTotal; + } // Wholesale replacement must not drop provider-annotated extras (Copilot // premium-request accounting): the failed/cancelled paths throw right after diff --git a/packages/ai/test/models-cost.test.ts b/packages/ai/test/models-cost.test.ts index 875bc8baf..72db1bb96 100644 --- a/packages/ai/test/models-cost.test.ts +++ b/packages/ai/test/models-cost.test.ts @@ -71,6 +71,38 @@ describe("calculateCost", () => { expect(usage.cost.total).toBeCloseTo(2.18, 8); }); + it("prices provider orchestration tokens without changing visible usage buckets", () => { + const model = { + ...getBundledModel("openai", "gpt-4o-mini"), + cost: { + input: 1000, + output: 2000, + cacheRead: 500, + cacheWrite: 800, + }, + }; + const usage: Usage = { + input: 100, + output: 20, + cacheRead: 50, + cacheWrite: 10, + totalTokens: 250, + orchestration: { input: 25, output: 40, cacheRead: 5 }, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }; + + calculateCost(model, usage); + + expect(usage.input).toBe(100); + expect(usage.output).toBe(20); + expect(usage.cacheRead).toBe(50); + expect(usage.cost.input).toBeCloseTo(0.125, 8); + expect(usage.cost.output).toBeCloseTo(0.12, 8); + expect(usage.cost.cacheRead).toBeCloseTo(0.0275, 8); + expect(usage.cost.cacheWrite).toBeCloseTo(0.008, 8); + expect(usage.cost.total).toBeCloseTo(0.2805, 8); + }); + it("prices OpenAI Codex GPT models from the matching OpenAI catalog entry", () => { const openAIModel = getBundledModel("openai", "gpt-5.4"); const codexModel = getBundledModel("openai-codex", "gpt-5.4"); diff --git a/packages/ai/test/openai-codex-stream.test.ts b/packages/ai/test/openai-codex-stream.test.ts index 931604f05..c27c36893 100644 --- a/packages/ai/test/openai-codex-stream.test.ts +++ b/packages/ai/test/openai-codex-stream.test.ts @@ -860,6 +860,58 @@ describe("openai-codex streaming", () => { } }); + it("separates websocket terminal orchestration usage from prompt cache buckets", async () => { + const tempDir = TempDir.createSync("@pi-codex-stream-"); + setAgentDir(tempDir.path()); + const token = createCodexTestToken(); + + class UsageWebSocket extends MockWebSocket { + constructor(url: string, options?: { headers?: WsHeaders }) { + super(url, options); + this.scheduleOpen(); + } + + send(): void { + this.sendJson({ + type: "response.done", + response: { + id: "resp_usage", + status: "completed", + usage: { + input_tokens: 185_853, + output_tokens: 29, + total_tokens: 185_882, + input_tokens_details: { + cached_tokens: 180_224, + orchestration_input_tokens: 5_629, + orchestration_input_cached_tokens: 0, + }, + }, + }, + }); + } + } + global.WebSocket = UsageWebSocket as unknown as typeof WebSocket; + + const model = { + ...createCodexTestModel("https://chatgpt.com/backend-api"), + cost: { input: 1000, output: 2000, cacheRead: 500, cacheWrite: 0 }, + }; + const result = await streamOpenAICodexResponses(model, createCodexTestContext(), { + apiKey: token, + sessionId: "ws-orchestration-usage-session", + providerSessionState: new Map(), + }).result(); + + expect(result.usage.input).toBe(0); + expect(result.usage.cacheRead).toBe(180_224); + expect(result.usage.output).toBe(29); + expect(result.usage.orchestration).toEqual({ input: 5_629 }); + expect(result.usage.totalTokens).toBe(185_882); + expect(result.usage.cost.input).toBeCloseTo(5.629, 8); + expect(result.usage.cost.cacheRead).toBeCloseTo(90.112, 8); + }); + it("omits request-body headers and replaces stale beta headers for websocket handshakes", async () => { const tempDir = TempDir.createSync("@pi-codex-stream-"); setAgentDir(tempDir.path()); diff --git a/packages/ai/test/usage-attribution.test.ts b/packages/ai/test/usage-attribution.test.ts index de03e870f..44d4c5dd8 100644 --- a/packages/ai/test/usage-attribution.test.ts +++ b/packages/ai/test/usage-attribution.test.ts @@ -261,7 +261,7 @@ describe("shared OpenAI usage accounting", () => { }); describe("openai-responses usage attribution", () => { - it("folds Fugu Ultra orchestration token details into billable usage", () => { + it("separates Responses orchestration tokens from conversation usage", () => { const output: AssistantMessage = { role: "assistant", content: [], @@ -287,11 +287,42 @@ describe("openai-responses usage attribution", () => { }, }); - expect(output.usage.input).toBe(135); - expect(output.usage.cacheRead).toBe(15); - expect(output.usage.output).toBe(120); + expect(output.usage.input).toBe(110); + expect(output.usage.cacheRead).toBe(10); + expect(output.usage.output).toBe(80); + expect(output.usage.orchestration).toEqual({ input: 25, cacheRead: 5, output: 40 }); expect(output.usage.totalTokens).toBe(270); }); + + it("does not label Codex orchestration input as an uncached prompt miss when primary totals include it", () => { + const output: AssistantMessage = { + role: "assistant", + content: [], + api: "openai-codex-responses", + provider: "openai-codex", + model: "gpt-5.5", + usage: blankUsage(), + stopReason: "toolUse", + timestamp: 0, + }; + + populateResponsesUsageFromResponse(output, { + input_tokens: 185_853, + output_tokens: 29, + total_tokens: 185_882, + input_tokens_details: { + cached_tokens: 180_224, + orchestration_input_tokens: 5_629, + orchestration_input_cached_tokens: 0, + }, + }); + + expect(output.usage.input).toBe(0); + expect(output.usage.cacheRead).toBe(180_224); + expect(output.usage.output).toBe(29); + expect(output.usage.orchestration).toEqual({ input: 5_629 }); + expect(output.usage.totalTokens).toBe(185_882); + }); }); describe("anthropic applyAnthropicUsageExtras", () => { diff --git a/packages/catalog/CHANGELOG.md b/packages/catalog/CHANGELOG.md index 3370361e7..2465288c8 100644 --- a/packages/catalog/CHANGELOG.md +++ b/packages/catalog/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Fixed + +- Fixed usage cost calculation to include provider orchestration token sidecars without forcing those tokens into normal input/output/cache buckets. ([#4469](https://github.com/can1357/oh-my-pi/issues/4469)) + ## [16.3.4] - 2026-07-03 ### Added diff --git a/packages/catalog/src/models.ts b/packages/catalog/src/models.ts index 363ac1b6f..2130bbdb8 100644 --- a/packages/catalog/src/models.ts +++ b/packages/catalog/src/models.ts @@ -44,9 +44,10 @@ export function getBundledModels(provider: GeneratedProvider): Model[] { } export function calculateCost(model: Model, usage: Usage): Usage["cost"] { - usage.cost.input = (model.cost.input / 1000000) * usage.input; - usage.cost.output = (model.cost.output / 1000000) * usage.output; - usage.cost.cacheRead = (model.cost.cacheRead / 1000000) * usage.cacheRead; + const orchestration = usage.orchestration; + usage.cost.input = (model.cost.input / 1000000) * (usage.input + (orchestration?.input ?? 0)); + usage.cost.output = (model.cost.output / 1000000) * (usage.output + (orchestration?.output ?? 0)); + usage.cost.cacheRead = (model.cost.cacheRead / 1000000) * (usage.cacheRead + (orchestration?.cacheRead ?? 0)); usage.cost.cacheWrite = (model.cost.cacheWrite / 1000000) * usage.cacheWrite; usage.cost.total = usage.cost.input + usage.cost.output + usage.cost.cacheRead + usage.cost.cacheWrite; return usage.cost; diff --git a/packages/catalog/src/types.ts b/packages/catalog/src/types.ts index 794024dcb..103549d0f 100644 --- a/packages/catalog/src/types.ts +++ b/packages/catalog/src/types.ts @@ -93,16 +93,25 @@ export type Provider = string; export type ThinkingBudgets = { [key in Effort]?: number }; export interface Usage { - /** Non-cached input tokens (matches the bucket the provider bills as new input). */ + /** Non-cached conversation input tokens (matches the bucket the provider bills as new input). */ input: number; - /** Total output tokens for the turn, including thinking, assistant text, and tool-call argument tokens. */ + /** Total conversation output tokens for the turn, including thinking, assistant text, and tool-call argument tokens. */ output: number; - /** Tokens read from the prompt cache. */ + /** Conversation tokens read from the prompt cache. */ cacheRead: number; - /** Tokens written to the prompt cache (cache creation). */ + /** Conversation tokens written to the prompt cache (cache creation). */ cacheWrite: number; - /** Sum of input + output + cacheRead + cacheWrite. */ + /** Sum of input + output + cacheRead + cacheWrite plus provider-side orchestration tokens when reported. */ totalTokens: number; + /** Provider-side orchestration tokens, billed but not part of the conversation prompt/cache buckets. */ + orchestration?: { + /** Non-cached orchestration input tokens. */ + input?: number; + /** Orchestration tokens read from provider-side cache. */ + cacheRead?: number; + /** Orchestration output tokens. */ + output?: number; + }; /** Copilot premium-request counter, when applicable. */ premiumRequests?: number; /** diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 860266c53..9f7203d47 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Fixed + +- Fixed session/status usage totals to preserve provider-reported orchestration tokens separately from ordinary input and cache-hit buckets. ([#4469](https://github.com/can1357/oh-my-pi/issues/4469)) + ## [16.3.4] - 2026-07-03 ### Fixed diff --git a/packages/coding-agent/src/eval/__tests__/budget-bridge.test.ts b/packages/coding-agent/src/eval/__tests__/budget-bridge.test.ts index 9e3af45e3..37e773779 100644 --- a/packages/coding-agent/src/eval/__tests__/budget-bridge.test.ts +++ b/packages/coding-agent/src/eval/__tests__/budget-bridge.test.ts @@ -23,7 +23,18 @@ function goalState(extra: Partial): GoalModeState { } function usage(output: number): UsageStatistics { - return { input: 0, output, cacheRead: 0, cacheWrite: 0, premiumRequests: 0, cost: 0 }; + return { + input: 0, + output, + cacheRead: 0, + cacheWrite: 0, + totalTokens: output, + orchestrationInput: 0, + orchestrationOutput: 0, + orchestrationCacheRead: 0, + premiumRequests: 0, + cost: 0, + }; } describe("runEvalBudget", () => { diff --git a/packages/coding-agent/src/modes/acp/acp-agent.ts b/packages/coding-agent/src/modes/acp/acp-agent.ts index 66797ae1a..8d4083c6c 100644 --- a/packages/coding-agent/src/modes/acp/acp-agent.ts +++ b/packages/coding-agent/src/modes/acp/acp-agent.ts @@ -1823,6 +1823,10 @@ export class AcpAgent implements Agent { output: usage.output, cacheRead: usage.cacheRead, cacheWrite: usage.cacheWrite, + totalTokens: usage.totalTokens, + orchestrationInput: usage.orchestrationInput, + orchestrationOutput: usage.orchestrationOutput, + orchestrationCacheRead: usage.orchestrationCacheRead, premiumRequests: usage.premiumRequests, cost: usage.cost, }; @@ -1833,7 +1837,7 @@ export class AcpAgent implements Agent { const outputTokens = Math.max(0, current.output - previous.output); const cachedReadTokens = Math.max(0, current.cacheRead - previous.cacheRead); const cachedWriteTokens = Math.max(0, current.cacheWrite - previous.cacheWrite); - const totalTokens = inputTokens + outputTokens + cachedReadTokens + cachedWriteTokens; + const totalTokens = Math.max(0, current.totalTokens - previous.totalTokens); if (totalTokens === 0) { return undefined; diff --git a/packages/coding-agent/src/modes/components/status-line/component.ts b/packages/coding-agent/src/modes/components/status-line/component.ts index 0d3554b7f..380e0d5e6 100644 --- a/packages/coding-agent/src/modes/components/status-line/component.ts +++ b/packages/coding-agent/src/modes/components/status-line/component.ts @@ -998,6 +998,10 @@ export class StatusLineComponent implements Component { output: 0, cacheRead: 0, cacheWrite: 0, + totalTokens: 0, + orchestrationInput: 0, + orchestrationOutput: 0, + orchestrationCacheRead: 0, premiumRequests: 0, cost: 0, }; diff --git a/packages/coding-agent/src/modes/components/status-line/segments.ts b/packages/coding-agent/src/modes/components/status-line/segments.ts index 0b1487a7d..05403bdb2 100644 --- a/packages/coding-agent/src/modes/components/status-line/segments.ts +++ b/packages/coding-agent/src/modes/components/status-line/segments.ts @@ -362,10 +362,11 @@ const tokenTotalSegment: StatusLineSegment = { id: "token_total", render(ctx) { // Excludes cacheRead: that field re-reads the full cached context every - // turn, making the cumulative sum N×context_size. The dedicated cache_read - // segment handles cache monitoring; the cost segment handles billing. - const { input, output, cacheWrite } = ctx.usageStats; - const total = input + output + cacheWrite; + // turn, making the cumulative sum N×context_size. Orchestration cache read + // follows the same rule; orchestration input/output remain in the total so + // provider-side service work is preserved without labeling it prompt input. + const { input, output, cacheWrite, orchestrationInput, orchestrationOutput } = ctx.usageStats; + const total = input + output + cacheWrite + orchestrationInput + orchestrationOutput; if (!total) return { content: "", visible: false }; const content = withIcon(theme.icon.tokens, formatNumber(total)); diff --git a/packages/coding-agent/src/modes/components/status-line/types.ts b/packages/coding-agent/src/modes/components/status-line/types.ts index ab8a71b95..ae02e7168 100644 --- a/packages/coding-agent/src/modes/components/status-line/types.ts +++ b/packages/coding-agent/src/modes/components/status-line/types.ts @@ -74,6 +74,10 @@ export interface SegmentContext { output: number; cacheRead: number; cacheWrite: number; + totalTokens: number; + orchestrationInput: number; + orchestrationOutput: number; + orchestrationCacheRead: number; premiumRequests: number; cost: number; tokensPerSecond: number | null; diff --git a/packages/coding-agent/src/session/agent-session.ts b/packages/coding-agent/src/session/agent-session.ts index f271f21bf..06a85d16b 100644 --- a/packages/coding-agent/src/session/agent-session.ts +++ b/packages/coding-agent/src/session/agent-session.ts @@ -14869,6 +14869,7 @@ export class AgentSession { let totalCacheRead = 0; let totalReasoning = 0; let totalCacheWrite = 0; + let totalTokens = 0; let totalCost = 0; let totalPremiumRequests = 0; @@ -14889,6 +14890,7 @@ export class AgentSession { totalReasoning += assistantMsg.usage.reasoningTokens ?? 0; totalCacheRead += assistantMsg.usage.cacheRead; totalCacheWrite += assistantMsg.usage.cacheWrite; + totalTokens += assistantMsg.usage.totalTokens; totalPremiumRequests += assistantMsg.usage.premiumRequests ?? 0; totalCost += assistantMsg.usage.cost.total; } @@ -14901,6 +14903,7 @@ export class AgentSession { totalReasoning += usage.reasoningTokens ?? 0; totalCacheRead += usage.cacheRead; totalCacheWrite += usage.cacheWrite; + totalTokens += usage.totalTokens; totalPremiumRequests += usage.premiumRequests ?? 0; totalCost += usage.cost.total; } @@ -14921,7 +14924,7 @@ export class AgentSession { reasoning: totalReasoning, cacheRead: totalCacheRead, cacheWrite: totalCacheWrite, - total: totalInput + totalOutput + totalCacheRead + totalCacheWrite, + total: totalTokens, }, cost: totalCost, premiumRequests: totalPremiumRequests, @@ -15567,6 +15570,7 @@ export class AgentSession { let reasoning = 0; let cacheRead = 0; let cacheWrite = 0; + let totalTokens = 0; let cost = 0; let user = 0; let assistant = 0; @@ -15580,6 +15584,7 @@ export class AgentSession { reasoning += assistantMsg.usage.reasoningTokens ?? 0; cacheRead += assistantMsg.usage.cacheRead; cacheWrite += assistantMsg.usage.cacheWrite; + totalTokens += assistantMsg.usage.totalTokens; cost += assistantMsg.usage.cost.total; } } @@ -15588,7 +15593,7 @@ export class AgentSession { model, contextWindow: model.contextWindow ?? 0, contextTokens, - tokens: { input, output, reasoning, cacheRead, cacheWrite, total: input + output + cacheRead + cacheWrite }, + tokens: { input, output, reasoning, cacheRead, cacheWrite, total: totalTokens }, cost, messages: { user, assistant, total: messages.length }, }; diff --git a/packages/coding-agent/src/session/session-entries.ts b/packages/coding-agent/src/session/session-entries.ts index 830062697..bccfb9624 100644 --- a/packages/coding-agent/src/session/session-entries.ts +++ b/packages/coding-agent/src/session/session-entries.ts @@ -236,6 +236,10 @@ export interface UsageStatistics { output: number; cacheRead: number; cacheWrite: number; + totalTokens: number; + orchestrationInput: number; + orchestrationOutput: number; + orchestrationCacheRead: number; premiumRequests: number; cost: number; } diff --git a/packages/coding-agent/src/session/session-manager.ts b/packages/coding-agent/src/session/session-manager.ts index 896255836..aabb96d55 100644 --- a/packages/coding-agent/src/session/session-manager.ts +++ b/packages/coding-agent/src/session/session-manager.ts @@ -114,7 +114,18 @@ function resolveBreadcrumbToInteractiveRoot(sessionFile: string): string { } function emptyUsageStatistics(): UsageStatistics { - return { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, premiumRequests: 0, cost: 0 }; + return { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + orchestrationInput: 0, + orchestrationOutput: 0, + orchestrationCacheRead: 0, + premiumRequests: 0, + cost: 0, + }; } function taskUsageFrom(details: unknown): Usage | undefined { @@ -137,6 +148,10 @@ function addUsage(target: UsageStatistics, usage: Usage | undefined): void { target.output += usage.output; target.cacheRead += usage.cacheRead; target.cacheWrite += usage.cacheWrite; + target.totalTokens += usage.totalTokens; + target.orchestrationInput += usage.orchestration?.input ?? 0; + target.orchestrationOutput += usage.orchestration?.output ?? 0; + target.orchestrationCacheRead += usage.orchestration?.cacheRead ?? 0; target.premiumRequests += usage.premiumRequests ?? 0; target.cost += usage.cost.total; } diff --git a/packages/coding-agent/src/slash-commands/helpers/usage-report.ts b/packages/coding-agent/src/slash-commands/helpers/usage-report.ts index 55f61df06..120f57385 100644 --- a/packages/coding-agent/src/slash-commands/helpers/usage-report.ts +++ b/packages/coding-agent/src/slash-commands/helpers/usage-report.ts @@ -150,12 +150,15 @@ export async function buildUsageReportText(runtime: SlashCommandRuntime): Promis } const stats = runtime.session.sessionManager.getUsageStatistics(); + const orchestrationTokens = stats.orchestrationInput + stats.orchestrationOutput + stats.orchestrationCacheRead; return [ "Usage", `Input tokens: ${stats.input}`, `Output tokens: ${stats.output}`, `Cache read tokens: ${stats.cacheRead}`, `Cache write tokens: ${stats.cacheWrite}`, + `Total tokens: ${stats.totalTokens}`, + ...(orchestrationTokens > 0 ? [`Orchestration tokens: ${orchestrationTokens}`] : []), `Premium requests: ${stats.premiumRequests}`, `Cost: $${stats.cost.toFixed(6)}`, ].join("\n"); diff --git a/packages/coding-agent/test/collab/guest-idle-reconciler.test.ts b/packages/coding-agent/test/collab/guest-idle-reconciler.test.ts index 5f8385490..a8e8a6fec 100644 --- a/packages/coding-agent/test/collab/guest-idle-reconciler.test.ts +++ b/packages/coding-agent/test/collab/guest-idle-reconciler.test.ts @@ -73,6 +73,10 @@ function makeSession(): ConstructorParameters[0] { output: 0, cacheRead: 0, cacheWrite: 0, + totalTokens: 0, + orchestrationInput: 0, + orchestrationOutput: 0, + orchestrationCacheRead: 0, premiumRequests: 0, cost: 0, }), diff --git a/packages/coding-agent/test/core/js-workflow-helpers.test.ts b/packages/coding-agent/test/core/js-workflow-helpers.test.ts index 9beb785b9..84ac9eaef 100644 --- a/packages/coding-agent/test/core/js-workflow-helpers.test.ts +++ b/packages/coding-agent/test/core/js-workflow-helpers.test.ts @@ -89,6 +89,10 @@ describe("executeJs workflow helpers", () => { output: 777, cacheRead: 0, cacheWrite: 0, + totalTokens: 787, + orchestrationInput: 0, + orchestrationOutput: 0, + orchestrationCacheRead: 0, premiumRequests: 0, cost: 0, }), diff --git a/packages/coding-agent/test/session-manager/usage-statistics.test.ts b/packages/coding-agent/test/session-manager/usage-statistics.test.ts index 0881f2ebc..85cf27940 100644 --- a/packages/coding-agent/test/session-manager/usage-statistics.test.ts +++ b/packages/coding-agent/test/session-manager/usage-statistics.test.ts @@ -50,6 +50,37 @@ describe("SessionManager usage statistics", () => { expect(usage.premiumRequests).toBe(3); }); + it("keeps orchestration usage out of ordinary input while preserving total tokens", () => { + const session = SessionManager.inMemory(); + + session.appendMessage({ role: "user", content: "hello", timestamp: 1 }); + session.appendMessage({ + role: "assistant", + content: [{ type: "text", text: "" }], + api: "openai-codex-responses", + provider: "openai-codex", + model: "gpt-5.5", + usage: { + input: 0, + output: 29, + cacheRead: 180_224, + cacheWrite: 0, + totalTokens: 185_882, + orchestration: { input: 5_629 }, + cost: { input: 5.629, output: 0, cacheRead: 0, cacheWrite: 0, total: 5.629 }, + }, + stopReason: "toolUse", + timestamp: 2, + }); + + const usage = session.getUsageStatistics(); + expect(usage.input).toBe(0); + expect(usage.cacheRead).toBe(180_224); + expect(usage.totalTokens).toBe(185_882); + expect(usage.orchestrationInput).toBe(5_629); + expect(usage.cost).toBeCloseTo(5.629, 8); + }); + it("preserves fractional premium request multipliers", () => { const session = SessionManager.inMemory(); diff --git a/packages/coding-agent/test/status-line-cache-hit.test.ts b/packages/coding-agent/test/status-line-cache-hit.test.ts index ad8fc950d..4ac06b8ed 100644 --- a/packages/coding-agent/test/status-line-cache-hit.test.ts +++ b/packages/coding-agent/test/status-line-cache-hit.test.ts @@ -15,6 +15,10 @@ function ctxWith(usage: Partial): SegmentContext { output: 0, cacheRead: 0, cacheWrite: 0, + totalTokens: 0, + orchestrationInput: 0, + orchestrationOutput: 0, + orchestrationCacheRead: 0, premiumRequests: 0, cost: 0, tokensPerSecond: null, diff --git a/packages/coding-agent/test/status-line-context-cache.test.ts b/packages/coding-agent/test/status-line-context-cache.test.ts index bac25f39a..c465653c4 100644 --- a/packages/coding-agent/test/status-line-context-cache.test.ts +++ b/packages/coding-agent/test/status-line-context-cache.test.ts @@ -58,6 +58,10 @@ function makeSession(opts: { messages: unknown[]; contextWindow?: number; usage? output: 0, cacheRead: 0, cacheWrite: 0, + totalTokens: 0, + orchestrationInput: 0, + orchestrationOutput: 0, + orchestrationCacheRead: 0, premiumRequests: 0, cost: 0, }), diff --git a/packages/coding-agent/test/status-line-dispose-async-leak.test.ts b/packages/coding-agent/test/status-line-dispose-async-leak.test.ts index 53f675528..a167a1ea2 100644 --- a/packages/coding-agent/test/status-line-dispose-async-leak.test.ts +++ b/packages/coding-agent/test/status-line-dispose-async-leak.test.ts @@ -66,6 +66,10 @@ function makeSession() { output: 0, cacheRead: 0, cacheWrite: 0, + totalTokens: 0, + orchestrationInput: 0, + orchestrationOutput: 0, + orchestrationCacheRead: 0, premiumRequests: 0, cost: 0, }), diff --git a/packages/coding-agent/test/status-line-model.test.ts b/packages/coding-agent/test/status-line-model.test.ts index 03ba9ce85..bbf8f1bcd 100644 --- a/packages/coding-agent/test/status-line-model.test.ts +++ b/packages/coding-agent/test/status-line-model.test.ts @@ -29,6 +29,10 @@ function createModelContext(advisorActive: boolean): SegmentContext { output: 0, cacheRead: 0, cacheWrite: 0, + totalTokens: 0, + orchestrationInput: 0, + orchestrationOutput: 0, + orchestrationCacheRead: 0, premiumRequests: 0, cost: 0, tokensPerSecond: null, diff --git a/packages/coding-agent/test/status-line-overflow.test.ts b/packages/coding-agent/test/status-line-overflow.test.ts index 16f543c2e..fd6307971 100644 --- a/packages/coding-agent/test/status-line-overflow.test.ts +++ b/packages/coding-agent/test/status-line-overflow.test.ts @@ -52,6 +52,10 @@ function createCtx(overrides?: { pathMaxLength?: number; branch?: string | null output: 0, cacheRead: 0, cacheWrite: 0, + totalTokens: 0, + orchestrationInput: 0, + orchestrationOutput: 0, + orchestrationCacheRead: 0, premiumRequests: 0, cost: 0, tokensPerSecond: null, @@ -87,6 +91,10 @@ function createStatusLineSession(sessionName: string) { output: 0, cacheRead: 0, cacheWrite: 0, + totalTokens: 0, + orchestrationInput: 0, + orchestrationOutput: 0, + orchestrationCacheRead: 0, premiumRequests: 0, cost: 0, }), diff --git a/packages/coding-agent/test/status-line-path.test.ts b/packages/coding-agent/test/status-line-path.test.ts index 3c8a6b431..587ec20bb 100644 --- a/packages/coding-agent/test/status-line-path.test.ts +++ b/packages/coding-agent/test/status-line-path.test.ts @@ -38,6 +38,10 @@ function createPathContext(): SegmentContext { output: 0, cacheRead: 0, cacheWrite: 0, + totalTokens: 0, + orchestrationInput: 0, + orchestrationOutput: 0, + orchestrationCacheRead: 0, premiumRequests: 0, cost: 0, tokensPerSecond: null, diff --git a/packages/coding-agent/test/status-line-pr-lookup-timeout.test.ts b/packages/coding-agent/test/status-line-pr-lookup-timeout.test.ts index 3420fc4c8..312f905b5 100644 --- a/packages/coding-agent/test/status-line-pr-lookup-timeout.test.ts +++ b/packages/coding-agent/test/status-line-pr-lookup-timeout.test.ts @@ -69,6 +69,10 @@ function makeSession() { output: 0, cacheRead: 0, cacheWrite: 0, + totalTokens: 0, + orchestrationInput: 0, + orchestrationOutput: 0, + orchestrationCacheRead: 0, premiumRequests: 0, cost: 0, }), diff --git a/packages/coding-agent/test/status-line-settings-cache.test.ts b/packages/coding-agent/test/status-line-settings-cache.test.ts index 847e03ad1..4113da1b6 100644 --- a/packages/coding-agent/test/status-line-settings-cache.test.ts +++ b/packages/coding-agent/test/status-line-settings-cache.test.ts @@ -57,6 +57,10 @@ function makeSession(sessionName = "Cache Session") { output: 0, cacheRead: 0, cacheWrite: 0, + totalTokens: 0, + orchestrationInput: 0, + orchestrationOutput: 0, + orchestrationCacheRead: 0, premiumRequests: 0, cost: 0, }), diff --git a/packages/coding-agent/test/status-line-time-spent.test.ts b/packages/coding-agent/test/status-line-time-spent.test.ts index 19def6cea..230434fcc 100644 --- a/packages/coding-agent/test/status-line-time-spent.test.ts +++ b/packages/coding-agent/test/status-line-time-spent.test.ts @@ -48,6 +48,10 @@ function createCtx(activeMs: number): SegmentContext { output: 0, cacheRead: 0, cacheWrite: 0, + totalTokens: 0, + orchestrationInput: 0, + orchestrationOutput: 0, + orchestrationCacheRead: 0, premiumRequests: 0, cost: 0, tokensPerSecond: null, @@ -95,6 +99,10 @@ function makeSession( output: 0, cacheRead: 0, cacheWrite: 0, + totalTokens: 0, + orchestrationInput: 0, + orchestrationOutput: 0, + orchestrationCacheRead: 0, premiumRequests: 0, cost: 0, }), diff --git a/packages/coding-agent/test/status-line-transparent.test.ts b/packages/coding-agent/test/status-line-transparent.test.ts index 0c5c73424..559a988a6 100644 --- a/packages/coding-agent/test/status-line-transparent.test.ts +++ b/packages/coding-agent/test/status-line-transparent.test.ts @@ -40,6 +40,10 @@ function makeSession() { output: 0, cacheRead: 0, cacheWrite: 0, + totalTokens: 0, + orchestrationInput: 0, + orchestrationOutput: 0, + orchestrationCacheRead: 0, premiumRequests: 0, cost: 0, }), diff --git a/packages/coding-agent/test/status-line-usage-refresh.test.ts b/packages/coding-agent/test/status-line-usage-refresh.test.ts index 2488638dd..161a19e39 100644 --- a/packages/coding-agent/test/status-line-usage-refresh.test.ts +++ b/packages/coding-agent/test/status-line-usage-refresh.test.ts @@ -24,6 +24,10 @@ function makeSession(fetchUsageReports: (signal?: AbortSignal) => Promise { output: 0, cacheRead: 0, cacheWrite: 0, + totalTokens: 0, + orchestrationInput: 0, + orchestrationOutput: 0, + orchestrationCacheRead: 0, premiumRequests: 0, cost: 0, }), From 38454d33210940d2a616b759b5d59e919920466f Mon Sep 17 00:00:00 2001 From: roboomp Date: Fri, 3 Jul 2026 16:53:11 +0000 Subject: [PATCH 2/2] fix(agent): excluded orchestration tokens from context sizing calculateContextTokens returned usage.totalTokens which, with the new Usage.orchestration sidecar, folds provider-side orchestration back into the context size used by auto-compaction/context promotion thresholds. Subtract the orchestration sidecar so context sizing stays conversation-only while cost and totalTokens keep the orchestration spend visible. Refs #4469 --- packages/agent/CHANGELOG.md | 4 ++ packages/agent/src/compaction/compaction.ts | 10 ++++- .../test/context-tokens-orchestration.test.ts | 37 +++++++++++++++++++ 3 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 packages/agent/test/context-tokens-orchestration.test.ts diff --git a/packages/agent/CHANGELOG.md b/packages/agent/CHANGELOG.md index 4cac16f77..2e031c82d 100644 --- a/packages/agent/CHANGELOG.md +++ b/packages/agent/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Fixed + +- Fixed `calculateContextTokens` including provider orchestration tokens in context sizing, which could trigger premature auto-compaction and context promotion on Codex/Fugu turns with sizable provider-side orchestration. ([#4469](https://github.com/can1357/oh-my-pi/issues/4469)) + ## [16.3.3] - 2026-07-02 ### Changed diff --git a/packages/agent/src/compaction/compaction.ts b/packages/agent/src/compaction/compaction.ts index 3c7e6ecbd..9f502bac5 100644 --- a/packages/agent/src/compaction/compaction.ts +++ b/packages/agent/src/compaction/compaction.ts @@ -204,9 +204,17 @@ export const DEFAULT_COMPACTION_SETTINGS: CompactionSettings = { /** * Calculate total context tokens from usage. * Uses the native totalTokens field when available, falls back to computing from components. + * Provider-side orchestration tokens are billable but never replay into the + * conversation prefix, so they are excluded from context sizing to keep + * auto-compaction and context-promotion thresholds honest. */ export function calculateContextTokens(usage: Usage): number { - return usage.totalTokens || usage.input + usage.output + usage.cacheRead + usage.cacheWrite; + const orchestration = usage.orchestration; + const orchestrationTotal = orchestration + ? (orchestration.input ?? 0) + (orchestration.output ?? 0) + (orchestration.cacheRead ?? 0) + : 0; + const raw = usage.totalTokens || usage.input + usage.output + usage.cacheRead + usage.cacheWrite; + return Math.max(0, raw - orchestrationTotal); } export function calculatePromptTokens(usage: Usage): number { diff --git a/packages/agent/test/context-tokens-orchestration.test.ts b/packages/agent/test/context-tokens-orchestration.test.ts new file mode 100644 index 000000000..f38333021 --- /dev/null +++ b/packages/agent/test/context-tokens-orchestration.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, it } from "bun:test"; +import { calculateContextTokens, calculatePromptTokens } from "@oh-my-pi/pi-agent-core/compaction"; +import type { Usage } from "@oh-my-pi/pi-ai"; + +function usage(overrides: Partial): Usage { + return { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + ...overrides, + }; +} + +describe("calculateContextTokens", () => { + it("excludes provider orchestration tokens from context sizing", () => { + // Codex-style turn: conversation prefix is ~186k, orchestration adds 5.5k; + // context sizing must stay on the conversation, not the billable total. + const u = usage({ + input: 5_517, + output: 29, + cacheRead: 181_248, + cacheWrite: 0, + totalTokens: 186_794 + 5_629, + orchestration: { input: 5_629 }, + }); + expect(calculateContextTokens(u)).toBe(186_794); + expect(calculatePromptTokens(u)).toBe(5_517 + 181_248); + }); + + it("keeps native totalTokens when no orchestration sidecar is present", () => { + const u = usage({ input: 10, output: 5, cacheRead: 100, cacheWrite: 0, totalTokens: 115 }); + expect(calculateContextTokens(u)).toBe(115); + }); +});