Merge PR #7980: fix(session): attribute a run to the model that produced its output (@enieuwy)
This commit is contained in:
@@ -73,6 +73,7 @@
|
||||
### Removed
|
||||
|
||||
- Removed the `resolveAgentModelSource` model-resolver export, whose only use was being fed to `resolveExplicitModelRole`. Replaced by `resolveAgentModelSelection`, which returns the expanded `patterns` and the pre-expansion `role` together so a spawn path cannot derive one without the other ([#7910](https://github.com/can1357/oh-my-pi/pull/7910) by [@enieuwy](https://github.com/enieuwy)).
|
||||
- A run is now attributed to the model that actually produced its output, not whichever model the session was last pointed at. A retry fallback that errored on its first request — an exhausted quota, a hard provider error — was credited with the whole run in the Agent Hub row and the settled task result, even when the previous model did every turn. Sessions expose the serving model directly, holding the last model that produced output while a candidate is armed but unproven, and transcript-derived history stops at the newest turn that produced output.
|
||||
|
||||
## [17.2.10] - 2026-08-06
|
||||
|
||||
|
||||
@@ -95,19 +95,25 @@ function formatResolvedModelBadge(resolved: string, preserveProvider = false, fa
|
||||
/**
|
||||
* Resolved model + reasoning level for a hub row. Exact executor progress is
|
||||
* authoritative (and survives completion); direct live sessions are the
|
||||
* fallback for agents without an observer snapshot.
|
||||
* fallback for agents without an observer snapshot — the main session has no
|
||||
* snapshot at all, so its row is read straight off the live session.
|
||||
*
|
||||
* Every source reports the model that produced the row's work, never the one
|
||||
* the session merely points at: an armed fallback that has not served yet stays
|
||||
* attributed to whichever model last actually spoke.
|
||||
*/
|
||||
export function modelBadge(ref: AgentRef, observed: ObservableSession | undefined): string | undefined {
|
||||
const progress = observed?.progress;
|
||||
const liveThinkingLevel = ref.session?.thinkingLevel;
|
||||
const serving = ref.session?.servingModel;
|
||||
const fallbackSelector =
|
||||
ref.session?.retryFallbackModel ??
|
||||
(serving?.isFallback ? serving.selector : undefined) ??
|
||||
(progress?.resolvedModelIsFallback ? progress.resolvedModel : undefined) ??
|
||||
(ref.history?.resolvedModelIsFallback ? ref.history.resolvedModel : undefined);
|
||||
if (fallbackSelector) {
|
||||
return `${theme.fg("warning", "fallback →")} ${formatResolvedModelBadge(fallbackSelector, true, liveThinkingLevel)}`;
|
||||
}
|
||||
const resolvedModel = progress?.resolvedModel ?? ref.history?.resolvedModel;
|
||||
const resolvedModel = progress?.resolvedModel ?? ref.history?.resolvedModel ?? serving?.selector;
|
||||
if (resolvedModel) return formatResolvedModelBadge(resolvedModel, false, liveThinkingLevel);
|
||||
const model = ref.session?.model;
|
||||
if (!model) return undefined;
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
import * as fs from "node:fs";
|
||||
import * as path from "node:path";
|
||||
import type { AssistantMessage } from "@oh-my-pi/pi-ai";
|
||||
import { ADVISOR_TRANSCRIPT_FILENAME, isAdvisorTranscriptName } from "../advisor/transcript-recorder";
|
||||
import { resolveExplicitModelRole } from "../config/model-resolver";
|
||||
import { assistantTurnProducedOutput } from "../session/messages";
|
||||
import { EPHEMERAL_MODEL_CHANGE_ROLE } from "../session/session-entries";
|
||||
import { visitEntriesFromFileStream } from "../session/session-loader";
|
||||
import { loadBundledAgents } from "../task/agents";
|
||||
@@ -90,6 +92,12 @@ interface AssistantMetrics {
|
||||
cost: number;
|
||||
contextTokens?: number;
|
||||
resolvedModel?: string;
|
||||
/**
|
||||
* True when this turn produced output, making its model the run's. Uses the
|
||||
* same predicate as the live session, so replaying a transcript reaches the
|
||||
* same verdict the session reached while running it.
|
||||
*/
|
||||
served: boolean;
|
||||
}
|
||||
|
||||
function assistantMetrics(message: Record<string, unknown>): AssistantMetrics {
|
||||
@@ -104,6 +112,10 @@ function assistantMetrics(message: Record<string, unknown>): AssistantMetrics {
|
||||
cost: finiteNumber(cost?.total),
|
||||
contextTokens: finiteNumber(usage.totalTokens) || undefined,
|
||||
resolvedModel: provider && model ? `${provider}/${model}` : undefined,
|
||||
served: assistantTurnProducedOutput({
|
||||
stopReason: message.stopReason,
|
||||
content,
|
||||
} as Pick<AssistantMessage, "stopReason" | "content">),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -159,27 +171,42 @@ async function readPersistedAgentHistory(
|
||||
),
|
||||
durationKind: "span",
|
||||
};
|
||||
// Attribution walks leaf → root and stops at the newest turn that actually
|
||||
// produced output: that model did this run's work. A `model_change` newer
|
||||
// than it was never served (a fallback the session died on), so crediting the
|
||||
// run to it would report work the previous model did.
|
||||
let resolvedModel: string | undefined;
|
||||
let resolvedModelIsFallback: boolean | undefined;
|
||||
let modelRole: string | undefined;
|
||||
let contextTokens: number | undefined;
|
||||
let modelChangeFound = false;
|
||||
let servedModel: string | undefined;
|
||||
let latestModelChange: { model: string; resolvedModelIsFallback: boolean } | undefined;
|
||||
const visited = new Set<string>();
|
||||
for (let id = leafId; id && !visited.has(id); id = parents.get(id)) {
|
||||
visited.add(id);
|
||||
const modelChange = modelChangeById.get(id);
|
||||
if (modelChange && !modelChangeFound) {
|
||||
modelChangeFound = true;
|
||||
resolvedModel = modelChange.model;
|
||||
resolvedModelIsFallback = modelChange.resolvedModelIsFallback;
|
||||
if (modelChange) {
|
||||
latestModelChange ??= modelChange;
|
||||
if (modelChange.role && modelChange.role !== EPHEMERAL_MODEL_CHANGE_ROLE) {
|
||||
modelRole = modelChange.role;
|
||||
modelRole ??= modelChange.role;
|
||||
}
|
||||
// The transition that installed the serving model: it carries the
|
||||
// fallback flag the raw message lacks. Every writer records the selector
|
||||
// through `formatModelStringWithRouting`, which appends an `@upstream`
|
||||
// gateway route the message's bare `provider/model` never has.
|
||||
if (
|
||||
servedModel !== undefined &&
|
||||
resolvedModel === undefined &&
|
||||
(modelChange.model === servedModel || modelChange.model.startsWith(`${servedModel}@`))
|
||||
) {
|
||||
resolvedModel = modelChange.model;
|
||||
resolvedModelIsFallback = modelChange.resolvedModelIsFallback;
|
||||
}
|
||||
}
|
||||
const assistant = assistantById.get(id);
|
||||
if (!assistant) continue;
|
||||
if (!modelChangeFound && resolvedModel === undefined && assistant.resolvedModel) {
|
||||
resolvedModel = assistant.resolvedModel;
|
||||
if (servedModel === undefined && assistant.served && assistant.resolvedModel) {
|
||||
servedModel = assistant.resolvedModel;
|
||||
}
|
||||
metrics.requests++;
|
||||
metrics.tokens += assistant.tokens;
|
||||
@@ -187,6 +214,14 @@ async function readPersistedAgentHistory(
|
||||
metrics.cost += assistant.cost;
|
||||
contextTokens ??= assistant.contextTokens;
|
||||
}
|
||||
// No transition described the serving model (pre-`model_change` transcript, or
|
||||
// the spawn record was pruned) — the message's own model still beats a
|
||||
// transition that never ran. Nothing served at all leaves only the last
|
||||
// transition to report.
|
||||
if (resolvedModel === undefined) {
|
||||
resolvedModel = servedModel ?? latestModelChange?.model;
|
||||
resolvedModelIsFallback = servedModel !== undefined ? false : latestModelChange?.resolvedModelIsFallback;
|
||||
}
|
||||
if (contextTokens !== undefined) metrics.contextTokens = contextTokens;
|
||||
return {
|
||||
...(metrics.requests > 0 ? { metrics } : {}),
|
||||
|
||||
@@ -313,6 +313,7 @@ import {
|
||||
queueChipText,
|
||||
toRestoredQueuedMessage,
|
||||
} from "./queued-messages";
|
||||
import type { ServingModel } from "./retry-fallback-chains";
|
||||
import { type AdvisorStats, SessionAdvisors, type SessionAdvisorsHost } from "./session-advisors";
|
||||
import type { BuildSessionContextOptions, SessionContext } from "./session-context";
|
||||
import { getRestorableSessionModels } from "./session-context";
|
||||
@@ -4097,9 +4098,13 @@ export class AgentSession {
|
||||
return this.agent.state.model;
|
||||
}
|
||||
|
||||
/** Resolved selector while retry routing is using a fallback model. */
|
||||
get retryFallbackModel(): string | undefined {
|
||||
return this.#recovery.retryFallbackModel;
|
||||
/**
|
||||
* Model this session's produced work is attributed to. Holds the last model
|
||||
* that actually served while a fallback is armed but unproven, so observers
|
||||
* never credit a run to a candidate that produced nothing.
|
||||
*/
|
||||
get servingModel(): ServingModel | undefined {
|
||||
return this.#recovery.servingModel;
|
||||
}
|
||||
|
||||
/** Install the interactive decision surface for reserve-triggered model changes. */
|
||||
@@ -6497,6 +6502,7 @@ export class AgentSession {
|
||||
async fork(): Promise<boolean> {
|
||||
this.#assertVibeSessionTransitionAllowed("fork the session");
|
||||
const previousSessionFile = this.sessionFile;
|
||||
const previousSessionId = this.sessionManager.getSessionId();
|
||||
|
||||
// Emit session_before_switch event with reason "fork" (can be cancelled)
|
||||
if (this.#extensionRunner?.hasHandlers("session_before_switch")) {
|
||||
@@ -6535,6 +6541,9 @@ export class AgentSession {
|
||||
}
|
||||
this.#bash.markSessionTransition(bashTransition);
|
||||
this.#bash.finishSessionTransition(bashTransition, true);
|
||||
// The fork clones the transcript and keeps this recovery state running
|
||||
// under a fresh id, so the work already produced is still this session's.
|
||||
this.#recovery.reanchorServedAttribution(previousSessionId);
|
||||
|
||||
// Copy artifacts directory if it exists
|
||||
const oldArtifactDir = forkResult.oldSessionFile.slice(0, -6);
|
||||
|
||||
@@ -498,6 +498,76 @@ export function isEmptyErrorTurn(message: Pick<AssistantMessage, "stopReason" |
|
||||
});
|
||||
}
|
||||
|
||||
/** Non-whitespace text. Tolerates malformed blocks: transcripts replayed off
|
||||
* disk predate current shapes, and a missing field must not throw. */
|
||||
function hasText(content: { text?: unknown }): boolean {
|
||||
return typeof content.text === "string" && content.text.trim().length > 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* A block that is real output from the model.
|
||||
*
|
||||
* Everything the assistant can emit counts except two: unsigned thinking, which
|
||||
* is not provider-authenticated and not actionable, and Anthropic's `fallback`
|
||||
* marker, which records that the request was routed elsewhere rather than
|
||||
* carrying any output. A native image response often arrives with no text and
|
||||
* no tool call at all, so recognising only those would call it nothing.
|
||||
*/
|
||||
function isActionableContent(content: AssistantMessage["content"][number] | undefined): boolean {
|
||||
switch (content?.type) {
|
||||
case "toolCall":
|
||||
case "image":
|
||||
case "redactedThinking":
|
||||
case "anthropicServerTool":
|
||||
return true;
|
||||
case "text":
|
||||
return hasText(content);
|
||||
case "thinking":
|
||||
return typeof content.thinkingSignature === "string" && content.thinkingSignature.trim().length > 0;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/** A `stop`/`toolUse` turn that produced nothing actionable. Any other stop
|
||||
* reason is not an "empty stop": an `error`/`aborted` turn is a failure rather
|
||||
* than an empty completion, and a `length` stop was cut off mid-output. */
|
||||
export function isEmptyAssistantStop(message: Pick<AssistantMessage, "stopReason" | "content">): boolean {
|
||||
switch (message.stopReason) {
|
||||
case "stop":
|
||||
return !message.content.some(isActionableContent);
|
||||
case "toolUse":
|
||||
// An orphaned toolUse stop (no tool_use block) corrupts Anthropic history:
|
||||
// a later tool_result has nothing to anchor to. Thinking alone cannot anchor
|
||||
// a tool_result, so it does not rescue a toolUse stop here.
|
||||
return !message.content.some(
|
||||
content => content?.type === "toolCall" || (content?.type === "text" && hasText(content)),
|
||||
);
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* True when this assistant turn actually produced output, making its model the
|
||||
* one that served the run.
|
||||
*
|
||||
* Attribution asks this from two places that MUST reach the same verdict: the
|
||||
* live session, which flips a fallback to "served", and the offline walk
|
||||
* replaying a transcript. `error` and `aborted` are both failures — a stalled or
|
||||
* dropped stream is finalized as `aborted` with its partial block still
|
||||
* attached, so a stop reason alone is not proof.
|
||||
*
|
||||
* Actionable content is required on top of the empty-stop rule, which only
|
||||
* inspects `stop`/`toolUse`. A `length` stop burns the whole output budget
|
||||
* without necessarily emitting anything usable, and every other stop reason
|
||||
* bypasses that rule entirely.
|
||||
*/
|
||||
export function assistantTurnProducedOutput(message: Pick<AssistantMessage, "stopReason" | "content">): boolean {
|
||||
if (message.stopReason === "error" || message.stopReason === "aborted") return false;
|
||||
return !isEmptyAssistantStop(message) && message.content.some(isActionableContent);
|
||||
}
|
||||
|
||||
/** Sentinel `errorMessage` the agent stamps on any abort that carried no custom
|
||||
* reason (bare `abort()`). Renderers treat it as "no specific reason given". */
|
||||
export const GENERIC_ABORT_SENTINEL = "Request was aborted";
|
||||
|
||||
@@ -50,6 +50,20 @@ export interface ActiveRetryFallbackState {
|
||||
originalThinkingLevel: ConfiguredThinkingLevel | undefined;
|
||||
lastAppliedFallbackThinkingLevel: ConfiguredThinkingLevel | undefined;
|
||||
pinned: boolean;
|
||||
/**
|
||||
* Set once a turn on the fallback target settles successfully. Until then the
|
||||
* switch is only a routing decision — nothing has been produced by the new
|
||||
* model, so no observer may report the run as having used it.
|
||||
*/
|
||||
served?: boolean;
|
||||
}
|
||||
|
||||
/** Model a session's produced work is attributed to. */
|
||||
export interface ServingModel {
|
||||
/** Full selector including routing and thinking level. */
|
||||
selector: string;
|
||||
/** Whether fallback routing, rather than the configured primary, owns it. */
|
||||
isFallback: boolean;
|
||||
}
|
||||
|
||||
const RETRY_BACKOFF_MAX_DELAY_MS = 8_000;
|
||||
|
||||
@@ -44,7 +44,7 @@ import type {
|
||||
UsageFallbackConfirmation,
|
||||
UsageFallbackConfirmer,
|
||||
} from "./agent-session-types";
|
||||
import { isEmptyErrorTurn } from "./messages";
|
||||
import { assistantTurnProducedOutput, isEmptyAssistantStop, isEmptyErrorTurn } from "./messages";
|
||||
import {
|
||||
type ActiveRetryFallbackState,
|
||||
calculateRetryBackoffDelayMs,
|
||||
@@ -58,6 +58,7 @@ import {
|
||||
type RetryFallbackRevertPolicy,
|
||||
type RetryFallbackSelector,
|
||||
resolveRetryFallbackChainKey,
|
||||
type ServingModel,
|
||||
validateRetryFallbackChains,
|
||||
} from "./retry-fallback-chains";
|
||||
import { getLatestCompactionEntry } from "./session-context";
|
||||
@@ -189,6 +190,33 @@ export class TurnRecovery {
|
||||
#emptyStopRetryCount = 0;
|
||||
#unexpectedStopRetryCount = 0;
|
||||
#acceptTerminalEmptyStopForPrompt = false;
|
||||
// Three fields sit near the word "serve" and are deliberately distinct:
|
||||
// `#activeRetryFallback.served` gates the one-shot `retry_fallback_succeeded`
|
||||
// event for the current arm, `#fallbackRouted` says how the CURRENT model was
|
||||
// reached, and `#lastServed` is the session's attribution. A fallback flipping
|
||||
// to served does not by itself move attribution — only a settled turn does.
|
||||
/**
|
||||
* Attribution of the newest turn that produced output, tagged with the
|
||||
* session it belongs to. Anchoring rather than resetting follows
|
||||
* `#ensurePersistedMessageKeys`: every real switch mints a new session id, so
|
||||
* stale attribution drops itself and no mutation call site has to remember to
|
||||
* clear it. The id — not the file — is the anchor because an unpersisted
|
||||
* session has no file, and comparing two `undefined`s would never invalidate.
|
||||
*/
|
||||
#lastServed: { attribution: ServingModel; sessionId: string } | undefined;
|
||||
/**
|
||||
* Session whose current model was reached by fallback routing rather than by
|
||||
* the configured primary, or `undefined` when it was not. Tracked separately
|
||||
* from {@link #activeRetryFallback} because the Fireworks Fast degrade swaps
|
||||
* models without arming a chain, and anchored like {@link #lastServed}:
|
||||
* switching transcripts in place must not describe a fresh session's model
|
||||
* with how the previous one was routed.
|
||||
*/
|
||||
#fallbackRoutedFor: string | undefined;
|
||||
/** Memoized bootstrap answer, for the window before anything has served. */
|
||||
#bootstrapCache:
|
||||
| { model: Model; level: ThinkingLevel | undefined; routed: boolean; value: ServingModel }
|
||||
| undefined;
|
||||
|
||||
constructor(host: TurnRecoveryHost, options: TurnRecoveryOptions = {}) {
|
||||
this.#host = host;
|
||||
@@ -198,6 +226,7 @@ export class TurnRecovery {
|
||||
lastAppliedFallbackThinkingLevel: host.configuredThinkingLevel(),
|
||||
pinned: options.initialRetryFallback.pinned ?? false,
|
||||
};
|
||||
this.#markFallbackRouted();
|
||||
}
|
||||
this.#validateRetryFallbackChains();
|
||||
}
|
||||
@@ -212,12 +241,70 @@ export class TurnRecovery {
|
||||
return this.#retryPromise;
|
||||
}
|
||||
|
||||
/** Resolved selector while fallback routing owns the current model. */
|
||||
get retryFallbackModel(): string | undefined {
|
||||
/** Whether the CURRENT session's model was reached by fallback routing. */
|
||||
get #fallbackRouted(): boolean {
|
||||
return (
|
||||
this.#fallbackRoutedFor !== undefined && this.#fallbackRoutedFor === this.#host.sessionManager.getSessionId()
|
||||
);
|
||||
}
|
||||
|
||||
#markFallbackRouted(): void {
|
||||
this.#fallbackRoutedFor = this.#host.sessionManager.getSessionId();
|
||||
}
|
||||
|
||||
/**
|
||||
* Model this session's produced work is attributed to.
|
||||
*
|
||||
* A model switch is a routing decision, not evidence the target can produce
|
||||
* anything: a candidate that errors on its first request produced none of the
|
||||
* turns already in this session. So attribution only ever names a model that
|
||||
* has settled a turn here, and a switch — into a fallback, back to a restored
|
||||
* primary, or anywhere else — moves it only once the new model answers.
|
||||
*
|
||||
* Before anything has served there is no earlier work to miscredit, so the
|
||||
* configured model is both the only available answer and a safe one.
|
||||
*/
|
||||
get servingModel(): ServingModel | undefined {
|
||||
const served = this.#lastServed;
|
||||
if (served && served.sessionId === this.#host.sessionManager.getSessionId()) return served.attribution;
|
||||
const model = this.#host.model();
|
||||
return this.#activeRetryFallback && model
|
||||
? formatRetryFallbackSelector(model, this.#host.thinkingLevel())
|
||||
: undefined;
|
||||
if (!model) return undefined;
|
||||
// Polled per streaming event and per render, so the pre-first-turn window
|
||||
// must not format a selector on every call.
|
||||
const level = this.#host.thinkingLevel();
|
||||
const cached = this.#bootstrapCache;
|
||||
if (cached && cached.model === model && cached.level === level && cached.routed === this.#fallbackRouted) {
|
||||
return cached.value;
|
||||
}
|
||||
const value: ServingModel = {
|
||||
selector: formatRetryFallbackSelector(model, level),
|
||||
isFallback: this.#fallbackRouted,
|
||||
};
|
||||
this.#bootstrapCache = { model, level, routed: this.#fallbackRouted, value };
|
||||
return value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Carries attribution onto a new session id that continues this conversation.
|
||||
*
|
||||
* The session-id anchor assumes a new id means an unrelated transcript, which
|
||||
* holds for `/new` and for resuming something else. A fork breaks that
|
||||
* assumption on purpose: it clones the transcript and keeps running the same
|
||||
* recovery state under a fresh id. Dropping attribution there would bootstrap
|
||||
* an unproven fallback as the primary and re-credit it with the work the
|
||||
* previous model did — the very bug the anchor exists to prevent.
|
||||
*
|
||||
* Only state belonging to `previousSessionId` moves, so an id left behind by
|
||||
* an earlier switch stays expired.
|
||||
*/
|
||||
reanchorServedAttribution(previousSessionId: string): void {
|
||||
const sessionId = this.#host.sessionManager.getSessionId();
|
||||
if (this.#lastServed?.sessionId === previousSessionId) {
|
||||
this.#lastServed = { ...this.#lastServed, sessionId };
|
||||
}
|
||||
if (this.#fallbackRoutedFor === previousSessionId) {
|
||||
this.#fallbackRoutedFor = sessionId;
|
||||
}
|
||||
}
|
||||
|
||||
/** Resets per-prompt recovery counters and terminal-stop acceptance. */
|
||||
@@ -232,24 +319,41 @@ export class TurnRecovery {
|
||||
this.#acceptTerminalEmptyStopForPrompt = accept;
|
||||
}
|
||||
|
||||
/** Closes a successful retry saga and annotates recovered persisted errors. */
|
||||
/**
|
||||
* Records which model produced this turn, marks an active fallback as having
|
||||
* served, then closes a successful retry saga and annotates recovered
|
||||
* persisted errors.
|
||||
*/
|
||||
async onAssistantSettledSuccessfully(message: AssistantMessage): Promise<void> {
|
||||
if (
|
||||
message.stopReason === "error" ||
|
||||
message.stopReason === "aborted" ||
|
||||
this.#isEmptyAssistantStop(message) ||
|
||||
this.#retryAttempt === 0
|
||||
) {
|
||||
if (!assistantTurnProducedOutput(message)) {
|
||||
return;
|
||||
}
|
||||
const model = this.#host.model();
|
||||
if (this.#activeRetryFallback && model) {
|
||||
if (model) {
|
||||
this.#lastServed = {
|
||||
attribution: {
|
||||
selector: formatRetryFallbackSelector(model, this.#host.thinkingLevel()),
|
||||
isFallback: this.#fallbackRouted,
|
||||
},
|
||||
sessionId: this.#host.sessionManager.getSessionId(),
|
||||
};
|
||||
}
|
||||
// Independent of the retry saga below: a usage-aware fallback is applied
|
||||
// before a request without ever incrementing `#retryAttempt`, and it still
|
||||
// owns every turn it serves. Gating this on the saga left such a fallback
|
||||
// permanently unproven, hiding it from observers for the whole session.
|
||||
if (this.#activeRetryFallback && !this.#activeRetryFallback.served && model) {
|
||||
this.#activeRetryFallback.served = true;
|
||||
await this.#host.emitSessionEvent({
|
||||
type: "retry_fallback_succeeded",
|
||||
model: formatRetryFallbackSelector(model, this.#host.thinkingLevel()),
|
||||
model:
|
||||
this.#lastServed?.attribution.selector ?? formatRetryFallbackSelector(model, this.#host.thinkingLevel()),
|
||||
role: this.#activeRetryFallback.role,
|
||||
});
|
||||
}
|
||||
if (this.#retryAttempt === 0) {
|
||||
return;
|
||||
}
|
||||
const retryErrors = await this.#markPendingRetryErrors({
|
||||
status: "recovered",
|
||||
supersedingMessage: message,
|
||||
@@ -538,7 +642,7 @@ export class TurnRecovery {
|
||||
}
|
||||
|
||||
async #handleEmptyAssistantStop(assistantMessage: AssistantMessage): Promise<boolean> {
|
||||
if (!this.#isEmptyAssistantStop(assistantMessage)) {
|
||||
if (!isEmptyAssistantStop(assistantMessage)) {
|
||||
this.#emptyStopRetryCount = 0;
|
||||
return false;
|
||||
}
|
||||
@@ -587,31 +691,6 @@ export class TurnRecovery {
|
||||
return true;
|
||||
}
|
||||
|
||||
#isEmptyAssistantStop(assistantMessage: AssistantMessage): boolean {
|
||||
switch (assistantMessage.stopReason) {
|
||||
case "stop":
|
||||
// Unsigned thinking alone is not actionable, but a signature is
|
||||
// provider-authenticated content and makes the stop terminal.
|
||||
for (const content of assistantMessage.content) {
|
||||
if (content.type === "toolCall") return false;
|
||||
if (content.type === "text" && hasNonWhitespace(content.text)) return false;
|
||||
if (content.type === "thinking" && hasNonWhitespace(content.thinkingSignature ?? "")) return false;
|
||||
}
|
||||
return true;
|
||||
case "toolUse":
|
||||
// An orphaned toolUse stop (no tool_use block) corrupts Anthropic history:
|
||||
// a later tool_result has nothing to anchor to. Thinking alone cannot anchor
|
||||
// a tool_result, so it does not rescue a toolUse stop here.
|
||||
for (const content of assistantMessage.content) {
|
||||
if (content.type === "toolCall") return false;
|
||||
if (content.type === "text" && hasNonWhitespace(content.text)) return false;
|
||||
}
|
||||
return true;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
#emptyStopRetryReminder(): string {
|
||||
return prompt.render(emptyStopRetryTemplate, {
|
||||
retryCount: this.#emptyStopRetryCount,
|
||||
@@ -1069,9 +1148,10 @@ export class TurnRecovery {
|
||||
return getRetryFallbackRevertPolicy(this.#host.settings);
|
||||
}
|
||||
|
||||
/** Clears fallback ownership after an explicit model change. */
|
||||
/** Clears fallback ownership after an explicit model change or a restore. */
|
||||
clearActiveRetryFallback(): void {
|
||||
this.#activeRetryFallback = undefined;
|
||||
this.#fallbackRoutedFor = undefined;
|
||||
}
|
||||
|
||||
/** Checks whether a fallback selector remains in cooldown. */
|
||||
@@ -1312,14 +1392,29 @@ export class TurnRecovery {
|
||||
: clampThinkingLevelToCeiling(candidate, requestedThinkingLevel, this.#host.thinkingLevelCeiling());
|
||||
const candidateSelector = formatModelStringWithRouting(candidate);
|
||||
const previousModel = this.#host.model();
|
||||
// Mark routing BEFORE the swap: `setModelWithProviderSessionReset` moves the
|
||||
// model and fans `model_changed` out to subscribers synchronously, and a
|
||||
// listener reading attribution in that window must already see the incoming
|
||||
// candidate as fallback-routed. Attribution itself is safe regardless — it
|
||||
// names the last model that served, which this swap has not changed.
|
||||
const routedBeforeSwap = this.#fallbackRoutedFor;
|
||||
const servedBeforeSwap = this.#activeRetryFallback?.served;
|
||||
this.#markFallbackRouted();
|
||||
if (this.#activeRetryFallback) this.#activeRetryFallback.served = false;
|
||||
await this.#host.setModelWithProviderSessionReset(candidate);
|
||||
if (options?.signal?.aborted) {
|
||||
this.#fallbackRoutedFor = routedBeforeSwap;
|
||||
if (this.#activeRetryFallback) this.#activeRetryFallback.served = servedBeforeSwap;
|
||||
if (previousModel && this.#host.model() === candidate) {
|
||||
await this.#host.setModelWithProviderSessionReset(previousModel);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
if (this.#host.model() !== candidate) return false;
|
||||
if (this.#host.model() !== candidate) {
|
||||
this.#fallbackRoutedFor = routedBeforeSwap;
|
||||
if (this.#activeRetryFallback) this.#activeRetryFallback.served = servedBeforeSwap;
|
||||
return false;
|
||||
}
|
||||
this.#host.sessionManager.appendModelChange(candidateSelector, EPHEMERAL_MODEL_CHANGE_ROLE, true);
|
||||
this.#host.settings.getStorage()?.recordModelUsage(candidateSelector);
|
||||
this.#host.setThinkingLevel(nextThinkingLevel);
|
||||
@@ -1439,6 +1534,9 @@ export class TurnRecovery {
|
||||
const apiKey = await this.#host.modelRegistry.getApiKey(baseModel, this.#host.sessionId());
|
||||
if (!apiKey) return false;
|
||||
const baseSelector = formatModelStringWithRouting(baseModel);
|
||||
// A capability degrade is fallback routing too, even though it arms no
|
||||
// chain: the base model must not be reported as the configured primary.
|
||||
this.#markFallbackRouted();
|
||||
await this.#host.setModelWithProviderSessionReset(baseModel);
|
||||
this.#host.sessionManager.appendModelChange(baseSelector, EPHEMERAL_MODEL_CHANGE_ROLE, true);
|
||||
this.#host.settings.getStorage()?.recordModelUsage(baseSelector);
|
||||
@@ -1463,7 +1561,12 @@ export class TurnRecovery {
|
||||
} = this.#activeRetryFallback;
|
||||
const originalSelector = parseRetryFallbackSelector(originalSelectorRaw, this.#host.modelRegistry);
|
||||
if (!originalSelector) {
|
||||
this.clearActiveRetryFallback();
|
||||
// Defensive: the stored selector is always produced by
|
||||
// `formatRetryFallbackSelector`, so it should never fail to parse. If it
|
||||
// somehow does, nothing is restored and the session keeps running on the
|
||||
// fallback — so drop the chain record but NOT `#fallbackRouted`, whose
|
||||
// clearing would report the fallback's remaining turns as the primary.
|
||||
this.#activeRetryFallback = undefined;
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -1493,11 +1596,15 @@ export class TurnRecovery {
|
||||
const thinkingToApply =
|
||||
currentThinkingLevel === lastAppliedFallbackThinkingLevel ? originalThinkingLevel : currentThinkingLevel;
|
||||
const primarySelector = formatModelStringWithRouting(primaryModel);
|
||||
// Clear before the swap: `setModelWithProviderSessionReset` and
|
||||
// `setThinkingLevel` both notify subscribers, and an observer reading
|
||||
// attribution in that window would see the restored primary still tagged
|
||||
// as fallback-served.
|
||||
this.clearActiveRetryFallback();
|
||||
await this.#host.setModelWithProviderSessionReset(primaryModel);
|
||||
this.#host.sessionManager.appendModelChange(primarySelector, EPHEMERAL_MODEL_CHANGE_ROLE);
|
||||
this.#host.settings.getStorage()?.recordModelUsage(primarySelector);
|
||||
this.#host.setThinkingLevel(thinkingToApply);
|
||||
this.clearActiveRetryFallback();
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
@@ -1664,15 +1664,28 @@ function createSubagentRunMonitor(args: RunMonitorArgs): SubagentRunMonitor {
|
||||
};
|
||||
|
||||
const attach = (session: AgentSession): (() => void) => {
|
||||
let activeModel = session.model ? formatModelStringWithRouting(session.model) : undefined;
|
||||
// The session owns attribution: it knows which model produced its output
|
||||
// and withholds an armed-but-unproven fallback. Re-deriving that here from
|
||||
// the event stream got it wrong twice over — the stream also carries
|
||||
// advisor turns running on a different model, and a routing switch was
|
||||
// read as evidence the target had served.
|
||||
const publishServingModel = (): void => {
|
||||
const serving = session.servingModel;
|
||||
if (!serving) return;
|
||||
const isFallback = serving.isFallback;
|
||||
if (
|
||||
serving.selector === progress.resolvedModel &&
|
||||
(progress.resolvedModelIsFallback ?? false) === isFallback
|
||||
) {
|
||||
return;
|
||||
}
|
||||
progress.resolvedModel = serving.selector;
|
||||
progress.resolvedModelIsFallback = isFallback;
|
||||
scheduleProgress(true);
|
||||
};
|
||||
return session.subscribe(event => {
|
||||
emitSubagentEvent(event);
|
||||
const nextModel = session.model ? formatModelStringWithRouting(session.model) : undefined;
|
||||
if (nextModel && nextModel !== activeModel) {
|
||||
activeModel = nextModel;
|
||||
progress.resolvedModel = nextModel;
|
||||
scheduleProgress(true);
|
||||
}
|
||||
publishServingModel();
|
||||
if (event.type === "auto_retry_start") {
|
||||
progress.retryState = {
|
||||
attempt: event.attempt,
|
||||
@@ -1712,18 +1725,6 @@ function createSubagentRunMonitor(args: RunMonitorArgs): SubagentRunMonitor {
|
||||
popLoopPhase();
|
||||
}
|
||||
}
|
||||
if (event.type === "retry_fallback_applied") {
|
||||
progress.resolvedModel = event.to;
|
||||
progress.resolvedModelIsFallback = true;
|
||||
scheduleProgress(true);
|
||||
return;
|
||||
}
|
||||
if (event.type === "retry_fallback_succeeded") {
|
||||
progress.resolvedModel = event.model;
|
||||
progress.resolvedModelIsFallback = true;
|
||||
scheduleProgress(true);
|
||||
return;
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
|
||||
@@ -442,14 +442,64 @@ describe("Agent hub row ordering", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("reads a live row's model off the session's served attribution, not its current pointer", () => {
|
||||
geometry = stubStdoutGeometry(120);
|
||||
const agents = new AgentRegistry();
|
||||
// The main session has no executor progress and no persisted history, so
|
||||
// its row comes straight off the live session. With a fallback armed but
|
||||
// unproven, `model` already points at the candidate that has produced
|
||||
// nothing — reporting it credits the run to a model that never spoke.
|
||||
const session = {
|
||||
model: { id: "gpt-5.6-sol", thinking: true },
|
||||
thinkingLevel: "high",
|
||||
servingModel: { selector: "anthropic/claude-sonnet-5", isFallback: false },
|
||||
} as unknown as AgentSession;
|
||||
agents.register({ id: "MainAgent", displayName: "Main Agent", kind: "sub", session });
|
||||
|
||||
const hub = makeHub(agents, { observers: new SessionObserverRegistry() });
|
||||
|
||||
try {
|
||||
const rendered = Bun.stripANSI(hub.render(120).join("\n"));
|
||||
expect(rendered).toContain("claude-sonnet-5");
|
||||
expect(rendered).not.toContain("gpt-5.6-sol");
|
||||
expect(rendered).not.toContain("fallback →");
|
||||
} finally {
|
||||
hub.dispose();
|
||||
}
|
||||
});
|
||||
|
||||
it("marks an armed fallback that has served nothing yet", () => {
|
||||
geometry = stubStdoutGeometry(120);
|
||||
const agents = new AgentRegistry();
|
||||
// Nothing has served in this session, so there is no earlier work to
|
||||
// miscredit — but the row must still say the model was reached by a
|
||||
// fallback rather than presenting it as the plain configured model.
|
||||
const session = {
|
||||
model: { id: "gpt-5.6-sol", thinking: true },
|
||||
thinkingLevel: "high",
|
||||
// Nothing served, so the session names what it currently points at —
|
||||
// still flagged as fallback-routed.
|
||||
servingModel: { selector: "openai-codex/gpt-5.6-sol", isFallback: true },
|
||||
} as unknown as AgentSession;
|
||||
agents.register({ id: "UnprovenAgent", displayName: "Unproven Agent", kind: "sub", session });
|
||||
|
||||
const hub = makeHub(agents, { observers: new SessionObserverRegistry() });
|
||||
|
||||
try {
|
||||
expect(Bun.stripANSI(hub.render(120).join("\n"))).toContain("fallback → openai-codex/gpt-5.6-sol");
|
||||
} finally {
|
||||
hub.dispose();
|
||||
}
|
||||
});
|
||||
|
||||
it("flags a fallback badge for a live row whose fallback armed no session retry state", () => {
|
||||
geometry = stubStdoutGeometry(120);
|
||||
const agents = new AgentRegistry();
|
||||
// Live session with a resolved model but no `retryFallbackModel` — the
|
||||
// Live session with a resolved model but no served fallback — the
|
||||
// Fireworks Fast → base degrade emits `retry_fallback_applied` without
|
||||
// arming `#activeRetryFallback`, so the badge must fall back to the
|
||||
// executor-reported progress flag.
|
||||
const session = { model: { id: "kimi-k2" }, retryFallbackModel: undefined } as unknown as AgentSession;
|
||||
const session = { model: { id: "kimi-k2" }, servingModel: undefined } as unknown as AgentSession;
|
||||
agents.register({ id: "FastAgent", displayName: "Fast Agent", kind: "sub", session });
|
||||
|
||||
const observers = new SessionObserverRegistry();
|
||||
|
||||
@@ -25,6 +25,7 @@ import { initTheme } from "@oh-my-pi/pi-coding-agent/modes/theme/theme";
|
||||
import { AgentRegistry } from "@oh-my-pi/pi-coding-agent/registry/agent-registry";
|
||||
import { AgentSession, type AgentSessionEvent } from "@oh-my-pi/pi-coding-agent/session/agent-session";
|
||||
import { AuthStorage } from "@oh-my-pi/pi-coding-agent/session/auth-storage";
|
||||
import type { ServingModel } from "@oh-my-pi/pi-coding-agent/session/retry-fallback-chains";
|
||||
import { SessionManager } from "@oh-my-pi/pi-coding-agent/session/session-manager";
|
||||
import { TempDir } from "@oh-my-pi/pi-utils";
|
||||
|
||||
@@ -1245,8 +1246,20 @@ describe("AgentSession retry fallback", () => {
|
||||
originalThinkingLevel: undefined,
|
||||
},
|
||||
});
|
||||
session.subscribe(event => {
|
||||
if (event.type === "retry_fallback_applied") fallbackAppliedEvents.push(event);
|
||||
// Startup-owned: selected before the session ran, so it owns every turn
|
||||
// from the first request — there is no earlier model's work to misattribute.
|
||||
expect(session.servingModel).toEqual({
|
||||
selector: `${firstFallback.provider}/${firstFallback.id}`,
|
||||
isFallback: true,
|
||||
});
|
||||
|
||||
const swapProbe: Array<ServingModel | undefined> = [];
|
||||
const observed = session;
|
||||
observed.subscribe(event => {
|
||||
if (event.type === "retry_fallback_applied") {
|
||||
fallbackAppliedEvents.push(event);
|
||||
swapProbe.push(observed.servingModel);
|
||||
}
|
||||
});
|
||||
|
||||
await session.prompt("Continue the startup fallback chain");
|
||||
@@ -1266,6 +1279,14 @@ describe("AgentSession retry fallback", () => {
|
||||
role: "slow",
|
||||
},
|
||||
]);
|
||||
// Nothing had served when the chain advanced, so there was no earlier work
|
||||
// to miscredit and the candidate being attempted is the only answer — but
|
||||
// it is still reported as fallback-routed.
|
||||
expect(swapProbe).toEqual([{ selector: `${secondFallback.provider}/${secondFallback.id}`, isFallback: true }]);
|
||||
expect(session.servingModel).toEqual({
|
||||
selector: `${secondFallback.provider}/${secondFallback.id}`,
|
||||
isFallback: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("applies a model-keyed fallback chain to advisor quota failures", async () => {
|
||||
@@ -3401,6 +3422,130 @@ describe("AgentSession retry fallback", () => {
|
||||
]);
|
||||
expect(session.model?.provider).toBe(primaryModel.provider);
|
||||
expect(session.model?.id).toBe(primaryModel.id);
|
||||
// The restored primary answered, so attribution moves back with it.
|
||||
expect(session.servingModel).toEqual({
|
||||
selector: `${primaryModel.provider}/${primaryModel.id}`,
|
||||
isFallback: false,
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps credit with the fallback when a restored primary fails without serving", async () => {
|
||||
const primaryModel = getBundledModel("anthropic", "claude-sonnet-4-5");
|
||||
const fallbackModel = getBundledModel("openai", "gpt-4o-mini");
|
||||
if (!primaryModel || !fallbackModel) {
|
||||
throw new Error("Expected bundled test models to exist");
|
||||
}
|
||||
|
||||
const requestedModels: string[] = [];
|
||||
const mock = createMockModel();
|
||||
const agent = new Agent({
|
||||
getApiKey: model => `${model.provider}-test-key`,
|
||||
initialState: { model: primaryModel, systemPrompt: ["Test"], tools: [], messages: [] },
|
||||
streamFn: (model, context, options) => {
|
||||
requestedModels.push(`${model.provider}/${model.id}`);
|
||||
// Only the fallback ever produces anything; the primary rate-limits on
|
||||
// every request, including after its cooldown expires and it is
|
||||
// restored. `retry-after-ms` keeps the cooldown short enough to expire
|
||||
// within the test's clock jump.
|
||||
mock.push(
|
||||
model.id === fallbackModel.id
|
||||
? { content: ["the fallback did the work"] }
|
||||
: { throw: "rate limit exceeded retry-after-ms=200" },
|
||||
);
|
||||
return mock.stream(model, context, options);
|
||||
},
|
||||
});
|
||||
|
||||
const settings = Settings.isolated({
|
||||
"compaction.enabled": false,
|
||||
"retry.baseDelayMs": 5,
|
||||
"retry.maxRetries": 2,
|
||||
"retry.fallbackChains": { default: [`${fallbackModel.provider}/${fallbackModel.id}`] },
|
||||
"retry.fallbackRevertPolicy": "cooldown-expiry",
|
||||
});
|
||||
settings.setModelRole("default", `${primaryModel.provider}/${primaryModel.id}`);
|
||||
|
||||
session = new AgentSession({
|
||||
agent,
|
||||
sessionManager: SessionManager.inMemory(),
|
||||
settings,
|
||||
modelRegistry,
|
||||
});
|
||||
let now = Date.now();
|
||||
vi.spyOn(Date, "now").mockImplementation(() => now);
|
||||
|
||||
await session.prompt("Fail over to the fallback");
|
||||
await session.waitForIdle();
|
||||
expect(session.servingModel).toEqual({
|
||||
selector: `${fallbackModel.provider}/${fallbackModel.id}`,
|
||||
isFallback: true,
|
||||
});
|
||||
|
||||
// Capture attribution inside the restore's synchronous `model_changed`
|
||||
// fan-out, which is the window the restore path reopens.
|
||||
const servingDuringSwaps: Array<ServingModel | undefined> = [];
|
||||
const restoring = session;
|
||||
restoring.subscribe(event => {
|
||||
if (event.type === "model_changed") servingDuringSwaps.push(restoring.servingModel);
|
||||
});
|
||||
|
||||
now += 240;
|
||||
await session.prompt("Cooldown expired: revert to the primary and fail there");
|
||||
await session.waitForIdle();
|
||||
|
||||
// A restore is a routing decision like a fallback is: the primary produced
|
||||
// nothing after coming back, so the work still belongs to the fallback.
|
||||
expect(requestedModels).toContain(`${primaryModel.provider}/${primaryModel.id}`);
|
||||
expect(servingDuringSwaps.length).toBeGreaterThan(0);
|
||||
for (const serving of servingDuringSwaps) {
|
||||
expect(serving?.selector).not.toBe(`${primaryModel.provider}/${primaryModel.id}`);
|
||||
}
|
||||
});
|
||||
|
||||
it("reports a Fireworks Fast degrade as fallback-routed even though it arms no chain", async () => {
|
||||
const fastModel = getBundledModel("fireworks", "kimi-k2.6-fast");
|
||||
if (!fastModel) throw new Error("Expected the bundled Fireworks Fast model to exist");
|
||||
const baseId = fastModel.id.replace(/-fast$/, "");
|
||||
|
||||
const requestedModels: string[] = [];
|
||||
const mock = createMockModel();
|
||||
const agent = new Agent({
|
||||
getApiKey: model => `${model.provider}-test-key`,
|
||||
initialState: { model: fastModel, systemPrompt: ["Test"], tools: [], messages: [] },
|
||||
streamFn: (model, context, options) => {
|
||||
requestedModels.push(`${model.provider}/${model.id}`);
|
||||
// Fast rejects the request; the base model answers it.
|
||||
mock.push(
|
||||
model.id === fastModel.id
|
||||
? { throw: "rate limit exceeded retry-after-ms=200" }
|
||||
: { content: ["the base model did the work"] },
|
||||
);
|
||||
return mock.stream(model, context, options);
|
||||
},
|
||||
});
|
||||
|
||||
const settings = Settings.isolated({ "compaction.enabled": false, "retry.baseDelayMs": 5 });
|
||||
settings.setModelRole("default", `${fastModel.provider}/${fastModel.id}`);
|
||||
session = new AgentSession({
|
||||
agent,
|
||||
sessionManager: SessionManager.inMemory(),
|
||||
settings,
|
||||
modelRegistry,
|
||||
});
|
||||
|
||||
await session.prompt("Degrade off Fast and answer on the base model");
|
||||
await session.waitForIdle();
|
||||
|
||||
// The degrade swaps models without arming a retry-fallback chain, but it is
|
||||
// still fallback routing — a bare model badge would hide that.
|
||||
expect(requestedModels).toEqual([`${fastModel.provider}/${fastModel.id}`, `fireworks/${baseId}`]);
|
||||
expect(session.servingModel).toEqual({ selector: `fireworks/${baseId}`, isFallback: true });
|
||||
|
||||
// How the previous transcript was routed says nothing about a freshly
|
||||
// loaded one: switching sessions in place must not describe the new
|
||||
// session's model as fallback-routed.
|
||||
vi.spyOn(session.sessionManager, "getSessionId").mockReturnValue("some-other-session");
|
||||
expect(session.servingModel).toEqual({ selector: `fireworks/${baseId}`, isFallback: false });
|
||||
});
|
||||
|
||||
it("re-checks context before a cooldown-expiry revert onto a smaller-window model in the auto-continue path", async () => {
|
||||
@@ -4038,4 +4183,282 @@ describe("AgentSession retry fallback", () => {
|
||||
expect(session.isRetrying).toBe(false);
|
||||
expect(getLastAssistantMessage(session).stopReason).toBe("stop");
|
||||
});
|
||||
|
||||
// `session.servingModel` is what the Agent Hub row reads for a live or
|
||||
// parked agent. A fallback that errors on its first request produced none of
|
||||
// the session's work, so announcing it credits the primary's output to it.
|
||||
it("withholds the fallback selector until the target has served a turn", async () => {
|
||||
const primaryModel = getBundledModel("anthropic", "claude-sonnet-4-5");
|
||||
const fallbackModel = getBundledModel("openai", "gpt-4o-mini");
|
||||
if (!primaryModel || !fallbackModel) {
|
||||
throw new Error("Expected bundled test models to exist");
|
||||
}
|
||||
|
||||
const requestedModels: string[] = [];
|
||||
const mock = createMockModel();
|
||||
const agent = new Agent({
|
||||
getApiKey: model => `${model.provider}-test-key`,
|
||||
initialState: { model: primaryModel, systemPrompt: ["Test"], tools: [], messages: [] },
|
||||
streamFn: (model, context, options) => {
|
||||
requestedModels.push(`${model.provider}/${model.id}`);
|
||||
// The primary serves one real turn, then both models fail: the chain
|
||||
// switches but the target never produces anything.
|
||||
if (requestedModels.length === 1) {
|
||||
mock.push({ content: ["primary did the work"] });
|
||||
} else {
|
||||
mock.push({ throw: "overloaded_error: provider returned error 503" });
|
||||
}
|
||||
return mock.stream(model, context, options);
|
||||
},
|
||||
});
|
||||
|
||||
const settings = Settings.isolated({
|
||||
"compaction.enabled": false,
|
||||
"retry.baseDelayMs": 5,
|
||||
"retry.maxRetries": 1,
|
||||
"retry.fallbackChains": { default: [`${fallbackModel.provider}/${fallbackModel.id}`] },
|
||||
});
|
||||
settings.setModelRole("default", `${primaryModel.provider}/${primaryModel.id}`);
|
||||
|
||||
session = new AgentSession({
|
||||
agent,
|
||||
sessionManager: SessionManager.inMemory(),
|
||||
settings,
|
||||
modelRegistry,
|
||||
});
|
||||
|
||||
await session.prompt("Do the work on the primary");
|
||||
await session.waitForIdle();
|
||||
expect(session.servingModel?.isFallback).toBeFalsy();
|
||||
expect(session.servingModel).toEqual({
|
||||
selector: `${primaryModel.provider}/${primaryModel.id}`,
|
||||
isFallback: false,
|
||||
});
|
||||
|
||||
await session.prompt("Fail over and die on the fallback");
|
||||
await session.waitForIdle();
|
||||
|
||||
// Routing moved; attribution stayed with the model that produced the work.
|
||||
expect(session.model?.id).toBe(fallbackModel.id);
|
||||
expect(requestedModels).toContain(`${fallbackModel.provider}/${fallbackModel.id}`);
|
||||
expect(session.servingModel?.isFallback).toBeFalsy();
|
||||
expect(session.servingModel).toEqual({
|
||||
selector: `${primaryModel.provider}/${primaryModel.id}`,
|
||||
isFallback: false,
|
||||
});
|
||||
// Both attribution and how the model was routed belong to the session they
|
||||
// were earned in. Every real switch mints a new session id — including for
|
||||
// an unpersisted session, which has no file to compare — so both drop
|
||||
// themselves, leaving only the model this session currently points at,
|
||||
// described without a claim about how it got there.
|
||||
vi.spyOn(session.sessionManager, "getSessionId").mockReturnValue("some-other-session");
|
||||
expect(session.servingModel).toEqual({
|
||||
selector: `${fallbackModel.provider}/${fallbackModel.id}`,
|
||||
isFallback: false,
|
||||
});
|
||||
});
|
||||
|
||||
it("reports the fallback selector once the target serves a turn", async () => {
|
||||
const primaryModel = getBundledModel("anthropic", "claude-sonnet-4-5");
|
||||
const fallbackModel = getBundledModel("openai", "gpt-4o-mini");
|
||||
if (!primaryModel || !fallbackModel) {
|
||||
throw new Error("Expected bundled test models to exist");
|
||||
}
|
||||
|
||||
const requestedModels: string[] = [];
|
||||
const agent = createFallbackAgent(primaryModel, requestedModels);
|
||||
const settings = Settings.isolated({
|
||||
"compaction.enabled": false,
|
||||
"retry.baseDelayMs": 5,
|
||||
"retry.fallbackChains": { default: [`${fallbackModel.provider}/${fallbackModel.id}`] },
|
||||
});
|
||||
settings.setModelRole("default", `${primaryModel.provider}/${primaryModel.id}`);
|
||||
|
||||
session = new AgentSession({
|
||||
agent,
|
||||
sessionManager: SessionManager.inMemory(),
|
||||
settings,
|
||||
modelRegistry,
|
||||
});
|
||||
|
||||
// Observers poll this per streaming event and per render. Before anything
|
||||
// has served the answer is computed rather than stored, so that is the
|
||||
// window where a fresh allocation per call would show up.
|
||||
expect(session.servingModel).toBe(session.servingModel);
|
||||
|
||||
await session.prompt("Fail over to a working fallback");
|
||||
await session.waitForIdle();
|
||||
|
||||
expect(session.model?.id).toBe(fallbackModel.id);
|
||||
expect(session.servingModel).toEqual({
|
||||
selector: `${fallbackModel.provider}/${fallbackModel.id}`,
|
||||
isFallback: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("carries attribution across a fork, which continues the conversation under a new id", async () => {
|
||||
using tempDir = TempDir.createSync("@omp-fallback-fork-");
|
||||
const primaryModel = getBundledModel("anthropic", "claude-sonnet-4-5");
|
||||
const fallbackModel = getBundledModel("openai", "gpt-4o-mini");
|
||||
if (!primaryModel || !fallbackModel) {
|
||||
throw new Error("Expected bundled test models to exist");
|
||||
}
|
||||
|
||||
const requestedModels: string[] = [];
|
||||
const agent = createFallbackAgent(primaryModel, requestedModels);
|
||||
const settings = Settings.isolated({
|
||||
"compaction.enabled": false,
|
||||
"retry.baseDelayMs": 5,
|
||||
"retry.fallbackChains": { default: [`${fallbackModel.provider}/${fallbackModel.id}`] },
|
||||
});
|
||||
settings.setModelRole("default", `${primaryModel.provider}/${primaryModel.id}`);
|
||||
|
||||
const sessionManager = SessionManager.create(tempDir.path(), tempDir.path());
|
||||
session = new AgentSession({ agent, sessionManager, settings, modelRegistry });
|
||||
|
||||
await session.prompt("Fail over to the fallback");
|
||||
await session.waitForIdle();
|
||||
const served = {
|
||||
selector: `${fallbackModel.provider}/${fallbackModel.id}`,
|
||||
isFallback: true,
|
||||
};
|
||||
expect(session.servingModel).toEqual(served);
|
||||
|
||||
const sessionIdBeforeFork = sessionManager.getSessionId();
|
||||
expect(await session.fork()).toBe(true);
|
||||
expect(sessionManager.getSessionId()).not.toBe(sessionIdBeforeFork);
|
||||
|
||||
// A fork clones the transcript and keeps running the same session, so the
|
||||
// work the fallback produced is still this session's — unlike a switch to
|
||||
// an unrelated transcript, which expires it.
|
||||
expect(session.servingModel).toEqual(served);
|
||||
});
|
||||
|
||||
it("keeps attribution on a served fallback while the next candidate is unproven", async () => {
|
||||
const primaryModel = getBundledModel("anthropic", "claude-sonnet-4-5");
|
||||
const firstFallback = getBundledModel("openai", "gpt-4o-mini");
|
||||
const secondFallback = getBundledModel("google", "gemini-2.0-flash");
|
||||
if (!primaryModel || !firstFallback || !secondFallback) {
|
||||
throw new Error("Expected bundled test models to exist");
|
||||
}
|
||||
|
||||
const requestedModels: string[] = [];
|
||||
const mock = createMockModel();
|
||||
const agent = new Agent({
|
||||
getApiKey: model => `${model.provider}-test-key`,
|
||||
initialState: { model: primaryModel, systemPrompt: ["Test"], tools: [], messages: [] },
|
||||
streamFn: (model, context, options) => {
|
||||
requestedModels.push(`${model.provider}/${model.id}`);
|
||||
// Primary fails, candidate A serves, then everything fails again so
|
||||
// candidate B is armed but never produces anything.
|
||||
mock.push(
|
||||
requestedModels.length === 2
|
||||
? { content: ["candidate A did the work"] }
|
||||
: { throw: "overloaded_error: provider returned error 503" },
|
||||
);
|
||||
return mock.stream(model, context, options);
|
||||
},
|
||||
});
|
||||
|
||||
const settings = Settings.isolated({
|
||||
"compaction.enabled": false,
|
||||
"retry.baseDelayMs": 5,
|
||||
"retry.maxRetries": 1,
|
||||
"retry.fallbackChains": {
|
||||
default: [
|
||||
`${firstFallback.provider}/${firstFallback.id}`,
|
||||
`${secondFallback.provider}/${secondFallback.id}`,
|
||||
],
|
||||
},
|
||||
});
|
||||
settings.setModelRole("default", `${primaryModel.provider}/${primaryModel.id}`);
|
||||
|
||||
session = new AgentSession({
|
||||
agent,
|
||||
sessionManager: SessionManager.inMemory(),
|
||||
settings,
|
||||
modelRegistry,
|
||||
});
|
||||
|
||||
await session.prompt("Fail over to candidate A");
|
||||
await session.waitForIdle();
|
||||
expect(session.servingModel).toEqual({
|
||||
selector: `${firstFallback.provider}/${firstFallback.id}`,
|
||||
isFallback: true,
|
||||
});
|
||||
|
||||
// `model_changed` fans out synchronously from inside the swap, which is the
|
||||
// window where the incoming candidate could inherit the previous one's proof.
|
||||
const servingAtModelChange: Array<ServingModel | undefined> = [];
|
||||
const advancing = session;
|
||||
advancing.subscribe(event => {
|
||||
if (event.type === "model_changed") servingAtModelChange.push(advancing.servingModel);
|
||||
});
|
||||
await session.prompt("Advance to candidate B and die there");
|
||||
await session.waitForIdle();
|
||||
|
||||
// Candidate B owns the routing but produced nothing, so the work still
|
||||
// belongs to candidate A — and it was reached by a fallback.
|
||||
expect(session.model?.id).toBe(secondFallback.id);
|
||||
expect(session.servingModel).toEqual({
|
||||
selector: `${firstFallback.provider}/${firstFallback.id}`,
|
||||
isFallback: true,
|
||||
});
|
||||
// Never the incoming candidate: mid-swap it has produced nothing.
|
||||
expect(servingAtModelChange.length).toBeGreaterThan(0);
|
||||
for (const serving of servingAtModelChange) {
|
||||
expect(serving?.selector).not.toBe(`${secondFallback.provider}/${secondFallback.id}`);
|
||||
}
|
||||
});
|
||||
|
||||
// A usage-aware fallback is applied before a request and never increments the
|
||||
// retry counter, so gating "served" on a retry saga hid it for the whole
|
||||
// session — most visibly on the Main Session row, which has no executor
|
||||
// progress to fall back on.
|
||||
it("reports a usage-aware fallback selector without any retry saga", async () => {
|
||||
const primaryModel = getBundledModel("anthropic", "claude-sonnet-4-5");
|
||||
const fallbackModel = getBundledModel("openai", "gpt-4o-mini");
|
||||
if (!primaryModel || !fallbackModel) {
|
||||
throw new Error("Expected bundled test models to exist");
|
||||
}
|
||||
|
||||
const requestedModels: string[] = [];
|
||||
const mock = createMockModel({ responses: [{ content: ["served on the fallback"] }] });
|
||||
const agent = new Agent({
|
||||
getApiKey: model => `${model.provider}-test-key`,
|
||||
initialState: { model: primaryModel, systemPrompt: ["Test"], tools: [], messages: [] },
|
||||
streamFn: (model, context, options) => {
|
||||
requestedModels.push(`${model.provider}/${model.id}`);
|
||||
return mock.stream(model, context, options);
|
||||
},
|
||||
});
|
||||
const settings = Settings.isolated({
|
||||
"compaction.enabled": false,
|
||||
"retry.usageAwareFallback": true,
|
||||
"retry.fallbackChains": { default: [`${fallbackModel.provider}/${fallbackModel.id}`] },
|
||||
});
|
||||
settings.setModelRole("default", `${primaryModel.provider}/${primaryModel.id}`);
|
||||
vi.spyOn(modelRegistry.authStorage, "getModelUsageHealth").mockImplementation(async provider =>
|
||||
provider === primaryModel.provider
|
||||
? { state: "depleted", accounts: [{ credentialId: 1, credentialType: "oauth", state: "depleted" }] }
|
||||
: { state: "healthy", accounts: [] },
|
||||
);
|
||||
|
||||
session = new AgentSession({
|
||||
agent,
|
||||
sessionManager: SessionManager.inMemory(),
|
||||
settings,
|
||||
modelRegistry,
|
||||
});
|
||||
|
||||
await session.prompt("Work on the healthy model");
|
||||
await session.waitForIdle();
|
||||
|
||||
// Proactive: the primary was never requested, so no retry saga ran.
|
||||
expect(requestedModels).toEqual([`${fallbackModel.provider}/${fallbackModel.id}`]);
|
||||
expect(session.servingModel).toEqual({
|
||||
selector: `${fallbackModel.provider}/${fallbackModel.id}`,
|
||||
isFallback: true,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -4,6 +4,7 @@ import { buildModel } from "@oh-my-pi/pi-catalog/build";
|
||||
import { Settings } from "@oh-my-pi/pi-coding-agent/config/settings";
|
||||
import * as sdkModule from "@oh-my-pi/pi-coding-agent/sdk";
|
||||
import type { AgentSession } from "@oh-my-pi/pi-coding-agent/session/agent-session";
|
||||
import type { ServingModel } from "@oh-my-pi/pi-coding-agent/session/retry-fallback-chains";
|
||||
import { runSubprocess } from "@oh-my-pi/pi-coding-agent/task/executor";
|
||||
import type { AgentDefinition } from "@oh-my-pi/pi-coding-agent/task/types";
|
||||
|
||||
@@ -22,11 +23,26 @@ function model(provider: string, id: string): Model<Api> {
|
||||
});
|
||||
}
|
||||
|
||||
function createYieldingSession(): AgentSession {
|
||||
/**
|
||||
* Fake session that runs a turn on its primary, applies a retry fallback, and
|
||||
* yields.
|
||||
*
|
||||
* `servingModel` mirrors the real session contract: it names the model that
|
||||
* produced output and holds the previous one while a fallback is armed but
|
||||
* unproven, so the executor is exercised against the same shape production
|
||||
* gives it.
|
||||
*
|
||||
* `fallback` picks what the target does with the switch it was handed:
|
||||
* - `"served"` settles a real turn on it, which moves attribution.
|
||||
* - `"unproven"` errors on its first request, producing none of the run's work.
|
||||
*/
|
||||
function createYieldingSession(fallback: "served" | "unproven" = "served"): AgentSession {
|
||||
const listeners: Array<(event: { type: string; [key: string]: unknown }) => void> = [];
|
||||
const session = {
|
||||
agent: { state: { systemPrompt: ["test"] } },
|
||||
state: { messages: [] },
|
||||
model: model("primary", "bad-runtime-model"),
|
||||
servingModel: { selector: "primary/bad-runtime-model", isFallback: false } as ServingModel | undefined,
|
||||
extensionRunner: undefined,
|
||||
sessionManager: { appendSessionInit: () => {} },
|
||||
getActiveToolNames: () => ["yield"],
|
||||
@@ -38,21 +54,29 @@ function createYieldingSession(): AgentSession {
|
||||
return () => {};
|
||||
},
|
||||
prompt: async () => {
|
||||
for (const listener of listeners) {
|
||||
listener({
|
||||
type: "retry_fallback_applied",
|
||||
from: "primary/bad-runtime-model",
|
||||
to: "fallback/working-model",
|
||||
role: "subagent:issue-2750",
|
||||
});
|
||||
listener({
|
||||
type: "tool_execution_end",
|
||||
toolCallId: "tool-yield",
|
||||
toolName: "yield",
|
||||
result: { content: [{ type: "text", text: "Result submitted." }], details: { status: "success" } },
|
||||
isError: false,
|
||||
});
|
||||
// Broadcast per event, not per subscriber: every observer must see the
|
||||
// same session state at the same point in the sequence.
|
||||
const emit = (event: { type: string; [key: string]: unknown }): void => {
|
||||
for (const listener of listeners) listener(event);
|
||||
};
|
||||
session.model = model("fallback", "working-model");
|
||||
emit({
|
||||
type: "retry_fallback_applied",
|
||||
from: "primary/bad-runtime-model",
|
||||
to: "fallback/working-model",
|
||||
role: "subagent:issue-2750",
|
||||
});
|
||||
if (fallback === "served") {
|
||||
session.servingModel = { selector: "fallback/working-model", isFallback: true };
|
||||
emit({ type: "retry_fallback_succeeded", model: "fallback/working-model", role: "subagent:issue-2750" });
|
||||
}
|
||||
emit({
|
||||
type: "tool_execution_end",
|
||||
toolCallId: "tool-yield",
|
||||
toolName: "yield",
|
||||
result: { content: [{ type: "text", text: "Result submitted." }], details: { status: "success" } },
|
||||
isError: false,
|
||||
});
|
||||
},
|
||||
waitForIdle: async () => {},
|
||||
getLastAssistantMessage: () => undefined,
|
||||
@@ -120,6 +144,46 @@ describe("subagent runtime model resolution", () => {
|
||||
expect(inheritedFallbackChain).toEqual(["global/inherited-model"]);
|
||||
expect(result.modelOverride).toEqual(["primary/bad-runtime-model", "fallback/working-model"]);
|
||||
expect(result.resolvedModel).toBe("fallback/working-model");
|
||||
expect(result.resolvedModelIsFallback).toBe(true);
|
||||
});
|
||||
|
||||
it("does not attribute the run to a fallback that never served a turn", async () => {
|
||||
// The incident shape: the primary does all the work, a transient error
|
||||
// routes the child onto a chain candidate, and that candidate errors on its
|
||||
// first request. Crediting the run to it reports 0 tokens of its output as
|
||||
// the whole run — to the Agent Hub row and, via the hub job snapshot, to
|
||||
// the parent model.
|
||||
const primary = model("primary", "bad-runtime-model");
|
||||
const fallback = model("fallback", "working-model");
|
||||
vi.spyOn(sdkModule, "createAgentSession").mockImplementation(async () => {
|
||||
return {
|
||||
session: createYieldingSession("unproven"),
|
||||
extensionsResult: {},
|
||||
setToolUIContext: () => {},
|
||||
} as never;
|
||||
});
|
||||
|
||||
const agent: AgentDefinition = { name: "task", description: "test", systemPrompt: "test", source: "bundled" };
|
||||
const settings = Settings.isolated({});
|
||||
settings.setModelRole("default", "primary/bad-runtime-model");
|
||||
const result = await runSubprocess({
|
||||
cwd: "/tmp",
|
||||
agent,
|
||||
task: "work",
|
||||
index: 0,
|
||||
id: "unproven-fallback",
|
||||
modelOverride: ["primary/bad-runtime-model"],
|
||||
settings,
|
||||
modelRegistry: {
|
||||
refresh: async () => {},
|
||||
getAvailable: () => [primary, fallback],
|
||||
getApiKey: async () => "test-key",
|
||||
} as never,
|
||||
enableLsp: false,
|
||||
});
|
||||
|
||||
expect(result.resolvedModel).toBe("primary/bad-runtime-model");
|
||||
expect(result.resolvedModelIsFallback).toBeFalsy();
|
||||
});
|
||||
|
||||
it("inherits an explicitly configured default fallback chain for a single subagent model", async () => {
|
||||
|
||||
@@ -0,0 +1,227 @@
|
||||
import { describe, expect, it } from "bun:test";
|
||||
import * as path from "node:path";
|
||||
import { AgentRegistry } from "@oh-my-pi/pi-coding-agent/registry/agent-registry";
|
||||
import { registerPersistedSubagents } from "@oh-my-pi/pi-coding-agent/registry/persisted-agents";
|
||||
import { TempDir } from "@oh-my-pi/pi-utils";
|
||||
|
||||
const SONNET = { provider: "anthropic", model: "claude-sonnet-5" };
|
||||
const SOL = { provider: "openai-codex", model: "gpt-5.6-sol" };
|
||||
|
||||
function assistant(
|
||||
id: string,
|
||||
parentId: string,
|
||||
who: { provider: string; model: string },
|
||||
stopReason: string,
|
||||
content: unknown[],
|
||||
): string {
|
||||
return JSON.stringify({
|
||||
type: "message",
|
||||
id,
|
||||
parentId,
|
||||
timestamp: "2026-08-07T11:00:00.000Z",
|
||||
message: {
|
||||
role: "assistant",
|
||||
content,
|
||||
provider: who.provider,
|
||||
model: who.model,
|
||||
stopReason,
|
||||
usage: { input: 10, output: 20, totalTokens: 30, cost: { total: 0.5 } },
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function modelChange(id: string, parentId: string, model: string, role: string, isFallback: boolean): string {
|
||||
return JSON.stringify({
|
||||
type: "model_change",
|
||||
id,
|
||||
parentId,
|
||||
timestamp: "2026-08-07T11:00:00.000Z",
|
||||
model,
|
||||
role,
|
||||
resolvedModelIsFallback: isFallback,
|
||||
});
|
||||
}
|
||||
|
||||
/** Head every transcript shares: a session that started on sonnet under the `task` role. */
|
||||
function transcriptHead(): string[] {
|
||||
return [
|
||||
JSON.stringify({ type: "session", id: "s0", parentId: null, timestamp: "2026-08-07T10:34:37.300Z" }),
|
||||
modelChange("m1", "s0", "anthropic/claude-sonnet-5", "task", false),
|
||||
JSON.stringify({
|
||||
type: "session_init",
|
||||
id: "si",
|
||||
parentId: "m1",
|
||||
timestamp: "2026-08-07T10:34:38.000Z",
|
||||
agent: "task",
|
||||
task: "build the thing",
|
||||
}),
|
||||
];
|
||||
}
|
||||
|
||||
/** Writes a worker transcript beside an empty root session and registers it. */
|
||||
async function historyFor(dir: string, id: string, records: string[]): Promise<AgentRegistry> {
|
||||
await Bun.write(path.join(dir, "main.jsonl"), "");
|
||||
await Bun.write(path.join(dir, "main", `${id}.jsonl`), `${records.join("\n")}\n`);
|
||||
const registry = new AgentRegistry();
|
||||
await registerPersistedSubagents(registry, path.join(dir, "main.jsonl"));
|
||||
return registry;
|
||||
}
|
||||
|
||||
describe("persisted agent model attribution", () => {
|
||||
it("reports the model that produced output, not a fallback that never served", async () => {
|
||||
using tempDir = TempDir.createSync("@omp-attribution-incident-");
|
||||
// The incident: sonnet does the work, a chain candidate errors instantly.
|
||||
const registry = await historyFor(tempDir.path(), "BuildThing", [
|
||||
...transcriptHead(),
|
||||
assistant("a1", "si", SONNET, "toolUse", [{ type: "toolCall", id: "t1", name: "read" }]),
|
||||
assistant("e1", "a1", SONNET, "error", []),
|
||||
modelChange("m2", "e1", "openai-codex/gpt-5.6-sol", "fallback", true),
|
||||
assistant("e2", "m2", SOL, "error", []),
|
||||
]);
|
||||
|
||||
const history = registry.get("BuildThing")?.history;
|
||||
expect(history?.resolvedModel).toBe("anthropic/claude-sonnet-5");
|
||||
expect(history?.resolvedModelIsFallback).toBe(false);
|
||||
// The role label survives the ephemeral fallback transition on top of it.
|
||||
expect(history?.modelRole).toBe("task");
|
||||
// Every assistant turn still counts toward the row's telemetry.
|
||||
expect(history?.metrics?.requests).toBe(3);
|
||||
});
|
||||
|
||||
it("treats a stall aborted mid-tool-call as unserved despite its partial content", async () => {
|
||||
using tempDir = TempDir.createSync("@omp-attribution-stall-");
|
||||
// A dropped stream is finalized as `aborted` with the partially streamed
|
||||
// tool call still attached, so content alone does not prove it completed.
|
||||
const registry = await historyFor(tempDir.path(), "Stalled", [
|
||||
...transcriptHead(),
|
||||
assistant("a1", "si", SONNET, "stop", [{ type: "text", text: "sonnet did the work" }]),
|
||||
assistant("e1", "a1", SONNET, "error", []),
|
||||
modelChange("m2", "e1", "openai-codex/gpt-5.6-sol", "fallback", true),
|
||||
assistant("e2", "m2", SOL, "aborted", [{ type: "toolCall", id: "t2", name: "read" }]),
|
||||
]);
|
||||
|
||||
const history = registry.get("Stalled")?.history;
|
||||
expect(history?.resolvedModel).toBe("anthropic/claude-sonnet-5");
|
||||
expect(history?.resolvedModelIsFallback).toBe(false);
|
||||
});
|
||||
|
||||
it("treats a substantively empty stop as unserved despite a non-empty content array", async () => {
|
||||
using tempDir = TempDir.createSync("@omp-attribution-empty-stop-");
|
||||
// A `stop` carrying only whitespace text and unsigned thinking produced
|
||||
// nothing actionable — it is what the empty-stop retry machinery exists to
|
||||
// recover from. A content-length check alone would credit the candidate.
|
||||
const registry = await historyFor(tempDir.path(), "EmptyStop", [
|
||||
...transcriptHead(),
|
||||
assistant("a1", "si", SONNET, "stop", [{ type: "text", text: "sonnet did the work" }]),
|
||||
assistant("e1", "a1", SONNET, "error", []),
|
||||
modelChange("m2", "e1", "openai-codex/gpt-5.6-sol", "fallback", true),
|
||||
assistant("e2", "m2", SOL, "stop", [
|
||||
{ type: "thinking", thinking: " ", thinkingSignature: "" },
|
||||
{ type: "text", text: " " },
|
||||
]),
|
||||
]);
|
||||
|
||||
const history = registry.get("EmptyStop")?.history;
|
||||
expect(history?.resolvedModel).toBe("anthropic/claude-sonnet-5");
|
||||
expect(history?.resolvedModelIsFallback).toBe(false);
|
||||
});
|
||||
|
||||
it("credits a turn whose only output is an image", async () => {
|
||||
using tempDir = TempDir.createSync("@omp-attribution-image-");
|
||||
// A native image response can arrive with no text and no tool call at all.
|
||||
// Recognising only those would call it nothing and leave the run credited
|
||||
// to whichever model spoke before it.
|
||||
const registry = await historyFor(tempDir.path(), "Painter", [
|
||||
...transcriptHead(),
|
||||
assistant("a1", "si", SONNET, "stop", [{ type: "text", text: "sonnet did the work" }]),
|
||||
assistant("e1", "a1", SONNET, "error", []),
|
||||
modelChange("m2", "e1", "openai-codex/gpt-5.6-sol", "fallback", true),
|
||||
assistant("a2", "m2", SOL, "stop", [{ type: "image", data: "aGk=", mimeType: "image/png" }]),
|
||||
]);
|
||||
|
||||
const history = registry.get("Painter")?.history;
|
||||
expect(history?.resolvedModel).toBe("openai-codex/gpt-5.6-sol");
|
||||
expect(history?.resolvedModelIsFallback).toBe(true);
|
||||
});
|
||||
|
||||
it("treats a budget-exhausted length stop with nothing usable as unserved", async () => {
|
||||
using tempDir = TempDir.createSync("@omp-attribution-length-");
|
||||
// `length` is not an "empty stop", so the empty-stop rule never inspects it:
|
||||
// a candidate that burned its whole output budget on unsigned thinking still
|
||||
// produced nothing to attribute.
|
||||
const registry = await historyFor(tempDir.path(), "OutOfBudget", [
|
||||
...transcriptHead(),
|
||||
assistant("a1", "si", SONNET, "stop", [{ type: "text", text: "sonnet did the work" }]),
|
||||
assistant("e1", "a1", SONNET, "error", []),
|
||||
modelChange("m2", "e1", "openai-codex/gpt-5.6-sol", "fallback", true),
|
||||
assistant("e2", "m2", SOL, "length", [{ type: "thinking", thinking: "spent it all", thinkingSignature: "" }]),
|
||||
]);
|
||||
|
||||
const history = registry.get("OutOfBudget")?.history;
|
||||
expect(history?.resolvedModel).toBe("anthropic/claude-sonnet-5");
|
||||
expect(history?.resolvedModelIsFallback).toBe(false);
|
||||
});
|
||||
|
||||
it("summarizes a transcript carrying malformed content blocks", async () => {
|
||||
using tempDir = TempDir.createSync("@omp-attribution-malformed-");
|
||||
// Transcripts outlive the shapes that wrote them. A block that is null or
|
||||
// missing its `text` must not throw: the reader catches and returns an
|
||||
// empty summary, blanking the whole row over one bad line.
|
||||
const registry = await historyFor(tempDir.path(), "Legacy", [
|
||||
...transcriptHead(),
|
||||
assistant("a1", "si", SONNET, "stop", [null, { type: "text" }]),
|
||||
assistant("a2", "a1", SONNET, "stop", [{ type: "text", text: "recovered and did the work" }]),
|
||||
]);
|
||||
|
||||
const history = registry.get("Legacy")?.history;
|
||||
expect(history?.resolvedModel).toBe("anthropic/claude-sonnet-5");
|
||||
expect(history?.metrics?.requests).toBe(2);
|
||||
});
|
||||
|
||||
it("labels the row with the newest role a transition assigned, skipping ephemeral ones", async () => {
|
||||
using tempDir = TempDir.createSync("@omp-attribution-role-");
|
||||
// Two real role transitions plus an ephemeral fallback on top: the label
|
||||
// must be the latest deliberate role, not the first one nor the fallback.
|
||||
const registry = await historyFor(tempDir.path(), "Rerolled", [
|
||||
...transcriptHead(),
|
||||
assistant("a1", "si", SONNET, "stop", [{ type: "text", text: "first role" }]),
|
||||
modelChange("m2", "a1", "openai-codex/gpt-5.6-sol", "slow", false),
|
||||
assistant("a2", "m2", SOL, "stop", [{ type: "text", text: "second role" }]),
|
||||
modelChange("m3", "a2", "openai-codex/gpt-5.6-sol", "fallback", true),
|
||||
]);
|
||||
|
||||
const history = registry.get("Rerolled")?.history;
|
||||
expect(history?.modelRole).toBe("slow");
|
||||
});
|
||||
|
||||
it("reports the fallback once it has served a turn", async () => {
|
||||
using tempDir = TempDir.createSync("@omp-attribution-served-");
|
||||
const registry = await historyFor(tempDir.path(), "Worker", [
|
||||
...transcriptHead(),
|
||||
assistant("e1", "si", SONNET, "error", []),
|
||||
modelChange("m2", "e1", "openai-codex/gpt-5.6-sol", "fallback", true),
|
||||
assistant("a1", "m2", SOL, "toolUse", [{ type: "toolCall", id: "t1", name: "read" }]),
|
||||
]);
|
||||
|
||||
const history = registry.get("Worker")?.history;
|
||||
expect(history?.resolvedModel).toBe("openai-codex/gpt-5.6-sol");
|
||||
expect(history?.resolvedModelIsFallback).toBe(true);
|
||||
});
|
||||
|
||||
it("matches a served model to a transition carrying a gateway route", async () => {
|
||||
using tempDir = TempDir.createSync("@omp-attribution-routed-");
|
||||
// Writers record the selector through `formatModelStringWithRouting`, which
|
||||
// appends an `@upstream` gateway route the raw message never carries.
|
||||
// Failing to match drops the fallback flag.
|
||||
const registry = await historyFor(tempDir.path(), "Routed", [
|
||||
...transcriptHead(),
|
||||
assistant("e1", "si", SONNET, "error", []),
|
||||
modelChange("m2", "e1", "openai-codex/gpt-5.6-sol@vercel-gw", "fallback", true),
|
||||
assistant("a1", "m2", SOL, "stop", [{ type: "text", text: "served via the gateway" }]),
|
||||
]);
|
||||
|
||||
const history = registry.get("Routed")?.history;
|
||||
expect(history?.resolvedModel).toBe("openai-codex/gpt-5.6-sol@vercel-gw");
|
||||
expect(history?.resolvedModelIsFallback).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -30,10 +30,15 @@ function yieldEmittingSession(
|
||||
): AgentSession {
|
||||
const listeners: Array<(event: AgentSessionEvent) => void> = [];
|
||||
let activeTools = initialTools;
|
||||
// `servingModel` mirrors the real session: attribution names the model that
|
||||
// produced output, so a prewalk hand-off moves it along with `model`.
|
||||
const serving = (model: Model | undefined): { selector: string; isFallback: boolean } | undefined =>
|
||||
model ? { selector: `${model.provider}/${model.id}`, isFallback: false } : undefined;
|
||||
const session = {
|
||||
state: { messages: [] },
|
||||
agent: { state: { systemPrompt: ["test"] } },
|
||||
model: modelSwitch?.from,
|
||||
servingModel: serving(modelSwitch?.from),
|
||||
extensionRunner: undefined,
|
||||
sessionManager: { appendSessionInit: () => {} },
|
||||
getActiveToolNames: () => activeTools,
|
||||
@@ -52,6 +57,7 @@ function yieldEmittingSession(
|
||||
prompt: async (_text: string, _options?: PromptOptions) => {
|
||||
if (modelSwitch) {
|
||||
session.model = modelSwitch.to;
|
||||
session.servingModel = serving(modelSwitch.to);
|
||||
for (const listener of listeners) {
|
||||
listener({ type: "notice", level: "info", message: "Prewalk switched", source: "prewalk" });
|
||||
}
|
||||
|
||||
@@ -111,6 +111,7 @@ describe("TurnRecovery replay-unsafe output classification", () => {
|
||||
host.model = () => activeModel;
|
||||
host.sessionManager = {
|
||||
appendModelChange: (selector: string) => modelChanges.push(selector),
|
||||
getSessionId: () => "replay-unsafe-session",
|
||||
} as never;
|
||||
host.setModelWithProviderSessionReset = async nextModel => {
|
||||
activeModel = nextModel;
|
||||
@@ -161,6 +162,7 @@ describe("TurnRecovery replay-unsafe output classification", () => {
|
||||
host.model = () => activeModel;
|
||||
host.sessionManager = {
|
||||
appendModelChange: (selector: string) => modelChanges.push(selector),
|
||||
getSessionId: () => "replay-unsafe-session",
|
||||
} as never;
|
||||
host.setThinkingLevel = level => thinkingChanges.push(level);
|
||||
host.setModelWithProviderSessionReset = async nextModel => {
|
||||
@@ -208,6 +210,7 @@ describe("TurnRecovery replay-unsafe output classification", () => {
|
||||
host.model = () => activeModel;
|
||||
host.sessionManager = {
|
||||
appendModelChange: (selector: string) => modelChanges.push(selector),
|
||||
getSessionId: () => "replay-unsafe-session",
|
||||
} as never;
|
||||
host.setModelWithProviderSessionReset = async nextModel => {
|
||||
activeModel = nextModel;
|
||||
|
||||
Reference in New Issue
Block a user