fix(coding-agent): added configurable retry delay cap and surfaced rate-limit failure state
- Added `retry.maxDelayMs` to the settings schema and interfaces, with a default cap for provider backoff delays. - Updated session auto-retry logic to fail fast when a requested wait exceeds the cap without fallback, emitting terminal auto-retry failure state. - Propagated retry state and failure data into task progress and rendering so children show retry/wait details and reminder prompts stop after terminal errors.
This commit is contained in:
can1357
@@ -836,6 +836,16 @@ export const SETTINGS_SCHEMA = {
|
||||
},
|
||||
|
||||
"retry.baseDelayMs": { type: "number", default: 2000 },
|
||||
"retry.maxDelayMs": {
|
||||
type: "number",
|
||||
default: 5 * 60 * 1000,
|
||||
ui: {
|
||||
tab: "model",
|
||||
label: "Max Retry Delay",
|
||||
description:
|
||||
"Maximum wait between retries, in ms. When the provider asks us to wait longer than this and no credential or model fallback succeeds, the request fails fast instead of sleeping (e.g. 3-hour Anthropic rate-limit windows).",
|
||||
},
|
||||
},
|
||||
"retry.fallbackChains": { type: "record", default: {} as Record<string, string[]> },
|
||||
"retry.fallbackRevertPolicy": {
|
||||
type: "enum",
|
||||
@@ -2859,6 +2869,7 @@ export interface RetrySettings {
|
||||
enabled: boolean;
|
||||
maxRetries: number;
|
||||
baseDelayMs: number;
|
||||
maxDelayMs: number;
|
||||
}
|
||||
|
||||
export interface MemoriesSettings {
|
||||
|
||||
@@ -7108,6 +7108,27 @@ export class AgentSession {
|
||||
}
|
||||
}
|
||||
|
||||
// Fail-fast cap: if the provider asks us to wait longer than
|
||||
// retry.maxDelayMs and we have no fallback credential or model to
|
||||
// switch to, surface the error instead of sleeping. Defends against
|
||||
// 3-hour Anthropic rate-limit windows that would otherwise leave a
|
||||
// subagent (or interactive session) silently hung. The original
|
||||
// assistant error message is preserved in agent state so the caller
|
||||
// can act on it.
|
||||
const maxDelayMs = retrySettings.maxDelayMs;
|
||||
if (maxDelayMs > 0 && delayMs > maxDelayMs && !switchedCredential && !switchedModel) {
|
||||
const attempt = this.#retryAttempt;
|
||||
this.#retryAttempt = 0;
|
||||
await this.#emitSessionEvent({
|
||||
type: "auto_retry_end",
|
||||
success: false,
|
||||
attempt,
|
||||
finalError: `Provider requested ${delayMs}ms wait, exceeds retry.maxDelayMs (${maxDelayMs}ms). Original error: ${errorMessage}`,
|
||||
});
|
||||
this.#resolveRetry();
|
||||
return false;
|
||||
}
|
||||
|
||||
await this.#emitSessionEvent({
|
||||
type: "auto_retry_start",
|
||||
attempt: this.#retryAttempt,
|
||||
|
||||
@@ -1350,6 +1350,30 @@ export async function runSubprocess(options: ExecutorOptions): Promise<SingleRes
|
||||
|
||||
const MAX_YIELD_RETRIES = 3;
|
||||
unsubscribe = session.subscribe(event => {
|
||||
if (event.type === "auto_retry_start") {
|
||||
progress.retryState = {
|
||||
attempt: event.attempt,
|
||||
maxAttempts: event.maxAttempts,
|
||||
delayMs: event.delayMs,
|
||||
errorMessage: event.errorMessage,
|
||||
startedAtMs: Date.now(),
|
||||
};
|
||||
progress.retryFailure = undefined;
|
||||
scheduleProgress(true);
|
||||
return;
|
||||
}
|
||||
if (event.type === "auto_retry_end") {
|
||||
const attempt = progress.retryState?.attempt ?? event.attempt;
|
||||
progress.retryState = undefined;
|
||||
if (!event.success) {
|
||||
progress.retryFailure = {
|
||||
attempt,
|
||||
errorMessage: event.finalError ?? "Auto-retry failed",
|
||||
};
|
||||
}
|
||||
scheduleProgress(true);
|
||||
return;
|
||||
}
|
||||
if (isAgentEvent(event)) {
|
||||
try {
|
||||
processEvent(event);
|
||||
@@ -1385,6 +1409,12 @@ export async function runSubprocess(options: ExecutorOptions): Promise<SingleRes
|
||||
|
||||
let retryCount = 0;
|
||||
while (!yieldCalled && retryCount < MAX_YIELD_RETRIES && !abortSignal.aborted) {
|
||||
// Skip reminders when the model returned a terminal error (e.g.
|
||||
// rate-limit cap hit, auth failure). Re-prompting would just
|
||||
// hit the same wall, multiplying the failure noise without
|
||||
// any chance of producing a yield.
|
||||
const lastBeforeReminder = session.getLastAssistantMessage();
|
||||
if (lastBeforeReminder?.stopReason === "error") break;
|
||||
try {
|
||||
retryCount++;
|
||||
const reminder = prompt.render(submitReminderTemplate, {
|
||||
@@ -1584,6 +1614,7 @@ export async function runSubprocess(options: ExecutorOptions): Promise<SingleRes
|
||||
usage: hasUsage ? accumulatedUsage : undefined,
|
||||
outputPath,
|
||||
extractedToolData: progress.extractedToolData,
|
||||
retryFailure: progress.retryFailure,
|
||||
outputMeta,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -410,6 +410,8 @@ export class TaskTool implements AgentTool<TaskToolSchemaInstance, TaskToolDetai
|
||||
progress.contextWindow = singleResult?.contextWindow;
|
||||
progress.cost = singleResult?.usage?.cost.total ?? 0;
|
||||
progress.extractedToolData = singleResult?.extractedToolData;
|
||||
progress.retryFailure = singleResult?.retryFailure;
|
||||
progress.retryState = undefined;
|
||||
}
|
||||
completedJobs += 1;
|
||||
if (singleResult && ((singleResult.aborted ?? false) || singleResult.exitCode !== 0)) {
|
||||
|
||||
@@ -551,8 +551,15 @@ function renderAgentProgress(
|
||||
const titlePart = description ? `${theme.bold(displayId)}: ${description}` : displayId;
|
||||
let statusLine = `${prefix} ${theme.fg(iconColor, icon)} ${theme.fg("accent", titlePart)}`;
|
||||
|
||||
// Only show badge for non-running states (spinner already indicates running)
|
||||
if (progress.status === "failed" || progress.status === "aborted") {
|
||||
// Show retry-blocked badge so the parent immediately sees that a child
|
||||
// is sleeping on a provider 429, not silently progressing. Wins over the
|
||||
// generic running spinner because "we're waiting on a quota window" is
|
||||
// the operationally meaningful state.
|
||||
if (progress.retryState && progress.status === "running") {
|
||||
statusLine += ` ${formatBadge("retrying", "warning", theme)}`;
|
||||
} else if (progress.retryFailure && (progress.status === "failed" || progress.status === "aborted")) {
|
||||
statusLine += ` ${formatBadge("rate-limited", "error", theme)}`;
|
||||
} else if (progress.status === "failed" || progress.status === "aborted") {
|
||||
const statusLabel = progress.status === "failed" ? "failed" : "aborted";
|
||||
statusLine += ` ${formatBadge(statusLabel, iconColor, theme)}`;
|
||||
}
|
||||
@@ -598,6 +605,26 @@ function renderAgentProgress(
|
||||
}
|
||||
}
|
||||
|
||||
// Retry detail line: surface why the subagent is paused and roughly how
|
||||
// long until the next attempt. Without this, the parent UI would just
|
||||
// keep spinning while a child sleeps on a 3-hour provider rate-limit.
|
||||
if (progress.retryState && progress.status === "running") {
|
||||
const remainingMs = Math.max(
|
||||
0,
|
||||
progress.retryState.startedAtMs + progress.retryState.delayMs - Date.now(),
|
||||
);
|
||||
const waitLabel = remainingMs > 0 ? `in ${formatDuration(remainingMs)}` : "now";
|
||||
const summary =
|
||||
`retrying ${progress.retryState.attempt}/${progress.retryState.maxAttempts} ${waitLabel}: ` +
|
||||
truncateToWidth(replaceTabs(progress.retryState.errorMessage), 60);
|
||||
lines.push(`${continuePrefix}${theme.tree.hook} ${theme.fg("warning", summary)}`);
|
||||
} else if (progress.retryFailure && progress.status !== "running") {
|
||||
const summary = `auto-retry gave up after ${progress.retryFailure.attempt} attempt${
|
||||
progress.retryFailure.attempt === 1 ? "" : "s"
|
||||
}: ${truncateToWidth(replaceTabs(progress.retryFailure.errorMessage), 80)}`;
|
||||
lines.push(`${continuePrefix}${theme.tree.hook} ${theme.fg("error", summary)}`);
|
||||
}
|
||||
|
||||
// Render extracted tool data inline (e.g., review findings)
|
||||
if (progress.extractedToolData) {
|
||||
// For completed tasks, check for review verdict from yield tool
|
||||
|
||||
@@ -212,6 +212,30 @@ export interface AgentProgress {
|
||||
modelOverride?: string | string[];
|
||||
/** Data extracted by registered subprocess tool handlers (keyed by tool name) */
|
||||
extractedToolData?: Record<string, unknown[]>;
|
||||
/**
|
||||
* Auto-retry state when the subagent is sleeping between provider retries
|
||||
* (e.g. 429 rate-limit with retry-after). Cleared when the retry resolves
|
||||
* or fails. Surfacing this to the parent prevents the task tool from
|
||||
* looking indefinitely "in progress" when a child is actually blocked on
|
||||
* provider quota.
|
||||
*/
|
||||
retryState?: {
|
||||
attempt: number;
|
||||
maxAttempts: number;
|
||||
delayMs: number;
|
||||
errorMessage: string;
|
||||
startedAtMs: number;
|
||||
};
|
||||
/**
|
||||
* Terminal retry failure surfaced once the subagent gave up retrying
|
||||
* (e.g. retry-after exceeded the cap, or all attempts exhausted). Carries
|
||||
* the final error so the parent UI can render "blocked: rate-limited"
|
||||
* instead of waiting for a status that never arrives.
|
||||
*/
|
||||
retryFailure?: {
|
||||
attempt: number;
|
||||
errorMessage: string;
|
||||
};
|
||||
}
|
||||
|
||||
/** Result from a single agent execution */
|
||||
@@ -251,6 +275,16 @@ export interface SingleResult {
|
||||
nestedPatches?: NestedRepoPatch[];
|
||||
/** Data extracted by registered subprocess tool handlers (keyed by tool name) */
|
||||
extractedToolData?: Record<string, unknown[]>;
|
||||
/**
|
||||
* Terminal retry failure, when the subagent exited because the auto-retry
|
||||
* loop gave up (retry-after exceeded the cap, or all attempts exhausted).
|
||||
* Lets the parent task tool surface a "blocked: rate-limited" outcome
|
||||
* instead of a generic failure.
|
||||
*/
|
||||
retryFailure?: {
|
||||
attempt: number;
|
||||
errorMessage: string;
|
||||
};
|
||||
/** Output metadata for agent:// URL integration */
|
||||
outputMeta?: { lineCount: number; charCount: number };
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user