feat(ai,coding-agent): added Anthropic fast mode with auto-fallback

Wires `speed: "fast"` and the `fast-mode-2026-02-01` beta into the
Anthropic provider, plumbs a matching `speed` option through
`SimpleStreamOptions` and the Agent, and teaches `/fast` to dispatch
on the active model's API (Anthropic -> speed=fast, OpenAI -> existing
serviceTier=priority path). Server is the authority on which models
support fast mode.

When the server rejects an unsupported model, the provider mirrors the
strict-tools fallback: drops the field, retries the same turn
transparently, persists the disable via `providerSessionState`, and
surfaces the action through the new `AssistantMessage.disabledFeatures`
marker so the session can sync the toggle off and warn the user.
This commit is contained in:
Lucas Szwarcberg
2026-05-18 11:48:40 -07:00
parent 12e92262e8
commit d0efcea5ad
11 changed files with 247 additions and 2 deletions
+13
View File
@@ -167,6 +167,8 @@ export interface AgentOptions {
presencePenalty?: number;
repetitionPenalty?: number;
serviceTier?: ServiceTier;
/** Anthropic fast mode (`speed: "fast"`). Ignored by non-Anthropic providers. */
speed?: "fast" | "standard";
/**
* If true, request that the underlying provider omit reasoning/thinking summaries
* from the response. The model still reasons internally; only the human-readable
@@ -273,6 +275,7 @@ export class Agent {
#presencePenalty?: number;
#repetitionPenalty?: number;
#serviceTier?: ServiceTier;
#speed?: "fast" | "standard";
#hideThinkingSummary?: boolean;
#maxRetryDelayMs?: number;
#getToolContext?: (toolCall?: ToolCallContext) => AgentToolContext | undefined;
@@ -326,6 +329,7 @@ export class Agent {
this.#presencePenalty = opts.presencePenalty;
this.#repetitionPenalty = opts.repetitionPenalty;
this.#serviceTier = opts.serviceTier;
this.#speed = opts.speed;
this.#hideThinkingSummary = opts.hideThinkingSummary;
this.#maxRetryDelayMs = opts.maxRetryDelayMs;
this.getApiKey = opts.getApiKey;
@@ -513,6 +517,14 @@ export class Agent {
this.#serviceTier = value;
}
get speed(): "fast" | "standard" | undefined {
return this.#speed;
}
set speed(value: "fast" | "standard" | undefined) {
this.#speed = value;
}
get hideThinkingSummary(): boolean | undefined {
return this.#hideThinkingSummary;
}
@@ -892,6 +904,7 @@ export class Agent {
presencePenalty: this.#presencePenalty,
repetitionPenalty: this.#repetitionPenalty,
serviceTier: this.#serviceTier,
speed: this.#speed,
hideThinkingSummary: this.#hideThinkingSummary,
interruptMode: this.#interruptMode,
sessionId: this.#sessionId,
+6
View File
@@ -31,6 +31,11 @@ export interface ServiceTierChangeEntry extends SessionEntryBase {
serviceTier: ServiceTier | null;
}
export interface SpeedChangeEntry extends SessionEntryBase {
type: "speed_change";
speed: "fast" | "standard" | null;
}
export interface CompactionEntry<T = unknown> extends SessionEntryBase {
type: "compaction";
summary: string;
@@ -116,6 +121,7 @@ export type SessionEntry =
| ThinkingLevelChangeEntry
| ModelChangeEntry
| ServiceTierChangeEntry
| SpeedChangeEntry
| CompactionEntry
| BranchSummaryEntry
| CustomEntry
+4
View File
@@ -1,6 +1,10 @@
# Changelog
## [Unreleased]
### Added
- Added Anthropic fast mode support via `speed: "fast" | "standard"` on `SimpleStreamOptions` and `AnthropicOptions`. The provider sets the top-level `speed` field on the Messages request and appends the `fast-mode-2026-02-01` beta to the client's `Anthropic-Beta` header. When the server rejects an unsupported model with `invalid_request_error`, the provider transparently retries the same turn without `speed` (mirroring the strict-tools fallback pattern), persists the disable via a new `providerSessionState.fastModeDisabled` flag so subsequent requests in the session skip the field, and surfaces the action via the new `AssistantMessage.disabledFeatures` array so callers can sync user-facing toggles.
### Changed
- Updated auth-gateway format and pi-native request handling to invalidate the failed API key and retry the provider request with a replacement key when authentication fails
+65 -1
View File
@@ -111,6 +111,7 @@ const claudeCodeBetaDefaults = [
];
const fineGrainedToolStreamingBeta = "fine-grained-tool-streaming-2025-05-14";
const interleavedThinkingBeta = "interleaved-thinking-2025-05-14";
const fastModeBeta = "fast-mode-2026-02-01";
function getHeaderCaseInsensitive(headers: Record<string, string> | undefined, headerName: string): string | undefined {
if (!headers) return undefined;
@@ -224,13 +225,16 @@ const ANTHROPIC_PROVIDER_SESSION_STATE_KEY = "anthropic-messages";
type AnthropicProviderSessionState = ProviderSessionState & {
strictToolsDisabled: boolean;
fastModeDisabled: boolean;
};
function createAnthropicProviderSessionState(): AnthropicProviderSessionState {
const state: AnthropicProviderSessionState = {
strictToolsDisabled: false,
fastModeDisabled: false,
close: () => {
state.strictToolsDisabled = false;
state.fastModeDisabled = false;
},
};
return state;
@@ -258,11 +262,22 @@ function isAnthropicStrictGrammarTooLargeError(error: unknown): boolean {
return /invalid_request_error/i.test(message) && (isStrictGrammarTooLarge || isSchemaCompilationTooComplex);
}
function isAnthropicFastModeUnsupportedError(error: unknown): boolean {
if (extractHttpStatusFromError(error) !== 400) return false;
const message = error instanceof Error ? error.message : String(error);
// Server message: "'claude-opus-4-5-20251101' does not support the `speed` parameter."
return /invalid_request_error/i.test(message) && /`speed`/i.test(message) && /does not support/i.test(message);
}
function hasStrictAnthropicTools(params: MessageCreateParamsStreaming): boolean {
const tools = params.tools as Array<{ strict?: unknown }> | undefined;
return tools?.some(tool => tool.strict === true) ?? false;
}
function dropAnthropicFastMode(params: MessageCreateParamsStreaming): void {
delete (params as unknown as Record<string, unknown>).speed;
}
function dropAnthropicStrictTools(params: MessageCreateParamsStreaming): void {
const tools = params.tools as Array<{ strict?: unknown }> | undefined;
if (!tools) return;
@@ -526,6 +541,12 @@ export interface AnthropicOptions extends StreamOptions {
interleavedThinking?: boolean;
toolChoice?: "auto" | "any" | "none" | { type: "tool"; name: string };
betas?: string[] | string;
/**
* Anthropic fast mode. When `"fast"`, sets the `speed` request field and
* sends the `fast-mode-2026-02-01` beta header. Server rejects unsupported
* models with `invalid_request_error`.
*/
speed?: "fast" | "standard";
/** Force OAuth bearer auth mode for proxy tokens that don't match Anthropic token prefixes. */
isOAuth?: boolean;
/**
@@ -961,10 +982,15 @@ export const streamAnthropic: StreamFunction<"anthropic-messages"> = (
} else {
const apiKey = options?.apiKey ?? getEnvApiKey(model.provider) ?? "";
const extraBetas = normalizeExtraBetas(options?.betas);
if (options?.speed === "fast" && !extraBetas.includes(fastModeBeta)) {
extraBetas.push(fastModeBeta);
}
const created = createClient(model, {
model,
apiKey,
extraBetas: normalizeExtraBetas(options?.betas),
extraBetas,
stream: true,
interleavedThinking: options?.interleavedThinking ?? true,
headers: options?.headers,
@@ -984,6 +1010,7 @@ export const streamAnthropic: StreamFunction<"anthropic-messages"> = (
let disableStrictTools =
(providerSessionState?.strictToolsDisabled ?? false) || (model.compat?.disableStrictTools ?? false);
let strictFallbackErrorMessage: string | undefined;
let dropFastMode = providerSessionState?.fastModeDisabled ?? false;
const prepareParams = async (): Promise<MessageCreateParamsStreaming> => {
let nextParams = buildParams(model, baseUrl, context, isOAuthToken, options, disableStrictTools);
const replacementPayload = await options?.onPayload?.(nextParams, model);
@@ -993,6 +1020,9 @@ export const streamAnthropic: StreamFunction<"anthropic-messages"> = (
if (disableStrictTools) {
dropAnthropicStrictTools(nextParams);
}
if (dropFastMode) {
dropAnthropicFastMode(nextParams);
}
rawRequestDump = {
provider: model.provider,
api: output.api,
@@ -1284,6 +1314,30 @@ export const streamAnthropic: StreamFunction<"anthropic-messages"> = (
firstTokenTime = undefined;
continue;
}
if (
!dropFastMode &&
options?.speed === "fast" &&
firstTokenTime === undefined &&
isAnthropicFastModeUnsupportedError(streamFailure)
) {
logger.debug("anthropic: fast mode unsupported, retrying without speed", {
model: model.id,
error: streamFailure instanceof Error ? streamFailure.message : String(streamFailure),
});
if (providerSessionState) {
providerSessionState.fastModeDisabled = true;
}
dropFastMode = true;
params = await prepareParams();
providerRetryAttempt = 0;
output.content.length = 0;
output.responseId = undefined;
output.providerPayload = undefined;
output.usage = createEmptyUsage(copilotDynamicHeaders?.premiumRequests);
output.stopReason = "stop";
firstTokenTime = undefined;
continue;
}
const isTransientEnvelopeFailure =
isTransientStreamParseError(streamFailure) || isTransientStreamEnvelopeError(streamFailure);
const canRetryTransientEnvelopeFailure = isTransientEnvelopeFailure && !streamedReplayUnsafeContent;
@@ -1315,6 +1369,9 @@ export const streamAnthropic: StreamFunction<"anthropic-messages"> = (
output.duration = Date.now() - startTime;
if (firstTokenTime) output.ttft = firstTokenTime - startTime;
if (dropFastMode && options?.speed === "fast") {
output.disabledFeatures = [...(output.disabledFeatures ?? []), "anthropic.fast_mode"];
}
stream.push({ type: "done", reason: output.stopReason, message: output });
stream.end();
} catch (error) {
@@ -1862,6 +1919,13 @@ function buildParams(
params.metadata = { user_id: metadataUserId };
}
if (options?.speed === "fast") {
// `speed` is typed on `BetaMessageCreateParams` (client.beta.messages),
// but this provider posts via `client.messages.create` whose param
// type doesn't include it. Cast to inject the field.
(params as unknown as Record<string, unknown>).speed = "fast";
}
if (options?.toolChoice) {
if (typeof options.toolChoice === "string") {
params.tool_choice = { type: options.toolChoice };
+6
View File
@@ -580,6 +580,7 @@ function mapOptionsForApi<TApi extends Api>(
thinkingEnabled: false,
toolChoice: mapAnthropicToolChoice(options?.toolChoice),
thinkingDisplay: options?.hideThinkingSummary ? "omitted" : undefined,
speed: options?.speed,
});
}
@@ -590,6 +591,7 @@ function mapOptionsForApi<TApi extends Api>(
thinkingEnabled: false,
toolChoice: mapAnthropicToolChoice(options?.toolChoice),
thinkingDisplay: options?.hideThinkingSummary ? "omitted" : undefined,
speed: options?.speed,
});
}
@@ -603,6 +605,7 @@ function mapOptionsForApi<TApi extends Api>(
effort,
toolChoice: mapAnthropicToolChoice(options?.toolChoice),
thinkingDisplay: options?.hideThinkingSummary ? "omitted" : undefined,
speed: options?.speed,
});
}
@@ -613,6 +616,7 @@ function mapOptionsForApi<TApi extends Api>(
thinkingBudgetTokens: thinkingBudget,
toolChoice: mapAnthropicToolChoice(options?.toolChoice),
thinkingDisplay: options?.hideThinkingSummary ? "omitted" : undefined,
speed: options?.speed,
});
}
@@ -631,6 +635,7 @@ function mapOptionsForApi<TApi extends Api>(
thinkingEnabled: false,
toolChoice: mapAnthropicToolChoice(options?.toolChoice),
thinkingDisplay: options?.hideThinkingSummary ? "omitted" : undefined,
speed: options?.speed,
});
} else {
return castApi<"anthropic-messages">({
@@ -640,6 +645,7 @@ function mapOptionsForApi<TApi extends Api>(
thinkingBudgetTokens: thinkingBudget,
toolChoice: mapAnthropicToolChoice(options?.toolChoice),
thinkingDisplay: options?.hideThinkingSummary ? "omitted" : undefined,
speed: options?.speed,
});
}
}
+10
View File
@@ -348,6 +348,8 @@ export interface SimpleStreamOptions extends StreamOptions {
toolChoice?: ToolChoice;
/** OpenAI service tier for processing priority/cost control. Ignored by non-OpenAI providers. */
serviceTier?: ServiceTier;
/** Anthropic fast mode (`speed: "fast"`, beta `fast-mode-2026-02-01`). Ignored by non-Anthropic providers. */
speed?: "fast" | "standard";
/** API format for Kimi Code provider: "openai" or "anthropic" (default: "anthropic") */
kimiApiFormat?: "openai" | "anthropic";
/** API format for Synthetic provider: "openai" or "anthropic" (default: "openai") */
@@ -502,6 +504,14 @@ export interface AssistantMessage {
errorMessage?: string;
/** HTTP status surfaced by the provider when the request failed. Populated by every provider's catch block alongside `errorMessage` so consumers (auth retry, telemetry, UI) can branch without regex-scraping the message. */
errorStatus?: number;
/**
* Stable identifiers for request features the provider silently dropped
* during this turn (e.g. `"anthropic.fast_mode"`). Set when a server-side
* rejection triggered an in-provider fallback retry that succeeded
* without the feature. Callers can use this to sync user-facing toggles
* back to the server's actual state.
*/
disabledFeatures?: string[];
/** Provider-specific opaque payload used to reconstruct transport-native history. */
providerPayload?: ProviderPayload;
timestamp: number; // Unix timestamp in milliseconds
@@ -0,0 +1,85 @@
import { describe, expect, it } from "bun:test";
import { streamAnthropic } from "@oh-my-pi/pi-ai/providers/anthropic";
import type { Context, Model } from "@oh-my-pi/pi-ai/types";
function makeAnthropicModel(id: string): Model<"anthropic-messages"> {
return {
id,
name: id,
api: "anthropic-messages",
provider: "anthropic",
baseUrl: "https://api.anthropic.com",
reasoning: true,
input: ["text", "image"],
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
contextWindow: 200_000,
maxTokens: 8_192,
};
}
const CONTEXT: Context = {
systemPrompt: ["Stay concise."],
messages: [{ role: "user", content: "Hi", timestamp: Date.now() }],
};
function abortedSignal(): AbortSignal {
const controller = new AbortController();
controller.abort();
return controller.signal;
}
function capturePayload(model: Model<"anthropic-messages">, speed: "fast" | "standard" | undefined): Promise<unknown> {
const { promise, resolve } = Promise.withResolvers<unknown>();
streamAnthropic(model, CONTEXT, {
apiKey: "sk-ant-oat-test",
isOAuth: true,
signal: abortedSignal(),
speed,
onPayload: payload => resolve(payload),
});
return promise;
}
describe("Anthropic fast mode (speed: 'fast')", () => {
it("sets speed='fast' on the request body for Claude Opus 4.7", async () => {
const payload = (await capturePayload(makeAnthropicModel("claude-opus-4-7"), "fast")) as {
speed?: string;
};
expect(payload.speed).toBe("fast");
});
it("sets speed='fast' on the request body for Claude Opus 4.6", async () => {
const payload = (await capturePayload(makeAnthropicModel("claude-opus-4-6"), "fast")) as {
speed?: string;
};
expect(payload.speed).toBe("fast");
});
it("forwards speed='fast' for any model and lets the server decide what's supported", async () => {
// We deliberately don't gate client-side so future model additions
// (Opus 4.8, Sonnet 4.x, etc.) don't require an SDK release. The server
// returns invalid_request_error naming the model when unsupported.
const payload = (await capturePayload(makeAnthropicModel("claude-opus-4-5"), "fast")) as {
speed?: string;
};
expect(payload.speed).toBe("fast");
});
it("omits the speed field when not requested", async () => {
const payload = (await capturePayload(makeAnthropicModel("claude-opus-4-7"), undefined)) as Record<
string,
unknown
>;
expect(payload.speed).toBeUndefined();
});
it("omits the speed field when explicitly 'standard'", async () => {
const payload = (await capturePayload(makeAnthropicModel("claude-opus-4-7"), "standard")) as Record<
string,
unknown
>;
// "standard" is the API default; we only forward "fast" to avoid invalidating
// prompt caches that were primed without it.
expect(payload.speed).toBeUndefined();
});
});
+4
View File
@@ -2,6 +2,10 @@
## [Unreleased]
### Changed
- Changed `/fast` to dispatch on the active model's API: Anthropic models toggle `speed: "fast"` (the `fast-mode-2026-02-01` beta supported on Claude Opus 4.6 and 4.7 today), other providers continue to set `serviceTier: "priority"`. A new `speed_change` session-history entry persists the Anthropic toggle across reloads. When the provider auto-drops fast mode because the server rejected the request, the session syncs the toggle off and emits a warning notice based on the new `AssistantMessage.disabledFeatures` marker; subsequent requests in the same session skip `speed` entirely via `providerSessionState.fastModeDisabled`.
### Fixed
- Fixed ACP command and custom tool-call notifications to carry the original tool arguments in replayed and final updates, so command text is preserved and raw input is no longer wrapped
@@ -1578,6 +1578,14 @@ export class AgentSession {
if (event.message.role === "assistant") {
this.#lastAssistantMessage = event.message;
const assistantMsg = event.message as AssistantMessage;
if (assistantMsg.disabledFeatures?.includes("anthropic.fast_mode") && this.speed === "fast") {
this.setSpeed(undefined);
this.emitNotice(
"warning",
"Fast mode rejected for this model; retried without it. Fast mode is now off.",
"anthropic.fast_mode",
);
}
// Resolve TTSR resume gate before checking for new deferred injections.
// Gate on #ttsrAbortPending, not stopReason: a non-TTSR abort (e.g. streaming
// edit) also produces stopReason === "aborted" but has no continuation coming.
@@ -2761,6 +2769,10 @@ export class AgentSession {
return this.agent.serviceTier;
}
get speed(): "fast" | "standard" | undefined {
return this.agent.speed;
}
/** Whether agent is currently streaming a response */
get isStreaming(): boolean {
return this.agent.state.isStreaming || this.#promptInFlightCount > 0;
@@ -5111,6 +5123,9 @@ export class AgentSession {
}
isFastModeEnabled(): boolean {
if (this.model?.api === "anthropic-messages") {
return this.speed === "fast";
}
return this.serviceTier === "priority";
}
@@ -5120,7 +5135,17 @@ export class AgentSession {
this.sessionManager.appendServiceTierChange(serviceTier ?? null);
}
setSpeed(speed: "fast" | "standard" | undefined): void {
if (this.speed === speed) return;
this.agent.speed = speed;
this.sessionManager.appendSpeedChange(speed ?? null);
}
setFastMode(enabled: boolean): void {
if (this.model?.api === "anthropic-messages") {
this.setSpeed(enabled ? "fast" : undefined);
return;
}
this.setServiceTier(enabled ? "priority" : undefined);
}
@@ -7582,6 +7607,7 @@ export class AgentSession {
reasoning: toReasoningEffort(this.thinkingLevel),
hideThinkingSummary: this.agent.hideThinkingSummary,
serviceTier: this.serviceTier,
speed: this.speed,
signal: args.signal,
toolChoice: "none",
},
@@ -7821,6 +7847,7 @@ export class AgentSession {
const hasServiceTierEntry = this.sessionManager
.getBranch()
.some(entry => entry.type === "service_tier_change");
const hasSpeedEntry = this.sessionManager.getBranch().some(entry => entry.type === "speed_change");
const defaultThinkingLevel = this.settings.get("defaultThinkingLevel");
const configuredServiceTier = this.settings.get("serviceTier");
const nextThinkingLevel = resolveThinkingLevelForModel(
@@ -7834,6 +7861,7 @@ export class AgentSession {
: configuredServiceTier === "none"
? undefined
: configuredServiceTier;
this.agent.speed = hasSpeedEntry ? sessionContext.speed : undefined;
if (switchingToDifferentSession) {
this.#resetHindsightConversationTrackingIfHindsight();
@@ -103,6 +103,11 @@ export interface ServiceTierChangeEntry extends SessionEntryBase {
serviceTier: ServiceTier | null;
}
export interface SpeedChangeEntry extends SessionEntryBase {
type: "speed_change";
speed: "fast" | "standard" | null;
}
export interface CompactionEntry<T = unknown> extends SessionEntryBase {
type: "compaction";
summary: string;
@@ -214,6 +219,7 @@ export type SessionEntry =
| ThinkingLevelChangeEntry
| ModelChangeEntry
| ServiceTierChangeEntry
| SpeedChangeEntry
| CompactionEntry
| BranchSummaryEntry
| CustomEntry
@@ -239,6 +245,7 @@ export interface SessionContext {
messages: AgentMessage[];
thinkingLevel?: string;
serviceTier?: ServiceTier;
speed?: "fast" | "standard";
/** Model roles: { default: "provider/modelId", small: "provider/modelId", ... } */
models: Record<string, string>;
/** Names of TTSR rules that have been injected this session */
@@ -524,6 +531,7 @@ export function buildSessionContext(
messages: [],
thinkingLevel: "off",
serviceTier: undefined,
speed: undefined,
models: {},
injectedTtsrRules: [],
selectedMCPToolNames: [],
@@ -544,6 +552,7 @@ export function buildSessionContext(
messages: [],
thinkingLevel: "off",
serviceTier: undefined,
speed: undefined,
models: {},
injectedTtsrRules: [],
selectedMCPToolNames: [],
@@ -563,6 +572,7 @@ export function buildSessionContext(
// Extract settings and find compaction
let thinkingLevel: string | undefined = "off";
let serviceTier: ServiceTier | undefined;
let speed: "fast" | "standard" | undefined;
const models: Record<string, string> = {};
let compaction: CompactionEntry | null = null;
const injectedTtsrRulesSet = new Set<string>();
@@ -593,6 +603,8 @@ export function buildSessionContext(
}
} else if (entry.type === "service_tier_change") {
serviceTier = entry.serviceTier ?? undefined;
} else if (entry.type === "speed_change") {
speed = entry.speed ?? undefined;
} else if (entry.type === "message" && entry.message.role === "assistant") {
// Legacy fallback: infer default model from assistant messages only
// when no explicit `model_change` (role=default) entry has been
@@ -705,6 +717,7 @@ export function buildSessionContext(
messages,
thinkingLevel,
serviceTier,
speed,
models,
injectedTtsrRules,
selectedMCPToolNames,
@@ -2564,6 +2577,18 @@ export class SessionManager {
return entry.id;
}
appendSpeedChange(speed: "fast" | "standard" | null): string {
const entry: SpeedChangeEntry = {
type: "speed_change",
id: generateId(this.#byId),
parentId: this.#leafId,
timestamp: new Date().toISOString(),
speed,
};
this.#appendEntry(entry);
return entry.id;
}
/** Append a mode change as child of current leaf, then advance leaf. Returns entry id. */
appendModeChange(mode: string, data?: Record<string, unknown>): string {
const entry: ModeChangeEntry = {
@@ -147,7 +147,7 @@ const BUILTIN_SLASH_COMMAND_REGISTRY: ReadonlyArray<SlashCommandSpec> = [
},
{
name: "fast",
description: "Toggle fast mode (OpenAI service tier priority)",
description: "Toggle fast mode (Anthropic speed=fast on supported Opus models, OpenAI service_tier=priority)",
acpDescription: "Toggle fast mode",
acpInputHint: "[on|off|status]",
subcommands: [