fix(ai): retry GitHub Copilot fleet-skew model 400s
Any Copilot model in the middle of a rollout (claude-sonnet-4.6,
claude-opus-4.6, gpt-5.4, gpt-5.3-codex, ...) returned a raw HTTP 400 on
roughly half of all turns. GET /models on api.githubcopilot.com returns
two different catalogs across repeated calls: part of the fleet serves
those ids, part rejects them with
400 {"error":{"message":"The requested model is not available for
integrator \"copilot-language-server\". ...",
"code":"model_not_available_for_integrator", ...}}
The absorb machinery already existed and was correct
(isCopilotTransientModelError -> isProviderRetryableError -> the
Anthropic transport's PROVIDER_MAX_RETRIES). Only the classifier missed:
it matched the older model_not_supported code and probed err.code /
err.error.code, while the real code is model_not_available_for_integrator
sitting at err.error.error.code (the SDK stores the parsed body on
.error, and Copilot's body is itself {error:{code}}). So
isProviderRetryableError fell through to "4xx => terminal".
Fix the classifier: providerErrorCode() walks the error envelope up to
depth 3 instead of hardcoding a shape, both Copilot model-availability
codes are accepted, and a wire-body text match backs it up because SDK
envelope shapes drift between provider families while the stringified
message does not. This alone restores the retry path, because
isProviderRetryableError consults the provider hook before its
4xx short-circuit.
Retry shape, since a rejection is a per-request replica reroll rather
than upstream backpressure:
- both transports wait a flat delay between model-flap attempts instead
of the growing backoff; generic retryable failures (429/5xx/transport)
keep their linear ramp and Retry-After handling
- the OpenAI-transport budget goes 3 -> 8 attempts, because a measured
~70% flap window produced a turn that needed 6 wire attempts and
exhausting the budget escalates to the agent-level retry, which
restarts the whole turn
Absorbed attempts cost no tokens: rejections are gateway-side, carry no
usage block, and return in ~208ms median versus ~1884ms for a served
request.
Also refresh the exhausted-retry guidance text, which cited a
nonexistent model id and described the cause as a per-client rollout gap
rather than fleet skew.
This commit is contained in:
@@ -2,6 +2,10 @@
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
### Fixed
|
||||
|
||||
- Fixed GitHub Copilot requests failing with a raw `HTTP 400 model_not_available_for_integrator` on roughly half of all turns for recently rolled-out models. Copilot's fleet is not uniform — part of it rejects models that `/models` advertises on the same host — and the transient classifier matched only the older `model_not_supported` code at a fixed envelope depth, so these rejections surfaced as terminal errors instead of entering the existing retry path. Model-availability 400s are now recognized at any envelope depth and retried on a flat delay, with the OpenAI-transport retry budget raised from 3 to 8 attempts.
|
||||
|
||||
## [17.2.7] - 2026-08-03
|
||||
|
||||
### Changed
|
||||
|
||||
@@ -107,10 +107,17 @@ const STALE_RESPONSE_ITEM_DETAIL_PATTERN = /not[ _]?found|invalid|expired|stale|
|
||||
export const LLAMA_CPP_TOOL_CALL_PARSE_PATTERN =
|
||||
/failed to parse tool call arguments as json|\[json\.exception\.parse_error\.101\]/i;
|
||||
|
||||
// Copilot routing flap: HTTP 400 `model_not_supported` (structural code on the
|
||||
// error, also surfaced in text). Treated as transient — a retry usually lands
|
||||
// on a backend that has the model.
|
||||
const COPILOT_MODEL_NOT_SUPPORTED_PATTERN = /model_not_supported/i;
|
||||
// Copilot fleet skew: HTTP 400 rejecting a model that `/models` advertised on
|
||||
// the very same host. Two codes appear in the wild — `model_not_supported`
|
||||
// (per-OAuth-client rollout gap) and `model_not_available_for_integrator`
|
||||
// (replicas whose integrator allowlist predates the model). Both flap
|
||||
// request-to-request, so a retry usually lands on a backend that has the model.
|
||||
const COPILOT_TRANSIENT_MODEL_CODES: Record<string, true> = {
|
||||
model_not_supported: true,
|
||||
model_not_available_for_integrator: true,
|
||||
};
|
||||
const COPILOT_MODEL_UNAVAILABLE_PATTERN =
|
||||
/model_not_supported|model_not_available_for_integrator|not available for integrator/i;
|
||||
// Anthropic strict-tool grammar too large / schema too complex (400 invalid_request_error).
|
||||
// Feature-gated deployments (Azure Foundry, Baseten, …) reject `strict: true`
|
||||
// tools outright when the hosted model lacks structured outputs, e.g.
|
||||
@@ -345,8 +352,8 @@ function classifyText(errorMessage: string | undefined, errorStatus: number | un
|
||||
kinds |= Flag.StaleResponsesItem;
|
||||
}
|
||||
|
||||
// Copilot per-client routing flap is transient.
|
||||
if (statusClean === 400 && COPILOT_MODEL_NOT_SUPPORTED_PATTERN.test(cleanMessage)) kinds |= Flag.Transient;
|
||||
// Copilot fleet-skew model rejection is transient.
|
||||
if (statusClean === 400 && COPILOT_MODEL_UNAVAILABLE_PATTERN.test(cleanMessage)) kinds |= Flag.Transient;
|
||||
if (matchesStrictToolsRejection(cleanMessage, statusClean)) kinds |= Flag.Grammar;
|
||||
if (matchesFastModeUnsupported(cleanMessage, statusClean)) kinds |= Flag.FastModeUnsupported;
|
||||
}
|
||||
@@ -460,16 +467,35 @@ export function isFastModeUnsupported(error: unknown): boolean {
|
||||
}
|
||||
|
||||
/**
|
||||
* GitHub Copilot 400 `model_not_supported` routing flap — transient. Reads the
|
||||
* structural `code` (and falls back to {@link Flag.Transient} text classification).
|
||||
* Depth-bounded search for a provider error `code`. SDK error objects keep the
|
||||
* parsed response body on `.error`, and Copilot's body is itself
|
||||
* `{ error: { code } }`, so the code sits up to two envelopes below the thrown
|
||||
* error depending on which SDK produced it.
|
||||
*/
|
||||
function providerErrorCode(error: object): string | undefined {
|
||||
let node: object = error;
|
||||
for (let depth = 0; depth < 3; depth++) {
|
||||
if ("code" in node && typeof node.code === "string") return node.code;
|
||||
if (!("error" in node)) return undefined;
|
||||
const nested: unknown = node.error;
|
||||
if (!nested || typeof nested !== "object") return undefined;
|
||||
node = nested;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* GitHub Copilot 400 rejecting a model its own `/models` catalog advertises —
|
||||
* transient fleet skew, not a malformed request. Reads the structural `code`
|
||||
* through the SDK/body envelopes, then falls back to the stringified body both
|
||||
* SDK families put in `message` (shapes drift; the wire text does not).
|
||||
*/
|
||||
export function isCopilotTransientModelError(error: unknown): boolean {
|
||||
if (status(error) === 400 && error && typeof error === "object") {
|
||||
const info = error as { code?: unknown; error?: { code?: unknown } | null };
|
||||
const code = typeof info.code === "string" ? info.code : info.error?.code;
|
||||
if (code === "model_not_supported") return true;
|
||||
}
|
||||
return false;
|
||||
if (!error || typeof error !== "object" || status(error) !== 400) return false;
|
||||
const code = providerErrorCode(error);
|
||||
if (code !== undefined && COPILOT_TRANSIENT_MODEL_CODES[code]) return true;
|
||||
const message: unknown = "message" in error ? error.message : undefined;
|
||||
return typeof message === "string" && COPILOT_MODEL_UNAVAILABLE_PATTERN.test(message);
|
||||
}
|
||||
|
||||
export function classifyMessage(message: {
|
||||
|
||||
@@ -33,7 +33,7 @@ function isTransientTransportMessage(message: string): boolean {
|
||||
export interface ProviderRetryableHooks {
|
||||
/** Provider id of the failing request, used to gate provider-specific checks. */
|
||||
provider?: string;
|
||||
/** Provider-specific transient predicate (e.g. Copilot `model_not_supported`). */
|
||||
/** Provider-specific transient predicate (e.g. Copilot model-availability 400s). */
|
||||
isProviderTransient?: (error: Error) => boolean;
|
||||
}
|
||||
|
||||
|
||||
@@ -1531,6 +1531,13 @@ async function* observeDecodedAnthropicSdkEvents(
|
||||
|
||||
const PROVIDER_MAX_RETRIES = 10;
|
||||
|
||||
/**
|
||||
* Flat delay between attempts when Copilot 400s a model its own `/models`
|
||||
* catalog advertises. Part of the fleet carries the model and part doesn't, so
|
||||
* the retry is a reroll rather than a wait for capacity to free up.
|
||||
*/
|
||||
const COPILOT_MODEL_FLAP_RETRY_DELAY_MS = 400;
|
||||
|
||||
/**
|
||||
* How long `ping` keepalives may keep extending the idle deadline without any
|
||||
* semantic stream progress, as a multiple of the idle timeout. Anthropic pings
|
||||
@@ -1561,8 +1568,8 @@ function shouldIgnoreAnthropicPreambleEvent(eventType: unknown): boolean {
|
||||
/**
|
||||
* Whether an Anthropic (or Copilot-over-Anthropic) stream error should be
|
||||
* retried. The classification lives in {@link AIError.isProviderRetryableError};
|
||||
* this wrapper injects the Copilot-specific `model_not_supported` transient
|
||||
* check, which the error module must not import directly.
|
||||
* this wrapper injects the Copilot-specific model-availability transient check,
|
||||
* which the error module must not import directly.
|
||||
*/
|
||||
export function isProviderRetryableError(error: unknown, provider?: string): boolean {
|
||||
return AIError.isProviderRetryableError(error, {
|
||||
@@ -2709,7 +2716,12 @@ const streamAnthropicOnce = (
|
||||
throw streamFailure;
|
||||
}
|
||||
providerRetryAttempt++;
|
||||
const backoffDelayMs = calculateAnthropicRetryDelayMs(providerRetryAttempt - 1);
|
||||
// Copilot's model-availability 400 is a per-request replica reroll, not
|
||||
// upstream backpressure — the exponential curve would just add dead
|
||||
// time to a coin flip that the next attempt is as likely to win.
|
||||
const backoffDelayMs = AIError.isCopilotTransientModelError(streamFailure)
|
||||
? COPILOT_MODEL_FLAP_RETRY_DELAY_MS
|
||||
: calculateAnthropicRetryDelayMs(providerRetryAttempt - 1);
|
||||
// Honor the server's retry hint (`retry-after-ms`/`retry-after`) on
|
||||
// 429/529-style failures: retrying sooner than the server asked is a
|
||||
// guaranteed failure that just burns the retry budget.
|
||||
|
||||
@@ -95,10 +95,11 @@ export async function finalizeErrorMessage(
|
||||
* Rewrite error message for GitHub Copilot request failures.
|
||||
* Must run AFTER finalizeErrorMessage since it replaces the message entirely.
|
||||
*
|
||||
* 400 `model_not_supported` = Copilot routing rollout gap for our OAuth client.
|
||||
* A preview model (gpt-5.3-codex, gpt-5.4*, ...) flaps between 200 and
|
||||
* 400 because only some of Copilot's backends have the model. After the
|
||||
* in-request retry exhausts, surface guidance rather than the raw error.
|
||||
* 400 model-unavailable = Copilot fleet skew. A model that `/models` advertises
|
||||
* (claude-sonnet-4.6, claude-opus-4.6, gpt-5.4, gpt-5.3-codex, ...)
|
||||
* flaps between 200 and 400 because only part of Copilot's fleet has it
|
||||
* in the integrator allowlist. After the in-request retry exhausts,
|
||||
* surface guidance rather than the raw error.
|
||||
* 401 = token invalid/expired → credential removal is safe, prompt re-login.
|
||||
* 403 = token valid but access denied (plan, model policy, org restriction) →
|
||||
* do NOT reuse the auth-failed string (which triggers credential removal).
|
||||
@@ -113,7 +114,7 @@ export function rewriteCopilotError(errorMessage: string, error: unknown, provid
|
||||
return `GitHub Copilot access denied (HTTP 403). Your account may not have access to this model or feature. Check your Copilot plan or model policy settings.`;
|
||||
}
|
||||
if (isCopilotTransientModelError(error)) {
|
||||
return `GitHub Copilot rejected this model (HTTP 400 model_not_supported) after retries. This is a known intermittent rollout gap for preview models on OAuth clients other than VS Code. Try again in a few seconds, switch to a GA model (gpt-5-mini, gpt-5.2), or run this model from VS Code.`;
|
||||
return `GitHub Copilot rejected this model (HTTP 400) after retries: only part of its fleet currently serves this model id, even though /models advertises it. Try again in a few seconds or switch to a model Copilot serves fleet-wide (claude-opus-4.7, claude-sonnet-4.5, gpt-4.1).`;
|
||||
}
|
||||
return errorMessage;
|
||||
}
|
||||
|
||||
@@ -7,14 +7,22 @@ import { getHeadersFromError, getRetryAfterMsFromHeaders } from "./retry-after";
|
||||
// home). Re-exported here so existing `../utils/retry` importers keep working.
|
||||
export { isCopilotTransientModelError };
|
||||
|
||||
const COPILOT_MODEL_RETRY_MAX_ATTEMPTS = 3;
|
||||
// Copilot's model-availability flap is a per-request coin flip across fleet
|
||||
// replicas, not backpressure. Measured per-attempt rejection rates for models
|
||||
// mid-rollout reach ~70% (gpt-5.4, 2026-08-04), and a live 10-turn run needed 6
|
||||
// attempts on one turn — so a small budget just pushes the failure up to the
|
||||
// agent-level retry, which restarts the whole turn. Eight attempts on a flat
|
||||
// delay keep the residual near 5% at p=0.7 and under 1% at p=0.5, bounded at
|
||||
// ~2.8s of dead time in the pathological case.
|
||||
const COPILOT_MODEL_RETRY_MAX_ATTEMPTS = 8;
|
||||
const COPILOT_MODEL_RETRY_BASE_DELAY_MS = 400;
|
||||
/** Longest server-requested backoff we are willing to sit out before giving up. */
|
||||
const COPILOT_RETRY_AFTER_MAX_WAIT_MS = 30_000;
|
||||
|
||||
/**
|
||||
* Wrap an initial Copilot request so transient `model_not_supported` 400s are
|
||||
* retried a small number of times. No-op for non-Copilot providers.
|
||||
* Wrap an initial Copilot request so transient model-availability 400s
|
||||
* (`model_not_supported`, `model_not_available_for_integrator`) are retried a
|
||||
* small number of times. No-op for non-Copilot providers.
|
||||
*
|
||||
* The callback **MUST** create a fresh in-flight request each invocation — a
|
||||
* once-consumed AsyncIterable cannot be re-iterated.
|
||||
@@ -39,7 +47,10 @@ export async function callWithCopilotModelRetry<T>(
|
||||
const transientModelError = isCopilotTransientModelError(error);
|
||||
if (!transientModelError && !isRetryableError(error)) throw error;
|
||||
if (attempt === COPILOT_MODEL_RETRY_MAX_ATTEMPTS - 1) break;
|
||||
let delayMs = retryBaseDelayMs * (attempt + 1);
|
||||
// Reroll the model flap on a flat delay: a ramp only adds dead time to a
|
||||
// coin flip the next attempt is equally likely to win. Generic retryable
|
||||
// failures (429/5xx/transport) keep the linear backoff below.
|
||||
let delayMs = transientModelError ? retryBaseDelayMs : retryBaseDelayMs * (attempt + 1);
|
||||
if (!transientModelError) {
|
||||
const errorStatus = status(error);
|
||||
if (errorStatus !== undefined) {
|
||||
|
||||
@@ -105,4 +105,22 @@ describe("isProviderRetryableError", () => {
|
||||
expect(isProviderRetryableError(err, "anthropic")).toBe(false);
|
||||
expect(isProviderRetryableError(err)).toBe(false);
|
||||
});
|
||||
|
||||
it("retries Copilot's model_not_available_for_integrator 400 from the Anthropic messages proxy", () => {
|
||||
// Shape thrown by @anthropic-ai/sdk against api.githubcopilot.com/v1/messages:
|
||||
// the parsed body lands on `.error` and is itself `{ error: { code } }`.
|
||||
const body = {
|
||||
error: {
|
||||
message:
|
||||
'The requested model is not available for integrator "copilot-language-server". Available models: [gpt-4.1 claude-opus-4.7]. Verify the correct Copilot-Integration-Id header is being sent.',
|
||||
code: "model_not_available_for_integrator",
|
||||
param: "model",
|
||||
type: "invalid_request_error",
|
||||
},
|
||||
};
|
||||
const err = new Error(`400 ${JSON.stringify(body)}`);
|
||||
Object.assign(err, { status: 400, error: body });
|
||||
expect(isProviderRetryableError(err, "github-copilot")).toBe(true);
|
||||
expect(isProviderRetryableError(err, "anthropic")).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,8 +1,18 @@
|
||||
import { describe, expect, it } from "bun:test";
|
||||
import { afterEach, describe, expect, it, vi } from "bun:test";
|
||||
import { scheduler } from "node:timers/promises";
|
||||
import { callWithCopilotModelRetry, isCopilotTransientModelError } from "@oh-my-pi/pi-ai/utils/retry";
|
||||
import { isRetryableError } from "@oh-my-pi/pi-utils";
|
||||
|
||||
type ErrorShape = { status: number; code?: string; error?: { code?: string; message?: string }; message: string };
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
type ErrorShape = {
|
||||
status: number;
|
||||
code?: string;
|
||||
error?: { code?: string; message?: string } | { error: { code?: string; message?: string } };
|
||||
message: string;
|
||||
};
|
||||
|
||||
function copilotError({ status, code, error, message }: ErrorShape): Error {
|
||||
const err = new Error(message);
|
||||
@@ -31,6 +41,31 @@ describe("isCopilotTransientModelError", () => {
|
||||
expect(isCopilotTransientModelError(err)).toBe(true);
|
||||
});
|
||||
|
||||
it("matches 400 model_not_available_for_integrator nested two envelopes deep (Anthropic SDK shape)", () => {
|
||||
// api.githubcopilot.com/v1/messages: the SDK stores the parsed body on
|
||||
// `.error`, and that body is itself `{ error: { code } }`.
|
||||
const err = copilotError({
|
||||
status: 400,
|
||||
error: {
|
||||
error: {
|
||||
code: "model_not_available_for_integrator",
|
||||
message: 'The requested model is not available for integrator "copilot-language-server".',
|
||||
},
|
||||
},
|
||||
message:
|
||||
'400 {"error":{"message":"The requested model is not available for integrator \\"copilot-language-server\\". Available models: [gpt-4.1 claude-opus-4.7]","code":"model_not_available_for_integrator","param":"model","type":"invalid_request_error"}}',
|
||||
});
|
||||
expect(isCopilotTransientModelError(err)).toBe(true);
|
||||
});
|
||||
|
||||
it("falls back to the stringified body when no envelope exposes a code", () => {
|
||||
const err = copilotError({
|
||||
status: 400,
|
||||
message: '400 {"error":{"message":"The requested model is not available for integrator \\"x\\"."}}',
|
||||
});
|
||||
expect(isCopilotTransientModelError(err)).toBe(true);
|
||||
});
|
||||
|
||||
it("does not match other 400 codes", () => {
|
||||
const err = copilotError({
|
||||
status: 400,
|
||||
@@ -74,7 +109,7 @@ describe("callWithCopilotModelRetry", () => {
|
||||
expect(calls).toBe(1);
|
||||
});
|
||||
|
||||
it("retries up to 3 attempts for Copilot transient errors and eventually throws the last error", async () => {
|
||||
it("retries up to 8 attempts for Copilot transient errors and eventually throws the last error", async () => {
|
||||
let calls = 0;
|
||||
const err = copilotError({ status: 400, code: "model_not_supported", message: "transient" });
|
||||
await expect(
|
||||
@@ -86,7 +121,7 @@ describe("callWithCopilotModelRetry", () => {
|
||||
{ provider: "github-copilot", retryBaseDelayMs: 0 },
|
||||
),
|
||||
).rejects.toBe(err);
|
||||
expect(calls).toBe(3);
|
||||
expect(calls).toBe(8);
|
||||
});
|
||||
|
||||
it("succeeds on the second attempt when the first is transient", async () => {
|
||||
@@ -171,6 +206,45 @@ describe("callWithCopilotModelRetry", () => {
|
||||
expect(calls).toBe(2);
|
||||
});
|
||||
|
||||
it("keeps the model-flap delay flat while generic retryable failures ramp", async () => {
|
||||
const flatWaits: number[] = [];
|
||||
const rampWaits: number[] = [];
|
||||
const record = (into: number[]) => {
|
||||
const spy = vi.spyOn(scheduler, "wait");
|
||||
spy.mockImplementation(async (delay?: number) => {
|
||||
into.push(delay ?? 0);
|
||||
});
|
||||
return spy;
|
||||
};
|
||||
|
||||
record(flatWaits);
|
||||
await expect(
|
||||
callWithCopilotModelRetry(
|
||||
async () => {
|
||||
throw copilotError({ status: 400, code: "model_not_available_for_integrator", message: "flap" });
|
||||
},
|
||||
{ provider: "github-copilot", retryBaseDelayMs: 100 },
|
||||
),
|
||||
).rejects.toBeInstanceOf(Error);
|
||||
vi.restoreAllMocks();
|
||||
|
||||
record(rampWaits);
|
||||
await expect(
|
||||
callWithCopilotModelRetry(
|
||||
async () => {
|
||||
throw new Error(
|
||||
'HTTP2StreamReset fetching "https://api.example.com/x". For more information, pass `verbose: true` in the second argument to fetch()',
|
||||
);
|
||||
},
|
||||
{ provider: "github-copilot", retryBaseDelayMs: 100 },
|
||||
),
|
||||
).rejects.toBeInstanceOf(Error);
|
||||
vi.restoreAllMocks();
|
||||
|
||||
expect(flatWaits).toEqual([100, 100, 100, 100, 100, 100, 100]);
|
||||
expect(rampWaits).toEqual([100, 200, 300, 400, 500, 600, 700]);
|
||||
});
|
||||
|
||||
it("stops retrying when the caller aborts during backoff", async () => {
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
|
||||
@@ -0,0 +1,125 @@
|
||||
import { afterEach, describe, expect, it, vi } from "bun:test";
|
||||
import { streamAnthropic } from "@oh-my-pi/pi-ai/providers/anthropic";
|
||||
import type { Context, Model } from "@oh-my-pi/pi-ai/types";
|
||||
import { buildModel } from "@oh-my-pi/pi-catalog/build";
|
||||
import { OPENCODE_HEADERS } from "@oh-my-pi/pi-catalog/wire/github-copilot";
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
function makeCopilotClaudeModel(): Model<"anthropic-messages"> {
|
||||
return buildModel({
|
||||
id: "claude-sonnet-4.6",
|
||||
name: "Claude Sonnet 4.6",
|
||||
api: "anthropic-messages",
|
||||
provider: "github-copilot",
|
||||
baseUrl: "https://api.githubcopilot.com",
|
||||
headers: { ...OPENCODE_HEADERS },
|
||||
reasoning: true,
|
||||
input: ["text"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 128_000,
|
||||
maxTokens: 16_000,
|
||||
});
|
||||
}
|
||||
|
||||
const testContext: Context = {
|
||||
messages: [{ role: "user", content: "hello", timestamp: Date.now() }],
|
||||
};
|
||||
|
||||
/**
|
||||
* Verbatim body served by `api.githubcopilot.com/v1/messages` when the request
|
||||
* lands on a fleet replica whose integrator allowlist predates the model, even
|
||||
* though `/models` on the same host advertises it.
|
||||
*/
|
||||
const FLEET_SKEW_BODY = {
|
||||
error: {
|
||||
message:
|
||||
'The requested model is not available for integrator "copilot-language-server". Available models: [gpt-4.1 claude-opus-4.7 claude-sonnet-4.5]. Verify the correct Copilot-Integration-Id header is being sent.',
|
||||
code: "model_not_available_for_integrator",
|
||||
param: "model",
|
||||
type: "invalid_request_error",
|
||||
},
|
||||
};
|
||||
|
||||
const SSE_EVENTS = [
|
||||
{
|
||||
type: "message_start",
|
||||
message: {
|
||||
id: "msg_fleet",
|
||||
type: "message",
|
||||
role: "assistant",
|
||||
model: "claude-sonnet-4.6",
|
||||
content: [],
|
||||
stop_reason: null,
|
||||
stop_sequence: null,
|
||||
usage: { input_tokens: 7, output_tokens: 0 },
|
||||
},
|
||||
},
|
||||
{ type: "content_block_start", index: 0, content_block: { type: "text", text: "" } },
|
||||
{ type: "content_block_delta", index: 0, delta: { type: "text_delta", text: "second try" } },
|
||||
{ type: "content_block_stop", index: 0 },
|
||||
{
|
||||
type: "message_delta",
|
||||
delta: { stop_reason: "end_turn", stop_sequence: null },
|
||||
usage: { output_tokens: 3 },
|
||||
},
|
||||
{ type: "message_stop" },
|
||||
];
|
||||
|
||||
function sseResponse(): Response {
|
||||
const body = `${SSE_EVENTS.map(event => `event: ${event.type}\ndata: ${JSON.stringify(event)}\n`).join("\n")}\n`;
|
||||
return new Response(body, { status: 200, headers: { "Content-Type": "text/event-stream" } });
|
||||
}
|
||||
|
||||
describe("GitHub Copilot Anthropic fleet skew", () => {
|
||||
it("retries the model-availability 400 and completes on the next replica", async () => {
|
||||
let attempts = 0;
|
||||
const fetchMock = vi.fn(async () => {
|
||||
attempts += 1;
|
||||
if (attempts === 1) {
|
||||
return new Response(JSON.stringify(FLEET_SKEW_BODY), {
|
||||
status: 400,
|
||||
headers: { "Content-Type": "application/json" },
|
||||
});
|
||||
}
|
||||
return sseResponse();
|
||||
});
|
||||
|
||||
const result = await streamAnthropic(makeCopilotClaudeModel(), testContext, {
|
||||
apiKey: "ghu_test_copilot_token",
|
||||
fetch: fetchMock as unknown as typeof fetch,
|
||||
providerRetryWait: async () => {},
|
||||
}).result();
|
||||
|
||||
expect(attempts).toBe(2);
|
||||
expect(result.stopReason).toBe("stop");
|
||||
expect(result.errorMessage).toBeUndefined();
|
||||
expect(result.content).toMatchObject([{ type: "text", text: "second try" }]);
|
||||
// Billing is per user prompt, not per wire attempt: a turn that burned an
|
||||
// extra gateway-rejected attempt must still report one premium request.
|
||||
expect(result.usage.premiumRequests).toBe(1);
|
||||
});
|
||||
|
||||
it("surfaces fleet-skew guidance once every retry lands on a stale replica", async () => {
|
||||
const fetchMock = vi.fn(
|
||||
async () =>
|
||||
new Response(JSON.stringify(FLEET_SKEW_BODY), {
|
||||
status: 400,
|
||||
headers: { "Content-Type": "application/json" },
|
||||
}),
|
||||
);
|
||||
|
||||
const result = await streamAnthropic(makeCopilotClaudeModel(), testContext, {
|
||||
apiKey: "ghu_test_copilot_token",
|
||||
fetch: fetchMock as unknown as typeof fetch,
|
||||
providerRetryWait: async () => {},
|
||||
}).result();
|
||||
|
||||
expect(result.stopReason).toBe("error");
|
||||
expect(result.errorMessage).toContain("only part of its fleet");
|
||||
// Every attempt is a fresh wire request, not a replayed promise.
|
||||
expect(fetchMock.mock.calls.length).toBeGreaterThan(1);
|
||||
});
|
||||
});
|
||||
@@ -33,14 +33,16 @@ describe("rewriteCopilotError", () => {
|
||||
expect(result).not.toContain("/login github-copilot");
|
||||
});
|
||||
|
||||
it("rewrites 400 model_not_supported with rollout-gap guidance", () => {
|
||||
const err = new Error("400 The requested model is not supported.");
|
||||
(err as unknown as { status: number; code: string }).status = 400;
|
||||
(err as unknown as { status: number; code: string }).code = "model_not_supported";
|
||||
const result = rewriteCopilotError("original", err, "github-copilot");
|
||||
expect(result).toContain("HTTP 400 model_not_supported");
|
||||
expect(result).toContain("rollout gap");
|
||||
expect(result).not.toContain("authentication failed");
|
||||
it("rewrites 400 model-unavailable codes with fleet-skew guidance", () => {
|
||||
for (const code of ["model_not_supported", "model_not_available_for_integrator"]) {
|
||||
const err = new Error("400 The requested model is not available.");
|
||||
(err as unknown as { status: number; code: string }).status = 400;
|
||||
(err as unknown as { status: number; code: string }).code = code;
|
||||
const result = rewriteCopilotError("original", err, "github-copilot");
|
||||
expect(result).toContain("HTTP 400");
|
||||
expect(result).toContain("only part of its fleet");
|
||||
expect(result).not.toContain("authentication failed");
|
||||
}
|
||||
});
|
||||
|
||||
it("leaves non-copilot 400 model_not_supported untouched", () => {
|
||||
|
||||
Reference in New Issue
Block a user