- Extend USAGE_LIMIT_PATTERN with quota.?reached so auth-retry and AuthStorage.markUsageLimitReached recognise Antigravity's 'Individual quota reached' 429 as a credential-rotatable usage limit instead of a terminal provider error. The parseRateLimitReason classifier already mapped this phrasing to QUOTA_EXHAUSTED via the generic quota check. - Add antigravityRankingStrategy: picks the lowest-remainingFraction counter as primary and the next-lowest as secondary, with 24h windowDefaults matching the daily-cloudcode-pa.googleapis.com reset cadence (Antigravity windows omit durationMs). Register it in DEFAULT_RANKING_STRATEGIES so new google-antigravity sessions consult usage reports before assignment. - Regression coverage: rate-limit-utils.test.ts pins the matcher against 'Individual quota reached' and bare quota reached / quota_reached; auth-storage-antigravity-selection.test.ts proves an exhausted Gemini counter on one OAuth credential causes getApiKey to return the healthy sibling and that a less-pressured account is preferred when neither is exhausted. Fixes #2198
120 lines
5.0 KiB
TypeScript
120 lines
5.0 KiB
TypeScript
import { describe, expect, it } from "bun:test";
|
||
import { calculateRateLimitBackoffMs, isUsageLimitError, parseRateLimitReason } from "@oh-my-pi/pi-ai/rate-limit-utils";
|
||
|
||
describe("parseRateLimitReason", () => {
|
||
it("classifies Google Quota exceeded as QUOTA_EXHAUSTED", () => {
|
||
expect(
|
||
parseRateLimitReason("Cloud Code Assist API error (429): Quota exceeded for aiplatform.googleapis.com"),
|
||
).toBe("QUOTA_EXHAUSTED");
|
||
});
|
||
|
||
// "Resource has been exhausted (e.g. check quota)" is a quota/daily-limit error — long wait.
|
||
// Only the literal phrase "resource exhausted" (gRPC status name) is MODEL_CAPACITY.
|
||
it("classifies 'Resource has been exhausted (e.g. check quota)' as QUOTA_EXHAUSTED", () => {
|
||
expect(
|
||
parseRateLimitReason("Cloud Code Assist API error (429): Resource has been exhausted (e.g. check quota)."),
|
||
).toBe("QUOTA_EXHAUSTED");
|
||
});
|
||
|
||
it("classifies 'resource exhausted' (exact gRPC phrase) as MODEL_CAPACITY_EXHAUSTED", () => {
|
||
expect(parseRateLimitReason("resource exhausted")).toBe("MODEL_CAPACITY_EXHAUSTED");
|
||
});
|
||
|
||
it("classifies Too many requests as RATE_LIMIT_EXCEEDED", () => {
|
||
expect(parseRateLimitReason("Cloud Code Assist API error (429): Too many requests")).toBe("RATE_LIMIT_EXCEEDED");
|
||
});
|
||
|
||
it("classifies per minute errors as RATE_LIMIT_EXCEEDED", () => {
|
||
expect(parseRateLimitReason("Requests per minute limit reached")).toBe("RATE_LIMIT_EXCEEDED");
|
||
});
|
||
|
||
it("classifies overloaded 529 as MODEL_CAPACITY_EXHAUSTED", () => {
|
||
expect(parseRateLimitReason("Service overloaded 529")).toBe("MODEL_CAPACITY_EXHAUSTED");
|
||
});
|
||
|
||
it("classifies internal server error as SERVER_ERROR", () => {
|
||
expect(parseRateLimitReason("Internal Server Error (500)")).toBe("SERVER_ERROR");
|
||
});
|
||
|
||
it("returns UNKNOWN for unrecognised messages", () => {
|
||
expect(parseRateLimitReason("Something completely unexpected happened")).toBe("UNKNOWN");
|
||
});
|
||
|
||
it("classifies Codex usage limit error as QUOTA_EXHAUSTED", () => {
|
||
expect(
|
||
parseRateLimitReason("Codex error event: The usage limit has been reached (code=usage_limit_reached)"),
|
||
).toBe("QUOTA_EXHAUSTED");
|
||
});
|
||
|
||
it("classifies account rate limits as QUOTA_EXHAUSTED", () => {
|
||
expect(
|
||
parseRateLimitReason(
|
||
'429 {"type":"error","error":{"type":"rate_limit_error","message":"This request would exceed your account\'s rate limit. Please try again later."}}',
|
||
),
|
||
).toBe("QUOTA_EXHAUSTED");
|
||
});
|
||
|
||
it("classifies Antigravity capacity-exhausted as QUOTA_EXHAUSTED, not transient MODEL_CAPACITY", () => {
|
||
// Antigravity returns "You have exhausted your capacity on this model. Your
|
||
// quota will reset after 3h6m38s." The literal "capacity" used to win the
|
||
// classifier race and land in MODEL_CAPACITY_EXHAUSTED (45-75s backoff),
|
||
// blocking the agent from rotating to another OAuth account even though the
|
||
// "quota will reset" suffix is the long-wait, switch-account signal.
|
||
expect(
|
||
parseRateLimitReason(
|
||
"Cloud Code Assist API error (429): You have exhausted your capacity on this model. Your quota will reset after 3h6m38s.",
|
||
),
|
||
).toBe("QUOTA_EXHAUSTED");
|
||
});
|
||
});
|
||
|
||
describe("isUsageLimitError", () => {
|
||
it("detects account rate limits as credential-rotatable usage limits", () => {
|
||
expect(
|
||
isUsageLimitError(
|
||
'429 {"type":"error","error":{"type":"rate_limit_error","message":"This request would exceed your account\'s rate limit. Please try again later."}}',
|
||
),
|
||
).toBe(true);
|
||
});
|
||
|
||
it("detects Antigravity capacity-exhausted message as a usage-limit error", () => {
|
||
// Without this branch `markUsageLimitReached` is never invoked, so the
|
||
// session sticks to the exhausted OAuth account instead of rotating —
|
||
// see `agent-session.ts` line 8314 and `auth-storage.ts` line 3457.
|
||
expect(
|
||
isUsageLimitError(
|
||
"Cloud Code Assist API error (429): You have exhausted your capacity on this model. Your quota will reset after 3h6m38s.",
|
||
),
|
||
).toBe(true);
|
||
});
|
||
|
||
// Antigravity / Cloud Code Assist returns this phrasing for an exhausted
|
||
// project quota; `parseRateLimitReason` already maps it to QUOTA_EXHAUSTED
|
||
// via the generic `quota` substring, but `isUsageLimitError` decides
|
||
// whether the auth layer rotates to a sibling OAuth credential, so it
|
||
// must match too — otherwise the session stays pinned to the exhausted
|
||
// account (see issue #2198).
|
||
it("detects Antigravity 'Individual quota reached' as a credential-rotatable usage limit", () => {
|
||
expect(
|
||
isUsageLimitError(
|
||
"Cloud Code Assist API error (429): Individual quota reached. Contact your administrator to enable overages.",
|
||
),
|
||
).toBe(true);
|
||
});
|
||
|
||
it("detects bare 'quota reached' phrasing", () => {
|
||
expect(isUsageLimitError("quota reached")).toBe(true);
|
||
expect(isUsageLimitError("quota_reached")).toBe(true);
|
||
});
|
||
});
|
||
|
||
describe("calculateRateLimitBackoffMs", () => {
|
||
it("returns 45–75s range for MODEL_CAPACITY_EXHAUSTED (jitter)", () => {
|
||
for (let i = 0; i < 20; i++) {
|
||
const ms = calculateRateLimitBackoffMs("MODEL_CAPACITY_EXHAUSTED");
|
||
expect(ms).toBeGreaterThanOrEqual(45_000);
|
||
expect(ms).toBeLessThanOrEqual(75_000);
|
||
}
|
||
});
|
||
});
|