roboomp
d87eeaa2ac
Treat Synthetic discovery as an authoritative catalog so deprecated bundled IDs are removed from resolved model lists and cache snapshots. Validate Synthetic API keys through the models endpoint instead of a model-specific chat request.\n\nFixes #1417
2340 lines
79 KiB
TypeScript
2340 lines
79 KiB
TypeScript
import { afterEach, beforeEach, describe, expect, test } from "bun:test";
|
|
import * as fs from "node:fs";
|
|
import * as os from "node:os";
|
|
import * as path from "node:path";
|
|
import { Effort, type Model, type OpenAICompat, type ThinkingConfig, writeModelCache } from "@oh-my-pi/pi-ai";
|
|
import { kNoAuth, ModelRegistry } from "@oh-my-pi/pi-coding-agent/config/model-registry";
|
|
import { resetSettingsForTest, Settings } from "@oh-my-pi/pi-coding-agent/config/settings";
|
|
import { AuthStorage } from "@oh-my-pi/pi-coding-agent/session/auth-storage";
|
|
import { hookFetch, Snowflake } from "@oh-my-pi/pi-utils";
|
|
|
|
describe("ModelRegistry", () => {
|
|
let tempDir: string;
|
|
let modelsJsonPath: string;
|
|
let cacheDbPath: string;
|
|
let authStorage: AuthStorage;
|
|
|
|
beforeEach(async () => {
|
|
resetSettingsForTest();
|
|
tempDir = path.join(os.tmpdir(), `pi-test-model-registry-${Snowflake.next()}`);
|
|
fs.mkdirSync(tempDir, { recursive: true });
|
|
modelsJsonPath = path.join(tempDir, "models.json");
|
|
cacheDbPath = path.join(tempDir, "models.db");
|
|
authStorage = await AuthStorage.create(path.join(tempDir, "testauth.db"));
|
|
});
|
|
|
|
afterEach(() => {
|
|
resetSettingsForTest();
|
|
authStorage.close();
|
|
if (tempDir && fs.existsSync(tempDir)) {
|
|
fs.rmSync(tempDir, { recursive: true });
|
|
}
|
|
});
|
|
|
|
type ProviderConfig = {
|
|
baseUrl: string;
|
|
apiKey: string;
|
|
api: string;
|
|
models: Array<{
|
|
id: string;
|
|
name: string;
|
|
reasoning: boolean;
|
|
thinking?: ThinkingConfig;
|
|
input: string[];
|
|
cost: { input: number; output: number; cacheRead: number; cacheWrite: number };
|
|
contextWindow: number;
|
|
maxTokens: number;
|
|
}>;
|
|
};
|
|
|
|
/** Create minimal provider config */
|
|
function providerConfig(
|
|
baseUrl: string,
|
|
models: Array<{
|
|
id: string;
|
|
name?: string;
|
|
reasoning?: boolean;
|
|
thinking?: ThinkingConfig;
|
|
contextWindow?: number;
|
|
}>,
|
|
api: string = "anthropic-messages",
|
|
) {
|
|
return {
|
|
baseUrl,
|
|
apiKey: "TEST_KEY",
|
|
api,
|
|
models: models.map(m => ({
|
|
id: m.id,
|
|
name: m.name ?? m.id,
|
|
reasoning: m.reasoning ?? false,
|
|
thinking: m.thinking,
|
|
input: ["text"],
|
|
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
|
contextWindow: m.contextWindow ?? 100000,
|
|
maxTokens: 8000,
|
|
})),
|
|
};
|
|
}
|
|
|
|
function writeModelsJson(providers: Record<string, ProviderConfig>) {
|
|
fs.writeFileSync(modelsJsonPath, JSON.stringify({ providers }));
|
|
}
|
|
|
|
function writeCachedOllamaModels(models: Model<"openai-completions">[]) {
|
|
writeModelCache("ollama", Date.now(), models, true, "", cacheDbPath);
|
|
}
|
|
|
|
function getModelsForProvider(registry: ModelRegistry, provider: string) {
|
|
return registry.getAll().filter(m => m.provider === provider);
|
|
}
|
|
|
|
function getOpenAICompat(model: Model | undefined): OpenAICompat | undefined {
|
|
// All custom-model compat overrides flow through OpenAICompatSchema regardless of
|
|
// the underlying api ("openai-completions" vs "openai-responses"), so we can read
|
|
// the field for any model in this fixture.
|
|
return model?.compat as OpenAICompat | undefined;
|
|
}
|
|
|
|
/** Create a baseUrl-only override (no custom models) */
|
|
function overrideConfig(baseUrl: string, headers?: Record<string, string>) {
|
|
return { baseUrl, ...(headers && { headers }) };
|
|
}
|
|
|
|
/** Write raw providers config (for mixed override/replacement scenarios) */
|
|
function writeRawModelsJson(providers: Record<string, unknown>) {
|
|
fs.writeFileSync(modelsJsonPath, JSON.stringify({ providers }));
|
|
}
|
|
|
|
function writeRawModelsConfig(config: Record<string, unknown>) {
|
|
fs.writeFileSync(modelsJsonPath, JSON.stringify(config));
|
|
}
|
|
|
|
function mockOpenAiCompatibleModels(url: string, modelIds: string[]) {
|
|
return hookFetch(input => {
|
|
const requestUrl = String(input);
|
|
if (requestUrl === url) {
|
|
return new Response(JSON.stringify({ data: modelIds.map(id => ({ id })) }), {
|
|
status: 200,
|
|
headers: { "Content-Type": "application/json" },
|
|
});
|
|
}
|
|
throw new Error(`Unexpected URL: ${requestUrl}`);
|
|
});
|
|
}
|
|
|
|
function mockOllamaDiscovery(modelNames: string[]) {
|
|
return hookFetch(input => {
|
|
const url = String(input);
|
|
if (url === "http://127.0.0.1:11434/api/tags") {
|
|
return new Response(JSON.stringify({ models: modelNames.map(name => ({ name })) }), {
|
|
status: 200,
|
|
headers: { "Content-Type": "application/json" },
|
|
});
|
|
}
|
|
if (url === "http://127.0.0.1:11434/api/show") {
|
|
return new Response(JSON.stringify({ capabilities: ["completion"] }), {
|
|
status: 200,
|
|
headers: { "Content-Type": "application/json" },
|
|
});
|
|
}
|
|
throw new Error(`Unexpected URL: ${url}`);
|
|
});
|
|
}
|
|
|
|
describe("canonical equivalence", () => {
|
|
test("groups dotted provider variants under the bundled canonical id", () => {
|
|
writeRawModelsJson({
|
|
demo: providerConfig("https://demo.example.com/v1", [{ id: "anthropic/claude-sonnet-4.5" }]),
|
|
});
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
const variants = registry.getCanonicalVariants("claude-sonnet-4-5");
|
|
|
|
expect(variants.some(variant => variant.selector === "anthropic/claude-sonnet-4-5")).toBe(true);
|
|
expect(variants.some(variant => variant.selector === "demo/anthropic/claude-sonnet-4.5")).toBe(true);
|
|
});
|
|
|
|
test("collapses wrapped, dated, and tuned anthropic variants under the base canonical id", () => {
|
|
writeRawModelsJson({
|
|
demo: providerConfig("https://demo.example.com/v1", [
|
|
{ id: "anthropic/claude-opus-4.5" },
|
|
{ id: "claude-opus-4-5-20251101" },
|
|
{ id: "claude-4.5-opus-high-thinking" },
|
|
]),
|
|
});
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
const variants = registry.getCanonicalVariants("claude-opus-4-5");
|
|
|
|
expect(variants.some(variant => variant.selector === "demo/anthropic/claude-opus-4.5")).toBe(true);
|
|
expect(variants.some(variant => variant.selector === "demo/claude-opus-4-5-20251101")).toBe(true);
|
|
expect(variants.some(variant => variant.selector === "demo/claude-4.5-opus-high-thinking")).toBe(true);
|
|
});
|
|
|
|
test("collapses gitlab duo chat wrapper ids into the upstream canonical id", () => {
|
|
writeRawModelsJson({
|
|
"gitlab-duo": providerConfig("https://demo.example.com/v1", [{ id: "duo-chat-opus-4-6" }]),
|
|
});
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
const variants = registry.getCanonicalVariants("claude-opus-4-6");
|
|
|
|
expect(variants.some(variant => variant.selector === "gitlab-duo/duo-chat-opus-4-6")).toBe(true);
|
|
});
|
|
|
|
test("collapses synthetic and vendor-prefixed glm wrappers into the upstream canonical id", () => {
|
|
writeRawModelsJson({
|
|
demo: providerConfig("https://demo.example.com/v1", [{ id: "hf:zai-org/GLM-4.7" }, { id: "zai-glm-4.7" }]),
|
|
});
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
const variants = registry.getCanonicalVariants("glm-4.7");
|
|
|
|
expect(variants.some(variant => variant.selector === "demo/hf:zai-org/GLM-4.7")).toBe(true);
|
|
expect(variants.some(variant => variant.selector === "demo/zai-glm-4.7")).toBe(true);
|
|
});
|
|
|
|
test("collapses compact and reordered claude aliases into the upstream canonical id", () => {
|
|
writeRawModelsJson({
|
|
demo: providerConfig("https://demo.example.com/v1", [
|
|
{ id: "claude-opus-45" },
|
|
{ id: "claude-4.5-sonnet" },
|
|
]),
|
|
});
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
const opusVariants = registry.getCanonicalVariants("claude-opus-4-5");
|
|
const sonnetVariants = registry.getCanonicalVariants("claude-sonnet-4-5");
|
|
|
|
expect(opusVariants.some(variant => variant.selector === "demo/claude-opus-45")).toBe(true);
|
|
expect(sonnetVariants.some(variant => variant.selector === "demo/claude-4.5-sonnet")).toBe(true);
|
|
});
|
|
|
|
test("collapses nitro-suffixed OpenRouter variants under the upstream canonical id", () => {
|
|
writeRawModelsJson({
|
|
openrouter: providerConfig("https://openrouter.ai/api/v1", [{ id: "z-ai/glm-4.7-20251222:nitro" }]),
|
|
});
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
const variants = registry.getCanonicalVariants("glm-4.7");
|
|
|
|
expect(variants.some(variant => variant.selector === "openrouter/z-ai/glm-4.7-20251222:nitro")).toBe(true);
|
|
});
|
|
|
|
test("uses bundled metadata for Ollama cloud aliases in custom local-proxy configs", () => {
|
|
writeRawModelsJson({
|
|
ollama: {
|
|
baseUrl: "http://127.0.0.1:11434/v1",
|
|
api: "openai-completions",
|
|
auth: "none",
|
|
models: [
|
|
{
|
|
id: "deepseek-v4-pro:cloud",
|
|
name: "DeepSeek V4 Pro (Ollama Cloud)",
|
|
reasoning: true,
|
|
input: ["text"],
|
|
contextWindow: 1_048_576,
|
|
maxTokens: 65_536,
|
|
},
|
|
],
|
|
},
|
|
});
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
const model = registry.find("ollama", "deepseek-v4-pro:cloud");
|
|
const variants = registry.getCanonicalVariants("deepseek-v4-pro");
|
|
|
|
expect(model?.cost.cacheRead).toBeGreaterThan(0);
|
|
expect(model?.thinking?.maxLevel).toBe(Effort.XHigh);
|
|
expect(variants.some(variant => variant.selector === "ollama/deepseek-v4-pro:cloud")).toBe(true);
|
|
});
|
|
|
|
test("collapses anthropic latest aliases into the best upstream claude family id", () => {
|
|
writeRawModelsJson({
|
|
demo: providerConfig("https://demo.example.com/v1", [
|
|
{ id: "anthropic/claude-opus-latest" },
|
|
{ id: "anthropic/claude-haiku-latest" },
|
|
]),
|
|
});
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
const opusVariants = registry.getCanonicalVariants("claude-opus-4-7");
|
|
const haikuVariants = registry.getCanonicalVariants("claude-haiku-4-5");
|
|
|
|
expect(opusVariants.some(variant => variant.selector === "demo/anthropic/claude-opus-latest")).toBe(true);
|
|
expect(haikuVariants.some(variant => variant.selector === "demo/anthropic/claude-haiku-latest")).toBe(true);
|
|
expect(
|
|
registry
|
|
.getCanonicalVariants("claude-haiku-4-5-20251001-thinking")
|
|
.some(variant => variant.selector === "demo/anthropic/claude-haiku-latest"),
|
|
).toBe(false);
|
|
});
|
|
|
|
test("collapses wrapped gemini tool and tuning variants under the base preview id", () => {
|
|
writeRawModelsJson({
|
|
demo: providerConfig("https://demo.example.com/v1", [
|
|
{ id: "google/gemini-3.1-pro-preview" },
|
|
{ id: "google/gemini-3.1-pro-preview-customtools" },
|
|
{ id: "google/gemini-3.1-pro-preview-high" },
|
|
]),
|
|
});
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
const variants = registry.getCanonicalVariants("gemini-3.1-pro-preview");
|
|
|
|
expect(variants.some(variant => variant.selector === "demo/google/gemini-3.1-pro-preview")).toBe(true);
|
|
expect(variants.some(variant => variant.selector === "demo/google/gemini-3.1-pro-preview-customtools")).toBe(
|
|
true,
|
|
);
|
|
expect(variants.some(variant => variant.selector === "demo/google/gemini-3.1-pro-preview-high")).toBe(true);
|
|
});
|
|
|
|
test("collapses compact version aliases and hardware suffixes into clean canonical ids", () => {
|
|
writeRawModelsJson({
|
|
demo: providerConfig("https://demo.example.com/v1", [
|
|
{ id: "hf:nvidia/Kimi-K2.5-NVFP4" },
|
|
{ id: "kimi-k2-5" },
|
|
{ id: "z-ai/glm4.7" },
|
|
{ id: "z-ai/glm5" },
|
|
]),
|
|
});
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
const kimiVariants = registry.getCanonicalVariants("kimi-k2.5");
|
|
const glm47Variants = registry.getCanonicalVariants("glm-4.7");
|
|
const glm5Variants = registry.getCanonicalVariants("glm-5");
|
|
|
|
expect(kimiVariants.some(variant => variant.selector === "demo/hf:nvidia/Kimi-K2.5-NVFP4")).toBe(true);
|
|
expect(kimiVariants.some(variant => variant.selector === "demo/kimi-k2-5")).toBe(true);
|
|
expect(glm47Variants.some(variant => variant.selector === "demo/z-ai/glm4.7")).toBe(true);
|
|
expect(glm5Variants.some(variant => variant.selector === "demo/z-ai/glm5")).toBe(true);
|
|
});
|
|
|
|
test("prefers clean canonical ids over bundled wrapper ids when available", () => {
|
|
writeRawModelsJson({
|
|
demo: providerConfig("https://demo.example.com/v1", [
|
|
{ id: "zai/glm-4.6v-flash" },
|
|
{ id: "hf:deepseek-ai/DeepSeek-V3" },
|
|
{ id: "google/gemini-pro-latest" },
|
|
]),
|
|
});
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
|
|
expect(
|
|
registry
|
|
.getCanonicalVariants("glm-4.6v-flash")
|
|
.some(variant => variant.selector === "demo/zai/glm-4.6v-flash"),
|
|
).toBe(true);
|
|
expect(
|
|
registry
|
|
.getCanonicalVariants("deepseek-v3")
|
|
.some(variant => variant.selector === "demo/hf:deepseek-ai/DeepSeek-V3"),
|
|
).toBe(true);
|
|
expect(
|
|
registry
|
|
.getCanonicalVariants("gemini-pro")
|
|
.some(variant => variant.selector === "demo/google/gemini-pro-latest"),
|
|
).toBe(true);
|
|
});
|
|
|
|
test("applies explicit equivalence overrides from config", () => {
|
|
writeRawModelsConfig({
|
|
providers: {
|
|
"proxy-anthropic": providerConfig("https://demo.example.com/v1", [{ id: "corp-sonnet" }]),
|
|
},
|
|
equivalence: {
|
|
overrides: {
|
|
"proxy-anthropic/corp-sonnet": "claude-sonnet-4-5",
|
|
},
|
|
},
|
|
});
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
const variants = registry.getCanonicalVariants("claude-sonnet-4-5");
|
|
|
|
expect(variants.some(variant => variant.selector === "proxy-anthropic/corp-sonnet")).toBe(true);
|
|
});
|
|
|
|
test("exclusions keep variants out of canonical grouping", () => {
|
|
writeRawModelsConfig({
|
|
providers: {
|
|
demo: providerConfig("https://demo.example.com/v1", [{ id: "anthropic/claude-sonnet-4.5" }]),
|
|
},
|
|
equivalence: {
|
|
exclude: ["demo/anthropic/claude-sonnet-4.5"],
|
|
},
|
|
});
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
const grouped = registry.getCanonicalVariants("claude-sonnet-4-5");
|
|
const fallback = registry.getCanonicalVariants("anthropic/claude-sonnet-4.5");
|
|
|
|
expect(grouped.some(variant => variant.selector === "demo/anthropic/claude-sonnet-4.5")).toBe(false);
|
|
expect(fallback.some(variant => variant.selector === "demo/anthropic/claude-sonnet-4.5")).toBe(true);
|
|
});
|
|
|
|
test("resolves canonical models using configured provider order", async () => {
|
|
await Settings.init({
|
|
inMemory: true,
|
|
overrides: {
|
|
modelProviderOrder: ["demo", "anthropic"],
|
|
},
|
|
});
|
|
writeRawModelsJson({
|
|
demo: providerConfig("https://demo.example.com/v1", [{ id: "anthropic/claude-sonnet-4.5" }]),
|
|
});
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
const resolved = registry.resolveCanonicalModel("claude-sonnet-4-5", {
|
|
availableOnly: false,
|
|
candidates: registry.getAll(),
|
|
});
|
|
|
|
expect(resolved?.provider).toBe("demo");
|
|
expect(resolved?.id).toBe("anthropic/claude-sonnet-4.5");
|
|
});
|
|
});
|
|
|
|
describe("OpenRouter routed suffix fallback", () => {
|
|
test("find synthesizes a routed model id from the base OpenRouter metadata", () => {
|
|
writeRawModelsJson({
|
|
openrouter: providerConfig("https://openrouter.ai/api/v1", [{ id: "z-ai/glm-4.7" }]),
|
|
});
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
const model = registry.find("openrouter", "z-ai/glm-4.7-20251222:nitro");
|
|
|
|
expect(model?.provider).toBe("openrouter");
|
|
expect(model?.id).toBe("z-ai/glm-4.7-20251222:nitro");
|
|
expect(model?.name).toBe("z-ai/glm-4.7-20251222:nitro");
|
|
});
|
|
});
|
|
|
|
describe("baseUrl override (no custom models)", () => {
|
|
test("overriding baseUrl keeps all built-in models", () => {
|
|
writeRawModelsJson({
|
|
anthropic: overrideConfig("https://my-proxy.example.com/v1"),
|
|
});
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
const anthropicModels = getModelsForProvider(registry, "anthropic");
|
|
|
|
// Should have multiple built-in models, not just one
|
|
expect(anthropicModels.length).toBeGreaterThan(1);
|
|
expect(anthropicModels.some(m => m.id.includes("claude"))).toBe(true);
|
|
});
|
|
|
|
test("overriding baseUrl changes URL on all built-in models", () => {
|
|
writeRawModelsJson({
|
|
anthropic: overrideConfig("https://my-proxy.example.com/v1"),
|
|
});
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
const anthropicModels = getModelsForProvider(registry, "anthropic");
|
|
|
|
// All models should have the new baseUrl
|
|
for (const model of anthropicModels) {
|
|
expect(model.baseUrl).toBe("https://my-proxy.example.com/v1");
|
|
}
|
|
});
|
|
|
|
test("overriding headers merges with model headers", () => {
|
|
writeRawModelsJson({
|
|
anthropic: overrideConfig("https://my-proxy.example.com/v1", {
|
|
"X-Custom-Header": "custom-value",
|
|
}),
|
|
});
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
const anthropicModels = getModelsForProvider(registry, "anthropic");
|
|
|
|
for (const model of anthropicModels) {
|
|
expect(model.headers?.["X-Custom-Header"]).toBe("custom-value");
|
|
}
|
|
});
|
|
|
|
test("headers-only override applies to built-in models", () => {
|
|
writeRawModelsJson({
|
|
anthropic: {
|
|
headers: { "X-Custom-Header": "custom-only" },
|
|
},
|
|
});
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
const anthropicModels = getModelsForProvider(registry, "anthropic");
|
|
|
|
expect(anthropicModels.length).toBeGreaterThan(1);
|
|
for (const model of anthropicModels) {
|
|
expect(model.headers?.["X-Custom-Header"]).toBe("custom-only");
|
|
}
|
|
});
|
|
|
|
test("authHeader override applies bearer auth to built-in models without custom models", () => {
|
|
writeRawModelsJson({
|
|
anthropic: {
|
|
baseUrl: "https://anthropic-proxy.example.com/v1",
|
|
apiKey: "issue-929-key",
|
|
authHeader: true,
|
|
},
|
|
});
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
const anthropicModels = getModelsForProvider(registry, "anthropic");
|
|
|
|
expect(anthropicModels.length).toBeGreaterThan(1);
|
|
for (const model of anthropicModels) {
|
|
expect(model.headers?.Authorization).toBe("Bearer issue-929-key");
|
|
}
|
|
});
|
|
|
|
test("apiKey-only override supplies fallback auth for built-in models", async () => {
|
|
const originalOpenAiKey = Bun.env.OPENAI_API_KEY;
|
|
delete Bun.env.OPENAI_API_KEY;
|
|
try {
|
|
writeRawModelsJson({
|
|
openai: {
|
|
apiKey: "issue-typed-key",
|
|
},
|
|
});
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
const openaiModels = getModelsForProvider(registry, "openai");
|
|
|
|
expect(openaiModels.length).toBeGreaterThan(0);
|
|
await expect(registry.getApiKey(openaiModels[0])).resolves.toBe("issue-typed-key");
|
|
} finally {
|
|
if (originalOpenAiKey === undefined) delete Bun.env.OPENAI_API_KEY;
|
|
else Bun.env.OPENAI_API_KEY = originalOpenAiKey;
|
|
}
|
|
});
|
|
test("baseUrl-only override does not affect other providers", () => {
|
|
writeRawModelsJson({
|
|
anthropic: overrideConfig("https://my-proxy.example.com/v1"),
|
|
});
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
const googleModels = getModelsForProvider(registry, "google");
|
|
|
|
// Google models should still have their original baseUrl
|
|
expect(googleModels.length).toBeGreaterThan(0);
|
|
expect(googleModels[0].baseUrl).not.toBe("https://my-proxy.example.com/v1");
|
|
});
|
|
|
|
test("can mix baseUrl override and models merge", () => {
|
|
writeRawModelsJson({
|
|
// baseUrl-only for anthropic
|
|
anthropic: overrideConfig("https://anthropic-proxy.example.com/v1"),
|
|
// Add custom model for google (merged with built-ins)
|
|
google: providerConfig(
|
|
"https://google-proxy.example.com/v1",
|
|
[{ id: "gemini-custom" }],
|
|
"google-generative-ai",
|
|
),
|
|
});
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
|
|
// Anthropic: multiple built-in models with new baseUrl
|
|
const anthropicModels = getModelsForProvider(registry, "anthropic");
|
|
expect(anthropicModels.length).toBeGreaterThan(1);
|
|
expect(anthropicModels[0].baseUrl).toBe("https://anthropic-proxy.example.com/v1");
|
|
|
|
// Google: built-ins plus custom model
|
|
const googleModels = getModelsForProvider(registry, "google");
|
|
expect(googleModels.length).toBeGreaterThan(1);
|
|
expect(googleModels.some(m => m.id === "gemini-custom")).toBe(true);
|
|
});
|
|
|
|
test("refresh() picks up baseUrl override changes", async () => {
|
|
writeRawModelsJson({
|
|
anthropic: overrideConfig("https://first-proxy.example.com/v1"),
|
|
});
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
|
|
expect(getModelsForProvider(registry, "anthropic")[0].baseUrl).toBe("https://first-proxy.example.com/v1");
|
|
|
|
// Update and refresh
|
|
writeRawModelsJson({
|
|
anthropic: overrideConfig("https://second-proxy.example.com/v1"),
|
|
});
|
|
await registry.refresh("offline");
|
|
|
|
expect(getModelsForProvider(registry, "anthropic")[0].baseUrl).toBe("https://second-proxy.example.com/v1");
|
|
});
|
|
});
|
|
|
|
describe("provider compat overrides", () => {
|
|
test("provider-level compat applies to built-in models", () => {
|
|
writeRawModelsJson({
|
|
openrouter: {
|
|
compat: {
|
|
supportsUsageInStreaming: false,
|
|
supportsStrictMode: false,
|
|
supportsMultipleSystemMessages: false,
|
|
disableReasoningOnToolChoice: true,
|
|
allowsSyntheticReasoningContentForToolCalls: false,
|
|
},
|
|
},
|
|
});
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
const models = getModelsForProvider(registry, "openrouter");
|
|
expect(models.length).toBeGreaterThan(0);
|
|
for (const model of models) {
|
|
expect(getOpenAICompat(model)?.supportsUsageInStreaming).toBe(false);
|
|
expect(getOpenAICompat(model)?.supportsStrictMode).toBe(false);
|
|
expect(getOpenAICompat(model)?.supportsMultipleSystemMessages).toBe(false);
|
|
expect(getOpenAICompat(model)?.disableReasoningOnToolChoice).toBe(true);
|
|
expect(getOpenAICompat(model)?.allowsSyntheticReasoningContentForToolCalls).toBe(false);
|
|
}
|
|
});
|
|
|
|
test("provider-level compat applies to custom models", () => {
|
|
writeRawModelsJson({
|
|
demo: {
|
|
baseUrl: "https://example.com/v1",
|
|
apiKey: "DEMO_KEY",
|
|
api: "openai-completions",
|
|
compat: {
|
|
supportsUsageInStreaming: false,
|
|
maxTokensField: "max_tokens",
|
|
},
|
|
models: [
|
|
{
|
|
id: "demo-model",
|
|
reasoning: false,
|
|
input: ["text"],
|
|
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
|
contextWindow: 1000,
|
|
maxTokens: 100,
|
|
},
|
|
],
|
|
},
|
|
});
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
const model = registry.find("demo", "demo-model");
|
|
const compat = getOpenAICompat(model);
|
|
expect(compat?.supportsUsageInStreaming).toBe(false);
|
|
expect(compat?.maxTokensField).toBe("max_tokens");
|
|
});
|
|
|
|
test("model-level compat overrides provider-level compat for custom models", () => {
|
|
writeRawModelsJson({
|
|
demo: {
|
|
baseUrl: "https://example.com/v1",
|
|
apiKey: "DEMO_KEY",
|
|
api: "openai-completions",
|
|
compat: {
|
|
supportsUsageInStreaming: false,
|
|
maxTokensField: "max_tokens",
|
|
},
|
|
models: [
|
|
{
|
|
id: "demo-model",
|
|
reasoning: false,
|
|
input: ["text"],
|
|
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
|
contextWindow: 1000,
|
|
maxTokens: 100,
|
|
compat: {
|
|
supportsUsageInStreaming: true,
|
|
maxTokensField: "max_completion_tokens",
|
|
},
|
|
},
|
|
],
|
|
},
|
|
});
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
const model = registry.find("demo", "demo-model");
|
|
const compat = getOpenAICompat(model);
|
|
expect(compat?.supportsUsageInStreaming).toBe(true);
|
|
expect(compat?.maxTokensField).toBe("max_completion_tokens");
|
|
});
|
|
});
|
|
|
|
describe("custom models merge behavior", () => {
|
|
test("custom provider with same name as built-in merges with built-in models", () => {
|
|
writeModelsJson({
|
|
anthropic: providerConfig("https://my-proxy.example.com/v1", [{ id: "claude-custom" }]),
|
|
});
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
const anthropicModels = getModelsForProvider(registry, "anthropic");
|
|
|
|
// Built-in models still present, custom model merged in
|
|
expect(anthropicModels.length).toBeGreaterThan(1);
|
|
const custom = anthropicModels.find(m => m.id === "claude-custom");
|
|
expect(custom).toBeDefined();
|
|
expect(custom!.baseUrl).toBe("https://my-proxy.example.com/v1");
|
|
});
|
|
|
|
test("custom model with same id replaces built-in model by id", () => {
|
|
writeModelsJson({
|
|
openrouter: providerConfig(
|
|
"https://my-proxy.example.com/v1",
|
|
[{ id: "anthropic/claude-sonnet-4" }],
|
|
"openai-completions",
|
|
),
|
|
});
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
const models = getModelsForProvider(registry, "openrouter");
|
|
const sonnetModels = models.filter(m => m.id === "anthropic/claude-sonnet-4");
|
|
|
|
expect(sonnetModels).toHaveLength(1);
|
|
expect(sonnetModels[0].baseUrl).toBe("https://my-proxy.example.com/v1");
|
|
});
|
|
|
|
test("custom same-id replacement does not keep bundled headers", () => {
|
|
writeRawModelsJson({
|
|
"github-copilot": {
|
|
baseUrl: "https://proxy.example.com/v1",
|
|
headers: { "X-Proxy": "proxy" },
|
|
apiKey: "TEST_KEY",
|
|
api: "openai-completions",
|
|
models: [{ id: "gpt-4o" }],
|
|
},
|
|
});
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
const model = registry.find("github-copilot", "gpt-4o");
|
|
|
|
expect(model?.headers).toEqual({ "X-Proxy": "proxy" });
|
|
expect(model?.headers?.["User-Agent"]).toBeUndefined();
|
|
expect(model?.headers?.["Editor-Version"]).toBeUndefined();
|
|
});
|
|
|
|
test("custom provider with same name as built-in does not affect other built-in providers", () => {
|
|
writeModelsJson({
|
|
anthropic: providerConfig("https://my-proxy.example.com/v1", [{ id: "claude-custom" }]),
|
|
});
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
|
|
expect(getModelsForProvider(registry, "google").length).toBeGreaterThan(0);
|
|
expect(getModelsForProvider(registry, "openai").length).toBeGreaterThan(0);
|
|
});
|
|
|
|
test("provider-level baseUrl applies to both built-in and custom models", () => {
|
|
writeModelsJson({
|
|
anthropic: providerConfig("https://merged-proxy.example.com/v1", [{ id: "claude-custom" }]),
|
|
});
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
const anthropicModels = getModelsForProvider(registry, "anthropic");
|
|
|
|
for (const model of anthropicModels) {
|
|
expect(model.baseUrl).toBe("https://merged-proxy.example.com/v1");
|
|
}
|
|
});
|
|
|
|
test("model-level baseUrl overrides provider-level baseUrl for custom models", () => {
|
|
writeRawModelsJson({
|
|
"opencode-go": {
|
|
baseUrl: "https://opencode.ai/zen/go/v1",
|
|
apiKey: "TEST_KEY",
|
|
models: [
|
|
{
|
|
id: "minimax-m2.5",
|
|
api: "anthropic-messages",
|
|
baseUrl: "https://opencode.ai/zen/go",
|
|
reasoning: true,
|
|
input: ["text"],
|
|
cost: { input: 0.3, output: 1.2, cacheRead: 0.03, cacheWrite: 0 },
|
|
contextWindow: 204800,
|
|
maxTokens: 131072,
|
|
},
|
|
{
|
|
id: "glm-5",
|
|
api: "openai-completions",
|
|
reasoning: true,
|
|
input: ["text"],
|
|
cost: { input: 1, output: 3.2, cacheRead: 0.2, cacheWrite: 0 },
|
|
contextWindow: 204800,
|
|
maxTokens: 131072,
|
|
},
|
|
],
|
|
},
|
|
});
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
const m25 = registry.find("opencode-go", "minimax-m2.5");
|
|
const glm5 = registry.find("opencode-go", "glm-5");
|
|
|
|
expect(m25?.baseUrl).toBe("https://opencode.ai/zen/go");
|
|
expect(glm5?.baseUrl).toBe("https://opencode.ai/zen/go/v1");
|
|
});
|
|
|
|
test("modelOverrides still apply when provider also defines models", () => {
|
|
writeRawModelsJson({
|
|
openrouter: {
|
|
baseUrl: "https://my-proxy.example.com/v1",
|
|
apiKey: "OPENROUTER_API_KEY",
|
|
api: "openai-completions",
|
|
models: [
|
|
{
|
|
id: "custom/openrouter-model",
|
|
name: "Custom OpenRouter Model",
|
|
reasoning: false,
|
|
input: ["text"],
|
|
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
|
contextWindow: 128000,
|
|
maxTokens: 16384,
|
|
},
|
|
],
|
|
modelOverrides: {
|
|
"anthropic/claude-sonnet-4": {
|
|
name: "Overridden Built-in Sonnet",
|
|
},
|
|
},
|
|
},
|
|
});
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
const models = getModelsForProvider(registry, "openrouter");
|
|
|
|
expect(models.some(m => m.id === "custom/openrouter-model")).toBe(true);
|
|
expect(models.some(m => m.id === "anthropic/claude-sonnet-4" && m.name === "Overridden Built-in Sonnet")).toBe(
|
|
true,
|
|
);
|
|
});
|
|
|
|
test("refresh() reloads merged custom models from disk", async () => {
|
|
writeModelsJson({
|
|
anthropic: providerConfig("https://first-proxy.example.com/v1", [{ id: "claude-custom" }]),
|
|
});
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
expect(getModelsForProvider(registry, "anthropic").some(m => m.id === "claude-custom")).toBe(true);
|
|
|
|
// Update and refresh
|
|
writeModelsJson({
|
|
anthropic: providerConfig("https://second-proxy.example.com/v1", [{ id: "claude-custom-2" }]),
|
|
});
|
|
await registry.refresh("offline");
|
|
|
|
const anthropicModels = getModelsForProvider(registry, "anthropic");
|
|
expect(anthropicModels.some(m => m.id === "claude-custom")).toBe(false);
|
|
expect(anthropicModels.some(m => m.id === "claude-custom-2")).toBe(true);
|
|
expect(anthropicModels.some(m => m.id.includes("claude"))).toBe(true);
|
|
});
|
|
|
|
test("built-in gpt-5.4 applies the hardcoded context window policy", () => {
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
expect(registry.find("openai", "gpt-5.4")?.contextWindow).toBe(1_000_000);
|
|
});
|
|
|
|
test("custom gpt-5.4 replacement keeps the hardcoded context window when contextWindow is omitted", () => {
|
|
writeRawModelsJson({
|
|
openai: {
|
|
baseUrl: "https://my-proxy.example.com/v1",
|
|
apiKey: "TEST_KEY",
|
|
api: "openai-responses",
|
|
models: [{ id: "gpt-5.4" }],
|
|
},
|
|
});
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
const model = registry.find("openai", "gpt-5.4");
|
|
expect(model?.contextWindow).toBe(1_000_000);
|
|
expect(model?.baseUrl).toBe("https://my-proxy.example.com/v1");
|
|
});
|
|
|
|
test("custom-only gpt-5.4 provider keeps the hardcoded context window when contextWindow is omitted", () => {
|
|
writeRawModelsJson({
|
|
"my-proxy": {
|
|
baseUrl: "https://my-proxy.example.com/v1",
|
|
apiKey: "TEST_KEY",
|
|
api: "openai-responses",
|
|
models: [{ id: "gpt-5.4" }],
|
|
},
|
|
});
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
const model = registry.find("my-proxy", "gpt-5.4");
|
|
expect(model?.contextWindow).toBe(1_000_000);
|
|
expect(model?.baseUrl).toBe("https://my-proxy.example.com/v1");
|
|
});
|
|
|
|
test("custom gpt-5.4 replacement preserves its explicit context window", () => {
|
|
writeModelsJson({
|
|
openai: providerConfig(
|
|
"https://my-proxy.example.com/v1",
|
|
[{ id: "gpt-5.4", contextWindow: 256000 }],
|
|
"openai-responses",
|
|
),
|
|
});
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
expect(registry.find("openai", "gpt-5.4")?.contextWindow).toBe(256000);
|
|
});
|
|
|
|
test("modelOverrides can still patch a custom gpt-5.4 replacement", () => {
|
|
writeRawModelsJson({
|
|
openai: {
|
|
baseUrl: "https://my-proxy.example.com/v1",
|
|
apiKey: "TEST_KEY",
|
|
api: "openai-responses",
|
|
models: [
|
|
{
|
|
id: "gpt-5.4",
|
|
name: "gpt-5.4",
|
|
reasoning: false,
|
|
input: ["text"],
|
|
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
|
contextWindow: 256000,
|
|
maxTokens: 128000,
|
|
},
|
|
],
|
|
modelOverrides: {
|
|
"gpt-5.4": {
|
|
contextWindow: 512000,
|
|
},
|
|
},
|
|
},
|
|
});
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
expect(registry.find("openai", "gpt-5.4")?.contextWindow).toBe(512000);
|
|
});
|
|
|
|
test("discoverable bundled replacement survives refresh", async () => {
|
|
writeModelsJson({
|
|
openai: providerConfig(
|
|
"https://my-proxy.example.com/v1",
|
|
[{ id: "gpt-5.4", name: "Proxy GPT-5.4", contextWindow: 256000 }],
|
|
"openai-responses",
|
|
),
|
|
});
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
expect(registry.find("openai", "gpt-5.4")?.name).toBe("Proxy GPT-5.4");
|
|
expect(registry.find("openai", "gpt-5.4")?.contextWindow).toBe(256000);
|
|
|
|
using _hook = mockOpenAiCompatibleModels("https://my-proxy.example.com/v1/models", ["gpt-5.4"]);
|
|
await registry.refreshProvider("openai", "online");
|
|
|
|
const model = registry.find("openai", "gpt-5.4");
|
|
expect(model?.name).toBe("Proxy GPT-5.4");
|
|
expect(model?.contextWindow).toBe(256000);
|
|
expect(model?.baseUrl).toBe("https://my-proxy.example.com/v1");
|
|
});
|
|
|
|
test("discoverable custom-only gpt-5.4 survives refresh", async () => {
|
|
writeRawModelsJson({
|
|
"custom-local": {
|
|
baseUrl: "http://127.0.0.1:8080",
|
|
apiKey: "TEST_KEY",
|
|
api: "openai-responses",
|
|
discovery: { type: "llama.cpp" },
|
|
models: [{ id: "gpt-5.4" }],
|
|
},
|
|
});
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
expect(registry.find("custom-local", "gpt-5.4")?.contextWindow).toBe(1_000_000);
|
|
|
|
using _hook = mockOpenAiCompatibleModels("http://127.0.0.1:8080/models", ["gpt-5.4"]);
|
|
await registry.refreshProvider("custom-local", "online");
|
|
|
|
const model = registry.find("custom-local", "gpt-5.4");
|
|
expect(model?.contextWindow).toBe(1_000_000);
|
|
expect(model?.baseUrl).toBe("http://127.0.0.1:8080");
|
|
});
|
|
|
|
test("discoverable custom compat survives refresh", async () => {
|
|
writeRawModelsJson({
|
|
openai: {
|
|
baseUrl: "https://my-proxy.example.com/v1",
|
|
apiKey: "TEST_KEY",
|
|
api: "openai-responses",
|
|
models: [
|
|
{
|
|
id: "gpt-5.4",
|
|
compat: {
|
|
extraBody: { source: "proxy" },
|
|
},
|
|
},
|
|
],
|
|
},
|
|
});
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
expect(getOpenAICompat(registry.find("openai", "gpt-5.4"))?.extraBody).toEqual({ source: "proxy" });
|
|
|
|
using _hook = mockOpenAiCompatibleModels("https://my-proxy.example.com/v1/models", ["gpt-5.4"]);
|
|
await registry.refreshProvider("openai", "online");
|
|
|
|
expect(getOpenAICompat(registry.find("openai", "gpt-5.4"))?.extraBody).toEqual({ source: "proxy" });
|
|
});
|
|
|
|
test("modelOverrides still apply after discoverable refresh", async () => {
|
|
writeRawModelsJson({
|
|
openai: {
|
|
baseUrl: "https://my-proxy.example.com/v1",
|
|
apiKey: "TEST_KEY",
|
|
api: "openai-responses",
|
|
models: [
|
|
{
|
|
id: "gpt-5.4",
|
|
contextWindow: 256000,
|
|
},
|
|
],
|
|
modelOverrides: {
|
|
"gpt-5.4": {
|
|
contextWindow: 512000,
|
|
},
|
|
},
|
|
},
|
|
});
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
expect(registry.find("openai", "gpt-5.4")?.contextWindow).toBe(512000);
|
|
|
|
using _hook = mockOpenAiCompatibleModels("https://my-proxy.example.com/v1/models", ["gpt-5.4"]);
|
|
await registry.refreshProvider("openai", "online");
|
|
|
|
expect(registry.find("openai", "gpt-5.4")?.contextWindow).toBe(512000);
|
|
});
|
|
|
|
test("newly discovered ids inherit provider fields, not another model's custom fields", async () => {
|
|
writeRawModelsJson({
|
|
openai: {
|
|
baseUrl: "https://provider.example.com/v1",
|
|
headers: { "X-Provider": "provider" },
|
|
apiKey: "TEST_KEY",
|
|
api: "openai-responses",
|
|
models: [
|
|
{
|
|
id: "gpt-5.4",
|
|
baseUrl: "https://special.example.com/v1",
|
|
headers: { "X-Model": "special" },
|
|
},
|
|
],
|
|
},
|
|
});
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
expect(registry.find("openai", "gpt-5.4")?.baseUrl).toBe("https://special.example.com/v1");
|
|
|
|
using _hook = mockOpenAiCompatibleModels("https://provider.example.com/v1/models", ["gpt-5.4", "gpt-5.5"]);
|
|
await registry.refreshProvider("openai", "online");
|
|
|
|
const discovered = registry.find("openai", "gpt-5.5");
|
|
expect(discovered?.baseUrl).toBe("https://provider.example.com/v1");
|
|
expect(discovered?.headers?.["X-Provider"]).toBe("provider");
|
|
expect(discovered?.headers?.["X-Model"]).toBeUndefined();
|
|
});
|
|
|
|
test("same-id replacement uses configured compat without bundled compat leak", () => {
|
|
writeRawModelsJson({
|
|
"minimax-code": {
|
|
baseUrl: "https://proxy.example.com/v1",
|
|
apiKey: "TEST_KEY",
|
|
api: "openai-completions",
|
|
compat: {
|
|
extraBody: { source: "proxy" },
|
|
},
|
|
models: [{ id: "MiniMax-M2.5" }],
|
|
},
|
|
});
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
const model = registry.find("minimax-code", "MiniMax-M2.5");
|
|
const compat = getOpenAICompat(model);
|
|
expect(compat?.thinkingFormat).toBeUndefined();
|
|
expect(compat?.reasoningContentField).toBeUndefined();
|
|
expect(compat?.extraBody).toEqual({ source: "proxy" });
|
|
});
|
|
|
|
test("removing custom models from models.json keeps built-in provider models", async () => {
|
|
writeModelsJson({
|
|
anthropic: providerConfig("https://proxy.example.com/v1", [{ id: "claude-custom" }]),
|
|
});
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
|
|
expect(getModelsForProvider(registry, "anthropic").some(m => m.id === "claude-custom")).toBe(true);
|
|
|
|
// Remove custom models and refresh
|
|
writeModelsJson({});
|
|
await registry.refresh("offline");
|
|
|
|
const anthropicModels = getModelsForProvider(registry, "anthropic");
|
|
expect(anthropicModels.length).toBeGreaterThan(1);
|
|
expect(anthropicModels.some(m => m.id === "claude-custom")).toBe(false);
|
|
expect(anthropicModels.some(m => m.id.includes("claude"))).toBe(true);
|
|
});
|
|
});
|
|
|
|
describe("thinking metadata normalization", () => {
|
|
test("custom models preserve explicit thinking", () => {
|
|
const thinking: ThinkingConfig = {
|
|
mode: "anthropic-adaptive",
|
|
minLevel: Effort.Minimal,
|
|
maxLevel: Effort.High,
|
|
levels: [Effort.Minimal, Effort.High],
|
|
};
|
|
|
|
writeModelsJson({
|
|
anthropic: providerConfig("https://my-proxy.example.com/v1", [
|
|
{ id: "claude-custom", reasoning: true, thinking },
|
|
]),
|
|
});
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
const model = getModelsForProvider(registry, "anthropic").find(m => m.id === "claude-custom");
|
|
|
|
expect(model?.thinking).toEqual(thinking);
|
|
});
|
|
|
|
test("model overrides can replace canonical thinking metadata", () => {
|
|
writeRawModelsJson({
|
|
openrouter: {
|
|
modelOverrides: {
|
|
"anthropic/claude-sonnet-4": {
|
|
thinking: { mode: "budget", minLevel: Effort.Low, maxLevel: Effort.Medium },
|
|
},
|
|
},
|
|
},
|
|
});
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
const model = getModelsForProvider(registry, "openrouter").find(m => m.id === "anthropic/claude-sonnet-4");
|
|
|
|
expect(model?.thinking).toEqual({
|
|
mode: "budget",
|
|
minLevel: Effort.Low,
|
|
maxLevel: Effort.Medium,
|
|
});
|
|
});
|
|
});
|
|
|
|
describe("modelOverrides (per-model customization)", () => {
|
|
test("model override applies to a single built-in model", () => {
|
|
writeRawModelsJson({
|
|
openrouter: {
|
|
modelOverrides: {
|
|
"anthropic/claude-sonnet-4": {
|
|
name: "Custom Sonnet Name",
|
|
},
|
|
},
|
|
},
|
|
});
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
const models = getModelsForProvider(registry, "openrouter");
|
|
|
|
const sonnet = models.find(m => m.id === "anthropic/claude-sonnet-4");
|
|
expect(sonnet?.name).toBe("Custom Sonnet Name");
|
|
|
|
// Other models should be unchanged
|
|
const opus = models.find(m => m.id === "anthropic/claude-opus-4");
|
|
expect(opus?.name).not.toBe("Custom Sonnet Name");
|
|
});
|
|
|
|
test("model override with compat.openRouterRouting", () => {
|
|
writeRawModelsJson({
|
|
openrouter: {
|
|
modelOverrides: {
|
|
"anthropic/claude-sonnet-4": {
|
|
compat: {
|
|
openRouterRouting: { only: ["amazon-bedrock"] },
|
|
},
|
|
},
|
|
},
|
|
},
|
|
});
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
const models = getModelsForProvider(registry, "openrouter");
|
|
|
|
const sonnet = models.find(m => m.id === "anthropic/claude-sonnet-4");
|
|
const compat = sonnet?.compat as OpenAICompat | undefined;
|
|
expect(compat?.openRouterRouting).toEqual({ only: ["amazon-bedrock"] });
|
|
});
|
|
|
|
test("model override deep merges compat settings", () => {
|
|
writeRawModelsJson({
|
|
openrouter: {
|
|
modelOverrides: {
|
|
"anthropic/claude-sonnet-4": {
|
|
compat: {
|
|
openRouterRouting: { order: ["anthropic", "together"] },
|
|
},
|
|
},
|
|
},
|
|
},
|
|
});
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
const models = getModelsForProvider(registry, "openrouter");
|
|
const sonnet = models.find(m => m.id === "anthropic/claude-sonnet-4");
|
|
|
|
const compat = sonnet?.compat as OpenAICompat | undefined;
|
|
expect(compat?.openRouterRouting).toEqual({ order: ["anthropic", "together"] });
|
|
});
|
|
|
|
test("model override merges compat.extraBody across provider+model", () => {
|
|
writeRawModelsJson({
|
|
openrouter: {
|
|
compat: {
|
|
extraBody: {
|
|
gateway: "default-gateway",
|
|
controller: "provider-controller",
|
|
},
|
|
},
|
|
modelOverrides: {
|
|
"anthropic/claude-sonnet-4": {
|
|
compat: {
|
|
extraBody: {
|
|
controller: "model-controller",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
});
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
const models = getModelsForProvider(registry, "openrouter");
|
|
const sonnet = models.find(m => m.id === "anthropic/claude-sonnet-4");
|
|
|
|
const compat = sonnet?.compat as OpenAICompat | undefined;
|
|
expect(compat?.extraBody).toEqual({ gateway: "default-gateway", controller: "model-controller" });
|
|
});
|
|
|
|
test("multiple model overrides on same provider", () => {
|
|
writeRawModelsJson({
|
|
openrouter: {
|
|
modelOverrides: {
|
|
"anthropic/claude-sonnet-4": {
|
|
compat: { openRouterRouting: { only: ["amazon-bedrock"] } },
|
|
},
|
|
"anthropic/claude-opus-4": {
|
|
compat: { openRouterRouting: { only: ["anthropic"] } },
|
|
},
|
|
},
|
|
},
|
|
});
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
const models = getModelsForProvider(registry, "openrouter");
|
|
|
|
const sonnet = models.find(m => m.id === "anthropic/claude-sonnet-4");
|
|
const opus = models.find(m => m.id === "anthropic/claude-opus-4");
|
|
|
|
const sonnetCompat = sonnet?.compat as OpenAICompat | undefined;
|
|
const opusCompat = opus?.compat as OpenAICompat | undefined;
|
|
expect(sonnetCompat?.openRouterRouting).toEqual({ only: ["amazon-bedrock"] });
|
|
expect(opusCompat?.openRouterRouting).toEqual({ only: ["anthropic"] });
|
|
});
|
|
|
|
test("model override combined with baseUrl override", () => {
|
|
writeRawModelsJson({
|
|
openrouter: {
|
|
baseUrl: "https://my-proxy.example.com/v1",
|
|
modelOverrides: {
|
|
"anthropic/claude-sonnet-4": {
|
|
name: "Proxied Sonnet",
|
|
},
|
|
},
|
|
},
|
|
});
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
const models = getModelsForProvider(registry, "openrouter");
|
|
const sonnet = models.find(m => m.id === "anthropic/claude-sonnet-4");
|
|
|
|
// Both overrides should apply
|
|
expect(sonnet?.baseUrl).toBe("https://my-proxy.example.com/v1");
|
|
expect(sonnet?.name).toBe("Proxied Sonnet");
|
|
|
|
// Other models should have the baseUrl but not the name override
|
|
const opus = models.find(m => m.id === "anthropic/claude-opus-4");
|
|
expect(opus?.baseUrl).toBe("https://my-proxy.example.com/v1");
|
|
expect(opus?.name).not.toBe("Proxied Sonnet");
|
|
});
|
|
|
|
test("model override for non-existent model ID is ignored", () => {
|
|
writeRawModelsJson({
|
|
openrouter: {
|
|
modelOverrides: {
|
|
"nonexistent/model-id": {
|
|
name: "This should not appear",
|
|
},
|
|
},
|
|
},
|
|
});
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
const models = getModelsForProvider(registry, "openrouter");
|
|
|
|
// Should not create a new model
|
|
expect(models.find(m => m.id === "nonexistent/model-id")).toBeUndefined();
|
|
// Should not crash or show error
|
|
expect(registry.getError()).toBeUndefined();
|
|
});
|
|
|
|
test("model override can change cost fields partially", () => {
|
|
writeRawModelsJson({
|
|
openrouter: {
|
|
modelOverrides: {
|
|
"anthropic/claude-sonnet-4": {
|
|
cost: { input: 99 },
|
|
},
|
|
},
|
|
},
|
|
});
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
const models = getModelsForProvider(registry, "openrouter");
|
|
const sonnet = models.find(m => m.id === "anthropic/claude-sonnet-4");
|
|
|
|
// Input cost should be overridden
|
|
expect(sonnet?.cost.input).toBe(99);
|
|
// Other cost fields should be preserved from built-in
|
|
expect(sonnet?.cost.output).toBeGreaterThan(0);
|
|
});
|
|
|
|
test("model override can add headers", () => {
|
|
writeRawModelsJson({
|
|
openrouter: {
|
|
modelOverrides: {
|
|
"anthropic/claude-sonnet-4": {
|
|
headers: { "X-Custom-Model-Header": "value" },
|
|
},
|
|
},
|
|
},
|
|
});
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
const models = getModelsForProvider(registry, "openrouter");
|
|
const sonnet = models.find(m => m.id === "anthropic/claude-sonnet-4");
|
|
|
|
expect(sonnet?.headers?.["X-Custom-Model-Header"]).toBe("value");
|
|
});
|
|
|
|
test("refresh() picks up model override changes", async () => {
|
|
writeRawModelsJson({
|
|
openrouter: {
|
|
modelOverrides: {
|
|
"anthropic/claude-sonnet-4": {
|
|
name: "First Name",
|
|
},
|
|
},
|
|
},
|
|
});
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
expect(
|
|
getModelsForProvider(registry, "openrouter").find(m => m.id === "anthropic/claude-sonnet-4")?.name,
|
|
).toBe("First Name");
|
|
|
|
// Update and refresh
|
|
writeRawModelsJson({
|
|
openrouter: {
|
|
modelOverrides: {
|
|
"anthropic/claude-sonnet-4": {
|
|
name: "Second Name",
|
|
},
|
|
},
|
|
},
|
|
});
|
|
await registry.refresh("offline");
|
|
|
|
expect(
|
|
getModelsForProvider(registry, "openrouter").find(m => m.id === "anthropic/claude-sonnet-4")?.name,
|
|
).toBe("Second Name");
|
|
});
|
|
|
|
test("removing model override restores built-in values", async () => {
|
|
writeRawModelsJson({
|
|
openrouter: {
|
|
modelOverrides: {
|
|
"anthropic/claude-sonnet-4": {
|
|
name: "Custom Name",
|
|
},
|
|
},
|
|
},
|
|
});
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
const customName = getModelsForProvider(registry, "openrouter").find(
|
|
m => m.id === "anthropic/claude-sonnet-4",
|
|
)?.name;
|
|
expect(customName).toBe("Custom Name");
|
|
|
|
// Remove override and refresh
|
|
writeRawModelsJson({});
|
|
await registry.refresh("offline");
|
|
|
|
const restoredName = getModelsForProvider(registry, "openrouter").find(
|
|
m => m.id === "anthropic/claude-sonnet-4",
|
|
)?.name;
|
|
expect(restoredName).not.toBe("Custom Name");
|
|
});
|
|
});
|
|
|
|
describe("github-copilot oauth endpoint alignment", () => {
|
|
test("getApiKey does not mutate bundled github-copilot baseUrl", async () => {
|
|
await authStorage.set("github-copilot", [
|
|
{
|
|
type: "oauth",
|
|
access: "ghu_individual_token_123",
|
|
refresh: "ghu_individual_token_123",
|
|
expires: Date.now() + 60_000,
|
|
},
|
|
{
|
|
type: "oauth",
|
|
access: "ghu_enterprise_token_456",
|
|
refresh: "ghu_enterprise_token_456",
|
|
expires: Date.now() + 60_000,
|
|
enterpriseUrl: "ghe.example.com",
|
|
},
|
|
]);
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
const model = registry.find("github-copilot", "gpt-4o");
|
|
expect(model).toBeDefined();
|
|
if (!model) throw new Error("Expected github-copilot/gpt-4o model");
|
|
|
|
const initialBaseUrl = model.baseUrl;
|
|
const firstApiKey = await registry.getApiKey(model);
|
|
expect(firstApiKey).toBeDefined();
|
|
const firstParsed = JSON.parse(firstApiKey!) as { token?: string; enterpriseUrl?: string };
|
|
expect(firstParsed.token).toBe("ghu_individual_token_123");
|
|
expect(firstParsed.enterpriseUrl).toBeUndefined();
|
|
const secondApiKey = await registry.getApiKey(model);
|
|
expect(secondApiKey).toBeDefined();
|
|
const secondParsed = JSON.parse(secondApiKey!) as { token?: string; enterpriseUrl?: string };
|
|
expect(secondParsed.token).toBe("ghu_enterprise_token_456");
|
|
expect(secondParsed.enterpriseUrl).toBe("ghe.example.com");
|
|
expect(model.baseUrl).toBe(initialBaseUrl);
|
|
});
|
|
|
|
test("refreshProvider uses enterprise Copilot discovery host for peeked credentials", async () => {
|
|
await authStorage.set("github-copilot", [
|
|
{
|
|
type: "oauth",
|
|
access: "ghu_enterprise_token_456",
|
|
refresh: "ghu_enterprise_token_456",
|
|
expires: Date.now() + 60_000,
|
|
enterpriseUrl: "ghe.example.com",
|
|
},
|
|
]);
|
|
|
|
const requestedUrls: string[] = [];
|
|
using _hook = hookFetch((input: string | URL | Request, init?: RequestInit) => {
|
|
const url = input instanceof Request ? input.url : String(input);
|
|
requestedUrls.push(url);
|
|
if (url === "https://copilot-api.ghe.example.com/models") {
|
|
const authHeader =
|
|
input instanceof Request
|
|
? input.headers.get("Authorization")
|
|
: new Headers(init?.headers).get("Authorization");
|
|
expect(authHeader).toBe("Bearer ghu_enterprise_token_456");
|
|
return new Response(
|
|
JSON.stringify({
|
|
data: [
|
|
{
|
|
id: "gpt-5-mini",
|
|
name: "GPT-5 mini",
|
|
},
|
|
],
|
|
}),
|
|
{ status: 200, headers: { "Content-Type": "application/json" } },
|
|
);
|
|
}
|
|
throw new Error(`Unexpected URL: ${url}`);
|
|
});
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
await registry.refreshProvider("github-copilot", "online");
|
|
expect(requestedUrls).toContain("https://copilot-api.ghe.example.com/models");
|
|
expect(requestedUrls).not.toContain("https://api.githubcopilot.com/models");
|
|
});
|
|
});
|
|
|
|
describe("disabled provider filtering", () => {
|
|
test("getAvailable and getDiscoverableProviders exclude disabled providers from settings", async () => {
|
|
writeRawModelsJson({
|
|
ollama: {
|
|
baseUrl: "http://127.0.0.1:11434/v1",
|
|
api: "openai-completions",
|
|
auth: "none",
|
|
discovery: { type: "ollama" },
|
|
},
|
|
});
|
|
await authStorage.set("github-copilot", [
|
|
{
|
|
type: "oauth",
|
|
access: "ghu_test_token_for_disabled",
|
|
refresh: "ghu_test_token_for_disabled",
|
|
expires: Date.now() + 60_000,
|
|
},
|
|
]);
|
|
await Settings.init({
|
|
inMemory: true,
|
|
overrides: {
|
|
disabledProviders: ["github-copilot", "ollama"],
|
|
},
|
|
});
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
|
|
expect(registry.getAvailable().some(model => model.provider === "github-copilot")).toBe(false);
|
|
expect(registry.getDiscoverableProviders()).not.toContain("ollama");
|
|
});
|
|
|
|
test("refresh skips discovery probes for disabled local providers", async () => {
|
|
await Settings.init({
|
|
inMemory: true,
|
|
overrides: {
|
|
disabledProviders: ["llama.cpp", "lm-studio", "ollama"],
|
|
},
|
|
});
|
|
const requestedUrls: string[] = [];
|
|
using _hook = hookFetch(input => {
|
|
requestedUrls.push(String(input));
|
|
throw new Error(`Unexpected URL: ${String(input)}`);
|
|
});
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
await registry.refresh("online");
|
|
|
|
const disabledProbeUrls = requestedUrls.filter(
|
|
url => url.includes("127.0.0.1:11434") || url.includes("127.0.0.1:8080") || url.includes("127.0.0.1:1234"),
|
|
);
|
|
expect(disabledProbeUrls).toEqual([]);
|
|
});
|
|
});
|
|
describe("runtime discovery", () => {
|
|
test("auto-discovers ollama models without provider config", async () => {
|
|
using _hook = mockOllamaDiscovery(["phi4-mini"]);
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
await registry.refresh();
|
|
const ollamaModels = getModelsForProvider(registry, "ollama");
|
|
expect(ollamaModels.some(m => m.id === "phi4-mini")).toBe(true);
|
|
expect(registry.getAvailable().some(m => m.provider === "ollama" && m.id === "phi4-mini")).toBe(true);
|
|
expect(await registry.getApiKey(ollamaModels[0])).toBe(kNoAuth);
|
|
});
|
|
|
|
test("discovers ollama-cloud through built-in descriptor flow without regressing local implicit ollama", async () => {
|
|
authStorage.setRuntimeApiKey("ollama-cloud", "cloud-test-key");
|
|
|
|
using _hook = hookFetch((input, init) => {
|
|
const url = String(input);
|
|
if (url === "http://127.0.0.1:11434/api/tags") {
|
|
return new Response(JSON.stringify({ models: [{ name: "phi4-mini" }] }), {
|
|
status: 200,
|
|
headers: { "Content-Type": "application/json" },
|
|
});
|
|
}
|
|
if (url === "http://127.0.0.1:11434/api/show") {
|
|
return new Response(JSON.stringify({ capabilities: ["completion"] }), {
|
|
status: 200,
|
|
headers: { "Content-Type": "application/json" },
|
|
});
|
|
}
|
|
if (url === "https://ollama.com/api/tags") {
|
|
const headers = new Headers(init?.headers);
|
|
expect(headers.get("Authorization")).toBe("Bearer cloud-test-key");
|
|
return new Response(JSON.stringify({ models: [{ name: "gpt-oss:120b" }] }), {
|
|
status: 200,
|
|
headers: { "Content-Type": "application/json" },
|
|
});
|
|
}
|
|
if (url === "https://ollama.com/api/show") {
|
|
const headers = new Headers(init?.headers);
|
|
expect(headers.get("Authorization")).toBe("Bearer cloud-test-key");
|
|
const body = JSON.parse(String(init?.body ?? "{}")) as { model?: string };
|
|
expect(body.model).toBe("gpt-oss:120b");
|
|
return new Response(
|
|
JSON.stringify({
|
|
capabilities: ["completion", "thinking"],
|
|
model_info: { "gpt-oss.context_length": 262144 },
|
|
}),
|
|
{ status: 200, headers: { "Content-Type": "application/json" } },
|
|
);
|
|
}
|
|
throw new Error(`Unexpected URL: ${url}`);
|
|
});
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
await registry.refresh();
|
|
|
|
const local = registry.find("ollama", "phi4-mini");
|
|
const cloud = registry.find("ollama-cloud", "gpt-oss:120b");
|
|
|
|
expect(local?.provider).toBe("ollama");
|
|
expect(local?.api).toBe("openai-responses");
|
|
expect(cloud?.provider).toBe("ollama-cloud");
|
|
expect(cloud?.api).toBe("ollama-chat");
|
|
expect(cloud?.baseUrl).toBe("https://ollama.com");
|
|
expect(cloud?.reasoning).toBe(true);
|
|
expect(cloud?.contextWindow).toBe(262144);
|
|
expect(await registry.getApiKey(cloud!)).toBe("cloud-test-key");
|
|
expect(registry.getAvailable().some(model => model.provider === "ollama" && model.id === "phi4-mini")).toBe(
|
|
true,
|
|
);
|
|
expect(
|
|
registry.getAvailable().some(model => model.provider === "ollama-cloud" && model.id === "gpt-oss:120b"),
|
|
).toBe(true);
|
|
});
|
|
test("discovers ollama models at runtime and treats auth:none providers as available", async () => {
|
|
writeRawModelsJson({
|
|
ollama: {
|
|
baseUrl: "http://127.0.0.1:11434/v1",
|
|
api: "openai-completions",
|
|
auth: "none",
|
|
discovery: { type: "ollama" },
|
|
},
|
|
});
|
|
|
|
using _hook = hookFetch(input => {
|
|
const url = String(input);
|
|
if (url === "http://127.0.0.1:11434/api/tags") {
|
|
return new Response(
|
|
JSON.stringify({
|
|
models: [{ name: "qwen2.5-coder:7b" }, { model: "llama3.2:3b", name: "llama3.2:3b" }],
|
|
}),
|
|
{ status: 200, headers: { "Content-Type": "application/json" } },
|
|
);
|
|
}
|
|
if (url === "http://127.0.0.1:11434/api/show") {
|
|
return new Response(JSON.stringify({ capabilities: ["completion"] }), {
|
|
status: 200,
|
|
headers: { "Content-Type": "application/json" },
|
|
});
|
|
}
|
|
throw new Error(`Unexpected URL: ${url}`);
|
|
});
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
await registry.refresh();
|
|
|
|
const ollamaModels = getModelsForProvider(registry, "ollama");
|
|
expect(ollamaModels.some(m => m.id === "qwen2.5-coder:7b")).toBe(true);
|
|
expect(ollamaModels.some(m => m.id === "llama3.2:3b")).toBe(true);
|
|
|
|
const available = registry.getAvailable().filter(m => m.provider === "ollama");
|
|
expect(available.length).toBe(2);
|
|
expect(await registry.getApiKey(available[0])).toBe(kNoAuth);
|
|
});
|
|
|
|
test("normalizes cached ollama completions rows to responses on load", () => {
|
|
writeRawModelsJson({
|
|
ollama: {
|
|
baseUrl: "http://127.0.0.1:11434/v1",
|
|
api: "openai-responses",
|
|
auth: "none",
|
|
discovery: { type: "ollama" },
|
|
},
|
|
});
|
|
writeCachedOllamaModels([
|
|
{
|
|
id: "phi4-mini",
|
|
name: "phi4-mini",
|
|
api: "openai-completions",
|
|
provider: "ollama",
|
|
baseUrl: "http://127.0.0.1:11434/v1",
|
|
reasoning: false,
|
|
input: ["text"],
|
|
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
|
contextWindow: 128000,
|
|
maxTokens: 8192,
|
|
},
|
|
]);
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
const ollama = registry.find("ollama", "phi4-mini");
|
|
|
|
expect(ollama?.api).toBe("openai-responses");
|
|
expect(ollama?.baseUrl).toBe("http://127.0.0.1:11434/v1");
|
|
expect(registry.getProviderDiscoveryState("ollama")?.status).toBe("cached");
|
|
});
|
|
|
|
test("discovers ollama thinking capabilities from show metadata", async () => {
|
|
writeRawModelsJson({
|
|
ollama: {
|
|
baseUrl: "http://127.0.0.1:11434/v1",
|
|
api: "openai-completions",
|
|
auth: "none",
|
|
discovery: { type: "ollama" },
|
|
},
|
|
});
|
|
|
|
using _hook = hookFetch((input, init) => {
|
|
const url = String(input);
|
|
if (url === "http://127.0.0.1:11434/api/tags") {
|
|
return new Response(
|
|
JSON.stringify({
|
|
models: [{ name: "qwen3.5:397b-cloud" }, { name: "llama3.2:3b" }],
|
|
}),
|
|
{ status: 200, headers: { "Content-Type": "application/json" } },
|
|
);
|
|
}
|
|
if (url === "http://127.0.0.1:11434/api/show") {
|
|
const body = JSON.parse(String(init?.body ?? "{}")) as { model?: string };
|
|
if (body.model === "qwen3.5:397b-cloud") {
|
|
return new Response(JSON.stringify({ capabilities: ["completion", "thinking"] }), {
|
|
status: 200,
|
|
headers: { "Content-Type": "application/json" },
|
|
});
|
|
}
|
|
if (body.model === "llama3.2:3b") {
|
|
return new Response(JSON.stringify({ capabilities: ["completion"] }), {
|
|
status: 200,
|
|
headers: { "Content-Type": "application/json" },
|
|
});
|
|
}
|
|
}
|
|
throw new Error(`Unexpected request: ${url}`);
|
|
});
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
await registry.refresh();
|
|
|
|
const qwen = registry.find("ollama", "qwen3.5:397b-cloud");
|
|
expect(qwen?.reasoning).toBe(true);
|
|
expect(qwen?.thinking).toEqual({
|
|
mode: "effort",
|
|
minLevel: Effort.Minimal,
|
|
maxLevel: Effort.High,
|
|
});
|
|
|
|
const llama = registry.find("ollama", "llama3.2:3b");
|
|
expect(llama?.reasoning).toBe(false);
|
|
});
|
|
|
|
test("discovers ollama context window from show model_info", async () => {
|
|
using _hook = hookFetch((input, init) => {
|
|
const url = String(input);
|
|
if (url === "http://127.0.0.1:11434/api/tags") {
|
|
return new Response(JSON.stringify({ models: [{ name: "gemma3:4b" }] }), {
|
|
status: 200,
|
|
headers: { "Content-Type": "application/json" },
|
|
});
|
|
}
|
|
if (url === "http://127.0.0.1:11434/api/show") {
|
|
const body = JSON.parse(String(init?.body ?? "{}")) as { model?: string };
|
|
if (body.model === "gemma3:4b") {
|
|
return new Response(
|
|
JSON.stringify({
|
|
model_info: {
|
|
"gemma3.context_length": 131072,
|
|
},
|
|
}),
|
|
{
|
|
status: 200,
|
|
headers: { "Content-Type": "application/json" },
|
|
},
|
|
);
|
|
}
|
|
}
|
|
throw new Error(`Unexpected request: ${url}`);
|
|
});
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
await registry.refresh();
|
|
|
|
const gemma = registry.find("ollama", "gemma3:4b");
|
|
expect(gemma?.contextWindow).toBe(131072);
|
|
expect(gemma?.maxTokens).toBe(8192);
|
|
expect(gemma?.input).toEqual(["text"]);
|
|
expect(gemma?.reasoning).toBe(false);
|
|
});
|
|
|
|
test("discovery failure does not fail model registry refresh", async () => {
|
|
writeRawModelsJson({
|
|
ollama: {
|
|
baseUrl: "http://127.0.0.1:11434",
|
|
api: "openai-completions",
|
|
auth: "none",
|
|
discovery: { type: "ollama" },
|
|
},
|
|
});
|
|
|
|
using _hook = hookFetch(() => {
|
|
throw new Error("connection refused");
|
|
});
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
await registry.refresh();
|
|
expect(getModelsForProvider(registry, "ollama")).toHaveLength(0);
|
|
expect(registry.getError()).toBeUndefined();
|
|
});
|
|
test("loads cached local models before live refresh and preserves them on failure", async () => {
|
|
writeRawModelsJson({
|
|
ollama: {
|
|
baseUrl: "http://127.0.0.1:11434/v1",
|
|
api: "openai-completions",
|
|
auth: "none",
|
|
discovery: { type: "ollama" },
|
|
},
|
|
});
|
|
|
|
{
|
|
using _hook = mockOllamaDiscovery(["phi4-mini"]);
|
|
const primedRegistry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
await primedRegistry.refresh();
|
|
}
|
|
|
|
const cachedRegistry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
expect(getModelsForProvider(cachedRegistry, "ollama").some(model => model.id === "phi4-mini")).toBe(true);
|
|
expect(cachedRegistry.getProviderDiscoveryState("ollama")?.status).toBe("cached");
|
|
|
|
{
|
|
using _hook = hookFetch(() => {
|
|
throw new Error("connection refused");
|
|
});
|
|
await cachedRegistry.refreshProvider("ollama");
|
|
}
|
|
|
|
expect(getModelsForProvider(cachedRegistry, "ollama").some(model => model.id === "phi4-mini")).toBe(true);
|
|
const state = cachedRegistry.getProviderDiscoveryState("ollama");
|
|
expect(state?.status).toBe("cached");
|
|
expect(state?.error).toContain("connection refused");
|
|
});
|
|
|
|
test("reports unauthenticated discoverable providers without discarding cached models", async () => {
|
|
writeRawModelsJson({
|
|
"custom-local": {
|
|
baseUrl: "http://127.0.0.1:11434/v1",
|
|
api: "openai-completions",
|
|
discovery: { type: "ollama" },
|
|
},
|
|
});
|
|
authStorage.setRuntimeApiKey("custom-local", "test-key");
|
|
|
|
{
|
|
using _hook = hookFetch(input => {
|
|
const url = String(input);
|
|
if (url === "http://127.0.0.1:11434/api/tags") {
|
|
return new Response(JSON.stringify({ models: [{ name: "local-coder" }] }), {
|
|
status: 200,
|
|
headers: { "Content-Type": "application/json" },
|
|
});
|
|
}
|
|
if (url === "http://127.0.0.1:11434/api/show") {
|
|
return new Response(JSON.stringify({ capabilities: ["completion"] }), {
|
|
status: 200,
|
|
headers: { "Content-Type": "application/json" },
|
|
});
|
|
}
|
|
throw new Error(`Unexpected URL: ${url}`);
|
|
});
|
|
const primedRegistry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
await primedRegistry.refreshProvider("custom-local");
|
|
}
|
|
|
|
authStorage.setRuntimeApiKey("custom-local", "");
|
|
const cachedRegistry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
await cachedRegistry.refreshProvider("custom-local");
|
|
|
|
expect(getModelsForProvider(cachedRegistry, "custom-local").some(model => model.id === "local-coder")).toBe(
|
|
true,
|
|
);
|
|
const state = cachedRegistry.getProviderDiscoveryState("custom-local");
|
|
expect(state?.status).toBe("unauthenticated");
|
|
expect(state?.models).toContain("local-coder");
|
|
});
|
|
test("llama.cpp discovery honors configured API key", async () => {
|
|
authStorage.setRuntimeApiKey("llama.cpp", "test-llama-key");
|
|
using _hook = hookFetch((input, init) => {
|
|
const url = String(input);
|
|
if (url === "http://127.0.0.1:8080/models") {
|
|
const headers = init?.headers as Headers | Record<string, string> | undefined;
|
|
let authHeader: string | null = null;
|
|
if (headers instanceof Headers) {
|
|
authHeader = headers.get("Authorization");
|
|
} else if (typeof headers === "object") {
|
|
authHeader = headers.Authorization;
|
|
}
|
|
expect(String(authHeader ?? "")).toBe("Bearer test-llama-key");
|
|
return new Response(JSON.stringify({ data: [{ id: "llama-3.2:3b" }, { id: "mistral:7b" }] }), {
|
|
status: 200,
|
|
headers: { "Content-Type": "application/json" },
|
|
});
|
|
}
|
|
if (url === "http://127.0.0.1:8080/props") {
|
|
const headers = init?.headers as Headers | Record<string, string> | undefined;
|
|
let authHeader: string | null = null;
|
|
if (headers instanceof Headers) {
|
|
authHeader = headers.get("Authorization");
|
|
} else if (typeof headers === "object") {
|
|
authHeader = headers.Authorization;
|
|
}
|
|
expect(String(authHeader ?? "")).toBe("Bearer test-llama-key");
|
|
return new Response(JSON.stringify({ default_generation_settings: { n_ctx: 262144 } }), {
|
|
status: 200,
|
|
headers: { "Content-Type": "application/json" },
|
|
});
|
|
}
|
|
throw new Error(`Unexpected URL: ${url}`);
|
|
});
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
await registry.refresh();
|
|
const llamaModels = getModelsForProvider(registry, "llama.cpp");
|
|
expect(llamaModels.some(m => m.id === "llama-3.2:3b")).toBe(true);
|
|
const apiKey = await registry.getApiKey(llamaModels[0]);
|
|
expect(apiKey).toBe("test-llama-key");
|
|
expect(apiKey).not.toBe(kNoAuth);
|
|
});
|
|
test("llama.cpp discovery without API key is treated as keyless", async () => {
|
|
using _hook = hookFetch((input, init) => {
|
|
const url = String(input);
|
|
if (url === "http://127.0.0.1:8080/models") {
|
|
const headers = init?.headers as Headers | Record<string, string> | undefined;
|
|
let authHeader: string | null = null;
|
|
if (headers instanceof Headers) {
|
|
authHeader = headers.get("Authorization");
|
|
} else if (typeof headers === "object") {
|
|
authHeader = headers.Authorization;
|
|
}
|
|
// When no API key, headers should be empty object or undefined
|
|
expect(authHeader).toBeUndefined();
|
|
return new Response(JSON.stringify({ data: [{ id: "llama-3.2:3b" }] }), {
|
|
status: 200,
|
|
headers: { "Content-Type": "application/json" },
|
|
});
|
|
}
|
|
if (url === "http://127.0.0.1:8080/props") {
|
|
const headers = init?.headers as Headers | Record<string, string> | undefined;
|
|
let authHeader: string | null = null;
|
|
if (headers instanceof Headers) {
|
|
authHeader = headers.get("Authorization");
|
|
} else if (typeof headers === "object") {
|
|
authHeader = headers.Authorization;
|
|
}
|
|
expect(authHeader).toBeUndefined();
|
|
return new Response(JSON.stringify({ default_generation_settings: { n_ctx: 262144 } }), {
|
|
status: 200,
|
|
headers: { "Content-Type": "application/json" },
|
|
});
|
|
}
|
|
throw new Error(`Unexpected URL: ${url}`);
|
|
});
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
await registry.refresh();
|
|
const state = registry.getProviderDiscoveryState("llama.cpp");
|
|
if (state?.status !== "ok") {
|
|
throw new Error(`Discovery failed with status ${state?.status}: ${state?.error}`);
|
|
}
|
|
const llamaModels = getModelsForProvider(registry, "llama.cpp");
|
|
const apiKey = await registry.getApiKey(llamaModels[0]);
|
|
expect(apiKey).toBe(kNoAuth);
|
|
});
|
|
test("llama.cpp discovery reads context window from props n_ctx", async () => {
|
|
using _hook = hookFetch(input => {
|
|
const url = String(input);
|
|
if (url === "http://127.0.0.1:8080/models") {
|
|
return new Response(JSON.stringify({ data: [{ id: "qwen35-35b-a3b" }] }), {
|
|
status: 200,
|
|
headers: { "Content-Type": "application/json" },
|
|
});
|
|
}
|
|
if (url === "http://127.0.0.1:8080/props") {
|
|
return new Response(
|
|
JSON.stringify({
|
|
default_generation_settings: {
|
|
n_ctx: 262144,
|
|
},
|
|
modalities: {
|
|
vision: true,
|
|
audio: false,
|
|
},
|
|
}),
|
|
{
|
|
status: 200,
|
|
headers: { "Content-Type": "application/json" },
|
|
},
|
|
);
|
|
}
|
|
throw new Error(`Unexpected URL: ${url}`);
|
|
});
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
await registry.refresh();
|
|
const llama = registry.find("llama.cpp", "qwen35-35b-a3b");
|
|
expect(llama?.contextWindow).toBe(262144);
|
|
expect(llama?.maxTokens).toBe(8192);
|
|
expect(llama?.input).toEqual(["text", "image"]);
|
|
});
|
|
});
|
|
describe("bundled Anthropic catalog availability", () => {
|
|
test("includes native Opus 4.7 in available models when Anthropic auth exists", async () => {
|
|
await authStorage.set("anthropic", [{ type: "api_key", key: "sk-ant-api-test" }]);
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
await registry.refresh("offline");
|
|
|
|
expect(
|
|
registry.getAvailable().some(model => model.provider === "anthropic" && model.id === "claude-opus-4-7"),
|
|
).toBe(true);
|
|
});
|
|
});
|
|
describe("disableStrictTools", () => {
|
|
test("custom provider with models gets disableStrictTools merged into compat", () => {
|
|
writeRawModelsJson({
|
|
"bedrock-anthropic": {
|
|
baseUrl: "https://bedrock-runtime.us-east-1.amazonaws.com/anthropic",
|
|
apiKey: "TEST_KEY",
|
|
api: "anthropic-messages",
|
|
disableStrictTools: true,
|
|
models: [
|
|
{
|
|
id: "claude-sonnet-4-20250514",
|
|
name: "Claude Sonnet 4",
|
|
reasoning: false,
|
|
input: ["text", "image"],
|
|
cost: { input: 3.0, output: 15.0, cacheRead: 0.3, cacheWrite: 3.75 },
|
|
contextWindow: 200000,
|
|
maxTokens: 16384,
|
|
},
|
|
],
|
|
},
|
|
});
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
const model = registry.find("bedrock-anthropic", "claude-sonnet-4-20250514");
|
|
|
|
expect(model).toBeDefined();
|
|
expect((model?.compat as { disableStrictTools?: boolean } | undefined)?.disableStrictTools).toBe(true);
|
|
});
|
|
|
|
test("disableStrictTools on override-only provider applies to built-in models", () => {
|
|
writeRawModelsJson({ anthropic: { disableStrictTools: true } });
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
const models = getModelsForProvider(registry, "anthropic");
|
|
|
|
expect(models.length).toBeGreaterThan(0);
|
|
for (const model of models) {
|
|
expect((model.compat as { disableStrictTools?: boolean } | undefined)?.disableStrictTools).toBe(true);
|
|
}
|
|
});
|
|
|
|
test("disableStrictTools is absent on built-in models without override", () => {
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
const models = getModelsForProvider(registry, "anthropic");
|
|
|
|
expect(models.length).toBeGreaterThan(0);
|
|
for (const model of models) {
|
|
expect((model.compat as { disableStrictTools?: boolean } | undefined)?.disableStrictTools).toBeUndefined();
|
|
}
|
|
});
|
|
|
|
test("disableStrictTools is merged with explicit compat on custom provider", () => {
|
|
writeRawModelsJson({
|
|
"my-proxy": {
|
|
baseUrl: "https://proxy.example.com/anthropic",
|
|
apiKey: "TEST_KEY",
|
|
api: "anthropic-messages",
|
|
disableStrictTools: true,
|
|
models: [
|
|
{
|
|
id: "claude-sonnet-4",
|
|
name: "Sonnet",
|
|
reasoning: false,
|
|
input: ["text"],
|
|
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
|
contextWindow: 200000,
|
|
maxTokens: 16384,
|
|
},
|
|
],
|
|
},
|
|
});
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
const model = registry.find("my-proxy", "claude-sonnet-4");
|
|
|
|
expect(model).toBeDefined();
|
|
expect((model?.compat as { disableStrictTools?: boolean } | undefined)?.disableStrictTools).toBe(true);
|
|
});
|
|
});
|
|
|
|
describe("provider auth: oauth", () => {
|
|
test("models from a provider with auth: oauth are marked isOAuth=true", async () => {
|
|
writeRawModelsJson({
|
|
"proxy-anthropic": {
|
|
baseUrl: "https://proxy.example.com",
|
|
apiKey: "literal-key",
|
|
api: "anthropic-messages",
|
|
auth: "oauth",
|
|
models: [
|
|
{
|
|
id: "claude-sonnet-4-5",
|
|
name: "Claude Sonnet 4.5",
|
|
reasoning: true,
|
|
input: ["text"],
|
|
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
|
contextWindow: 200000,
|
|
maxTokens: 8000,
|
|
},
|
|
],
|
|
},
|
|
});
|
|
await authStorage.setRuntimeApiKey("proxy-anthropic", "literal-key");
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
await registry.refresh("offline");
|
|
|
|
const model = registry.find("proxy-anthropic", "claude-sonnet-4-5");
|
|
expect(model).toBeDefined();
|
|
expect(model?.isOAuth).toBe(true);
|
|
});
|
|
|
|
test("anthropic-messages providers default to isOAuth=true even without explicit auth", async () => {
|
|
writeRawModelsJson({
|
|
"proxy-anthropic": {
|
|
baseUrl: "https://proxy.example.com",
|
|
apiKey: "literal-key",
|
|
api: "anthropic-messages",
|
|
models: [
|
|
{
|
|
id: "claude-sonnet-4-5",
|
|
name: "Claude Sonnet 4.5",
|
|
reasoning: true,
|
|
input: ["text"],
|
|
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
|
contextWindow: 200000,
|
|
maxTokens: 8000,
|
|
},
|
|
],
|
|
},
|
|
});
|
|
await authStorage.setRuntimeApiKey("proxy-anthropic", "literal-key");
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
await registry.refresh("offline");
|
|
|
|
const model = registry.find("proxy-anthropic", "claude-sonnet-4-5");
|
|
expect(model).toBeDefined();
|
|
expect(model?.isOAuth).toBe(true);
|
|
});
|
|
|
|
test("auth: apiKey opts out of the anthropic-messages default", async () => {
|
|
writeRawModelsJson({
|
|
"proxy-anthropic": {
|
|
baseUrl: "https://proxy.example.com",
|
|
apiKey: "literal-key",
|
|
api: "anthropic-messages",
|
|
auth: "apiKey",
|
|
models: [
|
|
{
|
|
id: "claude-sonnet-4-5",
|
|
name: "Claude Sonnet 4.5",
|
|
reasoning: true,
|
|
input: ["text"],
|
|
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
|
contextWindow: 200000,
|
|
maxTokens: 8000,
|
|
},
|
|
],
|
|
},
|
|
});
|
|
await authStorage.setRuntimeApiKey("proxy-anthropic", "literal-key");
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
await registry.refresh("offline");
|
|
|
|
const model = registry.find("proxy-anthropic", "claude-sonnet-4-5");
|
|
expect(model).toBeDefined();
|
|
expect(model?.isOAuth).toBeUndefined();
|
|
});
|
|
|
|
test("non-anthropic apis do not get the OAuth default", async () => {
|
|
writeRawModelsJson({
|
|
"proxy-openai": {
|
|
baseUrl: "https://proxy.example.com/v1",
|
|
apiKey: "literal-key",
|
|
api: "openai-completions",
|
|
models: [
|
|
{
|
|
id: "gpt-5",
|
|
name: "GPT-5",
|
|
reasoning: true,
|
|
input: ["text"],
|
|
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
|
contextWindow: 200000,
|
|
maxTokens: 8000,
|
|
},
|
|
],
|
|
},
|
|
});
|
|
await authStorage.setRuntimeApiKey("proxy-openai", "literal-key");
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
await registry.refresh("offline");
|
|
|
|
const model = registry.find("proxy-openai", "gpt-5");
|
|
expect(model).toBeDefined();
|
|
expect(model?.isOAuth).toBeUndefined();
|
|
});
|
|
});
|
|
|
|
test("cached discovery with UNK contextWindow preserves bundled value", () => {
|
|
// Configure openai as a discoverable provider through models.json
|
|
writeRawModelsJson({
|
|
openai: {
|
|
baseUrl: "https://my-proxy.example.com/v1",
|
|
apiKey: "TEST_KEY",
|
|
api: "openai-completions",
|
|
discovery: { type: "openai-models-list" },
|
|
models: [],
|
|
},
|
|
});
|
|
// Pre-populate the cache with a model that has UNK sentinel values
|
|
// (simulating a discovery that didn't return limit.context)
|
|
writeModelCache<"openai-completions">(
|
|
"openai",
|
|
Date.now(),
|
|
[
|
|
{
|
|
id: "gpt-4o",
|
|
name: "GPT-4o",
|
|
api: "openai-completions",
|
|
provider: "openai",
|
|
baseUrl: "https://my-proxy.example.com/v1",
|
|
reasoning: false,
|
|
input: ["text"],
|
|
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
|
contextWindow: 222_222, // UNK_CONTEXT_WINDOW
|
|
maxTokens: 8_888, // UNK_MAX_TOKENS
|
|
},
|
|
],
|
|
true,
|
|
cacheDbPath,
|
|
);
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
const model = registry.find("openai", "gpt-4o");
|
|
|
|
expect(model).toBeDefined();
|
|
// The bundled gpt-4o has a correct contextWindow, not the UNK sentinel
|
|
expect(model!.contextWindow).not.toBe(222_222);
|
|
expect(model!.contextWindow).toBeGreaterThan(100_000);
|
|
expect(model!.maxTokens).not.toBe(8_888);
|
|
expect(model!.maxTokens).toBeGreaterThan(1000);
|
|
});
|
|
|
|
test("loads cached standard provider discovery models on startup", () => {
|
|
const cachedModel: Model<"ollama-chat"> = {
|
|
id: "deepseek-v4-pro",
|
|
name: "DeepSeek V4 Pro",
|
|
api: "ollama-chat",
|
|
provider: "ollama-cloud",
|
|
baseUrl: "https://ollama.com",
|
|
reasoning: true,
|
|
input: ["text"],
|
|
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
|
contextWindow: 1_000_000,
|
|
maxTokens: 384_000,
|
|
};
|
|
writeModelCache("ollama-cloud", Date.now(), [cachedModel], true, "", cacheDbPath);
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
|
|
expect(registry.find("ollama-cloud", "deepseek-v4-pro")?.maxTokens).toBe(384_000);
|
|
});
|
|
|
|
test("replaces bundled google-vertex models with authoritative Vertex project discovery", () => {
|
|
const cachedModel: Model<"openai-completions"> = {
|
|
id: "zai-org/glm-4.7-maas",
|
|
name: "GLM-4.7",
|
|
api: "openai-completions",
|
|
provider: "google-vertex",
|
|
baseUrl: "https://aiplatform.googleapis.com/v1/projects/vertex-project/locations/global/endpoints/openapi",
|
|
reasoning: true,
|
|
input: ["text"],
|
|
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
|
contextWindow: 222_222,
|
|
maxTokens: 8_888,
|
|
};
|
|
writeModelCache("google-vertex", Date.now(), [cachedModel], true, "", cacheDbPath);
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
const vertexModels = getModelsForProvider(registry, "google-vertex");
|
|
|
|
expect(vertexModels.map(model => model.id)).toEqual(["zai-org/glm-4.7-maas"]);
|
|
expect(registry.find("google-vertex", "gemini-1.5-pro")).toBeUndefined();
|
|
});
|
|
|
|
test("does not re-add bundled synthetic models after authoritative cache load", () => {
|
|
const cachedModel: Model<"openai-completions"> = {
|
|
id: "hf:zai-org/GLM-5.1",
|
|
name: "GLM 5.1",
|
|
api: "openai-completions",
|
|
provider: "synthetic",
|
|
baseUrl: "https://api.synthetic.new/openai/v1",
|
|
reasoning: true,
|
|
input: ["text"],
|
|
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
|
contextWindow: 128_000,
|
|
maxTokens: 8_192,
|
|
};
|
|
writeModelCache("synthetic", Date.now(), [cachedModel], true, "authoritative:test", cacheDbPath);
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
const syntheticModels = getModelsForProvider(registry, "synthetic");
|
|
|
|
expect(syntheticModels.map(model => model.id)).toEqual(["hf:zai-org/GLM-5.1"]);
|
|
expect(registry.find("synthetic", "hf:moonshotai/Kimi-K2.5")).toBeUndefined();
|
|
});
|
|
|
|
test("does not re-add bundled synthetic models after authoritative refresh", async () => {
|
|
authStorage.setRuntimeApiKey("synthetic", "synthetic-test-key");
|
|
using _hook = mockOpenAiCompatibleModels("https://api.synthetic.new/openai/v1/models", ["hf:zai-org/GLM-5.1"]);
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
|
|
await registry.refresh("online");
|
|
const syntheticModels = getModelsForProvider(registry, "synthetic");
|
|
|
|
expect(syntheticModels.map(model => model.id)).toEqual(["hf:zai-org/GLM-5.1"]);
|
|
expect(registry.find("synthetic", "hf:moonshotai/Kimi-K2.5")).toBeUndefined();
|
|
});
|
|
|
|
test("keeps bundled google-vertex fallback when cached project catalog is non-authoritative", () => {
|
|
const cachedModel: Model<"openai-completions"> = {
|
|
id: "zai-org/glm-4.7-maas",
|
|
name: "GLM-4.7",
|
|
api: "openai-completions",
|
|
provider: "google-vertex",
|
|
baseUrl: "https://aiplatform.googleapis.com/v1/projects/vertex-project/locations/global/endpoints/openapi",
|
|
reasoning: true,
|
|
input: ["text"],
|
|
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
|
contextWindow: 222_222,
|
|
maxTokens: 8_888,
|
|
};
|
|
writeModelCache("google-vertex", Date.now(), [cachedModel], false, "", cacheDbPath);
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
const vertexModels = getModelsForProvider(registry, "google-vertex");
|
|
|
|
expect(vertexModels.some(model => model.id === "zai-org/glm-4.7-maas")).toBe(true);
|
|
expect(vertexModels.some(model => model.id.startsWith("gemini-"))).toBe(true);
|
|
});
|
|
|
|
test("keeps bundled google-vertex fallback when cached project catalog is stale", () => {
|
|
const cachedModel: Model<"openai-completions"> = {
|
|
id: "zai-org/glm-4.7-maas",
|
|
name: "GLM-4.7",
|
|
api: "openai-completions",
|
|
provider: "google-vertex",
|
|
baseUrl: "https://aiplatform.googleapis.com/v1/projects/vertex-project/locations/global/endpoints/openapi",
|
|
reasoning: true,
|
|
input: ["text"],
|
|
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
|
contextWindow: 222_222,
|
|
maxTokens: 8_888,
|
|
};
|
|
// 25h old > 24h TTL → cache.fresh === false even though authoritative === true.
|
|
const staleTimestamp = Date.now() - 25 * 60 * 60 * 1000;
|
|
writeModelCache("google-vertex", staleTimestamp, [cachedModel], true, "", cacheDbPath);
|
|
|
|
const registry = new ModelRegistry(authStorage, modelsJsonPath);
|
|
const vertexModels = getModelsForProvider(registry, "google-vertex");
|
|
|
|
expect(vertexModels.some(model => model.id === "zai-org/glm-4.7-maas")).toBe(true);
|
|
expect(vertexModels.some(model => model.id.startsWith("gemini-"))).toBe(true);
|
|
});
|
|
});
|