Files
oh-my-pi/packages/coding-agent/test/model-registry.test.ts
T
can1357 6645a7a059 fix(model-registry): filtered disabled providers from model listing
- Applied `disabledProviders` setting to `getAvailable()` and `getDiscoverableProviders()` so `--list-models` and `/model` hide models from disabled providers.
- Initialized settings before model refresh in the `--list-models` code path.
- Added tests verifying disabled providers are excluded from both methods.
Fixes #588
2026-04-01 04:52:03 +02:00

1463 lines
48 KiB
TypeScript

import { afterEach, beforeEach, describe, expect, test } from "bun:test";
import * as fs from "node:fs";
import * as os from "node:os";
import * as path from "node:path";
import { Effort, type OpenAICompat, type ThinkingConfig } from "@oh-my-pi/pi-ai";
import { kNoAuth, MODEL_ROLES, ModelRegistry } from "@oh-my-pi/pi-coding-agent/config/model-registry";
import { _resetSettingsForTest, Settings } from "@oh-my-pi/pi-coding-agent/config/settings";
import { AuthStorage } from "@oh-my-pi/pi-coding-agent/session/auth-storage";
import { hookFetch, Snowflake } from "@oh-my-pi/pi-utils";
describe("ModelRegistry", () => {
let tempDir: string;
let modelsJsonPath: string;
let authStorage: AuthStorage;
test("commit role includes a visible badge tag", () => {
expect(MODEL_ROLES.commit.tag).toBe("COMMIT");
expect(MODEL_ROLES.commit.color).toBe("dim");
});
beforeEach(async () => {
_resetSettingsForTest();
tempDir = path.join(os.tmpdir(), `pi-test-model-registry-${Snowflake.next()}`);
fs.mkdirSync(tempDir, { recursive: true });
modelsJsonPath = path.join(tempDir, "models.json");
authStorage = await AuthStorage.create(path.join(tempDir, "testauth.db"));
});
afterEach(() => {
_resetSettingsForTest();
authStorage.close();
if (tempDir && fs.existsSync(tempDir)) {
fs.rmSync(tempDir, { recursive: true });
}
});
type ProviderConfig = {
baseUrl: string;
apiKey: string;
api: string;
models: Array<{
id: string;
name: string;
reasoning: boolean;
thinking?: ThinkingConfig;
input: string[];
cost: { input: number; output: number; cacheRead: number; cacheWrite: number };
contextWindow: number;
maxTokens: number;
}>;
};
/** Create minimal provider config */
function providerConfig(
baseUrl: string,
models: Array<{
id: string;
name?: string;
reasoning?: boolean;
thinking?: ThinkingConfig;
contextWindow?: number;
}>,
api: string = "anthropic-messages",
) {
return {
baseUrl,
apiKey: "TEST_KEY",
api,
models: models.map(m => ({
id: m.id,
name: m.name ?? m.id,
reasoning: m.reasoning ?? false,
thinking: m.thinking,
input: ["text"],
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
contextWindow: m.contextWindow ?? 100000,
maxTokens: 8000,
})),
};
}
function writeModelsJson(providers: Record<string, ProviderConfig>) {
fs.writeFileSync(modelsJsonPath, JSON.stringify({ providers }));
}
function getModelsForProvider(registry: ModelRegistry, provider: string) {
return registry.getAll().filter(m => m.provider === provider);
}
/** Create a baseUrl-only override (no custom models) */
function overrideConfig(baseUrl: string, headers?: Record<string, string>) {
return { baseUrl, ...(headers && { headers }) };
}
/** Write raw providers config (for mixed override/replacement scenarios) */
function writeRawModelsJson(providers: Record<string, unknown>) {
fs.writeFileSync(modelsJsonPath, JSON.stringify({ providers }));
}
function mockOpenAiCompatibleModels(url: string, modelIds: string[]) {
return hookFetch(input => {
const requestUrl = String(input);
if (requestUrl === url) {
return new Response(JSON.stringify({ data: modelIds.map(id => ({ id })) }), {
status: 200,
headers: { "Content-Type": "application/json" },
});
}
throw new Error(`Unexpected URL: ${requestUrl}`);
});
}
describe("baseUrl override (no custom models)", () => {
test("overriding baseUrl keeps all built-in models", () => {
writeRawModelsJson({
anthropic: overrideConfig("https://my-proxy.example.com/v1"),
});
const registry = new ModelRegistry(authStorage, modelsJsonPath);
const anthropicModels = getModelsForProvider(registry, "anthropic");
// Should have multiple built-in models, not just one
expect(anthropicModels.length).toBeGreaterThan(1);
expect(anthropicModels.some(m => m.id.includes("claude"))).toBe(true);
});
test("overriding baseUrl changes URL on all built-in models", () => {
writeRawModelsJson({
anthropic: overrideConfig("https://my-proxy.example.com/v1"),
});
const registry = new ModelRegistry(authStorage, modelsJsonPath);
const anthropicModels = getModelsForProvider(registry, "anthropic");
// All models should have the new baseUrl
for (const model of anthropicModels) {
expect(model.baseUrl).toBe("https://my-proxy.example.com/v1");
}
});
test("overriding headers merges with model headers", () => {
writeRawModelsJson({
anthropic: overrideConfig("https://my-proxy.example.com/v1", {
"X-Custom-Header": "custom-value",
}),
});
const registry = new ModelRegistry(authStorage, modelsJsonPath);
const anthropicModels = getModelsForProvider(registry, "anthropic");
for (const model of anthropicModels) {
expect(model.headers?.["X-Custom-Header"]).toBe("custom-value");
}
});
test("baseUrl-only override does not affect other providers", () => {
writeRawModelsJson({
anthropic: overrideConfig("https://my-proxy.example.com/v1"),
});
const registry = new ModelRegistry(authStorage, modelsJsonPath);
const googleModels = getModelsForProvider(registry, "google");
// Google models should still have their original baseUrl
expect(googleModels.length).toBeGreaterThan(0);
expect(googleModels[0].baseUrl).not.toBe("https://my-proxy.example.com/v1");
});
test("can mix baseUrl override and models merge", () => {
writeRawModelsJson({
// baseUrl-only for anthropic
anthropic: overrideConfig("https://anthropic-proxy.example.com/v1"),
// Add custom model for google (merged with built-ins)
google: providerConfig(
"https://google-proxy.example.com/v1",
[{ id: "gemini-custom" }],
"google-generative-ai",
),
});
const registry = new ModelRegistry(authStorage, modelsJsonPath);
// Anthropic: multiple built-in models with new baseUrl
const anthropicModels = getModelsForProvider(registry, "anthropic");
expect(anthropicModels.length).toBeGreaterThan(1);
expect(anthropicModels[0].baseUrl).toBe("https://anthropic-proxy.example.com/v1");
// Google: built-ins plus custom model
const googleModels = getModelsForProvider(registry, "google");
expect(googleModels.length).toBeGreaterThan(1);
expect(googleModels.some(m => m.id === "gemini-custom")).toBe(true);
});
test("refresh() picks up baseUrl override changes", async () => {
writeRawModelsJson({
anthropic: overrideConfig("https://first-proxy.example.com/v1"),
});
const registry = new ModelRegistry(authStorage, modelsJsonPath);
expect(getModelsForProvider(registry, "anthropic")[0].baseUrl).toBe("https://first-proxy.example.com/v1");
// Update and refresh
writeRawModelsJson({
anthropic: overrideConfig("https://second-proxy.example.com/v1"),
});
await registry.refresh("offline");
expect(getModelsForProvider(registry, "anthropic")[0].baseUrl).toBe("https://second-proxy.example.com/v1");
});
});
describe("provider compat overrides", () => {
test("provider-level compat applies to built-in models", () => {
writeRawModelsJson({
openrouter: {
compat: {
supportsUsageInStreaming: false,
supportsStrictMode: false,
},
},
});
const registry = new ModelRegistry(authStorage, modelsJsonPath);
const models = getModelsForProvider(registry, "openrouter");
expect(models.length).toBeGreaterThan(0);
for (const model of models) {
expect(model.compat?.supportsUsageInStreaming).toBe(false);
expect(model.compat?.supportsStrictMode).toBe(false);
}
});
test("provider-level compat applies to custom models", () => {
writeRawModelsJson({
demo: {
baseUrl: "https://example.com/v1",
apiKey: "DEMO_KEY",
api: "openai-completions",
compat: {
supportsUsageInStreaming: false,
maxTokensField: "max_tokens",
},
models: [
{
id: "demo-model",
reasoning: false,
input: ["text"],
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
contextWindow: 1000,
maxTokens: 100,
},
],
},
});
const registry = new ModelRegistry(authStorage, modelsJsonPath);
const model = registry.find("demo", "demo-model");
expect(model?.compat?.supportsUsageInStreaming).toBe(false);
expect(model?.compat?.maxTokensField).toBe("max_tokens");
});
test("model-level compat overrides provider-level compat for custom models", () => {
writeRawModelsJson({
demo: {
baseUrl: "https://example.com/v1",
apiKey: "DEMO_KEY",
api: "openai-completions",
compat: {
supportsUsageInStreaming: false,
maxTokensField: "max_tokens",
},
models: [
{
id: "demo-model",
reasoning: false,
input: ["text"],
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
contextWindow: 1000,
maxTokens: 100,
compat: {
supportsUsageInStreaming: true,
maxTokensField: "max_completion_tokens",
},
},
],
},
});
const registry = new ModelRegistry(authStorage, modelsJsonPath);
const model = registry.find("demo", "demo-model");
expect(model?.compat?.supportsUsageInStreaming).toBe(true);
expect(model?.compat?.maxTokensField).toBe("max_completion_tokens");
});
});
describe("custom models merge behavior", () => {
test("custom provider with same name as built-in merges with built-in models", () => {
writeModelsJson({
anthropic: providerConfig("https://my-proxy.example.com/v1", [{ id: "claude-custom" }]),
});
const registry = new ModelRegistry(authStorage, modelsJsonPath);
const anthropicModels = getModelsForProvider(registry, "anthropic");
// Built-in models still present, custom model merged in
expect(anthropicModels.length).toBeGreaterThan(1);
const custom = anthropicModels.find(m => m.id === "claude-custom");
expect(custom).toBeDefined();
expect(custom!.baseUrl).toBe("https://my-proxy.example.com/v1");
});
test("custom model with same id replaces built-in model by id", () => {
writeModelsJson({
openrouter: providerConfig(
"https://my-proxy.example.com/v1",
[{ id: "anthropic/claude-sonnet-4" }],
"openai-completions",
),
});
const registry = new ModelRegistry(authStorage, modelsJsonPath);
const models = getModelsForProvider(registry, "openrouter");
const sonnetModels = models.filter(m => m.id === "anthropic/claude-sonnet-4");
expect(sonnetModels).toHaveLength(1);
expect(sonnetModels[0].baseUrl).toBe("https://my-proxy.example.com/v1");
});
test("custom same-id replacement does not keep bundled headers", () => {
writeRawModelsJson({
"github-copilot": {
baseUrl: "https://proxy.example.com/v1",
headers: { "X-Proxy": "proxy" },
apiKey: "TEST_KEY",
api: "openai-completions",
models: [{ id: "gpt-4o" }],
},
});
const registry = new ModelRegistry(authStorage, modelsJsonPath);
const model = registry.find("github-copilot", "gpt-4o");
expect(model?.headers).toEqual({ "X-Proxy": "proxy" });
expect(model?.headers?.["User-Agent"]).toBeUndefined();
expect(model?.headers?.["Editor-Version"]).toBeUndefined();
});
test("custom provider with same name as built-in does not affect other built-in providers", () => {
writeModelsJson({
anthropic: providerConfig("https://my-proxy.example.com/v1", [{ id: "claude-custom" }]),
});
const registry = new ModelRegistry(authStorage, modelsJsonPath);
expect(getModelsForProvider(registry, "google").length).toBeGreaterThan(0);
expect(getModelsForProvider(registry, "openai").length).toBeGreaterThan(0);
});
test("provider-level baseUrl applies to both built-in and custom models", () => {
writeModelsJson({
anthropic: providerConfig("https://merged-proxy.example.com/v1", [{ id: "claude-custom" }]),
});
const registry = new ModelRegistry(authStorage, modelsJsonPath);
const anthropicModels = getModelsForProvider(registry, "anthropic");
for (const model of anthropicModels) {
expect(model.baseUrl).toBe("https://merged-proxy.example.com/v1");
}
});
test("model-level baseUrl overrides provider-level baseUrl for custom models", () => {
writeRawModelsJson({
"opencode-go": {
baseUrl: "https://opencode.ai/zen/go/v1",
apiKey: "TEST_KEY",
models: [
{
id: "minimax-m2.5",
api: "anthropic-messages",
baseUrl: "https://opencode.ai/zen/go",
reasoning: true,
input: ["text"],
cost: { input: 0.3, output: 1.2, cacheRead: 0.03, cacheWrite: 0 },
contextWindow: 204800,
maxTokens: 131072,
},
{
id: "glm-5",
api: "openai-completions",
reasoning: true,
input: ["text"],
cost: { input: 1, output: 3.2, cacheRead: 0.2, cacheWrite: 0 },
contextWindow: 204800,
maxTokens: 131072,
},
],
},
});
const registry = new ModelRegistry(authStorage, modelsJsonPath);
const m25 = registry.find("opencode-go", "minimax-m2.5");
const glm5 = registry.find("opencode-go", "glm-5");
expect(m25?.baseUrl).toBe("https://opencode.ai/zen/go");
expect(glm5?.baseUrl).toBe("https://opencode.ai/zen/go/v1");
});
test("modelOverrides still apply when provider also defines models", () => {
writeRawModelsJson({
openrouter: {
baseUrl: "https://my-proxy.example.com/v1",
apiKey: "OPENROUTER_API_KEY",
api: "openai-completions",
models: [
{
id: "custom/openrouter-model",
name: "Custom OpenRouter Model",
reasoning: false,
input: ["text"],
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
contextWindow: 128000,
maxTokens: 16384,
},
],
modelOverrides: {
"anthropic/claude-sonnet-4": {
name: "Overridden Built-in Sonnet",
},
},
},
});
const registry = new ModelRegistry(authStorage, modelsJsonPath);
const models = getModelsForProvider(registry, "openrouter");
expect(models.some(m => m.id === "custom/openrouter-model")).toBe(true);
expect(models.some(m => m.id === "anthropic/claude-sonnet-4" && m.name === "Overridden Built-in Sonnet")).toBe(
true,
);
});
test("refresh() reloads merged custom models from disk", async () => {
writeModelsJson({
anthropic: providerConfig("https://first-proxy.example.com/v1", [{ id: "claude-custom" }]),
});
const registry = new ModelRegistry(authStorage, modelsJsonPath);
expect(getModelsForProvider(registry, "anthropic").some(m => m.id === "claude-custom")).toBe(true);
// Update and refresh
writeModelsJson({
anthropic: providerConfig("https://second-proxy.example.com/v1", [{ id: "claude-custom-2" }]),
});
await registry.refresh("offline");
const anthropicModels = getModelsForProvider(registry, "anthropic");
expect(anthropicModels.some(m => m.id === "claude-custom")).toBe(false);
expect(anthropicModels.some(m => m.id === "claude-custom-2")).toBe(true);
expect(anthropicModels.some(m => m.id.includes("claude"))).toBe(true);
});
test("built-in gpt-5.4 applies the hardcoded context window policy", () => {
const registry = new ModelRegistry(authStorage, modelsJsonPath);
expect(registry.find("openai", "gpt-5.4")?.contextWindow).toBe(1_000_000);
});
test("custom gpt-5.4 replacement keeps the hardcoded context window when contextWindow is omitted", () => {
writeRawModelsJson({
openai: {
baseUrl: "https://my-proxy.example.com/v1",
apiKey: "TEST_KEY",
api: "openai-responses",
models: [{ id: "gpt-5.4" }],
},
});
const registry = new ModelRegistry(authStorage, modelsJsonPath);
const model = registry.find("openai", "gpt-5.4");
expect(model?.contextWindow).toBe(1_000_000);
expect(model?.baseUrl).toBe("https://my-proxy.example.com/v1");
});
test("custom-only gpt-5.4 provider keeps the hardcoded context window when contextWindow is omitted", () => {
writeRawModelsJson({
"my-proxy": {
baseUrl: "https://my-proxy.example.com/v1",
apiKey: "TEST_KEY",
api: "openai-responses",
models: [{ id: "gpt-5.4" }],
},
});
const registry = new ModelRegistry(authStorage, modelsJsonPath);
const model = registry.find("my-proxy", "gpt-5.4");
expect(model?.contextWindow).toBe(1_000_000);
expect(model?.baseUrl).toBe("https://my-proxy.example.com/v1");
});
test("custom gpt-5.4 replacement preserves its explicit context window", () => {
writeModelsJson({
openai: providerConfig(
"https://my-proxy.example.com/v1",
[{ id: "gpt-5.4", contextWindow: 256000 }],
"openai-responses",
),
});
const registry = new ModelRegistry(authStorage, modelsJsonPath);
expect(registry.find("openai", "gpt-5.4")?.contextWindow).toBe(256000);
});
test("modelOverrides can still patch a custom gpt-5.4 replacement", () => {
writeRawModelsJson({
openai: {
baseUrl: "https://my-proxy.example.com/v1",
apiKey: "TEST_KEY",
api: "openai-responses",
models: [
{
id: "gpt-5.4",
name: "gpt-5.4",
reasoning: false,
input: ["text"],
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
contextWindow: 256000,
maxTokens: 128000,
},
],
modelOverrides: {
"gpt-5.4": {
contextWindow: 512000,
},
},
},
});
const registry = new ModelRegistry(authStorage, modelsJsonPath);
expect(registry.find("openai", "gpt-5.4")?.contextWindow).toBe(512000);
});
test("discoverable bundled replacement survives refresh", async () => {
writeModelsJson({
openai: providerConfig(
"https://my-proxy.example.com/v1",
[{ id: "gpt-5.4", name: "Proxy GPT-5.4", contextWindow: 256000 }],
"openai-responses",
),
});
const registry = new ModelRegistry(authStorage, modelsJsonPath);
expect(registry.find("openai", "gpt-5.4")?.name).toBe("Proxy GPT-5.4");
expect(registry.find("openai", "gpt-5.4")?.contextWindow).toBe(256000);
using _hook = mockOpenAiCompatibleModels("https://my-proxy.example.com/v1/models", ["gpt-5.4"]);
await registry.refreshProvider("openai", "online");
const model = registry.find("openai", "gpt-5.4");
expect(model?.name).toBe("Proxy GPT-5.4");
expect(model?.contextWindow).toBe(256000);
expect(model?.baseUrl).toBe("https://my-proxy.example.com/v1");
});
test("discoverable custom-only gpt-5.4 survives refresh", async () => {
writeRawModelsJson({
"custom-local": {
baseUrl: "http://127.0.0.1:8080",
apiKey: "TEST_KEY",
api: "openai-responses",
discovery: { type: "llama.cpp" },
models: [{ id: "gpt-5.4" }],
},
});
const registry = new ModelRegistry(authStorage, modelsJsonPath);
expect(registry.find("custom-local", "gpt-5.4")?.contextWindow).toBe(1_000_000);
using _hook = mockOpenAiCompatibleModels("http://127.0.0.1:8080/models", ["gpt-5.4"]);
await registry.refreshProvider("custom-local", "online");
const model = registry.find("custom-local", "gpt-5.4");
expect(model?.contextWindow).toBe(1_000_000);
expect(model?.baseUrl).toBe("http://127.0.0.1:8080");
});
test("discoverable custom compat survives refresh", async () => {
writeRawModelsJson({
openai: {
baseUrl: "https://my-proxy.example.com/v1",
apiKey: "TEST_KEY",
api: "openai-responses",
models: [
{
id: "gpt-5.4",
compat: {
extraBody: { source: "proxy" },
},
},
],
},
});
const registry = new ModelRegistry(authStorage, modelsJsonPath);
expect(registry.find("openai", "gpt-5.4")?.compat?.extraBody).toEqual({ source: "proxy" });
using _hook = mockOpenAiCompatibleModels("https://my-proxy.example.com/v1/models", ["gpt-5.4"]);
await registry.refreshProvider("openai", "online");
expect(registry.find("openai", "gpt-5.4")?.compat?.extraBody).toEqual({ source: "proxy" });
});
test("modelOverrides still apply after discoverable refresh", async () => {
writeRawModelsJson({
openai: {
baseUrl: "https://my-proxy.example.com/v1",
apiKey: "TEST_KEY",
api: "openai-responses",
models: [
{
id: "gpt-5.4",
contextWindow: 256000,
},
],
modelOverrides: {
"gpt-5.4": {
contextWindow: 512000,
},
},
},
});
const registry = new ModelRegistry(authStorage, modelsJsonPath);
expect(registry.find("openai", "gpt-5.4")?.contextWindow).toBe(512000);
using _hook = mockOpenAiCompatibleModels("https://my-proxy.example.com/v1/models", ["gpt-5.4"]);
await registry.refreshProvider("openai", "online");
expect(registry.find("openai", "gpt-5.4")?.contextWindow).toBe(512000);
});
test("newly discovered ids inherit provider fields, not another model's custom fields", async () => {
writeRawModelsJson({
openai: {
baseUrl: "https://provider.example.com/v1",
headers: { "X-Provider": "provider" },
apiKey: "TEST_KEY",
api: "openai-responses",
models: [
{
id: "gpt-5.4",
baseUrl: "https://special.example.com/v1",
headers: { "X-Model": "special" },
},
],
},
});
const registry = new ModelRegistry(authStorage, modelsJsonPath);
expect(registry.find("openai", "gpt-5.4")?.baseUrl).toBe("https://special.example.com/v1");
using _hook = mockOpenAiCompatibleModels("https://provider.example.com/v1/models", ["gpt-5.4", "gpt-5.5"]);
await registry.refreshProvider("openai", "online");
const discovered = registry.find("openai", "gpt-5.5");
expect(discovered?.baseUrl).toBe("https://provider.example.com/v1");
expect(discovered?.headers?.["X-Provider"]).toBe("provider");
expect(discovered?.headers?.["X-Model"]).toBeUndefined();
});
test("same-id replacement uses configured compat without bundled compat leak", () => {
writeRawModelsJson({
"minimax-code": {
baseUrl: "https://proxy.example.com/v1",
apiKey: "TEST_KEY",
api: "openai-completions",
compat: {
extraBody: { source: "proxy" },
},
models: [{ id: "MiniMax-M2.5" }],
},
});
const registry = new ModelRegistry(authStorage, modelsJsonPath);
const model = registry.find("minimax-code", "MiniMax-M2.5");
expect(model?.compat?.thinkingFormat).toBeUndefined();
expect(model?.compat?.reasoningContentField).toBeUndefined();
expect(model?.compat?.extraBody).toEqual({ source: "proxy" });
});
test("removing custom models from models.json keeps built-in provider models", async () => {
writeModelsJson({
anthropic: providerConfig("https://proxy.example.com/v1", [{ id: "claude-custom" }]),
});
const registry = new ModelRegistry(authStorage, modelsJsonPath);
expect(getModelsForProvider(registry, "anthropic").some(m => m.id === "claude-custom")).toBe(true);
// Remove custom models and refresh
writeModelsJson({});
await registry.refresh("offline");
const anthropicModels = getModelsForProvider(registry, "anthropic");
expect(anthropicModels.length).toBeGreaterThan(1);
expect(anthropicModels.some(m => m.id === "claude-custom")).toBe(false);
expect(anthropicModels.some(m => m.id.includes("claude"))).toBe(true);
});
});
describe("thinking metadata normalization", () => {
test("custom models preserve explicit thinking", () => {
const thinking: ThinkingConfig = {
mode: "anthropic-adaptive",
minLevel: Effort.Minimal,
maxLevel: Effort.High,
};
writeModelsJson({
anthropic: providerConfig("https://my-proxy.example.com/v1", [
{ id: "claude-custom", reasoning: true, thinking },
]),
});
const registry = new ModelRegistry(authStorage, modelsJsonPath);
const model = getModelsForProvider(registry, "anthropic").find(m => m.id === "claude-custom");
expect(model?.thinking).toEqual(thinking);
});
test("model overrides can replace canonical thinking metadata", () => {
writeRawModelsJson({
openrouter: {
modelOverrides: {
"anthropic/claude-sonnet-4": {
thinking: { mode: "budget", minLevel: Effort.Low, maxLevel: Effort.Medium },
},
},
},
});
const registry = new ModelRegistry(authStorage, modelsJsonPath);
const model = getModelsForProvider(registry, "openrouter").find(m => m.id === "anthropic/claude-sonnet-4");
expect(model?.thinking).toEqual({
mode: "budget",
minLevel: Effort.Low,
maxLevel: Effort.Medium,
});
});
});
describe("modelOverrides (per-model customization)", () => {
test("model override applies to a single built-in model", () => {
writeRawModelsJson({
openrouter: {
modelOverrides: {
"anthropic/claude-sonnet-4": {
name: "Custom Sonnet Name",
},
},
},
});
const registry = new ModelRegistry(authStorage, modelsJsonPath);
const models = getModelsForProvider(registry, "openrouter");
const sonnet = models.find(m => m.id === "anthropic/claude-sonnet-4");
expect(sonnet?.name).toBe("Custom Sonnet Name");
// Other models should be unchanged
const opus = models.find(m => m.id === "anthropic/claude-opus-4");
expect(opus?.name).not.toBe("Custom Sonnet Name");
});
test("model override with compat.openRouterRouting", () => {
writeRawModelsJson({
openrouter: {
modelOverrides: {
"anthropic/claude-sonnet-4": {
compat: {
openRouterRouting: { only: ["amazon-bedrock"] },
},
},
},
},
});
const registry = new ModelRegistry(authStorage, modelsJsonPath);
const models = getModelsForProvider(registry, "openrouter");
const sonnet = models.find(m => m.id === "anthropic/claude-sonnet-4");
const compat = sonnet?.compat as OpenAICompat | undefined;
expect(compat?.openRouterRouting).toEqual({ only: ["amazon-bedrock"] });
});
test("model override deep merges compat settings", () => {
writeRawModelsJson({
openrouter: {
modelOverrides: {
"anthropic/claude-sonnet-4": {
compat: {
openRouterRouting: { order: ["anthropic", "together"] },
},
},
},
},
});
const registry = new ModelRegistry(authStorage, modelsJsonPath);
const models = getModelsForProvider(registry, "openrouter");
const sonnet = models.find(m => m.id === "anthropic/claude-sonnet-4");
const compat = sonnet?.compat as OpenAICompat | undefined;
expect(compat?.openRouterRouting).toEqual({ order: ["anthropic", "together"] });
});
test("model override merges compat.extraBody across provider+model", () => {
writeRawModelsJson({
openrouter: {
compat: {
extraBody: {
gateway: "default-gateway",
controller: "provider-controller",
},
},
modelOverrides: {
"anthropic/claude-sonnet-4": {
compat: {
extraBody: {
controller: "model-controller",
},
},
},
},
},
});
const registry = new ModelRegistry(authStorage, modelsJsonPath);
const models = getModelsForProvider(registry, "openrouter");
const sonnet = models.find(m => m.id === "anthropic/claude-sonnet-4");
const compat = sonnet?.compat as OpenAICompat | undefined;
expect(compat?.extraBody).toEqual({ gateway: "default-gateway", controller: "model-controller" });
});
test("multiple model overrides on same provider", () => {
writeRawModelsJson({
openrouter: {
modelOverrides: {
"anthropic/claude-sonnet-4": {
compat: { openRouterRouting: { only: ["amazon-bedrock"] } },
},
"anthropic/claude-opus-4": {
compat: { openRouterRouting: { only: ["anthropic"] } },
},
},
},
});
const registry = new ModelRegistry(authStorage, modelsJsonPath);
const models = getModelsForProvider(registry, "openrouter");
const sonnet = models.find(m => m.id === "anthropic/claude-sonnet-4");
const opus = models.find(m => m.id === "anthropic/claude-opus-4");
const sonnetCompat = sonnet?.compat as OpenAICompat | undefined;
const opusCompat = opus?.compat as OpenAICompat | undefined;
expect(sonnetCompat?.openRouterRouting).toEqual({ only: ["amazon-bedrock"] });
expect(opusCompat?.openRouterRouting).toEqual({ only: ["anthropic"] });
});
test("model override combined with baseUrl override", () => {
writeRawModelsJson({
openrouter: {
baseUrl: "https://my-proxy.example.com/v1",
modelOverrides: {
"anthropic/claude-sonnet-4": {
name: "Proxied Sonnet",
},
},
},
});
const registry = new ModelRegistry(authStorage, modelsJsonPath);
const models = getModelsForProvider(registry, "openrouter");
const sonnet = models.find(m => m.id === "anthropic/claude-sonnet-4");
// Both overrides should apply
expect(sonnet?.baseUrl).toBe("https://my-proxy.example.com/v1");
expect(sonnet?.name).toBe("Proxied Sonnet");
// Other models should have the baseUrl but not the name override
const opus = models.find(m => m.id === "anthropic/claude-opus-4");
expect(opus?.baseUrl).toBe("https://my-proxy.example.com/v1");
expect(opus?.name).not.toBe("Proxied Sonnet");
});
test("model override for non-existent model ID is ignored", () => {
writeRawModelsJson({
openrouter: {
modelOverrides: {
"nonexistent/model-id": {
name: "This should not appear",
},
},
},
});
const registry = new ModelRegistry(authStorage, modelsJsonPath);
const models = getModelsForProvider(registry, "openrouter");
// Should not create a new model
expect(models.find(m => m.id === "nonexistent/model-id")).toBeUndefined();
// Should not crash or show error
expect(registry.getError()).toBeUndefined();
});
test("model override can change cost fields partially", () => {
writeRawModelsJson({
openrouter: {
modelOverrides: {
"anthropic/claude-sonnet-4": {
cost: { input: 99 },
},
},
},
});
const registry = new ModelRegistry(authStorage, modelsJsonPath);
const models = getModelsForProvider(registry, "openrouter");
const sonnet = models.find(m => m.id === "anthropic/claude-sonnet-4");
// Input cost should be overridden
expect(sonnet?.cost.input).toBe(99);
// Other cost fields should be preserved from built-in
expect(sonnet?.cost.output).toBeGreaterThan(0);
});
test("model override can add headers", () => {
writeRawModelsJson({
openrouter: {
modelOverrides: {
"anthropic/claude-sonnet-4": {
headers: { "X-Custom-Model-Header": "value" },
},
},
},
});
const registry = new ModelRegistry(authStorage, modelsJsonPath);
const models = getModelsForProvider(registry, "openrouter");
const sonnet = models.find(m => m.id === "anthropic/claude-sonnet-4");
expect(sonnet?.headers?.["X-Custom-Model-Header"]).toBe("value");
});
test("refresh() picks up model override changes", async () => {
writeRawModelsJson({
openrouter: {
modelOverrides: {
"anthropic/claude-sonnet-4": {
name: "First Name",
},
},
},
});
const registry = new ModelRegistry(authStorage, modelsJsonPath);
expect(
getModelsForProvider(registry, "openrouter").find(m => m.id === "anthropic/claude-sonnet-4")?.name,
).toBe("First Name");
// Update and refresh
writeRawModelsJson({
openrouter: {
modelOverrides: {
"anthropic/claude-sonnet-4": {
name: "Second Name",
},
},
},
});
await registry.refresh("offline");
expect(
getModelsForProvider(registry, "openrouter").find(m => m.id === "anthropic/claude-sonnet-4")?.name,
).toBe("Second Name");
});
test("removing model override restores built-in values", async () => {
writeRawModelsJson({
openrouter: {
modelOverrides: {
"anthropic/claude-sonnet-4": {
name: "Custom Name",
},
},
},
});
const registry = new ModelRegistry(authStorage, modelsJsonPath);
const customName = getModelsForProvider(registry, "openrouter").find(
m => m.id === "anthropic/claude-sonnet-4",
)?.name;
expect(customName).toBe("Custom Name");
// Remove override and refresh
writeRawModelsJson({});
await registry.refresh("offline");
const restoredName = getModelsForProvider(registry, "openrouter").find(
m => m.id === "anthropic/claude-sonnet-4",
)?.name;
expect(restoredName).not.toBe("Custom Name");
});
});
describe("github-copilot oauth endpoint alignment", () => {
test("getApiKey does not mutate bundled github-copilot baseUrl", async () => {
await authStorage.set("github-copilot", [
{
type: "oauth",
access: "tid=1;proxy-ep=proxy.individual.githubcopilot.com;exp=9999999999",
refresh: "refresh-individual",
expires: Date.now() + 60_000,
},
{
type: "oauth",
access: "tid=2;proxy-ep=proxy.enterprise.githubcopilot.com;exp=9999999999",
refresh: "refresh-enterprise",
expires: Date.now() + 60_000,
},
]);
const registry = new ModelRegistry(authStorage, modelsJsonPath);
const model = registry.find("github-copilot", "gpt-4o");
expect(model).toBeDefined();
if (!model) throw new Error("Expected github-copilot/gpt-4o model");
const initialBaseUrl = model.baseUrl;
const firstApiKey = await registry.getApiKey(model);
expect(firstApiKey).toContain("proxy.individual.githubcopilot.com");
const secondApiKey = await registry.getApiKey(model);
expect(secondApiKey).toContain("proxy.enterprise.githubcopilot.com");
expect(model.baseUrl).toBe(initialBaseUrl);
});
});
describe("disabled provider filtering", () => {
test("getAvailable and getDiscoverableProviders exclude disabled providers from settings", async () => {
writeRawModelsJson({
ollama: {
baseUrl: "http://127.0.0.1:11434/v1",
api: "openai-completions",
auth: "none",
discovery: { type: "ollama" },
},
});
await authStorage.set("github-copilot", [
{
type: "oauth",
access: "tid=1;proxy-ep=proxy.individual.githubcopilot.com;exp=9999999999",
refresh: "refresh-individual",
expires: Date.now() + 60_000,
},
]);
await Settings.init({
inMemory: true,
overrides: {
disabledProviders: ["github-copilot", "ollama"],
},
});
const registry = new ModelRegistry(authStorage, modelsJsonPath);
expect(registry.getAvailable().some(model => model.provider === "github-copilot")).toBe(false);
expect(registry.getDiscoverableProviders()).not.toContain("ollama");
});
});
describe("runtime discovery", () => {
test("auto-discovers ollama models without provider config", async () => {
using _hook = hookFetch(input => {
const url = String(input);
if (url === "http://127.0.0.1:11434/api/tags") {
return new Response(JSON.stringify({ models: [{ name: "phi4-mini" }] }), {
status: 200,
headers: { "Content-Type": "application/json" },
});
}
if (url === "http://127.0.0.1:11434/api/show") {
return new Response(JSON.stringify({ capabilities: ["completion"] }), {
status: 200,
headers: { "Content-Type": "application/json" },
});
}
throw new Error(`Unexpected URL: ${url}`);
});
const registry = new ModelRegistry(authStorage, modelsJsonPath);
await registry.refresh();
const ollamaModels = getModelsForProvider(registry, "ollama");
expect(ollamaModels.some(m => m.id === "phi4-mini")).toBe(true);
expect(registry.getAvailable().some(m => m.provider === "ollama" && m.id === "phi4-mini")).toBe(true);
expect(await registry.getApiKey(ollamaModels[0])).toBe(kNoAuth);
});
test("discovers ollama models at runtime and treats auth:none providers as available", async () => {
writeRawModelsJson({
ollama: {
baseUrl: "http://127.0.0.1:11434/v1",
api: "openai-completions",
auth: "none",
discovery: { type: "ollama" },
},
});
using _hook = hookFetch(input => {
const url = String(input);
if (url === "http://127.0.0.1:11434/api/tags") {
return new Response(
JSON.stringify({
models: [{ name: "qwen2.5-coder:7b" }, { model: "llama3.2:3b", name: "llama3.2:3b" }],
}),
{ status: 200, headers: { "Content-Type": "application/json" } },
);
}
if (url === "http://127.0.0.1:11434/api/show") {
return new Response(JSON.stringify({ capabilities: ["completion"] }), {
status: 200,
headers: { "Content-Type": "application/json" },
});
}
throw new Error(`Unexpected URL: ${url}`);
});
const registry = new ModelRegistry(authStorage, modelsJsonPath);
await registry.refresh();
const ollamaModels = getModelsForProvider(registry, "ollama");
expect(ollamaModels.some(m => m.id === "qwen2.5-coder:7b")).toBe(true);
expect(ollamaModels.some(m => m.id === "llama3.2:3b")).toBe(true);
const available = registry.getAvailable().filter(m => m.provider === "ollama");
expect(available.length).toBe(2);
expect(await registry.getApiKey(available[0])).toBe(kNoAuth);
});
test("discovers ollama thinking capabilities from show metadata", async () => {
writeRawModelsJson({
ollama: {
baseUrl: "http://127.0.0.1:11434/v1",
api: "openai-completions",
auth: "none",
discovery: { type: "ollama" },
},
});
using _hook = hookFetch((input, init) => {
const url = String(input);
if (url === "http://127.0.0.1:11434/api/tags") {
return new Response(
JSON.stringify({
models: [{ name: "qwen3.5:397b-cloud" }, { name: "llama3.2:3b" }],
}),
{ status: 200, headers: { "Content-Type": "application/json" } },
);
}
if (url === "http://127.0.0.1:11434/api/show") {
const body = JSON.parse(String(init?.body ?? "{}")) as { model?: string };
if (body.model === "qwen3.5:397b-cloud") {
return new Response(JSON.stringify({ capabilities: ["completion", "thinking"] }), {
status: 200,
headers: { "Content-Type": "application/json" },
});
}
if (body.model === "llama3.2:3b") {
return new Response(JSON.stringify({ capabilities: ["completion"] }), {
status: 200,
headers: { "Content-Type": "application/json" },
});
}
}
throw new Error(`Unexpected request: ${url}`);
});
const registry = new ModelRegistry(authStorage, modelsJsonPath);
await registry.refresh();
const qwen = registry.find("ollama", "qwen3.5:397b-cloud");
expect(qwen?.reasoning).toBe(true);
expect(qwen?.thinking).toEqual({
mode: "effort",
minLevel: Effort.Minimal,
maxLevel: Effort.High,
});
const llama = registry.find("ollama", "llama3.2:3b");
expect(llama?.reasoning).toBe(false);
});
test("discovers ollama context window from show model_info", async () => {
using _hook = hookFetch((input, init) => {
const url = String(input);
if (url === "http://127.0.0.1:11434/api/tags") {
return new Response(JSON.stringify({ models: [{ name: "gemma3:4b" }] }), {
status: 200,
headers: { "Content-Type": "application/json" },
});
}
if (url === "http://127.0.0.1:11434/api/show") {
const body = JSON.parse(String(init?.body ?? "{}")) as { model?: string };
if (body.model === "gemma3:4b") {
return new Response(
JSON.stringify({
model_info: {
"gemma3.context_length": 131072,
},
}),
{
status: 200,
headers: { "Content-Type": "application/json" },
},
);
}
}
throw new Error(`Unexpected request: ${url}`);
});
const registry = new ModelRegistry(authStorage, modelsJsonPath);
await registry.refresh();
const gemma = registry.find("ollama", "gemma3:4b");
expect(gemma?.contextWindow).toBe(131072);
expect(gemma?.maxTokens).toBe(8192);
expect(gemma?.input).toEqual(["text"]);
expect(gemma?.reasoning).toBe(false);
});
test("discovery failure does not fail model registry refresh", async () => {
writeRawModelsJson({
ollama: {
baseUrl: "http://127.0.0.1:11434",
api: "openai-completions",
auth: "none",
discovery: { type: "ollama" },
},
});
using _hook = hookFetch(() => {
throw new Error("connection refused");
});
const registry = new ModelRegistry(authStorage, modelsJsonPath);
await registry.refresh();
expect(getModelsForProvider(registry, "ollama")).toHaveLength(0);
expect(registry.getError()).toBeUndefined();
});
test("loads cached local models before live refresh and preserves them on failure", async () => {
writeRawModelsJson({
ollama: {
baseUrl: "http://127.0.0.1:11434/v1",
api: "openai-completions",
auth: "none",
discovery: { type: "ollama" },
},
});
{
using _hook = hookFetch(input => {
const url = String(input);
if (url === "http://127.0.0.1:11434/api/tags") {
return new Response(JSON.stringify({ models: [{ name: "phi4-mini" }] }), {
status: 200,
headers: { "Content-Type": "application/json" },
});
}
if (url === "http://127.0.0.1:11434/api/show") {
return new Response(JSON.stringify({ capabilities: ["completion"] }), {
status: 200,
headers: { "Content-Type": "application/json" },
});
}
throw new Error(`Unexpected URL: ${url}`);
});
const primedRegistry = new ModelRegistry(authStorage, modelsJsonPath);
await primedRegistry.refresh();
}
const cachedRegistry = new ModelRegistry(authStorage, modelsJsonPath);
expect(getModelsForProvider(cachedRegistry, "ollama").some(model => model.id === "phi4-mini")).toBe(true);
expect(cachedRegistry.getProviderDiscoveryState("ollama")?.status).toBe("cached");
{
using _hook = hookFetch(() => {
throw new Error("connection refused");
});
await cachedRegistry.refreshProvider("ollama");
}
expect(getModelsForProvider(cachedRegistry, "ollama").some(model => model.id === "phi4-mini")).toBe(true);
const state = cachedRegistry.getProviderDiscoveryState("ollama");
expect(state?.status).toBe("cached");
expect(state?.error).toContain("connection refused");
});
test("reports unauthenticated discoverable providers without discarding cached models", async () => {
writeRawModelsJson({
"custom-local": {
baseUrl: "http://127.0.0.1:11434/v1",
api: "openai-completions",
discovery: { type: "ollama" },
},
});
authStorage.setRuntimeApiKey("custom-local", "test-key");
{
using _hook = hookFetch(input => {
const url = String(input);
if (url === "http://127.0.0.1:11434/api/tags") {
return new Response(JSON.stringify({ models: [{ name: "local-coder" }] }), {
status: 200,
headers: { "Content-Type": "application/json" },
});
}
if (url === "http://127.0.0.1:11434/api/show") {
return new Response(JSON.stringify({ capabilities: ["completion"] }), {
status: 200,
headers: { "Content-Type": "application/json" },
});
}
throw new Error(`Unexpected URL: ${url}`);
});
const primedRegistry = new ModelRegistry(authStorage, modelsJsonPath);
await primedRegistry.refreshProvider("custom-local");
}
authStorage.setRuntimeApiKey("custom-local", "");
const cachedRegistry = new ModelRegistry(authStorage, modelsJsonPath);
await cachedRegistry.refreshProvider("custom-local");
expect(getModelsForProvider(cachedRegistry, "custom-local").some(model => model.id === "local-coder")).toBe(
true,
);
const state = cachedRegistry.getProviderDiscoveryState("custom-local");
expect(state?.status).toBe("unauthenticated");
expect(state?.models).toContain("local-coder");
});
test("llama.cpp discovery honors configured API key", async () => {
authStorage.setRuntimeApiKey("llama.cpp", "test-llama-key");
using _hook = hookFetch((input, init) => {
const url = String(input);
if (url === "http://127.0.0.1:8080/models") {
const headers = init?.headers as Headers | Record<string, string> | undefined;
let authHeader: string | null = null;
if (headers instanceof Headers) {
authHeader = headers.get("Authorization");
} else if (typeof headers === "object") {
authHeader = headers.Authorization;
}
expect(String(authHeader ?? "")).toBe("Bearer test-llama-key");
return new Response(JSON.stringify({ data: [{ id: "llama-3.2:3b" }, { id: "mistral:7b" }] }), {
status: 200,
headers: { "Content-Type": "application/json" },
});
}
if (url === "http://127.0.0.1:8080/props") {
const headers = init?.headers as Headers | Record<string, string> | undefined;
let authHeader: string | null = null;
if (headers instanceof Headers) {
authHeader = headers.get("Authorization");
} else if (typeof headers === "object") {
authHeader = headers.Authorization;
}
expect(String(authHeader ?? "")).toBe("Bearer test-llama-key");
return new Response(JSON.stringify({ default_generation_settings: { n_ctx: 262144 } }), {
status: 200,
headers: { "Content-Type": "application/json" },
});
}
throw new Error(`Unexpected URL: ${url}`);
});
const registry = new ModelRegistry(authStorage, modelsJsonPath);
await registry.refresh();
const llamaModels = getModelsForProvider(registry, "llama.cpp");
expect(llamaModels.some(m => m.id === "llama-3.2:3b")).toBe(true);
const apiKey = await registry.getApiKey(llamaModels[0]);
expect(apiKey).toBe("test-llama-key");
expect(apiKey).not.toBe(kNoAuth);
});
test("llama.cpp discovery without API key is treated as keyless", async () => {
using _hook = hookFetch((input, init) => {
const url = String(input);
if (url === "http://127.0.0.1:8080/models") {
const headers = init?.headers as Headers | Record<string, string> | undefined;
let authHeader: string | null = null;
if (headers instanceof Headers) {
authHeader = headers.get("Authorization");
} else if (typeof headers === "object") {
authHeader = headers.Authorization;
}
// When no API key, headers should be empty object or undefined
expect(authHeader).toBeUndefined();
return new Response(JSON.stringify({ data: [{ id: "llama-3.2:3b" }] }), {
status: 200,
headers: { "Content-Type": "application/json" },
});
}
if (url === "http://127.0.0.1:8080/props") {
const headers = init?.headers as Headers | Record<string, string> | undefined;
let authHeader: string | null = null;
if (headers instanceof Headers) {
authHeader = headers.get("Authorization");
} else if (typeof headers === "object") {
authHeader = headers.Authorization;
}
expect(authHeader).toBeUndefined();
return new Response(JSON.stringify({ default_generation_settings: { n_ctx: 262144 } }), {
status: 200,
headers: { "Content-Type": "application/json" },
});
}
throw new Error(`Unexpected URL: ${url}`);
});
const registry = new ModelRegistry(authStorage, modelsJsonPath);
await registry.refresh();
const state = registry.getProviderDiscoveryState("llama.cpp");
if (state?.status !== "ok") {
throw new Error(`Discovery failed with status ${state?.status}: ${state?.error}`);
}
const llamaModels = getModelsForProvider(registry, "llama.cpp");
const apiKey = await registry.getApiKey(llamaModels[0]);
expect(apiKey).toBe(kNoAuth);
});
test("llama.cpp discovery reads context window from props n_ctx", async () => {
using _hook = hookFetch(input => {
const url = String(input);
if (url === "http://127.0.0.1:8080/models") {
return new Response(JSON.stringify({ data: [{ id: "qwen35-35b-a3b" }] }), {
status: 200,
headers: { "Content-Type": "application/json" },
});
}
if (url === "http://127.0.0.1:8080/props") {
return new Response(
JSON.stringify({
default_generation_settings: {
n_ctx: 262144,
},
modalities: {
vision: true,
audio: false,
},
}),
{
status: 200,
headers: { "Content-Type": "application/json" },
},
);
}
throw new Error(`Unexpected URL: ${url}`);
});
const registry = new ModelRegistry(authStorage, modelsJsonPath);
await registry.refresh();
const llama = registry.find("llama.cpp", "qwen35-35b-a3b");
expect(llama?.contextWindow).toBe(262144);
expect(llama?.maxTokens).toBe(8192);
expect(llama?.input).toEqual(["text", "image"]);
});
});
});