Gerben Meijer
4a6eec624b
Read vLLM max_model_len and OpenAI-compatible context_length metadata during model discovery, route providers.vllm.baseUrl into built-in discovery before cached models exist, and avoid sending local placeholder bearer tokens. Scope the vLLM model cache to the discovery base URL so endpoint changes refetch immediately, and add focused regression coverage for configured and built-in vLLM discovery.
422 lines
14 KiB
TypeScript
422 lines
14 KiB
TypeScript
import { afterEach, beforeEach, describe, expect, test, vi } from "bun:test";
|
|
import * as fs from "node:fs";
|
|
import * as os from "node:os";
|
|
import * as path from "node:path";
|
|
import { getBundledModel } from "@oh-my-pi/pi-catalog/models";
|
|
import { ModelRegistry } from "@oh-my-pi/pi-coding-agent/config/model-registry";
|
|
import { Settings } from "@oh-my-pi/pi-coding-agent/config/settings";
|
|
import { createAgentSession, type ExtensionFactory } from "@oh-my-pi/pi-coding-agent/sdk";
|
|
import { AuthStorage } from "@oh-my-pi/pi-coding-agent/session/auth-storage";
|
|
import { SessionManager } from "@oh-my-pi/pi-coding-agent/session/session-manager";
|
|
import { Snowflake } from "@oh-my-pi/pi-utils";
|
|
|
|
describe("createAgentSession deferred model pattern resolution", () => {
|
|
let tempDir: string;
|
|
const authStoragesToClose: AuthStorage[] = [];
|
|
|
|
beforeEach(() => {
|
|
tempDir = path.join(os.tmpdir(), `pi-sdk-model-selection-${Snowflake.next()}`);
|
|
fs.mkdirSync(tempDir, { recursive: true });
|
|
});
|
|
|
|
afterEach(() => {
|
|
for (const authStorage of authStoragesToClose) {
|
|
authStorage.close();
|
|
}
|
|
authStoragesToClose.length = 0;
|
|
if (tempDir && fs.existsSync(tempDir)) {
|
|
fs.rmSync(tempDir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
const providerExtension: ExtensionFactory = pi => {
|
|
pi.registerProvider("runtime-provider", {
|
|
baseUrl: "https://runtime.example.com/v1",
|
|
apiKey: "RUNTIME_KEY",
|
|
api: "openai-completions",
|
|
models: [
|
|
{
|
|
id: "runtime-model",
|
|
name: "Runtime Model",
|
|
reasoning: false,
|
|
input: ["text"],
|
|
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
|
contextWindow: 128000,
|
|
maxTokens: 8192,
|
|
},
|
|
{
|
|
id: "runtime-reasoning-model",
|
|
name: "Runtime Reasoning Model",
|
|
reasoning: true,
|
|
input: ["text"],
|
|
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
|
contextWindow: 128000,
|
|
maxTokens: 8192,
|
|
},
|
|
],
|
|
});
|
|
};
|
|
|
|
async function buildSessionOptions(modelPattern: string) {
|
|
// Pass an explicit ModelRegistry so createAgentSession skips its implicit
|
|
// ModelRegistry.refreshInBackground() — a network model-discovery pass
|
|
// (~250ms/session) that contributes nothing here: the model resolves from
|
|
// the inline extension provider, never from network catalogs. Mirrors the
|
|
// explicit-registry pattern the resume tests below already rely on.
|
|
const authStorage = await AuthStorage.create(path.join(tempDir, "auth.db"));
|
|
authStoragesToClose.push(authStorage);
|
|
const modelRegistry = new ModelRegistry(authStorage, path.join(tempDir, "models.yml"));
|
|
return {
|
|
cwd: tempDir,
|
|
agentDir: tempDir,
|
|
authStorage,
|
|
modelRegistry,
|
|
sessionManager: SessionManager.inMemory(),
|
|
disableExtensionDiscovery: true,
|
|
extensions: [providerExtension],
|
|
skills: [],
|
|
contextFiles: [],
|
|
promptTemplates: [],
|
|
slashCommands: [],
|
|
enableMCP: false,
|
|
enableLsp: false,
|
|
modelPattern,
|
|
};
|
|
}
|
|
|
|
test("resolves explicit modelPattern after extension providers register", async () => {
|
|
const { session, modelFallbackMessage } = await createAgentSession(
|
|
await buildSessionOptions("runtime-provider/runtime-model"),
|
|
);
|
|
|
|
expect(session.model).toBeDefined();
|
|
expect(session.model?.provider).toBe("runtime-provider");
|
|
expect(session.model?.id).toBe("runtime-model");
|
|
expect(modelFallbackMessage).toBeUndefined();
|
|
});
|
|
|
|
test("does not silently fallback when explicit modelPattern is unresolved", async () => {
|
|
const { session, modelFallbackMessage } = await createAgentSession(
|
|
await buildSessionOptions("missing-provider/missing-model"),
|
|
);
|
|
|
|
expect(session.model).toBeUndefined();
|
|
expect(modelFallbackMessage).toBe('Model "missing-provider/missing-model" not found');
|
|
});
|
|
|
|
test("does not apply default role thinking override when modelPattern is explicit", async () => {
|
|
const settings = Settings.isolated({ defaultThinkingLevel: "off" });
|
|
settings.setModelRole("smol", "runtime-provider/runtime-reasoning-model");
|
|
settings.setModelRole("default", "pi/smol:high");
|
|
|
|
const { session } = await createAgentSession({
|
|
...(await buildSessionOptions("runtime-provider/runtime-reasoning-model")),
|
|
settings,
|
|
});
|
|
|
|
expect(session.model?.provider).toBe("runtime-provider");
|
|
expect(session.model?.id).toBe("runtime-reasoning-model");
|
|
expect(session.thinkingLevel).toBe("off");
|
|
});
|
|
|
|
test("selects the settings default model without synchronously validating auth", async () => {
|
|
const defaultModel = getBundledModel("anthropic", "claude-sonnet-4-5");
|
|
if (!defaultModel) {
|
|
throw new Error("Expected bundled anthropic default model");
|
|
}
|
|
|
|
const authStorage = await AuthStorage.create(path.join(tempDir, "testauth.db"));
|
|
authStorage.setRuntimeApiKey(defaultModel.provider, "test-key");
|
|
const modelRegistry = new ModelRegistry(authStorage, path.join(tempDir, "models.yml"));
|
|
const settings = Settings.isolated();
|
|
settings.setModelRole("default", `${defaultModel.provider}/${defaultModel.id}`);
|
|
|
|
const getApiKeySpy = vi
|
|
.spyOn(modelRegistry, "getApiKey")
|
|
.mockRejectedValue(new Error("settings default model should not validate auth during startup"));
|
|
|
|
try {
|
|
const { session } = await createAgentSession({
|
|
cwd: tempDir,
|
|
agentDir: tempDir,
|
|
authStorage,
|
|
modelRegistry,
|
|
settings,
|
|
sessionManager: SessionManager.inMemory(),
|
|
disableExtensionDiscovery: true,
|
|
skills: [],
|
|
contextFiles: [],
|
|
promptTemplates: [],
|
|
slashCommands: [],
|
|
enableMCP: false,
|
|
enableLsp: false,
|
|
});
|
|
|
|
try {
|
|
expect(session.model?.provider).toBe(defaultModel.provider);
|
|
expect(session.model?.id).toBe(defaultModel.id);
|
|
expect(getApiKeySpy).not.toHaveBeenCalled();
|
|
} finally {
|
|
await session.dispose();
|
|
}
|
|
} finally {
|
|
getApiKeySpy.mockRestore();
|
|
authStorage.close();
|
|
}
|
|
});
|
|
|
|
test("restores the saved session model without resolving auth over the network", async () => {
|
|
// Regression: `restoreSessionModel` probed each saved-model candidate with
|
|
// the async `getApiKey`, which refreshes OAuth tokens and hits the auth
|
|
// broker. When the broker was unreachable that blocked resume for the full
|
|
// ~10s refresh timeout — the "Still starting … restoreSessionModel" hang.
|
|
// Selection now uses the synchronous, side-effect-free `hasConfiguredAuth`
|
|
// probe; the real key is resolved lazily per request via the resolver.
|
|
const savedModel = getBundledModel("anthropic", "claude-sonnet-4-5");
|
|
if (!savedModel) {
|
|
throw new Error("Expected bundled anthropic default model");
|
|
}
|
|
|
|
const authStorage = await AuthStorage.create(path.join(tempDir, "resume-saved-auth.db"));
|
|
authStoragesToClose.push(authStorage);
|
|
authStorage.setRuntimeApiKey(savedModel.provider, "test-key");
|
|
const modelRegistry = new ModelRegistry(authStorage, path.join(tempDir, "models.yml"));
|
|
|
|
const targetSessionFile = path.join(tempDir, "resume-saved-model.jsonl");
|
|
const timestamp = "2026-06-01T00:00:00.000Z";
|
|
await Bun.write(
|
|
targetSessionFile,
|
|
`${[
|
|
{ type: "session", version: 3, id: "resume-saved", timestamp, cwd: tempDir },
|
|
{
|
|
type: "model_change",
|
|
id: "default-model",
|
|
parentId: null,
|
|
timestamp,
|
|
model: `${savedModel.provider}/${savedModel.id}`,
|
|
role: "default",
|
|
},
|
|
]
|
|
.map(entry => JSON.stringify(entry))
|
|
.join("\n")}\n`,
|
|
);
|
|
const sessionManager = await SessionManager.open(targetSessionFile, path.join(tempDir, "resume-saved-sessions"));
|
|
|
|
// A rejecting getApiKey stands in for the unreachable broker / hanging
|
|
// OAuth refresh: if startup awaits it to pick the restore model, it surfaces.
|
|
const getApiKeySpy = vi
|
|
.spyOn(modelRegistry, "getApiKey")
|
|
.mockRejectedValue(new Error("startup model restore must not resolve auth over the network"));
|
|
|
|
try {
|
|
const { session } = await createAgentSession({
|
|
cwd: tempDir,
|
|
agentDir: tempDir,
|
|
authStorage,
|
|
modelRegistry,
|
|
sessionManager,
|
|
settings: Settings.isolated(),
|
|
disableExtensionDiscovery: true,
|
|
skills: [],
|
|
contextFiles: [],
|
|
promptTemplates: [],
|
|
slashCommands: [],
|
|
enableMCP: false,
|
|
enableLsp: false,
|
|
skipPythonPreflight: true,
|
|
});
|
|
|
|
try {
|
|
expect(session.model?.provider).toBe(savedModel.provider);
|
|
expect(session.model?.id).toBe(savedModel.id);
|
|
expect(getApiKeySpy).not.toHaveBeenCalled();
|
|
} finally {
|
|
await session.dispose();
|
|
}
|
|
} finally {
|
|
getApiKeySpy.mockRestore();
|
|
}
|
|
});
|
|
|
|
test("prefers the provider default over catalog order in the startup fallback", async () => {
|
|
// Regression: with an Anthropic key but no configured `default` role and no
|
|
// session/CLI model, the step-4 startup fallback used to pick the first
|
|
// anthropic model in models.json catalog order (claude-3-5-sonnet-20240620)
|
|
// instead of the provider's configured default from DEFAULT_MODEL_PER_PROVIDER
|
|
// (claude-opus-4-8).
|
|
const providerDefault = getBundledModel("anthropic", "claude-opus-4-8");
|
|
const catalogFirst = getBundledModel("anthropic", "claude-3-5-sonnet-20240620");
|
|
if (!providerDefault || !catalogFirst) {
|
|
throw new Error("Expected bundled anthropic models for fallback regression");
|
|
}
|
|
|
|
const authStorage = await AuthStorage.create(path.join(tempDir, "fallbackauth.db"));
|
|
authStoragesToClose.push(authStorage);
|
|
authStorage.setRuntimeApiKey("anthropic", "test-key");
|
|
const modelRegistry = new ModelRegistry(authStorage, path.join(tempDir, "models.yml"));
|
|
// No `default` model role configured: forces the step-4 startup fallback.
|
|
const settings = Settings.isolated();
|
|
|
|
const { session } = await createAgentSession({
|
|
cwd: tempDir,
|
|
agentDir: tempDir,
|
|
authStorage,
|
|
modelRegistry,
|
|
settings,
|
|
sessionManager: SessionManager.inMemory(),
|
|
disableExtensionDiscovery: true,
|
|
skills: [],
|
|
contextFiles: [],
|
|
promptTemplates: [],
|
|
slashCommands: [],
|
|
enableMCP: false,
|
|
enableLsp: false,
|
|
skipPythonPreflight: true,
|
|
});
|
|
|
|
try {
|
|
expect(session.model?.provider).toBe("anthropic");
|
|
expect(session.model?.id).toBe(providerDefault.id);
|
|
expect(session.model?.id).not.toBe(catalogFirst.id);
|
|
} finally {
|
|
await session.dispose();
|
|
}
|
|
});
|
|
|
|
test("restores role model from extension provider after startup resume", async () => {
|
|
const defaultModel = getBundledModel("anthropic", "claude-sonnet-4-5");
|
|
if (!defaultModel) {
|
|
throw new Error("Expected bundled anthropic default model");
|
|
}
|
|
|
|
const authStorage = await AuthStorage.create(path.join(tempDir, "testauth.db"));
|
|
authStorage.setRuntimeApiKey(defaultModel.provider, "test-key");
|
|
const modelRegistry = new ModelRegistry(authStorage, path.join(tempDir, "models.yml"));
|
|
|
|
const targetSessionFile = path.join(tempDir, "resume-extension.jsonl");
|
|
const timestamp = "2026-06-01T00:00:00.000Z";
|
|
await Bun.write(
|
|
targetSessionFile,
|
|
`${[
|
|
{ type: "session", version: 3, id: "resume-ext", timestamp, cwd: tempDir },
|
|
{
|
|
type: "model_change",
|
|
id: "default-model",
|
|
parentId: null,
|
|
timestamp,
|
|
model: `${defaultModel.provider}/${defaultModel.id}`,
|
|
role: "default",
|
|
},
|
|
{
|
|
type: "model_change",
|
|
id: "smol-model",
|
|
parentId: "default-model",
|
|
timestamp,
|
|
model: "runtime-provider/runtime-model",
|
|
role: "smol",
|
|
},
|
|
]
|
|
.map(entry => JSON.stringify(entry))
|
|
.join("\n")}\n`,
|
|
);
|
|
const sessionManager = await SessionManager.open(targetSessionFile, path.join(tempDir, "sessions"));
|
|
|
|
const { session } = await createAgentSession({
|
|
cwd: tempDir,
|
|
agentDir: tempDir,
|
|
authStorage,
|
|
modelRegistry,
|
|
sessionManager,
|
|
settings: Settings.isolated(),
|
|
disableExtensionDiscovery: true,
|
|
extensions: [providerExtension],
|
|
skills: [],
|
|
contextFiles: [],
|
|
promptTemplates: [],
|
|
slashCommands: [],
|
|
enableMCP: false,
|
|
enableLsp: false,
|
|
skipPythonPreflight: true,
|
|
});
|
|
|
|
try {
|
|
expect(session.model?.provider).toBe("runtime-provider");
|
|
expect(session.model?.id).toBe("runtime-model");
|
|
} finally {
|
|
await session.dispose();
|
|
authStorage.close();
|
|
}
|
|
});
|
|
|
|
test("restores extension role model when saved default cannot be restored before extensions load", async () => {
|
|
const settingsDefaultModel = getBundledModel("anthropic", "claude-sonnet-4-5");
|
|
if (!settingsDefaultModel) {
|
|
throw new Error("Expected bundled anthropic default model");
|
|
}
|
|
|
|
const authStorage = await AuthStorage.create(path.join(tempDir, "testauth.db"));
|
|
authStorage.setRuntimeApiKey(settingsDefaultModel.provider, "test-key");
|
|
const modelRegistry = new ModelRegistry(authStorage, path.join(tempDir, "models.yml"));
|
|
|
|
// Saved default points at a provider that has no usable credentials. The
|
|
// last active role (`smol`) is supplied by the inline extension and is
|
|
// only resolvable once provider registrations are processed.
|
|
const targetSessionFile = path.join(tempDir, "resume-extension-default-missing.jsonl");
|
|
const timestamp = "2026-06-01T00:00:00.000Z";
|
|
await Bun.write(
|
|
targetSessionFile,
|
|
`${[
|
|
{ type: "session", version: 3, id: "resume-ext-no-default", timestamp, cwd: tempDir },
|
|
{
|
|
type: "model_change",
|
|
id: "default-model",
|
|
parentId: null,
|
|
timestamp,
|
|
model: "anthropic/not-available",
|
|
role: "default",
|
|
},
|
|
{
|
|
type: "model_change",
|
|
id: "smol-model",
|
|
parentId: "default-model",
|
|
timestamp,
|
|
model: "runtime-provider/runtime-model",
|
|
role: "smol",
|
|
},
|
|
]
|
|
.map(entry => JSON.stringify(entry))
|
|
.join("\n")}\n`,
|
|
);
|
|
const sessionManager = await SessionManager.open(targetSessionFile, path.join(tempDir, "sessions-no-default"));
|
|
|
|
const settings = Settings.isolated();
|
|
settings.setModelRole("default", `${settingsDefaultModel.provider}/${settingsDefaultModel.id}`);
|
|
|
|
const { session } = await createAgentSession({
|
|
cwd: tempDir,
|
|
agentDir: tempDir,
|
|
authStorage,
|
|
modelRegistry,
|
|
sessionManager,
|
|
settings,
|
|
disableExtensionDiscovery: true,
|
|
extensions: [providerExtension],
|
|
skills: [],
|
|
contextFiles: [],
|
|
promptTemplates: [],
|
|
slashCommands: [],
|
|
enableMCP: false,
|
|
enableLsp: false,
|
|
skipPythonPreflight: true,
|
|
});
|
|
|
|
try {
|
|
expect(session.model?.provider).toBe("runtime-provider");
|
|
expect(session.model?.id).toBe("runtime-model");
|
|
} finally {
|
|
await session.dispose();
|
|
authStorage.close();
|
|
}
|
|
});
|
|
});
|