can1357
476 lines
16 KiB
TypeScript
476 lines
16 KiB
TypeScript
import { afterEach, beforeEach, describe, expect, test, vi } from "bun:test";
|
|
import * as fs from "node:fs";
|
|
import * as os from "node:os";
|
|
import * as path from "node:path";
|
|
import { Effort } from "@oh-my-pi/pi-ai";
|
|
import { getBundledModel } from "@oh-my-pi/pi-catalog/models";
|
|
import { ModelRegistry } from "@oh-my-pi/pi-coding-agent/config/model-registry";
|
|
import { Settings } from "@oh-my-pi/pi-coding-agent/config/settings";
|
|
import { createAgentSession, type ExtensionFactory } from "@oh-my-pi/pi-coding-agent/sdk";
|
|
import { AuthStorage } from "@oh-my-pi/pi-coding-agent/session/auth-storage";
|
|
import { SessionManager } from "@oh-my-pi/pi-coding-agent/session/session-manager";
|
|
import { removeSyncWithRetries, Snowflake } from "@oh-my-pi/pi-utils";
|
|
|
|
describe("createAgentSession deferred model pattern resolution", () => {
|
|
let tempDir: string;
|
|
const authStoragesToClose: AuthStorage[] = [];
|
|
|
|
beforeEach(() => {
|
|
tempDir = path.join(os.tmpdir(), `pi-sdk-model-selection-${Snowflake.next()}`);
|
|
fs.mkdirSync(tempDir, { recursive: true });
|
|
});
|
|
|
|
afterEach(() => {
|
|
for (const authStorage of authStoragesToClose) {
|
|
authStorage.close();
|
|
}
|
|
authStoragesToClose.length = 0;
|
|
if (tempDir && fs.existsSync(tempDir)) {
|
|
removeSyncWithRetries(tempDir);
|
|
}
|
|
});
|
|
|
|
const providerExtension: ExtensionFactory = pi => {
|
|
pi.registerProvider("runtime-provider", {
|
|
baseUrl: "https://runtime.example.com/v1",
|
|
apiKey: "RUNTIME_KEY",
|
|
api: "openai-completions",
|
|
models: [
|
|
{
|
|
id: "runtime-model",
|
|
name: "Runtime Model",
|
|
reasoning: false,
|
|
input: ["text"],
|
|
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
|
contextWindow: 128000,
|
|
maxTokens: 8192,
|
|
},
|
|
{
|
|
id: "runtime-reasoning-model",
|
|
name: "Runtime Reasoning Model",
|
|
reasoning: true,
|
|
input: ["text"],
|
|
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
|
contextWindow: 128000,
|
|
maxTokens: 8192,
|
|
},
|
|
],
|
|
});
|
|
};
|
|
|
|
async function buildSessionOptions(modelPattern: string) {
|
|
// Pass an explicit ModelRegistry so createAgentSession skips its implicit
|
|
// ModelRegistry.refreshInBackground() — a network model-discovery pass
|
|
// (~250ms/session) that contributes nothing here: the model resolves from
|
|
// the inline extension provider, never from network catalogs. Mirrors the
|
|
// explicit-registry pattern the resume tests below already rely on.
|
|
const authStorage = await AuthStorage.create(path.join(tempDir, "auth.db"));
|
|
authStoragesToClose.push(authStorage);
|
|
const modelRegistry = new ModelRegistry(authStorage, path.join(tempDir, "models.yml"));
|
|
return {
|
|
cwd: tempDir,
|
|
agentDir: tempDir,
|
|
authStorage,
|
|
modelRegistry,
|
|
sessionManager: SessionManager.inMemory(),
|
|
disableExtensionDiscovery: true,
|
|
extensions: [providerExtension],
|
|
skills: [],
|
|
contextFiles: [],
|
|
promptTemplates: [],
|
|
slashCommands: [],
|
|
enableMCP: false,
|
|
enableLsp: false,
|
|
modelPattern,
|
|
};
|
|
}
|
|
|
|
test("resolves explicit modelPattern after extension providers register", async () => {
|
|
const { session, modelFallbackMessage } = await createAgentSession(
|
|
await buildSessionOptions("runtime-provider/runtime-model"),
|
|
);
|
|
|
|
expect(session.model).toBeDefined();
|
|
expect(session.model?.provider).toBe("runtime-provider");
|
|
expect(session.model?.id).toBe("runtime-model");
|
|
expect(modelFallbackMessage).toBeUndefined();
|
|
});
|
|
|
|
test("does not silently fallback when explicit modelPattern is unresolved", async () => {
|
|
const { session, modelFallbackMessage } = await createAgentSession(
|
|
await buildSessionOptions("missing-provider/missing-model"),
|
|
);
|
|
|
|
expect(session.model).toBeUndefined();
|
|
expect(modelFallbackMessage).toBe('Model "missing-provider/missing-model" not found');
|
|
});
|
|
|
|
test("does not apply default role thinking override when modelPattern is explicit", async () => {
|
|
const settings = Settings.isolated({ defaultThinkingLevel: "off" });
|
|
settings.setModelRole("smol", "runtime-provider/runtime-reasoning-model");
|
|
settings.setModelRole("default", "pi/smol:high");
|
|
|
|
const { session } = await createAgentSession({
|
|
...(await buildSessionOptions("runtime-provider/runtime-reasoning-model")),
|
|
settings,
|
|
});
|
|
|
|
expect(session.model?.provider).toBe("runtime-provider");
|
|
expect(session.model?.id).toBe("runtime-reasoning-model");
|
|
expect(session.thinkingLevel).toBe("off");
|
|
});
|
|
|
|
test("normalizes max default thinking level from settings", async () => {
|
|
const settings = Settings.isolated({ defaultThinkingLevel: "max" });
|
|
|
|
const { session } = await createAgentSession({
|
|
...(await buildSessionOptions("runtime-provider/runtime-reasoning-model")),
|
|
settings,
|
|
});
|
|
|
|
expect(session.model?.provider).toBe("runtime-provider");
|
|
expect(session.model?.id).toBe("runtime-reasoning-model");
|
|
expect(session.thinkingLevel).toBe(Effort.XHigh);
|
|
});
|
|
|
|
test("selects the settings default model without synchronously validating auth", async () => {
|
|
const defaultModel = getBundledModel("anthropic", "claude-sonnet-4-5");
|
|
if (!defaultModel) {
|
|
throw new Error("Expected bundled anthropic default model");
|
|
}
|
|
|
|
const authStorage = await AuthStorage.create(path.join(tempDir, "testauth.db"));
|
|
authStorage.setRuntimeApiKey(defaultModel.provider, "test-key");
|
|
const modelRegistry = new ModelRegistry(authStorage, path.join(tempDir, "models.yml"));
|
|
const settings = Settings.isolated();
|
|
settings.setModelRole("default", `${defaultModel.provider}/${defaultModel.id}`);
|
|
|
|
const getApiKeySpy = vi
|
|
.spyOn(modelRegistry, "getApiKey")
|
|
.mockRejectedValue(new Error("settings default model should not validate auth during startup"));
|
|
|
|
try {
|
|
const { session } = await createAgentSession({
|
|
cwd: tempDir,
|
|
agentDir: tempDir,
|
|
authStorage,
|
|
modelRegistry,
|
|
settings,
|
|
sessionManager: SessionManager.inMemory(),
|
|
disableExtensionDiscovery: true,
|
|
skills: [],
|
|
contextFiles: [],
|
|
promptTemplates: [],
|
|
slashCommands: [],
|
|
enableMCP: false,
|
|
enableLsp: false,
|
|
});
|
|
|
|
try {
|
|
expect(session.model?.provider).toBe(defaultModel.provider);
|
|
expect(session.model?.id).toBe(defaultModel.id);
|
|
expect(getApiKeySpy).not.toHaveBeenCalled();
|
|
} finally {
|
|
await session.dispose();
|
|
}
|
|
} finally {
|
|
getApiKeySpy.mockRestore();
|
|
authStorage.close();
|
|
}
|
|
});
|
|
|
|
test("restores the saved session model without resolving auth over the network", async () => {
|
|
// Regression: `restoreSessionModel` probed each saved-model candidate with
|
|
// the async `getApiKey`, which refreshes OAuth tokens and hits the auth
|
|
// broker. When the broker was unreachable that blocked resume for the full
|
|
// ~10s refresh timeout — the "Still starting … restoreSessionModel" hang.
|
|
// Selection now uses the synchronous, side-effect-free `hasConfiguredAuth`
|
|
// probe; the real key is resolved lazily per request via the resolver.
|
|
const savedModel = getBundledModel("anthropic", "claude-sonnet-4-5");
|
|
if (!savedModel) {
|
|
throw new Error("Expected bundled anthropic default model");
|
|
}
|
|
|
|
const authStorage = await AuthStorage.create(path.join(tempDir, "resume-saved-auth.db"));
|
|
authStoragesToClose.push(authStorage);
|
|
authStorage.setRuntimeApiKey(savedModel.provider, "test-key");
|
|
const modelRegistry = new ModelRegistry(authStorage, path.join(tempDir, "models.yml"));
|
|
|
|
const targetSessionFile = path.join(tempDir, "resume-saved-model.jsonl");
|
|
const timestamp = "2026-06-01T00:00:00.000Z";
|
|
await Bun.write(
|
|
targetSessionFile,
|
|
`${[
|
|
{ type: "session", version: 3, id: "resume-saved", timestamp, cwd: tempDir },
|
|
{
|
|
type: "model_change",
|
|
id: "default-model",
|
|
parentId: null,
|
|
timestamp,
|
|
model: `${savedModel.provider}/${savedModel.id}`,
|
|
role: "default",
|
|
},
|
|
]
|
|
.map(entry => JSON.stringify(entry))
|
|
.join("\n")}\n`,
|
|
);
|
|
const sessionManager = await SessionManager.open(targetSessionFile, path.join(tempDir, "resume-saved-sessions"));
|
|
|
|
// A rejecting getApiKey stands in for the unreachable broker / hanging
|
|
// OAuth refresh: if startup awaits it to pick the restore model, it surfaces.
|
|
const getApiKeySpy = vi
|
|
.spyOn(modelRegistry, "getApiKey")
|
|
.mockRejectedValue(new Error("startup model restore must not resolve auth over the network"));
|
|
|
|
try {
|
|
const { session } = await createAgentSession({
|
|
cwd: tempDir,
|
|
agentDir: tempDir,
|
|
authStorage,
|
|
modelRegistry,
|
|
sessionManager,
|
|
settings: Settings.isolated(),
|
|
disableExtensionDiscovery: true,
|
|
skills: [],
|
|
contextFiles: [],
|
|
promptTemplates: [],
|
|
slashCommands: [],
|
|
enableMCP: false,
|
|
enableLsp: false,
|
|
skipPythonPreflight: true,
|
|
});
|
|
|
|
try {
|
|
expect(session.model?.provider).toBe(savedModel.provider);
|
|
expect(session.model?.id).toBe(savedModel.id);
|
|
expect(getApiKeySpy).not.toHaveBeenCalled();
|
|
} finally {
|
|
await session.dispose();
|
|
}
|
|
} finally {
|
|
getApiKeySpy.mockRestore();
|
|
}
|
|
});
|
|
|
|
test("prefers the provider default over catalog order in the startup fallback", async () => {
|
|
// Regression: with an Anthropic key but no configured `default` role and no
|
|
// session/CLI model, the step-4 startup fallback used to pick the first
|
|
// anthropic model in models.json catalog order (claude-3-5-sonnet-20240620)
|
|
// instead of the provider's configured default from DEFAULT_MODEL_PER_PROVIDER
|
|
// (claude-opus-4-8).
|
|
const providerDefault = getBundledModel("anthropic", "claude-opus-4-8");
|
|
const catalogFirst = getBundledModel("anthropic", "claude-3-5-sonnet-20240620");
|
|
if (!providerDefault || !catalogFirst) {
|
|
throw new Error("Expected bundled anthropic models for fallback regression");
|
|
}
|
|
|
|
const authStorage = await AuthStorage.create(path.join(tempDir, "fallbackauth.db"));
|
|
authStoragesToClose.push(authStorage);
|
|
authStorage.setRuntimeApiKey("anthropic", "test-key");
|
|
const modelRegistry = new ModelRegistry(authStorage, path.join(tempDir, "models.yml"));
|
|
// No `default` model role configured: forces the step-4 startup fallback.
|
|
const settings = Settings.isolated({ enabledModels: ["anthropic/*"] });
|
|
|
|
const { session } = await createAgentSession({
|
|
cwd: tempDir,
|
|
agentDir: tempDir,
|
|
authStorage,
|
|
modelRegistry,
|
|
settings,
|
|
sessionManager: SessionManager.inMemory(),
|
|
disableExtensionDiscovery: true,
|
|
skills: [],
|
|
contextFiles: [],
|
|
promptTemplates: [],
|
|
slashCommands: [],
|
|
enableMCP: false,
|
|
enableLsp: false,
|
|
skipPythonPreflight: true,
|
|
});
|
|
|
|
try {
|
|
expect(session.model?.provider).toBe("anthropic");
|
|
expect(session.model?.id).toBe(providerDefault.id);
|
|
expect(session.model?.id).not.toBe(catalogFirst.id);
|
|
} finally {
|
|
await session.dispose();
|
|
}
|
|
});
|
|
|
|
test("prefers Codex OAuth over plain OpenAI for the shared startup default", async () => {
|
|
const openaiDefault = getBundledModel("openai", "gpt-5.5");
|
|
const codexDefault = getBundledModel("openai-codex", "gpt-5.5");
|
|
if (!openaiDefault || !codexDefault) {
|
|
throw new Error("Expected bundled OpenAI and Codex GPT-5.5 defaults");
|
|
}
|
|
|
|
const authStorage = await AuthStorage.create(path.join(tempDir, "codex-fallback-auth.db"));
|
|
authStoragesToClose.push(authStorage);
|
|
authStorage.setRuntimeApiKey("openai", "sk-or-v1-invalid-openai-key");
|
|
authStorage.setRuntimeApiKey("openai-codex", "codex-oauth-token");
|
|
const modelRegistry = new ModelRegistry(authStorage, path.join(tempDir, "models.yml"));
|
|
|
|
const { session } = await createAgentSession({
|
|
cwd: tempDir,
|
|
agentDir: tempDir,
|
|
authStorage,
|
|
modelRegistry,
|
|
settings: Settings.isolated({ enabledModels: ["openai/gpt-5.5", "openai-codex/gpt-5.5"] }),
|
|
sessionManager: SessionManager.inMemory(),
|
|
disableExtensionDiscovery: true,
|
|
skills: [],
|
|
contextFiles: [],
|
|
promptTemplates: [],
|
|
slashCommands: [],
|
|
enableMCP: false,
|
|
enableLsp: false,
|
|
skipPythonPreflight: true,
|
|
});
|
|
|
|
try {
|
|
expect(session.model?.provider).toBe("openai-codex");
|
|
expect(session.model?.id).toBe(codexDefault.id);
|
|
expect(session.model?.id).toBe(openaiDefault.id);
|
|
} finally {
|
|
await session.dispose();
|
|
}
|
|
});
|
|
|
|
test("restores role model max selector from extension provider after startup resume", async () => {
|
|
const defaultModel = getBundledModel("anthropic", "claude-sonnet-4-5");
|
|
if (!defaultModel) {
|
|
throw new Error("Expected bundled anthropic default model");
|
|
}
|
|
|
|
const authStorage = await AuthStorage.create(path.join(tempDir, "testauth.db"));
|
|
authStorage.setRuntimeApiKey(defaultModel.provider, "test-key");
|
|
const modelRegistry = new ModelRegistry(authStorage, path.join(tempDir, "models.yml"));
|
|
|
|
const targetSessionFile = path.join(tempDir, "resume-extension.jsonl");
|
|
const timestamp = "2026-06-01T00:00:00.000Z";
|
|
await Bun.write(
|
|
targetSessionFile,
|
|
`${[
|
|
{ type: "session", version: 3, id: "resume-ext", timestamp, cwd: tempDir },
|
|
{
|
|
type: "model_change",
|
|
id: "default-model",
|
|
parentId: null,
|
|
timestamp,
|
|
model: `${defaultModel.provider}/${defaultModel.id}`,
|
|
role: "default",
|
|
},
|
|
{
|
|
type: "model_change",
|
|
id: "smol-model",
|
|
parentId: "default-model",
|
|
timestamp,
|
|
model: "runtime-provider/runtime-reasoning-model:max",
|
|
role: "smol",
|
|
},
|
|
]
|
|
.map(entry => JSON.stringify(entry))
|
|
.join("\n")}\n`,
|
|
);
|
|
const sessionManager = await SessionManager.open(targetSessionFile, path.join(tempDir, "sessions"));
|
|
|
|
const { session } = await createAgentSession({
|
|
cwd: tempDir,
|
|
agentDir: tempDir,
|
|
authStorage,
|
|
modelRegistry,
|
|
sessionManager,
|
|
settings: Settings.isolated(),
|
|
disableExtensionDiscovery: true,
|
|
extensions: [providerExtension],
|
|
skills: [],
|
|
contextFiles: [],
|
|
promptTemplates: [],
|
|
slashCommands: [],
|
|
enableMCP: false,
|
|
enableLsp: false,
|
|
skipPythonPreflight: true,
|
|
});
|
|
|
|
try {
|
|
expect(session.model?.provider).toBe("runtime-provider");
|
|
expect(session.model?.id).toBe("runtime-reasoning-model");
|
|
expect(session.thinkingLevel).toBe(Effort.XHigh);
|
|
} finally {
|
|
await session.dispose();
|
|
authStorage.close();
|
|
}
|
|
});
|
|
|
|
test("restores extension role model when saved default cannot be restored before extensions load", async () => {
|
|
const settingsDefaultModel = getBundledModel("anthropic", "claude-sonnet-4-5");
|
|
if (!settingsDefaultModel) {
|
|
throw new Error("Expected bundled anthropic default model");
|
|
}
|
|
|
|
const authStorage = await AuthStorage.create(path.join(tempDir, "testauth.db"));
|
|
authStorage.setRuntimeApiKey(settingsDefaultModel.provider, "test-key");
|
|
const modelRegistry = new ModelRegistry(authStorage, path.join(tempDir, "models.yml"));
|
|
|
|
// Saved default points at a provider that has no usable credentials. The
|
|
// last active role (`smol`) is supplied by the inline extension and is
|
|
// only resolvable once provider registrations are processed.
|
|
const targetSessionFile = path.join(tempDir, "resume-extension-default-missing.jsonl");
|
|
const timestamp = "2026-06-01T00:00:00.000Z";
|
|
await Bun.write(
|
|
targetSessionFile,
|
|
`${[
|
|
{ type: "session", version: 3, id: "resume-ext-no-default", timestamp, cwd: tempDir },
|
|
{
|
|
type: "model_change",
|
|
id: "default-model",
|
|
parentId: null,
|
|
timestamp,
|
|
model: "anthropic/not-available",
|
|
role: "default",
|
|
},
|
|
{
|
|
type: "model_change",
|
|
id: "smol-model",
|
|
parentId: "default-model",
|
|
timestamp,
|
|
model: "runtime-provider/runtime-model",
|
|
role: "smol",
|
|
},
|
|
]
|
|
.map(entry => JSON.stringify(entry))
|
|
.join("\n")}\n`,
|
|
);
|
|
const sessionManager = await SessionManager.open(targetSessionFile, path.join(tempDir, "sessions-no-default"));
|
|
|
|
const settings = Settings.isolated();
|
|
settings.setModelRole("default", `${settingsDefaultModel.provider}/${settingsDefaultModel.id}`);
|
|
|
|
const { session } = await createAgentSession({
|
|
cwd: tempDir,
|
|
agentDir: tempDir,
|
|
authStorage,
|
|
modelRegistry,
|
|
sessionManager,
|
|
settings,
|
|
disableExtensionDiscovery: true,
|
|
extensions: [providerExtension],
|
|
skills: [],
|
|
contextFiles: [],
|
|
promptTemplates: [],
|
|
slashCommands: [],
|
|
enableMCP: false,
|
|
enableLsp: false,
|
|
skipPythonPreflight: true,
|
|
});
|
|
|
|
try {
|
|
expect(session.model?.provider).toBe("runtime-provider");
|
|
expect(session.model?.id).toBe("runtime-model");
|
|
} finally {
|
|
await session.dispose();
|
|
authStorage.close();
|
|
}
|
|
});
|
|
});
|