From 09427086d99d7d9ce275f700af85f9f55db4aca5 Mon Sep 17 00:00:00 2001 From: can1357 Date: Sat, 10 Jan 2026 23:24:35 +0100 Subject: [PATCH] feat(coding-agent): migrated settings and auth storage from JSON files to SQLite database - Migrated settings and auth credential storage from JSON files (settings.json, auth.json) to SQLite database (agent.db). - Added AgentStorage class implementing unified SQLite storage with WAL mode, schema versioning, and prepared statement caching. - Added storage-migration module to handle one-time migration from legacy JSON files to SQLite with backup creation. - Updated credential migration message to reference agent.db instead of auth.json. - Removed file locking mechanism from AuthStorage since SQLite handles concurrency natively. - Added getAgentDbPath() config function returning path to SQLite database file. --- packages/coding-agent/CHANGELOG.md | 3 +- packages/coding-agent/src/config.ts | 9 + .../coding-agent/src/core/agent-storage.ts | 435 ++++++++++++++++++ .../coding-agent/src/core/auth-storage.ts | 283 ++++-------- packages/coding-agent/src/core/sdk.ts | 8 + .../coding-agent/src/core/settings-manager.ts | 84 ++-- .../src/core/storage-migration.ts | 205 +++++++++ .../src/core/tools/web-search/auth.ts | 103 ++++- .../tools/web-search/providers/anthropic.ts | 50 +- packages/coding-agent/src/main.ts | 2 +- packages/coding-agent/src/migrations.ts | 30 +- .../test/settings-manager.test.ts | 100 ++-- 12 files changed, 1001 insertions(+), 311 deletions(-) create mode 100644 packages/coding-agent/src/core/agent-storage.ts create mode 100644 packages/coding-agent/src/core/storage-migration.ts diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index f33fdad65..40bbdca61 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -1,13 +1,14 @@ # Changelog ## [Unreleased] - ### Added - Added support for custom share scripts at ~/.omp/agent/share.ts to replace default GitHub Gist sharing ### Changed +- Migrated settings and auth credential storage from JSON files to SQLite database (agent.db) +- Updated credential migration message to reference agent.db instead of auth.json - Renamed Glob tool references to Find tool throughout prompts and documentation - Updated project context formatting to use XML-style tags for clearer structure - Refined bash tool guidance to prefer dedicated tools (read/grep/find/ls) over bash for file operations diff --git a/packages/coding-agent/src/config.ts b/packages/coding-agent/src/config.ts index 75b5d871e..cf9fe8990 100644 --- a/packages/coding-agent/src/config.ts +++ b/packages/coding-agent/src/config.ts @@ -79,6 +79,15 @@ export function getSettingsPath(): string { return join(getAgentDir(), "settings.json"); } +/** + * Gets the path to agent.db (SQLite database for settings and auth storage). + * @param agentDir - Base agent directory, defaults to ~/.omp/agent + * @returns Absolute path to the agent.db file + */ +export function getAgentDbPath(agentDir: string = getAgentDir()): string { + return join(agentDir, "agent.db"); +} + /** Get path to tools directory */ export function getToolsDir(): string { return join(getAgentDir(), "tools"); diff --git a/packages/coding-agent/src/core/agent-storage.ts b/packages/coding-agent/src/core/agent-storage.ts new file mode 100644 index 000000000..c143fc7cf --- /dev/null +++ b/packages/coding-agent/src/core/agent-storage.ts @@ -0,0 +1,435 @@ +import { Database } from "bun:sqlite"; +import { mkdirSync } from "node:fs"; +import { dirname } from "node:path"; +import { getAgentDbPath } from "../config"; +import type { AuthCredential } from "./auth-storage"; +import { logger } from "./logger"; +import type { Settings } from "./settings-manager"; + +/** Prepared SQLite statement type from bun:sqlite */ +type Statement = ReturnType; + +/** Row shape for settings table queries */ +type SettingsRow = { + key: string; + value: string; +}; + +/** Row shape for auth_credentials table queries */ +type AuthRow = { + id: number; + provider: string; + credential_type: string; + data: string; +}; + +/** + * Auth credential with database row ID for updates/deletes. + * Wraps AuthCredential with storage metadata. + */ +export interface StoredAuthCredential { + id: number; + provider: string; + credential: AuthCredential; +} + +/** Bump when schema changes require migration */ +const SCHEMA_VERSION = 2; + +/** + * Type guard for plain objects. + * @param value - Value to check + * @returns True if value is a non-null, non-array object + */ +function isRecord(value: unknown): value is Record { + return !!value && typeof value === "object" && !Array.isArray(value); +} + +/** + * Converts credential to DB format, stripping the type discriminant from the data blob. + * @param credential - The credential to serialize + * @returns Object with credentialType and JSON data string, or null for unknown types + */ +function serializeCredential( + credential: AuthCredential, +): { credentialType: AuthCredential["type"]; data: string } | null { + if (credential.type === "api_key") { + return { + credentialType: "api_key", + data: JSON.stringify({ key: credential.key }), + }; + } + if (credential.type === "oauth") { + const { type: _type, ...rest } = credential; + return { + credentialType: "oauth", + data: JSON.stringify(rest), + }; + } + return null; +} + +/** + * Reconstructs credential from DB row, re-adding the type discriminant. + * @param row - Database row containing credential data + * @returns Reconstructed AuthCredential, or null if parsing fails or type is unknown + */ +function deserializeCredential(row: AuthRow): AuthCredential | null { + let parsed: unknown; + try { + parsed = JSON.parse(row.data); + } catch (error) { + logger.warn("AgentStorage failed to parse auth credential", { + provider: row.provider, + id: row.id, + error: String(error), + }); + return null; + } + if (!isRecord(parsed)) { + logger.warn("AgentStorage auth credential data invalid", { + provider: row.provider, + id: row.id, + }); + return null; + } + if (row.credential_type === "api_key") { + return { type: "api_key", ...(parsed as Record) } as AuthCredential; + } + if (row.credential_type === "oauth") { + return { type: "oauth", ...(parsed as Record) } as AuthCredential; + } + logger.warn("AgentStorage unknown credential type", { + provider: row.provider, + id: row.id, + type: row.credential_type, + }); + return null; +} + +/** + * Unified SQLite storage for agent settings and auth credentials. + * Uses singleton pattern per database path; access via AgentStorage.open(). + */ +export class AgentStorage { + private db: Database; + private static instances = new Map(); + + private listSettingsStmt: Statement; + private insertSettingStmt: Statement; + private deleteSettingsStmt: Statement; + private listAuthStmt: Statement; + private listAuthByProviderStmt: Statement; + private insertAuthStmt: Statement; + private updateAuthStmt: Statement; + private deleteAuthStmt: Statement; + private deleteAuthByProviderStmt: Statement; + private countAuthStmt: Statement; + + private constructor(dbPath: string) { + this.ensureDir(dbPath); + this.db = new Database(dbPath); + + this.initializeSchema(); + + this.listSettingsStmt = this.db.prepare("SELECT key, value FROM settings"); + this.insertSettingStmt = this.db.prepare( + "INSERT INTO settings (key, value, updated_at) VALUES (?, ?, unixepoch())", + ); + this.deleteSettingsStmt = this.db.prepare("DELETE FROM settings"); + + this.listAuthStmt = this.db.prepare( + "SELECT id, provider, credential_type, data FROM auth_credentials ORDER BY id ASC", + ); + this.listAuthByProviderStmt = this.db.prepare( + "SELECT id, provider, credential_type, data FROM auth_credentials WHERE provider = ? ORDER BY id ASC", + ); + this.insertAuthStmt = this.db.prepare( + "INSERT INTO auth_credentials (provider, credential_type, data) VALUES (?, ?, ?) RETURNING id", + ); + this.updateAuthStmt = this.db.prepare( + "UPDATE auth_credentials SET credential_type = ?, data = ?, updated_at = unixepoch() WHERE id = ?", + ); + this.deleteAuthStmt = this.db.prepare("DELETE FROM auth_credentials WHERE id = ?"); + this.deleteAuthByProviderStmt = this.db.prepare("DELETE FROM auth_credentials WHERE provider = ?"); + this.countAuthStmt = this.db.prepare("SELECT COUNT(*) as count FROM auth_credentials"); + } + + /** + * Creates tables if missing and migrates legacy single-blob settings to key-value format. + * Handles v1 to v2 schema migration for settings table. + */ + private initializeSchema(): void { + this.db.exec(` +PRAGMA journal_mode=WAL; +PRAGMA synchronous=NORMAL; + +CREATE TABLE IF NOT EXISTS auth_credentials ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + provider TEXT NOT NULL, + credential_type TEXT NOT NULL, + data TEXT NOT NULL, + created_at INTEGER NOT NULL DEFAULT (unixepoch()), + updated_at INTEGER NOT NULL DEFAULT (unixepoch()) +); +CREATE INDEX IF NOT EXISTS idx_auth_provider ON auth_credentials(provider); + +CREATE TABLE IF NOT EXISTS schema_version (version INTEGER PRIMARY KEY); +`); + + const settingsInfo = this.db.prepare("PRAGMA table_info(settings)").all() as Array<{ name?: string }>; + const hasSettingsTable = settingsInfo.length > 0; + const hasKey = settingsInfo.some((column) => column.name === "key"); + const hasValue = settingsInfo.some((column) => column.name === "value"); + + if (!hasSettingsTable) { + this.db.exec(` +CREATE TABLE settings ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL, + updated_at INTEGER NOT NULL DEFAULT (unixepoch()) +); +`); + } else if (!hasKey || !hasValue) { + // Migrate v1 schema: single JSON blob in `data` column → per-key rows + let legacySettings: Record | null = null; + const row = this.db.prepare("SELECT data FROM settings WHERE id = 1").get() as + | { data?: string } + | undefined; + if (row?.data) { + try { + const parsed = JSON.parse(row.data); + if (isRecord(parsed)) { + legacySettings = parsed; + } else { + logger.warn("AgentStorage legacy settings invalid shape"); + } + } catch (error) { + logger.warn("AgentStorage failed to parse legacy settings", { error: String(error) }); + } + } + + const migrate = this.db.transaction((settings: Record | null) => { + this.db.exec("DROP TABLE settings"); + this.db.exec(` +CREATE TABLE settings ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL, + updated_at INTEGER NOT NULL DEFAULT (unixepoch()) +); +`); + if (settings) { + const insert = this.db.prepare( + "INSERT INTO settings (key, value, updated_at) VALUES (?, ?, unixepoch())", + ); + for (const [key, value] of Object.entries(settings)) { + if (value === undefined) continue; + const serialized = JSON.stringify(value); + if (serialized === undefined) continue; + insert.run(key, serialized); + } + } + }); + + migrate(legacySettings); + } + + const versionRow = this.db.prepare("SELECT version FROM schema_version ORDER BY version DESC LIMIT 1").get() as + | { version?: number } + | undefined; + if (versionRow?.version !== undefined && versionRow.version !== SCHEMA_VERSION) { + logger.warn("AgentStorage schema version mismatch", { + current: versionRow.version, + expected: SCHEMA_VERSION, + }); + } + this.db.prepare("INSERT OR REPLACE INTO schema_version(version) VALUES (?)").run(SCHEMA_VERSION); + } + + /** + * Returns singleton instance for the given database path, creating if needed. + * @param dbPath - Path to the SQLite database file (defaults to config path) + * @returns AgentStorage instance for the given path + */ + static open(dbPath: string = getAgentDbPath()): AgentStorage { + const existing = AgentStorage.instances.get(dbPath); + if (existing) return existing; + const storage = new AgentStorage(dbPath); + AgentStorage.instances.set(dbPath, storage); + return storage; + } + + /** + * Retrieves all settings from storage. + * @returns Settings object, or null if no settings are stored + */ + getSettings(): Settings | null { + const rows = (this.listSettingsStmt.all() as SettingsRow[]) ?? []; + if (rows.length === 0) return null; + const settings: Record = {}; + for (const row of rows) { + try { + settings[row.key] = JSON.parse(row.value) as unknown; + } catch (error) { + logger.warn("AgentStorage failed to parse setting", { + key: row.key, + error: String(error), + }); + } + } + return settings as Settings; + } + + /** + * Atomically replaces all settings in storage. + * Uses delete-then-insert within a transaction for consistency. + * @param settings - Settings object to persist + */ + saveSettings(settings: Settings): void { + const entries = Object.entries(settings).filter(([, value]) => value !== undefined); + const replace = this.db.transaction((rows: Array<[string, unknown]>) => { + this.deleteSettingsStmt.run(); + for (const [key, value] of rows) { + const serialized = JSON.stringify(value); + if (serialized === undefined) continue; + this.insertSettingStmt.run(key, serialized); + } + }); + + try { + replace(entries); + } catch (error) { + logger.error("AgentStorage failed to save settings", { error: String(error) }); + } + } + + /** + * Checks if any auth credentials exist in storage. + * @returns True if at least one credential is stored + */ + hasAuthCredentials(): boolean { + const row = this.countAuthStmt.get() as { count?: number } | undefined; + return (row?.count ?? 0) > 0; + } + + /** + * Lists auth credentials, optionally filtered by provider. + * @param provider - Optional provider name to filter by + * @returns Array of stored credentials with their database IDs + */ + listAuthCredentials(provider?: string): StoredAuthCredential[] { + const rows = + (provider + ? (this.listAuthByProviderStmt.all(provider) as AuthRow[]) + : (this.listAuthStmt.all() as AuthRow[])) ?? []; + + const results: StoredAuthCredential[] = []; + for (const row of rows) { + const credential = deserializeCredential(row); + if (!credential) continue; + results.push({ id: row.id, provider: row.provider, credential }); + } + return results; + } + + /** + * Atomically replaces all credentials for a provider. + * Useful for OAuth token refresh where old tokens should be discarded. + * @param provider - Provider name (e.g., "anthropic", "openai") + * @param credentials - New credentials to store + * @returns Array of newly stored credentials with their database IDs + */ + replaceAuthCredentialsForProvider(provider: string, credentials: AuthCredential[]): StoredAuthCredential[] { + const replace = this.db.transaction((providerName: string, items: AuthCredential[]) => { + this.deleteAuthByProviderStmt.run(providerName); + const inserted: StoredAuthCredential[] = []; + for (const credential of items) { + const record = this.insertAuthCredential(providerName, credential); + if (record) inserted.push(record); + } + return inserted; + }); + + return replace(provider, credentials); + } + + /** + * Updates an existing auth credential by ID. + * @param id - Database row ID of the credential to update + * @param credential - New credential data + */ + updateAuthCredential(id: number, credential: AuthCredential): void { + const serialized = serializeCredential(credential); + if (!serialized) { + logger.warn("AgentStorage updateAuthCredential invalid type", { id, type: credential.type }); + return; + } + try { + this.updateAuthStmt.run(serialized.credentialType, serialized.data, id); + } catch (error) { + logger.warn("AgentStorage updateAuthCredential failed", { id, error: String(error) }); + } + } + + /** + * Deletes an auth credential by ID. + * @param id - Database row ID of the credential to delete + */ + deleteAuthCredential(id: number): void { + try { + this.deleteAuthStmt.run(id); + } catch (error) { + logger.warn("AgentStorage deleteAuthCredential failed", { id, error: String(error) }); + } + } + + /** + * Deletes all auth credentials for a provider. + * @param provider - Provider name whose credentials should be deleted + */ + deleteAuthCredentialsForProvider(provider: string): void { + try { + this.deleteAuthByProviderStmt.run(provider); + } catch (error) { + logger.warn("AgentStorage deleteAuthCredentialsForProvider failed", { + provider, + error: String(error), + }); + } + } + + /** + * Inserts a new auth credential for a provider. + * @param provider - Provider name (e.g., "anthropic", "openai") + * @param credential - Credential to insert + * @returns Stored credential with database ID, or null on failure + */ + private insertAuthCredential(provider: string, credential: AuthCredential): StoredAuthCredential | null { + const serialized = serializeCredential(credential); + if (!serialized) { + logger.warn("AgentStorage insertAuthCredential invalid type", { provider, type: credential.type }); + return null; + } + try { + const row = this.insertAuthStmt.get(provider, serialized.credentialType, serialized.data) as + | { id?: number } + | undefined; + if (!row?.id) { + logger.warn("AgentStorage insertAuthCredential missing id", { provider }); + return null; + } + return { id: row.id, provider, credential }; + } catch (error) { + logger.warn("AgentStorage insertAuthCredential failed", { provider, error: String(error) }); + return null; + } + } + + /** + * Ensures the parent directory for the database file exists. + * @param dbPath - Path to the database file + */ + private ensureDir(dbPath: string): void { + mkdirSync(dirname(dbPath), { recursive: true }); + } +} diff --git a/packages/coding-agent/src/core/auth-storage.ts b/packages/coding-agent/src/core/auth-storage.ts index b7fcdbf9b..915da99ee 100644 --- a/packages/coding-agent/src/core/auth-storage.ts +++ b/packages/coding-agent/src/core/auth-storage.ts @@ -1,23 +1,9 @@ /** * Credential storage for API keys and OAuth tokens. - * Handles loading, saving, and refreshing credentials from auth.json. - * - * Uses file locking to prevent race conditions when multiple pi instances - * try to refresh tokens simultaneously. + * Handles loading, saving, and refreshing credentials from agent.db. */ -import { - chmodSync, - closeSync, - existsSync, - openSync, - readFileSync, - renameSync, - statSync, - unlinkSync, - writeFileSync, -} from "node:fs"; -import { dirname } from "node:path"; +import { dirname, join } from "node:path"; import { getEnvApiKey, getOAuthApiKey, @@ -29,7 +15,10 @@ import { type OAuthCredentials, type OAuthProvider, } from "@oh-my-pi/pi-ai"; +import { getAgentDbPath } from "../config"; +import { AgentStorage } from "./agent-storage"; import { logger } from "./logger"; +import { migrateJsonStorage } from "./storage-migration"; export type ApiKeyCredential = { type: "api_key"; @@ -46,6 +35,12 @@ export type AuthCredentialEntry = AuthCredential | AuthCredential[]; export type AuthStorageData = Record; +/** + * In-memory representation pairing DB row ID with credential. + * The ID is required for update/delete operations against agent.db. + */ +type StoredCredential = { id: number; credential: AuthCredential }; + /** Rate limit window from Codex usage API (primary or secondary quota). */ type CodexUsageWindow = { usedPercent?: number; @@ -86,18 +81,18 @@ function toBoolean(value: unknown): boolean | undefined { } /** - * Credential storage backed by a JSON file. - * Reads from multiple fallback paths, writes to primary path. + * Credential storage backed by agent.db. + * Reads from SQLite and migrates legacy auth.json paths. */ export class AuthStorage { - // File locking configuration for concurrent access protection - private static readonly lockRetryDelayMs = 50; // Polling interval when waiting for lock - private static readonly lockTimeoutMs = 5000; // Max wait time before failing - private static readonly lockStaleMs = 30000; // Age threshold for auto-removing orphaned locks private static readonly codexUsageCacheTtlMs = 60_000; // Cache usage data for 1 minute private static readonly defaultBackoffMs = 60_000; // Default backoff when no reset time available - private data: AuthStorageData = {}; + /** Provider -> credentials cache, populated from agent.db on reload(). */ + private data: Map = new Map(); + private storage: AgentStorage; + /** Resolved path to agent.db (derived from authPath or used directly if .db). */ + private dbPath: string; private runtimeOverrides: Map = new Map(); /** Tracks next credential index per provider:type key for round-robin distribution (non-session use). */ private providerRoundRobinIndex: Map = new Map(); @@ -110,13 +105,28 @@ export class AuthStorage { private fallbackResolver?: (provider: string) => string | undefined; /** - * @param authPath - Primary path for reading/writing auth.json - * @param fallbackPaths - Additional paths to check when reading (legacy support) + * @param authPath - Legacy auth.json path used for migration and locating agent.db + * @param fallbackPaths - Additional auth.json paths to migrate (legacy support) */ constructor( private authPath: string, private fallbackPaths: string[] = [], - ) {} + ) { + this.dbPath = AuthStorage.resolveDbPath(authPath); + this.storage = AgentStorage.open(this.dbPath); + } + + /** + * Converts legacy auth.json path to agent.db path, or returns .db path as-is. + * @param authPath - Path to auth.json or agent.db + * @returns Resolved path to agent.db + */ + private static resolveDbPath(authPath: string): string { + if (authPath.endsWith(".db")) { + return authPath; + } + return getAgentDbPath(dirname(authPath)); + } /** * Set a runtime API key override (not persisted to disk). @@ -134,7 +144,7 @@ export class AuthStorage { } /** - * Set a fallback resolver for API keys not found in auth.json or env vars. + * Set a fallback resolver for API keys not found in agent.db or env vars. * Used for custom provider keys from models.json. */ setFallbackResolver(resolver: (provider: string) => string | undefined): void { @@ -142,138 +152,53 @@ export class AuthStorage { } /** - * Reload credentials from disk. - * Checks primary path first, then fallback paths. + * Reload credentials from agent.db. + * Migrates legacy auth.json/settings.json on first load. */ async reload(): Promise { - const pathsToCheck = [this.authPath, ...this.fallbackPaths]; + const agentDir = dirname(this.dbPath); + await migrateJsonStorage({ + agentDir, + settingsPath: join(agentDir, "settings.json"), + authPaths: [this.authPath, ...this.fallbackPaths], + }); - logger.debug("AuthStorage.reload checking paths", { paths: pathsToCheck }); - - for (const authPath of pathsToCheck) { - const exists = existsSync(authPath); - logger.debug("AuthStorage.reload path check", { path: authPath, exists }); - - if (exists) { - try { - this.data = JSON.parse(readFileSync(authPath, "utf-8")); - logger.debug("AuthStorage.reload loaded", { path: authPath, providers: Object.keys(this.data) }); - return; - } catch (e) { - logger.error("AuthStorage failed to parse auth file", { path: authPath, error: String(e) }); - // Continue to next path on parse error - } - } + const records = this.storage.listAuthCredentials(); + const grouped = new Map(); + for (const record of records) { + const list = grouped.get(record.provider) ?? []; + list.push({ id: record.id, credential: record.credential }); + grouped.set(record.provider, list); } - - logger.warn("AuthStorage no auth file found", { checkedPaths: pathsToCheck }); - this.data = {}; + this.data = grouped; } /** - * Save credentials to disk. + * Gets cached credentials for a provider. + * @param provider - Provider name (e.g., "anthropic", "openai") + * @returns Array of stored credentials, empty if none exist */ - private async save(): Promise { - const lockFd = await this.acquireLock(); - const tempPath = this.getTempPath(); - - try { - writeFileSync(tempPath, JSON.stringify(this.data, null, 2), { mode: 0o600 }); - renameSync(tempPath, this.authPath); - chmodSync(this.authPath, 0o600); - const dir = dirname(this.authPath); - chmodSync(dir, 0o700); - } finally { - this.safeUnlink(tempPath); - this.releaseLock(lockFd); - } - } - - /** Returns the lock file path (auth.json.lock) */ - private getLockPath(): string { - return `${this.authPath}.lock`; - } - - /** Returns a unique temp file path using pid and timestamp to avoid collisions */ - private getTempPath(): string { - return `${this.authPath}.tmp-${process.pid}-${Date.now()}`; - } - - /** Checks if lock file is older than lockStaleMs (orphaned by crashed process) */ - private isLockStale(lockPath: string): boolean { - try { - const stats = statSync(lockPath); - return Date.now() - stats.mtimeMs > AuthStorage.lockStaleMs; - } catch { - return false; - } + private getStoredCredentials(provider: string): StoredCredential[] { + return this.data.get(provider) ?? []; } /** - * Acquires exclusive file lock using O_EXCL atomic create. - * Polls with exponential backoff, removes stale locks from crashed processes. - * @returns File descriptor for the lock (must be passed to releaseLock) + * Updates in-memory credential cache for a provider. + * Removes the provider entry entirely if credentials array is empty. + * @param provider - Provider name (e.g., "anthropic", "openai") + * @param credentials - Array of stored credentials to cache */ - private async acquireLock(): Promise { - const lockPath = this.getLockPath(); - const start = Date.now(); - const timeoutMs = AuthStorage.lockTimeoutMs; - const retryDelayMs = AuthStorage.lockRetryDelayMs; - - while (true) { - try { - // O_EXCL fails if file exists, providing atomic lock acquisition - return openSync(lockPath, "wx", 0o600); - } catch (error) { - const err = error as NodeJS.ErrnoException; - if (err.code !== "EEXIST") { - throw err; - } - if (this.isLockStale(lockPath)) { - this.safeUnlink(lockPath); - logger.warn("AuthStorage lock was stale, removing", { path: lockPath }); - continue; - } - if (Date.now() - start > timeoutMs) { - throw new Error(`Timed out waiting for auth lock: ${lockPath}`); - } - await new Promise((resolve) => setTimeout(resolve, retryDelayMs)); - } + private setStoredCredentials(provider: string, credentials: StoredCredential[]): void { + if (credentials.length === 0) { + this.data.delete(provider); + } else { + this.data.set(provider, credentials); } } - /** Releases file lock by closing fd and removing lock file */ - private releaseLock(lockFd: number): void { - const lockPath = this.getLockPath(); - try { - closeSync(lockFd); - } catch (error) { - logger.warn("AuthStorage failed to close lock file", { error: String(error) }); - } - this.safeUnlink(lockPath); - } - - /** Removes file if it exists, ignoring ENOENT errors */ - private safeUnlink(path: string): void { - try { - unlinkSync(path); - } catch (error) { - const err = error as NodeJS.ErrnoException; - if (err.code !== "ENOENT") { - logger.warn("AuthStorage failed to remove file", { path, error: String(error) }); - } - } - } - - /** Normalizes credential storage format: single credential becomes array of one */ - private normalizeCredentialEntry(entry: AuthCredentialEntry | undefined): AuthCredential[] { - if (!entry) return []; - return Array.isArray(entry) ? entry : [entry]; - } - /** Returns all credentials for a provider as an array */ private getCredentialsForProvider(provider: string): AuthCredential[] { - return this.normalizeCredentialEntry(this.data[provider]); + return this.getStoredCredentials(provider).map((entry) => entry.credential); } /** Composite key for round-robin tracking: "anthropic:oauth" or "openai:api_key" */ @@ -423,21 +348,13 @@ export class AuthStorage { /** Updates credential at index in-place (used for OAuth token refresh) */ private replaceCredentialAt(provider: string, index: number, credential: AuthCredential): void { - const entry = this.data[provider]; - if (!entry) return; - - if (Array.isArray(entry)) { - if (index >= 0 && index < entry.length) { - const updated = [...entry]; - updated[index] = credential; - this.data[provider] = updated; - } - return; - } - - if (index === 0) { - this.data[provider] = credential; - } + const entries = this.getStoredCredentials(provider); + if (index < 0 || index >= entries.length) return; + const target = entries[index]; + this.storage.updateAuthCredential(target.id, credential); + const updated = [...entries]; + updated[index] = { id: target.id, credential }; + this.setStoredCredentials(provider, updated); } /** @@ -445,20 +362,11 @@ export class AuthStorage { * Cleans up provider entry if last credential removed. */ private removeCredentialAt(provider: string, index: number): void { - const entry = this.data[provider]; - if (!entry) return; - - if (Array.isArray(entry)) { - const updated = entry.filter((_value, idx) => idx !== index); - if (updated.length > 0) { - this.data[provider] = updated; - } else { - delete this.data[provider]; - } - } else { - delete this.data[provider]; - } - + const entries = this.getStoredCredentials(provider); + if (index < 0 || index >= entries.length) return; + this.storage.deleteAuthCredential(entries[index].id); + const updated = entries.filter((_value, idx) => idx !== index); + this.setStoredCredentials(provider, updated); this.resetProviderAssignments(provider); } @@ -473,29 +381,33 @@ export class AuthStorage { * Set credential for a provider. */ async set(provider: string, credential: AuthCredentialEntry): Promise { - this.data[provider] = credential; + const normalized = Array.isArray(credential) ? credential : [credential]; + const stored = this.storage.replaceAuthCredentialsForProvider(provider, normalized); + this.setStoredCredentials( + provider, + stored.map((record) => ({ id: record.id, credential: record.credential })), + ); this.resetProviderAssignments(provider); - await this.save(); } /** * Remove credential for a provider. */ async remove(provider: string): Promise { - delete this.data[provider]; + this.storage.deleteAuthCredentialsForProvider(provider); + this.data.delete(provider); this.resetProviderAssignments(provider); - await this.save(); } /** * List all providers with credentials. */ list(): string[] { - return Object.keys(this.data); + return [...this.data.keys()]; } /** - * Check if credentials exist for a provider in auth.json. + * Check if credentials exist for a provider in agent.db. */ has(provider: string): boolean { return this.getCredentialsForProvider(provider).length > 0; @@ -533,7 +445,16 @@ export class AuthStorage { * Get all credentials. */ getAll(): AuthStorageData { - return { ...this.data }; + const result: AuthStorageData = {}; + for (const [provider, entries] of this.data.entries()) { + const credentials = entries.map((entry) => entry.credential); + if (credentials.length === 1) { + result[provider] = credentials[0]; + } else if (credentials.length > 1) { + result[provider] = credentials; + } + } + return result; } /** @@ -887,7 +808,6 @@ export class AuthStorage { const updated: OAuthCredential = { type: "oauth", ...result.newCredentials }; this.replaceCredentialAt(provider, selection.index, updated); - await this.save(); if (checkUsage) { const usage = await this.getCodexUsage(updated, options?.baseUrl); @@ -906,7 +826,6 @@ export class AuthStorage { return result.apiKey; } catch { this.removeCredentialAt(provider, selection.index); - await this.save(); if (this.getCredentialsForProvider(provider).some((credential) => credential.type === "oauth")) { return this.getApiKey(provider, sessionId, options); } @@ -919,8 +838,8 @@ export class AuthStorage { * Get API key for a provider. * Priority: * 1. Runtime override (CLI --api-key) - * 2. API key from auth.json - * 3. OAuth token from auth.json (auto-refreshed) + * 2. API key from agent.db + * 3. OAuth token from agent.db (auto-refreshed) * 4. Environment variable * 5. Fallback resolver (models.json custom providers) */ diff --git a/packages/coding-agent/src/core/sdk.ts b/packages/coding-agent/src/core/sdk.ts index 62528e05a..8639d371b 100644 --- a/packages/coding-agent/src/core/sdk.ts +++ b/packages/coding-agent/src/core/sdk.ts @@ -71,6 +71,7 @@ import { loadSkills as loadSkillsInternal, type Skill, type SkillWarning } from import { type FileSlashCommand, loadSlashCommands as loadSlashCommandsInternal } from "./slash-commands"; import { closeAllConnections } from "./ssh/connection-manager"; import { unmountAll } from "./ssh/sshfs-mount"; +import { migrateJsonStorage } from "./storage-migration"; import { buildSystemPrompt as buildSystemPromptInternal, loadProjectContextFiles as loadContextFilesInternal, @@ -242,6 +243,13 @@ export async function discoverAuthStorage(agentDir: string = getDefaultAgentDir( logger.debug("discoverAuthStorage", { agentDir, primaryPath, allPaths, fallbackPaths }); + // Migrate legacy JSON files (settings.json, auth.json) to SQLite before loading + await migrateJsonStorage({ + agentDir, + settingsPath: join(agentDir, "settings.json"), + authPaths: [primaryPath, ...fallbackPaths], + }); + const storage = new AuthStorage(primaryPath, fallbackPaths); await storage.reload(); return storage; diff --git a/packages/coding-agent/src/core/settings-manager.ts b/packages/coding-agent/src/core/settings-manager.ts index f4513920e..38df938ec 100644 --- a/packages/coding-agent/src/core/settings-manager.ts +++ b/packages/coding-agent/src/core/settings-manager.ts @@ -1,9 +1,9 @@ -import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; -import { dirname, join } from "node:path"; import { type Settings as SettingsItem, settingsCapability } from "../capability/settings"; -import { getAgentDir } from "../config"; +import { getAgentDbPath, getAgentDir } from "../config"; import { loadSync } from "../discovery"; import type { SymbolPreset } from "../modes/interactive/theme/theme"; +import { AgentStorage } from "./agent-storage"; +import { logger } from "./logger"; export interface CompactionSettings { enabled?: boolean; // default: true @@ -377,15 +377,23 @@ function deepMergeSettings(base: Settings, overrides: Settings): Settings { } export class SettingsManager { - private settingsPath: string | null; + /** SQLite storage for persisted settings (null for in-memory mode) */ + private storage: AgentStorage | null; private cwd: string | null; private globalSettings: Settings; private overrides: Settings; private settings!: Settings; private persist: boolean; - private constructor(settingsPath: string | null, cwd: string | null, initialSettings: Settings, persist: boolean) { - this.settingsPath = settingsPath; + /** + * Private constructor - use static factory methods instead. + * @param storage - SQLite storage instance for persistence, or null for in-memory mode + * @param cwd - Current working directory for project settings discovery + * @param initialSettings - Initial global settings to use + * @param persist - Whether to persist settings changes to storage + */ + private constructor(storage: AgentStorage | null, cwd: string | null, initialSettings: Settings, persist: boolean) { + this.storage = storage; this.cwd = cwd; this.persist = persist; this.globalSettings = initialSettings; @@ -416,9 +424,14 @@ export class SettingsManager { } } - /** Create a SettingsManager that loads from files */ + /** + * Create a SettingsManager that loads from persistent SQLite storage. + * @param cwd - Current working directory for project settings discovery + * @param agentDir - Agent directory containing agent.db + * @returns Configured SettingsManager with merged global and user settings + */ static create(cwd: string = process.cwd(), agentDir: string = getAgentDir()): SettingsManager { - const settingsPath = join(agentDir, "settings.json"); + const storage = AgentStorage.open(getAgentDbPath(agentDir)); // Use capability API to load user-level settings from all providers const result = loadSync(settingsCapability.id, { cwd }); @@ -431,30 +444,36 @@ export class SettingsManager { } } - // Also load from agentDir for backward compatibility (if not covered by providers) - const legacySettings = SettingsManager.loadFromFile(settingsPath); - globalSettings = deepMergeSettings(globalSettings, legacySettings); + // Load persisted settings from agent.db (legacy settings.json is migrated separately) + const storedSettings = SettingsManager.loadFromStorage(storage); + globalSettings = deepMergeSettings(globalSettings, storedSettings); - return new SettingsManager(settingsPath, cwd, globalSettings, true); + return new SettingsManager(storage, cwd, globalSettings, true); } - /** Create an in-memory SettingsManager (no file I/O) */ + /** + * Create an in-memory SettingsManager without persistence. + * @param settings - Initial settings to use + * @returns SettingsManager that won't persist changes to disk + */ static inMemory(settings: Partial = {}): SettingsManager { return new SettingsManager(null, null, settings, false); } - private static loadFromFile(path: string): Settings { - if (!existsSync(path)) { + /** + * Load settings from SQLite storage, applying any schema migrations. + * @param storage - AgentStorage instance, or null for in-memory mode + * @returns Parsed and migrated settings, or empty object if storage is null/empty + */ + private static loadFromStorage(storage: AgentStorage | null): Settings { + if (!storage) { return {}; } - try { - const content = readFileSync(path, "utf-8"); - const settings = JSON.parse(content); - return SettingsManager.migrateSettings(settings as Record); - } catch (error) { - console.error(`Warning: Could not read settings file ${path}: ${error}`); + const settings = storage.getSettings(); + if (!settings) { return {}; } + return SettingsManager.migrateSettings(settings as Record); } /** Migrate old settings format to new format */ @@ -497,24 +516,19 @@ export class SettingsManager { this.rebuildSettings(); } + /** + * Persist current global settings to SQLite storage and rebuild merged settings. + * Merges with any concurrent changes in storage before saving. + */ private save(): void { - if (this.persist && this.settingsPath) { + if (this.persist && this.storage) { try { - const dir = dirname(this.settingsPath); - if (!existsSync(dir)) { - mkdirSync(dir, { recursive: true }); - } - - // Re-read current file to preserve any settings added externally while running - const currentFileSettings = SettingsManager.loadFromFile(this.settingsPath); - // Merge: file settings as base, globalSettings (in-memory changes) as overrides - const mergedSettings = deepMergeSettings(currentFileSettings, this.globalSettings); + const currentSettings = this.storage.getSettings() ?? {}; + const mergedSettings = deepMergeSettings(currentSettings, this.globalSettings); this.globalSettings = mergedSettings; - - // Save merged settings (project settings are read-only) - writeFileSync(this.settingsPath, JSON.stringify(this.globalSettings, null, 2), "utf-8"); + this.storage.saveSettings(this.globalSettings); } catch (error) { - console.error(`Warning: Could not save settings file: ${error}`); + logger.warn("SettingsManager save failed", { error: String(error) }); } } diff --git a/packages/coding-agent/src/core/storage-migration.ts b/packages/coding-agent/src/core/storage-migration.ts new file mode 100644 index 000000000..e0dc67983 --- /dev/null +++ b/packages/coding-agent/src/core/storage-migration.ts @@ -0,0 +1,205 @@ +/** + * Migrates legacy JSON storage (settings.json, auth.json) to SQLite-based agent.db. + * Runs once on startup; skips migration if agent.db already has data (DB is authoritative). + * Original JSON files are backed up to .bak and removed after successful migration. + */ + +import { getAgentDbPath } from "../config"; +import { AgentStorage } from "./agent-storage"; +import type { AuthCredential, AuthCredentialEntry, AuthStorageData } from "./auth-storage"; +import { logger } from "./logger"; +import type { Settings } from "./settings-manager"; + +/** Paths configuration for the storage migration process. */ +type MigrationPaths = { + /** Directory containing agent.db */ + agentDir: string; + /** Path to legacy settings.json file */ + settingsPath: string; + /** Candidate paths to search for auth.json (checked in order) */ + authPaths: string[]; +}; + +/** Result of the JSON-to-SQLite storage migration. */ +export interface StorageMigrationResult { + /** Whether settings.json was migrated to agent.db */ + migratedSettings: boolean; + /** Whether auth.json was migrated to agent.db */ + migratedAuth: boolean; + /** Non-fatal issues encountered during migration */ + warnings: string[]; +} + +/** + * Type guard for plain objects. + * @param value - Value to check + * @returns True if value is a non-null, non-array object + */ +function isRecord(value: unknown): value is Record { + return !!value && typeof value === "object" && !Array.isArray(value); +} + +/** + * Transforms legacy settings to current schema (e.g., queueMode -> steeringMode). + * @param settings - Settings object potentially containing deprecated keys + * @returns Settings with deprecated keys renamed to current equivalents + */ +function migrateLegacySettings(settings: Settings): Settings { + const migrated = { ...settings } as Record; + if ("queueMode" in migrated && !("steeringMode" in migrated)) { + migrated.steeringMode = migrated.queueMode; + delete migrated.queueMode; + } + return migrated as Settings; +} + +/** + * Normalizes credential entries to array format (legacy stored single credentials). + * @param entry - Single credential or array of credentials + * @returns Array of credentials (empty if entry is undefined) + */ +function normalizeCredentialEntry(entry: AuthCredentialEntry | undefined): AuthCredential[] { + if (!entry) return []; + return Array.isArray(entry) ? entry : [entry]; +} + +/** + * Reads and parses a JSON file. + * @param path - Path to the JSON file + * @returns Parsed JSON content, or null if file doesn't exist or parsing fails + */ +async function readJsonFile(path: string): Promise { + try { + const file = Bun.file(path); + if (!(await file.exists())) return null; + const content = await file.text(); + return JSON.parse(content) as T; + } catch (error) { + logger.warn("Storage migration failed to read JSON", { path, error: String(error) }); + return null; + } +} + +/** + * Backs up a JSON file to .bak and removes the original. + * Prevents re-migration on subsequent runs. + * @param path - Path to the JSON file to backup + */ +async function backupJson(path: string): Promise { + const file = Bun.file(path); + if (!(await file.exists())) return; + + const backupPath = `${path}.bak`; + try { + const content = await file.arrayBuffer(); + await Bun.write(backupPath, content); + await file.unlink(); + } catch (error) { + logger.warn("Storage migration failed to backup JSON", { path, error: String(error) }); + } +} + +/** + * Migrates settings.json to SQLite storage if DB is empty. + * @param storage - AgentStorage instance to migrate into + * @param settingsPath - Path to legacy settings.json + * @param warnings - Array to collect non-fatal warnings + * @returns True if migration was performed + */ +async function migrateSettings(storage: AgentStorage, settingsPath: string, warnings: string[]): Promise { + const settingsFile = Bun.file(settingsPath); + const settingsExists = await settingsFile.exists(); + const hasDbSettings = storage.getSettings() !== null; + + if (!settingsExists) return false; + if (hasDbSettings) { + warnings.push(`settings.json exists but agent.db is authoritative: ${settingsPath}`); + return false; + } + + const settingsJson = await readJsonFile(settingsPath); + if (!settingsJson) return false; + + storage.saveSettings(migrateLegacySettings(settingsJson)); + await backupJson(settingsPath); + return true; +} + +/** + * Finds the first valid auth.json from candidate paths (checked in priority order). + * @param authPaths - Candidate paths to search (e.g., project-local before global) + * @returns First valid auth file with its path and parsed data, or null if none found + */ +async function findFirstAuthJson(authPaths: string[]): Promise<{ path: string; data: AuthStorageData } | null> { + for (const authPath of authPaths) { + const data = await readJsonFile(authPath); + if (data && isRecord(data)) { + return { path: authPath, data }; + } + } + return null; +} + +/** + * Validates that a credential has a recognized type. + * @param entry - Credential to validate + * @returns True if credential type is api_key or oauth + */ +function isValidCredential(entry: AuthCredential): boolean { + return entry.type === "api_key" || entry.type === "oauth"; +} + +/** + * Migrates auth.json to SQLite storage if DB has no credentials. + * @param storage - AgentStorage instance to migrate into + * @param authPaths - Candidate paths to search for auth.json + * @param warnings - Array to collect non-fatal warnings + * @returns True if migration was performed + */ +async function migrateAuth(storage: AgentStorage, authPaths: string[], warnings: string[]): Promise { + const hasDbAuth = storage.hasAuthCredentials(); + const authJson = await findFirstAuthJson(authPaths); + + if (!authJson) return false; + if (hasDbAuth) { + warnings.push(`auth.json exists but agent.db is authoritative: ${authJson.path}`); + return false; + } + + for (const [provider, entry] of Object.entries(authJson.data)) { + const credentials = normalizeCredentialEntry(entry) + .filter(isValidCredential) + .map((credential) => credential); + + if (credentials.length > 0) { + storage.replaceAuthCredentialsForProvider(provider, credentials); + } + } + + await backupJson(authJson.path); + return true; +} + +/** + * Migrates legacy JSON files (settings.json, auth.json) to SQLite-based agent.db. + * Skips migration if DB already contains data (DB is authoritative). + * @param paths - Configuration specifying locations of legacy files and target DB + * @returns Result indicating what was migrated and any warnings encountered + */ +export async function migrateJsonStorage(paths: MigrationPaths): Promise { + const storage = AgentStorage.open(getAgentDbPath(paths.agentDir)); + const warnings: string[] = []; + + const [migratedSettings, migratedAuth] = await Promise.all([ + migrateSettings(storage, paths.settingsPath, warnings), + migrateAuth(storage, paths.authPaths, warnings), + ]); + + if (warnings.length > 0) { + for (const warning of warnings) { + logger.warn("Storage migration warning", { warning }); + } + } + + return { migratedSettings, migratedAuth, warnings }; +} diff --git a/packages/coding-agent/src/core/tools/web-search/auth.ts b/packages/coding-agent/src/core/tools/web-search/auth.ts index def9b5fa6..4ad85dc0b 100644 --- a/packages/coding-agent/src/core/tools/web-search/auth.ts +++ b/packages/coding-agent/src/core/tools/web-search/auth.ts @@ -4,19 +4,26 @@ * 4-tier auth resolution: * 1. ANTHROPIC_SEARCH_API_KEY / ANTHROPIC_SEARCH_BASE_URL env vars * 2. Provider with api="anthropic-messages" in ~/.omp/agent/models.json - * 3. OAuth credentials in ~/.omp/agent/auth.json (with expiry check) + * 3. OAuth credentials in ~/.omp/agent/agent.db (with expiry check) * 4. ANTHROPIC_API_KEY / ANTHROPIC_BASE_URL fallback */ import * as os from "node:os"; import * as path from "node:path"; import { buildBetaHeader, claudeCodeHeaders, claudeCodeVersion } from "@oh-my-pi/pi-ai"; -import { getConfigDirPaths } from "../../../config"; -import type { AnthropicAuthConfig, AnthropicOAuthCredential, AuthJson, ModelsJson } from "./types"; +import { getAgentDbPath, getConfigDirPaths } from "../../../config"; +import { AgentStorage } from "../../agent-storage"; +import type { AuthCredential } from "../../auth-storage"; +import { migrateJsonStorage } from "../../storage-migration"; +import type { AnthropicAuthConfig, AnthropicOAuthCredential, ModelsJson } from "./types"; const DEFAULT_BASE_URL = "https://api.anthropic.com"; -/** Parse a .env file and return key-value pairs */ +/** + * Parses a .env file and extracts key-value pairs. + * @param filePath - Path to the .env file + * @returns Object containing parsed environment variables + */ async function parseEnvFile(filePath: string): Promise> { const result: Record = {}; try { @@ -47,7 +54,11 @@ async function parseEnvFile(filePath: string): Promise> { return result; } -/** Get env var from process.env or .env files */ +/** + * Gets an environment variable from process.env or .env files. + * @param key - The environment variable name to look up + * @returns The value if found, undefined otherwise + */ export async function getEnv(key: string): Promise { if (process.env[key]) return process.env[key]; @@ -60,7 +71,11 @@ export async function getEnv(key: string): Promise { return undefined; } -/** Read JSON file safely */ +/** + * Reads and parses a JSON file safely. + * @param filePath - Path to the JSON file + * @returns Parsed JSON content, or null if file doesn't exist or parsing fails + */ async function readJson(filePath: string): Promise { try { const file = Bun.file(filePath); @@ -72,22 +87,62 @@ async function readJson(filePath: string): Promise { } } -/** Check if a token is an OAuth token (sk-ant-oat* prefix) */ +/** + * Checks if a token is an OAuth token by looking for sk-ant-oat prefix. + * @param apiKey - The API key to check + * @returns True if the token is an OAuth token + */ export function isOAuthToken(apiKey: string): boolean { return apiKey.includes("sk-ant-oat"); } -function normalizeAnthropicOAuthCredentials(entry: AuthJson["anthropic"] | undefined): AnthropicOAuthCredential[] { - if (!entry) return []; - return Array.isArray(entry) ? entry : [entry]; +/** + * Converts a generic AuthCredential to AnthropicOAuthCredential if it's a valid OAuth entry. + * @param credential - The credential to convert + * @returns The converted OAuth credential, or null if not a valid OAuth type + */ +function toAnthropicOAuthCredential(credential: AuthCredential): AnthropicOAuthCredential | null { + if (credential.type !== "oauth") return null; + if (typeof credential.access !== "string" || typeof credential.expires !== "number") return null; + return { + type: "oauth", + access: credential.access, + refresh: credential.refresh, + expires: credential.expires, + }; } /** - * Find Anthropic auth config using 4-tier priority: + * Reads Anthropic OAuth credentials from agent.db, migrating from legacy auth.json if needed. + * @param configDir - Path to the config directory containing agent.db + * @returns Array of valid Anthropic OAuth credentials + */ +async function readAnthropicOAuthCredentials(configDir: string): Promise { + await migrateJsonStorage({ + agentDir: configDir, + settingsPath: path.join(configDir, "settings.json"), + authPaths: [path.join(configDir, "auth.json")], + }); + + const storage = AgentStorage.open(getAgentDbPath(configDir)); + const records = storage.listAuthCredentials("anthropic"); + const credentials: AnthropicOAuthCredential[] = []; + for (const record of records) { + const mapped = toAnthropicOAuthCredential(record.credential); + if (mapped) { + credentials.push(mapped); + } + } + return credentials; +} + +/** + * Finds Anthropic auth config using 4-tier priority: * 1. ANTHROPIC_SEARCH_API_KEY / ANTHROPIC_SEARCH_BASE_URL * 2. Provider with api="anthropic-messages" in models.json - * 3. OAuth in auth.json (with 5-minute expiry buffer) + * 3. OAuth in agent.db (with 5-minute expiry buffer) * 4. ANTHROPIC_API_KEY / ANTHROPIC_BASE_URL fallback + * @returns The first valid auth configuration found, or null if none available */ export async function findAnthropicAuth(): Promise { // Get all config directories (user-level only) for fallback support @@ -131,14 +186,13 @@ export async function findAnthropicAuth(): Promise { } } - // 3. OAuth credentials in auth.json (with 5-minute expiry buffer, check all config dirs) + // 3. OAuth credentials in agent.db (with 5-minute expiry buffer, check all config dirs) const expiryBuffer = 5 * 60 * 1000; // 5 minutes const now = Date.now(); for (const configDir of configDirs) { - const authJson = await readJson(path.join(configDir, "auth.json")); - const credentials = normalizeAnthropicOAuthCredentials(authJson?.anthropic); + const credentials = await readAnthropicOAuthCredentials(configDir); for (const credential of credentials) { - if (credential.type !== "oauth" || !credential.access) continue; + if (!credential.access) continue; if (credential.expires > now + expiryBuffer) { return { apiKey: credential.access, @@ -163,6 +217,11 @@ export async function findAnthropicAuth(): Promise { return null; } +/** + * Checks if a base URL points to the official Anthropic API. + * @param baseUrl - The base URL to check + * @returns True if the URL is for api.anthropic.com over HTTPS + */ function isAnthropicBaseUrl(baseUrl: string): boolean { try { const url = new URL(baseUrl); @@ -172,7 +231,11 @@ function isAnthropicBaseUrl(baseUrl: string): boolean { } } -/** Build headers for Anthropic API request */ +/** + * Builds HTTP headers for Anthropic API requests. + * @param auth - The authentication configuration + * @returns Headers object ready for use in fetch requests + */ export function buildAnthropicHeaders(auth: AnthropicAuthConfig): Record { const baseBetas = auth.isOAuth ? [ @@ -205,7 +268,11 @@ export function buildAnthropicHeaders(auth: AnthropicAuthConfig): Record { return isOAuth ? applyClaudeToolPrefix(name) : name; }; @@ -33,11 +39,21 @@ export interface AnthropicSearchParams { num_results?: number; } -/** Get model from env or use default */ +/** + * Gets the model to use for web search from environment or default. + * @returns Model identifier string + */ async function getModel(): Promise { return (await getEnv("ANTHROPIC_SEARCH_MODEL")) ?? DEFAULT_MODEL; } +/** + * Builds system instruction blocks for the Anthropic API request. + * @param auth - Authentication configuration + * @param model - Model identifier (affects whether Claude Code instruction is included) + * @param systemPrompt - Optional custom system prompt + * @returns Array of system blocks for the API request + */ function buildSystemBlocks( auth: AnthropicAuthConfig, model: string, @@ -53,7 +69,16 @@ function buildSystemBlocks( }); } -/** Call Anthropic API with web search */ +/** + * Calls the Anthropic API with web search tool enabled. + * @param auth - Authentication configuration (API key or OAuth) + * @param model - Model identifier to use + * @param query - Search query from the user + * @param systemPrompt - Optional custom system prompt + * @param maxTokens - Maximum tokens for the response + * @returns Raw API response from Anthropic + * @throws {WebSearchProviderError} If the API request fails + */ async function callWebSearch( auth: AnthropicAuthConfig, model: string, @@ -100,7 +125,11 @@ async function callWebSearch( return response.json() as Promise; } -/** Parse page_age string into seconds (e.g., "2 days ago", "3h ago", "1 week ago") */ +/** + * Parses a human-readable page age string into seconds. + * @param pageAge - Age string like "2 days ago", "3h ago", "1 week ago" + * @returns Age in seconds, or undefined if parsing fails + */ function parsePageAge(pageAge: string | null | undefined): number | undefined { if (!pageAge) return undefined; @@ -132,7 +161,11 @@ function parsePageAge(pageAge: string | null | undefined): number | undefined { return value * (multipliers[unit] ?? 86400); } -/** Parse API response into unified WebSearchResponse */ +/** + * Parses the Anthropic API response into a unified WebSearchResponse. + * @param response - Raw API response containing content blocks + * @returns Normalized response with answer, sources, citations, and usage + */ function parseResponse(response: AnthropicApiResponse): WebSearchResponse { const answerParts: string[] = []; const searchQueries: string[] = []; @@ -193,12 +226,17 @@ function parseResponse(response: AnthropicApiResponse): WebSearchResponse { }; } -/** Execute Anthropic web search */ +/** + * Executes a web search using Anthropic's Claude with built-in web search tool. + * @param params - Search parameters including query and optional settings + * @returns Search response with synthesized answer, sources, and citations + * @throws {Error} If no Anthropic credentials are configured + */ export async function searchAnthropic(params: AnthropicSearchParams): Promise { const auth = await findAnthropicAuth(); if (!auth) { throw new Error( - "No Anthropic credentials found. Set ANTHROPIC_API_KEY or configure OAuth in ~/.omp/agent/auth.json", + "No Anthropic credentials found. Set ANTHROPIC_API_KEY or configure OAuth in ~/.omp/agent/agent.db", ); } diff --git a/packages/coding-agent/src/main.ts b/packages/coding-agent/src/main.ts index a6edc2556..8adee3e30 100644 --- a/packages/coding-agent/src/main.ts +++ b/packages/coding-agent/src/main.ts @@ -76,7 +76,7 @@ async function runInteractiveMode( mode.renderInitialMessages(); if (migratedProviders.length > 0) { - mode.showWarning(`Migrated credentials to auth.json: ${migratedProviders.join(", ")}`); + mode.showWarning(`Migrated credentials to agent.db: ${migratedProviders.join(", ")}`); } if (modelsJsonError) { diff --git a/packages/coding-agent/src/migrations.ts b/packages/coding-agent/src/migrations.ts index 7f4b2f334..9ab7393ba 100644 --- a/packages/coding-agent/src/migrations.ts +++ b/packages/coding-agent/src/migrations.ts @@ -3,9 +3,11 @@ */ import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs"; -import { dirname, join } from "node:path"; +import { join } from "node:path"; import chalk from "chalk"; -import { getAgentDir, getBinDir } from "./config"; +import { getAgentDbPath, getAgentDir, getBinDir } from "./config"; +import { AgentStorage } from "./core/agent-storage"; +import type { AuthCredential } from "./core/auth-storage"; /** * Migrate PI_* environment variables to OMP_* equivalents. @@ -29,28 +31,26 @@ export function migrateEnvVars(): string[] { } /** - * Migrate legacy oauth.json and settings.json apiKeys to auth.json. + * Migrate legacy oauth.json and settings.json apiKeys to agent.db. * * @returns Array of provider names that were migrated */ -export function migrateAuthToAuthJson(): string[] { +export function migrateAuthToAgentDb(): string[] { const agentDir = getAgentDir(); - const authPath = join(agentDir, "auth.json"); const oauthPath = join(agentDir, "oauth.json"); const settingsPath = join(agentDir, "settings.json"); + const storage = AgentStorage.open(getAgentDbPath(agentDir)); - // Skip if auth.json already exists - if (existsSync(authPath)) return []; + if (storage.hasAuthCredentials()) return []; - const migrated: Record = {}; + const migrated: Record = {}; const providers: string[] = []; - // Migrate oauth.json if (existsSync(oauthPath)) { try { const oauth = JSON.parse(readFileSync(oauthPath, "utf-8")); for (const [provider, cred] of Object.entries(oauth)) { - migrated[provider] = { type: "oauth", ...(cred as object) }; + migrated[provider] = [{ type: "oauth", ...(cred as object) } as AuthCredential]; providers.push(provider); } renameSync(oauthPath, `${oauthPath}.migrated`); @@ -59,7 +59,6 @@ export function migrateAuthToAuthJson(): string[] { } } - // Migrate settings.json apiKeys if (existsSync(settingsPath)) { try { const content = readFileSync(settingsPath, "utf-8"); @@ -67,7 +66,7 @@ export function migrateAuthToAuthJson(): string[] { if (settings.apiKeys && typeof settings.apiKeys === "object") { for (const [provider, key] of Object.entries(settings.apiKeys)) { if (!migrated[provider] && typeof key === "string") { - migrated[provider] = { type: "api_key", key }; + migrated[provider] = [{ type: "api_key", key }]; providers.push(provider); } } @@ -79,9 +78,8 @@ export function migrateAuthToAuthJson(): string[] { } } - if (Object.keys(migrated).length > 0) { - mkdirSync(dirname(authPath), { recursive: true }); - writeFileSync(authPath, JSON.stringify(migrated, null, 2), { mode: 0o600 }); + for (const [provider, credentials] of Object.entries(migrated)) { + storage.replaceAuthCredentialsForProvider(provider, credentials); } return providers; @@ -201,7 +199,7 @@ export async function runMigrations(_cwd: string): Promise<{ const migratedEnvVars = migrateEnvVars(); // Then: run data migrations - const migratedAuthProviders = migrateAuthToAuthJson(); + const migratedAuthProviders = migrateAuthToAgentDb(); migrateSessionsFromAgentRoot(); migrateToolsToBin(); diff --git a/packages/coding-agent/test/settings-manager.test.ts b/packages/coding-agent/test/settings-manager.test.ts index 779483c18..11d2efd5a 100644 --- a/packages/coding-agent/test/settings-manager.test.ts +++ b/packages/coding-agent/test/settings-manager.test.ts @@ -1,15 +1,21 @@ import { afterEach, beforeEach, describe, expect, it } from "bun:test"; -import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { existsSync, mkdirSync, rmSync } from "node:fs"; import { join } from "node:path"; +import { getAgentDbPath } from "../src/config"; +import { AgentStorage } from "../src/core/agent-storage"; import { SettingsManager } from "../src/core/settings-manager"; describe("SettingsManager", () => { - const testDir = join(process.cwd(), "test-settings-tmp"); - const agentDir = join(testDir, "agent"); - const projectDir = join(testDir, "project"); + let testDir: string; + let agentDir: string; + let projectDir: string; beforeEach(() => { - // Clean up and create fresh directories + // Use random UUID to isolate parallel test runs (SQLite files can't be shared) + testDir = join(process.cwd(), "test-settings-tmp", crypto.randomUUID()); + agentDir = join(testDir, "agent"); + projectDir = join(testDir, "project"); + if (existsSync(testDir)) { rmSync(testDir, { recursive: true }); } @@ -23,85 +29,75 @@ describe("SettingsManager", () => { } }); + // Tests that SettingsManager merges with DB state on save rather than blindly overwriting. + // This ensures external edits (via AgentStorage directly) aren't lost when the app saves. describe("preserves externally added settings", () => { it("should preserve enabledModels when changing thinking level", () => { - // Create initial settings file - const settingsPath = join(agentDir, "settings.json"); - writeFileSync( - settingsPath, - JSON.stringify({ - theme: "dark", - defaultModel: "claude-sonnet", - }), - ); + // Seed initial settings in DB + const storage = AgentStorage.open(getAgentDbPath(agentDir)); + storage.saveSettings({ + theme: "dark", + modelRoles: { default: "claude-sonnet" }, + }); - // Create SettingsManager (simulates pi starting up) + // Manager loads the initial state const manager = SettingsManager.create(projectDir, agentDir); - // Simulate user editing settings.json externally to add enabledModels - const currentSettings = JSON.parse(readFileSync(settingsPath, "utf-8")); - currentSettings.enabledModels = ["claude-opus-4-5", "gpt-5.2-codex"]; - writeFileSync(settingsPath, JSON.stringify(currentSettings, null, 2)); + // Simulate external edit (e.g., user modifying DB directly or another process) + storage.saveSettings({ + theme: "dark", + modelRoles: { default: "claude-sonnet" }, + enabledModels: ["claude-opus-4-5", "gpt-5.2-codex"], + }); - // User changes thinking level via Shift+Tab + // Manager saves a change - should merge, not overwrite manager.setDefaultThinkingLevel("high"); - // Verify enabledModels is preserved - const savedSettings = JSON.parse(readFileSync(settingsPath, "utf-8")); + const savedSettings = storage.getSettings() ?? {}; expect(savedSettings.enabledModels).toEqual(["claude-opus-4-5", "gpt-5.2-codex"]); expect(savedSettings.defaultThinkingLevel).toBe("high"); expect(savedSettings.theme).toBe("dark"); - expect(savedSettings.defaultModel).toBe("claude-sonnet"); + expect(savedSettings.modelRoles?.default).toBe("claude-sonnet"); }); it("should preserve custom settings when changing theme", () => { - const settingsPath = join(agentDir, "settings.json"); - writeFileSync( - settingsPath, - JSON.stringify({ - defaultModel: "claude-sonnet", - }), - ); + const storage = AgentStorage.open(getAgentDbPath(agentDir)); + storage.saveSettings({ + modelRoles: { default: "claude-sonnet" }, + }); const manager = SettingsManager.create(projectDir, agentDir); - // User adds custom settings externally - const currentSettings = JSON.parse(readFileSync(settingsPath, "utf-8")); - currentSettings.shellPath = "/bin/zsh"; - currentSettings.extensions = ["/path/to/extension.ts"]; - writeFileSync(settingsPath, JSON.stringify(currentSettings, null, 2)); + storage.saveSettings({ + modelRoles: { default: "claude-sonnet" }, + shellPath: "/bin/zsh", + extensions: ["/path/to/extension.ts"], + }); - // User changes theme manager.setTheme("light"); - // Verify all settings preserved - const savedSettings = JSON.parse(readFileSync(settingsPath, "utf-8")); + const savedSettings = storage.getSettings() ?? {}; expect(savedSettings.shellPath).toBe("/bin/zsh"); expect(savedSettings.extensions).toEqual(["/path/to/extension.ts"]); expect(savedSettings.theme).toBe("light"); }); it("should let in-memory changes override file changes for same key", () => { - const settingsPath = join(agentDir, "settings.json"); - writeFileSync( - settingsPath, - JSON.stringify({ - theme: "dark", - }), - ); + const storage = AgentStorage.open(getAgentDbPath(agentDir)); + storage.saveSettings({ + theme: "dark", + }); const manager = SettingsManager.create(projectDir, agentDir); - // User externally sets thinking level to "low" - const currentSettings = JSON.parse(readFileSync(settingsPath, "utf-8")); - currentSettings.defaultThinkingLevel = "low"; - writeFileSync(settingsPath, JSON.stringify(currentSettings, null, 2)); + storage.saveSettings({ + theme: "dark", + defaultThinkingLevel: "low", + }); - // But then changes it via UI to "high" manager.setDefaultThinkingLevel("high"); - // In-memory change should win - const savedSettings = JSON.parse(readFileSync(settingsPath, "utf-8")); + const savedSettings = storage.getSettings() ?? {}; expect(savedSettings.defaultThinkingLevel).toBe("high"); }); });