diff --git a/packages/mnemopi/CHANGELOG.md b/packages/mnemopi/CHANGELOG.md index 3608b7e55..4a4c46e36 100644 --- a/packages/mnemopi/CHANGELOG.md +++ b/packages/mnemopi/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Added + +- Added opt-in SQLite page-size selection for file-backed databases. Set `MNEMOPI_DB_PAGE_SIZE` to a valid SQLite page size (a power of two from 512 to 65536) or `os`, or pass `pageSize` to `openDatabase`. The default remains SQLite's own page size because the virtual-memory page size does not establish storage-block behavior and larger WAL pages can increase sparse-write cost. Existing databases retain their page size. + ## [17.2.3] - 2026-08-01 ### Fixed diff --git a/packages/mnemopi/README.md b/packages/mnemopi/README.md index 68a09c1cf..f99c2ec5c 100644 --- a/packages/mnemopi/README.md +++ b/packages/mnemopi/README.md @@ -83,6 +83,7 @@ In `per-project-tagged`, the wrapper is responsible for combining project-local Common environment fallbacks: - `MNEMOPI_DATA_DIR` / `MNEMOPI_DB_PATH`: default storage location. +- `MNEMOPI_DB_PAGE_SIZE`: optional SQLite page size for new file-backed databases; use a valid power of two from 512 to 65536 or `os` to request the detected system page size. Unset preserves SQLite's default. - `MNEMOPI_NO_EMBEDDINGS=1`: force FTS-only recall. - `MNEMOPI_EMBEDDING_MODEL`: defaults to `BAAI/bge-small-en-v1.5`. - `MNEMOPI_EMBEDDING_API_URL` and `MNEMOPI_EMBEDDING_API_KEY`: OpenAI-compatible embedding endpoint. diff --git a/packages/mnemopi/src/core/cost-log.ts b/packages/mnemopi/src/core/cost-log.ts index dea8ed533..77fd59c23 100644 --- a/packages/mnemopi/src/core/cost-log.ts +++ b/packages/mnemopi/src/core/cost-log.ts @@ -1,10 +1,9 @@ -import { Database } from "bun:sqlite"; -import { mkdirSync } from "node:fs"; -import { homedir } from "node:os"; -import { dirname, join } from "node:path"; - -export const DEFAULT_LOG_DIR = join(homedir(), ".mnemopi", "data"); -export const DEFAULT_LOG_DB = join(DEFAULT_LOG_DIR, "cost_log.db"); +import type { Database } from "bun:sqlite"; +import * as os from "node:os"; +import * as path from "node:path"; +import { openDatabase } from "../db"; +export const DEFAULT_LOG_DIR = path.join(os.homedir(), ".mnemopi", "data"); +export const DEFAULT_LOG_DB = path.join(DEFAULT_LOG_DIR, "cost_log.db"); export interface CostStats { total_calls: number; @@ -22,8 +21,7 @@ type AggregateRow = { export function getConn(dbPath?: string): Database { const path = dbPath ?? DEFAULT_LOG_DB; - mkdirSync(dirname(path), { recursive: true }); - return new Database(path, { create: true, readwrite: true, strict: true }); + return openDatabase(path, { pragmas: false }); } export function initCostLog(dbPath?: string): void { diff --git a/packages/mnemopi/src/core/query-cache.ts b/packages/mnemopi/src/core/query-cache.ts index 915c52846..11aed1402 100644 --- a/packages/mnemopi/src/core/query-cache.ts +++ b/packages/mnemopi/src/core/query-cache.ts @@ -1,7 +1,6 @@ -import { Database } from "bun:sqlite"; -import { mkdirSync } from "node:fs"; -import { dirname } from "node:path"; +import type { Database } from "bun:sqlite"; import { type Env, enhancedRecallEnabled } from "../config"; +import { openDatabase } from "../db"; import { cosineSimilarity } from "./vector-math"; export type QueryCacheResult = Record; @@ -80,8 +79,7 @@ export class QueryCache { } #initDb(dbPath: string): void { - if (dbPath !== ":memory:") mkdirSync(dirname(dbPath), { recursive: true }); - const db = new Database(dbPath, { create: true, readwrite: true, strict: true }); + const db = openDatabase(dbPath, { pragmas: false }); this.#conn = db; if (dbPath !== ":memory:") db.exec("PRAGMA journal_mode=WAL"); db.exec(` diff --git a/packages/mnemopi/src/db.ts b/packages/mnemopi/src/db.ts index 2ce63b94f..564979db6 100644 --- a/packages/mnemopi/src/db.ts +++ b/packages/mnemopi/src/db.ts @@ -3,6 +3,59 @@ import { mkdirSync } from "node:fs"; import { dirname } from "node:path"; import { dbPath } from "./config"; +export type SqlitePageSize = number | "os"; + +const MIN_SQLITE_PAGE_SIZE = 512; +const MAX_SQLITE_PAGE_SIZE = 65536; +let detectedSystemPageSize: number | null | undefined; + +function isValidPageSize(size: number): boolean { + return ( + Number.isInteger(size) && + size >= MIN_SQLITE_PAGE_SIZE && + size <= MAX_SQLITE_PAGE_SIZE && + (size & (size - 1)) === 0 + ); +} + +function detectSystemPageSize(): number | undefined { + if (detectedSystemPageSize !== undefined) return detectedSystemPageSize ?? undefined; + try { + const proc = Bun.spawnSync(["getconf", "PAGE_SIZE"], { stdout: "pipe" }); + if (proc.exitCode === 0) { + const size = Number(proc.stdout.toString().trim()); + if (isValidPageSize(size)) { + detectedSystemPageSize = size; + return size; + } + } + } catch { + /* fall through */ + } + detectedSystemPageSize = null; + return undefined; +} + +function configuredPageSize(): SqlitePageSize | undefined { + const raw = process.env.MNEMOPI_DB_PAGE_SIZE?.trim(); + if (!raw) return undefined; + if (raw.toLowerCase() === "os") return "os"; + const size = Number(raw); + return isValidPageSize(size) ? size : undefined; +} + +function resolvePageSize(requested?: SqlitePageSize): number | undefined { + const pageSize = requested ?? configuredPageSize(); + if (pageSize === "os") return detectSystemPageSize(); + return pageSize !== undefined && isValidPageSize(pageSize) ? pageSize : undefined; +} + +function applyPageSize(db: Database, path: DatabasePath, requested?: SqlitePageSize): void { + if (path === ":memory:") return; + const pageSize = resolvePageSize(requested); + if (pageSize !== undefined) db.exec(`PRAGMA page_size=${pageSize}`); +} + export type DatabasePath = string | ":memory:"; export interface OpenDatabaseOptions { @@ -11,6 +64,7 @@ export interface OpenDatabaseOptions { readonly strict?: boolean; readonly loadExtension?: string | readonly string[]; readonly pragmas?: boolean; + readonly pageSize?: SqlitePageSize; } interface TxState { @@ -29,15 +83,19 @@ export function openDatabase(path: DatabasePath = dbPath(), options: OpenDatabas readwrite: options.readwrite ?? true, strict: options.strict ?? true, }); - if (options.pragmas !== false) enablePragmas(db, path); + if (options.pragmas !== false) enablePragmas(db, path, options.pageSize); + else if (options.readwrite !== false) applyPageSize(db, path, options.pageSize); if (options.loadExtension !== undefined) loadExtensions(db, options.loadExtension); return db; } -export function enablePragmas(db: Database, path?: DatabasePath): void { +export function enablePragmas(db: Database, path?: DatabasePath, pageSize?: SqlitePageSize): void { db.exec("PRAGMA foreign_keys=ON"); db.exec("PRAGMA busy_timeout=5000"); - if (path !== ":memory:") db.exec("PRAGMA journal_mode=WAL"); + if (path !== ":memory:") { + applyPageSize(db, path ?? dbPath(), pageSize); + db.exec("PRAGMA journal_mode=WAL"); + } } export function loadExtensions(db: Database, extensions: string | readonly string[]): void { diff --git a/packages/mnemopi/src/dr/recovery.ts b/packages/mnemopi/src/dr/recovery.ts index cd4aa8d43..50507c215 100644 --- a/packages/mnemopi/src/dr/recovery.ts +++ b/packages/mnemopi/src/dr/recovery.ts @@ -1,4 +1,4 @@ -import { Database } from "bun:sqlite"; +import type { Database } from "bun:sqlite"; import { createHash } from "node:crypto"; import { copyFileSync, @@ -188,7 +188,7 @@ function isSqliteFile(bytes: Uint8Array): boolean { function writeGzippedSqlDump(sql: string, tempPath: string): void { let db: Database | null = null; try { - db = new Database(tempPath, { create: true, readwrite: true, strict: true }); + db = openDatabase(tempPath, { pragmas: false }); db.exec(sql); } finally { closeQuietly(db); diff --git a/packages/mnemopi/test/db-page-size.test.ts b/packages/mnemopi/test/db-page-size.test.ts new file mode 100644 index 000000000..b451d1db1 --- /dev/null +++ b/packages/mnemopi/test/db-page-size.test.ts @@ -0,0 +1,77 @@ +import type { Database } from "bun:sqlite"; +import { afterEach, describe, expect, it } from "bun:test"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { closeQuietly, openDatabase } from "../src/db"; + +const roots: string[] = []; + +function tempDb(): string { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "mnemopi-page-size-")); + roots.push(root); + return path.join(root, "test.db"); +} + +interface PageSizeRow { + page_size: number; +} + +function pageSize(db: Database): number { + const row = db.query("PRAGMA page_size").get() as PageSizeRow; + return row.page_size; +} + +afterEach(() => { + for (;;) { + const root = roots.pop(); + if (root === undefined) break; + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +describe("db page size", () => { + it("uses an explicitly requested page size for a new file-backed database", () => { + const db = openDatabase(tempDb(), { pageSize: 16384 }); + try { + expect(pageSize(db)).toBe(16384); + } finally { + closeQuietly(db); + } + }); + + it("ignores invalid explicit page sizes", () => { + const db = openDatabase(tempDb(), { pageSize: 12345 }); + try { + expect(pageSize(db)).toBe(4096); + } finally { + closeQuietly(db); + } + }); + + it("keeps the existing page size when reopening a database with another request", () => { + const dbPath = tempDb(); + const initial = openDatabase(dbPath, { pageSize: 4096 }); + try { + initial.run("CREATE TABLE existing_data (value TEXT NOT NULL)"); + } finally { + closeQuietly(initial); + } + + const reopened = openDatabase(dbPath, { pageSize: 16384 }); + try { + expect(pageSize(reopened)).toBe(4096); + } finally { + closeQuietly(reopened); + } + }); + + it("does not set page size on in-memory databases", () => { + const db = openDatabase(":memory:", { pageSize: 16384 }); + try { + expect(pageSize(db)).toBe(4096); + } finally { + closeQuietly(db); + } + }); +});