perf(sqlite-reader): replaced full COUNT(*) scan with bounded row probing

- Added ROW_COUNT_PROBE_CAP to limit rows scanned when counting tables, preventing JS thread freezes on large databases.
- Used sqlite_stat1 estimates for tables exceeding the cap; exact counts only for provably small tables.
- Introduced TableRowCount type with exact/estimate/atLeast variants reflected in rendered output.
This commit is contained in:
can1357
2026-06-02 05:24:21 +02:00
parent 783971f575
commit b522fde56d
3 changed files with 178 additions and 10 deletions
@@ -12,6 +12,15 @@ const MAX_QUERY_LIMIT = 500;
const MAX_RENDER_WIDTH = 120;
const MAX_COLUMN_WIDTH = 40;
const MIN_COLUMN_WIDTH = 1;
/**
* Upper bound on rows scanned when counting a table for the listing. SQLite has
* no stored row count, so `COUNT(*)` is a full b-tree scan — multi-second on a
* multi-GB database, and `bun:sqlite` runs it synchronously on the JS thread
* that also drives the TUI, freezing rendering and input. The listing instead
* trusts the planner's `sqlite_stat1` estimate for large tables and only counts
* exactly when a table is provably small, reading at most this many rows.
*/
const ROW_COUNT_PROBE_CAP = 50_000;
type SqliteBinding = Exclude<SQLQueryBindings, Record<string, unknown>>;
@@ -26,6 +35,11 @@ interface SqliteCountRow {
count: number;
}
interface SqliteStat1Row {
tbl: string;
stat: string | null;
}
interface SqliteTableInfoRow {
cid: number;
name: string;
@@ -50,6 +64,23 @@ export type SqliteSelector =
export type SqliteRowLookup = { kind: "pk"; column: string; type: string } | { kind: "rowid" };
/**
* Row count for a table in the listing.
* - `exact`: counted in full (the table is small enough to count cheaply).
* - `estimate`: the planner's `sqlite_stat1` figure; the table is too large to
* scan, so this may be stale.
* - `atLeast`: a lower bound; counting was capped before reaching the end.
*/
export type TableRowCount =
| { kind: "exact"; rows: number }
| { kind: "estimate"; rows: number }
| { kind: "atLeast"; rows: number };
export interface SqliteTableSummary {
name: string;
count: TableRowCount;
}
function splitSqliteRemainder(remainder: string): { subPath: string; queryString: string } {
const queryIndex = remainder.indexOf("?");
if (queryIndex === -1) {
@@ -495,20 +526,61 @@ export function parseSqliteSelector(subPath: string, queryString: string): Sqlit
return { kind: "schema", table, sampleLimit: DEFAULT_SCHEMA_SAMPLE_LIMIT };
}
export function listTables(db: Database): { name: string; rowCount: number }[] {
/**
* Reads the planner's per-table row estimate from `sqlite_stat1` (populated by
* `ANALYZE`). The first integer of each `stat` string is the number of rows in
* that index; for a full (non-partial) index it equals the table's row count,
* so the max across a table's entries is the table estimate. Returns an empty
* map when the database was never analyzed. One small indexed read — no scan.
*/
function loadRowEstimates(db: Database): Map<string, number> {
const estimates = new Map<string, number>();
const hasStat1 = db
.prepare<Pick<SqliteMasterRow, "name">, []>(
"SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'sqlite_stat1'",
)
.get();
if (!hasStat1) return estimates;
for (const { tbl, stat } of db.prepare<SqliteStat1Row, []>("SELECT tbl, stat FROM sqlite_stat1").all()) {
if (!stat) continue;
const rows = Number.parseInt(stat, 10);
if (!Number.isFinite(rows)) continue;
const prev = estimates.get(tbl);
if (prev === undefined || rows > prev) estimates.set(tbl, rows);
}
return estimates;
}
/**
* Counts a table while reading at most `cap + 1` rows. Returns an exact count
* when the table holds `cap` rows or fewer, otherwise a lower bound of `cap`.
* Bounds the worst-case scan so a stale or missing estimate can never trigger a
* full-table scan on the JS thread.
*/
function probeRowCount(db: Database, table: string, cap: number): TableRowCount {
const sql = `SELECT COUNT(*) AS count FROM (SELECT 1 FROM ${quoteSqliteIdentifier(table)} LIMIT ${cap + 1})`;
const counted = db.prepare<SqliteCountRow, []>(sql).get()?.count ?? 0;
return counted > cap ? { kind: "atLeast", rows: cap } : { kind: "exact", rows: counted };
}
export function listTables(db: Database, options: { probeCap?: number } = {}): SqliteTableSummary[] {
const cap = options.probeCap ?? ROW_COUNT_PROBE_CAP;
const names = db
.prepare<Pick<SqliteMasterRow, "name">, []>(
"SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name COLLATE NOCASE",
)
.all();
const estimates = loadRowEstimates(db);
return names.map(({ name }) => {
const countRow =
db.prepare<SqliteCountRow, []>(`SELECT COUNT(*) AS count FROM ${quoteSqliteIdentifier(name)}`).get() ?? null;
return {
name,
rowCount: countRow?.count ?? 0,
};
const estimate = estimates.get(name);
// Trust the planner only when it says the table is too large to count
// cheaply; otherwise count exactly (bounded), which also corrects a
// stale-low estimate without ever scanning more than `cap` rows.
const count: TableRowCount =
estimate !== undefined && estimate > cap ? { kind: "estimate", rows: estimate } : probeRowCount(db, name, cap);
return { name, count };
});
}
@@ -679,13 +751,24 @@ export function deleteRowByRowId(db: Database, table: string, key: string): numb
return statement.run(binding).changes;
}
export function renderTableList(tables: { name: string; rowCount: number }[]): string {
function formatRowCount(count: TableRowCount): string {
switch (count.kind) {
case "exact":
return `${count.rows} rows`;
case "estimate":
return `~${count.rows} rows`;
case "atLeast":
return `${count.rows}+ rows`;
}
}
export function renderTableList(tables: SqliteTableSummary[]): string {
if (tables.length === 0) {
return "(no tables)";
}
return tables
.map(table => truncateToWidth(replaceTabs(`${table.name} (${table.rowCount} rows)`), MAX_RENDER_WIDTH))
.map(table => truncateToWidth(replaceTabs(`${table.name} (${formatRowCount(table.count)})`), MAX_RENDER_WIDTH))
.join("\n");
}