perf(coding-agent): optimized documentation indexing via compressed blob
- Replaced raw TypeScript documentation map with a lazily-inflated gzip blob. - Reduced bundled binary/npm package size by approximately 0.9MB. - Encapsulated index logic into `docs-index.ts` to separate header metadata from content. - Added `postpack` and robust `try/finally` patterns in build scripts to ensure clean artifacts. - Implemented disk-based fallback during development to maintain existing developer experience.
This commit is contained in:
can1357
@@ -20,8 +20,6 @@
|
||||
"ignore": [
|
||||
// Generated from `packages/natives/scripts/native-index.template.js` via gen-enums.ts.
|
||||
"packages/natives/native/index.js",
|
||||
// Generated by `packages/coding-agent/scripts/generate-docs-index.ts`.
|
||||
"packages/coding-agent/src/internal-urls/docs-index.generated.ts",
|
||||
// Embedded HTML asset shipped as a static template, not a code module.
|
||||
"packages/coding-agent/src/export/html/template.js",
|
||||
// Generated/owned upstream — see packages/ai/scripts/generate-models.ts.
|
||||
|
||||
@@ -3,7 +3,9 @@
|
||||
## [Unreleased]
|
||||
### Changed
|
||||
|
||||
- Optimized `omp://` documentation indexing by compressing doc bodies into a lazily-inflated blob
|
||||
- Changed Mermaid fenced-block ASCII rendering to use the first-party vendored renderer in `@oh-my-pi/pi-utils` (`src/vendor/mermaid-ascii`), dropping the `beautiful-mermaid` npm package, its transitive `elkjs` (~3.13MB), and the `beautiful-mermaid` `bun patch`; CJK/emoji width handling and the layout-direction override are preserved.
|
||||
- Changed `omp://` documentation embedding to a gzipped base64 index (`docs-index.generated.txt`, populated at build time and reset afterward) inflated on first read, instead of a ~1.6MB raw TypeScript map. The compiled binary / npm bundle drops ~0.9MB; the dev tree and source checkouts read `docs/` from disk.
|
||||
|
||||
### Fixed
|
||||
|
||||
@@ -11954,4 +11956,4 @@ Initial public release.
|
||||
|
||||
## [0.7.6] - 2025-11-13
|
||||
|
||||
Previous releases did not maintain a changelog.
|
||||
Previous releases did not maintain a changelog.
|
||||
@@ -36,11 +36,12 @@
|
||||
"check:types": "tsgo -p tsconfig.json --noEmit",
|
||||
"lint": "biome lint .",
|
||||
"test": "bun test --parallel=4",
|
||||
"fix": "biome check --write --unsafe . && bun run format-prompts && bun run generate-docs-index",
|
||||
"fix": "biome check --write --unsafe . && bun run format-prompts",
|
||||
"fmt": "biome format --write . && bun run format-prompts",
|
||||
"format-prompts": "bun scripts/format-prompts.ts",
|
||||
"generate-docs-index": "bun scripts/generate-docs-index.ts",
|
||||
"prepack": "bun scripts/generate-docs-index.ts && bun --cwd=../collab-web run build:tool-views && bun scripts/bundle-dist.ts",
|
||||
"generate-docs-index": "bun scripts/generate-docs-index.ts --generate",
|
||||
"prepack": "bun scripts/generate-docs-index.ts --generate && bun --cwd=../collab-web run build:tool-views && bun scripts/bundle-dist.ts || ( bun scripts/generate-docs-index.ts --reset; exit 1 )",
|
||||
"postpack": "bun scripts/generate-docs-index.ts --reset",
|
||||
"bench:guard": "bun scripts/bench-guard.ts"
|
||||
},
|
||||
"dependencies": {
|
||||
|
||||
@@ -38,8 +38,11 @@ async function runCommand(
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
await runCommand(["bun", "--cwd=../stats", "scripts/generate-client-bundle.ts", "--generate"]);
|
||||
// Generate inside the try so the finally always restores the empty checked-in
|
||||
// placeholders (stats client archive, docs index) even on failure.
|
||||
try {
|
||||
await runCommand(["bun", "--cwd=../stats", "scripts/generate-client-bundle.ts", "--generate"]);
|
||||
await runCommand(["bun", "scripts/generate-docs-index.ts", "--generate"]);
|
||||
await runCommand(["bun", "--cwd=../natives", "run", "embed:native"]);
|
||||
try {
|
||||
const buildEnv = shouldAdhocSignDarwinBinary() ? { ...Bun.env, BUN_NO_CODESIGN_MACHO_BINARY: "1" } : Bun.env;
|
||||
@@ -98,6 +101,7 @@ async function main(): Promise<void> {
|
||||
}
|
||||
} finally {
|
||||
await runCommand(["bun", "--cwd=../stats", "scripts/generate-client-bundle.ts", "--reset"]);
|
||||
await runCommand(["bun", "scripts/generate-docs-index.ts", "--reset"]);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -19,8 +19,8 @@ const ALWAYS_EXTERNAL = ["mupdf", "@oh-my-pi/pi-natives", "@huggingface/transfor
|
||||
// redundant copy that bloats dist/cli.js. NEVER add a patched dependency here — the
|
||||
// bundle is where a root `patchedDependencies` patch is baked in, so an externalized
|
||||
// import would load the unpatched npm package in users' installs (currently
|
||||
// beautiful-mermaid and @ark/schema are patched, so they — and arktype, which pulls
|
||||
// @ark/schema — stay bundled).
|
||||
// @ark/schema is patched, so it — and arktype, which pulls @ark/schema — stay
|
||||
// bundled).
|
||||
const RUNTIME_EXTERNAL = [
|
||||
"puppeteer-core",
|
||||
"@puppeteer/browsers",
|
||||
@@ -30,7 +30,6 @@ const RUNTIME_EXTERNAL = [
|
||||
"turndown-plugin-gfm",
|
||||
"@mozilla/readability",
|
||||
"linkedom",
|
||||
"markit-ai",
|
||||
"@agentclientprotocol/sdk",
|
||||
];
|
||||
|
||||
|
||||
Regular → Executable
+47
-31
@@ -1,40 +1,56 @@
|
||||
#!/usr/bin/env bun
|
||||
|
||||
/**
|
||||
* Populate (or reset) the embedded harness documentation index for `omp://`.
|
||||
*
|
||||
* `--generate` writes `src/internal-urls/docs-index.generated.txt` as two lines:
|
||||
* a plain JSON array of the sorted `docs/**\/*.md` file names, then a base64
|
||||
* gzip blob of the index-aligned doc bodies (`string[]`). Keeping the filename
|
||||
* list out of the blob lets the loader list docs without inflating it.
|
||||
* Compiled binaries and the prepacked npm bundle inline this (~0.5MB) instead of
|
||||
* the ~1.6MB raw map; `--reset` restores the checked-in empty placeholder so the
|
||||
* dev tree reads `docs/` from disk. Mirrors the stats / model-catalog embeds.
|
||||
*/
|
||||
|
||||
import * as path from "node:path";
|
||||
import { gzipSync } from "node:zlib";
|
||||
import { Glob } from "bun";
|
||||
|
||||
const docsDir = path.resolve(import.meta.dir, "../../../docs");
|
||||
const outputPath = path.resolve(import.meta.dir, "../src/internal-urls/docs-index.generated.ts");
|
||||
const outputPath = path.resolve(import.meta.dir, "../src/internal-urls/docs-index.generated.txt");
|
||||
const GENERATE_FLAG = "--generate";
|
||||
const RESET_FLAG = "--reset";
|
||||
|
||||
const glob = new Glob("**/*.md");
|
||||
const entries: string[] = [];
|
||||
for await (const relativePath of glob.scan(docsDir)) {
|
||||
entries.push(relativePath.split(path.sep).join("/"));
|
||||
async function main(): Promise<void> {
|
||||
const rel = path.relative(process.cwd(), outputPath);
|
||||
|
||||
if (process.argv.includes(RESET_FLAG)) {
|
||||
await Bun.write(outputPath, "");
|
||||
console.log(`Reset ${rel}`);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!process.argv.includes(GENERATE_FLAG)) {
|
||||
console.log(`Skipping ${rel}; pass ${GENERATE_FLAG} to embed docs (the dev tree reads docs/ from disk)`);
|
||||
return;
|
||||
}
|
||||
|
||||
const glob = new Glob("**/*.md");
|
||||
const files: string[] = [];
|
||||
for await (const relativePath of glob.scan(docsDir)) {
|
||||
files.push(relativePath.split(path.sep).join("/"));
|
||||
}
|
||||
files.sort();
|
||||
|
||||
// Index-aligned bodies (Promise.all preserves order), kept separate from the
|
||||
// filename list so the loader can list docs without inflating the blob.
|
||||
const bodies = await Promise.all(files.map(file => Bun.file(path.join(docsDir, file)).text()));
|
||||
|
||||
const bodiesB64 = Buffer.from(gzipSync(Buffer.from(JSON.stringify(bodies)), { level: 9 })).toString("base64");
|
||||
// Two lines: plain filename array, then the base64 gzip blob.
|
||||
const payload = `${JSON.stringify(files)}\n${bodiesB64}`;
|
||||
await Bun.write(outputPath, payload);
|
||||
console.log(`Generated ${rel} (${files.length} docs, ${payload.length} bytes)`);
|
||||
}
|
||||
entries.sort();
|
||||
|
||||
const docsWithContent = await Promise.all(
|
||||
entries.map(async relativePath => ({
|
||||
relativePath,
|
||||
content: await Bun.file(path.join(docsDir, relativePath)).text(),
|
||||
})),
|
||||
);
|
||||
|
||||
const filenamesLiteral = JSON.stringify(entries);
|
||||
|
||||
const mapEntries = docsWithContent
|
||||
.map(({ relativePath, content }) => `\t${JSON.stringify(relativePath)}: ${JSON.stringify(content)},`)
|
||||
.join("\n");
|
||||
const output = [
|
||||
"// Auto-generated by scripts/generate-docs-index.ts - DO NOT EDIT",
|
||||
"",
|
||||
`export const EMBEDDED_DOC_FILENAMES: readonly string[] = ${filenamesLiteral};`,
|
||||
"",
|
||||
`export const EMBEDDED_DOCS: Readonly<Record<string, string>> = {`,
|
||||
`${mapEntries}`,
|
||||
`};`,
|
||||
"",
|
||||
].join("\n");
|
||||
|
||||
await Bun.write(outputPath, output);
|
||||
console.log(`Generated ${path.relative(process.cwd(), outputPath)} (${entries.length} docs)`);
|
||||
await main();
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
/**
|
||||
* Harness documentation index for the `omp://` protocol.
|
||||
*
|
||||
* Compiled binaries and the prepacked npm bundle inline a compressed index from
|
||||
* `docs-index.generated.txt` (populated by `scripts/generate-docs-index.ts
|
||||
* --generate` at build time). The format is two lines:
|
||||
* 1. a plain JSON array of the sorted doc file names, and
|
||||
* 2. a base64 gzip blob of the index-aligned doc bodies (`string[]`).
|
||||
* Listing/completion (`getDocFilenames`) parses only the small first line and
|
||||
* never inflates the blob; the bodies are gunzipped off the event loop (via the
|
||||
* async `node:zlib` threadpool) lazily, once, on the first actual read. When the
|
||||
* placeholder is empty (dev tree, source checkout), the index is read from the
|
||||
* repo `docs/` directory on disk instead.
|
||||
*/
|
||||
import { readFileSync } from "node:fs";
|
||||
import * as path from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
import { gunzip } from "node:zlib";
|
||||
import { Glob } from "bun";
|
||||
import docsEmbed from "./docs-index.generated.txt";
|
||||
|
||||
const gunzipAsync = promisify(gunzip);
|
||||
|
||||
export interface DocsIndex {
|
||||
/** Sorted documentation file names, relative to `docs/`. */
|
||||
readonly filenames: readonly string[];
|
||||
/** Resolve a doc body by path; inflates the embedded bodies off-thread, lazily, on first call. */
|
||||
getBody(relativePath: string): Promise<string | undefined>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Decode a populated two-line embed (`<filenames JSON>\n<base64 gzip of bodies>`)
|
||||
* into a lazily-inflating index, or `null` when there is no newline separator
|
||||
* (the empty placeholder, or a malformed payload — the caller decides which).
|
||||
* Reading `filenames` never touches the blob; the bodies are gunzipped off the
|
||||
* event loop into a path→content table on the first `getBody` call, and that
|
||||
* work is shared across concurrent reads.
|
||||
*/
|
||||
export function decodeDocsIndex(embed: string): DocsIndex | null {
|
||||
const newline = embed.indexOf("\n");
|
||||
if (newline === -1) return null;
|
||||
const filenames = JSON.parse(embed.slice(0, newline)) as string[];
|
||||
let bodies: Promise<Record<string, string>> | undefined;
|
||||
return {
|
||||
filenames,
|
||||
getBody(relativePath: string): Promise<string | undefined> {
|
||||
bodies ??= (async () => {
|
||||
const inflated = await gunzipAsync(Buffer.from(embed.slice(newline + 1), "base64"));
|
||||
const decoded = JSON.parse(inflated.toString("utf8")) as string[];
|
||||
const map: Record<string, string> = {};
|
||||
for (let i = 0; i < filenames.length; i++) map[filenames[i]] = decoded[i];
|
||||
return map;
|
||||
})();
|
||||
return bodies.then(map => map[relativePath]);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** Dev tree / source checkout: build the index from the repo `docs/` directory. */
|
||||
function readDocsFromDisk(): DocsIndex {
|
||||
const docsDir = path.resolve(import.meta.dir, "../../../../docs");
|
||||
const filenames: string[] = [];
|
||||
const bodies: Record<string, string> = {};
|
||||
for (const relativePath of new Glob("**/*.md").scanSync(docsDir)) {
|
||||
const normalized = relativePath.split(path.sep).join("/");
|
||||
filenames.push(normalized);
|
||||
bodies[normalized] = readFileSync(path.join(docsDir, relativePath), "utf8");
|
||||
}
|
||||
filenames.sort();
|
||||
return { filenames, getBody: relativePath => Promise.resolve(bodies[relativePath]) };
|
||||
}
|
||||
|
||||
let index: DocsIndex | undefined;
|
||||
function getIndex(): DocsIndex {
|
||||
if (index !== undefined) return index;
|
||||
// Empty placeholder → dev tree / source checkout: read docs from disk.
|
||||
if (docsEmbed.length === 0) {
|
||||
index = readDocsFromDisk();
|
||||
return index;
|
||||
}
|
||||
// Populated embed in compiled binaries / npm bundle. A non-empty payload with
|
||||
// no newline is a broken build (truncated/corrupt embed), not a placeholder.
|
||||
const decoded = decodeDocsIndex(docsEmbed);
|
||||
if (decoded === null) {
|
||||
throw new Error(
|
||||
"Malformed embedded docs index (docs-index.generated.txt): non-empty payload without a newline separator. " +
|
||||
"Rebuild with `bun --cwd=packages/coding-agent scripts/generate-docs-index.ts --generate`.",
|
||||
);
|
||||
}
|
||||
index = decoded;
|
||||
return index;
|
||||
}
|
||||
|
||||
/** Sorted list of available documentation file names (relative to `docs/`). */
|
||||
export function getDocFilenames(): readonly string[] {
|
||||
return getIndex().filenames;
|
||||
}
|
||||
|
||||
/** Resolve a documentation file's content, or `undefined` when not found. */
|
||||
export function getEmbeddedDoc(relativePath: string): Promise<string | undefined> {
|
||||
return getIndex().getBody(relativePath);
|
||||
}
|
||||
@@ -8,7 +8,7 @@
|
||||
* - omp://<file>.md - Reads a specific documentation file
|
||||
*/
|
||||
import * as path from "node:path";
|
||||
import { EMBEDDED_DOC_FILENAMES, EMBEDDED_DOCS } from "./docs-index.generated";
|
||||
import { getDocFilenames, getEmbeddedDoc } from "./docs-index";
|
||||
import type { InternalResource, InternalUrl, ProtocolHandler, UrlCompletion } from "./types";
|
||||
|
||||
/**
|
||||
@@ -34,16 +34,17 @@ export class OmpProtocolHandler implements ProtocolHandler {
|
||||
}
|
||||
|
||||
async complete(): Promise<UrlCompletion[]> {
|
||||
return EMBEDDED_DOC_FILENAMES.map(value => ({ value }));
|
||||
return getDocFilenames().map(value => ({ value }));
|
||||
}
|
||||
|
||||
async #listDocs(url: InternalUrl): Promise<InternalResource> {
|
||||
if (EMBEDDED_DOC_FILENAMES.length === 0) {
|
||||
const filenames = getDocFilenames();
|
||||
if (filenames.length === 0) {
|
||||
throw new Error("No documentation files found");
|
||||
}
|
||||
|
||||
const listing = EMBEDDED_DOC_FILENAMES.map(f => `- [${f}](omp://${f})`).join("\n");
|
||||
const content = `# Documentation\n\n${EMBEDDED_DOC_FILENAMES.length} files available:\n\n${listing}\n`;
|
||||
const listing = filenames.map(f => `- [${f}](omp://${f})`).join("\n");
|
||||
const content = `# Documentation\n\n${filenames.length} files available:\n\n${listing}\n`;
|
||||
|
||||
return {
|
||||
url: url.href,
|
||||
@@ -70,12 +71,12 @@ export class OmpProtocolHandler implements ProtocolHandler {
|
||||
return this.#listDocs(url);
|
||||
}
|
||||
|
||||
const content = EMBEDDED_DOCS[docPath];
|
||||
const content = await getEmbeddedDoc(docPath);
|
||||
if (content === undefined) {
|
||||
const lookup = docPath.replace(/\.md$/, "");
|
||||
const suggestions = EMBEDDED_DOC_FILENAMES.filter(
|
||||
f => f.includes(lookup) || lookup.includes(f.replace(/\.md$/, "")),
|
||||
).slice(0, 5);
|
||||
const suggestions = getDocFilenames()
|
||||
.filter(f => f.includes(lookup) || lookup.includes(f.replace(/\.md$/, "")))
|
||||
.slice(0, 5);
|
||||
const suffix =
|
||||
suggestions.length > 0
|
||||
? `\nDid you mean: ${suggestions.join(", ")}`
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
import { describe, expect, it } from "bun:test";
|
||||
import { gzipSync } from "node:zlib";
|
||||
import { decodeDocsIndex } from "@oh-my-pi/pi-coding-agent/internal-urls/docs-index";
|
||||
|
||||
// The embed path only runs in compiled binaries / the npm bundle; dev tests
|
||||
// otherwise exercise the disk fallback (empty placeholder), so a regression in
|
||||
// the two-line `<filenames>\n<gzip bodies>` parsing would ship broken `omp://`
|
||||
// docs undetected. These cover the populated-embed decode directly.
|
||||
describe("decodeDocsIndex (embedded docs path)", () => {
|
||||
const files = ["agent.md", "tools/read.md"];
|
||||
const bodies = ["agent body", "read body"];
|
||||
const embed = `${JSON.stringify(files)}\n${Buffer.from(gzipSync(Buffer.from(JSON.stringify(bodies)))).toString("base64")}`;
|
||||
|
||||
it("lists filenames from the first line without inflating the blob", () => {
|
||||
// A deliberately corrupt blob: filenames must resolve anyway, proving the
|
||||
// listing path never decodes the gzip body.
|
||||
const index = decodeDocsIndex(`${JSON.stringify(files)}\n@@@not-a-valid-gzip-blob@@@`);
|
||||
expect(index?.filenames).toEqual(files);
|
||||
});
|
||||
|
||||
it("resolves bodies by index-aligned path, lazily, on first read", async () => {
|
||||
const index = decodeDocsIndex(embed);
|
||||
expect(index).not.toBeNull();
|
||||
expect(await index?.getBody("agent.md")).toBe("agent body");
|
||||
expect(await index?.getBody("tools/read.md")).toBe("read body");
|
||||
expect(await index?.getBody("missing.md")).toBeUndefined();
|
||||
});
|
||||
|
||||
it("returns null when there is no newline separator (empty placeholder)", () => {
|
||||
expect(decodeDocsIndex("")).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -153,19 +153,23 @@ async function buildBinary(target: BinaryTarget): Promise<void> {
|
||||
async function generateBundle(): Promise<void> {
|
||||
if (isDryRun) {
|
||||
console.log("DRY RUN bun --cwd=packages/stats scripts/generate-client-bundle.ts --generate");
|
||||
console.log("DRY RUN bun --cwd=packages/coding-agent scripts/generate-docs-index.ts --generate");
|
||||
return;
|
||||
}
|
||||
await runCommand(["bun", "--cwd=packages/stats", "scripts/generate-client-bundle.ts", "--generate"], repoRoot);
|
||||
await runCommand(["bun", "--cwd=packages/coding-agent", "scripts/generate-docs-index.ts", "--generate"], repoRoot);
|
||||
}
|
||||
|
||||
async function resetArtifacts(): Promise<void> {
|
||||
if (isDryRun) {
|
||||
console.log("DRY RUN bun --cwd=packages/natives run embed:native --reset");
|
||||
console.log("DRY RUN bun --cwd=packages/stats scripts/generate-client-bundle.ts --reset");
|
||||
console.log("DRY RUN bun --cwd=packages/coding-agent scripts/generate-docs-index.ts --reset");
|
||||
return;
|
||||
}
|
||||
await runCommand(["bun", "--cwd=packages/natives", "run", "embed:native", "--reset"], repoRoot);
|
||||
await runCommand(["bun", "--cwd=packages/stats", "scripts/generate-client-bundle.ts", "--reset"], repoRoot);
|
||||
await runCommand(["bun", "--cwd=packages/coding-agent", "scripts/generate-docs-index.ts", "--reset"], repoRoot);
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
@@ -188,8 +192,10 @@ async function main(): Promise<void> {
|
||||
}
|
||||
|
||||
await fs.mkdir(binariesDir, { recursive: true });
|
||||
await generateBundle();
|
||||
// Generate inside the try so resetArtifacts() always restores the empty
|
||||
// checked-in placeholders, even if a generate or build step throws.
|
||||
try {
|
||||
await generateBundle();
|
||||
for (const target of selectedTargets) {
|
||||
await buildBinary(target);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user