From 831b664a4c332219d4fd642559b99e93b15f3f13 Mon Sep 17 00:00:00 2001 From: can1357 Date: Thu, 18 Jun 2026 18:57:13 +0200 Subject: [PATCH] perf(coding-agent): optimized documentation indexing via compressed blob - Replaced raw TypeScript documentation map with a lazily-inflated gzip blob. - Reduced bundled binary/npm package size by approximately 0.9MB. - Encapsulated index logic into `docs-index.ts` to separate header metadata from content. - Added `postpack` and robust `try/finally` patterns in build scripts to ensure clean artifacts. - Implemented disk-based fallback during development to maintain existing developer experience. --- .fallowrc.jsonc | 2 - packages/coding-agent/CHANGELOG.md | 4 +- packages/coding-agent/package.json | 7 +- packages/coding-agent/scripts/build-binary.ts | 6 +- packages/coding-agent/scripts/bundle-dist.ts | 5 +- .../scripts/generate-docs-index.ts | 78 ++++++++------ .../src/internal-urls/docs-index.ts | 102 ++++++++++++++++++ .../src/internal-urls/omp-protocol.ts | 19 ++-- .../test/internal-urls/docs-index.test.ts | 32 ++++++ scripts/ci-release-build-binaries.ts | 8 +- 10 files changed, 212 insertions(+), 51 deletions(-) mode change 100644 => 100755 packages/coding-agent/scripts/generate-docs-index.ts create mode 100644 packages/coding-agent/src/internal-urls/docs-index.ts create mode 100644 packages/coding-agent/test/internal-urls/docs-index.test.ts diff --git a/.fallowrc.jsonc b/.fallowrc.jsonc index 5f4fc8844..d15167939 100644 --- a/.fallowrc.jsonc +++ b/.fallowrc.jsonc @@ -20,8 +20,6 @@ "ignore": [ // Generated from `packages/natives/scripts/native-index.template.js` via gen-enums.ts. "packages/natives/native/index.js", - // Generated by `packages/coding-agent/scripts/generate-docs-index.ts`. - "packages/coding-agent/src/internal-urls/docs-index.generated.ts", // Embedded HTML asset shipped as a static template, not a code module. "packages/coding-agent/src/export/html/template.js", // Generated/owned upstream — see packages/ai/scripts/generate-models.ts. diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index bb7fec891..b0af27b67 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -3,7 +3,9 @@ ## [Unreleased] ### Changed +- Optimized `omp://` documentation indexing by compressing doc bodies into a lazily-inflated blob - Changed Mermaid fenced-block ASCII rendering to use the first-party vendored renderer in `@oh-my-pi/pi-utils` (`src/vendor/mermaid-ascii`), dropping the `beautiful-mermaid` npm package, its transitive `elkjs` (~3.13MB), and the `beautiful-mermaid` `bun patch`; CJK/emoji width handling and the layout-direction override are preserved. +- Changed `omp://` documentation embedding to a gzipped base64 index (`docs-index.generated.txt`, populated at build time and reset afterward) inflated on first read, instead of a ~1.6MB raw TypeScript map. The compiled binary / npm bundle drops ~0.9MB; the dev tree and source checkouts read `docs/` from disk. ### Fixed @@ -11954,4 +11956,4 @@ Initial public release. ## [0.7.6] - 2025-11-13 -Previous releases did not maintain a changelog. +Previous releases did not maintain a changelog. \ No newline at end of file diff --git a/packages/coding-agent/package.json b/packages/coding-agent/package.json index e70754362..926667355 100644 --- a/packages/coding-agent/package.json +++ b/packages/coding-agent/package.json @@ -36,11 +36,12 @@ "check:types": "tsgo -p tsconfig.json --noEmit", "lint": "biome lint .", "test": "bun test --parallel=4", - "fix": "biome check --write --unsafe . && bun run format-prompts && bun run generate-docs-index", + "fix": "biome check --write --unsafe . && bun run format-prompts", "fmt": "biome format --write . && bun run format-prompts", "format-prompts": "bun scripts/format-prompts.ts", - "generate-docs-index": "bun scripts/generate-docs-index.ts", - "prepack": "bun scripts/generate-docs-index.ts && bun --cwd=../collab-web run build:tool-views && bun scripts/bundle-dist.ts", + "generate-docs-index": "bun scripts/generate-docs-index.ts --generate", + "prepack": "bun scripts/generate-docs-index.ts --generate && bun --cwd=../collab-web run build:tool-views && bun scripts/bundle-dist.ts || ( bun scripts/generate-docs-index.ts --reset; exit 1 )", + "postpack": "bun scripts/generate-docs-index.ts --reset", "bench:guard": "bun scripts/bench-guard.ts" }, "dependencies": { diff --git a/packages/coding-agent/scripts/build-binary.ts b/packages/coding-agent/scripts/build-binary.ts index 17e0f0cd2..2c122afcb 100644 --- a/packages/coding-agent/scripts/build-binary.ts +++ b/packages/coding-agent/scripts/build-binary.ts @@ -38,8 +38,11 @@ async function runCommand( } async function main(): Promise { - await runCommand(["bun", "--cwd=../stats", "scripts/generate-client-bundle.ts", "--generate"]); + // Generate inside the try so the finally always restores the empty checked-in + // placeholders (stats client archive, docs index) even on failure. try { + await runCommand(["bun", "--cwd=../stats", "scripts/generate-client-bundle.ts", "--generate"]); + await runCommand(["bun", "scripts/generate-docs-index.ts", "--generate"]); await runCommand(["bun", "--cwd=../natives", "run", "embed:native"]); try { const buildEnv = shouldAdhocSignDarwinBinary() ? { ...Bun.env, BUN_NO_CODESIGN_MACHO_BINARY: "1" } : Bun.env; @@ -98,6 +101,7 @@ async function main(): Promise { } } finally { await runCommand(["bun", "--cwd=../stats", "scripts/generate-client-bundle.ts", "--reset"]); + await runCommand(["bun", "scripts/generate-docs-index.ts", "--reset"]); } } diff --git a/packages/coding-agent/scripts/bundle-dist.ts b/packages/coding-agent/scripts/bundle-dist.ts index 19973a180..322121802 100755 --- a/packages/coding-agent/scripts/bundle-dist.ts +++ b/packages/coding-agent/scripts/bundle-dist.ts @@ -19,8 +19,8 @@ const ALWAYS_EXTERNAL = ["mupdf", "@oh-my-pi/pi-natives", "@huggingface/transfor // redundant copy that bloats dist/cli.js. NEVER add a patched dependency here — the // bundle is where a root `patchedDependencies` patch is baked in, so an externalized // import would load the unpatched npm package in users' installs (currently -// beautiful-mermaid and @ark/schema are patched, so they — and arktype, which pulls -// @ark/schema — stay bundled). +// @ark/schema is patched, so it — and arktype, which pulls @ark/schema — stay +// bundled). const RUNTIME_EXTERNAL = [ "puppeteer-core", "@puppeteer/browsers", @@ -30,7 +30,6 @@ const RUNTIME_EXTERNAL = [ "turndown-plugin-gfm", "@mozilla/readability", "linkedom", - "markit-ai", "@agentclientprotocol/sdk", ]; diff --git a/packages/coding-agent/scripts/generate-docs-index.ts b/packages/coding-agent/scripts/generate-docs-index.ts old mode 100644 new mode 100755 index cd5e8ba60..840f2d013 --- a/packages/coding-agent/scripts/generate-docs-index.ts +++ b/packages/coding-agent/scripts/generate-docs-index.ts @@ -1,40 +1,56 @@ #!/usr/bin/env bun +/** + * Populate (or reset) the embedded harness documentation index for `omp://`. + * + * `--generate` writes `src/internal-urls/docs-index.generated.txt` as two lines: + * a plain JSON array of the sorted `docs/**\/*.md` file names, then a base64 + * gzip blob of the index-aligned doc bodies (`string[]`). Keeping the filename + * list out of the blob lets the loader list docs without inflating it. + * Compiled binaries and the prepacked npm bundle inline this (~0.5MB) instead of + * the ~1.6MB raw map; `--reset` restores the checked-in empty placeholder so the + * dev tree reads `docs/` from disk. Mirrors the stats / model-catalog embeds. + */ + import * as path from "node:path"; +import { gzipSync } from "node:zlib"; import { Glob } from "bun"; const docsDir = path.resolve(import.meta.dir, "../../../docs"); -const outputPath = path.resolve(import.meta.dir, "../src/internal-urls/docs-index.generated.ts"); +const outputPath = path.resolve(import.meta.dir, "../src/internal-urls/docs-index.generated.txt"); +const GENERATE_FLAG = "--generate"; +const RESET_FLAG = "--reset"; -const glob = new Glob("**/*.md"); -const entries: string[] = []; -for await (const relativePath of glob.scan(docsDir)) { - entries.push(relativePath.split(path.sep).join("/")); +async function main(): Promise { + const rel = path.relative(process.cwd(), outputPath); + + if (process.argv.includes(RESET_FLAG)) { + await Bun.write(outputPath, ""); + console.log(`Reset ${rel}`); + return; + } + + if (!process.argv.includes(GENERATE_FLAG)) { + console.log(`Skipping ${rel}; pass ${GENERATE_FLAG} to embed docs (the dev tree reads docs/ from disk)`); + return; + } + + const glob = new Glob("**/*.md"); + const files: string[] = []; + for await (const relativePath of glob.scan(docsDir)) { + files.push(relativePath.split(path.sep).join("/")); + } + files.sort(); + + // Index-aligned bodies (Promise.all preserves order), kept separate from the + // filename list so the loader can list docs without inflating the blob. + const bodies = await Promise.all(files.map(file => Bun.file(path.join(docsDir, file)).text())); + + const bodiesB64 = Buffer.from(gzipSync(Buffer.from(JSON.stringify(bodies)), { level: 9 })).toString("base64"); + // Two lines: plain filename array, then the base64 gzip blob. + const payload = `${JSON.stringify(files)}\n${bodiesB64}`; + await Bun.write(outputPath, payload); + console.log(`Generated ${rel} (${files.length} docs, ${payload.length} bytes)`); } -entries.sort(); -const docsWithContent = await Promise.all( - entries.map(async relativePath => ({ - relativePath, - content: await Bun.file(path.join(docsDir, relativePath)).text(), - })), -); - -const filenamesLiteral = JSON.stringify(entries); - -const mapEntries = docsWithContent - .map(({ relativePath, content }) => `\t${JSON.stringify(relativePath)}: ${JSON.stringify(content)},`) - .join("\n"); -const output = [ - "// Auto-generated by scripts/generate-docs-index.ts - DO NOT EDIT", - "", - `export const EMBEDDED_DOC_FILENAMES: readonly string[] = ${filenamesLiteral};`, - "", - `export const EMBEDDED_DOCS: Readonly> = {`, - `${mapEntries}`, - `};`, - "", -].join("\n"); - -await Bun.write(outputPath, output); -console.log(`Generated ${path.relative(process.cwd(), outputPath)} (${entries.length} docs)`); +await main(); diff --git a/packages/coding-agent/src/internal-urls/docs-index.ts b/packages/coding-agent/src/internal-urls/docs-index.ts new file mode 100644 index 000000000..c35b794b1 --- /dev/null +++ b/packages/coding-agent/src/internal-urls/docs-index.ts @@ -0,0 +1,102 @@ +/** + * Harness documentation index for the `omp://` protocol. + * + * Compiled binaries and the prepacked npm bundle inline a compressed index from + * `docs-index.generated.txt` (populated by `scripts/generate-docs-index.ts + * --generate` at build time). The format is two lines: + * 1. a plain JSON array of the sorted doc file names, and + * 2. a base64 gzip blob of the index-aligned doc bodies (`string[]`). + * Listing/completion (`getDocFilenames`) parses only the small first line and + * never inflates the blob; the bodies are gunzipped off the event loop (via the + * async `node:zlib` threadpool) lazily, once, on the first actual read. When the + * placeholder is empty (dev tree, source checkout), the index is read from the + * repo `docs/` directory on disk instead. + */ +import { readFileSync } from "node:fs"; +import * as path from "node:path"; +import { promisify } from "node:util"; +import { gunzip } from "node:zlib"; +import { Glob } from "bun"; +import docsEmbed from "./docs-index.generated.txt"; + +const gunzipAsync = promisify(gunzip); + +export interface DocsIndex { + /** Sorted documentation file names, relative to `docs/`. */ + readonly filenames: readonly string[]; + /** Resolve a doc body by path; inflates the embedded bodies off-thread, lazily, on first call. */ + getBody(relativePath: string): Promise; +} + +/** + * Decode a populated two-line embed (`\n`) + * into a lazily-inflating index, or `null` when there is no newline separator + * (the empty placeholder, or a malformed payload — the caller decides which). + * Reading `filenames` never touches the blob; the bodies are gunzipped off the + * event loop into a path→content table on the first `getBody` call, and that + * work is shared across concurrent reads. + */ +export function decodeDocsIndex(embed: string): DocsIndex | null { + const newline = embed.indexOf("\n"); + if (newline === -1) return null; + const filenames = JSON.parse(embed.slice(0, newline)) as string[]; + let bodies: Promise> | undefined; + return { + filenames, + getBody(relativePath: string): Promise { + bodies ??= (async () => { + const inflated = await gunzipAsync(Buffer.from(embed.slice(newline + 1), "base64")); + const decoded = JSON.parse(inflated.toString("utf8")) as string[]; + const map: Record = {}; + for (let i = 0; i < filenames.length; i++) map[filenames[i]] = decoded[i]; + return map; + })(); + return bodies.then(map => map[relativePath]); + }, + }; +} + +/** Dev tree / source checkout: build the index from the repo `docs/` directory. */ +function readDocsFromDisk(): DocsIndex { + const docsDir = path.resolve(import.meta.dir, "../../../../docs"); + const filenames: string[] = []; + const bodies: Record = {}; + for (const relativePath of new Glob("**/*.md").scanSync(docsDir)) { + const normalized = relativePath.split(path.sep).join("/"); + filenames.push(normalized); + bodies[normalized] = readFileSync(path.join(docsDir, relativePath), "utf8"); + } + filenames.sort(); + return { filenames, getBody: relativePath => Promise.resolve(bodies[relativePath]) }; +} + +let index: DocsIndex | undefined; +function getIndex(): DocsIndex { + if (index !== undefined) return index; + // Empty placeholder → dev tree / source checkout: read docs from disk. + if (docsEmbed.length === 0) { + index = readDocsFromDisk(); + return index; + } + // Populated embed in compiled binaries / npm bundle. A non-empty payload with + // no newline is a broken build (truncated/corrupt embed), not a placeholder. + const decoded = decodeDocsIndex(docsEmbed); + if (decoded === null) { + throw new Error( + "Malformed embedded docs index (docs-index.generated.txt): non-empty payload without a newline separator. " + + "Rebuild with `bun --cwd=packages/coding-agent scripts/generate-docs-index.ts --generate`.", + ); + } + index = decoded; + return index; +} + +/** Sorted list of available documentation file names (relative to `docs/`). */ +export function getDocFilenames(): readonly string[] { + return getIndex().filenames; +} + +/** Resolve a documentation file's content, or `undefined` when not found. */ +export function getEmbeddedDoc(relativePath: string): Promise { + return getIndex().getBody(relativePath); +} diff --git a/packages/coding-agent/src/internal-urls/omp-protocol.ts b/packages/coding-agent/src/internal-urls/omp-protocol.ts index ee5e13534..9d0c2f532 100644 --- a/packages/coding-agent/src/internal-urls/omp-protocol.ts +++ b/packages/coding-agent/src/internal-urls/omp-protocol.ts @@ -8,7 +8,7 @@ * - omp://.md - Reads a specific documentation file */ import * as path from "node:path"; -import { EMBEDDED_DOC_FILENAMES, EMBEDDED_DOCS } from "./docs-index.generated"; +import { getDocFilenames, getEmbeddedDoc } from "./docs-index"; import type { InternalResource, InternalUrl, ProtocolHandler, UrlCompletion } from "./types"; /** @@ -34,16 +34,17 @@ export class OmpProtocolHandler implements ProtocolHandler { } async complete(): Promise { - return EMBEDDED_DOC_FILENAMES.map(value => ({ value })); + return getDocFilenames().map(value => ({ value })); } async #listDocs(url: InternalUrl): Promise { - if (EMBEDDED_DOC_FILENAMES.length === 0) { + const filenames = getDocFilenames(); + if (filenames.length === 0) { throw new Error("No documentation files found"); } - const listing = EMBEDDED_DOC_FILENAMES.map(f => `- [${f}](omp://${f})`).join("\n"); - const content = `# Documentation\n\n${EMBEDDED_DOC_FILENAMES.length} files available:\n\n${listing}\n`; + const listing = filenames.map(f => `- [${f}](omp://${f})`).join("\n"); + const content = `# Documentation\n\n${filenames.length} files available:\n\n${listing}\n`; return { url: url.href, @@ -70,12 +71,12 @@ export class OmpProtocolHandler implements ProtocolHandler { return this.#listDocs(url); } - const content = EMBEDDED_DOCS[docPath]; + const content = await getEmbeddedDoc(docPath); if (content === undefined) { const lookup = docPath.replace(/\.md$/, ""); - const suggestions = EMBEDDED_DOC_FILENAMES.filter( - f => f.includes(lookup) || lookup.includes(f.replace(/\.md$/, "")), - ).slice(0, 5); + const suggestions = getDocFilenames() + .filter(f => f.includes(lookup) || lookup.includes(f.replace(/\.md$/, ""))) + .slice(0, 5); const suffix = suggestions.length > 0 ? `\nDid you mean: ${suggestions.join(", ")}` diff --git a/packages/coding-agent/test/internal-urls/docs-index.test.ts b/packages/coding-agent/test/internal-urls/docs-index.test.ts new file mode 100644 index 000000000..980727ba1 --- /dev/null +++ b/packages/coding-agent/test/internal-urls/docs-index.test.ts @@ -0,0 +1,32 @@ +import { describe, expect, it } from "bun:test"; +import { gzipSync } from "node:zlib"; +import { decodeDocsIndex } from "@oh-my-pi/pi-coding-agent/internal-urls/docs-index"; + +// The embed path only runs in compiled binaries / the npm bundle; dev tests +// otherwise exercise the disk fallback (empty placeholder), so a regression in +// the two-line `\n` parsing would ship broken `omp://` +// docs undetected. These cover the populated-embed decode directly. +describe("decodeDocsIndex (embedded docs path)", () => { + const files = ["agent.md", "tools/read.md"]; + const bodies = ["agent body", "read body"]; + const embed = `${JSON.stringify(files)}\n${Buffer.from(gzipSync(Buffer.from(JSON.stringify(bodies)))).toString("base64")}`; + + it("lists filenames from the first line without inflating the blob", () => { + // A deliberately corrupt blob: filenames must resolve anyway, proving the + // listing path never decodes the gzip body. + const index = decodeDocsIndex(`${JSON.stringify(files)}\n@@@not-a-valid-gzip-blob@@@`); + expect(index?.filenames).toEqual(files); + }); + + it("resolves bodies by index-aligned path, lazily, on first read", async () => { + const index = decodeDocsIndex(embed); + expect(index).not.toBeNull(); + expect(await index?.getBody("agent.md")).toBe("agent body"); + expect(await index?.getBody("tools/read.md")).toBe("read body"); + expect(await index?.getBody("missing.md")).toBeUndefined(); + }); + + it("returns null when there is no newline separator (empty placeholder)", () => { + expect(decodeDocsIndex("")).toBeNull(); + }); +}); diff --git a/scripts/ci-release-build-binaries.ts b/scripts/ci-release-build-binaries.ts index b97c59b72..819bd5052 100644 --- a/scripts/ci-release-build-binaries.ts +++ b/scripts/ci-release-build-binaries.ts @@ -153,19 +153,23 @@ async function buildBinary(target: BinaryTarget): Promise { async function generateBundle(): Promise { if (isDryRun) { console.log("DRY RUN bun --cwd=packages/stats scripts/generate-client-bundle.ts --generate"); + console.log("DRY RUN bun --cwd=packages/coding-agent scripts/generate-docs-index.ts --generate"); return; } await runCommand(["bun", "--cwd=packages/stats", "scripts/generate-client-bundle.ts", "--generate"], repoRoot); + await runCommand(["bun", "--cwd=packages/coding-agent", "scripts/generate-docs-index.ts", "--generate"], repoRoot); } async function resetArtifacts(): Promise { if (isDryRun) { console.log("DRY RUN bun --cwd=packages/natives run embed:native --reset"); console.log("DRY RUN bun --cwd=packages/stats scripts/generate-client-bundle.ts --reset"); + console.log("DRY RUN bun --cwd=packages/coding-agent scripts/generate-docs-index.ts --reset"); return; } await runCommand(["bun", "--cwd=packages/natives", "run", "embed:native", "--reset"], repoRoot); await runCommand(["bun", "--cwd=packages/stats", "scripts/generate-client-bundle.ts", "--reset"], repoRoot); + await runCommand(["bun", "--cwd=packages/coding-agent", "scripts/generate-docs-index.ts", "--reset"], repoRoot); } async function main(): Promise { @@ -188,8 +192,10 @@ async function main(): Promise { } await fs.mkdir(binariesDir, { recursive: true }); - await generateBundle(); + // Generate inside the try so resetArtifacts() always restores the empty + // checked-in placeholders, even if a generate or build step throws. try { + await generateBundle(); for (const target of selectedTargets) { await buildBinary(target); }