feat(coding-agent): vendor relevant parts of markit

packages/coding-agent/src/utils/markit.ts

@@ -1,5 +1,5 @@
 import { logger, untilAborted } from "@oh-my-pi/pi-utils";
-import type { Markit, StreamInfo } from "markit-ai";
+import type { Markit, StreamInfo } from "../markit";
 import { ToolAbortError } from "../tools/tool-errors";
 
 export interface MarkitConversionResult {
@@ -23,26 +23,27 @@ interface MuPdfWasmModuleConfig {
 	printErr?: (...values: unknown[]) => void;
 }
 
-declare global {
-	var $libmupdf_wasm_Module: MuPdfWasmModuleConfig | undefined;
-}
-
 function logMuPdfWasmOutput(stream: "stdout" | "stderr", values: unknown[]): void {
 	const message = values.length === 1 && typeof values[0] === "string" ? values[0] : values.map(String).join(" ");
 	logger.debug("mupdf wasm output", { stream, message });
 }
 
+// `$libmupdf_wasm_Module` is declared globally (as `any`) by the mupdf package.
+// Install print hooks before the WASM module initializes so its stdout/stderr
+// route to the file logger instead of corrupting the TUI.
 function installMuPdfWasmLogger(): void {
-	const moduleConfig = globalThis.$libmupdf_wasm_Module ?? {};
-	moduleConfig.print = (...values) => logMuPdfWasmOutput("stdout", values);
-	moduleConfig.printErr = (...values) => logMuPdfWasmOutput("stderr", values);
+	const moduleConfig: MuPdfWasmModuleConfig = globalThis.$libmupdf_wasm_Module ?? {};
+	moduleConfig.print = (...values: unknown[]) => logMuPdfWasmOutput("stdout", values);
+	moduleConfig.printErr = (...values: unknown[]) => logMuPdfWasmOutput("stderr", values);
 	globalThis.$libmupdf_wasm_Module = moduleConfig;
 }
 
 installMuPdfWasmLogger();
 
 let markit: () => Markit | Promise<Markit> = async () => {
-	const promise = import("markit-ai").then(({ Markit }) => {
+	// Lazy: keep the document engine (mammoth/fflate/mupdf) off the startup
+	// import graph — it loads only when a document is first converted.
+	const promise = import("../markit").then(({ Markit }) => {
 		const instance = new Markit();
 		markit = () => instance;
 		return instance;

packages/coding-agent/src/utils/turndown.ts

@@ -0,0 +1,83 @@
+import TurndownService from "turndown";
+import { gfm } from "turndown-plugin-gfm";
+
+type TurndownListParent = {
+	nodeName: string;
+	getAttribute(name: string): string | null;
+	children: ArrayLike<unknown>;
+};
+
+/**
+ * Build a Turndown instance configured for GFM with the fixes omp relies on:
+ * `~~strikethrough~~`, unescaped heading periods, and single-space list markers.
+ *
+ * Shared by the web scrapers (HTML → markdown) and the markit document engine
+ * (`src/markit`). The rule set must stay identical across both call sites.
+ */
+export function createTurndown(): TurndownService {
+	const turndown = new TurndownService({
+		headingStyle: "atx",
+		codeBlockStyle: "fenced",
+		bulletListMarker: "-",
+	});
+	turndown.use(gfm);
+	// GFM spec uses ~~ (double tilde), not ~ (single)
+	turndown.addRule("strikethrough", {
+		filter: ["del", "s", "strike"],
+		replacement(content) {
+			return `~~${content}~~`;
+		},
+	});
+	// Unescape the backslash turndown inserts before periods in headings ("1." -> "1\.")
+	turndown.addRule("heading", {
+		filter: ["h1", "h2", "h3", "h4", "h5", "h6"],
+		replacement(content, node) {
+			const level = Number(node.nodeName.charAt(1));
+			const prefix = "#".repeat(level);
+			const cleaned = content.replace(/\\([.])/g, "$1").trim();
+			return `\n\n${prefix} ${cleaned}\n\n`;
+		},
+	});
+	// Single space after the marker (turndown hardcodes three)
+	turndown.addRule("listItem", {
+		filter: "li",
+		replacement(content, node, options) {
+			const body = content.replace(/^\n+/, "").replace(/\n+$/, "\n").replace(/\n/gm, "\n  ");
+			const parent = node.parentNode as unknown as TurndownListParent | null;
+			let prefix = `${options.bulletListMarker} `;
+			if (parent?.nodeName === "OL") {
+				const start = parent.getAttribute("start");
+				const index = Array.prototype.indexOf.call(parent.children, node);
+				prefix = `${(start ? Number(start) : 1) + index}. `;
+			}
+			return prefix + body + (node.nextSibling ? "\n" : "");
+		},
+	});
+	return turndown;
+}
+
+/**
+ * Normalize HTML tables so turndown-plugin-gfm can render them:
+ * - strip `<p>` tags inside `<td>`/`<th>` cells (joining paragraphs with a space)
+ * - wrap the first row in `<thead>` when missing
+ */
+export function normalizeTablesHtml(html: string): string {
+	let result = html.replace(
+		/<(td|th)([^>]*)>([\s\S]*?)<\/(td|th)>/gi,
+		(_match, tag: string, attrs: string, inner: string, closeTag: string) => {
+			const stripped = inner
+				.replace(/^\s*<p>/i, "")
+				.replace(/<\/p>\s*$/i, "")
+				.replace(/<\/p>\s*<p>/gi, " ");
+			return `<${tag}${attrs}>${stripped}</${closeTag}>`;
+		},
+	);
+	result = result.replace(
+		/<table([^>]*)>\s*(?:<tbody>\s*)?(<tr[\s\S]*?<\/tr>)([\s\S]*?)<\/(?:tbody>\s*<\/)?table>/gi,
+		(_match, attrs: string, firstRow: string, rest: string) => {
+			const theadRow = firstRow.replace(/<td/gi, "<th").replace(/<\/td>/gi, "</th>");
+			return `<table${attrs}><thead>${theadRow}</thead><tbody>${rest}</tbody></table>`;
+		},
+	);
+	return result;
+}

packages/coding-agent/src/utils/zip.ts

@@ -0,0 +1,29 @@
+// The single ZIP/DEFLATE boundary for the codebase. This is the ONLY module
+// that imports `fflate`; the markit document converters, the write tool, and
+// the archive reader all go through here so there is exactly one ZIP
+// implementation to reason about. Do not import `fflate` (or another archive
+// library) anywhere else.
+import type { Unzipped } from "fflate";
+import { inflateSync, strFromU8 } from "fflate";
+
+export type { Unzipped } from "fflate";
+export { unzipSync as unzip, zipSync as zip } from "fflate";
+
+/** Read a single ZIP entry as UTF-8 text, or `undefined` when the entry is absent. */
+export function unzipText(entries: Unzipped, entryPath: string): string | undefined {
+	const data = entries[entryPath];
+	return data ? strFromU8(data) : undefined;
+}
+
+/**
+ * Inflate a raw DEFLATE stream (a single deflate-compressed ZIP member). Pass a
+ * preallocated `into` buffer when the uncompressed size is known up front.
+ */
+export function inflateRaw(bytes: Uint8Array, into?: Uint8Array): Uint8Array {
+	return into ? inflateSync(bytes, { out: into }) : inflateSync(bytes);
+}
+
+/** Decode raw bytes as text — UTF-8 by default, latin1 when `latin1` is set. */
+export function bytesToText(bytes: Uint8Array, latin1?: boolean): string {
+	return strFromU8(bytes, latin1);
+}