feat(coding-agent): added HTML content extraction with Readability and GFM markdown support

- Added `extractReadableFromHtml()` utility function with dual-path content extraction using Readability library and CSS selector fallback.
- Integrated Turndown library with GitHub Flavored Markdown plugin for improved HTML-to-markdown conversion supporting tables, strikethrough, and task lists.
- Refactored `getPageReadable()` action to use new extraction function, consolidating content parsing logic and improving maintainability.
- Added TypeScript type declarations for turndown-plugin-gfm module with custom Turndown rules for enhanced markdown formatting.
This commit is contained in:
can1357
2026-04-10 21:31:28 +02:00
parent 7213aaf128
commit cc2801e6ef
8 changed files with 202 additions and 50 deletions
+80 -16
View File
@@ -501,6 +501,83 @@ export interface ReadableResult {
markdown?: string;
}
type ReadableFormat = "text" | "markdown";
/** Trim to non-empty string or undefined. */
function normalize(text: string | null | undefined): string | undefined {
const trimmed = text?.trim();
return trimmed || undefined;
}
/**
* Extract readable content from raw HTML.
* Tries Readability (article-isolation scoring) first, then falls back to a
* CSS selector chain over the same pre-parsed DOM. Returns null if neither
* path yields usable content.
*/
export function extractReadableFromHtml(html: string, url: string, format: ReadableFormat): ReadableResult | null {
const { document } = parseHTML(html);
// --- Primary: Readability article extraction ---
const article = new Readability(document).parse();
if (article) {
const result = toReadableResult(url, format, article.textContent, article.content, {
title: article.title,
byline: article.byline,
excerpt: article.excerpt,
length: article.length,
});
if (result) return result;
}
// --- Fallback: CSS selector chain ---
const candidates = [
document.querySelector("[data-pagefind-body]"),
document.querySelector("main article"),
document.querySelector("article"),
document.querySelector("main"),
document.querySelector("[role='main']"),
document.body,
];
for (const el of candidates) {
if (!el) continue;
const innerHTML = el.innerHTML?.trim();
const textContent = el.textContent?.trim();
if (!innerHTML || !textContent) continue;
const result = toReadableResult(url, format, textContent, innerHTML, {
title: document.title,
excerpt: textContent.slice(0, 240),
length: textContent.length,
});
if (result) return result;
}
return null;
}
/** Shared builder for both extraction paths. */
function toReadableResult(
url: string,
format: ReadableFormat,
textContent: string | null | undefined,
htmlContent: string | null | undefined,
meta: { title?: string | null; byline?: string | null; excerpt?: string | null; length?: number | null },
): ReadableResult | null {
const text = normalize(textContent);
const markdown = format === "markdown" ? (normalize(htmlToBasicMarkdown(htmlContent ?? "")) ?? text) : undefined;
const normalizedText = format === "text" ? text : undefined;
if (!normalizedText && !markdown) return null;
return {
url,
title: normalize(meta.title),
byline: normalize(meta.byline),
excerpt: normalize(meta.excerpt),
contentLength: meta.length ?? text?.length ?? markdown?.length ?? 0,
text: normalizedText,
markdown,
};
}
function ensureParam<T>(value: T | undefined, name: string, action: string): T {
if (value === undefined || value === null || value === "") {
throw new ToolError(`Missing required parameter '${name}' for action '${action}'.`);
@@ -1365,26 +1442,13 @@ export class BrowserTool implements AgentTool<typeof browserSchema, BrowserToolD
const format = params.format ?? "markdown";
const html = (await untilAborted(signal, () => page.content())) as string;
const url = page.url();
const { document } = parseHTML(html);
const reader = new Readability(document);
const article = reader.parse();
if (!article) {
const readable = extractReadableFromHtml(html, url, format);
if (!readable) {
throw new ToolError("Readable content not found");
}
const markdown = format === "markdown" ? htmlToBasicMarkdown(article.content ?? "") : undefined;
const text = format === "text" ? (article.textContent ?? "") : undefined;
const readable: ReadableResult = {
url,
title: article.title ?? undefined,
byline: article.byline ?? undefined,
excerpt: article.excerpt ?? undefined,
contentLength: article.length ?? article.textContent?.length ?? 0,
text,
markdown,
};
details.url = url;
details.readable = readable;
details.result = format === "markdown" ? (markdown ?? "") : (text ?? "");
details.result = format === "markdown" ? (readable.markdown ?? "") : (readable.text ?? "");
return toolResult(details)
.text(JSON.stringify(readable, null, 2))
.done();