can1357
5a7cf77ba9
- Added 25+ new specialized web scrapers for sites including MusicBrainz, Discourse, Lemmy, Sourcegraph, ORCID, and package registries. - Refactored web-fetch-handlers module to web-scrapers with modular architecture and centralized handler exports. - Added AbortSignal support throughout web-fetch tool and scraper utilities for improved request cancellation. - Fixed npm license parsing to handle object format with type property. - Fixed raw GitHub URL construction by removing refs/heads prefix from ref path.
1052 lines
30 KiB
TypeScript
1052 lines
30 KiB
TypeScript
import { tmpdir } from "node:os";
|
|
import * as path from "node:path";
|
|
import type { AgentTool } from "@oh-my-pi/pi-agent-core";
|
|
import { Type } from "@sinclair/typebox";
|
|
import { parse as parseHtml } from "node-html-parser";
|
|
import webFetchDescription from "../../prompts/tools/web-fetch.md" with { type: "text" };
|
|
import { ensureTool } from "../../utils/tools-manager";
|
|
import type { ToolSession } from "./index";
|
|
import { specialHandlers } from "./web-scrapers/index";
|
|
import type { RenderResult } from "./web-scrapers/types";
|
|
import { finalizeOutput, loadPage } from "./web-scrapers/types";
|
|
import { convertWithMarkitdown, fetchBinary } from "./web-scrapers/utils";
|
|
|
|
// =============================================================================
|
|
// Types and Constants
|
|
// =============================================================================
|
|
|
|
const DEFAULT_TIMEOUT = 20;
|
|
|
|
// Convertible document types (markitdown supported)
|
|
const CONVERTIBLE_MIMES = new Set([
|
|
"application/pdf",
|
|
"application/msword",
|
|
"application/vnd.ms-powerpoint",
|
|
"application/vnd.ms-excel",
|
|
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
|
"application/vnd.openxmlformats-officedocument.presentationml.presentation",
|
|
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
|
"application/rtf",
|
|
"application/epub+zip",
|
|
"application/zip",
|
|
"image/png",
|
|
"image/jpeg",
|
|
"image/gif",
|
|
"image/webp",
|
|
"audio/mpeg",
|
|
"audio/wav",
|
|
"audio/ogg",
|
|
]);
|
|
|
|
const CONVERTIBLE_EXTENSIONS = new Set([
|
|
".pdf",
|
|
".doc",
|
|
".docx",
|
|
".ppt",
|
|
".pptx",
|
|
".xls",
|
|
".xlsx",
|
|
".rtf",
|
|
".epub",
|
|
".png",
|
|
".jpg",
|
|
".jpeg",
|
|
".gif",
|
|
".webp",
|
|
".mp3",
|
|
".wav",
|
|
".ogg",
|
|
]);
|
|
|
|
// =============================================================================
|
|
// Utilities
|
|
// =============================================================================
|
|
|
|
type SpawnSyncOptions = NonNullable<Parameters<typeof Bun.spawnSync>[1]>;
|
|
|
|
/**
|
|
* Execute a command and return stdout
|
|
*/
|
|
function exec(
|
|
cmd: string,
|
|
args: string[],
|
|
options?: { timeout?: number; input?: string | Buffer },
|
|
): { stdout: string; stderr: string; ok: boolean } {
|
|
const stdin = (options?.input ?? "ignore") as SpawnSyncOptions["stdin"];
|
|
const result = Bun.spawnSync([cmd, ...args], {
|
|
stdin,
|
|
stdout: "pipe",
|
|
stderr: "pipe",
|
|
});
|
|
return {
|
|
stdout: result.stdout?.toString() ?? "",
|
|
stderr: result.stderr?.toString() ?? "",
|
|
ok: result.exitCode === 0,
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Check if a command exists (cross-platform)
|
|
*/
|
|
function hasCommand(cmd: string): boolean {
|
|
return Boolean(Bun.which(cmd));
|
|
}
|
|
|
|
/**
|
|
* Extract origin from URL
|
|
*/
|
|
function getOrigin(url: string): string {
|
|
try {
|
|
const parsed = new URL(url);
|
|
return `${parsed.protocol}//${parsed.host}`;
|
|
} catch {
|
|
return "";
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Normalize URL (add scheme if missing)
|
|
*/
|
|
function normalizeUrl(url: string): string {
|
|
if (!url.match(/^https?:\/\//i)) {
|
|
return `https://${url}`;
|
|
}
|
|
return url;
|
|
}
|
|
|
|
/**
|
|
* Normalize MIME type (lowercase, strip charset/params)
|
|
*/
|
|
function normalizeMime(contentType: string): string {
|
|
return contentType.split(";")[0].trim().toLowerCase();
|
|
}
|
|
|
|
/**
|
|
* Get extension from URL or Content-Disposition
|
|
*/
|
|
function getExtensionHint(url: string, contentDisposition?: string): string {
|
|
// Try Content-Disposition filename first
|
|
if (contentDisposition) {
|
|
const match = contentDisposition.match(/filename[*]?=["']?([^"';\n]+)/i);
|
|
if (match) {
|
|
const ext = path.extname(match[1]).toLowerCase();
|
|
if (ext) return ext;
|
|
}
|
|
}
|
|
|
|
// Fall back to URL path
|
|
try {
|
|
const pathname = new URL(url).pathname;
|
|
const ext = path.extname(pathname).toLowerCase();
|
|
if (ext) return ext;
|
|
} catch {}
|
|
|
|
return "";
|
|
}
|
|
|
|
/**
|
|
* Check if content type is convertible via markitdown
|
|
*/
|
|
function isConvertible(mime: string, extensionHint: string): boolean {
|
|
if (CONVERTIBLE_MIMES.has(mime)) return true;
|
|
if (mime === "application/octet-stream" && CONVERTIBLE_EXTENSIONS.has(extensionHint)) return true;
|
|
if (CONVERTIBLE_EXTENSIONS.has(extensionHint)) return true;
|
|
return false;
|
|
}
|
|
|
|
/**
|
|
* Check if content looks like HTML
|
|
*/
|
|
function looksLikeHtml(content: string): boolean {
|
|
const trimmed = content.trim().toLowerCase();
|
|
return (
|
|
trimmed.startsWith("<!doctype") ||
|
|
trimmed.startsWith("<html") ||
|
|
trimmed.startsWith("<head") ||
|
|
trimmed.startsWith("<body")
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Try fetching URL with .md appended (llms.txt convention)
|
|
*/
|
|
async function tryMdSuffix(url: string, timeout: number, signal?: AbortSignal): Promise<string | null> {
|
|
const candidates: string[] = [];
|
|
|
|
try {
|
|
const parsed = new URL(url);
|
|
const pathname = parsed.pathname;
|
|
|
|
if (pathname.endsWith("/")) {
|
|
// /foo/bar/ -> /foo/bar/index.html.md
|
|
candidates.push(`${parsed.origin}${pathname}index.html.md`);
|
|
} else if (pathname.includes(".")) {
|
|
// /foo/bar.html -> /foo/bar.html.md
|
|
candidates.push(`${parsed.origin}${pathname}.md`);
|
|
} else {
|
|
// /foo/bar -> /foo/bar.md
|
|
candidates.push(`${parsed.origin}${pathname}.md`);
|
|
}
|
|
} catch {
|
|
return null;
|
|
}
|
|
|
|
if (signal?.aborted) {
|
|
return null;
|
|
}
|
|
|
|
for (const candidate of candidates) {
|
|
if (signal?.aborted) {
|
|
return null;
|
|
}
|
|
const result = await loadPage(candidate, { timeout: Math.min(timeout, 5), signal });
|
|
if (result.ok && result.content.trim().length > 100 && !looksLikeHtml(result.content)) {
|
|
return result.content;
|
|
}
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
/**
|
|
* Try to fetch LLM-friendly endpoints
|
|
*/
|
|
async function tryLlmEndpoints(origin: string, timeout: number, signal?: AbortSignal): Promise<string | null> {
|
|
const endpoints = [`${origin}/.well-known/llms.txt`, `${origin}/llms.txt`, `${origin}/llms.md`];
|
|
|
|
if (signal?.aborted) {
|
|
return null;
|
|
}
|
|
|
|
for (const endpoint of endpoints) {
|
|
if (signal?.aborted) {
|
|
return null;
|
|
}
|
|
const result = await loadPage(endpoint, { timeout: Math.min(timeout, 5), signal });
|
|
if (result.ok && result.content.trim().length > 100 && !looksLikeHtml(result.content)) {
|
|
return result.content;
|
|
}
|
|
}
|
|
return null;
|
|
}
|
|
|
|
/**
|
|
* Try content negotiation for markdown/plain
|
|
*/
|
|
async function tryContentNegotiation(
|
|
url: string,
|
|
timeout: number,
|
|
signal?: AbortSignal,
|
|
): Promise<{ content: string; type: string } | null> {
|
|
if (signal?.aborted) {
|
|
return null;
|
|
}
|
|
|
|
const result = await loadPage(url, {
|
|
timeout,
|
|
headers: { Accept: "text/markdown, text/plain;q=0.9, text/html;q=0.8" },
|
|
signal,
|
|
});
|
|
|
|
if (!result.ok) return null;
|
|
|
|
const mime = normalizeMime(result.contentType);
|
|
if (mime.includes("markdown") || mime === "text/plain") {
|
|
return { content: result.content, type: result.contentType };
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
/**
|
|
* Parse alternate links from HTML head
|
|
*/
|
|
function parseAlternateLinks(html: string, pageUrl: string): string[] {
|
|
const links: string[] = [];
|
|
|
|
try {
|
|
const doc = parseHtml(html.slice(0, 262144));
|
|
const alternateLinks = doc.querySelectorAll('link[rel="alternate"]');
|
|
|
|
for (const link of alternateLinks) {
|
|
const href = link.getAttribute("href");
|
|
const type = link.getAttribute("type")?.toLowerCase() ?? "";
|
|
|
|
if (!href) continue;
|
|
|
|
// Skip site-wide feeds
|
|
if (
|
|
href.includes("RecentChanges") ||
|
|
href.includes("Special:") ||
|
|
href.includes("/feed/") ||
|
|
href.includes("action=feed")
|
|
) {
|
|
continue;
|
|
}
|
|
|
|
if (type.includes("markdown")) {
|
|
links.push(href);
|
|
} else if (
|
|
(type.includes("rss") || type.includes("atom") || type.includes("feed")) &&
|
|
(href.includes(new URL(pageUrl).pathname) || href.includes("comments"))
|
|
) {
|
|
links.push(href);
|
|
}
|
|
}
|
|
} catch {}
|
|
|
|
return links;
|
|
}
|
|
|
|
/**
|
|
* Extract document links from HTML (for PDF/DOCX wrapper pages)
|
|
*/
|
|
function extractDocumentLinks(html: string, baseUrl: string): string[] {
|
|
const links: string[] = [];
|
|
|
|
try {
|
|
const doc = parseHtml(html);
|
|
const anchors = doc.querySelectorAll("a[href]");
|
|
|
|
for (const anchor of anchors) {
|
|
const href = anchor.getAttribute("href");
|
|
if (!href) continue;
|
|
|
|
const ext = path.extname(href).toLowerCase();
|
|
if (CONVERTIBLE_EXTENSIONS.has(ext)) {
|
|
const resolved = href.startsWith("http") ? href : new URL(href, baseUrl).href;
|
|
links.push(resolved);
|
|
}
|
|
}
|
|
} catch {}
|
|
|
|
return links;
|
|
}
|
|
|
|
/**
|
|
* Strip CDATA wrapper and clean text
|
|
*/
|
|
function cleanFeedText(text: string): string {
|
|
return text
|
|
.replace(/<!\[CDATA\[/g, "")
|
|
.replace(/\]\]>/g, "")
|
|
.replace(/</g, "<")
|
|
.replace(/>/g, ">")
|
|
.replace(/&/g, "&")
|
|
.replace(/"/g, '"')
|
|
.replace(/<[^>]+>/g, "") // Strip HTML tags
|
|
.trim();
|
|
}
|
|
|
|
/**
|
|
* Parse RSS/Atom feed to markdown
|
|
*/
|
|
function parseFeedToMarkdown(content: string, maxItems = 10): string {
|
|
try {
|
|
const doc = parseHtml(content, { parseNoneClosedTags: true });
|
|
|
|
// Try RSS
|
|
const channel = doc.querySelector("channel");
|
|
if (channel) {
|
|
const title = cleanFeedText(channel.querySelector("title")?.text || "RSS Feed");
|
|
const items = channel.querySelectorAll("item").slice(0, maxItems);
|
|
|
|
let md = `# ${title}\n\n`;
|
|
for (const item of items) {
|
|
const itemTitle = cleanFeedText(item.querySelector("title")?.text || "Untitled");
|
|
const link = cleanFeedText(item.querySelector("link")?.text || "");
|
|
const pubDate = cleanFeedText(item.querySelector("pubDate")?.text || "");
|
|
const desc = cleanFeedText(item.querySelector("description")?.text || "");
|
|
|
|
md += `## ${itemTitle}\n`;
|
|
if (pubDate) md += `*${pubDate}*\n\n`;
|
|
if (desc) md += `${desc.slice(0, 500)}${desc.length > 500 ? "..." : ""}\n\n`;
|
|
if (link) md += `[Read more](${link})\n\n`;
|
|
md += "---\n\n";
|
|
}
|
|
return md;
|
|
}
|
|
|
|
// Try Atom
|
|
const feed = doc.querySelector("feed");
|
|
if (feed) {
|
|
const title = cleanFeedText(feed.querySelector("title")?.text || "Atom Feed");
|
|
const entries = feed.querySelectorAll("entry").slice(0, maxItems);
|
|
|
|
let md = `# ${title}\n\n`;
|
|
for (const entry of entries) {
|
|
const entryTitle = cleanFeedText(entry.querySelector("title")?.text || "Untitled");
|
|
const link = entry.querySelector("link")?.getAttribute("href") || "";
|
|
const updated = cleanFeedText(entry.querySelector("updated")?.text || "");
|
|
const summary = cleanFeedText(
|
|
entry.querySelector("summary")?.text || entry.querySelector("content")?.text || "",
|
|
);
|
|
|
|
md += `## ${entryTitle}\n`;
|
|
if (updated) md += `*${updated}*\n\n`;
|
|
if (summary) md += `${summary.slice(0, 500)}${summary.length > 500 ? "..." : ""}\n\n`;
|
|
if (link) md += `[Read more](${link})\n\n`;
|
|
md += "---\n\n";
|
|
}
|
|
return md;
|
|
}
|
|
} catch {}
|
|
|
|
return content; // Fall back to raw content
|
|
}
|
|
|
|
/**
|
|
* Render HTML to text using lynx or html2text fallback
|
|
*/
|
|
async function renderHtmlToText(
|
|
html: string,
|
|
timeout: number,
|
|
): Promise<{ content: string; ok: boolean; method: string }> {
|
|
const tmpDir = tmpdir();
|
|
const tmpFile = path.join(tmpDir, `omp-render-${Date.now()}.html`);
|
|
|
|
try {
|
|
await Bun.write(tmpFile, html);
|
|
|
|
// Try lynx first (can't auto-install, system package)
|
|
const lynx = hasCommand("lynx");
|
|
if (lynx) {
|
|
const normalizedPath = tmpFile.replace(/\\/g, "/");
|
|
const fileUrl = normalizedPath.startsWith("/") ? `file://${normalizedPath}` : `file:///${normalizedPath}`;
|
|
const result = exec("lynx", ["-dump", "-nolist", "-width", "120", fileUrl], { timeout });
|
|
if (result.ok) {
|
|
return { content: result.stdout, ok: true, method: "lynx" };
|
|
}
|
|
}
|
|
|
|
// Fall back to html2text (auto-install via uv/pip)
|
|
const html2text = await ensureTool("html2text", true);
|
|
if (html2text) {
|
|
const result = exec(html2text, [tmpFile], { timeout });
|
|
if (result.ok) {
|
|
return { content: result.stdout, ok: true, method: "html2text" };
|
|
}
|
|
}
|
|
|
|
return { content: "", ok: false, method: "none" };
|
|
} finally {
|
|
try {
|
|
await Bun.$`rm ${tmpFile}`.quiet();
|
|
} catch {}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Check if lynx output looks JS-gated or mostly navigation
|
|
*/
|
|
function isLowQualityOutput(content: string): boolean {
|
|
const lower = content.toLowerCase();
|
|
|
|
// JS-gated indicators
|
|
const jsGated = [
|
|
"enable javascript",
|
|
"javascript required",
|
|
"turn on javascript",
|
|
"please enable javascript",
|
|
"browser not supported",
|
|
];
|
|
if (content.length < 1024 && jsGated.some((t) => lower.includes(t))) {
|
|
return true;
|
|
}
|
|
|
|
// Mostly navigation (high link/menu density)
|
|
const lines = content.split("\n").filter((l) => l.trim());
|
|
const shortLines = lines.filter((l) => l.trim().length < 40);
|
|
if (lines.length > 10 && shortLines.length / lines.length > 0.7) {
|
|
return true;
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
/**
|
|
* Format JSON
|
|
*/
|
|
function formatJson(content: string): string {
|
|
try {
|
|
return JSON.stringify(JSON.parse(content), null, 2);
|
|
} catch {
|
|
return content;
|
|
}
|
|
}
|
|
|
|
// =============================================================================
|
|
// Unified Special Handler Dispatch
|
|
// =============================================================================
|
|
|
|
/**
|
|
* Try all special handlers
|
|
*/
|
|
async function handleSpecialUrls(url: string, timeout: number, signal?: AbortSignal): Promise<RenderResult | null> {
|
|
for (const handler of specialHandlers) {
|
|
if (signal?.aborted) {
|
|
throw new Error("Operation aborted");
|
|
}
|
|
const result = await handler(url, timeout);
|
|
if (result) return result;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
// =============================================================================
|
|
// Main Render Function
|
|
// =============================================================================
|
|
|
|
/**
|
|
* Main render function implementing the full pipeline
|
|
*/
|
|
async function renderUrl(
|
|
url: string,
|
|
timeout: number,
|
|
raw: boolean = false,
|
|
signal?: AbortSignal,
|
|
): Promise<RenderResult> {
|
|
const notes: string[] = [];
|
|
const fetchedAt = new Date().toISOString();
|
|
if (signal?.aborted) {
|
|
throw new Error("Operation aborted");
|
|
}
|
|
|
|
// Step 0: Normalize URL (ensure scheme for special handlers)
|
|
url = normalizeUrl(url);
|
|
const origin = getOrigin(url);
|
|
|
|
// Step 1: Try special handlers for known sites (unless raw mode)
|
|
if (!raw) {
|
|
const specialResult = await handleSpecialUrls(url, timeout, signal);
|
|
if (specialResult) return specialResult;
|
|
}
|
|
|
|
// Step 2: Fetch page
|
|
const response = await loadPage(url, { timeout, signal });
|
|
if (signal?.aborted) {
|
|
throw new Error("Operation aborted");
|
|
}
|
|
if (!response.ok) {
|
|
return {
|
|
url,
|
|
finalUrl: url,
|
|
contentType: "unknown",
|
|
method: "failed",
|
|
content: "",
|
|
fetchedAt,
|
|
truncated: false,
|
|
notes: ["Failed to fetch URL"],
|
|
};
|
|
}
|
|
|
|
const { finalUrl, content: rawContent } = response;
|
|
const mime = normalizeMime(response.contentType);
|
|
const extHint = getExtensionHint(finalUrl);
|
|
|
|
// Step 3: Handle convertible binary files (PDF, DOCX, etc.)
|
|
if (isConvertible(mime, extHint)) {
|
|
const binary = await fetchBinary(finalUrl, timeout, signal);
|
|
if (binary.ok) {
|
|
const ext = getExtensionHint(finalUrl, binary.contentDisposition) || extHint;
|
|
const converted = await convertWithMarkitdown(binary.buffer, ext, timeout, signal);
|
|
if (converted.ok) {
|
|
if (converted.content.trim().length > 50) {
|
|
notes.push("Converted with markitdown");
|
|
const output = finalizeOutput(converted.content);
|
|
return {
|
|
url,
|
|
finalUrl,
|
|
contentType: mime,
|
|
method: "markitdown",
|
|
content: output.content,
|
|
fetchedAt,
|
|
truncated: output.truncated,
|
|
notes,
|
|
};
|
|
}
|
|
notes.push("markitdown conversion produced no usable output");
|
|
} else if (converted.error) {
|
|
notes.push(`markitdown conversion failed: ${converted.error}`);
|
|
} else {
|
|
notes.push("markitdown conversion failed");
|
|
}
|
|
} else if (binary.error) {
|
|
notes.push(`Binary fetch failed: ${binary.error}`);
|
|
} else {
|
|
notes.push("Binary fetch failed");
|
|
}
|
|
}
|
|
|
|
// Step 4: Handle non-HTML text content
|
|
const isHtml = mime.includes("html") || mime.includes("xhtml");
|
|
const isJson = mime.includes("json");
|
|
const isXml = mime.includes("xml") && !isHtml;
|
|
const isText = mime.includes("text/plain") || mime.includes("text/markdown");
|
|
const isFeed = mime.includes("rss") || mime.includes("atom") || mime.includes("feed");
|
|
|
|
if (isJson) {
|
|
const output = finalizeOutput(formatJson(rawContent));
|
|
return {
|
|
url,
|
|
finalUrl,
|
|
contentType: mime,
|
|
method: "json",
|
|
content: output.content,
|
|
fetchedAt,
|
|
truncated: output.truncated,
|
|
notes,
|
|
};
|
|
}
|
|
|
|
if (isFeed || (isXml && (rawContent.includes("<rss") || rawContent.includes("<feed")))) {
|
|
const parsed = parseFeedToMarkdown(rawContent);
|
|
const output = finalizeOutput(parsed);
|
|
return {
|
|
url,
|
|
finalUrl,
|
|
contentType: mime,
|
|
method: "feed",
|
|
content: output.content,
|
|
fetchedAt,
|
|
truncated: output.truncated,
|
|
notes,
|
|
};
|
|
}
|
|
|
|
if (isText && !looksLikeHtml(rawContent)) {
|
|
const output = finalizeOutput(rawContent);
|
|
return {
|
|
url,
|
|
finalUrl,
|
|
contentType: mime,
|
|
method: "text",
|
|
content: output.content,
|
|
fetchedAt,
|
|
truncated: output.truncated,
|
|
notes,
|
|
};
|
|
}
|
|
|
|
// Step 5: For HTML, try digestible formats first (unless raw mode)
|
|
if (isHtml && !raw) {
|
|
// 5A: Check for page-specific markdown alternate
|
|
const alternates = parseAlternateLinks(rawContent, finalUrl);
|
|
const markdownAlt = alternates.find((alt) => alt.endsWith(".md") || alt.includes("markdown"));
|
|
if (markdownAlt) {
|
|
const resolved = markdownAlt.startsWith("http") ? markdownAlt : new URL(markdownAlt, finalUrl).href;
|
|
const altResult = await loadPage(resolved, { timeout, signal });
|
|
if (altResult.ok && altResult.content.trim().length > 100 && !looksLikeHtml(altResult.content)) {
|
|
notes.push(`Used markdown alternate: ${resolved}`);
|
|
const output = finalizeOutput(altResult.content);
|
|
return {
|
|
url,
|
|
finalUrl,
|
|
contentType: "text/markdown",
|
|
method: "alternate-markdown",
|
|
content: output.content,
|
|
fetchedAt,
|
|
truncated: output.truncated,
|
|
notes,
|
|
};
|
|
}
|
|
}
|
|
|
|
// 5B: Try URL.md suffix (llms.txt convention)
|
|
const mdSuffix = await tryMdSuffix(finalUrl, timeout, signal);
|
|
if (mdSuffix) {
|
|
notes.push("Found .md suffix version");
|
|
const output = finalizeOutput(mdSuffix);
|
|
return {
|
|
url,
|
|
finalUrl,
|
|
contentType: "text/markdown",
|
|
method: "md-suffix",
|
|
content: output.content,
|
|
fetchedAt,
|
|
truncated: output.truncated,
|
|
notes,
|
|
};
|
|
}
|
|
|
|
// 5C: LLM-friendly endpoints
|
|
const llmContent = await tryLlmEndpoints(origin, timeout, signal);
|
|
if (llmContent) {
|
|
notes.push("Found llms.txt");
|
|
const output = finalizeOutput(llmContent);
|
|
return {
|
|
url,
|
|
finalUrl,
|
|
contentType: "text/plain",
|
|
method: "llms.txt",
|
|
content: output.content,
|
|
fetchedAt,
|
|
truncated: output.truncated,
|
|
notes,
|
|
};
|
|
}
|
|
|
|
// 5D: Content negotiation
|
|
const negotiated = await tryContentNegotiation(url, timeout, signal);
|
|
if (negotiated) {
|
|
notes.push(`Content negotiation returned ${negotiated.type}`);
|
|
const output = finalizeOutput(negotiated.content);
|
|
return {
|
|
url,
|
|
finalUrl,
|
|
contentType: normalizeMime(negotiated.type),
|
|
method: "content-negotiation",
|
|
content: output.content,
|
|
fetchedAt,
|
|
truncated: output.truncated,
|
|
notes,
|
|
};
|
|
}
|
|
|
|
// 5E: Check for feed alternates
|
|
const feedAlternates = alternates.filter((alt) => !alt.endsWith(".md") && !alt.includes("markdown"));
|
|
for (const altUrl of feedAlternates.slice(0, 2)) {
|
|
const resolved = altUrl.startsWith("http") ? altUrl : new URL(altUrl, finalUrl).href;
|
|
const altResult = await loadPage(resolved, { timeout, signal });
|
|
if (altResult.ok && altResult.content.trim().length > 200) {
|
|
notes.push(`Used feed alternate: ${resolved}`);
|
|
const parsed = parseFeedToMarkdown(altResult.content);
|
|
const output = finalizeOutput(parsed);
|
|
return {
|
|
url,
|
|
finalUrl,
|
|
contentType: "application/feed",
|
|
method: "alternate-feed",
|
|
content: output.content,
|
|
fetchedAt,
|
|
truncated: output.truncated,
|
|
notes,
|
|
};
|
|
}
|
|
}
|
|
|
|
if (signal?.aborted) {
|
|
throw new Error("Operation aborted");
|
|
}
|
|
|
|
// Step 6: Render HTML with lynx or html2text
|
|
const htmlResult = await renderHtmlToText(rawContent, timeout);
|
|
if (!htmlResult.ok) {
|
|
notes.push("html rendering failed (lynx/html2text unavailable)");
|
|
const output = finalizeOutput(rawContent);
|
|
return {
|
|
url,
|
|
finalUrl,
|
|
contentType: mime,
|
|
method: "raw-html",
|
|
content: output.content,
|
|
fetchedAt,
|
|
truncated: output.truncated,
|
|
notes,
|
|
};
|
|
}
|
|
|
|
// Step 7: If lynx output is low quality, try extracting document links
|
|
if (isLowQualityOutput(htmlResult.content)) {
|
|
const docLinks = extractDocumentLinks(rawContent, finalUrl);
|
|
if (docLinks.length > 0) {
|
|
const docUrl = docLinks[0];
|
|
const binary = await fetchBinary(docUrl, timeout, signal);
|
|
if (binary.ok) {
|
|
const ext = getExtensionHint(docUrl, binary.contentDisposition);
|
|
const converted = await convertWithMarkitdown(binary.buffer, ext, timeout, signal);
|
|
if (converted.ok && converted.content.trim().length > htmlResult.content.length) {
|
|
notes.push(`Extracted and converted document: ${docUrl}`);
|
|
const output = finalizeOutput(converted.content);
|
|
return {
|
|
url,
|
|
finalUrl,
|
|
contentType: "application/document",
|
|
method: "extracted-document",
|
|
content: output.content,
|
|
fetchedAt,
|
|
truncated: output.truncated,
|
|
notes,
|
|
};
|
|
}
|
|
if (!converted.ok && converted.error) {
|
|
notes.push(`markitdown conversion failed: ${converted.error}`);
|
|
}
|
|
} else if (binary.error) {
|
|
notes.push(`Binary fetch failed: ${binary.error}`);
|
|
}
|
|
}
|
|
notes.push("Page appears to require JavaScript or is mostly navigation");
|
|
}
|
|
|
|
const output = finalizeOutput(htmlResult.content);
|
|
return {
|
|
url,
|
|
finalUrl,
|
|
contentType: mime,
|
|
method: htmlResult.method,
|
|
content: output.content,
|
|
fetchedAt,
|
|
truncated: output.truncated,
|
|
notes,
|
|
};
|
|
}
|
|
|
|
// Fallback: return raw content
|
|
const output = finalizeOutput(rawContent);
|
|
return {
|
|
url,
|
|
finalUrl,
|
|
contentType: mime,
|
|
method: "raw",
|
|
content: output.content,
|
|
fetchedAt,
|
|
truncated: output.truncated,
|
|
notes,
|
|
};
|
|
}
|
|
|
|
// =============================================================================
|
|
// Tool Definition
|
|
// =============================================================================
|
|
|
|
const webFetchSchema = Type.Object({
|
|
url: Type.String({ description: "The URL to fetch and render" }),
|
|
timeout: Type.Optional(Type.Number({ description: "Timeout in seconds (default: 20, max: 120)" })),
|
|
raw: Type.Optional(
|
|
Type.Boolean({ description: "Return raw content without site-specific rendering or LLM-friendly transforms" }),
|
|
),
|
|
});
|
|
|
|
export interface WebFetchToolDetails {
|
|
url: string;
|
|
finalUrl: string;
|
|
contentType: string;
|
|
method: string;
|
|
truncated: boolean;
|
|
notes: string[];
|
|
}
|
|
|
|
export function createWebFetchTool(_session: ToolSession): AgentTool<typeof webFetchSchema> {
|
|
return {
|
|
name: "web_fetch",
|
|
label: "Web Fetch",
|
|
description: webFetchDescription,
|
|
parameters: webFetchSchema,
|
|
execute: async (
|
|
_toolCallId: string,
|
|
{ url, timeout = DEFAULT_TIMEOUT, raw = false }: { url: string; timeout?: number; raw?: boolean },
|
|
signal?: AbortSignal,
|
|
) => {
|
|
if (signal?.aborted) {
|
|
throw new Error("Operation aborted");
|
|
}
|
|
|
|
// Clamp timeout
|
|
const effectiveTimeout = Math.min(Math.max(timeout, 1), 120);
|
|
|
|
const result = await renderUrl(url, effectiveTimeout, raw, signal);
|
|
|
|
// Format output
|
|
let output = "";
|
|
output += `URL: ${result.finalUrl}\n`;
|
|
output += `Content-Type: ${result.contentType}\n`;
|
|
output += `Method: ${result.method}\n`;
|
|
if (result.truncated) {
|
|
output += `Warning: Output was truncated\n`;
|
|
}
|
|
if (result.notes.length > 0) {
|
|
output += `Notes: ${result.notes.join("; ")}\n`;
|
|
}
|
|
output += `\n---\n\n`;
|
|
output += result.content;
|
|
|
|
const details: WebFetchToolDetails = {
|
|
url: result.url,
|
|
finalUrl: result.finalUrl,
|
|
contentType: result.contentType,
|
|
method: result.method,
|
|
truncated: result.truncated,
|
|
notes: result.notes,
|
|
};
|
|
|
|
return {
|
|
content: [{ type: "text", text: output }],
|
|
details,
|
|
};
|
|
},
|
|
};
|
|
}
|
|
|
|
// =============================================================================
|
|
// TUI Rendering
|
|
// =============================================================================
|
|
|
|
import type { Component } from "@oh-my-pi/pi-tui";
|
|
import { Text } from "@oh-my-pi/pi-tui";
|
|
import { type Theme, theme } from "../../modes/interactive/theme/theme";
|
|
import type { RenderResultOptions } from "../custom-tools/types";
|
|
|
|
/** Truncate text to max length with ellipsis */
|
|
function truncate(text: string, maxLen: number, ellipsis: string): string {
|
|
if (text.length <= maxLen) return text;
|
|
const sliceLen = Math.max(0, maxLen - ellipsis.length);
|
|
return `${text.slice(0, sliceLen)}${ellipsis}`;
|
|
}
|
|
|
|
/** Extract domain from URL */
|
|
function getDomain(url: string): string {
|
|
try {
|
|
const u = new URL(url);
|
|
return u.hostname.replace(/^www\./, "");
|
|
} catch {
|
|
return url;
|
|
}
|
|
}
|
|
|
|
/** Get first N lines of text as preview */
|
|
function getPreviewLines(text: string, maxLines: number, maxLineLen: number, ellipsis: string): string[] {
|
|
const lines = text.split("\n").filter((l) => l.trim());
|
|
return lines.slice(0, maxLines).map((l) => truncate(l.trim(), maxLineLen, ellipsis));
|
|
}
|
|
|
|
/** Count non-empty lines */
|
|
function countNonEmptyLines(text: string): number {
|
|
return text.split("\n").filter((l) => l.trim()).length;
|
|
}
|
|
|
|
/** Render web fetch call (URL preview) */
|
|
export function renderWebFetchCall(
|
|
args: { url: string; timeout?: number; raw?: boolean },
|
|
uiTheme: Theme = theme,
|
|
): Component {
|
|
const domain = getDomain(args.url);
|
|
const path = truncate(args.url.replace(/^https?:\/\/[^/]+/, ""), 50, uiTheme.format.ellipsis);
|
|
const icon = uiTheme.styledSymbol("status.pending", "muted");
|
|
const text = `${icon} ${uiTheme.fg("toolTitle", "Web Fetch")} ${uiTheme.fg("accent", domain)}${uiTheme.fg("dim", path)}`;
|
|
return new Text(text, 0, 0);
|
|
}
|
|
|
|
/** Render web fetch result with tree-based layout */
|
|
export function renderWebFetchResult(
|
|
result: { content: Array<{ type: string; text?: string }>; details?: WebFetchToolDetails },
|
|
options: RenderResultOptions,
|
|
uiTheme: Theme = theme,
|
|
): Component {
|
|
const { expanded } = options;
|
|
const details = result.details;
|
|
|
|
if (!details) {
|
|
return new Text(uiTheme.fg("error", "No response data"), 0, 0);
|
|
}
|
|
|
|
const domain = getDomain(details.finalUrl);
|
|
const hasRedirect = details.url !== details.finalUrl;
|
|
const hasNotes = details.notes.length > 0;
|
|
const statusIcon = details.truncated
|
|
? uiTheme.styledSymbol("status.warning", "warning")
|
|
: uiTheme.styledSymbol("status.success", "success");
|
|
const expandHint = expanded ? "" : uiTheme.fg("dim", " (Ctrl+O to expand)");
|
|
let text = `${statusIcon} ${uiTheme.fg("accent", `(${domain})`)}${uiTheme.sep.dot}${uiTheme.fg("dim", details.method)}${expandHint}`;
|
|
|
|
// Get content text
|
|
const contentText = result.content[0]?.text ?? "";
|
|
// Extract just the content part (after the --- separator)
|
|
const contentBody = contentText.includes("---\n\n")
|
|
? contentText.split("---\n\n").slice(1).join("---\n\n")
|
|
: contentText;
|
|
const lineCount = countNonEmptyLines(contentBody);
|
|
const charCount = contentBody.trim().length;
|
|
|
|
if (!expanded) {
|
|
// Collapsed view: metadata + preview
|
|
const metaLines: string[] = [
|
|
`${uiTheme.fg("muted", "Content-Type:")} ${details.contentType || "unknown"}`,
|
|
`${uiTheme.fg("muted", "Method:")} ${details.method}`,
|
|
];
|
|
if (hasRedirect) {
|
|
metaLines.push(`${uiTheme.fg("muted", "Final URL:")} ${uiTheme.fg("mdLinkUrl", details.finalUrl)}`);
|
|
}
|
|
if (details.truncated) {
|
|
metaLines.push(uiTheme.fg("warning", `${uiTheme.status.warning} Output truncated`));
|
|
}
|
|
if (hasNotes) {
|
|
metaLines.push(`${uiTheme.fg("muted", "Notes:")} ${details.notes.join("; ")}`);
|
|
}
|
|
|
|
const previewLines = getPreviewLines(contentBody, 3, 100, uiTheme.format.ellipsis);
|
|
const detailLines: string[] = [...metaLines];
|
|
|
|
if (previewLines.length === 0) {
|
|
detailLines.push(uiTheme.fg("dim", "(no content)"));
|
|
} else {
|
|
for (const line of previewLines) {
|
|
detailLines.push(uiTheme.fg("dim", line));
|
|
}
|
|
}
|
|
|
|
const remaining = Math.max(0, lineCount - previewLines.length);
|
|
if (remaining > 0) {
|
|
detailLines.push(uiTheme.fg("muted", `${uiTheme.format.ellipsis} ${remaining} more lines`));
|
|
} else {
|
|
const lineLabel = `${lineCount} line${lineCount === 1 ? "" : "s"}`;
|
|
detailLines.push(uiTheme.fg("muted", `${lineLabel}${uiTheme.sep.dot}${charCount} chars`));
|
|
}
|
|
|
|
for (let i = 0; i < detailLines.length; i++) {
|
|
const isLast = i === detailLines.length - 1;
|
|
const branch = isLast ? uiTheme.tree.last : uiTheme.tree.vertical;
|
|
text += `\n ${uiTheme.fg("dim", branch)} ${detailLines[i]}`;
|
|
}
|
|
} else {
|
|
// Expanded view: structured metadata + bounded content preview
|
|
const metaLines: string[] = [
|
|
`${uiTheme.fg("muted", "Content-Type:")} ${details.contentType || "unknown"}`,
|
|
`${uiTheme.fg("muted", "Method:")} ${details.method}`,
|
|
];
|
|
if (hasRedirect) {
|
|
metaLines.push(`${uiTheme.fg("muted", "Final URL:")} ${uiTheme.fg("mdLinkUrl", details.finalUrl)}`);
|
|
}
|
|
const lineLabel = `${lineCount} line${lineCount === 1 ? "" : "s"}`;
|
|
metaLines.push(`${uiTheme.fg("muted", "Lines:")} ${lineLabel}`);
|
|
metaLines.push(`${uiTheme.fg("muted", "Chars:")} ${charCount}`);
|
|
if (details.truncated) {
|
|
metaLines.push(uiTheme.fg("warning", `${uiTheme.status.warning} Output truncated`));
|
|
}
|
|
if (hasNotes) {
|
|
metaLines.push(`${uiTheme.fg("muted", "Notes:")} ${details.notes.join("; ")}`);
|
|
}
|
|
|
|
text += `\n ${uiTheme.fg("dim", uiTheme.tree.branch)} ${uiTheme.fg("accent", "Metadata")}`;
|
|
for (let i = 0; i < metaLines.length; i++) {
|
|
const isLast = i === metaLines.length - 1;
|
|
const branch = isLast ? uiTheme.tree.last : uiTheme.tree.branch;
|
|
text += `\n ${uiTheme.fg("dim", uiTheme.tree.vertical)} ${uiTheme.fg("dim", branch)} ${metaLines[i]}`;
|
|
}
|
|
|
|
text += `\n ${uiTheme.fg("dim", uiTheme.tree.last)} ${uiTheme.fg("accent", "Content Preview")}`;
|
|
const previewLines = getPreviewLines(contentBody, 12, 120, uiTheme.format.ellipsis);
|
|
const remaining = Math.max(0, lineCount - previewLines.length);
|
|
const contentPrefix = uiTheme.fg("dim", " ");
|
|
|
|
if (previewLines.length === 0) {
|
|
text += `\n ${contentPrefix} ${uiTheme.fg("dim", "(no content)")}`;
|
|
} else {
|
|
for (const line of previewLines) {
|
|
text += `\n ${contentPrefix} ${uiTheme.fg("dim", line)}`;
|
|
}
|
|
}
|
|
|
|
if (remaining > 0) {
|
|
text += `\n ${contentPrefix} ${uiTheme.fg("muted", `${uiTheme.format.ellipsis} ${remaining} more lines`)}`;
|
|
}
|
|
}
|
|
|
|
return new Text(text, 0, 0);
|
|
}
|
|
|
|
export const webFetchToolRenderer = {
|
|
renderCall: renderWebFetchCall,
|
|
renderResult: renderWebFetchResult,
|
|
};
|