From 5a7cf77ba97b2da3da3869d1ddb44ad3cb2ef8ed Mon Sep 17 00:00:00 2001 From: can1357 Date: Wed, 7 Jan 2026 20:34:33 +0100 Subject: [PATCH] feat(web-scrapers): added 25+ specialized scrapers with modular architecture - Added 25+ new specialized web scrapers for sites including MusicBrainz, Discourse, Lemmy, Sourcegraph, ORCID, and package registries. - Refactored web-fetch-handlers module to web-scrapers with modular architecture and centralized handler exports. - Added AbortSignal support throughout web-fetch tool and scraper utilities for improved request cancellation. - Fixed npm license parsing to handle object format with type property. - Fixed raw GitHub URL construction by removing refs/heads prefix from ref path. --- packages/coding-agent/CHANGELOG.md | 17 + .../core/tools/web-fetch-handlers/index.ts | 69 --- .../core/tools/web-fetch-handlers/utils.ts | 91 ---- .../coding-agent/src/core/tools/web-fetch.ts | 477 ++++-------------- .../academic.test.ts | 0 .../artifacthub.ts | 0 .../arxiv.ts | 0 .../aur.ts | 0 .../biorxiv.ts | 0 .../bluesky.ts | 0 .../brew.ts | 0 .../business.test.ts | 0 .../cheatsh.ts | 0 .../chocolatey.ts | 0 .../core/tools/web-scrapers/choosealicense.ts | 109 ++++ .../src/core/tools/web-scrapers/cisa-kev.ts | 95 ++++ .../src/core/tools/web-scrapers/clojars.ts | 175 +++++++ .../coingecko.ts | 0 .../crates-io.ts | 0 .../src/core/tools/web-scrapers/crossref.ts | 144 ++++++ .../dev-platforms.test.ts | 0 .../devto.ts | 0 .../discogs.ts | 0 .../src/core/tools/web-scrapers/discourse.ts | 217 ++++++++ .../dockerhub.ts | 0 .../documentation.test.ts | 0 .../src/core/tools/web-scrapers/fdroid.ts | 153 ++++++ .../finance-media.test.ts | 0 .../core/tools/web-scrapers/firefox-addons.ts | 213 ++++++++ .../src/core/tools/web-scrapers/flathub.ts | 235 +++++++++ .../git-hosting.test.ts | 0 .../github-gist.ts | 0 .../github.ts | 11 +- .../gitlab.ts | 0 .../go-pkg.ts | 0 .../hackage.ts | 0 .../hackernews.ts | 0 .../hex.ts | 0 .../huggingface.ts | 0 .../iacr.ts | 0 .../src/core/tools/web-scrapers/index.ts | 250 +++++++++ .../web-scrapers/jetbrains-marketplace.ts | 168 ++++++ .../src/core/tools/web-scrapers/lemmy.ts | 216 ++++++++ .../lobsters.ts | 0 .../mastodon.ts | 0 .../maven.ts | 0 .../mdn.ts | 0 .../media.test.ts | 0 .../metacpan.ts | 0 .../core/tools/web-scrapers/musicbrainz.ts | 268 ++++++++++ .../npm.ts | 7 +- .../nuget.ts | 0 .../nvd.ts | 0 .../src/core/tools/web-scrapers/ollama.ts | 263 ++++++++++ .../src/core/tools/web-scrapers/open-vsx.ts | 115 +++++ .../opencorporates.ts | 0 .../openlibrary.ts | 0 .../src/core/tools/web-scrapers/orcid.ts | 294 +++++++++++ .../osv.ts | 0 .../package-managers-2.test.ts | 0 .../package-managers.test.ts | 0 .../package-registries.test.ts | 0 .../packagist.ts | 0 .../pub-dev.ts | 0 .../pubmed.ts | 0 .../pypi.ts | 0 .../src/core/tools/web-scrapers/rawg.ts | 120 +++++ .../readthedocs.ts | 0 .../reddit.ts | 0 .../repology.ts | 0 .../research.test.ts | 0 .../rfc.ts | 0 .../rubygems.ts | 0 .../src/core/tools/web-scrapers/searchcode.ts | 213 ++++++++ .../sec-edgar.ts | 0 .../security.test.ts | 0 .../semantic-scholar.ts | 0 .../src/core/tools/web-scrapers/snapcraft.ts | 195 +++++++ .../social-extended.test.ts | 0 .../social.test.ts | 0 .../core/tools/web-scrapers/sourcegraph.ts | 353 +++++++++++++ .../src/core/tools/web-scrapers/spdx.ts | 116 +++++ .../spotify.ts | 0 .../stackexchange.test.ts | 0 .../stackoverflow.ts | 0 .../standards.test.ts | 0 .../terraform.ts | 0 .../tldr.ts | 0 .../twitter.ts | 0 .../types.ts | 123 ++++- .../src/core/tools/web-scrapers/utils.ts | 161 ++++++ .../vimeo.ts | 0 .../tools/web-scrapers/vscode-marketplace.ts | 193 +++++++ .../src/core/tools/web-scrapers/w3c.ts | 159 ++++++ .../wikidata.ts | 0 .../wikipedia.test.ts | 0 .../wikipedia.ts | 0 .../youtube.test.ts | 0 .../youtube.ts | 0 99 files changed, 4654 insertions(+), 566 deletions(-) delete mode 100644 packages/coding-agent/src/core/tools/web-fetch-handlers/index.ts delete mode 100644 packages/coding-agent/src/core/tools/web-fetch-handlers/utils.ts rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/academic.test.ts (100%) rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/artifacthub.ts (100%) rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/arxiv.ts (100%) rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/aur.ts (100%) rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/biorxiv.ts (100%) rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/bluesky.ts (100%) rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/brew.ts (100%) rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/business.test.ts (100%) rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/cheatsh.ts (100%) rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/chocolatey.ts (100%) create mode 100644 packages/coding-agent/src/core/tools/web-scrapers/choosealicense.ts create mode 100644 packages/coding-agent/src/core/tools/web-scrapers/cisa-kev.ts create mode 100644 packages/coding-agent/src/core/tools/web-scrapers/clojars.ts rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/coingecko.ts (100%) rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/crates-io.ts (100%) create mode 100644 packages/coding-agent/src/core/tools/web-scrapers/crossref.ts rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/dev-platforms.test.ts (100%) rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/devto.ts (100%) rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/discogs.ts (100%) create mode 100644 packages/coding-agent/src/core/tools/web-scrapers/discourse.ts rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/dockerhub.ts (100%) rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/documentation.test.ts (100%) create mode 100644 packages/coding-agent/src/core/tools/web-scrapers/fdroid.ts rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/finance-media.test.ts (100%) create mode 100644 packages/coding-agent/src/core/tools/web-scrapers/firefox-addons.ts create mode 100644 packages/coding-agent/src/core/tools/web-scrapers/flathub.ts rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/git-hosting.test.ts (100%) rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/github-gist.ts (100%) rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/github.ts (97%) rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/gitlab.ts (100%) rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/go-pkg.ts (100%) rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/hackage.ts (100%) rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/hackernews.ts (100%) rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/hex.ts (100%) rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/huggingface.ts (100%) rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/iacr.ts (100%) create mode 100644 packages/coding-agent/src/core/tools/web-scrapers/index.ts create mode 100644 packages/coding-agent/src/core/tools/web-scrapers/jetbrains-marketplace.ts create mode 100644 packages/coding-agent/src/core/tools/web-scrapers/lemmy.ts rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/lobsters.ts (100%) rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/mastodon.ts (100%) rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/maven.ts (100%) rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/mdn.ts (100%) rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/media.test.ts (100%) rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/metacpan.ts (100%) create mode 100644 packages/coding-agent/src/core/tools/web-scrapers/musicbrainz.ts rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/npm.ts (93%) rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/nuget.ts (100%) rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/nvd.ts (100%) create mode 100644 packages/coding-agent/src/core/tools/web-scrapers/ollama.ts create mode 100644 packages/coding-agent/src/core/tools/web-scrapers/open-vsx.ts rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/opencorporates.ts (100%) rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/openlibrary.ts (100%) create mode 100644 packages/coding-agent/src/core/tools/web-scrapers/orcid.ts rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/osv.ts (100%) rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/package-managers-2.test.ts (100%) rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/package-managers.test.ts (100%) rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/package-registries.test.ts (100%) rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/packagist.ts (100%) rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/pub-dev.ts (100%) rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/pubmed.ts (100%) rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/pypi.ts (100%) create mode 100644 packages/coding-agent/src/core/tools/web-scrapers/rawg.ts rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/readthedocs.ts (100%) rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/reddit.ts (100%) rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/repology.ts (100%) rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/research.test.ts (100%) rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/rfc.ts (100%) rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/rubygems.ts (100%) create mode 100644 packages/coding-agent/src/core/tools/web-scrapers/searchcode.ts rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/sec-edgar.ts (100%) rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/security.test.ts (100%) rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/semantic-scholar.ts (100%) create mode 100644 packages/coding-agent/src/core/tools/web-scrapers/snapcraft.ts rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/social-extended.test.ts (100%) rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/social.test.ts (100%) create mode 100644 packages/coding-agent/src/core/tools/web-scrapers/sourcegraph.ts create mode 100644 packages/coding-agent/src/core/tools/web-scrapers/spdx.ts rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/spotify.ts (100%) rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/stackexchange.test.ts (100%) rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/stackoverflow.ts (100%) rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/standards.test.ts (100%) rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/terraform.ts (100%) rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/tldr.ts (100%) rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/twitter.ts (100%) rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/types.ts (55%) create mode 100644 packages/coding-agent/src/core/tools/web-scrapers/utils.ts rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/vimeo.ts (100%) create mode 100644 packages/coding-agent/src/core/tools/web-scrapers/vscode-marketplace.ts create mode 100644 packages/coding-agent/src/core/tools/web-scrapers/w3c.ts rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/wikidata.ts (100%) rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/wikipedia.test.ts (100%) rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/wikipedia.ts (100%) rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/youtube.test.ts (100%) rename packages/coding-agent/src/core/tools/{web-fetch-handlers => web-scrapers}/youtube.ts (100%) diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index b3104a312..b9acc36d3 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -1,6 +1,23 @@ # Changelog ## [Unreleased] +### Added + +- Added 80+ specialized web scrapers for structured content extraction from popular sites including GitHub, GitLab, npm, PyPI, crates.io, Wikipedia, YouTube, Stack Overflow, Hacker News, Reddit, arXiv, PubMed, and many more +- Added site-specific API integrations for package registries (npm, PyPI, crates.io, Hex, Hackage, NuGet, Maven, RubyGems, Packagist, pub.dev, Go packages) +- Added scrapers for social platforms (Mastodon, Bluesky, Lemmy, Lobsters, Dev.to, Discourse) +- Added scrapers for academic sources (arXiv, bioRxiv, PubMed, Semantic Scholar, ORCID, CrossRef, IACR) +- Added scrapers for security databases (NVD, OSV, CISA KEV) +- Added scrapers for documentation sites (MDN, Read the Docs, RFC Editor, W3C, SPDX, tldr, cheat.sh) +- Added scrapers for media platforms (YouTube, Vimeo, Spotify, Discogs, MusicBrainz) +- Added scrapers for AI/ML platforms (Hugging Face, Ollama) +- Added scrapers for app stores and marketplaces (VS Code Marketplace, JetBrains Marketplace, Firefox Add-ons, Open VSX, Flathub, F-Droid, Snapcraft) +- Added scrapers for business data (SEC EDGAR, OpenCorporates, CoinGecko) +- Added scrapers for reference sources (Wikipedia, Wikidata, OpenLibrary, Choose a License) + +### Changed + +- Refactored web-fetch tool to use modular scraper architecture for improved maintainability ## [3.30.0] - 2026-01-07 ### Added diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/index.ts b/packages/coding-agent/src/core/tools/web-fetch-handlers/index.ts deleted file mode 100644 index 07b0da734..000000000 --- a/packages/coding-agent/src/core/tools/web-fetch-handlers/index.ts +++ /dev/null @@ -1,69 +0,0 @@ -/** - * Web Fetch Special Handlers Index - * - * Exports all special handlers for site-specific content extraction. - */ - -export { handleArtifactHub } from "./artifacthub"; -// Academic -export { handleArxiv } from "./arxiv"; -export { handleAur } from "./aur"; -export { handleBiorxiv } from "./biorxiv"; -export { handleBluesky } from "./bluesky"; -export { handleBrew } from "./brew"; -export { handleCheatSh } from "./cheatsh"; -export { handleChocolatey } from "./chocolatey"; -export { handleCoinGecko } from "./coingecko"; -export { handleCratesIo } from "./crates-io"; -export { handleDevTo } from "./devto"; -export { handleDiscogs } from "./discogs"; -export { handleDockerHub } from "./dockerhub"; -// Git hosting -export { fetchGitHubApi, handleGitHub } from "./github"; -export { handleGitHubGist } from "./github-gist"; -export { handleGitLab } from "./gitlab"; -export { handleGoPkg } from "./go-pkg"; -export { handleHackage } from "./hackage"; -export { handleHackerNews } from "./hackernews"; -export { handleHex } from "./hex"; -// ML/AI -export { handleHuggingFace } from "./huggingface"; -export { handleIacr } from "./iacr"; -export { handleLobsters } from "./lobsters"; -export { handleMastodon } from "./mastodon"; -export { handleMaven } from "./maven"; -export { handleMDN } from "./mdn"; -export { handleMetaCPAN } from "./metacpan"; -// Package registries -export { handleNpm } from "./npm"; -export { handleNuGet } from "./nuget"; -export { handleNvd } from "./nvd"; -export { handleOpenCorporates } from "./opencorporates"; -export { handleOpenLibrary } from "./openlibrary"; -export { handleOsv } from "./osv"; -export { handlePackagist } from "./packagist"; -export { handlePubDev } from "./pub-dev"; -export { handlePubMed } from "./pubmed"; -export { handlePyPI } from "./pypi"; -export { handleReadTheDocs } from "./readthedocs"; -export { handleReddit } from "./reddit"; -export { handleRepology } from "./repology"; -export { handleRfc } from "./rfc"; -export { handleRubyGems } from "./rubygems"; -export { handleSecEdgar } from "./sec-edgar"; -export { handleSemanticScholar } from "./semantic-scholar"; -export { handleSpotify } from "./spotify"; -// Developer content -export { handleStackOverflow } from "./stackoverflow"; -export { handleTerraform } from "./terraform"; -export { handleTldr } from "./tldr"; -// Social/News -export { handleTwitter } from "./twitter"; -export type { RenderResult, SpecialHandler } from "./types"; -export { handleVimeo } from "./vimeo"; - -// Reference -export { handleWikidata } from "./wikidata"; -export { handleWikipedia } from "./wikipedia"; -// Video/Media -export { handleYouTube } from "./youtube"; diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/utils.ts b/packages/coding-agent/src/core/tools/web-fetch-handlers/utils.ts deleted file mode 100644 index 2473cce9e..000000000 --- a/packages/coding-agent/src/core/tools/web-fetch-handlers/utils.ts +++ /dev/null @@ -1,91 +0,0 @@ -import { tmpdir } from "node:os"; -import * as path from "node:path"; -import { ensureTool } from "../../../utils/tools-manager"; - -const MAX_BYTES = 50 * 1024 * 1024; // 50MB for binary files - -function exec( - cmd: string, - args: string[], - options?: { timeout?: number; input?: string | Buffer }, -): { stdout: string; stderr: string; ok: boolean } { - const result = Bun.spawnSync([cmd, ...args], { - stdin: options?.input ? (options.input as any) : "ignore", - stdout: "pipe", - stderr: "pipe", - }); - return { - stdout: result.stdout?.toString() ?? "", - stderr: result.stderr?.toString() ?? "", - ok: result.exitCode === 0, - }; -} - -export async function convertWithMarkitdown( - content: Buffer, - extensionHint: string, - timeout: number, -): Promise<{ content: string; ok: boolean }> { - const markitdown = await ensureTool("markitdown", true); - if (!markitdown) { - return { content: "", ok: false }; - } - - // Write to temp file with extension hint - const ext = extensionHint || ".bin"; - const tmpDir = tmpdir(); - const tmpFile = path.join(tmpDir, `omp-convert-${Date.now()}${ext}`); - - try { - await Bun.write(tmpFile, content); - const result = exec(markitdown, [tmpFile], { timeout }); - return { content: result.stdout, ok: result.ok }; - } finally { - try { - await Bun.$`rm ${tmpFile}`.quiet(); - } catch {} - } -} - -export async function fetchBinary( - url: string, - timeout: number, -): Promise<{ buffer: Buffer; contentType: string; contentDisposition?: string; ok: boolean }> { - try { - const controller = new AbortController(); - const timeoutId = setTimeout(() => controller.abort(), timeout * 1000); - - const response = await fetch(url, { - signal: controller.signal, - headers: { - "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/131.0.0.0", - }, - redirect: "follow", - }); - - clearTimeout(timeoutId); - - if (!response.ok) { - return { buffer: Buffer.alloc(0), contentType: "", ok: false }; - } - - const contentType = response.headers.get("content-type") ?? ""; - const contentDisposition = response.headers.get("content-disposition") ?? undefined; - const contentLength = response.headers.get("content-length"); - if (contentLength) { - const size = Number.parseInt(contentLength, 10); - if (Number.isFinite(size) && size > MAX_BYTES) { - return { buffer: Buffer.alloc(0), contentType, contentDisposition, ok: false }; - } - } - - const buffer = Buffer.from(await response.arrayBuffer()); - if (buffer.length > MAX_BYTES) { - return { buffer: Buffer.alloc(0), contentType, contentDisposition, ok: false }; - } - - return { buffer, contentType, contentDisposition, ok: true }; - } catch { - return { buffer: Buffer.alloc(0), contentType: "", ok: false }; - } -} diff --git a/packages/coding-agent/src/core/tools/web-fetch.ts b/packages/coding-agent/src/core/tools/web-fetch.ts index 8f7b5ea29..329a6f275 100644 --- a/packages/coding-agent/src/core/tools/web-fetch.ts +++ b/packages/coding-agent/src/core/tools/web-fetch.ts @@ -5,82 +5,17 @@ import { Type } from "@sinclair/typebox"; import { parse as parseHtml } from "node-html-parser"; import webFetchDescription from "../../prompts/tools/web-fetch.md" with { type: "text" }; import { ensureTool } from "../../utils/tools-manager"; -import { logger } from "../logger"; import type { ToolSession } from "./index"; -import { - handleArtifactHub, - handleArxiv, - handleAur, - handleBiorxiv, - handleBluesky, - handleBrew, - handleCheatSh, - handleChocolatey, - handleCoinGecko, - handleCratesIo, - handleDevTo, - handleDiscogs, - handleDockerHub, - handleGitHub, - handleGitHubGist, - handleGitLab, - handleGoPkg, - handleHackage, - handleHackerNews, - handleHex, - handleHuggingFace, - handleIacr, - handleLobsters, - handleMastodon, - handleMaven, - handleMDN, - handleMetaCPAN, - handleNpm, - handleNuGet, - handleNvd, - handleOpenCorporates, - handleOpenLibrary, - handleOsv, - handlePackagist, - handlePubDev, - handlePubMed, - handlePyPI, - handleReadTheDocs, - handleReddit, - handleRepology, - handleRfc, - handleRubyGems, - handleSecEdgar, - handleSemanticScholar, - handleSpotify, - handleStackOverflow, - handleTerraform, - handleTldr, - handleTwitter, - handleVimeo, - handleWikidata, - handleWikipedia, - handleYouTube, -} from "./web-fetch-handlers/index"; +import { specialHandlers } from "./web-scrapers/index"; +import type { RenderResult } from "./web-scrapers/types"; +import { finalizeOutput, loadPage } from "./web-scrapers/types"; +import { convertWithMarkitdown, fetchBinary } from "./web-scrapers/utils"; // ============================================================================= // Types and Constants // ============================================================================= -interface RenderResult { - url: string; - finalUrl: string; - contentType: string; - method: string; - content: string; - fetchedAt: string; - truncated: boolean; - notes: string[]; -} - const DEFAULT_TIMEOUT = 20; -const MAX_BYTES = 50 * 1024 * 1024; // 50MB for binary files -const MAX_OUTPUT_CHARS = 500_000; // Convertible document types (markitdown supported) const CONVERTIBLE_MIMES = new Set([ @@ -123,124 +58,11 @@ const CONVERTIBLE_EXTENSIONS = new Set([ ".ogg", ]); -const USER_AGENTS = [ - "curl/8.0", - "Mozilla/5.0 (compatible; TextBot/1.0)", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36", -]; - // ============================================================================= // Utilities // ============================================================================= -interface LoadPageResult { - content: string; - contentType: string; - finalUrl: string; - ok: boolean; - status?: number; -} - -interface LoadPageOptions { - timeout?: number; - headers?: Record; - maxBytes?: number; -} - -/** - * Check if response indicates bot blocking (Cloudflare, etc.) - */ -function isBotBlocked(status: number, content: string): boolean { - if (status === 403 || status === 503) { - const lower = content.toLowerCase(); - return ( - lower.includes("cloudflare") || - lower.includes("captcha") || - lower.includes("challenge") || - lower.includes("blocked") || - lower.includes("access denied") || - lower.includes("bot detection") - ); - } - return false; -} - -/** - * Fetch a page with timeout, size limit, and automatic retry with browser UA if blocked - */ -async function loadPage(url: string, options: LoadPageOptions = {}): Promise { - const { timeout = 20, headers = {}, maxBytes = MAX_BYTES } = options; - - for (let attempt = 0; attempt < USER_AGENTS.length; attempt++) { - const userAgent = USER_AGENTS[attempt]; - - try { - const controller = new AbortController(); - const timeoutId = setTimeout(() => controller.abort(), timeout * 1000); - - const response = await fetch(url, { - signal: controller.signal, - headers: { - "User-Agent": userAgent, - Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", - "Accept-Language": "en-US,en;q=0.5", - ...headers, - }, - redirect: "follow", - }); - - clearTimeout(timeoutId); - - const contentType = response.headers.get("content-type")?.split(";")[0]?.trim().toLowerCase() ?? ""; - const finalUrl = response.url; - - // Read with size limit - const reader = response.body?.getReader(); - if (!reader) { - return { content: "", contentType, finalUrl, ok: false, status: response.status }; - } - - const chunks: Uint8Array[] = []; - let totalSize = 0; - - while (true) { - const { done, value } = await reader.read(); - if (done) break; - - chunks.push(value); - totalSize += value.length; - - if (totalSize > maxBytes) { - reader.cancel(); - break; - } - } - - const decoder = new TextDecoder(); - const content = decoder.decode(Buffer.concat(chunks)); - - // Check if we got blocked and should retry with browser UA - if (isBotBlocked(response.status, content) && attempt < USER_AGENTS.length - 1) { - continue; - } - - if (!response.ok) { - return { content, contentType, finalUrl, ok: false, status: response.status }; - } - - return { content, contentType, finalUrl, ok: true, status: response.status }; - } catch (err) { - // On last attempt, return failure - if (attempt === USER_AGENTS.length - 1) { - logger.debug("Web fetch failed after retries", { url, error: String(err) }); - return { content: "", contentType: "", finalUrl: url, ok: false }; - } - // Otherwise retry with next UA - } - } - - return { content: "", contentType: "", finalUrl: url, ok: false }; -} +type SpawnSyncOptions = NonNullable[1]>; /** * Execute a command and return stdout @@ -250,8 +72,9 @@ function exec( args: string[], options?: { timeout?: number; input?: string | Buffer }, ): { stdout: string; stderr: string; ok: boolean } { + const stdin = (options?.input ?? "ignore") as SpawnSyncOptions["stdin"]; const result = Bun.spawnSync([cmd, ...args], { - stdin: options?.input ? (options.input as any) : "ignore", + stdin, stdout: "pipe", stderr: "pipe", }); @@ -344,39 +167,10 @@ function looksLikeHtml(content: string): boolean { ); } -/** - * Convert binary file to markdown using markitdown - */ -async function convertWithMarkitdown( - content: Buffer, - extensionHint: string, - timeout: number, -): Promise<{ content: string; ok: boolean }> { - const markitdown = await ensureTool("markitdown", true); - if (!markitdown) { - return { content: "", ok: false }; - } - - // Write to temp file with extension hint - const ext = extensionHint || ".bin"; - const tmpDir = tmpdir(); - const tmpFile = path.join(tmpDir, `omp-convert-${Date.now()}${ext}`); - - try { - await Bun.write(tmpFile, content); - const result = exec(markitdown, [tmpFile], { timeout }); - return { content: result.stdout, ok: result.ok }; - } finally { - try { - await Bun.$`rm ${tmpFile}`.quiet(); - } catch {} - } -} - /** * Try fetching URL with .md appended (llms.txt convention) */ -async function tryMdSuffix(url: string, timeout: number): Promise { +async function tryMdSuffix(url: string, timeout: number, signal?: AbortSignal): Promise { const candidates: string[] = []; try { @@ -397,8 +191,15 @@ async function tryMdSuffix(url: string, timeout: number): Promise return null; } + if (signal?.aborted) { + return null; + } + for (const candidate of candidates) { - const result = await loadPage(candidate, { timeout: Math.min(timeout, 5) }); + if (signal?.aborted) { + return null; + } + const result = await loadPage(candidate, { timeout: Math.min(timeout, 5), signal }); if (result.ok && result.content.trim().length > 100 && !looksLikeHtml(result.content)) { return result.content; } @@ -410,11 +211,18 @@ async function tryMdSuffix(url: string, timeout: number): Promise /** * Try to fetch LLM-friendly endpoints */ -async function tryLlmEndpoints(origin: string, timeout: number): Promise { +async function tryLlmEndpoints(origin: string, timeout: number, signal?: AbortSignal): Promise { const endpoints = [`${origin}/.well-known/llms.txt`, `${origin}/llms.txt`, `${origin}/llms.md`]; + if (signal?.aborted) { + return null; + } + for (const endpoint of endpoints) { - const result = await loadPage(endpoint, { timeout: Math.min(timeout, 5) }); + if (signal?.aborted) { + return null; + } + const result = await loadPage(endpoint, { timeout: Math.min(timeout, 5), signal }); if (result.ok && result.content.trim().length > 100 && !looksLikeHtml(result.content)) { return result.content; } @@ -425,10 +233,19 @@ async function tryLlmEndpoints(origin: string, timeout: number): Promise { +async function tryContentNegotiation( + url: string, + timeout: number, + signal?: AbortSignal, +): Promise<{ content: string; type: string } | null> { + if (signal?.aborted) { + return null; + } + const result = await loadPage(url, { timeout, headers: { Accept: "text/markdown, text/plain;q=0.9, text/html;q=0.8" }, + signal, }); if (!result.ok) return null; @@ -658,64 +475,6 @@ function formatJson(content: string): string { } } -/** - * Truncate and cleanup output - */ -function finalizeOutput(content: string): { content: string; truncated: boolean } { - const cleaned = content.replace(/\n{3,}/g, "\n\n").trim(); - const truncated = cleaned.length > MAX_OUTPUT_CHARS; - return { - content: cleaned.slice(0, MAX_OUTPUT_CHARS), - truncated, - }; -} - -/** - * Fetch page as binary buffer (for convertible files) - */ -async function fetchBinary( - url: string, - timeout: number, -): Promise<{ buffer: Buffer; contentType: string; contentDisposition?: string; ok: boolean }> { - try { - const controller = new AbortController(); - const timeoutId = setTimeout(() => controller.abort(), timeout * 1000); - - const response = await fetch(url, { - signal: controller.signal, - headers: { - "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/131.0.0.0", - }, - redirect: "follow", - }); - - clearTimeout(timeoutId); - - if (!response.ok) { - return { buffer: Buffer.alloc(0), contentType: "", ok: false }; - } - - const contentType = response.headers.get("content-type") ?? ""; - const contentDisposition = response.headers.get("content-disposition") ?? undefined; - const contentLength = response.headers.get("content-length"); - if (contentLength) { - const size = Number.parseInt(contentLength, 10); - if (Number.isFinite(size) && size > MAX_BYTES) { - return { buffer: Buffer.alloc(0), contentType, contentDisposition, ok: false }; - } - } - - const buffer = Buffer.from(await response.arrayBuffer()); - if (buffer.length > MAX_BYTES) { - return { buffer: Buffer.alloc(0), contentType, contentDisposition, ok: false }; - } - - return { buffer, contentType, contentDisposition, ok: true }; - } catch { - return { buffer: Buffer.alloc(0), contentType: "", ok: false }; - } -} - // ============================================================================= // Unified Special Handler Dispatch // ============================================================================= @@ -723,74 +482,15 @@ async function fetchBinary( /** * Try all special handlers */ -async function handleSpecialUrls(url: string, timeout: number): Promise { - // Order matters - more specific first - return ( - // Git hosting - (await handleGitHubGist(url, timeout)) || - (await handleGitHub(url, timeout)) || - (await handleGitLab(url, timeout)) || - // Video/Media - (await handleYouTube(url, timeout)) || - (await handleVimeo(url, timeout)) || - (await handleSpotify(url, timeout)) || - (await handleDiscogs(url, timeout)) || - // Social/News - (await handleTwitter(url, timeout)) || - (await handleBluesky(url, timeout)) || - (await handleMastodon(url, timeout)) || - (await handleHackerNews(url, timeout)) || - (await handleLobsters(url, timeout)) || - (await handleReddit(url, timeout)) || - // Developer content - (await handleStackOverflow(url, timeout)) || - (await handleDevTo(url, timeout)) || - (await handleMDN(url, timeout)) || - (await handleReadTheDocs(url, timeout)) || - (await handleTldr(url, timeout)) || - (await handleCheatSh(url, timeout)) || - // Package registries - (await handleNpm(url, timeout)) || - (await handleNuGet(url, timeout)) || - (await handleChocolatey(url, timeout)) || - (await handleBrew(url, timeout)) || - (await handlePyPI(url, timeout)) || - (await handleCratesIo(url, timeout)) || - (await handleDockerHub(url, timeout)) || - (await handleGoPkg(url, timeout)) || - (await handleHex(url, timeout)) || - (await handlePackagist(url, timeout)) || - (await handlePubDev(url, timeout)) || - (await handleMaven(url, timeout)) || - (await handleArtifactHub(url, timeout)) || - (await handleRubyGems(url, timeout)) || - (await handleTerraform(url, timeout)) || - (await handleAur(url, timeout)) || - (await handleHackage(url, timeout)) || - (await handleMetaCPAN(url, timeout)) || - (await handleRepology(url, timeout)) || - // ML/AI - (await handleHuggingFace(url, timeout)) || - // Academic - (await handleArxiv(url, timeout)) || - (await handleBiorxiv(url, timeout)) || - (await handleIacr(url, timeout)) || - (await handleSemanticScholar(url, timeout)) || - (await handlePubMed(url, timeout)) || - (await handleRfc(url, timeout)) || - // Security - (await handleNvd(url, timeout)) || - (await handleOsv(url, timeout)) || - // Crypto - (await handleCoinGecko(url, timeout)) || - // Business - (await handleOpenCorporates(url, timeout)) || - (await handleSecEdgar(url, timeout)) || - // Reference - (await handleOpenLibrary(url, timeout)) || - (await handleWikidata(url, timeout)) || - (await handleWikipedia(url, timeout)) - ); +async function handleSpecialUrls(url: string, timeout: number, signal?: AbortSignal): Promise { + for (const handler of specialHandlers) { + if (signal?.aborted) { + throw new Error("Operation aborted"); + } + const result = await handler(url, timeout); + if (result) return result; + } + return null; } // ============================================================================= @@ -800,9 +500,17 @@ async function handleSpecialUrls(url: string, timeout: number): Promise { +async function renderUrl( + url: string, + timeout: number, + raw: boolean = false, + signal?: AbortSignal, +): Promise { const notes: string[] = []; const fetchedAt = new Date().toISOString(); + if (signal?.aborted) { + throw new Error("Operation aborted"); + } // Step 0: Normalize URL (ensure scheme for special handlers) url = normalizeUrl(url); @@ -810,12 +518,15 @@ async function renderUrl(url: string, timeout: number, raw: boolean = false): Pr // Step 1: Try special handlers for known sites (unless raw mode) if (!raw) { - const specialResult = await handleSpecialUrls(url, timeout); + const specialResult = await handleSpecialUrls(url, timeout, signal); if (specialResult) return specialResult; } // Step 2: Fetch page - const response = await loadPage(url, { timeout }); + const response = await loadPage(url, { timeout, signal }); + if (signal?.aborted) { + throw new Error("Operation aborted"); + } if (!response.ok) { return { url, @@ -835,26 +546,36 @@ async function renderUrl(url: string, timeout: number, raw: boolean = false): Pr // Step 3: Handle convertible binary files (PDF, DOCX, etc.) if (isConvertible(mime, extHint)) { - const binary = await fetchBinary(finalUrl, timeout); + const binary = await fetchBinary(finalUrl, timeout, signal); if (binary.ok) { const ext = getExtensionHint(finalUrl, binary.contentDisposition) || extHint; - const converted = await convertWithMarkitdown(binary.buffer, ext, timeout); - if (converted.ok && converted.content.trim().length > 50) { - notes.push(`Converted with markitdown`); - const output = finalizeOutput(converted.content); - return { - url, - finalUrl, - contentType: mime, - method: "markitdown", - content: output.content, - fetchedAt, - truncated: output.truncated, - notes, - }; + const converted = await convertWithMarkitdown(binary.buffer, ext, timeout, signal); + if (converted.ok) { + if (converted.content.trim().length > 50) { + notes.push("Converted with markitdown"); + const output = finalizeOutput(converted.content); + return { + url, + finalUrl, + contentType: mime, + method: "markitdown", + content: output.content, + fetchedAt, + truncated: output.truncated, + notes, + }; + } + notes.push("markitdown conversion produced no usable output"); + } else if (converted.error) { + notes.push(`markitdown conversion failed: ${converted.error}`); + } else { + notes.push("markitdown conversion failed"); } + } else if (binary.error) { + notes.push(`Binary fetch failed: ${binary.error}`); + } else { + notes.push("Binary fetch failed"); } - notes.push("markitdown conversion failed"); } // Step 4: Handle non-HTML text content @@ -914,7 +635,7 @@ async function renderUrl(url: string, timeout: number, raw: boolean = false): Pr const markdownAlt = alternates.find((alt) => alt.endsWith(".md") || alt.includes("markdown")); if (markdownAlt) { const resolved = markdownAlt.startsWith("http") ? markdownAlt : new URL(markdownAlt, finalUrl).href; - const altResult = await loadPage(resolved, { timeout }); + const altResult = await loadPage(resolved, { timeout, signal }); if (altResult.ok && altResult.content.trim().length > 100 && !looksLikeHtml(altResult.content)) { notes.push(`Used markdown alternate: ${resolved}`); const output = finalizeOutput(altResult.content); @@ -932,7 +653,7 @@ async function renderUrl(url: string, timeout: number, raw: boolean = false): Pr } // 5B: Try URL.md suffix (llms.txt convention) - const mdSuffix = await tryMdSuffix(finalUrl, timeout); + const mdSuffix = await tryMdSuffix(finalUrl, timeout, signal); if (mdSuffix) { notes.push("Found .md suffix version"); const output = finalizeOutput(mdSuffix); @@ -949,7 +670,7 @@ async function renderUrl(url: string, timeout: number, raw: boolean = false): Pr } // 5C: LLM-friendly endpoints - const llmContent = await tryLlmEndpoints(origin, timeout); + const llmContent = await tryLlmEndpoints(origin, timeout, signal); if (llmContent) { notes.push("Found llms.txt"); const output = finalizeOutput(llmContent); @@ -966,7 +687,7 @@ async function renderUrl(url: string, timeout: number, raw: boolean = false): Pr } // 5D: Content negotiation - const negotiated = await tryContentNegotiation(url, timeout); + const negotiated = await tryContentNegotiation(url, timeout, signal); if (negotiated) { notes.push(`Content negotiation returned ${negotiated.type}`); const output = finalizeOutput(negotiated.content); @@ -986,7 +707,7 @@ async function renderUrl(url: string, timeout: number, raw: boolean = false): Pr const feedAlternates = alternates.filter((alt) => !alt.endsWith(".md") && !alt.includes("markdown")); for (const altUrl of feedAlternates.slice(0, 2)) { const resolved = altUrl.startsWith("http") ? altUrl : new URL(altUrl, finalUrl).href; - const altResult = await loadPage(resolved, { timeout }); + const altResult = await loadPage(resolved, { timeout, signal }); if (altResult.ok && altResult.content.trim().length > 200) { notes.push(`Used feed alternate: ${resolved}`); const parsed = parseFeedToMarkdown(altResult.content); @@ -1004,6 +725,10 @@ async function renderUrl(url: string, timeout: number, raw: boolean = false): Pr } } + if (signal?.aborted) { + throw new Error("Operation aborted"); + } + // Step 6: Render HTML with lynx or html2text const htmlResult = await renderHtmlToText(rawContent, timeout); if (!htmlResult.ok) { @@ -1026,10 +751,10 @@ async function renderUrl(url: string, timeout: number, raw: boolean = false): Pr const docLinks = extractDocumentLinks(rawContent, finalUrl); if (docLinks.length > 0) { const docUrl = docLinks[0]; - const binary = await fetchBinary(docUrl, timeout); + const binary = await fetchBinary(docUrl, timeout, signal); if (binary.ok) { const ext = getExtensionHint(docUrl, binary.contentDisposition); - const converted = await convertWithMarkitdown(binary.buffer, ext, timeout); + const converted = await convertWithMarkitdown(binary.buffer, ext, timeout, signal); if (converted.ok && converted.content.trim().length > htmlResult.content.length) { notes.push(`Extracted and converted document: ${docUrl}`); const output = finalizeOutput(converted.content); @@ -1044,6 +769,11 @@ async function renderUrl(url: string, timeout: number, raw: boolean = false): Pr notes, }; } + if (!converted.ok && converted.error) { + notes.push(`markitdown conversion failed: ${converted.error}`); + } + } else if (binary.error) { + notes.push(`Binary fetch failed: ${binary.error}`); } } notes.push("Page appears to require JavaScript or is mostly navigation"); @@ -1106,11 +836,16 @@ export function createWebFetchTool(_session: ToolSession): AgentTool { + if (signal?.aborted) { + throw new Error("Operation aborted"); + } + // Clamp timeout const effectiveTimeout = Math.min(Math.max(timeout, 1), 120); - const result = await renderUrl(url, effectiveTimeout, raw); + const result = await renderUrl(url, effectiveTimeout, raw, signal); // Format output let output = ""; diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/academic.test.ts b/packages/coding-agent/src/core/tools/web-scrapers/academic.test.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/academic.test.ts rename to packages/coding-agent/src/core/tools/web-scrapers/academic.test.ts diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/artifacthub.ts b/packages/coding-agent/src/core/tools/web-scrapers/artifacthub.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/artifacthub.ts rename to packages/coding-agent/src/core/tools/web-scrapers/artifacthub.ts diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/arxiv.ts b/packages/coding-agent/src/core/tools/web-scrapers/arxiv.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/arxiv.ts rename to packages/coding-agent/src/core/tools/web-scrapers/arxiv.ts diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/aur.ts b/packages/coding-agent/src/core/tools/web-scrapers/aur.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/aur.ts rename to packages/coding-agent/src/core/tools/web-scrapers/aur.ts diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/biorxiv.ts b/packages/coding-agent/src/core/tools/web-scrapers/biorxiv.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/biorxiv.ts rename to packages/coding-agent/src/core/tools/web-scrapers/biorxiv.ts diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/bluesky.ts b/packages/coding-agent/src/core/tools/web-scrapers/bluesky.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/bluesky.ts rename to packages/coding-agent/src/core/tools/web-scrapers/bluesky.ts diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/brew.ts b/packages/coding-agent/src/core/tools/web-scrapers/brew.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/brew.ts rename to packages/coding-agent/src/core/tools/web-scrapers/brew.ts diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/business.test.ts b/packages/coding-agent/src/core/tools/web-scrapers/business.test.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/business.test.ts rename to packages/coding-agent/src/core/tools/web-scrapers/business.test.ts diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/cheatsh.ts b/packages/coding-agent/src/core/tools/web-scrapers/cheatsh.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/cheatsh.ts rename to packages/coding-agent/src/core/tools/web-scrapers/cheatsh.ts diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/chocolatey.ts b/packages/coding-agent/src/core/tools/web-scrapers/chocolatey.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/chocolatey.ts rename to packages/coding-agent/src/core/tools/web-scrapers/chocolatey.ts diff --git a/packages/coding-agent/src/core/tools/web-scrapers/choosealicense.ts b/packages/coding-agent/src/core/tools/web-scrapers/choosealicense.ts new file mode 100644 index 000000000..a12a6b47a --- /dev/null +++ b/packages/coding-agent/src/core/tools/web-scrapers/choosealicense.ts @@ -0,0 +1,109 @@ +import { parseFrontmatter } from "../../../discovery/helpers"; +import type { RenderResult, SpecialHandler } from "./types"; +import { finalizeOutput, loadPage } from "./types"; + +const ALLOWED_HOSTS = new Set(["choosealicense.com", "www.choosealicense.com"]); +const LICENSE_PATH = /^\/licenses\/([^/]+)\/?$/i; +const APPENDIX_PATH = /^\/appendix\/?$/i; + +function asString(value: unknown): string | undefined { + if (typeof value !== "string") return undefined; + const trimmed = value.trim(); + return trimmed.length > 0 ? trimmed : undefined; +} + +function normalizeList(value: unknown): string[] { + if (Array.isArray(value)) { + return value + .filter((item): item is string => typeof item === "string") + .map((item) => item.trim()) + .filter((item) => item.length > 0); + } + if (typeof value === "string") { + return value + .split(",") + .map((item) => item.trim()) + .filter((item) => item.length > 0); + } + return []; +} + +function formatLabel(value: string): string { + const cleaned = value.replace(/[-_]+/g, " ").replace(/\s+/g, " ").trim(); + if (!cleaned) return value; + return cleaned.charAt(0).toUpperCase() + cleaned.slice(1); +} + +function formatSection(title: string, items: string[]): string { + let md = `## ${title}\n\n`; + if (items.length === 0) { + md += "- None listed\n\n"; + return md; + } + for (const item of items) { + md += `- ${formatLabel(item)}\n`; + } + md += "\n"; + return md; +} + +export const handleChooseALicense: SpecialHandler = async ( + url: string, + timeout: number, +): Promise => { + try { + const parsed = new URL(url); + if (!ALLOWED_HOSTS.has(parsed.hostname)) return null; + + const licenseMatch = parsed.pathname.match(LICENSE_PATH); + const isAppendix = APPENDIX_PATH.test(parsed.pathname); + if (!licenseMatch && !isAppendix) return null; + + const licenseSlug = licenseMatch ? decodeURIComponent(licenseMatch[1]).toLowerCase() : "appendix"; + const rawUrl = licenseMatch + ? `https://raw.githubusercontent.com/github/choosealicense.com/gh-pages/_licenses/${licenseSlug}.txt` + : "https://raw.githubusercontent.com/github/choosealicense.com/gh-pages/_pages/appendix.md"; + + const fetchedAt = new Date().toISOString(); + const result = await loadPage(rawUrl, { timeout, headers: { Accept: "text/plain" } }); + if (!result.ok) return null; + + const { frontmatter, body } = parseFrontmatter(result.content); + + const title = asString(frontmatter.title) ?? formatLabel(licenseSlug); + const spdxId = asString(frontmatter["spdx-id"]) ?? "Unknown"; + const description = asString(frontmatter.description); + const permissions = normalizeList(frontmatter.permissions); + const conditions = normalizeList(frontmatter.conditions); + const limitations = normalizeList(frontmatter.limitations); + + let md = `# ${title}\n\n`; + if (description) md += `${description}\n\n`; + + md += `**SPDX ID:** ${spdxId}\n`; + md += `**Source:** https://choosealicense.com${isAppendix ? "/appendix" : `/licenses/${licenseSlug}/`}\n\n`; + + md += formatSection("Permissions", permissions); + md += formatSection("Conditions", conditions); + md += formatSection("Limitations", limitations); + + const licenseText = body.trim(); + if (licenseText.length > 0) { + md += `---\n\n## License Text\n\n${licenseText}\n`; + } + + const output = finalizeOutput(md); + return { + url, + finalUrl: url, + contentType: "text/markdown", + method: "choosealicense", + content: output.content, + fetchedAt, + truncated: output.truncated, + notes: ["Fetched via Choose a License"], + }; + } catch {} + + return null; +}; diff --git a/packages/coding-agent/src/core/tools/web-scrapers/cisa-kev.ts b/packages/coding-agent/src/core/tools/web-scrapers/cisa-kev.ts new file mode 100644 index 000000000..2ae0a4562 --- /dev/null +++ b/packages/coding-agent/src/core/tools/web-scrapers/cisa-kev.ts @@ -0,0 +1,95 @@ +import type { RenderResult, SpecialHandler } from "./types"; +import { finalizeOutput, loadPage } from "./types"; + +interface KevEntry { + cveID: string; + vendorProject?: string; + product?: string; + vulnerabilityName?: string; + shortDescription?: string; + requiredAction?: string; + dateAdded?: string; + dueDate?: string; +} + +interface KevCatalog { + title?: string; + catalogVersion?: string; + dateReleased?: string; + count?: number; + vulnerabilities?: KevEntry[]; +} + +const CVE_PATTERN = /CVE-\d{4}-\d{4,7}/i; +const KEV_FEED_URL = "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json"; + +/** + * Handle CISA Known Exploited Vulnerabilities (KEV) URLs + */ +export const handleCisaKev: SpecialHandler = async (url: string, timeout: number): Promise => { + try { + const parsed = new URL(url); + const hostname = parsed.hostname.toLowerCase(); + if (!hostname.endsWith("cisa.gov")) return null; + + const path = parsed.pathname.toLowerCase(); + if (!path.includes("known-exploited-vulnerabilities")) return null; + + const cveMatch = parsed.pathname.match(CVE_PATTERN) ?? parsed.search.match(CVE_PATTERN); + if (!cveMatch) return null; + + const cveId = cveMatch[0].toUpperCase(); + const fetchedAt = new Date().toISOString(); + + const result = await loadPage(KEV_FEED_URL, { + timeout, + headers: { Accept: "application/json" }, + }); + + if (!result.ok) return null; + + let data: KevCatalog; + try { + data = JSON.parse(result.content) as KevCatalog; + } catch { + return null; + } + + const entry = data.vulnerabilities?.find((item) => item.cveID?.toUpperCase() === cveId); + if (!entry) return null; + + let md = `# ${entry.cveID}\n\n`; + if (entry.vulnerabilityName) { + md += `${entry.vulnerabilityName}\n\n`; + } + + md += "## Metadata\n\n"; + if (entry.vendorProject) md += `**Vendor:** ${entry.vendorProject}\n`; + if (entry.product) md += `**Product:** ${entry.product}\n`; + if (entry.dateAdded) md += `**Date Added:** ${entry.dateAdded}\n`; + if (entry.dueDate) md += `**Due Date:** ${entry.dueDate}\n`; + md += "\n"; + + if (entry.shortDescription) { + md += `## Description\n\n${entry.shortDescription}\n\n`; + } + + if (entry.requiredAction) { + md += `## Required Action\n\n${entry.requiredAction}\n\n`; + } + + const output = finalizeOutput(md); + return { + url, + finalUrl: url, + contentType: "text/markdown", + method: "cisa-kev", + content: output.content, + fetchedAt, + truncated: output.truncated, + notes: ["Fetched via CISA KEV feed"], + }; + } catch {} + + return null; +}; diff --git a/packages/coding-agent/src/core/tools/web-scrapers/clojars.ts b/packages/coding-agent/src/core/tools/web-scrapers/clojars.ts new file mode 100644 index 000000000..be47cd0bc --- /dev/null +++ b/packages/coding-agent/src/core/tools/web-scrapers/clojars.ts @@ -0,0 +1,175 @@ +import type { RenderResult, SpecialHandler } from "./types"; +import { finalizeOutput, formatCount, loadPage } from "./types"; + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null; +} + +function asString(value: unknown): string | null { + if (typeof value !== "string") return null; + const trimmed = value.trim(); + return trimmed.length > 0 ? trimmed : null; +} + +function asNumber(value: unknown): number | null { + return typeof value === "number" && Number.isFinite(value) ? value : null; +} + +function formatLicenses(licenses: unknown): string[] { + if (!Array.isArray(licenses)) return []; + const output: string[] = []; + for (const license of licenses) { + if (typeof license === "string") { + const trimmed = license.trim(); + if (trimmed) output.push(trimmed); + continue; + } + if (isRecord(license)) { + const name = asString(license.name); + const url = asString(license.url); + if (name && url) { + output.push(`${name} (${url})`); + } else if (name) { + output.push(name); + } else if (url) { + output.push(url); + } + } + } + return output; +} + +function formatDependencies(deps: unknown): string[] { + const output: string[] = []; + if (Array.isArray(deps)) { + for (const dep of deps) { + if (typeof dep === "string") { + const trimmed = dep.trim(); + if (trimmed) output.push(trimmed); + continue; + } + if (Array.isArray(dep)) { + const name = asString(dep[0]); + const version = asString(dep[1]); + if (name && version) { + output.push(`${name}: ${version}`); + } else if (name) { + output.push(name); + } + continue; + } + if (isRecord(dep)) { + const name = asString(dep.name) ?? asString(dep.artifact) ?? asString(dep.jar_name); + const version = asString(dep.version); + if (name && version) { + output.push(`${name}: ${version}`); + } else if (name) { + output.push(name); + } + } + } + return output; + } + + if (isRecord(deps)) { + for (const [name, version] of Object.entries(deps)) { + const versionText = asString(version); + if (versionText) { + output.push(`${name}: ${versionText}`); + } else if (name.trim()) { + output.push(name); + } + } + } + + return output; +} + +/** + * Handle Clojars URLs via API + */ +export const handleClojars: SpecialHandler = async (url: string, timeout: number): Promise => { + try { + const parsed = new URL(url); + if (parsed.hostname !== "clojars.org" && parsed.hostname !== "www.clojars.org") return null; + + const path = parsed.pathname.replace(/^\/+|\/+$/g, ""); + if (!path) return null; + + const segments = path.split("/").filter(Boolean); + if (segments.length < 1 || segments.length > 2) return null; + + const groupFromUrl = segments.length === 2 ? decodeURIComponent(segments[0]) : null; + const artifactFromUrl = decodeURIComponent(segments[segments.length - 1]); + + const apiUrl = + segments.length === 2 + ? `https://clojars.org/api/artifacts/${encodeURIComponent(groupFromUrl ?? "")}/${encodeURIComponent(artifactFromUrl)}` + : `https://clojars.org/api/artifacts/${encodeURIComponent(artifactFromUrl)}`; + + const fetchedAt = new Date().toISOString(); + + const result = await loadPage(apiUrl, { + timeout, + headers: { Accept: "application/json" }, + }); + + if (!result.ok) return null; + + let payload: unknown; + try { + payload = JSON.parse(result.content); + } catch { + return null; + } + + const data = Array.isArray(payload) ? payload[0] : payload; + if (!isRecord(data)) return null; + + const groupName = asString(data.group_name) ?? asString(data.group) ?? groupFromUrl; + const artifactName = asString(data.jar_name) ?? asString(data.artifact) ?? asString(data.name) ?? artifactFromUrl; + const version = asString(data.latest_version) ?? asString(data.version); + const description = asString(data.description) ?? asString(data.summary); + const downloads = + asNumber(data.downloads) ?? asNumber(data.downloads_total) ?? asNumber(data.total_downloads) ?? null; + const homepage = asString(data.homepage) ?? asString(data.url); + const licenses = formatLicenses(data.licenses); + const dependencies = formatDependencies(data.dependencies ?? data.deps); + + const displayName = + groupName && artifactName && groupName !== artifactName + ? `${groupName}/${artifactName}` + : (artifactName ?? groupName ?? "Clojars artifact"); + + let md = `# ${displayName}\n\n`; + if (description) md += `${description}\n\n`; + + if (groupName) md += `**Group:** ${groupName}\n`; + if (artifactName) md += `**Artifact:** ${artifactName}\n`; + if (version) md += `**Latest:** ${version}\n`; + if (downloads !== null) md += `**Downloads:** ${formatCount(downloads)}\n`; + if (homepage) md += `**Homepage:** ${homepage}\n`; + if (licenses.length > 0) md += `**Licenses:** ${licenses.join(", ")}\n`; + + if (dependencies.length > 0) { + md += "\n## Dependencies\n\n"; + for (const dep of dependencies) { + md += `- ${dep}\n`; + } + } + + const output = finalizeOutput(md); + return { + url, + finalUrl: url, + contentType: "text/markdown", + method: "clojars", + content: output.content, + fetchedAt, + truncated: output.truncated, + notes: ["Fetched via Clojars API"], + }; + } catch {} + + return null; +}; diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/coingecko.ts b/packages/coding-agent/src/core/tools/web-scrapers/coingecko.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/coingecko.ts rename to packages/coding-agent/src/core/tools/web-scrapers/coingecko.ts diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/crates-io.ts b/packages/coding-agent/src/core/tools/web-scrapers/crates-io.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/crates-io.ts rename to packages/coding-agent/src/core/tools/web-scrapers/crates-io.ts diff --git a/packages/coding-agent/src/core/tools/web-scrapers/crossref.ts b/packages/coding-agent/src/core/tools/web-scrapers/crossref.ts new file mode 100644 index 000000000..1f4619fb8 --- /dev/null +++ b/packages/coding-agent/src/core/tools/web-scrapers/crossref.ts @@ -0,0 +1,144 @@ +import type { RenderResult, SpecialHandler } from "./types"; +import { finalizeOutput, htmlToBasicMarkdown, loadPage } from "./types"; + +interface CrossrefAuthor { + given?: string; + family?: string; + name?: string; +} + +interface CrossrefDate { + "date-parts"?: number[][]; +} + +interface CrossrefMessage { + title?: string[]; + author?: CrossrefAuthor[]; + "container-title"?: string[]; + "short-container-title"?: string[]; + publisher?: string; + published?: CrossrefDate; + "published-print"?: CrossrefDate; + "published-online"?: CrossrefDate; + issued?: CrossrefDate; + created?: CrossrefDate; + DOI?: string; + abstract?: string; + type?: string; +} + +interface CrossrefResponse { + message?: CrossrefMessage; +} + +const DOI_HOSTS = new Set(["doi.org", "dx.doi.org", "www.doi.org"]); + +function extractDoi(pathname: string): string | null { + const raw = pathname.replace(/^\/+/, ""); + if (!raw) return null; + return decodeURIComponent(raw); +} + +function formatAuthors(authors?: CrossrefAuthor[]): string | null { + if (!authors || authors.length === 0) return null; + const names = authors + .map((author) => { + if (author.name) return author.name; + const parts = [author.given, author.family].filter(Boolean); + return parts.length > 0 ? parts.join(" ") : null; + }) + .filter((name): name is string => Boolean(name)); + if (names.length === 0) return null; + return names.join(", "); +} + +function formatDate(date?: CrossrefDate): string | null { + const parts = date?.["date-parts"]?.[0]; + if (!parts || parts.length === 0) return null; + const [year, month, day] = parts; + if (!year) return null; + const formatted = [ + String(year), + month ? String(month).padStart(2, "0") : "", + day ? String(day).padStart(2, "0") : "", + ].filter(Boolean); + return formatted.join("-"); +} + +function formatAbstract(abstract?: string): string | null { + if (!abstract) return null; + const normalized = abstract.replace(/<\/?jats:p[^>]*>/g, (match) => (match.startsWith("" : "

")); + const markdown = htmlToBasicMarkdown(normalized); + return markdown.trim().length > 0 ? markdown : null; +} + +export const handleCrossref: SpecialHandler = async (url: string, timeout: number): Promise => { + try { + const parsed = new URL(url); + if (!DOI_HOSTS.has(parsed.hostname.toLowerCase())) return null; + + const doi = extractDoi(parsed.pathname); + if (!doi) return null; + + const fetchedAt = new Date().toISOString(); + const apiUrl = `https://api.crossref.org/works/${encodeURIComponent(doi)}`; + const result = await loadPage(apiUrl, { + timeout, + headers: { + Accept: "application/json", + }, + }); + + if (!result.ok) return null; + + let data: CrossrefResponse; + try { + data = JSON.parse(result.content); + } catch { + return null; + } + + const message = data.message; + if (!message) return null; + + const title = message.title?.[0]?.trim() || "CrossRef Record"; + const authors = formatAuthors(message.author); + const journal = message["container-title"]?.[0] || message["short-container-title"]?.[0]; + const publisher = message.publisher; + const published = + formatDate(message.published) || + formatDate(message["published-print"]) || + formatDate(message["published-online"]) || + formatDate(message.issued) || + formatDate(message.created); + const doiValue = message.DOI || doi; + const abstract = formatAbstract(message.abstract); + const type = message.type?.replace(/-/g, " "); + + let md = `# ${title}\n\n`; + if (authors) md += `**Authors:** ${authors}\n`; + if (journal) md += `**Journal:** ${journal}\n`; + if (publisher) md += `**Publisher:** ${publisher}\n`; + if (published) md += `**Published:** ${published}\n`; + md += `**DOI:** ${doiValue}\n`; + if (type) md += `**Type:** ${type}\n`; + md += "\n---\n\n"; + md += "## Abstract\n\n"; + md += abstract || "No abstract available."; + md += "\n"; + + const output = finalizeOutput(md); + return { + url, + finalUrl: url, + contentType: "text/markdown", + method: "crossref", + content: output.content, + fetchedAt, + truncated: output.truncated, + notes: ["Fetched via CrossRef API"], + }; + } catch {} + + return null; +}; diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/dev-platforms.test.ts b/packages/coding-agent/src/core/tools/web-scrapers/dev-platforms.test.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/dev-platforms.test.ts rename to packages/coding-agent/src/core/tools/web-scrapers/dev-platforms.test.ts diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/devto.ts b/packages/coding-agent/src/core/tools/web-scrapers/devto.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/devto.ts rename to packages/coding-agent/src/core/tools/web-scrapers/devto.ts diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/discogs.ts b/packages/coding-agent/src/core/tools/web-scrapers/discogs.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/discogs.ts rename to packages/coding-agent/src/core/tools/web-scrapers/discogs.ts diff --git a/packages/coding-agent/src/core/tools/web-scrapers/discourse.ts b/packages/coding-agent/src/core/tools/web-scrapers/discourse.ts new file mode 100644 index 000000000..4cbcb5a48 --- /dev/null +++ b/packages/coding-agent/src/core/tools/web-scrapers/discourse.ts @@ -0,0 +1,217 @@ +import type { RenderResult, SpecialHandler } from "./types"; +import { finalizeOutput, htmlToBasicMarkdown, loadPage } from "./types"; + +interface DiscourseUser { + username?: string; + name?: string; +} + +interface DiscoursePost { + id: number; + username?: string; + name?: string; + created_at?: string; + cooked?: string; + raw?: string; + like_count?: number; + post_number?: number; +} + +interface DiscoursePostResponse extends DiscoursePost { + topic_id?: number; +} + +interface DiscourseTopic { + id?: number; + title?: string; + fancy_title?: string; + posts_count?: number; + created_at?: string; + views?: number; + like_count?: number; + tags?: string[]; + category_id?: number; + category_slug?: string; + category?: { id?: number; name?: string; slug?: string }; + excerpt?: string; + details?: { created_by?: DiscourseUser }; + post_stream?: { posts?: DiscoursePost[] }; +} + +const MAX_POSTS = 20; + +function normalizeBasePath(basePath: string): string { + if (!basePath || basePath === "/") return ""; + return basePath.replace(/\/$/, ""); +} + +function parseTopicPath(pathname: string): { basePath: string; topicId: string } | null { + const match = pathname.match(/^(.*?)(?:\/t\/)(?:[^/]+\/)?(\d+)(?:\.json)?(?:\/|$)/); + if (!match) return null; + return { basePath: match[1] ?? "", topicId: match[2] }; +} + +function parsePostPath(pathname: string): { basePath: string; postId: string } | null { + const match = pathname.match(/^(.*?)(?:\/posts\/)(\d+)(?:\.json)?(?:\/|$)/); + if (!match) return null; + return { basePath: match[1] ?? "", postId: match[2] }; +} + +function formatAuthor(user?: DiscourseUser | null): string { + if (!user) return "unknown"; + const name = user.name?.trim(); + const username = user.username?.trim(); + if (name && username && name !== username) return `${name} (@${username})`; + if (username) return `@${username}`; + if (name) return name; + return "unknown"; +} + +function formatIsoDate(value?: string): string { + if (!value) return "unknown"; + const date = new Date(value); + if (Number.isNaN(date.getTime())) return value; + return date.toISOString().split("T")[0]; +} + +function formatCategory(topic: DiscourseTopic): string | null { + const parts: string[] = []; + const name = topic.category?.name ?? topic.category_slug; + if (name) parts.push(name); + const id = topic.category?.id ?? topic.category_id; + if (id != null) parts.push(`#${id}`); + return parts.length ? parts.join(" ") : null; +} + +function formatPostBody(post: DiscoursePost): string { + const raw = post.raw?.trim(); + if (raw) return raw; + const cooked = post.cooked?.trim(); + if (!cooked) return ""; + return htmlToBasicMarkdown(cooked); +} + +function buildTopicUrl(baseUrl: string, topicId: string): string { + const topicUrl = new URL(`${baseUrl}/t/${topicId}.json`); + topicUrl.searchParams.set("include_raw", "1"); + return topicUrl.toString(); +} + +function buildPostUrl(baseUrl: string, postId: string): string { + const postUrl = new URL(`${baseUrl}/posts/${postId}.json`); + postUrl.searchParams.set("include_raw", "1"); + return postUrl.toString(); +} + +/** + * Handle Discourse forum URLs via API + */ +export const handleDiscourse: SpecialHandler = async (url: string, timeout: number): Promise => { + try { + const parsed = new URL(url); + const topicMatch = parseTopicPath(parsed.pathname); + const postMatch = topicMatch ? null : parsePostPath(parsed.pathname); + if (!topicMatch && !postMatch) return null; + + const basePath = normalizeBasePath(topicMatch?.basePath ?? postMatch?.basePath ?? ""); + const baseUrl = `${parsed.origin}${basePath}`; + + let requestedPost: DiscoursePost | null = null; + let topicId = topicMatch?.topicId ?? null; + + if (!topicId && postMatch) { + const postResult = await loadPage(buildPostUrl(baseUrl, postMatch.postId), { timeout }); + if (!postResult.ok) return null; + + let postData: DiscoursePostResponse; + try { + postData = JSON.parse(postResult.content) as DiscoursePostResponse; + } catch { + return null; + } + + if (!postData.topic_id) return null; + topicId = String(postData.topic_id); + requestedPost = postData; + } + + if (!topicId) return null; + + const topicResult = await loadPage(buildTopicUrl(baseUrl, topicId), { timeout }); + if (!topicResult.ok) return null; + + let topic: DiscourseTopic; + try { + topic = JSON.parse(topicResult.content) as DiscourseTopic; + } catch { + return null; + } + + const title = topic.title || topic.fancy_title; + if (!title) return null; + + const fetchedAt = new Date().toISOString(); + + const posts: DiscoursePost[] = [...(topic.post_stream?.posts ?? [])]; + if (requestedPost && !posts.some((post) => post.id === requestedPost?.id)) { + posts.unshift(requestedPost); + } + + let md = `# ${title}\n\n`; + + const metaParts: string[] = []; + if (topic.id != null) metaParts.push(`**Topic ID:** ${topic.id}`); + if (topic.posts_count != null) metaParts.push(`**Posts:** ${topic.posts_count}`); + if (topic.views != null) metaParts.push(`**Views:** ${topic.views}`); + if (topic.like_count != null) metaParts.push(`**Likes:** ${topic.like_count}`); + if (metaParts.length) md += `${metaParts.join(" | ")}\n`; + + const categoryLabel = formatCategory(topic); + if (categoryLabel) md += `**Category:** ${categoryLabel}\n`; + if (topic.tags?.length) md += `**Tags:** ${topic.tags.join(", ")}\n`; + + const createdBy = formatAuthor(topic.details?.created_by ?? null); + if (createdBy !== "unknown" || topic.created_at) { + md += `**Created by:** ${createdBy} - ${formatIsoDate(topic.created_at)}\n`; + } + + md += "\n"; + + const description = topic.excerpt + ? htmlToBasicMarkdown(topic.excerpt) + : posts.length + ? formatPostBody(posts[0]) + : ""; + if (description) { + md += `## Description\n\n${description}\n\n`; + } + + if (posts.length) { + md += "## Posts\n\n"; + for (const post of posts.slice(0, MAX_POSTS)) { + const author = formatAuthor({ name: post.name, username: post.username }); + const date = formatIsoDate(post.created_at); + const likes = post.like_count ?? 0; + const content = formatPostBody(post); + const postLabel = post.post_number != null ? `Post ${post.post_number}` : `Post ${post.id}`; + + md += `### ${postLabel} - ${author} - ${date} - Likes: ${likes}\n\n`; + md += content ? `${content}\n\n---\n\n` : "_No content available._\n\n---\n\n"; + } + } + + const output = finalizeOutput(md); + return { + url, + finalUrl: url, + contentType: "text/markdown", + method: "discourse-api", + content: output.content, + fetchedAt, + truncated: output.truncated, + notes: ["Fetched via Discourse API"], + }; + } catch {} + + return null; +}; diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/dockerhub.ts b/packages/coding-agent/src/core/tools/web-scrapers/dockerhub.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/dockerhub.ts rename to packages/coding-agent/src/core/tools/web-scrapers/dockerhub.ts diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/documentation.test.ts b/packages/coding-agent/src/core/tools/web-scrapers/documentation.test.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/documentation.test.ts rename to packages/coding-agent/src/core/tools/web-scrapers/documentation.test.ts diff --git a/packages/coding-agent/src/core/tools/web-scrapers/fdroid.ts b/packages/coding-agent/src/core/tools/web-scrapers/fdroid.ts new file mode 100644 index 000000000..d38929e9e --- /dev/null +++ b/packages/coding-agent/src/core/tools/web-scrapers/fdroid.ts @@ -0,0 +1,153 @@ +import type { RenderResult, SpecialHandler } from "./types"; +import { finalizeOutput, loadPage } from "./types"; + +type LocalizedText = string | Record; + +type FdroidPackage = { + packageName?: string; + name?: LocalizedText; + summary?: LocalizedText; + description?: LocalizedText; + author?: string | { name?: string; email?: string }; + authorName?: string; + authorEmail?: string; + license?: string; + categories?: string[]; + antiFeatures?: string[]; + sourceCode?: string; + packages?: Array<{ + versionName?: string; + versionCode?: number; + added?: number; + antiFeatures?: string[]; + }>; + suggestedVersionCode?: number; + suggestedVersionName?: string; +}; + +function pickLocalizedText(value?: LocalizedText): string | undefined { + if (!value) return undefined; + if (typeof value === "string") return value; + const preferred = value["en-US"] ?? value.en_US ?? value.en; + if (preferred) return preferred; + const first = Object.values(value).find((entry) => typeof entry === "string"); + return first; +} + +function normalizeAuthor(data: FdroidPackage): string | undefined { + if (data.authorName) return data.authorName; + if (typeof data.author === "string") return data.author; + if (data.author && typeof data.author !== "string" && typeof data.author.name === "string") return data.author.name; + if (data.authorEmail) return data.authorEmail; + return undefined; +} + +function normalizeAuthorEmail(data: FdroidPackage): string | undefined { + if (data.authorEmail) return data.authorEmail; + if (data.author && typeof data.author !== "string" && typeof data.author.email === "string") + return data.author.email; + return undefined; +} + +function collectAntiFeatures(data: FdroidPackage): string[] { + const values = new Set(); + for (const feature of data.antiFeatures ?? []) values.add(feature); + for (const pkg of data.packages ?? []) { + for (const feature of pkg.antiFeatures ?? []) values.add(feature); + } + return Array.from(values); +} + +function resolveSuggestedVersion(data: FdroidPackage): string | undefined { + if (data.suggestedVersionName) return data.suggestedVersionName; + if (data.suggestedVersionCode) { + const match = data.packages?.find((pkg) => pkg.versionCode === data.suggestedVersionCode); + if (match?.versionName) return match.versionName; + } + return data.packages?.[0]?.versionName; +} + +/** + * Handle F-Droid URLs via API + */ +export const handleFdroid: SpecialHandler = async (url: string, timeout: number): Promise => { + try { + const parsed = new URL(url); + if (parsed.hostname !== "f-droid.org" && parsed.hostname !== "www.f-droid.org") return null; + + // Extract package name from /packages/{packageName} or /en/packages/{packageName} + const match = parsed.pathname.match(/^\/(?:en\/)?packages\/([^/]+)/); + if (!match) return null; + + const packageName = decodeURIComponent(match[1]); + const fetchedAt = new Date().toISOString(); + const apiUrl = `https://f-droid.org/api/v1/packages/${encodeURIComponent(packageName)}`; + + const result = await loadPage(apiUrl, { + timeout, + headers: { Accept: "application/json" }, + }); + + if (!result.ok) return null; + + let data: FdroidPackage; + try { + data = JSON.parse(result.content) as FdroidPackage; + } catch { + return null; + } + + const displayName = pickLocalizedText(data.name) ?? packageName; + const summary = pickLocalizedText(data.summary); + const description = pickLocalizedText(data.description); + const author = normalizeAuthor(data); + const authorEmail = normalizeAuthorEmail(data); + const antiFeatures = collectAntiFeatures(data); + const latestVersion = resolveSuggestedVersion(data); + + let md = `# ${displayName}\n\n`; + if (summary) md += `${summary}\n\n`; + + md += `**Package:** ${packageName}`; + if (latestVersion) md += ` · **Latest:** ${latestVersion}`; + if (data.license) md += ` · **License:** ${data.license}`; + md += "\n"; + + if (author) { + md += `**Author:** ${author}`; + if (authorEmail && authorEmail !== author) md += ` <${authorEmail}>`; + md += "\n"; + } + + if (data.sourceCode) md += `**Source Code:** ${data.sourceCode}\n`; + if (data.categories?.length) md += `**Categories:** ${data.categories.join(", ")}\n`; + if (antiFeatures.length) md += `**Anti-Features:** ${antiFeatures.join(", ")}\n`; + + if (description) { + md += `\n## Description\n\n${description}\n`; + } + + if (data.packages?.length) { + md += "\n## Version History\n\n"; + for (const version of data.packages.slice(0, 10)) { + const label = version.versionName ?? "unknown"; + const code = version.versionCode ? ` (${version.versionCode})` : ""; + md += `- ${label}${code}\n`; + } + } + + const output = finalizeOutput(md); + return { + url, + finalUrl: url, + contentType: "text/markdown", + method: "fdroid", + content: output.content, + fetchedAt, + truncated: output.truncated, + notes: ["Fetched via F-Droid API"], + }; + } catch {} + + return null; +}; diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/finance-media.test.ts b/packages/coding-agent/src/core/tools/web-scrapers/finance-media.test.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/finance-media.test.ts rename to packages/coding-agent/src/core/tools/web-scrapers/finance-media.test.ts diff --git a/packages/coding-agent/src/core/tools/web-scrapers/firefox-addons.ts b/packages/coding-agent/src/core/tools/web-scrapers/firefox-addons.ts new file mode 100644 index 000000000..490cd95fe --- /dev/null +++ b/packages/coding-agent/src/core/tools/web-scrapers/firefox-addons.ts @@ -0,0 +1,213 @@ +import type { RenderResult, SpecialHandler } from "./types"; +import { finalizeOutput, formatCount, htmlToBasicMarkdown, loadPage } from "./types"; + +type LocalizedText = string | Record | null | undefined; + +type AddonFile = { + permissions?: string[]; + host_permissions?: string[]; + optional_permissions?: string[]; + optional_host_permissions?: string[]; +}; + +type AddonLicense = { + name?: LocalizedText; + slug?: string; + url?: string; +}; + +type AddonVersion = { + version?: string; + license?: AddonLicense; + file?: AddonFile; +}; + +type AddonHomepage = { + url?: LocalizedText; + outgoing?: LocalizedText; +}; + +type AddonData = { + name?: LocalizedText; + summary?: LocalizedText; + description?: LocalizedText; + default_locale?: string; + authors?: Array<{ name?: string | null }>; + average_daily_users?: number; + weekly_downloads?: number; + ratings?: { average?: number; count?: number }; + current_version?: AddonVersion; + categories?: string[] | Record; + homepage?: AddonHomepage; + url?: string; +}; + +function getLocalizedText(value: LocalizedText, defaultLocale?: string): string | undefined { + if (!value) return undefined; + if (typeof value === "string") return value; + + const localized = value as Record; + if (defaultLocale && localized[defaultLocale]) return localized[defaultLocale] ?? undefined; + if (localized["en-US"]) return localized["en-US"] ?? undefined; + + for (const entry of Object.values(localized)) { + if (entry) return entry; + } + + return undefined; +} + +function normalizeCategories(categories?: string[] | Record): string[] { + if (!categories) return []; + if (Array.isArray(categories)) return categories.filter(Boolean); + + const values: string[] = []; + for (const list of Object.values(categories)) { + if (Array.isArray(list)) { + for (const item of list) { + if (item) values.push(item); + } + } + } + + const seen = new Set(); + return values.filter((item) => { + if (seen.has(item)) return false; + seen.add(item); + return true; + }); +} + +function collectPermissions(file?: AddonFile): string[] { + if (!file) return []; + const permissions: string[] = []; + const seen = new Set(); + + const add = (items?: string[]) => { + for (const item of items ?? []) { + if (!item || seen.has(item)) continue; + seen.add(item); + permissions.push(item); + } + }; + + add(file.permissions); + add(file.host_permissions); + add(file.optional_permissions); + add(file.optional_host_permissions); + + return permissions; +} + +export const handleFirefoxAddons: SpecialHandler = async ( + url: string, + timeout: number, +): Promise => { + try { + const parsed = new URL(url); + if (parsed.hostname !== "addons.mozilla.org") return null; + + const segments = parsed.pathname.split("/").filter(Boolean); + const addonIndex = segments.indexOf("addon"); + if (addonIndex === -1) return null; + + const slug = segments[addonIndex + 1] ? decodeURIComponent(segments[addonIndex + 1]) : ""; + if (!slug) return null; + + const apiUrl = `https://addons.mozilla.org/api/v5/addons/addon/${encodeURIComponent(slug)}/`; + const result = await loadPage(apiUrl, { timeout, headers: { Accept: "application/json" } }); + if (!result.ok) return null; + + let data: AddonData; + try { + data = JSON.parse(result.content) as AddonData; + } catch { + return null; + } + + const fetchedAt = new Date().toISOString(); + const defaultLocale = data.default_locale || "en-US"; + + const name = getLocalizedText(data.name, defaultLocale) ?? slug; + const summary = getLocalizedText(data.summary, defaultLocale); + const descriptionRaw = getLocalizedText(data.description, defaultLocale); + const description = descriptionRaw ? htmlToBasicMarkdown(descriptionRaw) : undefined; + + const authors = (data.authors ?? []) + .map((author) => author.name ?? "") + .map((author) => author.trim()) + .filter(Boolean); + + const ratingAverage = data.ratings?.average; + const ratingCount = data.ratings?.count; + const users = data.average_daily_users ?? data.weekly_downloads; + const version = data.current_version?.version; + const categories = normalizeCategories(data.categories); + + const licenseName = + getLocalizedText(data.current_version?.license?.name, defaultLocale) ?? data.current_version?.license?.slug; + const licenseUrl = data.current_version?.license?.url; + + const homepage = + getLocalizedText(data.homepage?.url, defaultLocale) ?? + getLocalizedText(data.homepage?.outgoing, defaultLocale); + + const permissions = collectPermissions(data.current_version?.file); + + let md = `# ${name}\n\n`; + if (summary) md += `${summary}\n\n`; + + if (authors.length > 0) { + md += `**Author${authors.length > 1 ? "s" : ""}:** ${authors.join(", ")}\n`; + } + + if (ratingAverage !== undefined) { + md += `**Rating:** ${ratingAverage.toFixed(2)}`; + if (ratingCount !== undefined) md += ` (${formatCount(ratingCount)} reviews)`; + md += "\n"; + } + + if (users !== undefined) md += `**Users:** ${formatCount(users)}\n`; + if (version) md += `**Version:** ${version}\n`; + if (categories.length > 0) md += `**Categories:** ${categories.join(", ")}\n`; + + if (licenseName && licenseUrl) { + md += `**License:** [${licenseName}](${licenseUrl})\n`; + } else if (licenseName) { + md += `**License:** ${licenseName}\n`; + } else if (licenseUrl) { + md += `**License:** ${licenseUrl}\n`; + } + + if (homepage) md += `**Homepage:** ${homepage}\n`; + + if (description) { + md += `\n## Description\n\n${description}\n`; + } + + if (permissions.length > 0) { + const preview = permissions.slice(0, 40); + md += `\n## Permissions (${permissions.length})\n\n`; + for (const permission of preview) { + md += `- ${permission}\n`; + } + if (permissions.length > preview.length) { + md += `\n*...and ${permissions.length - preview.length} more*\n`; + } + } + + const output = finalizeOutput(md); + return { + url, + finalUrl: data.url ?? result.finalUrl ?? url, + contentType: "text/markdown", + method: "firefox-addons", + content: output.content, + fetchedAt, + truncated: output.truncated, + notes: ["Fetched via Firefox Add-ons API"], + }; + } catch {} + + return null; +}; diff --git a/packages/coding-agent/src/core/tools/web-scrapers/flathub.ts b/packages/coding-agent/src/core/tools/web-scrapers/flathub.ts new file mode 100644 index 000000000..1230822a2 --- /dev/null +++ b/packages/coding-agent/src/core/tools/web-scrapers/flathub.ts @@ -0,0 +1,235 @@ +import type { RenderResult, SpecialHandler } from "./types"; +import { finalizeOutput, formatCount, htmlToBasicMarkdown, loadPage } from "./types"; + +interface FlathubScreenshotSize { + src?: string; + width?: string; + height?: string; + scale?: string; +} + +interface FlathubScreenshot { + caption?: string | null; + sizes?: FlathubScreenshotSize[]; +} + +interface FlathubRelease { + version?: string; + timestamp?: string; + description?: string | null; + url?: string | null; + type?: string | null; +} + +interface FlathubAppStream { + id?: string; + name?: string; + summary?: string; + description?: string; + developer_name?: string; + categories?: string[]; + screenshots?: FlathubScreenshot[]; + releases?: FlathubRelease[]; + metadata?: Record; + installs?: number | string; + permissions?: unknown; +} + +function extractAppId(pathname: string): string | null { + const detailsMatch = pathname.match(/^\/apps\/details\/([^/]+)\/?$/); + if (detailsMatch) return decodeURIComponent(detailsMatch[1]); + + const appMatch = pathname.match(/^\/apps\/([^/]+)\/?$/); + if (appMatch) return decodeURIComponent(appMatch[1]); + + return null; +} + +function parseNumber(value: unknown): number | null { + if (typeof value === "number" && Number.isFinite(value)) return value; + if (typeof value === "string") { + const cleaned = value.replace(/[^0-9.]/g, ""); + if (!cleaned) return null; + const parsed = Number(cleaned); + if (!Number.isNaN(parsed)) return parsed; + } + return null; +} + +function normalizeStringList(value: unknown): string[] { + if (Array.isArray(value)) { + return value.filter((item): item is string => typeof item === "string" && item.trim().length > 0); + } + if (typeof value === "string") { + return value + .split(/[,;\n]+/) + .map((item) => item.trim()) + .filter(Boolean); + } + return []; +} + +function extractInstalls(app: FlathubAppStream): number | null { + const direct = parseNumber(app.installs); + if (direct !== null) return direct; + + if (!app.metadata) return null; + for (const [key, value] of Object.entries(app.metadata)) { + if (!key.toLowerCase().includes("install")) continue; + const parsed = parseNumber(value); + if (parsed !== null) return parsed; + } + + return null; +} + +function extractPermissions(app: FlathubAppStream): string[] { + const permissions: string[] = []; + permissions.push(...normalizeStringList(app.permissions)); + + if (app.metadata) { + for (const [key, value] of Object.entries(app.metadata)) { + if (!key.toLowerCase().includes("permission")) continue; + const list = normalizeStringList(value); + if (list.length) { + permissions.push(...list); + continue; + } + if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") { + permissions.push(`${key}: ${String(value)}`); + } + } + } + + return Array.from(new Set(permissions)); +} + +function screenshotArea(size?: FlathubScreenshotSize): number { + if (!size) return 0; + const width = Number(size.width); + const height = Number(size.height); + if (!Number.isFinite(width) || !Number.isFinite(height)) return 0; + return width * height; +} + +function bestScreenshotUrl(sizes?: FlathubScreenshotSize[]): string | null { + if (!sizes || sizes.length === 0) return null; + + let best = sizes[0]; + let bestArea = screenshotArea(best); + + for (const size of sizes) { + const area = screenshotArea(size); + if (area > bestArea) { + best = size; + bestArea = area; + } + } + + return best.src ?? sizes[0].src ?? null; +} + +function formatReleaseDate(timestamp?: string | null): string | null { + if (!timestamp) return null; + const seconds = Number(timestamp); + if (!Number.isFinite(seconds)) return null; + const date = new Date(seconds * 1000); + if (Number.isNaN(date.getTime())) return null; + return date.toISOString().split("T")[0] ?? null; +} + +export const handleFlathub: SpecialHandler = async (url: string, timeout: number): Promise => { + try { + const parsed = new URL(url); + if (parsed.hostname !== "flathub.org" && parsed.hostname !== "www.flathub.org") return null; + + const appId = extractAppId(parsed.pathname); + if (!appId) return null; + + const apiUrl = `https://flathub.org/api/v2/appstream/${encodeURIComponent(appId)}`; + const result = await loadPage(apiUrl, { timeout, headers: { Accept: "application/json" } }); + if (!result.ok) return null; + + let app: FlathubAppStream; + try { + app = JSON.parse(result.content) as FlathubAppStream; + } catch { + return null; + } + + const fetchedAt = new Date().toISOString(); + const name = app.name ?? app.id ?? appId; + + let md = `# ${name}\n\n`; + if (app.summary) md += `${app.summary}\n\n`; + + md += "## Metadata\n\n"; + md += `**App ID:** ${app.id ?? appId}\n`; + if (app.developer_name) md += `**Developer:** ${app.developer_name}\n`; + + const installs = extractInstalls(app); + if (installs !== null) md += `**Installs:** ${formatCount(installs)}\n`; + + if (app.categories?.length) { + md += "\n## Categories\n\n"; + for (const category of app.categories) { + md += `- ${category}\n`; + } + } + + if (app.description) { + const description = htmlToBasicMarkdown(app.description); + if (description) md += `\n## Description\n\n${description}\n`; + } + + const permissions = extractPermissions(app); + if (permissions.length) { + md += "\n## Permissions\n\n"; + for (const permission of permissions) { + md += `- ${permission}\n`; + } + } + + if (app.screenshots?.length) { + md += "\n## Screenshots\n\n"; + for (const screenshot of app.screenshots.slice(0, 5)) { + const screenshotUrl = bestScreenshotUrl(screenshot.sizes); + if (!screenshotUrl) continue; + const caption = screenshot.caption ? ` - ${screenshot.caption}` : ""; + md += `- ${screenshotUrl}${caption}\n`; + } + } + + if (app.releases?.length) { + md += "\n## Releases\n\n"; + for (const release of app.releases.slice(0, 5)) { + const version = release.version ?? "unknown"; + let line = `- **${version}**`; + const date = formatReleaseDate(release.timestamp); + if (date) line += ` (${date})`; + if (release.type) line += ` · ${release.type}`; + if (release.url) line += ` · ${release.url}`; + md += `${line}\n`; + + if (release.description) { + const releaseDesc = htmlToBasicMarkdown(release.description).replace(/\n+/g, " ").trim(); + if (releaseDesc) md += ` - ${releaseDesc}\n`; + } + } + } + + const output = finalizeOutput(md); + return { + url, + finalUrl: result.finalUrl, + contentType: "text/markdown", + method: "flathub-appstream", + content: output.content, + fetchedAt, + truncated: output.truncated, + notes: ["Fetched via Flathub Appstream API"], + }; + } catch {} + + return null; +}; diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/git-hosting.test.ts b/packages/coding-agent/src/core/tools/web-scrapers/git-hosting.test.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/git-hosting.test.ts rename to packages/coding-agent/src/core/tools/web-scrapers/git-hosting.test.ts diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/github-gist.ts b/packages/coding-agent/src/core/tools/web-scrapers/github-gist.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/github-gist.ts rename to packages/coding-agent/src/core/tools/web-scrapers/github-gist.ts diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/github.ts b/packages/coding-agent/src/core/tools/web-scrapers/github.ts similarity index 97% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/github.ts rename to packages/coding-agent/src/core/tools/web-scrapers/github.ts index 67ae300df..196f11995 100644 --- a/packages/coding-agent/src/core/tools/web-fetch-handlers/github.ts +++ b/packages/coding-agent/src/core/tools/web-scrapers/github.ts @@ -64,7 +64,7 @@ function parseGitHubUrl(url: string): GitHubUrl | null { * Convert GitHub blob URL to raw URL */ function toRawGitHubUrl(gh: GitHubUrl): string { - return `https://raw.githubusercontent.com/${gh.owner}/${gh.repo}/refs/heads/${gh.ref}/${gh.path}`; + return `https://raw.githubusercontent.com/${gh.owner}/${gh.repo}/${gh.ref}/${gh.path}`; } /** @@ -72,8 +72,7 @@ function toRawGitHubUrl(gh: GitHubUrl): string { */ export async function fetchGitHubApi(endpoint: string, timeout: number): Promise<{ data: unknown; ok: boolean }> { try { - const controller = new AbortController(); - const timeoutId = setTimeout(() => controller.abort(), timeout * 1000); + const timeoutSignal = AbortSignal.timeout(timeout * 1000); const headers: Record = { Accept: "application/vnd.github.v3+json", @@ -87,12 +86,10 @@ export async function fetchGitHubApi(endpoint: string, timeout: number): Promise } const response = await fetch(`https://api.github.com${endpoint}`, { - signal: controller.signal, + signal: timeoutSignal, headers, }); - clearTimeout(timeoutId); - if (!response.ok) { return { data: null, ok: false }; } @@ -240,7 +237,7 @@ async function renderGitHubTree(gh: GitHubUrl, timeout: number): Promise<{ conte const readmeFile = items.find((item) => item.type === "file" && /^readme\.md$/i.test(item.name)); if (readmeFile) { const readmePath = dirPath ? `${dirPath}/${readmeFile.name}` : readmeFile.name; - const rawUrl = `https://raw.githubusercontent.com/${gh.owner}/${gh.repo}/refs/heads/${ref}/${readmePath}`; + const rawUrl = `https://raw.githubusercontent.com/${gh.owner}/${gh.repo}/${ref}/${readmePath}`; const readmeResult = await loadPage(rawUrl, { timeout }); if (readmeResult.ok) { md += `---\n\n## README\n\n${readmeResult.content}`; diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/gitlab.ts b/packages/coding-agent/src/core/tools/web-scrapers/gitlab.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/gitlab.ts rename to packages/coding-agent/src/core/tools/web-scrapers/gitlab.ts diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/go-pkg.ts b/packages/coding-agent/src/core/tools/web-scrapers/go-pkg.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/go-pkg.ts rename to packages/coding-agent/src/core/tools/web-scrapers/go-pkg.ts diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/hackage.ts b/packages/coding-agent/src/core/tools/web-scrapers/hackage.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/hackage.ts rename to packages/coding-agent/src/core/tools/web-scrapers/hackage.ts diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/hackernews.ts b/packages/coding-agent/src/core/tools/web-scrapers/hackernews.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/hackernews.ts rename to packages/coding-agent/src/core/tools/web-scrapers/hackernews.ts diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/hex.ts b/packages/coding-agent/src/core/tools/web-scrapers/hex.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/hex.ts rename to packages/coding-agent/src/core/tools/web-scrapers/hex.ts diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/huggingface.ts b/packages/coding-agent/src/core/tools/web-scrapers/huggingface.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/huggingface.ts rename to packages/coding-agent/src/core/tools/web-scrapers/huggingface.ts diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/iacr.ts b/packages/coding-agent/src/core/tools/web-scrapers/iacr.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/iacr.ts rename to packages/coding-agent/src/core/tools/web-scrapers/iacr.ts diff --git a/packages/coding-agent/src/core/tools/web-scrapers/index.ts b/packages/coding-agent/src/core/tools/web-scrapers/index.ts new file mode 100644 index 000000000..79a1ec976 --- /dev/null +++ b/packages/coding-agent/src/core/tools/web-scrapers/index.ts @@ -0,0 +1,250 @@ +/** + * Web Fetch Special Handlers Index + * + * Exports all special handlers for site-specific content extraction. + */ + +import { handleArtifactHub } from "./artifacthub"; +import { handleArxiv } from "./arxiv"; +import { handleAur } from "./aur"; +import { handleBiorxiv } from "./biorxiv"; +import { handleBluesky } from "./bluesky"; +import { handleBrew } from "./brew"; +import { handleCheatSh } from "./cheatsh"; +import { handleChocolatey } from "./chocolatey"; +import { handleChooseALicense } from "./choosealicense"; +import { handleCisaKev } from "./cisa-kev"; +import { handleClojars } from "./clojars"; +import { handleCoinGecko } from "./coingecko"; +import { handleCratesIo } from "./crates-io"; +import { handleCrossref } from "./crossref"; +import { handleDevTo } from "./devto"; +import { handleDiscogs } from "./discogs"; +import { handleDiscourse } from "./discourse"; +import { handleDockerHub } from "./dockerhub"; +import { handleFdroid } from "./fdroid"; +import { handleFirefoxAddons } from "./firefox-addons"; +import { handleFlathub } from "./flathub"; +import { fetchGitHubApi, handleGitHub } from "./github"; +import { handleGitHubGist } from "./github-gist"; +import { handleGitLab } from "./gitlab"; +import { handleGoPkg } from "./go-pkg"; +import { handleHackage } from "./hackage"; +import { handleHackerNews } from "./hackernews"; +import { handleHex } from "./hex"; +import { handleHuggingFace } from "./huggingface"; +import { handleIacr } from "./iacr"; +import { handleJetBrainsMarketplace } from "./jetbrains-marketplace"; +import { handleLemmy } from "./lemmy"; +import { handleLobsters } from "./lobsters"; +import { handleMastodon } from "./mastodon"; +import { handleMaven } from "./maven"; +import { handleMDN } from "./mdn"; +import { handleMetaCPAN } from "./metacpan"; +import { handleMusicBrainz } from "./musicbrainz"; +import { handleNpm } from "./npm"; +import { handleNuGet } from "./nuget"; +import { handleNvd } from "./nvd"; +import { handleOllama } from "./ollama"; +import { handleOpenVsx } from "./open-vsx"; +import { handleOpenCorporates } from "./opencorporates"; +import { handleOpenLibrary } from "./openlibrary"; +import { handleOrcid } from "./orcid"; +import { handleOsv } from "./osv"; +import { handlePackagist } from "./packagist"; +import { handlePubDev } from "./pub-dev"; +import { handlePubMed } from "./pubmed"; +import { handlePyPI } from "./pypi"; +import { handleRawg } from "./rawg"; +import { handleReadTheDocs } from "./readthedocs"; +import { handleReddit } from "./reddit"; +import { handleRepology } from "./repology"; +import { handleRfc } from "./rfc"; +import { handleRubyGems } from "./rubygems"; +import { handleSearchcode } from "./searchcode"; +import { handleSecEdgar } from "./sec-edgar"; +import { handleSemanticScholar } from "./semantic-scholar"; +import { handleSnapcraft } from "./snapcraft"; +import { handleSourcegraph } from "./sourcegraph"; +import { handleSpdx } from "./spdx"; +import { handleSpotify } from "./spotify"; +import { handleStackOverflow } from "./stackoverflow"; +import { handleTerraform } from "./terraform"; +import { handleTldr } from "./tldr"; +import { handleTwitter } from "./twitter"; +import type { SpecialHandler } from "./types"; +import { handleVimeo } from "./vimeo"; +import { handleVscodeMarketplace } from "./vscode-marketplace"; +import { handleW3c } from "./w3c"; +import { handleWikidata } from "./wikidata"; +import { handleWikipedia } from "./wikipedia"; +import { handleYouTube } from "./youtube"; + +export type { RenderResult, SpecialHandler } from "./types"; + +export { + fetchGitHubApi, + handleArtifactHub, + handleArxiv, + handleAur, + handleBiorxiv, + handleBluesky, + handleBrew, + handleCheatSh, + handleCisaKev, + handleChocolatey, + handleClojars, + handleChooseALicense, + handleCoinGecko, + handleCratesIo, + handleCrossref, + handleDevTo, + handleDiscogs, + handleDiscourse, + handleDockerHub, + handleFdroid, + handleFlathub, + handleFirefoxAddons, + handleGitHub, + handleGitHubGist, + handleGitLab, + handleGoPkg, + handleHackage, + handleHackerNews, + handleHex, + handleHuggingFace, + handleIacr, + handleJetBrainsMarketplace, + handleLemmy, + handleLobsters, + handleMastodon, + handleMaven, + handleMDN, + handleMetaCPAN, + handleMusicBrainz, + handleNpm, + handleNuGet, + handleNvd, + handleOllama, + handleOpenCorporates, + handleOpenLibrary, + handleOrcid, + handleOpenVsx, + handleOsv, + handlePackagist, + handlePubDev, + handlePubMed, + handlePyPI, + handleRawg, + handleReadTheDocs, + handleReddit, + handleRepology, + handleRfc, + handleRubyGems, + handleSecEdgar, + handleSearchcode, + handleSemanticScholar, + handleSnapcraft, + handleSourcegraph, + handleSpotify, + handleSpdx, + handleStackOverflow, + handleTerraform, + handleTldr, + handleTwitter, + handleVimeo, + handleVscodeMarketplace, + handleW3c, + handleWikidata, + handleWikipedia, + handleYouTube, +}; + +export const specialHandlers: SpecialHandler[] = [ + // Git hosting + handleGitHubGist, + handleGitHub, + handleGitLab, + // Video/Media + handleYouTube, + handleVimeo, + handleSpotify, + handleDiscogs, + handleMusicBrainz, + // Games + handleRawg, + // Social/News + handleTwitter, + handleBluesky, + handleMastodon, + handleLemmy, + handleHackerNews, + handleLobsters, + handleReddit, + handleDiscourse, + // Developer content + handleStackOverflow, + handleDevTo, + handleMDN, + handleReadTheDocs, + handleSearchcode, + handleSourcegraph, + handleTldr, + handleCheatSh, + // Package registries + handleNpm, + handleFirefoxAddons, + handleVscodeMarketplace, + handleNuGet, + handleChocolatey, + handleClojars, + handleBrew, + handlePyPI, + handleCratesIo, + handleDockerHub, + handleFdroid, + handleFlathub, + handleGoPkg, + handleHex, + handlePackagist, + handlePubDev, + handleMaven, + handleJetBrainsMarketplace, + handleOpenVsx, + handleArtifactHub, + handleRubyGems, + handleTerraform, + handleAur, + handleHackage, + handleMetaCPAN, + handleRepology, + handleSnapcraft, + // ML/AI + handleHuggingFace, + handleOllama, + // Academic + handleArxiv, + handleBiorxiv, + handleCrossref, + handleIacr, + handleOrcid, + handleSemanticScholar, + handlePubMed, + handleRfc, + // Security + handleCisaKev, + handleNvd, + handleOsv, + // Crypto + handleCoinGecko, + // Business + handleOpenCorporates, + handleSecEdgar, + // Reference + handleOpenLibrary, + handleChooseALicense, + handleW3c, + handleSpdx, + handleWikidata, + handleWikipedia, +]; diff --git a/packages/coding-agent/src/core/tools/web-scrapers/jetbrains-marketplace.ts b/packages/coding-agent/src/core/tools/web-scrapers/jetbrains-marketplace.ts new file mode 100644 index 000000000..9248e7907 --- /dev/null +++ b/packages/coding-agent/src/core/tools/web-scrapers/jetbrains-marketplace.ts @@ -0,0 +1,168 @@ +import type { RenderResult, SpecialHandler } from "./types"; +import { finalizeOutput, formatCount, htmlToBasicMarkdown, loadPage } from "./types"; + +interface PluginVendor { + name?: string; + publicName?: string; + url?: string; +} + +interface PluginTag { + name?: string; +} + +type PluginRating = + | number + | { + rating?: number; + value?: number; + score?: number; + votes?: number; + totalVotes?: number; + count?: number; + }; + +interface PluginData { + id?: number; + name?: string; + description?: string; + preview?: string; + vendor?: PluginVendor; + rating?: PluginRating; + ratingCount?: number; + downloads?: number; + tags?: PluginTag[]; + urls?: { + url?: string; + docUrl?: string; + sourceCodeUrl?: string; + bugtrackerUrl?: string; + }; +} + +interface UpdateData { + version?: string; + since?: string; + until?: string; + sinceUntil?: string; + channel?: string; + downloads?: number; + compatibleVersions?: Record; + cdate?: string | number; +} + +const MARKETPLACE_HOSTS = new Set(["plugins.jetbrains.com"]); + +function extractRating(plugin: PluginData): { value: number | null; votes: number | null } { + const rating = plugin.rating; + if (typeof rating === "number" && Number.isFinite(rating)) { + return { value: rating, votes: plugin.ratingCount ?? null }; + } + if (rating && typeof rating === "object") { + const value = rating.rating ?? rating.value ?? rating.score ?? null; + const votes = rating.votes ?? rating.totalVotes ?? rating.count ?? plugin.ratingCount ?? null; + return { value: typeof value === "number" ? value : null, votes: typeof votes === "number" ? votes : null }; + } + return { value: null, votes: plugin.ratingCount ?? null }; +} + +function formatBuildCompatibility(update: UpdateData): string | null { + if (update.sinceUntil) return update.sinceUntil; + if (update.since && update.until) return `${update.since} - ${update.until}`; + if (update.since) return `${update.since}+`; + return null; +} + +export const handleJetBrainsMarketplace: SpecialHandler = async ( + url: string, + timeout: number, +): Promise => { + try { + const parsed = new URL(url); + if (!MARKETPLACE_HOSTS.has(parsed.hostname)) return null; + + const match = parsed.pathname.match(/^\/plugin\/(\d+)(?:-[^/]+)?(?:\/|$)/); + if (!match) return null; + + const pluginId = match[1]; + const fetchedAt = new Date().toISOString(); + + const pluginUrl = `https://plugins.jetbrains.com/api/plugins/${pluginId}`; + const updatesUrl = `https://plugins.jetbrains.com/api/plugins/${pluginId}/updates?size=1`; + + const [pluginResult, updatesResult] = await Promise.all([ + loadPage(pluginUrl, { timeout }), + loadPage(updatesUrl, { timeout }), + ]); + + if (!pluginResult.ok || !updatesResult.ok) return null; + + let plugin: PluginData; + let updates: UpdateData[]; + + try { + plugin = JSON.parse(pluginResult.content) as PluginData; + updates = JSON.parse(updatesResult.content) as UpdateData[]; + } catch { + return null; + } + + const update = updates[0]; + if (!plugin?.name) return null; + + const vendorName = plugin.vendor?.name ?? plugin.vendor?.publicName; + const descriptionSource = plugin.description ?? plugin.preview ?? ""; + const description = descriptionSource ? htmlToBasicMarkdown(descriptionSource) : ""; + const tags = (plugin.tags ?? []).map((tag) => tag.name).filter(Boolean) as string[]; + const rating = extractRating(plugin); + const buildCompatibility = update ? formatBuildCompatibility(update) : null; + + let md = `# ${plugin.name}\n\n`; + if (description) md += `${description}\n\n`; + + md += `**Plugin ID:** ${pluginId}\n`; + if (vendorName) md += `**Vendor:** ${vendorName}\n`; + if (plugin.downloads !== undefined) { + md += `**Downloads:** ${formatCount(plugin.downloads)}\n`; + } + if (rating.value !== null) { + md += `**Rating:** ${rating.value.toFixed(2)}`; + if (rating.votes !== null) md += ` (${formatCount(rating.votes)} votes)`; + md += "\n"; + } + if (tags.length) md += `**Tags:** ${tags.join(", ")}\n`; + + if (update) { + md += "\n## Latest Release\n\n"; + if (update.version) md += `**Version:** ${update.version}\n`; + if (update.channel) md += `**Channel:** ${update.channel}\n`; + if (buildCompatibility) md += `**Build Compatibility:** ${buildCompatibility}\n`; + if (update.downloads !== undefined) { + md += `**Release Downloads:** ${formatCount(update.downloads)}\n`; + } + } + + const compatibility = update?.compatibleVersions ?? {}; + const compatibilityEntries = Object.entries(compatibility).sort(([a], [b]) => a.localeCompare(b)); + if (compatibilityEntries.length) { + md += "\n## IDE Compatibility\n\n"; + for (const [product, version] of compatibilityEntries) { + md += `- ${product}: ${version}\n`; + } + } + + const output = finalizeOutput(md); + return { + url, + finalUrl: url, + contentType: "text/markdown", + method: "jetbrains-marketplace", + content: output.content, + fetchedAt, + truncated: output.truncated, + notes: ["Fetched via JetBrains Marketplace API"], + }; + } catch {} + + return null; +}; diff --git a/packages/coding-agent/src/core/tools/web-scrapers/lemmy.ts b/packages/coding-agent/src/core/tools/web-scrapers/lemmy.ts new file mode 100644 index 000000000..456ca43b1 --- /dev/null +++ b/packages/coding-agent/src/core/tools/web-scrapers/lemmy.ts @@ -0,0 +1,216 @@ +import type { RenderResult, SpecialHandler } from "./types"; +import { finalizeOutput, loadPage } from "./types"; + +interface LemmyCreator { + name: string; + actor_id?: string; +} + +interface LemmyCommunity { + name: string; + actor_id?: string; +} + +interface LemmyCounts { + score: number; + comments?: number; +} + +interface LemmyPost { + id: number; + name: string; + body?: string; + url?: string; +} + +interface LemmyPostView { + post: LemmyPost; + creator: LemmyCreator; + community: LemmyCommunity; + counts: LemmyCounts; +} + +interface LemmyPostResponse { + post_view?: LemmyPostView; +} + +interface LemmyComment { + id: number; + content?: string; + path?: string; + parent_id?: number | null; + post_id?: number; +} + +interface LemmyCommentView { + comment: LemmyComment; + creator: LemmyCreator; + counts: LemmyCounts; +} + +interface LemmyCommentListResponse { + comments?: LemmyCommentView[]; +} + +interface LemmyCommentResponse { + comment_view?: LemmyCommentView; +} + +function parseJson(content: string): T | null { + try { + return JSON.parse(content) as T; + } catch { + return null; + } +} + +function formatCommunity(community: LemmyCommunity): string { + if (community.actor_id) { + try { + const host = new URL(community.actor_id).hostname; + return `!${community.name}@${host}`; + } catch {} + } + return `!${community.name}`; +} + +function formatAuthor(creator: LemmyCreator): string { + if (creator.actor_id) { + try { + const host = new URL(creator.actor_id).hostname; + return `@${creator.name}@${host}`; + } catch {} + } + return creator.name; +} + +function indentBlock(text: string, indent: string): string { + return text + .split("\n") + .map((line) => `${indent}${line}`) + .join("\n"); +} + +function renderComments(comments: LemmyCommentView[]): string { + const childrenByParent = new Map(); + + const commentIds = new Set(comments.map((view) => view.comment.id)); + + for (const commentView of comments) { + const parentId = commentView.comment.parent_id; + const resolvedParent = parentId && commentIds.has(parentId) ? parentId : 0; + const list = childrenByParent.get(resolvedParent); + if (list) { + list.push(commentView); + } else { + childrenByParent.set(resolvedParent, [commentView]); + } + } + + const renderThread = (parentId: number, depth: number): string => { + const items = childrenByParent.get(parentId) ?? []; + let output = ""; + + for (const view of items) { + const author = view.creator?.name ? formatAuthor(view.creator) : "unknown"; + const score = view.counts?.score ?? 0; + const content = (view.comment.content ?? "").trim(); + const indent = " ".repeat(depth); + + output += `${indent}- **${author}** · ${score} points\n`; + if (content) { + output += `${indentBlock(content, `${indent} `)}\n`; + } + + output += renderThread(view.comment.id, depth + 1); + output += "\n"; + } + + return output; + }; + + return renderThread(0, 0).trim(); +} + +export const handleLemmy: SpecialHandler = async (url: string, timeout: number): Promise => { + try { + const parsed = new URL(url); + const match = parsed.pathname.match(/^\/(post|comment)\/(\d+)/); + if (!match) return null; + + const kind = match[1]; + const id = Number.parseInt(match[2], 10); + if (!Number.isFinite(id)) return null; + + const baseUrl = parsed.origin; + const fetchedAt = new Date().toISOString(); + + let postId = id; + if (kind === "comment") { + const commentUrl = `${baseUrl}/api/v3/comment?id=${id}`; + const commentResult = await loadPage(commentUrl, { timeout }); + if (!commentResult.ok) return null; + + const commentData = parseJson(commentResult.content); + const commentView = commentData?.comment_view; + const commentPostId = commentView?.comment?.post_id; + if (!commentPostId) return null; + postId = commentPostId; + } + + const postUrl = `${baseUrl}/api/v3/post?id=${postId}`; + const commentsUrl = `${baseUrl}/api/v3/comment/list?post_id=${postId}`; + + const [postResult, commentsResult] = await Promise.all([ + loadPage(postUrl, { timeout }), + loadPage(commentsUrl, { timeout }), + ]); + + if (!postResult.ok || !commentsResult.ok) return null; + + const postData = parseJson(postResult.content); + const postView = postData?.post_view; + if (!postView) return null; + + const commentsData = parseJson(commentsResult.content); + const comments = commentsData?.comments ?? []; + + let md = `# ${postView.post.name}\n\n`; + + const communityLabel = formatCommunity(postView.community); + const authorLabel = formatAuthor(postView.creator); + const score = postView.counts?.score ?? 0; + const commentCount = postView.counts?.comments ?? comments.length; + + md += `**Community:** ${communityLabel} · **Author:** ${authorLabel} · **Score:** ${score} · **Comments:** ${commentCount}\n`; + if (postView.post.url) { + md += `**Link:** ${postView.post.url}\n`; + } + md += "\n"; + + if (postView.post.body) { + md += `---\n\n${postView.post.body}\n\n`; + } + + if (comments.length > 0) { + const threadedComments = renderComments(comments); + if (threadedComments) { + md += `---\n\n## Comments\n\n${threadedComments}\n`; + } + } + + const output = finalizeOutput(md); + return { + url, + finalUrl: url, + contentType: "text/markdown", + method: "lemmy-api", + content: output.content, + fetchedAt, + truncated: output.truncated, + notes: ["Fetched via Lemmy API"], + }; + } catch {} + + return null; +}; diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/lobsters.ts b/packages/coding-agent/src/core/tools/web-scrapers/lobsters.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/lobsters.ts rename to packages/coding-agent/src/core/tools/web-scrapers/lobsters.ts diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/mastodon.ts b/packages/coding-agent/src/core/tools/web-scrapers/mastodon.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/mastodon.ts rename to packages/coding-agent/src/core/tools/web-scrapers/mastodon.ts diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/maven.ts b/packages/coding-agent/src/core/tools/web-scrapers/maven.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/maven.ts rename to packages/coding-agent/src/core/tools/web-scrapers/maven.ts diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/mdn.ts b/packages/coding-agent/src/core/tools/web-scrapers/mdn.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/mdn.ts rename to packages/coding-agent/src/core/tools/web-scrapers/mdn.ts diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/media.test.ts b/packages/coding-agent/src/core/tools/web-scrapers/media.test.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/media.test.ts rename to packages/coding-agent/src/core/tools/web-scrapers/media.test.ts diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/metacpan.ts b/packages/coding-agent/src/core/tools/web-scrapers/metacpan.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/metacpan.ts rename to packages/coding-agent/src/core/tools/web-scrapers/metacpan.ts diff --git a/packages/coding-agent/src/core/tools/web-scrapers/musicbrainz.ts b/packages/coding-agent/src/core/tools/web-scrapers/musicbrainz.ts new file mode 100644 index 000000000..eed2a1261 --- /dev/null +++ b/packages/coding-agent/src/core/tools/web-scrapers/musicbrainz.ts @@ -0,0 +1,268 @@ +/** + * MusicBrainz URL handler for artists, releases, and recordings + */ + +import type { RenderResult, SpecialHandler } from "./types"; +import { finalizeOutput, loadPage } from "./types"; + +type MusicBrainzEntity = "artist" | "release" | "recording"; + +interface MusicBrainzLifeSpan { + begin?: string; + end?: string; + ended?: boolean; +} + +interface MusicBrainzArtist { + id: string; + name: string; + type?: string; + country?: string; + "life-span"?: MusicBrainzLifeSpan; +} + +interface MusicBrainzArtistCredit { + name?: string; + artist?: { + id?: string; + name: string; + }; +} + +interface MusicBrainzRecording { + id: string; + title: string; + length?: number; + "artist-credit"?: MusicBrainzArtistCredit[]; +} + +interface MusicBrainzTrack { + id?: string; + title?: string; + number?: string; + position?: number; + length?: number; + recording?: { + title?: string; + length?: number; + }; +} + +interface MusicBrainzMedium { + position?: number; + format?: string; + "track-count"?: number; + tracks?: MusicBrainzTrack[]; +} + +interface MusicBrainzRelease { + id: string; + title: string; + "track-count"?: number; + media?: MusicBrainzMedium[]; +} + +const MUSICBRAINZ_HOSTS = new Set(["musicbrainz.org", "www.musicbrainz.org"]); +const USER_AGENT = "omp-web-fetch/1.0 (https://github.com/anthropics)"; +const MAX_TRACKS = 50; + +function parseEntity(url: URL): { entity: MusicBrainzEntity; mbid: string } | null { + if (!MUSICBRAINZ_HOSTS.has(url.hostname)) return null; + + const parts = url.pathname.split("/").filter(Boolean); + if (parts.length < 2) return null; + + const entity = parts[0] as MusicBrainzEntity; + if (entity !== "artist" && entity !== "release" && entity !== "recording") return null; + + const mbid = parts[1]; + if (!/^[0-9a-fA-F-]{36}$/.test(mbid)) return null; + + return { entity, mbid }; +} + +async function fetchJson(apiUrl: string, timeout: number): Promise { + const result = await loadPage(apiUrl, { + timeout, + headers: { + "User-Agent": USER_AGENT, + Accept: "application/json", + }, + }); + + if (!result.ok) return null; + + try { + return JSON.parse(result.content) as T; + } catch { + return null; + } +} + +function formatLifeSpan(life: MusicBrainzLifeSpan | undefined): string | null { + if (!life) return null; + + const begin = life.begin?.trim(); + const end = life.end?.trim(); + + if (begin && end) return `${begin} - ${end}`; + if (begin && !end) return `${begin} - ${life.ended ? "ended" : "present"}`; + if (!begin && end) return `? - ${end}`; + if (life.ended !== undefined) return life.ended ? "ended" : "present"; + + return null; +} + +function formatDurationMs(lengthMs: number | undefined): string | null { + if (!lengthMs || lengthMs <= 0) return null; + + const totalSeconds = Math.round(lengthMs / 1000); + const hours = Math.floor(totalSeconds / 3600); + const minutes = Math.floor((totalSeconds % 3600) / 60); + const seconds = totalSeconds % 60; + + if (hours > 0) { + return `${hours}:${minutes.toString().padStart(2, "0")}:${seconds.toString().padStart(2, "0")}`; + } + + return `${minutes}:${seconds.toString().padStart(2, "0")}`; +} + +function formatArtistCredits(credits: MusicBrainzArtistCredit[] | undefined): string | null { + if (!credits?.length) return null; + + const names = credits + .map((credit) => credit.name || credit.artist?.name) + .filter((name): name is string => Boolean(name)); + + if (!names.length) return null; + return names.join(", "); +} + +function formatTrack(track: MusicBrainzTrack): string { + const title = track.title || track.recording?.title || "Untitled"; + const duration = formatDurationMs(track.length ?? track.recording?.length); + const number = track.number || (track.position ? String(track.position) : null); + + const prefix = number ? `${number}. ` : "- "; + let line = `${prefix}${title}`; + if (duration) line += ` (${duration})`; + return line; +} + +function buildMediumLabel(medium: MusicBrainzMedium, includePosition: boolean): string | null { + const parts: string[] = []; + if (includePosition && medium.position) parts.push(`Disc ${medium.position}`); + if (medium.format) parts.push(medium.format); + return parts.length ? parts.join(" - ") : null; +} + +function buildArtistMarkdown(artist: MusicBrainzArtist): string { + let md = `# ${artist.name}\n\n`; + const meta: string[] = []; + + if (artist.type) meta.push(`**Type**: ${artist.type}`); + if (artist.country) meta.push(`**Country**: ${artist.country}`); + + const lifeSpan = formatLifeSpan(artist["life-span"]); + if (lifeSpan) meta.push(`**Life Span**: ${lifeSpan}`); + + if (meta.length) md += `${meta.join("\n")}\n`; + + return md; +} + +function buildReleaseMarkdown(release: MusicBrainzRelease): string { + let md = `# ${release.title}\n\n`; + + const media = release.media ?? []; + const totalTracks = + release["track-count"] ?? + media.reduce((sum, medium) => sum + (medium["track-count"] ?? medium.tracks?.length ?? 0), 0); + + if (totalTracks) { + md += `**Tracks**: ${totalTracks}\n\n`; + } + + if (media.length) { + md += "## Tracks\n\n"; + const includePosition = media.length > 1; + + for (const medium of media) { + const label = buildMediumLabel(medium, includePosition); + if (label) md += `### ${label}\n\n`; + + const tracks = medium.tracks ?? []; + if (tracks.length) { + const lines = tracks.slice(0, MAX_TRACKS).map(formatTrack).join("\n"); + md += `${lines}\n\n`; + + if (tracks.length > MAX_TRACKS) { + md += `_Showing first ${MAX_TRACKS} of ${tracks.length} tracks._\n\n`; + } + } else if (medium["track-count"]) { + md += `- ${medium["track-count"]} tracks (details unavailable)\n\n`; + } + } + } + + return md; +} + +function buildRecordingMarkdown(recording: MusicBrainzRecording): string { + let md = `# ${recording.title}\n\n`; + const meta: string[] = []; + + const artists = formatArtistCredits(recording["artist-credit"]); + if (artists) meta.push(`**Artists**: ${artists}`); + + const length = formatDurationMs(recording.length); + if (length) meta.push(`**Length**: ${length}`); + + if (meta.length) md += `${meta.join("\n")}\n`; + + return md; +} + +export const handleMusicBrainz: SpecialHandler = async (url: string, timeout: number): Promise => { + try { + const parsed = new URL(url); + const parsedEntity = parseEntity(parsed); + if (!parsedEntity) return null; + + const { entity, mbid } = parsedEntity; + const fetchedAt = new Date().toISOString(); + let md = ""; + + if (entity === "artist") { + const apiUrl = `https://musicbrainz.org/ws/2/artist/${mbid}?fmt=json&inc=url-rels`; + const artist = await fetchJson(apiUrl, timeout); + if (!artist) return null; + md = buildArtistMarkdown(artist); + } else if (entity === "release") { + const apiUrl = `https://musicbrainz.org/ws/2/release/${mbid}?fmt=json&inc=recordings`; + const release = await fetchJson(apiUrl, timeout); + if (!release) return null; + md = buildReleaseMarkdown(release); + } else { + const apiUrl = `https://musicbrainz.org/ws/2/recording/${mbid}?fmt=json`; + const recording = await fetchJson(apiUrl, timeout); + if (!recording) return null; + md = buildRecordingMarkdown(recording); + } + + const output = finalizeOutput(md); + return { + url, + finalUrl: url, + contentType: "text/markdown", + method: "musicbrainz-api", + content: output.content, + fetchedAt, + truncated: output.truncated, + notes: ["Fetched via MusicBrainz API"], + }; + } catch {} + + return null; +}; diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/npm.ts b/packages/coding-agent/src/core/tools/web-scrapers/npm.ts similarity index 93% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/npm.ts rename to packages/coding-agent/src/core/tools/web-scrapers/npm.ts index f463e67c7..f273fbd5d 100644 --- a/packages/coding-agent/src/core/tools/web-fetch-handlers/npm.ts +++ b/packages/coding-agent/src/core/tools/web-scrapers/npm.ts @@ -47,7 +47,7 @@ export const handleNpm: SpecialHandler = async (url: string, timeout: number): P name: string; version: string; description?: string; - license?: string; + license?: string | { type: string }; homepage?: string; repository?: { url: string } | string; keywords?: string[]; @@ -66,7 +66,10 @@ export const handleNpm: SpecialHandler = async (url: string, timeout: number): P if (pkg.description) md += `${pkg.description}\n\n`; md += `**Latest:** ${pkg.version || "unknown"}`; - if (pkg.license) md += ` · **License:** ${typeof pkg.license === "string" ? pkg.license : pkg.license}`; + if (pkg.license) { + const license = typeof pkg.license === "string" ? pkg.license : (pkg.license.type ?? String(pkg.license)); + md += ` · **License:** ${license}`; + } md += "\n"; if (weeklyDownloads !== null) { md += `**Weekly Downloads:** ${formatCount(weeklyDownloads)}\n`; diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/nuget.ts b/packages/coding-agent/src/core/tools/web-scrapers/nuget.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/nuget.ts rename to packages/coding-agent/src/core/tools/web-scrapers/nuget.ts diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/nvd.ts b/packages/coding-agent/src/core/tools/web-scrapers/nvd.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/nvd.ts rename to packages/coding-agent/src/core/tools/web-scrapers/nvd.ts diff --git a/packages/coding-agent/src/core/tools/web-scrapers/ollama.ts b/packages/coding-agent/src/core/tools/web-scrapers/ollama.ts new file mode 100644 index 000000000..1d6141283 --- /dev/null +++ b/packages/coding-agent/src/core/tools/web-scrapers/ollama.ts @@ -0,0 +1,263 @@ +import type { RenderResult, SpecialHandler } from "./types"; +import { finalizeOutput, loadPage } from "./types"; + +interface OllamaTagDetails { + parent_model?: string; + format?: string; + family?: string; + families?: string[] | null; + parameter_size?: string; + quantization_level?: string; +} + +interface OllamaTagModel { + name?: string; + model?: string; + modified_at?: string; + size?: number; + digest?: string; + details?: OllamaTagDetails; +} + +interface OllamaTagsResponse { + models?: OllamaTagModel[]; +} + +const VALID_HOSTNAMES = new Set(["ollama.com", "www.ollama.com"]); +const RESERVED_ROOTS = new Set([ + "models", + "blog", + "docs", + "download", + "cloud", + "signin", + "signout", + "search", + "api", + "terms", + "privacy", + "license", + "settings", +]); + +function decodeHtmlEntities(value: string): string { + return value + .replace(/&/g, "&") + .replace(/</g, "<") + .replace(/>/g, ">") + .replace(/"/g, '"') + .replace(/'/g, "'") + .replace(/ /g, " "); +} + +function extractMetaDescription(html: string): string | null { + const patterns = [ + /]+name=["']description["'][^>]*content=["']([^"']+)["']/i, + /]+property=["']og:description["'][^>]*content=["']([^"']+)["']/i, + /]+property=["']twitter:description["'][^>]*content=["']([^"']+)["']/i, + ]; + + for (const pattern of patterns) { + const match = html.match(pattern); + if (match?.[1]) { + return decodeHtmlEntities(match[1].trim()); + } + } + + return null; +} + +function extractParameterSizes(html: string): string[] { + const sizes = new Set(); + const pattern = /x-test-size[^>]*>([^<]+)<\/span>/gi; + let match = pattern.exec(html); + while (match) { + const raw = match[1]?.trim(); + if (raw) { + sizes.add(raw.toUpperCase()); + } + match = pattern.exec(html); + } + + return Array.from(sizes); +} + +function extractTagsFromHtml(html: string, baseRef: string): string[] { + const tags = new Set(); + const pattern = /href=["']\/library\/([^"']+)["']/gi; + let match = pattern.exec(html); + while (match) { + const raw = match[1]?.trim(); + if (raw) { + const decoded = decodeHtmlEntities(raw); + if (decoded === baseRef || decoded.startsWith(`${baseRef}:`)) { + tags.add(decoded); + } + } + match = pattern.exec(html); + } + + return Array.from(tags); +} + +function formatSize(bytes: number): string { + if (bytes >= 1_000_000_000) return `${(bytes / 1_000_000_000).toFixed(1)}GB`; + if (bytes >= 1_000_000) return `${(bytes / 1_000_000).toFixed(1)}MB`; + if (bytes >= 1_000) return `${(bytes / 1_000).toFixed(1)}KB`; + return `${bytes}B`; +} + +function buildModelPath(parts: string[]): string { + return parts.map((part) => encodeURIComponent(part)).join("/"); +} + +function parseOllamaUrl(url: string): { modelRef: string; baseRef: string; pageUrl: string } | null { + try { + const parsed = new URL(url); + if (!VALID_HOSTNAMES.has(parsed.hostname)) return null; + + const parts = parsed.pathname.split("/").filter(Boolean); + if (parts.length === 0) return null; + + if (parts[0] === "library" && parts.length >= 2) { + const modelRef = decodeURIComponent(parts[1]); + const baseRef = modelRef.split(":")[0] ?? modelRef; + const pageUrl = `${parsed.origin}/${buildModelPath(["library", baseRef])}`; + return { modelRef, baseRef, pageUrl }; + } + + if (parts.length >= 2 && !RESERVED_ROOTS.has(parts[0])) { + const namespace = decodeURIComponent(parts[0]); + const model = decodeURIComponent(parts[1]); + const modelBase = model.split(":")[0] ?? model; + const modelRef = `${namespace}/${model}`; + const baseRef = `${namespace}/${modelBase}`; + const pageUrl = `${parsed.origin}/${buildModelPath([namespace, modelBase])}`; + return { modelRef, baseRef, pageUrl }; + } + } catch {} + + return null; +} + +function sortTags(tags: string[]): string[] { + return tags.sort((a, b) => { + const aLatest = a.endsWith(":latest"); + const bLatest = b.endsWith(":latest"); + if (aLatest && !bLatest) return -1; + if (!aLatest && bLatest) return 1; + return a.localeCompare(b); + }); +} + +function formatTagList(tags: string[], maxItems: number): string { + const limited = tags.slice(0, maxItems); + const formatted = limited.map((tag) => `\`${tag}\``).join(", "); + if (tags.length > maxItems) { + return `${formatted} (and ${tags.length - maxItems} more)`; + } + return formatted; +} + +function collectParameterSizes(models: OllamaTagModel[], htmlSizes: string[]): string[] { + const sizes = new Set(); + for (const model of models) { + const param = model.details?.parameter_size?.trim(); + if (param) sizes.add(param.toUpperCase()); + } + for (const size of htmlSizes) { + sizes.add(size); + } + return Array.from(sizes); +} + +export const handleOllama: SpecialHandler = async (url: string, timeout: number): Promise => { + try { + const parsed = parseOllamaUrl(url); + if (!parsed) return null; + + const { modelRef, baseRef, pageUrl } = parsed; + const fetchedAt = new Date().toISOString(); + + const tagsUrl = "https://ollama.com/api/tags"; + const [tagsResult, pageResult] = await Promise.all([ + loadPage(tagsUrl, { timeout, headers: { Accept: "application/json" } }), + loadPage(pageUrl, { timeout }), + ]); + + let tagsData: OllamaTagsResponse | null = null; + if (tagsResult.ok) { + try { + tagsData = JSON.parse(tagsResult.content) as OllamaTagsResponse; + } catch { + tagsData = null; + } + } + + const html = pageResult.ok ? pageResult.content : ""; + const description = html ? extractMetaDescription(html) : null; + const htmlParameterSizes = html ? extractParameterSizes(html) : []; + const htmlTags = html ? extractTagsFromHtml(html, baseRef) : []; + + const baseLower = baseRef.toLowerCase(); + const models = tagsData?.models ?? []; + const matchingModels = models.filter((model) => { + const name = (model.model ?? model.name ?? "").toLowerCase(); + return name === baseLower || name.startsWith(`${baseLower}:`); + }); + + const tagRef = modelRef.includes(":") ? modelRef : null; + const selectedTag = tagRef ? matchingModels.find((model) => (model.model ?? model.name ?? "") === tagRef) : null; + + const availableTagsRaw = matchingModels + .map((model) => model.model ?? model.name ?? "") + .filter((tag) => tag.length > 0); + const availableTags = sortTags(Array.from(new Set(availableTagsRaw))); + + const fallbackTags = sortTags(Array.from(new Set(htmlTags))); + const tagsToUse = availableTags.length > 0 ? availableTags : fallbackTags; + + const parameterSizes = collectParameterSizes(selectedTag ? [selectedTag] : matchingModels, htmlParameterSizes); + + const sizes = matchingModels + .map((model) => model.size) + .filter((size): size is number => typeof size === "number"); + let sizeLine: string | null = null; + + if (selectedTag?.size) { + sizeLine = formatSize(selectedTag.size); + } else if (sizes.length > 0) { + const minSize = Math.min(...sizes); + const maxSize = Math.max(...sizes); + sizeLine = minSize === maxSize ? formatSize(minSize) : `${formatSize(minSize)} - ${formatSize(maxSize)}`; + } + + let md = `# ${baseRef}\n\n`; + if (description) md += `${description}\n\n`; + + md += `**Model:** ${baseRef}\n`; + if (tagRef) md += `**Tag:** ${tagRef}\n`; + if (parameterSizes.length > 0) md += `**Parameters:** ${parameterSizes.join(", ")}\n`; + if (sizeLine) { + const label = sizeLine.includes(" - ") ? "Size Range" : "Size"; + md += `**${label}:** ${sizeLine}\n`; + } + if (tagsToUse.length > 0) { + md += `**Available Tags:** ${formatTagList(tagsToUse, 40)}\n`; + } + + const output = finalizeOutput(md); + return { + url, + finalUrl: pageResult.ok ? pageResult.finalUrl : url, + contentType: "text/markdown", + method: "ollama", + content: output.content, + fetchedAt, + truncated: output.truncated, + notes: ["Fetched via Ollama API"], + }; + } catch {} + + return null; +}; diff --git a/packages/coding-agent/src/core/tools/web-scrapers/open-vsx.ts b/packages/coding-agent/src/core/tools/web-scrapers/open-vsx.ts new file mode 100644 index 000000000..3a6a745a4 --- /dev/null +++ b/packages/coding-agent/src/core/tools/web-scrapers/open-vsx.ts @@ -0,0 +1,115 @@ +import type { RenderResult, SpecialHandler } from "./types"; +import { finalizeOutput, formatCount, loadPage } from "./types"; + +interface OpenVsxFileLinks { + readme?: string; +} + +interface OpenVsxExtension { + name: string; + namespace: string; + version: string; + displayName?: string; + description?: string; + downloadCount?: number; + averageRating?: number; + reviewCount?: number; + repository?: string | { url?: string }; + license?: string; + categories?: string[]; + homepage?: string; + files?: OpenVsxFileLinks; +} + +/** + * Handle Open VSX URLs via their API + */ +export const handleOpenVsx: SpecialHandler = async (url: string, timeout: number): Promise => { + try { + const parsed = new URL(url); + if (parsed.hostname !== "open-vsx.org" && parsed.hostname !== "www.open-vsx.org") return null; + + const match = parsed.pathname.match(/^\/extension\/([^/]+)\/([^/]+)(?:\/([^/]+))?\/?$/); + if (!match) return null; + + const namespace = decodeURIComponent(match[1]); + const extension = decodeURIComponent(match[2]); + const version = match[3] ? decodeURIComponent(match[3]) : null; + + const fetchedAt = new Date().toISOString(); + const baseUrl = `https://open-vsx.org/api/${encodeURIComponent(namespace)}/${encodeURIComponent(extension)}`; + const apiUrl = version ? `${baseUrl}/${encodeURIComponent(version)}` : baseUrl; + + const result = await loadPage(apiUrl, { timeout }); + if (!result.ok) return null; + + let data: OpenVsxExtension; + try { + data = JSON.parse(result.content); + } catch { + return null; + } + + let readme: string | null = null; + const readmeUrl = data.files?.readme; + if (readmeUrl) { + try { + const readmeResult = await loadPage(readmeUrl, { timeout: Math.min(timeout, 10) }); + if (readmeResult.ok) readme = readmeResult.content; + } catch {} + } + + const displayName = data.displayName || data.name || `${namespace}/${extension}`; + const displayNamespace = data.namespace || namespace; + const displayVersion = data.version || version || "unknown"; + const downloads = typeof data.downloadCount === "number" ? data.downloadCount : null; + const rating = typeof data.averageRating === "number" ? data.averageRating : null; + const reviews = typeof data.reviewCount === "number" ? data.reviewCount : null; + const repository = typeof data.repository === "string" ? data.repository : data.repository?.url || null; + + let md = `# ${displayName}\n\n`; + if (data.description) md += `${data.description}\n\n`; + + md += `**Namespace:** ${displayNamespace}\n`; + md += `**Extension:** ${data.name || extension}\n`; + md += `**Version:** ${displayVersion}`; + if (data.license) md += ` | **License:** ${data.license}`; + md += "\n"; + + if (downloads !== null) { + md += `**Downloads:** ${formatCount(downloads)}\n`; + } + + if (rating !== null) { + const reviewSuffix = reviews !== null ? ` (${reviews} reviews)` : ""; + md += `**Rating:** ${rating}${reviewSuffix}\n`; + } + + if (repository) { + const cleanedRepo = repository.replace(/^git\+/, "").replace(/\.git$/, ""); + md += `**Repository:** ${cleanedRepo}\n`; + } + + if (data.homepage) md += `**Homepage:** ${data.homepage}\n`; + if (data.categories?.length) md += `**Categories:** ${data.categories.join(", ")}\n`; + + if (readme) { + md += "\n---\n\n## README\n\n"; + md += `${readme}\n`; + } + + const output = finalizeOutput(md); + return { + url, + finalUrl: url, + contentType: "text/markdown", + method: "open-vsx", + content: output.content, + fetchedAt, + truncated: output.truncated, + notes: ["Fetched via Open VSX API"], + }; + } catch {} + + return null; +}; diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/opencorporates.ts b/packages/coding-agent/src/core/tools/web-scrapers/opencorporates.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/opencorporates.ts rename to packages/coding-agent/src/core/tools/web-scrapers/opencorporates.ts diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/openlibrary.ts b/packages/coding-agent/src/core/tools/web-scrapers/openlibrary.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/openlibrary.ts rename to packages/coding-agent/src/core/tools/web-scrapers/openlibrary.ts diff --git a/packages/coding-agent/src/core/tools/web-scrapers/orcid.ts b/packages/coding-agent/src/core/tools/web-scrapers/orcid.ts new file mode 100644 index 000000000..d5912f1b9 --- /dev/null +++ b/packages/coding-agent/src/core/tools/web-scrapers/orcid.ts @@ -0,0 +1,294 @@ +/** + * ORCID handler for web-fetch + */ + +import type { RenderResult, SpecialHandler } from "./types"; +import { finalizeOutput, loadPage } from "./types"; + +const MAX_WORKS = 50; +const ORCID_PATTERN = /\/(\d{4}-\d{4}-\d{4}-\d{3}[\dXx])(?:\/|$)/; + +interface OrcidName { + "given-names"?: { value?: string }; + "family-name"?: { value?: string }; + "credit-name"?: { value?: string }; +} + +interface OrcidBiography { + content?: string; +} + +interface OrcidPerson { + name?: OrcidName; + biography?: OrcidBiography; +} + +interface OrcidSummaryDate { + year?: { value?: string }; + month?: { value?: string }; + day?: { value?: string }; +} + +interface OrcidOrganizationAddress { + city?: string; + region?: string; + country?: string; +} + +interface OrcidOrganization { + name?: string; + address?: OrcidOrganizationAddress; +} + +interface OrcidAffiliationSummary { + organization?: OrcidOrganization; + "role-title"?: string; + "department-name"?: string; + "start-date"?: OrcidSummaryDate; + "end-date"?: OrcidSummaryDate; +} + +interface OrcidAffiliationGroupSummary { + "employment-summary"?: OrcidAffiliationSummary; + "education-summary"?: OrcidAffiliationSummary; +} + +interface OrcidAffiliationGroup { + summaries?: OrcidAffiliationGroupSummary[]; +} + +interface OrcidAffiliationsContainer { + "affiliation-group"?: OrcidAffiliationGroup[]; + "employment-summary"?: OrcidAffiliationSummary[]; + "education-summary"?: OrcidAffiliationSummary[]; +} + +interface OrcidWorkTitle { + title?: { value?: string }; +} + +interface OrcidWorkSummary { + title?: OrcidWorkTitle; +} + +interface OrcidWorkGroup { + "work-summary"?: OrcidWorkSummary[]; +} + +interface OrcidWorksContainer { + group?: OrcidWorkGroup[]; +} + +interface OrcidActivitiesSummary { + employments?: OrcidAffiliationsContainer; + educations?: OrcidAffiliationsContainer; + works?: OrcidWorksContainer; +} + +interface OrcidRecord { + "orcid-identifier"?: { path?: string; uri?: string }; + person?: OrcidPerson; + "activities-summary"?: OrcidActivitiesSummary; +} + +function isOrcidHost(hostname: string): boolean { + return hostname === "orcid.org" || hostname === "www.orcid.org"; +} + +function extractOrcidId(pathname: string): string | null { + const match = pathname.match(ORCID_PATTERN); + return match?.[1] ?? null; +} + +function formatName(name?: OrcidName): string | null { + const credit = name?.["credit-name"]?.value?.trim(); + if (credit) return credit; + + const given = name?.["given-names"]?.value?.trim(); + const family = name?.["family-name"]?.value?.trim(); + if (given && family) return `${given} ${family}`; + return given || family || null; +} + +function formatDate(date?: OrcidSummaryDate): string | null { + const year = date?.year?.value; + if (!year) return null; + + const month = date?.month?.value; + const day = date?.day?.value; + if (month && day) { + return `${year}-${month.padStart(2, "0")}-${day.padStart(2, "0")}`; + } + if (month) return `${year}-${month.padStart(2, "0")}`; + return year; +} + +function collectAffiliations( + container: OrcidAffiliationsContainer | undefined, + key: "employment-summary" | "education-summary", +): OrcidAffiliationSummary[] { + const summaries: OrcidAffiliationSummary[] = []; + + if (!container) return summaries; + + const direct = container[key]; + if (direct?.length) summaries.push(...direct); + + const groups = container["affiliation-group"]; + if (groups?.length) { + for (const group of groups) { + const groupSummaries = group.summaries || []; + for (const summary of groupSummaries) { + const entry = summary[key]; + if (entry) summaries.push(entry); + } + } + } + + return summaries; +} + +function formatAffiliation(summary: OrcidAffiliationSummary): string | null { + const organization = summary.organization?.name?.trim(); + const role = summary["role-title"]?.trim(); + const department = summary["department-name"]?.trim(); + + const address = summary.organization?.address; + const locationParts = [address?.city, address?.region, address?.country].filter(Boolean) as string[]; + const location = locationParts.length > 0 ? locationParts.join(", ") : null; + + const start = formatDate(summary["start-date"]); + const end = formatDate(summary["end-date"]); + let dates: string | null = null; + if (start && end) { + dates = `${start} - ${end}`; + } else if (start) { + dates = `${start} - Present`; + } else if (end) { + dates = `Until ${end}`; + } + + const label = organization || role || department; + if (!label) return null; + + const details: string[] = []; + if (organization && role) details.push(role); + if (!organization && role && department) details.push(department); + if (organization && department) details.push(`Dept: ${department}`); + if (location) details.push(`Location: ${location}`); + if (dates) details.push(`Dates: ${dates}`); + + if (details.length === 0) return label; + return `${label} (${details.join("; ")})`; +} + +function collectWorkTitles(container: OrcidWorksContainer | undefined): string[] { + const titles: string[] = []; + const seen = new Set(); + const groups = container?.group || []; + + for (const group of groups) { + const summaries = group["work-summary"] || []; + for (const summary of summaries) { + const title = summary.title?.title?.value?.trim(); + if (!title || seen.has(title)) continue; + seen.add(title); + titles.push(title); + if (titles.length >= MAX_WORKS) return titles; + } + } + + return titles; +} + +export const handleOrcid: SpecialHandler = async (url: string, timeout: number): Promise => { + try { + const parsed = new URL(url); + if (!isOrcidHost(parsed.hostname)) return null; + + const orcid = extractOrcidId(parsed.pathname); + if (!orcid) return null; + + const fetchedAt = new Date().toISOString(); + const apiUrl = `https://pub.orcid.org/v3.0/${orcid}/record`; + + const result = await loadPage(apiUrl, { + timeout, + headers: { Accept: "application/json" }, + }); + + if (!result.ok || !result.content) return null; + + let record: OrcidRecord; + try { + record = JSON.parse(result.content) as OrcidRecord; + } catch { + return null; + } + + const personName = formatName(record.person?.name); + const biography = record.person?.biography?.content?.trim(); + + const activities = record["activities-summary"]; + const employments = collectAffiliations(activities?.employments, "employment-summary"); + const educations = collectAffiliations(activities?.educations, "education-summary"); + const works = collectWorkTitles(activities?.works); + + let md = `# ${personName || "ORCID Profile"}\n\n`; + md += `**ORCID:** ${orcid}\n`; + md += `**ORCID Profile:** https://orcid.org/${orcid}\n\n`; + + md += "## Biography\n\n"; + md += biography ? `${biography}\n\n` : "No biography available.\n\n"; + + md += "## Affiliations\n\n"; + let hasAffiliations = false; + + if (employments.length > 0) { + hasAffiliations = true; + md += "### Employment\n\n"; + for (const summary of employments) { + const line = formatAffiliation(summary); + if (line) md += `- ${line}\n`; + } + md += "\n"; + } + + if (educations.length > 0) { + hasAffiliations = true; + md += "### Education\n\n"; + for (const summary of educations) { + const line = formatAffiliation(summary); + if (line) md += `- ${line}\n`; + } + md += "\n"; + } + + if (!hasAffiliations) { + md += "No affiliations available.\n\n"; + } + + md += "## Works\n\n"; + if (works.length > 0) { + for (const title of works) { + md += `- ${title}\n`; + } + } else { + md += "No works available.\n"; + } + + const output = finalizeOutput(md); + return { + url, + finalUrl: url, + contentType: "text/markdown", + method: "orcid-api", + content: output.content, + fetchedAt, + truncated: output.truncated, + notes: ["Fetched via ORCID Public API"], + }; + } catch { + return null; + } +}; diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/osv.ts b/packages/coding-agent/src/core/tools/web-scrapers/osv.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/osv.ts rename to packages/coding-agent/src/core/tools/web-scrapers/osv.ts diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/package-managers-2.test.ts b/packages/coding-agent/src/core/tools/web-scrapers/package-managers-2.test.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/package-managers-2.test.ts rename to packages/coding-agent/src/core/tools/web-scrapers/package-managers-2.test.ts diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/package-managers.test.ts b/packages/coding-agent/src/core/tools/web-scrapers/package-managers.test.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/package-managers.test.ts rename to packages/coding-agent/src/core/tools/web-scrapers/package-managers.test.ts diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/package-registries.test.ts b/packages/coding-agent/src/core/tools/web-scrapers/package-registries.test.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/package-registries.test.ts rename to packages/coding-agent/src/core/tools/web-scrapers/package-registries.test.ts diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/packagist.ts b/packages/coding-agent/src/core/tools/web-scrapers/packagist.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/packagist.ts rename to packages/coding-agent/src/core/tools/web-scrapers/packagist.ts diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/pub-dev.ts b/packages/coding-agent/src/core/tools/web-scrapers/pub-dev.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/pub-dev.ts rename to packages/coding-agent/src/core/tools/web-scrapers/pub-dev.ts diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/pubmed.ts b/packages/coding-agent/src/core/tools/web-scrapers/pubmed.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/pubmed.ts rename to packages/coding-agent/src/core/tools/web-scrapers/pubmed.ts diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/pypi.ts b/packages/coding-agent/src/core/tools/web-scrapers/pypi.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/pypi.ts rename to packages/coding-agent/src/core/tools/web-scrapers/pypi.ts diff --git a/packages/coding-agent/src/core/tools/web-scrapers/rawg.ts b/packages/coding-agent/src/core/tools/web-scrapers/rawg.ts new file mode 100644 index 000000000..00f0d0ecd --- /dev/null +++ b/packages/coding-agent/src/core/tools/web-scrapers/rawg.ts @@ -0,0 +1,120 @@ +import type { RenderResult, SpecialHandler } from "./types"; +import { finalizeOutput, htmlToBasicMarkdown, loadPage } from "./types"; + +interface RawgPlatformEntry { + platform?: { + name?: string; + }; +} + +interface RawgGenreEntry { + name?: string; +} + +interface RawgGameResponse { + name?: string; + released?: string; + rating?: number; + platforms?: RawgPlatformEntry[]; + genres?: RawgGenreEntry[]; + description?: string; + description_raw?: string; + detail?: string; + error?: string; +} + +export const handleRawg: SpecialHandler = async (url: string, timeout: number): Promise => { + try { + const parsed = new URL(url); + if (!isRawgHostname(parsed.hostname)) return null; + + const slug = extractGameSlug(parsed.pathname); + if (!slug) return null; + + const fetchedAt = new Date().toISOString(); + const apiUrl = `https://api.rawg.io/api/games/${encodeURIComponent(slug)}`; + const result = await loadPage(apiUrl, { timeout, headers: { Accept: "application/json" } }); + + if (!result.ok) return null; + + let game: RawgGameResponse; + try { + game = JSON.parse(result.content); + } catch { + return null; + } + + if (requiresApiKey(game)) return null; + + const title = game.name?.trim() || slug; + let md = `# ${title}\n\n`; + + if (game.released) md += `**Released:** ${game.released}\n`; + if (typeof game.rating === "number" && !Number.isNaN(game.rating)) { + md += `**Rating:** ${game.rating.toFixed(2)} / 5\n`; + } + + const platforms = collectNames(game.platforms?.map((entry) => entry.platform?.name)); + if (platforms.length) md += `**Platforms:** ${platforms.join(", ")}\n`; + + const genres = collectNames(game.genres?.map((entry) => entry.name)); + if (genres.length) md += `**Genres:** ${genres.join(", ")}\n`; + + md += `**RAWG:** https://rawg.io/games/${encodeURIComponent(slug)}\n`; + md += "\n"; + + const description = extractDescription(game); + if (description) { + md += `## Description\n\n${description}\n`; + } + + const output = finalizeOutput(md); + return { + url, + finalUrl: url, + contentType: "text/markdown", + method: "rawg", + content: output.content, + fetchedAt, + truncated: output.truncated, + notes: ["Fetched via RAWG API"], + }; + } catch {} + + return null; +}; + +function isRawgHostname(hostname: string): boolean { + return hostname === "rawg.io" || hostname === "www.rawg.io"; +} + +function extractGameSlug(pathname: string): string | null { + const match = pathname.match(/^\/games\/([^/?#]+)/); + if (!match) return null; + + const slug = decodeURIComponent(match[1]); + return slug ? slug.trim() : null; +} + +function requiresApiKey(game: RawgGameResponse): boolean { + const detail = `${game.detail ?? ""} ${game.error ?? ""}`.toLowerCase(); + return detail.includes("api key") || detail.includes("key is required") || detail.includes("apikey"); +} + +function extractDescription(game: RawgGameResponse): string | null { + if (game.description_raw) return game.description_raw.trim(); + if (!game.description) return null; + + const markdown = htmlToBasicMarkdown(game.description).trim(); + return markdown || null; +} + +function collectNames(values?: Array): string[] { + if (!values?.length) return []; + const names = new Set(); + for (const value of values) { + const trimmed = value?.trim(); + if (trimmed) names.add(trimmed); + } + return Array.from(names); +} diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/readthedocs.ts b/packages/coding-agent/src/core/tools/web-scrapers/readthedocs.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/readthedocs.ts rename to packages/coding-agent/src/core/tools/web-scrapers/readthedocs.ts diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/reddit.ts b/packages/coding-agent/src/core/tools/web-scrapers/reddit.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/reddit.ts rename to packages/coding-agent/src/core/tools/web-scrapers/reddit.ts diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/repology.ts b/packages/coding-agent/src/core/tools/web-scrapers/repology.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/repology.ts rename to packages/coding-agent/src/core/tools/web-scrapers/repology.ts diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/research.test.ts b/packages/coding-agent/src/core/tools/web-scrapers/research.test.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/research.test.ts rename to packages/coding-agent/src/core/tools/web-scrapers/research.test.ts diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/rfc.ts b/packages/coding-agent/src/core/tools/web-scrapers/rfc.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/rfc.ts rename to packages/coding-agent/src/core/tools/web-scrapers/rfc.ts diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/rubygems.ts b/packages/coding-agent/src/core/tools/web-scrapers/rubygems.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/rubygems.ts rename to packages/coding-agent/src/core/tools/web-scrapers/rubygems.ts diff --git a/packages/coding-agent/src/core/tools/web-scrapers/searchcode.ts b/packages/coding-agent/src/core/tools/web-scrapers/searchcode.ts new file mode 100644 index 000000000..b877de18a --- /dev/null +++ b/packages/coding-agent/src/core/tools/web-scrapers/searchcode.ts @@ -0,0 +1,213 @@ +import type { RenderResult, SpecialHandler } from "./types"; +import { finalizeOutput, formatCount, loadPage } from "./types"; + +interface SearchcodeResult { + id?: number | string; + filename?: string; + repo?: string; + language?: string; + code?: string; + lines?: number | string | Array; + location?: string; + url?: string; +} + +interface SearchcodeSearchResponse { + query?: string; + results?: SearchcodeResult[]; + total?: number; + total_results?: number; + nextpage?: number; +} + +const VALID_HOSTS = new Set(["searchcode.com", "www.searchcode.com"]); + +function parseLineNumbers(lines: SearchcodeResult["lines"]): number[] | null { + if (typeof lines === "number" && Number.isFinite(lines)) return [lines]; + + if (typeof lines === "string") { + const parts = lines.split(/[,\s]+/).filter(Boolean); + const parsed = parts.map((part) => Number.parseInt(part, 10)).filter((value) => Number.isFinite(value)); + return parsed.length ? parsed : null; + } + + if (Array.isArray(lines)) { + const parsed = lines.map((part) => Number.parseInt(String(part), 10)).filter((value) => Number.isFinite(value)); + return parsed.length ? parsed : null; + } + + return null; +} + +function formatLineNumbers(lines: number[] | null): string | null { + if (!lines || lines.length === 0) return null; + if (lines.length <= 10) return lines.join(", "); + const min = Math.min(...lines); + const max = Math.max(...lines); + return `${min}-${max} (${lines.length} lines)`; +} + +function formatCodeBlock( + code: string | undefined, + language: string | undefined, + lines: number[] | null, +): string | null { + if (!code) return null; + + const normalized = code.replace(/\r\n/g, "\n").trimEnd(); + const codeLines = normalized.split("\n"); + const languageTag = typeof language === "string" ? language.trim().toLowerCase() : ""; + + let displayLines = codeLines; + if (lines && lines.length === codeLines.length) { + displayLines = codeLines.map((line, index) => `${lines[index]}: ${line}`); + } + + const fence = languageTag ? languageTag : ""; + return `\n\n\`\`\`${fence}\n${displayLines.join("\n")}\n\`\`\`\n`; +} + +export const handleSearchcode: SpecialHandler = async (url: string, timeout: number): Promise => { + try { + const parsed = new URL(url); + if (!VALID_HOSTS.has(parsed.hostname)) return null; + + const fetchedAt = new Date().toISOString(); + const viewMatch = parsed.pathname.match(/^\/codesearch\/view\/([^/?#]+)/); + if (viewMatch) { + const id = viewMatch[1]; + const apiUrl = `https://searchcode.com/api/result/${encodeURIComponent(id)}/`; + const result = await loadPage(apiUrl, { timeout, headers: { Accept: "application/json" } }); + if (!result.ok) return null; + + let data: SearchcodeResult; + try { + data = JSON.parse(result.content) as SearchcodeResult; + } catch { + return null; + } + + const filename = data.filename || data.location || `Result ${id}`; + const lineNumbers = parseLineNumbers(data.lines); + const formattedLines = formatLineNumbers(lineNumbers); + const viewUrl = data.url || `https://searchcode.com/codesearch/view/${id}`; + const snippetBlock = formatCodeBlock(data.code, data.language, lineNumbers); + + let md = `# ${filename}\n\n`; + md += `## Description\n\n`; + md += "Code snippet from searchcode.com.\n\n"; + md += `## Metadata\n\n`; + if (data.repo) md += `**Repository:** ${data.repo}\n`; + if (data.language) md += `**Language:** ${data.language}\n`; + if (data.filename) md += `**File:** ${data.filename}\n`; + if (data.location) md += `**Location:** ${data.location}\n`; + if (formattedLines) md += `**Lines:** ${formattedLines}\n`; + md += `**Result ID:** ${id}\n`; + md += `**URL:** ${viewUrl}\n`; + + md += `\n## Snippet`; + if (snippetBlock) { + md += snippetBlock; + } else { + md += "\n\n_No snippet available._\n"; + } + + const output = finalizeOutput(md); + return { + url, + finalUrl: url, + contentType: "text/markdown", + method: "searchcode", + content: output.content, + fetchedAt, + truncated: output.truncated, + notes: ["Fetched via searchcode API"], + }; + } + + const query = parsed.searchParams.get("q"); + const isSearchPage = + parsed.pathname === "/" || parsed.pathname === "/codesearch" || parsed.pathname === "/codesearch/"; + if (!query || !isSearchPage) return null; + + const pageRaw = parsed.searchParams.get("p") ?? parsed.searchParams.get("page"); + const pageNumber = pageRaw ? Number.parseInt(pageRaw, 10) : 0; + const page = Number.isFinite(pageNumber) && pageNumber >= 0 ? pageNumber : 0; + const apiUrl = `https://searchcode.com/api/codesearch_I/?q=${encodeURIComponent(query)}&p=${page}`; + const result = await loadPage(apiUrl, { timeout, headers: { Accept: "application/json" } }); + if (!result.ok) return null; + + let data: SearchcodeSearchResponse; + try { + data = JSON.parse(result.content) as SearchcodeSearchResponse; + } catch { + return null; + } + + const results = Array.isArray(data.results) ? data.results : []; + const total = + typeof data.total === "number" + ? data.total + : typeof data.total_results === "number" + ? data.total_results + : null; + + let md = `# Searchcode Results\n\n`; + md += `## Description\n\n`; + md += `Search results for \`${query}\` on searchcode.com.\n\n`; + md += `## Metadata\n\n`; + md += `**Query:** \`${query}\`\n`; + md += `**Page:** ${page}\n`; + if (total !== null) md += `**Total Results:** ${formatCount(total)}\n`; + md += `**Result Count:** ${results.length}\n`; + if (typeof data.nextpage === "number") md += `**Next Page:** ${data.nextpage}\n`; + + md += `\n## Results\n\n`; + + if (results.length === 0) { + md += "_No results found._\n"; + } else { + const maxResults = 10; + for (const resultItem of results.slice(0, maxResults)) { + const id = resultItem.id !== undefined ? String(resultItem.id) : null; + const filename = resultItem.filename || resultItem.location || "Result"; + const lineNumbers = parseLineNumbers(resultItem.lines); + const formattedLines = formatLineNumbers(lineNumbers); + const viewUrl = resultItem.url || (id ? `https://searchcode.com/codesearch/view/${id}` : null); + const snippetBlock = formatCodeBlock(resultItem.code, resultItem.language, lineNumbers); + + md += `### ${filename}\n\n`; + if (resultItem.repo) md += `**Repository:** ${resultItem.repo}\n`; + if (resultItem.language) md += `**Language:** ${resultItem.language}\n`; + if (resultItem.filename) md += `**File:** ${resultItem.filename}\n`; + if (resultItem.location) md += `**Location:** ${resultItem.location}\n`; + if (formattedLines) md += `**Lines:** ${formattedLines}\n`; + if (viewUrl) md += `**URL:** ${viewUrl}\n`; + + if (snippetBlock) { + md += `${snippetBlock}\n`; + } + + md += "\n"; + } + + if (results.length > maxResults) { + md += `\n_Only showing first ${maxResults} results._\n`; + } + } + + const output = finalizeOutput(md); + return { + url, + finalUrl: url, + contentType: "text/markdown", + method: "searchcode", + content: output.content, + fetchedAt, + truncated: output.truncated, + notes: ["Fetched via searchcode API"], + }; + } catch {} + + return null; +}; diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/sec-edgar.ts b/packages/coding-agent/src/core/tools/web-scrapers/sec-edgar.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/sec-edgar.ts rename to packages/coding-agent/src/core/tools/web-scrapers/sec-edgar.ts diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/security.test.ts b/packages/coding-agent/src/core/tools/web-scrapers/security.test.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/security.test.ts rename to packages/coding-agent/src/core/tools/web-scrapers/security.test.ts diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/semantic-scholar.ts b/packages/coding-agent/src/core/tools/web-scrapers/semantic-scholar.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/semantic-scholar.ts rename to packages/coding-agent/src/core/tools/web-scrapers/semantic-scholar.ts diff --git a/packages/coding-agent/src/core/tools/web-scrapers/snapcraft.ts b/packages/coding-agent/src/core/tools/web-scrapers/snapcraft.ts new file mode 100644 index 000000000..a457cad53 --- /dev/null +++ b/packages/coding-agent/src/core/tools/web-scrapers/snapcraft.ts @@ -0,0 +1,195 @@ +import type { RenderResult, SpecialHandler } from "./types"; +import { finalizeOutput, formatCount, loadPage } from "./types"; + +interface SnapcraftPublisher { + "display-name"?: string; + username?: string; + id?: string; + validation?: string; +} + +interface SnapcraftChannel { + name?: string; + track?: string; + risk?: string; + branch?: string | null; + architecture?: string; + "released-at"?: string; +} + +interface SnapcraftDownload { + size?: number; + url?: string; + "sha3-384"?: string; +} + +interface SnapcraftChannelMapEntry { + channel?: SnapcraftChannel; + version?: string; + revision?: number | string; + download?: SnapcraftDownload; + type?: string; + "created-at"?: string; +} + +interface SnapcraftSnap { + name?: string; + title?: string; + summary?: string; + description?: string; + publisher?: SnapcraftPublisher; + version?: string; + confinement?: string; + base?: string; + downloads?: number; + download?: number; +} + +interface SnapcraftResponse { + name?: string; + title?: string; + summary?: string; + description?: string; + publisher?: SnapcraftPublisher; + version?: string; + confinement?: string; + base?: string; + downloads?: number; + download?: number; + snap?: SnapcraftSnap; + "channel-map"?: SnapcraftChannelMapEntry[]; +} + +function formatPublisher(publisher?: SnapcraftPublisher): string | null { + if (!publisher) return null; + const displayName = publisher["display-name"] ?? publisher.username ?? publisher.id; + if (!displayName) return null; + if (publisher.username && displayName !== publisher.username) { + return `${displayName} (@${publisher.username})`; + } + return displayName; +} + +function formatChannelName(channel?: SnapcraftChannel): string | null { + if (!channel) return null; + if (channel.name?.includes("/")) return channel.name; + if (channel.track && channel.risk) { + const branch = channel.branch ? `/${channel.branch}` : ""; + return `${channel.track}/${channel.risk}${branch}`; + } + return channel.name ?? null; +} + +function pickVersionFromChannels(entries: SnapcraftChannelMapEntry[]): string | undefined { + const stable = entries.find((entry) => entry.channel?.risk === "stable" && entry.version); + if (stable?.version) return stable.version; + const first = entries.find((entry) => entry.version); + return first?.version; +} + +function extractDownloads(snapInfo: SnapcraftSnap | SnapcraftResponse, data: SnapcraftResponse): number | null { + const candidates = [snapInfo.downloads, snapInfo.download, data.downloads, data.download]; + for (const value of candidates) { + if (typeof value === "number" && Number.isFinite(value)) return value; + } + return null; +} + +export const handleSnapcraft: SpecialHandler = async (url: string, timeout: number): Promise => { + try { + const parsed = new URL(url); + if (parsed.hostname !== "snapcraft.io" && parsed.hostname !== "www.snapcraft.io") return null; + + const installMatch = parsed.pathname.match(/^\/install\/([^/]+)\/?$/); + const directMatch = parsed.pathname.match(/^\/([^/]+)\/?$/); + if (!installMatch && !directMatch) return null; + + const snapName = decodeURIComponent((installMatch ?? directMatch)![1]); + const fetchedAt = new Date().toISOString(); + + const apiUrl = `https://api.snapcraft.io/v2/snaps/info/${encodeURIComponent(snapName)}`; + const result = await loadPage(apiUrl, { + timeout, + headers: { + Accept: "application/json", + "Snap-Device-Series": "16", + }, + }); + if (!result.ok) return null; + + let data: SnapcraftResponse; + try { + data = JSON.parse(result.content) as SnapcraftResponse; + } catch { + return null; + } + + const snapInfo = data.snap ?? data; + const name = snapInfo.title ?? snapInfo.name ?? data.name ?? snapName; + const summary = snapInfo.summary ?? data.summary; + const description = snapInfo.description ?? data.description; + const publisher = formatPublisher(snapInfo.publisher ?? data.publisher); + const confinement = snapInfo.confinement ?? data.confinement; + const base = snapInfo.base ?? data.base; + + const channelMap = data["channel-map"] ?? []; + let version = snapInfo.version ?? data.version; + if (!version && channelMap.length > 0) { + version = pickVersionFromChannels(channelMap); + } + + const downloads = extractDownloads(snapInfo, data); + + const channels = new Map }>(); + for (const entry of channelMap) { + const channelName = formatChannelName(entry.channel); + if (!channelName) continue; + const existing = channels.get(channelName) ?? { architectures: new Set() }; + if (!existing.version && entry.version) existing.version = entry.version; + if (entry.channel?.architecture) existing.architectures.add(entry.channel.architecture); + channels.set(channelName, existing); + } + + let md = `# ${name}\n\n`; + if (summary) md += `${summary}\n\n`; + + md += `**Version:** ${version ?? "unknown"}`; + if (confinement) md += ` · **Confinement:** ${confinement}`; + if (base) md += ` · **Base:** ${base}`; + md += "\n"; + if (publisher) md += `**Publisher:** ${publisher}\n`; + if (downloads !== null) md += `**Downloads:** ${formatCount(downloads)}\n`; + md += "\n"; + + if (channels.size > 0) { + md += "## Channels\n\n"; + const sortedChannels = Array.from(channels.entries()).sort((a, b) => a[0].localeCompare(b[0])); + for (const [channelName, info] of sortedChannels) { + const arches = Array.from(info.architectures).sort(); + const versionSuffix = info.version ? `: ${info.version}` : ""; + const archSuffix = arches.length > 0 ? ` (${arches.join(", ")})` : ""; + md += `- ${channelName}${versionSuffix}${archSuffix}\n`; + } + md += "\n"; + } + + const descriptionText = description ?? summary; + if (descriptionText) { + md += `## Description\n\n${descriptionText}\n`; + } + + const output = finalizeOutput(md); + return { + url, + finalUrl: url, + contentType: "text/markdown", + method: "snapcraft", + content: output.content, + fetchedAt, + truncated: output.truncated, + notes: ["Fetched via Snapcraft API"], + }; + } catch {} + + return null; +}; diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/social-extended.test.ts b/packages/coding-agent/src/core/tools/web-scrapers/social-extended.test.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/social-extended.test.ts rename to packages/coding-agent/src/core/tools/web-scrapers/social-extended.test.ts diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/social.test.ts b/packages/coding-agent/src/core/tools/web-scrapers/social.test.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/social.test.ts rename to packages/coding-agent/src/core/tools/web-scrapers/social.test.ts diff --git a/packages/coding-agent/src/core/tools/web-scrapers/sourcegraph.ts b/packages/coding-agent/src/core/tools/web-scrapers/sourcegraph.ts new file mode 100644 index 000000000..17e3b787e --- /dev/null +++ b/packages/coding-agent/src/core/tools/web-scrapers/sourcegraph.ts @@ -0,0 +1,353 @@ +import type { RenderResult, SpecialHandler } from "./types"; +import { finalizeOutput, loadPage } from "./types"; + +const GRAPHQL_ENDPOINT = "https://sourcegraph.com/.api/graphql"; +const GRAPHQL_HEADERS = { + Accept: "application/json", + "Content-Type": "application/json", +}; + +type SourcegraphTarget = + | { type: "search"; query: string } + | { type: "repo"; repoName: string; rev?: string } + | { type: "file"; repoName: string; rev?: string; filePath: string }; + +interface SourcegraphRepository { + name: string; + url: string; + description?: string | null; + defaultBranch?: { name: string } | null; +} + +interface RepoQueryData { + repository?: SourcegraphRepository | null; +} + +interface RepoFileQueryData { + repository?: + | (SourcegraphRepository & { + commit?: { + blob?: { content?: string | null } | null; + } | null; + }) + | null; +} + +interface SearchQueryData { + search?: { + results?: { + results?: SearchResultItem[] | null; + matchCount?: number | null; + limitHit?: boolean | null; + } | null; + } | null; +} + +interface FileMatchResult { + __typename: "FileMatch"; + repository?: { name?: string | null; url?: string | null } | null; + file?: { path?: string | null; url?: string | null } | null; + lineMatches?: Array<{ preview?: string | null; lineNumber?: number | null }> | null; +} + +interface RepositoryResult { + __typename: "Repository"; + name?: string | null; + url?: string | null; +} + +type SearchResultItem = FileMatchResult | RepositoryResult | { __typename: string }; + +const REPO_QUERY = `query Repo($name: String!) { + repository(name: $name) { + name + url + description + defaultBranch { + name + } + } +}`; + +const REPO_FILE_QUERY = `query RepoFile($name: String!, $path: String!, $rev: String!) { + repository(name: $name) { + name + url + description + defaultBranch { + name + } + commit(rev: $rev) { + blob(path: $path) { + content + } + } + } +}`; + +const SEARCH_QUERY = `query Search($query: String!) { + search(query: $query, version: V2) { + results { + results { + __typename + ... on FileMatch { + repository { + name + url + } + file { + path + url + } + lineMatches { + preview + lineNumber + } + } + ... on Repository { + name + url + } + } + matchCount + limitHit + } + } +}`; + +function parseSourcegraphUrl(url: string): SourcegraphTarget | null { + try { + const parsed = new URL(url); + if (parsed.hostname !== "sourcegraph.com" && parsed.hostname !== "www.sourcegraph.com") return null; + + if (parsed.pathname.startsWith("/search")) { + const query = parsed.searchParams.get("q")?.trim(); + if (!query) return null; + return { type: "search", query }; + } + + const parts = parsed.pathname + .split("/") + .filter(Boolean) + .map((part) => decodeURIComponent(part)); + if (parts.length < 3) return null; + + const hyphenIndex = parts.indexOf("-"); + const repoParts = hyphenIndex === -1 ? parts : parts.slice(0, hyphenIndex); + if (repoParts.length < 3) return null; + + const lastRepoPart = repoParts[repoParts.length - 1]; + const atIndex = lastRepoPart.indexOf("@"); + let rev: string | undefined; + let repoTail = lastRepoPart; + if (atIndex > 0) { + repoTail = lastRepoPart.slice(0, atIndex); + rev = lastRepoPart.slice(atIndex + 1) || undefined; + } + + repoParts[repoParts.length - 1] = repoTail; + const repoName = repoParts.join("/"); + + if (hyphenIndex !== -1 && parts[hyphenIndex + 1] === "blob") { + const filePath = parts.slice(hyphenIndex + 2).join("/"); + if (!filePath) return null; + return { type: "file", repoName, rev, filePath }; + } + + return { type: "repo", repoName, rev }; + } catch { + return null; + } +} + +function safeParseJson(content: string): T | null { + try { + return JSON.parse(content) as T; + } catch { + return null; + } +} + +async function fetchGraphql(query: string, variables: Record, timeout: number): Promise { + const body = JSON.stringify({ query, variables }); + const result = await loadPage(GRAPHQL_ENDPOINT, { + timeout, + headers: GRAPHQL_HEADERS, + method: "POST", + body, + }); + if (!result.ok) return null; + + const parsed = safeParseJson<{ data?: T; errors?: unknown }>(result.content); + if (!parsed?.data) return null; + if (Array.isArray(parsed.errors) && parsed.errors.length > 0) return null; + return parsed.data; +} + +function isFileMatchResult(result: SearchResultItem): result is FileMatchResult { + return result.__typename === "FileMatch"; +} + +function isRepositoryResult(result: SearchResultItem): result is RepositoryResult { + return result.__typename === "Repository"; +} + +function formatRepoMarkdown(repo: SourcegraphRepository): string { + let md = `# ${repo.name}\n\n`; + if (repo.description) md += `${repo.description}\n\n`; + md += `**URL:** ${repo.url}\n`; + if (repo.defaultBranch?.name) md += `**Default branch:** ${repo.defaultBranch.name}\n`; + return md; +} + +async function renderRepo(repoName: string, timeout: number): Promise<{ content: string; ok: boolean }> { + const data = await fetchGraphql(REPO_QUERY, { name: repoName }, timeout); + if (!data?.repository) return { content: "", ok: false }; + + return { content: formatRepoMarkdown(data.repository), ok: true }; +} + +async function renderFile( + repoName: string, + filePath: string, + rev: string, + timeout: number, +): Promise<{ content: string; ok: boolean }> { + const data = await fetchGraphql( + REPO_FILE_QUERY, + { name: repoName, path: filePath, rev }, + timeout, + ); + const repo = data?.repository; + const content = repo?.commit?.blob?.content ?? null; + if (!repo || content === null) return { content: "", ok: false }; + + let md = `${formatRepoMarkdown(repo)}\n`; + md += `**Path:** ${filePath}\n`; + md += `**Revision:** ${rev}\n\n`; + md += `---\n\n## File\n\n`; + md += "```text\n"; + md += `${content}\n`; + md += "```\n"; + return { content: md, ok: true }; +} + +async function renderSearch(query: string, timeout: number): Promise<{ content: string; ok: boolean }> { + const data = await fetchGraphql(SEARCH_QUERY, { query }, timeout); + const resultsData = data?.search?.results; + if (!resultsData) return { content: "", ok: false }; + const results = resultsData.results ?? []; + + let md = "# Sourcegraph Search\n\n"; + md += `**Query:** \`${query}\`\n`; + if (typeof resultsData?.matchCount === "number") { + md += `**Matches:** ${resultsData.matchCount}\n`; + } + if (typeof resultsData?.limitHit === "boolean") { + md += `**Limit hit:** ${resultsData.limitHit ? "yes" : "no"}\n`; + } + md += "\n"; + + if (!results || results.length === 0) { + md += "_No results._\n"; + return { content: md, ok: true }; + } + + const maxResults = 10; + md += "## Results\n\n"; + for (const result of results.slice(0, maxResults)) { + if (isFileMatchResult(result)) { + const repoName = result.repository?.name ?? "unknown"; + const filePath = result.file?.path ?? "unknown"; + md += `### ${repoName}/${filePath}\n\n`; + if (result.repository?.url) md += `**Repository:** ${result.repository.url}\n`; + if (result.file?.url) md += `**File:** ${result.file.url}\n`; + + const lineMatches = result.lineMatches ?? []; + if (lineMatches.length > 0) { + md += "\n```text\n"; + for (const line of lineMatches.slice(0, 5)) { + const preview = (line.preview ?? "").replace(/\n/g, " ").trim(); + const lineNumber = line.lineNumber ?? 0; + md += `L${lineNumber}: ${preview}\n`; + } + md += "```\n\n"; + } + continue; + } + + if (isRepositoryResult(result)) { + const name = result.name ?? "unknown"; + md += `### ${name}\n\n`; + if (result.url) md += `**Repository:** ${result.url}\n`; + md += "\n"; + } + } + + if (results.length > maxResults) { + md += `... and ${results.length - maxResults} more results\n`; + } + + return { content: md, ok: true }; +} + +export const handleSourcegraph: SpecialHandler = async (url: string, timeout: number): Promise => { + try { + const target = parseSourcegraphUrl(url); + if (!target) return null; + + const fetchedAt = new Date().toISOString(); + const notes = ["Fetched via Sourcegraph GraphQL API"]; + + switch (target.type) { + case "search": { + const result = await renderSearch(target.query, timeout); + if (!result.ok) return null; + const output = finalizeOutput(result.content); + return { + url, + finalUrl: url, + contentType: "text/markdown", + method: "sourcegraph-search", + content: output.content, + fetchedAt, + truncated: output.truncated, + notes, + }; + } + case "file": { + const rev = target.rev ?? "HEAD"; + const result = await renderFile(target.repoName, target.filePath, rev, timeout); + if (!result.ok) return null; + const output = finalizeOutput(result.content); + return { + url, + finalUrl: url, + contentType: "text/markdown", + method: "sourcegraph-file", + content: output.content, + fetchedAt, + truncated: output.truncated, + notes, + }; + } + case "repo": { + const result = await renderRepo(target.repoName, timeout); + if (!result.ok) return null; + const output = finalizeOutput(result.content); + return { + url, + finalUrl: url, + contentType: "text/markdown", + method: "sourcegraph-repo", + content: output.content, + fetchedAt, + truncated: output.truncated, + notes, + }; + } + } + } catch {} + + return null; +}; diff --git a/packages/coding-agent/src/core/tools/web-scrapers/spdx.ts b/packages/coding-agent/src/core/tools/web-scrapers/spdx.ts new file mode 100644 index 000000000..f3299938d --- /dev/null +++ b/packages/coding-agent/src/core/tools/web-scrapers/spdx.ts @@ -0,0 +1,116 @@ +import type { RenderResult, SpecialHandler } from "./types"; +import { finalizeOutput, htmlToBasicMarkdown, loadPage } from "./types"; + +interface SpdxCrossRef { + url?: string; + isValid?: boolean; + isLive?: boolean; + match?: string; + order?: number; +} + +interface SpdxLicense { + licenseId: string; + name: string; + isOsiApproved?: boolean; + isFsfLibre?: boolean; + licenseText?: string; + licenseTextHtml?: string; + seeAlso?: string[]; + crossRef?: SpdxCrossRef[]; + comment?: string; + licenseComments?: string; +} + +function formatYesNo(value?: boolean): string { + if (value === true) return "Yes"; + if (value === false) return "No"; + return "Unknown"; +} + +function collectCrossReferences(license: SpdxLicense): string[] { + const ordered = (license.crossRef ?? []) + .filter((ref) => ref.url) + .sort((a, b) => (a.order ?? 0) - (b.order ?? 0)) + .map((ref) => ref.url as string); + + const seeAlso = (license.seeAlso ?? []).filter((url) => url); + const combined = [...ordered, ...seeAlso]; + return combined.filter((url, index) => combined.indexOf(url) === index); +} + +/** + * Handle SPDX license URLs via SPDX JSON API + */ +export const handleSpdx: SpecialHandler = async (url: string, timeout: number): Promise => { + try { + const parsed = new URL(url); + if (parsed.hostname !== "spdx.org" && parsed.hostname !== "www.spdx.org") return null; + + const match = parsed.pathname.match(/^\/licenses\/([^/]+?)(?:\.html)?\/?$/i); + if (!match) return null; + + const licenseId = decodeURIComponent(match[1]); + if (!licenseId) return null; + + const fetchedAt = new Date().toISOString(); + const apiUrl = `https://spdx.org/licenses/${encodeURIComponent(licenseId)}.json`; + const result = await loadPage(apiUrl, { + timeout, + headers: { Accept: "application/json" }, + }); + + if (!result.ok) return null; + + let license: SpdxLicense; + try { + license = JSON.parse(result.content); + } catch { + return null; + } + + const title = license.name || license.licenseId || licenseId; + let md = `# ${title}\n\n`; + + md += `**License ID:** ${license.licenseId ? `\`${license.licenseId}\`` : `\`${licenseId}\``}\n`; + md += `**OSI Approved:** ${formatYesNo(license.isOsiApproved)}\n`; + md += `**FSF Libre:** ${formatYesNo(license.isFsfLibre)}\n`; + + const description = license.licenseComments ?? license.comment; + if (description) { + md += `\n## Description\n\n${description}\n`; + } + + const crossReferences = collectCrossReferences(license); + if (crossReferences.length) { + md += `\n## Cross References\n\n`; + for (const ref of crossReferences) { + md += `- ${ref}\n`; + } + } + + const licenseText = license.licenseText + ? license.licenseText + : license.licenseTextHtml + ? htmlToBasicMarkdown(license.licenseTextHtml) + : null; + + if (licenseText) { + md += `\n## License Text\n\n\`\`\`\n${licenseText}\n\`\`\`\n`; + } + + const output = finalizeOutput(md); + return { + url, + finalUrl: url, + contentType: "text/markdown", + method: "spdx-api", + content: output.content, + fetchedAt, + truncated: output.truncated, + notes: ["Fetched via SPDX license API"], + }; + } catch {} + + return null; +}; diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/spotify.ts b/packages/coding-agent/src/core/tools/web-scrapers/spotify.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/spotify.ts rename to packages/coding-agent/src/core/tools/web-scrapers/spotify.ts diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/stackexchange.test.ts b/packages/coding-agent/src/core/tools/web-scrapers/stackexchange.test.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/stackexchange.test.ts rename to packages/coding-agent/src/core/tools/web-scrapers/stackexchange.test.ts diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/stackoverflow.ts b/packages/coding-agent/src/core/tools/web-scrapers/stackoverflow.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/stackoverflow.ts rename to packages/coding-agent/src/core/tools/web-scrapers/stackoverflow.ts diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/standards.test.ts b/packages/coding-agent/src/core/tools/web-scrapers/standards.test.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/standards.test.ts rename to packages/coding-agent/src/core/tools/web-scrapers/standards.test.ts diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/terraform.ts b/packages/coding-agent/src/core/tools/web-scrapers/terraform.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/terraform.ts rename to packages/coding-agent/src/core/tools/web-scrapers/terraform.ts diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/tldr.ts b/packages/coding-agent/src/core/tools/web-scrapers/tldr.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/tldr.ts rename to packages/coding-agent/src/core/tools/web-scrapers/tldr.ts diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/twitter.ts b/packages/coding-agent/src/core/tools/web-scrapers/twitter.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/twitter.ts rename to packages/coding-agent/src/core/tools/web-scrapers/twitter.ts diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/types.ts b/packages/coding-agent/src/core/tools/web-scrapers/types.ts similarity index 55% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/types.ts rename to packages/coding-agent/src/core/tools/web-scrapers/types.ts index 6dc3b851f..dfdbf4e31 100644 --- a/packages/coding-agent/src/core/tools/web-fetch-handlers/types.ts +++ b/packages/coding-agent/src/core/tools/web-scrapers/types.ts @@ -16,6 +16,61 @@ export interface RenderResult { export type SpecialHandler = (url: string, timeout: number) => Promise; export const MAX_OUTPUT_CHARS = 500_000; +const MAX_BYTES = 50 * 1024 * 1024; + +const USER_AGENTS = [ + "curl/8.0", + "Mozilla/5.0 (compatible; TextBot/1.0)", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36", +]; + +export interface RequestSignal { + signal: AbortSignal; + cleanup: () => void; +} + +export function createRequestSignal(timeoutMs: number, signal?: AbortSignal): RequestSignal { + const controller = new AbortController(); + let timeoutId: ReturnType | undefined = setTimeout(() => controller.abort(), timeoutMs); + const abortHandler = () => controller.abort(); + + if (signal) { + if (signal.aborted) { + clearTimeout(timeoutId); + timeoutId = undefined; + controller.abort(); + } else { + signal.addEventListener("abort", abortHandler, { once: true }); + } + } + + const cleanup = () => { + if (timeoutId !== undefined) { + clearTimeout(timeoutId); + timeoutId = undefined; + } + if (signal) { + signal.removeEventListener("abort", abortHandler); + } + }; + + return { signal: controller.signal, cleanup }; +} + +function isBotBlocked(status: number, content: string): boolean { + if (status === 403 || status === 503) { + const lower = content.toLowerCase(); + return ( + lower.includes("cloudflare") || + lower.includes("captcha") || + lower.includes("challenge") || + lower.includes("blocked") || + lower.includes("access denied") || + lower.includes("bot detection") + ); + } + return false; +} /** * Truncate and cleanup output @@ -29,30 +84,41 @@ export function finalizeOutput(content: string): { content: string; truncated: b }; } +export interface LoadPageOptions { + timeout?: number; + headers?: Record; + method?: string; + body?: string; + maxBytes?: number; + signal?: AbortSignal; +} + +export interface LoadPageResult { + content: string; + contentType: string; + finalUrl: string; + ok: boolean; + status?: number; +} + /** * Fetch a page with timeout and size limit */ -export async function loadPage( - url: string, - options: { timeout?: number; headers?: Record; maxBytes?: number } = {}, -): Promise<{ content: string; contentType: string; finalUrl: string; ok: boolean; status?: number }> { - const { timeout = 20, headers = {}, maxBytes = 50 * 1024 * 1024 } = options; +export async function loadPage(url: string, options: LoadPageOptions = {}): Promise { + const { timeout = 20, headers = {}, maxBytes = MAX_BYTES, signal, method = "GET", body } = options; - const userAgents = [ - "curl/8.0", - "Mozilla/5.0 (compatible; TextBot/1.0)", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36", - ]; + for (let attempt = 0; attempt < USER_AGENTS.length; attempt++) { + if (signal?.aborted) { + return { content: "", contentType: "", finalUrl: url, ok: false }; + } - for (let attempt = 0; attempt < userAgents.length; attempt++) { - const userAgent = userAgents[attempt]; + const userAgent = USER_AGENTS[attempt]; + const { signal: requestSignal, cleanup } = createRequestSignal(timeout * 1000, signal); try { - const controller = new AbortController(); - const timeoutId = setTimeout(() => controller.abort(), timeout * 1000); - - const response = await fetch(url, { - signal: controller.signal, + const requestInit: RequestInit = { + signal: requestSignal, + method, headers: { "User-Agent": userAgent, Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", @@ -60,9 +126,13 @@ export async function loadPage( ...headers, }, redirect: "follow", - }); + }; - clearTimeout(timeoutId); + if (body !== undefined) { + requestInit.body = body; + } + + const response = await fetch(url, requestInit); const contentType = response.headers.get("content-type")?.split(";")[0]?.trim().toLowerCase() ?? ""; const finalUrl = response.url; @@ -91,12 +161,8 @@ export async function loadPage( const decoder = new TextDecoder(); const content = decoder.decode(Buffer.concat(chunks)); - // Check if blocked - if ((response.status === 403 || response.status === 503) && attempt < userAgents.length - 1) { - const lower = content.toLowerCase(); - if (lower.includes("cloudflare") || lower.includes("captcha") || lower.includes("blocked")) { - continue; - } + if (isBotBlocked(response.status, content) && attempt < USER_AGENTS.length - 1) { + continue; } if (!response.ok) { @@ -105,9 +171,14 @@ export async function loadPage( return { content, contentType, finalUrl, ok: true, status: response.status }; } catch (_err) { - if (attempt === userAgents.length - 1) { + if (signal?.aborted) { return { content: "", contentType: "", finalUrl: url, ok: false }; } + if (attempt === USER_AGENTS.length - 1) { + return { content: "", contentType: "", finalUrl: url, ok: false }; + } + } finally { + cleanup(); } } diff --git a/packages/coding-agent/src/core/tools/web-scrapers/utils.ts b/packages/coding-agent/src/core/tools/web-scrapers/utils.ts new file mode 100644 index 000000000..c2030189d --- /dev/null +++ b/packages/coding-agent/src/core/tools/web-scrapers/utils.ts @@ -0,0 +1,161 @@ +import { tmpdir } from "node:os"; +import * as path from "node:path"; +import { ensureTool } from "../../../utils/tools-manager"; +import { createRequestSignal } from "./types"; + +const MAX_BYTES = 50 * 1024 * 1024; // 50MB for binary files + +interface ExecResult { + stdout: string; + stderr: string; + ok: boolean; + exitCode: number; +} + +type SpawnSyncOptions = NonNullable[1]>; + +function exec(cmd: string, args: string[], options?: { timeout?: number; input?: string | Buffer }): ExecResult { + const stdin = (options?.input ?? "ignore") as SpawnSyncOptions["stdin"]; + const result = Bun.spawnSync([cmd, ...args], { + stdin, + stdout: "pipe", + stderr: "pipe", + }); + return { + stdout: result.stdout?.toString() ?? "", + stderr: result.stderr?.toString() ?? "", + ok: result.exitCode === 0, + exitCode: result.exitCode ?? -1, + }; +} + +export interface ConvertResult { + content: string; + ok: boolean; + error?: string; +} + +export interface BinaryFetchResult { + buffer: Buffer; + contentType: string; + contentDisposition?: string; + ok: boolean; + status?: number; + error?: string; +} + +export async function convertWithMarkitdown( + content: Buffer, + extensionHint: string, + timeout: number, + signal?: AbortSignal, +): Promise { + if (signal?.aborted) { + return { content: "", ok: false, error: "aborted" }; + } + + const markitdown = await ensureTool("markitdown", true); + if (!markitdown) { + return { content: "", ok: false, error: "markitdown not available" }; + } + + // Write to temp file with extension hint + const ext = extensionHint || ".bin"; + const tmpDir = tmpdir(); + const tmpFile = path.join(tmpDir, `omp-convert-${Date.now()}${ext}`); + + if (content.length > MAX_BYTES) { + return { content: "", ok: false, error: `content exceeds ${MAX_BYTES} bytes` }; + } + + try { + await Bun.write(tmpFile, content); + const result = exec(markitdown, [tmpFile], { timeout }); + if (!result.ok) { + const stderr = result.stderr.trim(); + return { + content: result.stdout, + ok: false, + error: stderr.length > 0 ? stderr : `markitdown failed (exit ${result.exitCode})`, + }; + } + return { content: result.stdout, ok: true }; + } finally { + try { + await Bun.$`rm ${tmpFile}`.quiet(); + } catch {} + } +} + +export async function fetchBinary(url: string, timeout: number, signal?: AbortSignal): Promise { + if (signal?.aborted) { + return { buffer: Buffer.alloc(0), contentType: "", ok: false, error: "aborted" }; + } + + const { signal: requestSignal, cleanup } = createRequestSignal(timeout * 1000, signal); + + try { + const response = await fetch(url, { + signal: requestSignal, + headers: { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/131.0.0.0", + }, + redirect: "follow", + }); + + const contentType = response.headers.get("content-type") ?? ""; + const contentDisposition = response.headers.get("content-disposition") ?? undefined; + + if (!response.ok) { + return { + buffer: Buffer.alloc(0), + contentType, + contentDisposition, + ok: false, + status: response.status, + error: `status ${response.status}`, + }; + } + + const contentLength = response.headers.get("content-length"); + if (contentLength) { + const size = Number.parseInt(contentLength, 10); + if (Number.isFinite(size) && size > MAX_BYTES) { + return { + buffer: Buffer.alloc(0), + contentType, + contentDisposition, + ok: false, + status: response.status, + error: `content-length ${size} exceeds ${MAX_BYTES}`, + }; + } + } + + const buffer = Buffer.from(await response.arrayBuffer()); + if (buffer.length > MAX_BYTES) { + return { + buffer: Buffer.alloc(0), + contentType, + contentDisposition, + ok: false, + status: response.status, + error: `response exceeds ${MAX_BYTES} bytes`, + }; + } + + return { buffer, contentType, contentDisposition, ok: true, status: response.status }; + } catch (err) { + if (signal?.aborted) { + return { buffer: Buffer.alloc(0), contentType: "", ok: false, error: "aborted" }; + } + return { + buffer: Buffer.alloc(0), + contentType: "", + ok: false, + error: `request failed: ${String(err)}`, + }; + } finally { + cleanup(); + } +} diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/vimeo.ts b/packages/coding-agent/src/core/tools/web-scrapers/vimeo.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/vimeo.ts rename to packages/coding-agent/src/core/tools/web-scrapers/vimeo.ts diff --git a/packages/coding-agent/src/core/tools/web-scrapers/vscode-marketplace.ts b/packages/coding-agent/src/core/tools/web-scrapers/vscode-marketplace.ts new file mode 100644 index 000000000..96d63cfbb --- /dev/null +++ b/packages/coding-agent/src/core/tools/web-scrapers/vscode-marketplace.ts @@ -0,0 +1,193 @@ +import type { RenderResult, SpecialHandler } from "./types"; +import { finalizeOutput, formatCount, loadPage } from "./types"; + +interface MarketplaceProperty { + key?: string; + value?: string; +} + +interface MarketplaceVersion { + version?: string; + properties?: MarketplaceProperty[]; +} + +interface MarketplaceStatistic { + statisticName?: string; + value?: number; +} + +interface MarketplacePublisher { + publisherName?: string; + displayName?: string; +} + +interface MarketplaceExtension { + extensionName?: string; + displayName?: string; + shortDescription?: string; + description?: string; + publisher?: MarketplacePublisher; + versions?: MarketplaceVersion[]; + statistics?: MarketplaceStatistic[]; + categories?: string[]; + tags?: string[]; + properties?: MarketplaceProperty[]; +} + +interface MarketplaceResponse { + results?: Array<{ extensions?: MarketplaceExtension[] }>; +} + +const MARKETPLACE_HOSTS = new Set(["marketplace.visualstudio.com", "www.marketplace.visualstudio.com"]); + +function getItemName(parsed: URL): string | null { + if (!parsed.pathname.startsWith("/items")) return null; + const itemName = parsed.searchParams.get("itemName"); + if (!itemName) return null; + const decoded = decodeURIComponent(itemName); + if (!decoded.includes(".")) return null; + return decoded; +} + +function toStatMap(stats: MarketplaceStatistic[] | undefined): Map { + const map = new Map(); + if (!stats) return map; + for (const stat of stats) { + if (!stat.statisticName || typeof stat.value !== "number") continue; + map.set(stat.statisticName.trim().toLowerCase(), stat.value); + } + return map; +} + +function formatRating(averageRating?: number, ratingCount?: number): string | null { + if (averageRating === undefined && ratingCount === undefined) return null; + if (averageRating !== undefined) { + const formatted = averageRating.toFixed(2).replace(/\.0+$/, "").replace(/\.$/, ""); + if (ratingCount !== undefined) { + return `${formatted} (${formatCount(ratingCount)} ratings)`; + } + return formatted; + } + if (ratingCount !== undefined) { + return `${formatCount(ratingCount)} ratings`; + } + return null; +} + +function extractRepoLink(properties: MarketplaceProperty[] | undefined): string | null { + if (!properties) return null; + for (const prop of properties) { + const key = prop.key?.trim().toLowerCase(); + const value = prop.value?.trim(); + if (!key || !value) continue; + if (!value.startsWith("http")) continue; + if (key.includes("links.source") || key.includes("repository")) return value; + } + for (const prop of properties) { + const key = prop.key?.trim().toLowerCase(); + const value = prop.value?.trim(); + if (!key || !value) continue; + if (!value.startsWith("http")) continue; + if (key === "source" || key.endsWith(".source")) return value; + } + return null; +} + +/** + * Handle VS Code Marketplace URLs via extension query API + */ +export const handleVscodeMarketplace: SpecialHandler = async ( + url: string, + timeout: number, +): Promise => { + try { + const parsed = new URL(url); + if (!MARKETPLACE_HOSTS.has(parsed.hostname)) return null; + + const itemName = getItemName(parsed); + if (!itemName) return null; + + const [publisherFromUrl, ...nameParts] = itemName.split("."); + const extensionFromUrl = nameParts.join("."); + + const fetchedAt = new Date().toISOString(); + const apiUrl = "https://marketplace.visualstudio.com/_apis/public/gallery/extensionquery"; + const payload = JSON.stringify({ + filters: [ + { + criteria: [{ filterType: 7, value: itemName }], + }, + ], + flags: 950, + }); + + const result = await loadPage(apiUrl, { + timeout, + method: "POST", + body: payload, + headers: { + "Content-Type": "application/json", + Accept: "application/json;api-version=7.2-preview.1", + }, + }); + + if (!result.ok) return null; + + let data: MarketplaceResponse; + try { + data = JSON.parse(result.content) as MarketplaceResponse; + } catch { + return null; + } + + const extension = data.results?.[0]?.extensions?.[0]; + if (!extension) return null; + + const extensionName = extension.extensionName ?? extensionFromUrl; + const displayName = extension.displayName ?? extensionName ?? itemName; + const description = extension.shortDescription ?? extension.description; + + const publisherName = extension.publisher?.publisherName ?? publisherFromUrl; + const publisherDisplayName = extension.publisher?.displayName; + const publisherLabel = + publisherDisplayName && publisherName && publisherDisplayName !== publisherName + ? `${publisherDisplayName} (${publisherName})` + : (publisherDisplayName ?? publisherName); + + const version = extension.versions?.[0]?.version; + const statMap = toStatMap(extension.statistics); + const installs = statMap.get("install") ?? statMap.get("installs"); + const averageRating = statMap.get("averagerating"); + const ratingCount = statMap.get("ratingcount"); + const ratingLabel = formatRating(averageRating, ratingCount); + + const repoLink = extractRepoLink(extension.versions?.[0]?.properties) ?? extractRepoLink(extension.properties); + + const identifier = publisherName && extensionName ? `${publisherName}.${extensionName}` : itemName; + + let md = `# ${displayName}\n\n`; + if (description) md += `${description}\n\n`; + md += `**Identifier:** ${identifier}\n`; + if (publisherLabel) md += `**Publisher:** ${publisherLabel}\n`; + if (version) md += `**Version:** ${version}\n`; + if (installs !== undefined) md += `**Installs:** ${formatCount(installs)}\n`; + if (ratingLabel) md += `**Rating:** ${ratingLabel}\n`; + if (extension.categories?.length) md += `**Categories:** ${extension.categories.join(", ")}\n`; + if (extension.tags?.length) md += `**Tags:** ${extension.tags.join(", ")}\n`; + if (repoLink) md += `**Repository:** ${repoLink}\n`; + + const output = finalizeOutput(md); + return { + url, + finalUrl: url, + contentType: "text/markdown", + method: "vscode-marketplace", + content: output.content, + fetchedAt, + truncated: output.truncated, + notes: ["Fetched via VS Code Marketplace API"], + }; + } catch {} + + return null; +}; diff --git a/packages/coding-agent/src/core/tools/web-scrapers/w3c.ts b/packages/coding-agent/src/core/tools/web-scrapers/w3c.ts new file mode 100644 index 000000000..525a108a4 --- /dev/null +++ b/packages/coding-agent/src/core/tools/web-scrapers/w3c.ts @@ -0,0 +1,159 @@ +import type { RenderResult, SpecialHandler } from "./types"; +import { finalizeOutput, htmlToBasicMarkdown, loadPage } from "./types"; + +type JsonRecord = Record; + +function asRecord(value: unknown): JsonRecord | null { + if (!value || typeof value !== "object" || Array.isArray(value)) return null; + return value as JsonRecord; +} + +function getString(record: JsonRecord | null, key: string): string | undefined { + if (!record) return undefined; + const value = record[key]; + return typeof value === "string" ? value : undefined; +} + +function getRecord(record: JsonRecord | null, key: string): JsonRecord | null { + if (!record) return null; + return asRecord(record[key]); +} + +function getArray(record: JsonRecord | null, key: string): unknown[] | undefined { + if (!record) return undefined; + const value = record[key]; + return Array.isArray(value) ? value : undefined; +} + +function extractShortname(pathname: string): string | null { + const trimmed = pathname.replace(/\/+$/g, ""); + const segments = trimmed.split("/").filter(Boolean); + + if (segments.length < 2 || segments[0] !== "TR") return null; + + if (segments.length === 2) { + const shortname = segments[1]; + if (/^\d{4}$/.test(shortname)) return null; + return decodeURIComponent(shortname); + } + + if (segments.length >= 3 && /^\d{4}$/.test(segments[1])) { + const version = segments[2]; + const match = version.match(/^[A-Za-z]+-(.+)-\d{8}$/); + if (match?.[1]) return decodeURIComponent(match[1]); + } + + return null; +} + +function normalizeStatus(status?: string): { code?: string; label?: string } { + if (!status) return {}; + const lower = status.toLowerCase(); + + if (lower.includes("working draft")) return { code: "WD", label: status }; + if (lower.includes("candidate recommendation")) return { code: "CR", label: status }; + if (lower.includes("proposed recommendation")) return { code: "PR", label: status }; + if (lower.includes("recommendation")) return { code: "REC", label: status }; + + return { label: status }; +} + +function extractEditors(editorsPayload: JsonRecord | null): string[] { + const links = getRecord(editorsPayload, "_links"); + const editors = getArray(links, "editors") ?? []; + const names: string[] = []; + + for (const entry of editors) { + const record = asRecord(entry); + const title = getString(record, "title"); + if (title) names.push(title); + } + + return names; +} + +export const handleW3c: SpecialHandler = async (url: string, timeout: number): Promise => { + try { + const parsed = new URL(url); + if (parsed.hostname !== "www.w3.org" && parsed.hostname !== "w3.org") return null; + + const shortname = extractShortname(parsed.pathname); + if (!shortname) return null; + + const fetchedAt = new Date().toISOString(); + + const specUrl = `https://api.w3.org/specifications/${encodeURIComponent(shortname)}`; + const latestUrl = `https://api.w3.org/specifications/${encodeURIComponent(shortname)}/versions/latest`; + + const [specResult, latestResult] = await Promise.all([ + loadPage(specUrl, { timeout, headers: { Accept: "application/json" } }), + loadPage(latestUrl, { timeout, headers: { Accept: "application/json" } }), + ]); + + if (!specResult.ok || !latestResult.ok) return null; + + const specPayload = asRecord(JSON.parse(specResult.content)); + const latestPayload = asRecord(JSON.parse(latestResult.content)); + if (!specPayload || !latestPayload) return null; + + const title = getString(specPayload, "title"); + const shortnameValue = getString(specPayload, "shortname") ?? shortname; + const description = getString(specPayload, "description") ?? getString(specPayload, "abstract"); + const abstract = description ? htmlToBasicMarkdown(description) : undefined; + + const latestVersionUrl = + getString(latestPayload, "uri") ?? + getString(latestPayload, "shortlink") ?? + getString(specPayload, "shortlink"); + + const latestStatus = getString(latestPayload, "status"); + const normalizedStatus = normalizeStatus(latestStatus); + + const specLinks = getRecord(specPayload, "_links"); + const historyUrl = getString(getRecord(specLinks, "version-history"), "href"); + + const latestLinks = getRecord(latestPayload, "_links"); + const editorsUrl = getString(getRecord(latestLinks, "editors"), "href"); + + let editors: string[] = []; + if (editorsUrl) { + const editorsResult = await loadPage(editorsUrl, { timeout: Math.min(timeout, 10) }); + if (editorsResult.ok) { + try { + const editorsPayload = asRecord(JSON.parse(editorsResult.content)); + editors = editorsPayload ? extractEditors(editorsPayload) : []; + } catch {} + } + } + + let md = `# ${title ?? shortnameValue}\n\n`; + if (abstract) md += `## Abstract\n\n${abstract}\n\n`; + + md += "## Metadata\n\n"; + md += `**Shortname:** ${shortnameValue}\n`; + if (normalizedStatus.code) { + md += `**Status:** ${normalizedStatus.code}`; + if (normalizedStatus.label) md += ` (${normalizedStatus.label})`; + md += "\n"; + } else if (normalizedStatus.label) { + md += `**Status:** ${normalizedStatus.label}\n`; + } + if (editors.length) md += `**Editors:** ${editors.join(", ")}\n`; + if (latestVersionUrl) md += `**Latest Version:** ${latestVersionUrl}\n`; + if (historyUrl) md += `**History:** ${historyUrl}\n`; + + const output = finalizeOutput(md); + return { + url, + finalUrl: latestVersionUrl ?? url, + contentType: "text/markdown", + method: "w3c-api", + content: output.content, + fetchedAt, + truncated: output.truncated, + notes: ["Fetched via W3C API"], + }; + } catch {} + + return null; +}; diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/wikidata.ts b/packages/coding-agent/src/core/tools/web-scrapers/wikidata.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/wikidata.ts rename to packages/coding-agent/src/core/tools/web-scrapers/wikidata.ts diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/wikipedia.test.ts b/packages/coding-agent/src/core/tools/web-scrapers/wikipedia.test.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/wikipedia.test.ts rename to packages/coding-agent/src/core/tools/web-scrapers/wikipedia.test.ts diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/wikipedia.ts b/packages/coding-agent/src/core/tools/web-scrapers/wikipedia.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/wikipedia.ts rename to packages/coding-agent/src/core/tools/web-scrapers/wikipedia.ts diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/youtube.test.ts b/packages/coding-agent/src/core/tools/web-scrapers/youtube.test.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/youtube.test.ts rename to packages/coding-agent/src/core/tools/web-scrapers/youtube.test.ts diff --git a/packages/coding-agent/src/core/tools/web-fetch-handlers/youtube.ts b/packages/coding-agent/src/core/tools/web-scrapers/youtube.ts similarity index 100% rename from packages/coding-agent/src/core/tools/web-fetch-handlers/youtube.ts rename to packages/coding-agent/src/core/tools/web-scrapers/youtube.ts