From cd737088311ded5ce0f57fb4d2c2e2254ef7aecb Mon Sep 17 00:00:00 2001 From: can1357 Date: Mon, 5 Jan 2026 01:46:38 +0100 Subject: [PATCH] feat(coding-agent/core): added crates.io API handler with parallel fetching - Added crates.io API handler for fetching Rust crate metadata, versions, and README content. - Added weekly download statistics to npm package fetch results. - Parallelized npm registry and download stats fetches for improved performance. --- .../coding-agent/src/core/tools/web-fetch.ts | 148 +++++++++++++++++- 1 file changed, 145 insertions(+), 3 deletions(-) diff --git a/packages/coding-agent/src/core/tools/web-fetch.ts b/packages/coding-agent/src/core/tools/web-fetch.ts index 9bf105b26..988703d86 100644 --- a/packages/coding-agent/src/core/tools/web-fetch.ts +++ b/packages/coding-agent/src/core/tools/web-fetch.ts @@ -1492,10 +1492,25 @@ async function handleNpm(url: string, timeout: number): Promise= 1_000_000 + ? `${(weeklyDownloads / 1_000_000).toFixed(1)}M` + : weeklyDownloads >= 1_000 + ? `${(weeklyDownloads / 1_000).toFixed(1)}K` + : String(weeklyDownloads); + md += `**Weekly Downloads:** ${formatted}\n`; + } + md += "\n"; if (pkg.homepage) md += `**Homepage:** ${pkg.homepage}\n`; const repoUrl = typeof pkg.repository === "string" ? pkg.repository : pkg.repository?.url; @@ -1555,6 +1580,122 @@ async function handleNpm(url: string, timeout: number): Promise { + try { + const parsed = new URL(url); + if (parsed.hostname !== "crates.io" && parsed.hostname !== "www.crates.io") return null; + + // Extract crate name from /crates/name or /crates/name/version + const match = parsed.pathname.match(/^\/crates\/([^/]+)/); + if (!match) return null; + + const crateName = decodeURIComponent(match[1]); + const fetchedAt = new Date().toISOString(); + + // Fetch from crates.io API + const apiUrl = `https://crates.io/api/v1/crates/${crateName}`; + const result = await loadPage(apiUrl, { + timeout, + headers: { "User-Agent": "omp-web-fetch/1.0 (https://github.com/anthropics)" }, + }); + + if (!result.ok) return null; + + let data: { + crate: { + name: string; + description: string | null; + downloads: number; + recent_downloads: number; + max_version: string; + repository: string | null; + homepage: string | null; + documentation: string | null; + categories: string[]; + keywords: string[]; + created_at: string; + updated_at: string; + }; + versions: Array<{ + num: string; + downloads: number; + created_at: string; + license: string | null; + rust_version: string | null; + }>; + }; + + try { + data = JSON.parse(result.content); + } catch { + return null; + } + + const crate = data.crate; + const latestVersion = data.versions?.[0]; + + // Format download counts + const formatDownloads = (n: number): string => + n >= 1_000_000 + ? `${(n / 1_000_000).toFixed(1)}M` + : n >= 1_000 + ? `${(n / 1_000).toFixed(1)}K` + : String(n); + + let md = `# ${crate.name}\n\n`; + if (crate.description) md += `${crate.description}\n\n`; + + md += `**Latest:** ${crate.max_version}`; + if (latestVersion?.license) md += ` · **License:** ${latestVersion.license}`; + if (latestVersion?.rust_version) md += ` · **MSRV:** ${latestVersion.rust_version}`; + md += "\n"; + md += `**Downloads:** ${formatDownloads(crate.downloads)} total · ${formatDownloads(crate.recent_downloads)} recent\n\n`; + + if (crate.repository) md += `**Repository:** ${crate.repository}\n`; + if (crate.homepage && crate.homepage !== crate.repository) md += `**Homepage:** ${crate.homepage}\n`; + if (crate.documentation) md += `**Docs:** ${crate.documentation}\n`; + if (crate.keywords?.length) md += `**Keywords:** ${crate.keywords.join(", ")}\n`; + if (crate.categories?.length) md += `**Categories:** ${crate.categories.join(", ")}\n`; + + // Show recent versions + if (data.versions?.length > 0) { + md += `\n## Recent Versions\n\n`; + for (const ver of data.versions.slice(0, 5)) { + const date = ver.created_at.split("T")[0]; + md += `- **${ver.num}** (${date}) - ${formatDownloads(ver.downloads)} downloads\n`; + } + } + + // Try to fetch README from docs.rs or repository + const docsRsUrl = `https://docs.rs/crate/${crateName}/${crate.max_version}/source/README.md`; + const readmeResult = await loadPage(docsRsUrl, { timeout: Math.min(timeout, 5) }); + if (readmeResult.ok && readmeResult.content.length > 100 && !looksLikeHtml(readmeResult.content)) { + md += `\n---\n\n## README\n\n${readmeResult.content}\n`; + } + + const output = finalizeOutput(md); + return { + url, + finalUrl: url, + contentType: "text/markdown", + method: "crates.io", + content: output.content, + fetchedAt, + truncated: output.truncated, + notes: ["Fetched via crates.io API"], + }; + } catch {} + + return null; +} + // ============================================================================= // arXiv Special Handling // ============================================================================= @@ -1803,6 +1944,7 @@ async function handleSpecialUrls(url: string, timeout: number): Promise