diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index ea002873f..f42a636e0 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -1,6 +1,13 @@ # Changelog ## [Unreleased] +### Added + +- Added package-level `google-gemini-headers` exports (`getGeminiCliHeaders`, `getGeminiCliUserAgent`, `getAntigravityHeaders`, `extractRetryDelay`, and `ANTIGRAVITY_SYSTEM_INSTRUCTION`) for header and retry handling reuse without importing full Google providers + +### Changed + +- Changed package exports and streaming/provider wiring to load heavy Google/Kimi/GitLab/synthetic provider modules lazily through `register-builtins`, reducing startup import overhead from optional provider SDKs ### Fixed diff --git a/packages/ai/src/index.ts b/packages/ai/src/index.ts index 8e1055a0c..09f9902ee 100644 --- a/packages/ai/src/index.ts +++ b/packages/ai/src/index.ts @@ -10,11 +10,12 @@ export * from "./provider-details"; export * from "./provider-models"; export * from "./providers/anthropic"; export * from "./providers/azure-openai-responses"; -export * from "./providers/cursor"; +export type * from "./providers/cursor"; export * from "./providers/gitlab-duo"; -export * from "./providers/google"; -export * from "./providers/google-gemini-cli"; -export * from "./providers/google-vertex"; +export type * from "./providers/google"; +export type * from "./providers/google-gemini-cli"; +export * from "./providers/google-gemini-headers"; +export type * from "./providers/google-vertex"; export * from "./providers/kimi"; export * from "./providers/ollama"; export type { OpenAICodexResponsesOptions } from "./providers/openai-codex-responses"; diff --git a/packages/ai/src/providers/antigravity-user-agent.ts b/packages/ai/src/providers/antigravity-user-agent.ts new file mode 100644 index 000000000..b65e239a4 --- /dev/null +++ b/packages/ai/src/providers/antigravity-user-agent.ts @@ -0,0 +1,19 @@ +/** + * Antigravity / Cloud Code Assist user agent. Lives in its own file so discovery + * and usage code can read it without pulling the heavy google-gemini-cli provider + * (and its @google/genai → google-auth-library dependency chain) into the startup + * parse graph. + */ +export let getAntigravityUserAgent = () => { + const DEFAULT_ANTIGRAVITY_VERSION = "1.104.0"; + const version = process.env.PI_AI_ANTIGRAVITY_VERSION || DEFAULT_ANTIGRAVITY_VERSION; + // Map Node.js platform/arch to Antigravity's expected format. + // Verified against Antigravity source: _qn() and wqn() in main.js. + // process.platform: win32→windows, others pass through (darwin, linux) + // process.arch: x64→amd64, ia32→386, others pass through (arm64) + const os = process.platform === "win32" ? "windows" : process.platform; + const arch = process.arch === "x64" ? "amd64" : process.arch === "ia32" ? "386" : process.arch; + const userAgent = `antigravity/${version} ${os}/${arch}`; + getAntigravityUserAgent = () => userAgent; + return userAgent; +}; diff --git a/packages/ai/src/providers/gitlab-duo.ts b/packages/ai/src/providers/gitlab-duo.ts index 15e15a2b5..61ffe78ae 100644 --- a/packages/ai/src/providers/gitlab-duo.ts +++ b/packages/ai/src/providers/gitlab-duo.ts @@ -1,11 +1,9 @@ import { ANTHROPIC_THINKING, mapAnthropicToolChoice } from "../stream"; import type { Api, Context, Model, SimpleStreamOptions } from "../types"; import { AssistantMessageEventStream } from "../utils/event-stream"; -import { streamAnthropic } from "./anthropic"; import type { OpenAICompletionsOptions } from "./openai-completions"; -import { streamOpenAICompletions } from "./openai-completions"; import type { OpenAIResponsesOptions } from "./openai-responses"; -import { streamOpenAIResponses } from "./openai-responses"; +import { streamAnthropic, streamOpenAICompletions, streamOpenAIResponses } from "./register-builtins"; import { createProviderErrorMessage } from "./shared/error-message"; const GITLAB_COM_URL = "https://gitlab.com"; diff --git a/packages/ai/src/providers/google-gemini-cli.ts b/packages/ai/src/providers/google-gemini-cli.ts index d49eda5ca..aaf6d24d4 100644 --- a/packages/ai/src/providers/google-gemini-cli.ts +++ b/packages/ai/src/providers/google-gemini-cli.ts @@ -24,6 +24,12 @@ import { refreshAntigravityToken } from "../utils/oauth/google-antigravity"; import { refreshGoogleCloudToken } from "../utils/oauth/google-gemini-cli"; import { extractHttpStatusFromError } from "../utils/retry"; import { sanitizeSchemaForCCA } from "../utils/schema"; +import { + ANTIGRAVITY_SYSTEM_INSTRUCTION, + extractRetryDelay, + getAntigravityHeaders, + getGeminiCliHeaders, +} from "./google-gemini-headers"; import { convertMessages, convertTools, @@ -63,75 +69,16 @@ const ANTIGRAVITY_DAILY_ENDPOINT = "https://daily-cloudcode-pa.googleapis.com"; const ANTIGRAVITY_SANDBOX_ENDPOINT = "https://daily-cloudcode-pa.sandbox.googleapis.com"; const ANTIGRAVITY_ENDPOINT_FALLBACKS = [ANTIGRAVITY_DAILY_ENDPOINT, ANTIGRAVITY_SANDBOX_ENDPOINT] as const; -/** - * Build a User-Agent string that identifies as Gemini CLI to unlock higher rate limits. - * Uses the same format as the official Gemini CLI (v0.35+): - * GeminiCLI/VERSION/MODEL (PLATFORM; ARCH; SURFACE) - */ -export function getGeminiCliUserAgent(modelId = "gemini-3.1-pro-preview"): string { - const version = process.env.PI_AI_GEMINI_CLI_VERSION || "0.35.3"; - const platform = process.platform === "win32" ? "win32" : process.platform; - const arch = process.arch === "x64" ? "x64" : process.arch; - return `GeminiCLI/${version}/${modelId} (${platform}; ${arch}; terminal)`; -} - -const ANTIGRAVITY_USER_AGENT = (() => { - const DEFAULT_ANTIGRAVITY_VERSION = "1.104.0"; - const version = process.env.PI_AI_ANTIGRAVITY_VERSION || DEFAULT_ANTIGRAVITY_VERSION; - // Map Node.js platform/arch to Antigravity's expected format. - // Verified against Antigravity source: _qn() and wqn() in main.js. - // process.platform: win32→windows, others pass through (darwin, linux) - // process.arch: x64→amd64, ia32→386, others pass through (arm64) - const os = process.platform === "win32" ? "windows" : process.platform; - const arch = process.arch === "x64" ? "amd64" : process.arch === "ia32" ? "386" : process.arch; - return `antigravity/${version} ${os}/${arch}`; -})(); - -const GEMINI_CLI_HEADERS = (modelId?: string) => - Object.freeze({ - "User-Agent": getGeminiCliUserAgent(modelId), - "Client-Metadata": "ideType=IDE_UNSPECIFIED,platform=PLATFORM_UNSPECIFIED,pluginType=GEMINI", - }); - -// Antigravity auth headers (project discovery/onboarding). -// Verified from binary: kae.w() and kae.y() send only Content-Type + User-Agent. -// X-Goog-Api-Client and Client-Metadata are NOT sent by the real client — product -// identification (ideType, ideName, ideVersion) goes in the protobuf request body. -const ANTIGRAVITY_AUTH_HEADERS = Object.freeze({ - "User-Agent": ANTIGRAVITY_USER_AGENT, -}); - -// Antigravity executor headers (streaming/generation). -// Same header set as auth calls — only User-Agent per binary analysis. -const ANTIGRAVITY_STREAMING_HEADERS = Object.freeze({ - "User-Agent": ANTIGRAVITY_USER_AGENT, -}); - -// Headers for Gemini CLI (prod endpoint) -export function getGeminiCliHeaders(modelId?: string) { - return GEMINI_CLI_HEADERS(modelId); -} -export function getGeminiCliUserAgentValue(modelId?: string) { - return getGeminiCliUserAgent(modelId); -} - -// Headers for Antigravity (sandbox endpoint) -export function getAntigravityAuthHeaders() { - return ANTIGRAVITY_AUTH_HEADERS; -} -export function getAntigravityHeaders() { - return ANTIGRAVITY_STREAMING_HEADERS; -} -export function getAntigravityUserAgent() { - return ANTIGRAVITY_USER_AGENT; -} - -// Antigravity system instruction (compact version from CLIProxyAPI). -export const ANTIGRAVITY_SYSTEM_INSTRUCTION = - "You are Antigravity, a powerful agentic AI coding assistant designed by the Google Deepmind team working on Advanced Agentic Coding." + - "You are pair programming with a USER to solve their coding task. The task may require creating a new codebase, modifying or debugging an existing codebase, or simply answering a question." + - "**Absolute paths only**" + - "**Proactiveness**"; +export { getAntigravityUserAgent } from "./antigravity-user-agent"; +export { + ANTIGRAVITY_SYSTEM_INSTRUCTION, + extractRetryDelay, + getAntigravityHeaders as getAntigravityAuthHeaders, + getAntigravityHeaders, + getGeminiCliHeaders, + getGeminiCliUserAgent, + getGeminiCliUserAgentValue, +} from "./google-gemini-headers"; // Counter for generating unique tool call IDs let toolCallCounter = 0; @@ -146,106 +93,6 @@ const CLAUDE_THINKING_BETA_HEADER = "interleaved-thinking-2025-05-14"; const GOOGLE_GEMINI_REFRESH_SKEW_MS = 60_000; const ANTIGRAVITY_REFRESH_SKEW_MS = 60_000; -/** - * Extract retry delay from Gemini error response (in milliseconds). - * Checks headers first (Retry-After, x-ratelimit-reset, x-ratelimit-reset-after), - * then parses body patterns like: - * - "Your quota will reset after 39s" - * - "Your quota will reset after 18h31m10s" - * - "Please retry in Xs" or "Please retry in Xms" - * - "retryDelay": "34.074824224s" (JSON field) - */ -export function extractRetryDelay(errorText: string, response?: Response | Headers): number | undefined { - const normalizeDelay = (ms: number): number | undefined => (ms > 0 ? Math.ceil(ms + 1000) : undefined); - - const headers = response instanceof Headers ? response : response?.headers; - if (headers) { - const retryAfter = headers.get("retry-after"); - if (retryAfter) { - const retryAfterSeconds = Number(retryAfter); - if (Number.isFinite(retryAfterSeconds)) { - const delay = normalizeDelay(retryAfterSeconds * 1000); - if (delay !== undefined) { - return delay; - } - } - const retryAfterDate = new Date(retryAfter); - const retryAfterMs = retryAfterDate.getTime(); - if (!Number.isNaN(retryAfterMs)) { - const delay = normalizeDelay(retryAfterMs - Date.now()); - if (delay !== undefined) { - return delay; - } - } - } - - const rateLimitReset = headers.get("x-ratelimit-reset"); - if (rateLimitReset) { - const resetSeconds = Number.parseInt(rateLimitReset, 10); - if (!Number.isNaN(resetSeconds)) { - const delay = normalizeDelay(resetSeconds * 1000 - Date.now()); - if (delay !== undefined) { - return delay; - } - } - } - - const rateLimitResetAfter = headers.get("x-ratelimit-reset-after"); - if (rateLimitResetAfter) { - const resetAfterSeconds = Number(rateLimitResetAfter); - if (Number.isFinite(resetAfterSeconds)) { - const delay = normalizeDelay(resetAfterSeconds * 1000); - if (delay !== undefined) { - return delay; - } - } - } - } - - // Pattern 1: "Your quota will reset after ..." (formats: "18h31m10s", "10m15s", "6s", "39s") - const durationMatch = errorText.match(/reset after (?:(\d+)h)?(?:(\d+)m)?(\d+(?:\.\d+)?)s/i); - if (durationMatch) { - const hours = durationMatch[1] ? parseInt(durationMatch[1], 10) : 0; - const minutes = durationMatch[2] ? parseInt(durationMatch[2], 10) : 0; - const seconds = parseFloat(durationMatch[3]); - if (!Number.isNaN(seconds)) { - const totalMs = ((hours * 60 + minutes) * 60 + seconds) * 1000; - const delay = normalizeDelay(totalMs); - if (delay !== undefined) { - return delay; - } - } - } - - // Pattern 2: "Please retry in X[ms|s]" - const retryInMatch = errorText.match(/Please retry in ([0-9.]+)(ms|s)/i); - if (retryInMatch?.[1]) { - const value = parseFloat(retryInMatch[1]); - if (!Number.isNaN(value) && value > 0) { - const ms = retryInMatch[2].toLowerCase() === "ms" ? value : value * 1000; - const delay = normalizeDelay(ms); - if (delay !== undefined) { - return delay; - } - } - } - - // Pattern 3: "retryDelay": "34.074824224s" (JSON field in error details) - const retryDelayMatch = errorText.match(/"retryDelay":\s*"([0-9.]+)(ms|s)"/i); - if (retryDelayMatch?.[1]) { - const value = parseFloat(retryDelayMatch[1]); - if (!Number.isNaN(value) && value > 0) { - const ms = retryDelayMatch[2].toLowerCase() === "ms" ? value : value * 1000; - const delay = normalizeDelay(ms); - if (delay !== undefined) { - return delay; - } - } - } - - return undefined; -} - function isClaudeModel(modelId: string): boolean { return modelId.toLowerCase().includes("claude"); } diff --git a/packages/ai/src/providers/google-gemini-headers.ts b/packages/ai/src/providers/google-gemini-headers.ts new file mode 100644 index 000000000..5e57f4ce7 --- /dev/null +++ b/packages/ai/src/providers/google-gemini-headers.ts @@ -0,0 +1,152 @@ +/** + * Lightweight header helpers for the Gemini CLI / Antigravity providers. + * + * Extracted from `./google-gemini-cli` so consumers (usage tracking, OAuth) can + * import these helpers without dragging the heavy `@google/genai` → + * `google-auth-library` dependency graph into the CLI startup parse path. + * + * Only depends on `node:` builtins, `process.env`, and `./antigravity-user-agent`. + */ +import { getAntigravityUserAgent } from "./antigravity-user-agent"; + +export { getAntigravityUserAgent } from "./antigravity-user-agent"; + +/** + * Build a User-Agent string that identifies as Gemini CLI to unlock higher rate limits. + * Uses the same format as the official Gemini CLI (v0.35+): + * GeminiCLI/VERSION/MODEL (PLATFORM; ARCH; SURFACE) + */ +export function getGeminiCliUserAgent(modelId = "gemini-3.1-pro-preview"): string { + const version = process.env.PI_AI_GEMINI_CLI_VERSION || "0.35.3"; + const platform = process.platform === "win32" ? "win32" : process.platform; + const arch = process.arch === "x64" ? "x64" : process.arch; + return `GeminiCLI/${version}/${modelId} (${platform}; ${arch}; terminal)`; +} + +const GEMINI_CLI_HEADERS = (modelId?: string) => + Object.freeze({ + "User-Agent": getGeminiCliUserAgent(modelId), + "Client-Metadata": "ideType=IDE_UNSPECIFIED,platform=PLATFORM_UNSPECIFIED,pluginType=GEMINI", + }); + +// Headers for Gemini CLI (prod endpoint) +export function getGeminiCliHeaders(modelId?: string) { + return GEMINI_CLI_HEADERS(modelId); +} +export function getGeminiCliUserAgentValue(modelId?: string) { + return getGeminiCliUserAgent(modelId); +} + +// Headers for Antigravity (sandbox endpoint) +export function getAntigravityHeaders() { + return { + "User-Agent": getAntigravityUserAgent(), + }; +} + +// Antigravity system instruction (compact version from CLIProxyAPI). +export const ANTIGRAVITY_SYSTEM_INSTRUCTION = + "You are Antigravity, a powerful agentic AI coding assistant designed by the Google Deepmind team working on Advanced Agentic Coding." + + "You are pair programming with a USER to solve their coding task. The task may require creating a new codebase, modifying or debugging an existing codebase, or simply answering a question." + + "**Absolute paths only**" + + "**Proactiveness**"; + +/** + * Extract retry delay from Gemini error response (in milliseconds). + * Checks headers first (Retry-After, x-ratelimit-reset, x-ratelimit-reset-after), + * then parses body patterns like: + * - "Your quota will reset after 39s" + * - "Your quota will reset after 18h31m10s" + * - "Please retry in Xs" or "Please retry in Xms" + * - "retryDelay": "34.074824224s" (JSON field) + */ +export function extractRetryDelay(errorText: string, response?: Response | Headers): number | undefined { + const normalizeDelay = (ms: number): number | undefined => (ms > 0 ? Math.ceil(ms + 1000) : undefined); + + const headers = response instanceof Headers ? response : response?.headers; + if (headers) { + const retryAfter = headers.get("retry-after"); + if (retryAfter) { + const retryAfterSeconds = Number(retryAfter); + if (Number.isFinite(retryAfterSeconds)) { + const delay = normalizeDelay(retryAfterSeconds * 1000); + if (delay !== undefined) { + return delay; + } + } + const retryAfterDate = new Date(retryAfter); + const retryAfterMs = retryAfterDate.getTime(); + if (!Number.isNaN(retryAfterMs)) { + const delay = normalizeDelay(retryAfterMs - Date.now()); + if (delay !== undefined) { + return delay; + } + } + } + + const rateLimitReset = headers.get("x-ratelimit-reset"); + if (rateLimitReset) { + const resetSeconds = Number.parseInt(rateLimitReset, 10); + if (!Number.isNaN(resetSeconds)) { + const delay = normalizeDelay(resetSeconds * 1000 - Date.now()); + if (delay !== undefined) { + return delay; + } + } + } + + const rateLimitResetAfter = headers.get("x-ratelimit-reset-after"); + if (rateLimitResetAfter) { + const resetAfterSeconds = Number(rateLimitResetAfter); + if (Number.isFinite(resetAfterSeconds)) { + const delay = normalizeDelay(resetAfterSeconds * 1000); + if (delay !== undefined) { + return delay; + } + } + } + } + + // Pattern 1: "Your quota will reset after ..." (formats: "18h31m10s", "10m15s", "6s", "39s") + const durationMatch = errorText.match(/reset after (?:(\d+)h)?(?:(\d+)m)?(\d+(?:\.\d+)?)s/i); + if (durationMatch) { + const hours = durationMatch[1] ? parseInt(durationMatch[1], 10) : 0; + const minutes = durationMatch[2] ? parseInt(durationMatch[2], 10) : 0; + const seconds = parseFloat(durationMatch[3]); + if (!Number.isNaN(seconds)) { + const totalMs = ((hours * 60 + minutes) * 60 + seconds) * 1000; + const delay = normalizeDelay(totalMs); + if (delay !== undefined) { + return delay; + } + } + } + + // Pattern 2: "Please retry in X[ms|s]" + const retryInMatch = errorText.match(/Please retry in ([0-9.]+)(ms|s)/i); + if (retryInMatch?.[1]) { + const value = parseFloat(retryInMatch[1]); + if (!Number.isNaN(value) && value > 0) { + const ms = retryInMatch[2].toLowerCase() === "ms" ? value : value * 1000; + const delay = normalizeDelay(ms); + if (delay !== undefined) { + return delay; + } + } + } + + // Pattern 3: "retryDelay": "34.074824224s" (JSON field in error details) + const retryDelayMatch = errorText.match(/"retryDelay":\s*"([0-9.]+)(ms|s)"/i); + if (retryDelayMatch?.[1]) { + const value = parseFloat(retryDelayMatch[1]); + if (!Number.isNaN(value) && value > 0) { + const ms = retryDelayMatch[2].toLowerCase() === "ms" ? value : value * 1000; + const delay = normalizeDelay(ms); + if (delay !== undefined) { + return delay; + } + } + } + + return undefined; +} diff --git a/packages/ai/src/providers/kimi.ts b/packages/ai/src/providers/kimi.ts index 0edfbac8c..8ec54e71a 100644 --- a/packages/ai/src/providers/kimi.ts +++ b/packages/ai/src/providers/kimi.ts @@ -13,8 +13,7 @@ import { ANTHROPIC_THINKING } from "../stream"; import type { Api, Context, Model, SimpleStreamOptions } from "../types"; import { AssistantMessageEventStream } from "../utils/event-stream"; import { getKimiCommonHeaders } from "../utils/oauth/kimi"; -import { streamAnthropic } from "./anthropic"; -import { streamOpenAICompletions } from "./openai-completions"; +import { streamAnthropic, streamOpenAICompletions } from "./register-builtins"; import { createProviderErrorMessage } from "./shared/error-message"; export type KimiApiFormat = "openai" | "anthropic"; diff --git a/packages/ai/src/providers/openai-completions-compat.ts b/packages/ai/src/providers/openai-completions-compat.ts index 5d91d89aa..5dddbf4f3 100644 --- a/packages/ai/src/providers/openai-completions-compat.ts +++ b/packages/ai/src/providers/openai-completions-compat.ts @@ -108,7 +108,7 @@ export function detectOpenAICompat(model: Model<"openai-completions">, resolvedB high: "default", xhigh: "default", } satisfies Partial>) - : isDeepseekFamily && Boolean(model.reasoning) + : isDeepseekFamily && model.reasoning ? { xhigh: "max" } : {}; @@ -145,7 +145,7 @@ export function detectOpenAICompat(model: Model<"openai-completions">, resolvedB ((provider === "openrouter" || baseUrl.includes("openrouter.ai")) && Boolean(model.reasoning)), // DeepSeek V4 rejects synthetic reasoning_content placeholders (".") on tool-call turns. // Kimi and OpenRouter accept them when actual reasoning is unavailable. - allowsSyntheticReasoningContentForToolCalls: !isDeepseekFamily || !Boolean(model.reasoning), + allowsSyntheticReasoningContentForToolCalls: !isDeepseekFamily || !model.reasoning, requiresAssistantContentForToolCalls: isKimiModel, openRouterRouting: undefined, vercelGatewayRouting: undefined, diff --git a/packages/ai/src/providers/synthetic.ts b/packages/ai/src/providers/synthetic.ts index 476e060b5..7d6540e24 100644 --- a/packages/ai/src/providers/synthetic.ts +++ b/packages/ai/src/providers/synthetic.ts @@ -11,8 +11,7 @@ import { ANTHROPIC_THINKING } from "../stream"; import type { Api, Context, Model, SimpleStreamOptions } from "../types"; import { AssistantMessageEventStream } from "../utils/event-stream"; -import { streamAnthropic } from "./anthropic"; -import { streamOpenAICompletions } from "./openai-completions"; +import { streamAnthropic, streamOpenAICompletions } from "./register-builtins"; import { createProviderErrorMessage } from "./shared/error-message"; export type SyntheticApiFormat = "openai" | "anthropic"; diff --git a/packages/ai/src/stream.ts b/packages/ai/src/stream.ts index 8ffe9804b..f7887739d 100644 --- a/packages/ai/src/stream.ts +++ b/packages/ai/src/stream.ts @@ -9,19 +9,37 @@ import { mapEffortToGoogleThinkingLevel, requireSupportedEffort, } from "./model-thinking"; -import { type BedrockOptions, streamBedrock } from "./providers/amazon-bedrock"; -import { type AnthropicOptions, streamAnthropic } from "./providers/anthropic"; -import { streamAzureOpenAIResponses } from "./providers/azure-openai-responses"; -import { type CursorOptions, streamCursor } from "./providers/cursor"; +import type { BedrockOptions } from "./providers/amazon-bedrock"; +import type { AnthropicOptions } from "./providers/anthropic"; +import type { CursorOptions } from "./providers/cursor"; import { isGitLabDuoModel, streamGitLabDuo } from "./providers/gitlab-duo"; -import { type GoogleOptions, streamGoogle } from "./providers/google"; -import { type GoogleGeminiCliOptions, streamGoogleGeminiCli } from "./providers/google-gemini-cli"; -import { type GoogleVertexOptions, streamGoogleVertex } from "./providers/google-vertex"; +import type { GoogleOptions } from "./providers/google"; +import type { GoogleGeminiCliOptions } from "./providers/google-gemini-cli"; +import type { GoogleVertexOptions } from "./providers/google-vertex"; import { isKimiModel, streamKimi } from "./providers/kimi"; -import { type OllamaChatOptions, streamOllama } from "./providers/ollama"; -import { streamOpenAICodexResponses } from "./providers/openai-codex-responses"; -import { type OpenAICompletionsOptions, streamOpenAICompletions } from "./providers/openai-completions"; -import { streamOpenAIResponses } from "./providers/openai-responses"; +import type { OllamaChatOptions } from "./providers/ollama"; +import type { OpenAICompletionsOptions } from "./providers/openai-completions"; +// Heavy provider stream functions are imported lazily via register-builtins, +// which wraps each provider module in a dynamic import. This keeps the +// AWS SDK, google-auth-library, @google/genai, @bufbuild/protobuf, and +// other provider SDKs out of the CLI startup parse graph. The +// gitlab-duo / kimi / synthetic providers stay eager because their modules +// export routing predicates (isGitLabDuoModel, isKimiModel, isSyntheticModel) +// that must be callable synchronously before streaming begins, and their +// modules are thin wrappers with no heavy SDK dependencies. +import { + streamAnthropic, + streamAzureOpenAIResponses, + streamBedrock, + streamCursor, + streamGoogle, + streamGoogleGeminiCli, + streamGoogleVertex, + streamOllama, + streamOpenAICodexResponses, + streamOpenAICompletions, + streamOpenAIResponses, +} from "./providers/register-builtins"; import { isSyntheticModel, streamSynthetic } from "./providers/synthetic"; import type { Api, diff --git a/packages/ai/src/utils/oauth/google-antigravity.ts b/packages/ai/src/utils/oauth/google-antigravity.ts index 769c96a85..e659dbfba 100644 --- a/packages/ai/src/utils/oauth/google-antigravity.ts +++ b/packages/ai/src/utils/oauth/google-antigravity.ts @@ -2,7 +2,7 @@ * Antigravity OAuth flow (Gemini 3, Claude, GPT-OSS via Google Cloud) * Uses different OAuth credentials than google-gemini-cli for access to additional models. */ -import { getAntigravityAuthHeaders } from "../../providers/google-gemini-cli"; +import { getAntigravityHeaders } from "../../providers/google-gemini-headers"; import { runGoogleOAuthLogin } from "./google-oauth-shared"; import type { OAuthController, OAuthCredentials } from "./types"; @@ -112,7 +112,7 @@ async function discoverProject(accessToken: string, onProgress?: (message: strin const headers = { Authorization: `Bearer ${accessToken}`, "Content-Type": "application/json", - ...getAntigravityAuthHeaders(), + ...getAntigravityHeaders(), }; onProgress?.("Checking for existing project..."); diff --git a/packages/ai/src/utils/oauth/google-gemini-cli.ts b/packages/ai/src/utils/oauth/google-gemini-cli.ts index 625fa2b3d..e3ea1e7c3 100644 --- a/packages/ai/src/utils/oauth/google-gemini-cli.ts +++ b/packages/ai/src/utils/oauth/google-gemini-cli.ts @@ -4,7 +4,7 @@ */ import { $env } from "@oh-my-pi/pi-utils"; -import { getGeminiCliHeaders } from "../../providers/google-gemini-cli"; +import { getGeminiCliHeaders } from "../../providers/google-gemini-headers"; import { runGoogleOAuthLogin } from "./google-oauth-shared"; import type { OAuthController, OAuthCredentials } from "./types"; diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index afd082d6c..a6608fa75 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -1,6 +1,13 @@ # Changelog ## [Unreleased] +### Changed + +- Changed markdown conversion and archive tooling to defer loading heavy dependencies (Turndown, fflate, and browser agent content) until first use, reducing startup overhead for CLI startup and command initialization + +### Fixed + +- Fixed changelog state tracking by flushing `lastChangelogVersion` to settings immediately when showing new entries, so the updated version is persisted across restarts ## [14.5.13] - 2026-05-01 diff --git a/packages/coding-agent/src/commit/pipeline.ts b/packages/coding-agent/src/commit/pipeline.ts index 4062f2ad6..7badfe4b7 100644 --- a/packages/coding-agent/src/commit/pipeline.ts +++ b/packages/coding-agent/src/commit/pipeline.ts @@ -25,7 +25,8 @@ import type { CommitCommandArgs, ConventionalAnalysis } from "./types"; const SUMMARY_MAX_CHARS = 72; const RECENT_COMMITS_COUNT = 8; -const TYPES_DESCRIPTION = prompt.render(typesDescriptionPrompt); +let _typesDescription: string | undefined; +const TYPES_DESCRIPTION = (): string => (_typesDescription ??= prompt.render(typesDescriptionPrompt)); /** * Execute the omp commit pipeline for staged changes. @@ -176,7 +177,7 @@ async function generateAnalysis(input: { diff: input.diff, stat: input.stat, scopeCandidates: input.scopeCandidates, - typesDescription: TYPES_DESCRIPTION, + typesDescription: TYPES_DESCRIPTION(), settings: { enabled: input.commitSettings.mapReduceEnabled, minFiles: input.commitSettings.mapReduceMinFiles, @@ -193,7 +194,7 @@ async function generateAnalysis(input: { thinkingLevel: input.primaryThinkingLevel, contextFiles: input.contextFiles, userContext: input.userContext, - typesDescription: TYPES_DESCRIPTION, + typesDescription: TYPES_DESCRIPTION(), recentCommits: input.recentCommits, scopeCandidates: input.scopeCandidates, stat: input.stat, diff --git a/packages/coding-agent/src/config.ts b/packages/coding-agent/src/config.ts index 9872fa7b6..488928f5d 100644 --- a/packages/coding-agent/src/config.ts +++ b/packages/coding-agent/src/config.ts @@ -11,7 +11,7 @@ import { } from "@oh-my-pi/pi-utils"; import type { TSchema } from "@sinclair/typebox"; import { Value } from "@sinclair/typebox/value"; -import { Ajv, type ErrorObject, type ValidateFunction } from "ajv"; +import type { ErrorObject } from "ajv"; import { JSONC, YAML } from "bun"; import { expandTilde } from "./tools/path-utils"; @@ -143,7 +143,6 @@ export type LoadResult = | { value: T; error?: undefined; status: "ok" } | { value?: null; error?: unknown; status: "not-found" }; -const ajv = new Ajv(); export class ConfigFile implements IConfigFile { readonly #basePath: string; #cache?: LoadResult; @@ -221,13 +220,17 @@ export class ConfigFile implements IConfigFile { throw new Error(`Invalid config file path: ${this.#basePath}`); } - const validate = ajv.compile(this.schema) as ValidateFunction; - if (!validate(parsed)) { - const error = new ConfigError(this.id, validate.errors); + if (!Value.Check(this.schema, parsed)) { + const schemaErrors: ErrorObject[] = []; + for (const err of Value.Errors(this.schema, parsed)) { + schemaErrors.push({ instancePath: err.path, message: err.message } as ErrorObject); + if (schemaErrors.length >= 50) break; + } + const error = new ConfigError(this.id, schemaErrors); logger.warn("Failed to parse config file", { path: this.path(), error }); return this.#storeCache({ error, status: "error" }); } - return this.#storeCache({ value: parsed, status: "ok" }); + return this.#storeCache({ value: parsed as T, status: "ok" }); } catch (error) { if (isEnoent(error)) { return this.#storeCache({ status: "not-found" }); diff --git a/packages/coding-agent/src/eval/py/kernel.ts b/packages/coding-agent/src/eval/py/kernel.ts index 7455e0ed5..d6845dc00 100644 --- a/packages/coding-agent/src/eval/py/kernel.ts +++ b/packages/coding-agent/src/eval/py/kernel.ts @@ -271,10 +271,10 @@ function normalizeDisplayText(text: string): string { } /** Renders a Jupyter display_data message into text and structured outputs. */ -export function renderKernelDisplay(content: Record): { +export async function renderKernelDisplay(content: Record): Promise<{ text: string; outputs: KernelDisplayOutput[]; -} { +}> { const data = content.data as Record | undefined; if (!data) return { text: "", outputs: [] }; @@ -307,7 +307,7 @@ export function renderKernelDisplay(content: Record): { return { text: normalizeDisplayText(String(data["text/plain"])), outputs }; } if (data["text/html"] !== undefined) { - const markdown = htmlToBasicMarkdown(String(data["text/html"])) || ""; + const markdown = (await htmlToBasicMarkdown(String(data["text/html"]))) || ""; return { text: markdown ? normalizeDisplayText(markdown) : "", outputs }; } return { text: "", outputs }; @@ -872,7 +872,7 @@ export class PythonKernel { } case "execute_result": case "display_data": { - const { text, outputs } = renderKernelDisplay(response.content); + const { text, outputs } = await renderKernelDisplay(response.content); if (text && options?.onChunk) { await options.onChunk(text); } diff --git a/packages/coding-agent/src/main.ts b/packages/coding-agent/src/main.ts index abc126f5f..69e682f45 100644 --- a/packages/coding-agent/src/main.ts +++ b/packages/coding-agent/src/main.ts @@ -253,12 +253,14 @@ async function getChangelogForDisplay(parsed: Args): Promise if (!lastVersion) { if (entries.length > 0) { settings.set("lastChangelogVersion", VERSION); + await flushChangelogVersion(); return entries.map(e => e.content).join("\n\n"); } } else { const newEntries = getNewEntries(entries, lastVersion); if (newEntries.length > 0) { settings.set("lastChangelogVersion", VERSION); + await flushChangelogVersion(); return newEntries.map(e => e.content).join("\n\n"); } } @@ -266,6 +268,14 @@ async function getChangelogForDisplay(parsed: Args): Promise return undefined; } +async function flushChangelogVersion(): Promise { + try { + await settings.flush(); + } catch (error: unknown) { + logger.warn("Failed to persist lastChangelogVersion", { error }); + } +} + async function createSessionManager(parsed: Args, cwd: string): Promise { if (parsed.fork) { if (parsed.noSession) { diff --git a/packages/coding-agent/src/sdk.ts b/packages/coding-agent/src/sdk.ts index abfddafd6..5c368ef53 100644 --- a/packages/coding-agent/src/sdk.ts +++ b/packages/coding-agent/src/sdk.ts @@ -300,7 +300,6 @@ function getDefaultAgentDir(): string { */ export async function discoverAuthStorage(agentDir: string = getDefaultAgentDir()): Promise { const dbPath = getAgentDbPath(agentDir); - logger.debug("discoverAuthStorage", { agentDir, dbPath }); const storage = await AuthStorage.create(dbPath, { configValueResolver: resolveConfigValue }); await storage.reload(); diff --git a/packages/coding-agent/src/task/agents.ts b/packages/coding-agent/src/task/agents.ts index 9118dd385..2f568a1d6 100644 --- a/packages/coding-agent/src/task/agents.ts +++ b/packages/coding-agent/src/task/agents.ts @@ -69,10 +69,7 @@ const EMBEDDED_AGENT_DEFS: EmbeddedAgentDef[] = [ }, ]; -const EMBEDDED_AGENTS: { name: string; content: string }[] = EMBEDDED_AGENT_DEFS.map(def => ({ - name: def.fileName, - content: buildAgentContent(def), -})); +// Computed lazily on first loadBundledAgents() call to avoid eager prompt.render at module load. export class AgentParsingError extends Error { constructor( @@ -133,7 +130,9 @@ export function loadBundledAgents(): AgentDefinition[] { if (bundledAgentsCache !== null) { return bundledAgentsCache; } - bundledAgentsCache = EMBEDDED_AGENTS.map(({ name, content }) => parseAgent(`embedded:${name}`, content, "bundled")); + bundledAgentsCache = EMBEDDED_AGENT_DEFS.map(def => + parseAgent(`embedded:${def.fileName}`, buildAgentContent(def), "bundled"), + ); return bundledAgentsCache; } diff --git a/packages/coding-agent/src/tools/archive-reader.ts b/packages/coding-agent/src/tools/archive-reader.ts index 56f49f3fa..eaf4c81b2 100644 --- a/packages/coding-agent/src/tools/archive-reader.ts +++ b/packages/coding-agent/src/tools/archive-reader.ts @@ -1,6 +1,11 @@ -import { unzipSync } from "fflate"; import { ToolError } from "./tool-errors"; +let fflateModulePromise: Promise | undefined; +async function loadFflate(): Promise { + if (!fflateModulePromise) fflateModulePromise = import("fflate"); + return fflateModulePromise; +} + export type ArchiveFormat = "zip" | "tar" | "tar.gz"; export interface ArchivePathCandidate { @@ -150,7 +155,8 @@ async function readTarEntries(bytes: Uint8Array): Promise { return entries; } -function readZipEntries(bytes: Uint8Array): ArchiveIndexEntry[] { +async function readZipEntries(bytes: Uint8Array): Promise { + const { unzipSync } = await loadFflate(); let files: Record; try { files = unzipSync(bytes); @@ -310,6 +316,6 @@ export async function openArchive(filePath: string): Promise { } const bytes = await Bun.file(filePath).bytes(); - const entries = format === "zip" ? readZipEntries(bytes) : await readTarEntries(bytes); + const entries = format === "zip" ? await readZipEntries(bytes) : await readTarEntries(bytes); return new ArchiveReader(format, entries); } diff --git a/packages/coding-agent/src/tools/browser.ts b/packages/coding-agent/src/tools/browser.ts index 1376beef0..08506afdd 100644 --- a/packages/coding-agent/src/tools/browser.ts +++ b/packages/coding-agent/src/tools/browser.ts @@ -110,12 +110,14 @@ function resolveBrowserKind(params: BrowserParams, session: ToolSession): Browse export class BrowserTool implements AgentTool { readonly name = "browser"; readonly label = "Browser"; - readonly description: string; readonly parameters = browserSchema; readonly strict = true; - constructor(private readonly session: ToolSession) { - this.description = prompt.render(browserDescription, {}); + constructor(private readonly session: ToolSession) {} + #description?: string; + get description(): string { + this.#description ??= prompt.render(browserDescription, {}); + return this.#description; } /** Restart browser to apply mode changes (e.g. headless toggle). Drops only headless browsers. */ diff --git a/packages/coding-agent/src/tools/browser/readable.ts b/packages/coding-agent/src/tools/browser/readable.ts index fa4505478..b3b6c3589 100644 --- a/packages/coding-agent/src/tools/browser/readable.ts +++ b/packages/coding-agent/src/tools/browser/readable.ts @@ -26,13 +26,17 @@ function normalize(text: string | null | undefined): string | undefined { * CSS selector chain over the same pre-parsed DOM. Returns null if neither * path yields usable content. */ -export function extractReadableFromHtml(html: string, url: string, format: ReadableFormat): ReadableResult | null { +export async function extractReadableFromHtml( + html: string, + url: string, + format: ReadableFormat, +): Promise { const { document } = parseHTML(html); // --- Primary: Readability article extraction --- const article = new Readability(document).parse(); if (article) { - const result = toReadableResult(url, format, article.textContent, article.content, { + const result = await toReadableResult(url, format, article.textContent, article.content, { title: article.title, byline: article.byline, excerpt: article.excerpt, @@ -55,7 +59,7 @@ export function extractReadableFromHtml(html: string, url: string, format: Reada const innerHTML = el.innerHTML?.trim(); const textContent = el.textContent?.trim(); if (!innerHTML || !textContent) continue; - const result = toReadableResult(url, format, textContent, innerHTML, { + const result = await toReadableResult(url, format, textContent, innerHTML, { title: document.title, excerpt: textContent.slice(0, 240), length: textContent.length, @@ -67,15 +71,16 @@ export function extractReadableFromHtml(html: string, url: string, format: Reada } /** Shared builder for both extraction paths. */ -function toReadableResult( +async function toReadableResult( url: string, format: ReadableFormat, textContent: string | null | undefined, htmlContent: string | null | undefined, meta: { title?: string | null; byline?: string | null; excerpt?: string | null; length?: number | null }, -): ReadableResult | null { +): Promise { const text = normalize(textContent); - const markdown = format === "markdown" ? (normalize(htmlToBasicMarkdown(htmlContent ?? "")) ?? text) : undefined; + const markdown = + format === "markdown" ? (normalize(await htmlToBasicMarkdown(htmlContent ?? "")) ?? text) : undefined; const normalizedText = format === "text" ? text : undefined; if (!normalizedText && !markdown) return null; return { diff --git a/packages/coding-agent/src/tools/browser/tab-supervisor.ts b/packages/coding-agent/src/tools/browser/tab-supervisor.ts index 7c2b67bf6..96bad2ae6 100644 --- a/packages/coding-agent/src/tools/browser/tab-supervisor.ts +++ b/packages/coding-agent/src/tools/browser/tab-supervisor.ts @@ -16,7 +16,6 @@ import type { WorkerInitPayload, WorkerOutbound, } from "./tab-protocol"; -import { WorkerCore } from "./tab-worker"; interface WorkerHandle { send(msg: WorkerInbound, transferList?: Transferable[]): void; @@ -398,7 +397,7 @@ function wrapBunWorker(worker: Worker): WorkerHandle { * entry. This preserves normal browser behavior but cannot interrupt synchronous * infinite loops because user code runs on the main thread. */ -function spawnInlineWorker(): WorkerHandle { +async function spawnInlineWorker(): Promise { const hostListeners = new Set<(message: WorkerOutbound) => void>(); const workerListeners = new Set<(message: WorkerInbound) => void>(); const workerTransport: Transport = { @@ -413,6 +412,7 @@ function spawnInlineWorker(): WorkerHandle { }, close: () => {}, }; + const { WorkerCore } = await import("./tab-worker"); new WorkerCore(workerTransport); return { mode: "inline", diff --git a/packages/coding-agent/src/tools/write.ts b/packages/coding-agent/src/tools/write.ts index 161c96483..a494a09b4 100644 --- a/packages/coding-agent/src/tools/write.ts +++ b/packages/coding-agent/src/tools/write.ts @@ -6,7 +6,6 @@ import type { Component } from "@oh-my-pi/pi-tui"; import { Text } from "@oh-my-pi/pi-tui"; import { isEnoent, isRecord, prompt, untilAborted } from "@oh-my-pi/pi-utils"; import { type Static, Type } from "@sinclair/typebox"; -import { unzipSync, zipSync } from "fflate"; import { stripHashlinePrefixes } from "../edit"; import type { RenderResultOptions } from "../extensibility/custom-tools/types"; import { createLspWritethrough, type FileDiagnosticsResult, type WritethroughCallback, writethroughNoop } from "../lsp"; @@ -44,6 +43,12 @@ import { import { ToolError } from "./tool-errors"; import { toolResult } from "./tool-result"; +let fflateModulePromise: Promise | undefined; +async function loadFflate(): Promise { + if (!fflateModulePromise) fflateModulePromise = import("fflate"); + return fflateModulePromise; +} + const writeSchema = Type.Object({ path: Type.String({ description: "file path", examples: ["src/new.ts"] }), content: Type.String({ description: "file content" }), @@ -229,6 +234,7 @@ export class WriteTool implements AgentTool { if (!abstract) return null; const normalized = abstract.replace(/<\/?jats:p[^>]*>/g, match => (match.startsWith("" : "

")); - const markdown = htmlToBasicMarkdown(normalized); + const markdown = await htmlToBasicMarkdown(normalized); return markdown.trim().length > 0 ? markdown : null; } @@ -114,7 +114,7 @@ export const handleCrossref: SpecialHandler = async ( formatDate(message.issued) || formatDate(message.created); const doiValue = message.DOI || doi; - const abstract = formatAbstract(message.abstract); + const abstract = await formatAbstract(message.abstract); const type = message.type?.replace(/-/g, " "); let md = `# ${title}\n\n`; diff --git a/packages/coding-agent/src/web/scrapers/devto.ts b/packages/coding-agent/src/web/scrapers/devto.ts index fc237e3dc..37430b413 100644 --- a/packages/coding-agent/src/web/scrapers/devto.ts +++ b/packages/coding-agent/src/web/scrapers/devto.ts @@ -133,7 +133,7 @@ export const handleDevTo: SpecialHandler = async ( if (article.body_markdown) { md += article.body_markdown; } else if (article.body_html) { - md += htmlToBasicMarkdown(article.body_html); + md += await htmlToBasicMarkdown(article.body_html); } notes.push("Fetched via dev.to API"); diff --git a/packages/coding-agent/src/web/scrapers/discourse.ts b/packages/coding-agent/src/web/scrapers/discourse.ts index 61e472b4b..864fca4a4 100644 --- a/packages/coding-agent/src/web/scrapers/discourse.ts +++ b/packages/coding-agent/src/web/scrapers/discourse.ts @@ -77,12 +77,12 @@ function formatCategory(topic: DiscourseTopic): string | null { return parts.length ? parts.join(" ") : null; } -function formatPostBody(post: DiscoursePost): string { +async function formatPostBody(post: DiscoursePost): Promise { const raw = post.raw?.trim(); if (raw) return raw; const cooked = post.cooked?.trim(); if (!cooked) return ""; - return htmlToBasicMarkdown(cooked); + return await htmlToBasicMarkdown(cooked); } function buildTopicUrl(baseUrl: string, topicId: string): string { @@ -168,9 +168,9 @@ export const handleDiscourse: SpecialHandler = async ( md += "\n"; const description = topic.excerpt - ? htmlToBasicMarkdown(topic.excerpt) + ? await htmlToBasicMarkdown(topic.excerpt) : posts.length - ? formatPostBody(posts[0]) + ? await formatPostBody(posts[0]) : ""; if (description) { md += `## Description\n\n${description}\n\n`; @@ -182,7 +182,7 @@ export const handleDiscourse: SpecialHandler = async ( const author = formatAuthor({ name: post.name, username: post.username }); const date = formatIsoDate(post.created_at); const likes = post.like_count ?? 0; - const content = formatPostBody(post); + const content = await formatPostBody(post); const postLabel = post.post_number != null ? `Post ${post.post_number}` : `Post ${post.id}`; md += `### ${postLabel} - ${author} - ${date} - Likes: ${likes}\n\n`; diff --git a/packages/coding-agent/src/web/scrapers/firefox-addons.ts b/packages/coding-agent/src/web/scrapers/firefox-addons.ts index f3394f649..a36976b1b 100644 --- a/packages/coding-agent/src/web/scrapers/firefox-addons.ts +++ b/packages/coding-agent/src/web/scrapers/firefox-addons.ts @@ -112,7 +112,7 @@ export const handleFirefoxAddons: SpecialHandler = async ( const name = getLocalizedText(data.name, defaultLocale) ?? slug; const summary = getLocalizedText(data.summary, defaultLocale); const descriptionRaw = getLocalizedText(data.description, defaultLocale); - const description = descriptionRaw ? htmlToBasicMarkdown(descriptionRaw) : undefined; + const description = descriptionRaw ? await htmlToBasicMarkdown(descriptionRaw) : undefined; const authors = (data.authors ?? []) .map(author => author.name ?? "") diff --git a/packages/coding-agent/src/web/scrapers/flathub.ts b/packages/coding-agent/src/web/scrapers/flathub.ts index 79ba5f667..a88f17295 100644 --- a/packages/coding-agent/src/web/scrapers/flathub.ts +++ b/packages/coding-agent/src/web/scrapers/flathub.ts @@ -170,7 +170,7 @@ export const handleFlathub: SpecialHandler = async ( } if (app.description) { - const description = htmlToBasicMarkdown(app.description); + const description = await htmlToBasicMarkdown(app.description); if (description) md += `\n## Description\n\n${description}\n`; } @@ -204,7 +204,7 @@ export const handleFlathub: SpecialHandler = async ( md += `${line}\n`; if (release.description) { - const releaseDesc = htmlToBasicMarkdown(release.description).replace(/\n+/g, " ").trim(); + const releaseDesc = (await htmlToBasicMarkdown(release.description)).replace(/\n+/g, " ").trim(); if (releaseDesc) md += ` - ${releaseDesc}\n`; } } diff --git a/packages/coding-agent/src/web/scrapers/gitlab.ts b/packages/coding-agent/src/web/scrapers/gitlab.ts index 0e04debb7..caec243ae 100644 --- a/packages/coding-agent/src/web/scrapers/gitlab.ts +++ b/packages/coding-agent/src/web/scrapers/gitlab.ts @@ -259,7 +259,7 @@ async function renderGitLabIssue( } md += `\n---\n\n## Description\n\n`; - md += issue.description ? htmlToBasicMarkdown(issue.description) : "*No description*"; + md += issue.description ? await htmlToBasicMarkdown(issue.description) : "*No description*"; return { content: md, ok: true }; } diff --git a/packages/coding-agent/src/web/scrapers/go-pkg.ts b/packages/coding-agent/src/web/scrapers/go-pkg.ts index af6cfcc1b..db9e36214 100644 --- a/packages/coding-agent/src/web/scrapers/go-pkg.ts +++ b/packages/coding-agent/src/web/scrapers/go-pkg.ts @@ -159,7 +159,7 @@ export const handleGoPkg: SpecialHandler = async ( // Get overview paragraph const overview = docSection.querySelector(".go-Message"); if (overview) { - const overviewMd = htmlToBasicMarkdown(overview.innerHTML); + const overviewMd = await htmlToBasicMarkdown(overview.innerHTML); sections.push(overviewMd); sections.push(""); } @@ -172,7 +172,7 @@ export const handleGoPkg: SpecialHandler = async ( const docParts: string[] = []; for (let i = 0; i < Math.min(3, paragraphs.length); i++) { const p = paragraphs[i]; - const text = htmlToBasicMarkdown(p.innerHTML).trim(); + const text = (await htmlToBasicMarkdown(p.innerHTML)).trim(); if (text) { docParts.push(text); } diff --git a/packages/coding-agent/src/web/scrapers/jetbrains-marketplace.ts b/packages/coding-agent/src/web/scrapers/jetbrains-marketplace.ts index 13aa03c5e..99d3ec22e 100644 --- a/packages/coding-agent/src/web/scrapers/jetbrains-marketplace.ts +++ b/packages/coding-agent/src/web/scrapers/jetbrains-marketplace.ts @@ -108,7 +108,7 @@ export const handleJetBrainsMarketplace: SpecialHandler = async ( const vendorName = plugin.vendor?.name ?? plugin.vendor?.publicName; const descriptionSource = plugin.description ?? plugin.preview ?? ""; - const description = descriptionSource ? htmlToBasicMarkdown(descriptionSource) : ""; + const description = descriptionSource ? await htmlToBasicMarkdown(descriptionSource) : ""; const tags = (plugin.tags ?? []).map(tag => tag.name).filter(Boolean) as string[]; const rating = extractRating(plugin); const buildCompatibility = update ? formatBuildCompatibility(update) : null; diff --git a/packages/coding-agent/src/web/scrapers/mastodon.ts b/packages/coding-agent/src/web/scrapers/mastodon.ts index b7297f1d8..0fdfb6451 100644 --- a/packages/coding-agent/src/web/scrapers/mastodon.ts +++ b/packages/coding-agent/src/web/scrapers/mastodon.ts @@ -89,11 +89,11 @@ function formatDate(isoDate: string): string { /** * Format a status/post as markdown */ -function formatStatus(status: MastodonStatus, isReblog = false): string { +async function formatStatus(status: MastodonStatus, isReblog = false): Promise { // Handle reblogs (boosts) if (status.reblog && !isReblog) { let md = `🔁 **${status.account.display_name || status.account.username}** boosted:\n\n`; - md += formatStatus(status.reblog, true); + md += await formatStatus(status.reblog, true); return md; } @@ -116,7 +116,7 @@ function formatStatus(status: MastodonStatus, isReblog = false): string { } // Main content (convert HTML to markdown) - const content = htmlToBasicMarkdown(status.content); + const content = await htmlToBasicMarkdown(status.content); md += `${content}\n\n`; // Poll @@ -152,7 +152,7 @@ function formatStatus(status: MastodonStatus, isReblog = false): string { /** * Format an account/profile as markdown */ -function formatAccount(account: MastodonAccount): string { +async function formatAccount(account: MastodonAccount): Promise { let md = `# ${account.display_name || account.username}\n\n`; md += `**@${account.acct}**`; @@ -161,7 +161,7 @@ function formatAccount(account: MastodonAccount): string { // Bio if (account.note) { - const bio = htmlToBasicMarkdown(account.note); + const bio = await htmlToBasicMarkdown(account.note); if (bio && bio !== account.display_name) { md += `${bio}\n\n`; } @@ -179,7 +179,7 @@ function formatAccount(account: MastodonAccount): string { if (account.fields && account.fields.length > 0) { md += "\n**Profile Fields:**\n"; for (const field of account.fields) { - const value = htmlToBasicMarkdown(field.value); + const value = await htmlToBasicMarkdown(field.value); md += `- **${field.name}:** ${value}\n`; } } @@ -228,7 +228,7 @@ export const handleMastodon: SpecialHandler = async ( const status = tryParseJson(result.content); if (!status) return null; - const md = formatStatus(status); + const md = await formatStatus(status); return buildResult(md, { url, @@ -263,7 +263,7 @@ export const handleMastodon: SpecialHandler = async ( signal, }); - let md = formatAccount(account); + let md = await formatAccount(account); if (statusesResult.ok) { const statuses = tryParseJson(statusesResult.content); @@ -271,7 +271,7 @@ export const handleMastodon: SpecialHandler = async ( md += "\n---\n\n## Recent Posts\n\n"; for (const status of statuses.slice(0, 5)) { md += `### ${formatDate(status.created_at)}\n\n`; - const content = htmlToBasicMarkdown(status.content); + const content = await htmlToBasicMarkdown(status.content); md += `${content}\n\n`; md += `\uD83D\uDCAC ${status.replies_count} \u00B7 \uD83D\uDD01 ${status.reblogs_count} \u00B7 \u2B50 ${status.favourites_count}\n\n`; } diff --git a/packages/coding-agent/src/web/scrapers/mdn.ts b/packages/coding-agent/src/web/scrapers/mdn.ts index 3371242f6..df244b960 100644 --- a/packages/coding-agent/src/web/scrapers/mdn.ts +++ b/packages/coding-agent/src/web/scrapers/mdn.ts @@ -29,7 +29,7 @@ interface MDNDoc { /** * Convert MDN body sections to markdown */ -function convertMDNBody(sections: MDNSection[]): string { +async function convertMDNBody(sections: MDNSection[]): Promise { const parts: string[] = []; for (const section of sections) { @@ -38,7 +38,7 @@ function convertMDNBody(sections: MDNSection[]): string { switch (type) { case "prose": if (value.content) { - const markdown = htmlToBasicMarkdown(value.content); + const markdown = await htmlToBasicMarkdown(value.content); if (value.title) { const level = value.isH3 ? "###" : "##"; parts.push(`${level} ${value.title}\n\n${markdown}`); @@ -74,7 +74,7 @@ function convertMDNBody(sections: MDNSection[]): string { if (value.items) { for (const item of value.items) { parts.push(`**${item.term}**`); - const desc = htmlToBasicMarkdown(item.description); + const desc = await htmlToBasicMarkdown(item.description); parts.push(desc); } } @@ -83,9 +83,13 @@ function convertMDNBody(sections: MDNSection[]): string { case "table": if (value.rows && value.rows.length > 0) { // Simple markdown table - const header = value.rows[0].map(cell => htmlToBasicMarkdown(cell)).join(" | "); + const header = (await Promise.all(value.rows[0].map(cell => htmlToBasicMarkdown(cell)))).join(" | "); const separator = value.rows[0].map(() => "---").join(" | "); - const bodyRows = value.rows.slice(1).map(row => row.map(cell => htmlToBasicMarkdown(cell)).join(" | ")); + const bodyRows = await Promise.all( + value.rows + .slice(1) + .map(async row => (await Promise.all(row.map(cell => htmlToBasicMarkdown(cell)))).join(" | ")), + ); parts.push(`| ${header} |`); parts.push(`| ${separator} |`); @@ -144,12 +148,12 @@ export const handleMDN: SpecialHandler = async (url: string, timeout: number, si parts.push(`# ${doc.title}`); if (doc.summary) { - const summary = htmlToBasicMarkdown(doc.summary); + const summary = await htmlToBasicMarkdown(doc.summary); parts.push(summary); } if (doc.body && doc.body.length > 0) { - const bodyMarkdown = convertMDNBody(doc.body); + const bodyMarkdown = await convertMDNBody(doc.body); parts.push(bodyMarkdown); } diff --git a/packages/coding-agent/src/web/scrapers/pub-dev.ts b/packages/coding-agent/src/web/scrapers/pub-dev.ts index 1bf9f02b7..b7ad5b3a2 100644 --- a/packages/coding-agent/src/web/scrapers/pub-dev.ts +++ b/packages/coding-agent/src/web/scrapers/pub-dev.ts @@ -125,7 +125,7 @@ export const handlePubDev: SpecialHandler = async (url: string, timeout: number, /]*class="[^"]*markdown-body[^"]*"[^>]*>([\s\S]*?)<\/div>/i, ); if (readmeMatch) { - const readme = htmlToBasicMarkdown(readmeMatch[1]); + const readme = await htmlToBasicMarkdown(readmeMatch[1]); if (readme.length > 100) { md += `## README\n\n${readme}\n`; diff --git a/packages/coding-agent/src/web/scrapers/rawg.ts b/packages/coding-agent/src/web/scrapers/rawg.ts index f44eb54f8..ff0e3c92b 100644 --- a/packages/coding-agent/src/web/scrapers/rawg.ts +++ b/packages/coding-agent/src/web/scrapers/rawg.ts @@ -63,7 +63,7 @@ export const handleRawg: SpecialHandler = async ( md += `**RAWG:** https://rawg.io/games/${encodeURIComponent(slug)}\n`; md += "\n"; - const description = extractDescription(game); + const description = await extractDescription(game); if (description) { md += `## Description\n\n${description}\n`; } @@ -91,11 +91,11 @@ function requiresApiKey(game: RawgGameResponse): boolean { return detail.includes("api key") || detail.includes("key is required") || detail.includes("apikey"); } -function extractDescription(game: RawgGameResponse): string | null { +async function extractDescription(game: RawgGameResponse): Promise { if (game.description_raw) return game.description_raw.trim(); if (!game.description) return null; - const markdown = htmlToBasicMarkdown(game.description).trim(); + const markdown = (await htmlToBasicMarkdown(game.description)).trim(); return markdown || null; } diff --git a/packages/coding-agent/src/web/scrapers/readthedocs.ts b/packages/coding-agent/src/web/scrapers/readthedocs.ts index f5a432a1a..921b1bb04 100644 --- a/packages/coding-agent/src/web/scrapers/readthedocs.ts +++ b/packages/coding-agent/src/web/scrapers/readthedocs.ts @@ -101,7 +101,7 @@ export const handleReadTheDocs: SpecialHandler = async ( // If no raw source, convert HTML to markdown if (!content && mainContent) { const html = mainContent.innerHTML; - content = htmlToBasicMarkdown(html); + content = await htmlToBasicMarkdown(html); } if (!content) { diff --git a/packages/coding-agent/src/web/scrapers/spdx.ts b/packages/coding-agent/src/web/scrapers/spdx.ts index 013f4f0f7..2a32d610a 100644 --- a/packages/coding-agent/src/web/scrapers/spdx.ts +++ b/packages/coding-agent/src/web/scrapers/spdx.ts @@ -94,7 +94,7 @@ export const handleSpdx: SpecialHandler = async ( const licenseText = license.licenseText ? license.licenseText : license.licenseTextHtml - ? htmlToBasicMarkdown(license.licenseTextHtml) + ? await htmlToBasicMarkdown(license.licenseTextHtml) : null; if (licenseText) { diff --git a/packages/coding-agent/src/web/scrapers/stackoverflow.ts b/packages/coding-agent/src/web/scrapers/stackoverflow.ts index 4334f2eef..992637119 100644 --- a/packages/coding-agent/src/web/scrapers/stackoverflow.ts +++ b/packages/coding-agent/src/web/scrapers/stackoverflow.ts @@ -90,7 +90,7 @@ export const handleStackOverflow: SpecialHandler = async ( md += question.is_answered ? " (Answered)" : ""; md += `\n**Tags:** ${question.tags.join(", ")}\n`; md += `**Asked by:** ${question.owner.display_name} · ${formatIsoDate(question.creation_date * 1000)}\n\n`; - md += `---\n\n## Question\n\n${htmlToBasicMarkdown(question.body)}\n\n`; + md += `---\n\n## Question\n\n${await htmlToBasicMarkdown(question.body)}\n\n`; // Fetch answers const aUrl = `https://api.stackexchange.com/2.3/questions/${questionId}/answers?order=desc&sort=votes&site=${site}&filter=withbody`; @@ -103,7 +103,7 @@ export const handleStackOverflow: SpecialHandler = async ( for (const answer of aData.items.slice(0, 5)) { const accepted = answer.is_accepted ? " (Accepted)" : ""; md += `### Score: ${answer.score}${accepted} · by ${answer.owner.display_name}\n\n`; - md += `${htmlToBasicMarkdown(answer.body)}\n\n---\n\n`; + md += `${await htmlToBasicMarkdown(answer.body)}\n\n---\n\n`; } } } diff --git a/packages/coding-agent/src/web/scrapers/types.ts b/packages/coding-agent/src/web/scrapers/types.ts index ce5f5e1cc..0ef64cc82 100644 --- a/packages/coding-agent/src/web/scrapers/types.ts +++ b/packages/coding-agent/src/web/scrapers/types.ts @@ -2,8 +2,8 @@ * Shared types and utilities for web-fetch handlers */ import { ptree } from "@oh-my-pi/pi-utils"; -import TurndownService from "turndown"; -import { gfm } from "turndown-plugin-gfm"; +import type TurndownService from "turndown"; + import { ToolAbortError } from "../../tools/tool-errors"; export { formatNumber } from "@oh-my-pi/pi-utils"; @@ -155,28 +155,8 @@ export async function loadPage(url: string, options: LoadPageOptions = {}): Prom return { content: "", contentType: "", finalUrl: url, ok: false }; } -/** Module-level Turndown instance — matches markit-ai's configuration. */ -const turndown = new TurndownService({ - headingStyle: "atx", - codeBlockStyle: "fenced", - bulletListMarker: "-", -}); -turndown.use(gfm); -turndown.addRule("strikethrough", { - filter: ["del", "s", "strike"], - replacement(content) { - return `~~${content}~~`; - }, -}); -turndown.addRule("heading", { - filter: ["h1", "h2", "h3", "h4", "h5", "h6"], - replacement(content, node) { - const level = Number(node.nodeName.charAt(1)); - const prefix = "#".repeat(level); - const cleaned = content.replace(/\\([.])/g, "$1").trim(); - return `\n\n${prefix} ${cleaned}\n\n`; - }, -}); +/** Module-level Turndown instance — built lazily on first use. */ +let turndownPromise: Promise | undefined; type TurndownListParent = { nodeName: string; @@ -184,27 +164,61 @@ type TurndownListParent = { children: ArrayLike; }; -turndown.addRule("listItem", { - filter: "li", - replacement(content, node, options) { - content = content.replace(/^\n+/, "").replace(/\n+$/, "\n").replace(/\n/gm, "\n "); - const parent = node.parentNode as unknown as TurndownListParent | null; - let prefix = `${options.bulletListMarker} `; - if (parent?.nodeName === "OL") { - const start = parent.getAttribute("start"); - const index = Array.prototype.indexOf.call(parent.children, node); - prefix = `${(start ? Number(start) : 1) + index}. `; - } - return prefix + content + (node.nextSibling ? "\n" : ""); - }, -}); +function getTurndown(): Promise { + turndownPromise ||= initTurndown(); + return turndownPromise; +} + +async function initTurndown(): Promise { + const [{ default: TurndownService }, { gfm }] = await Promise.all([ + import("turndown"), + import("turndown-plugin-gfm"), + ]); + const turndown = new TurndownService({ + headingStyle: "atx", + codeBlockStyle: "fenced", + bulletListMarker: "-", + }); + turndown.use(gfm); + turndown.addRule("strikethrough", { + filter: ["del", "s", "strike"], + replacement(content) { + return `~~${content}~~`; + }, + }); + turndown.addRule("heading", { + filter: ["h1", "h2", "h3", "h4", "h5", "h6"], + replacement(content, node) { + const level = Number(node.nodeName.charAt(1)); + const prefix = "#".repeat(level); + const cleaned = content.replace(/\\([.])/g, "$1").trim(); + return `\n\n${prefix} ${cleaned}\n\n`; + }, + }); + turndown.addRule("listItem", { + filter: "li", + replacement(content, node, options) { + content = content.replace(/^\n+/, "").replace(/\n+$/, "\n").replace(/\n/gm, "\n "); + const parent = node.parentNode as unknown as TurndownListParent | null; + let prefix = `${options.bulletListMarker} `; + if (parent?.nodeName === "OL") { + const start = parent.getAttribute("start"); + const index = Array.prototype.indexOf.call(parent.children, node); + prefix = `${(start ? Number(start) : 1) + index}. `; + } + return prefix + content + (node.nextSibling ? "\n" : ""); + }, + }); + return turndown; +} /** * Convert HTML to markdown using Turndown with GFM support. * Strips script/style tags before conversion. */ -export function htmlToBasicMarkdown(html: string): string { +export async function htmlToBasicMarkdown(html: string): Promise { const cleaned = html.replace(//gi, "").replace(//gi, ""); + const turndown = await getTurndown(); return turndown.turndown(cleaned).trim(); } diff --git a/packages/coding-agent/src/web/scrapers/w3c.ts b/packages/coding-agent/src/web/scrapers/w3c.ts index d884dc5ca..7c769aeed 100644 --- a/packages/coding-agent/src/web/scrapers/w3c.ts +++ b/packages/coding-agent/src/web/scrapers/w3c.ts @@ -100,7 +100,7 @@ export const handleW3c: SpecialHandler = async ( const title = getString(specPayload, "title"); const shortnameValue = getString(specPayload, "shortname") ?? shortname; const description = getString(specPayload, "description") ?? getString(specPayload, "abstract"); - const abstract = description ? htmlToBasicMarkdown(description) : undefined; + const abstract = description ? await htmlToBasicMarkdown(description) : undefined; const latestVersionUrl = getString(latestPayload, "uri") ?? diff --git a/packages/coding-agent/test/core/python-kernel-display.test.ts b/packages/coding-agent/test/core/python-kernel-display.test.ts index 307e523e1..6a9d92f8e 100644 --- a/packages/coding-agent/test/core/python-kernel-display.test.ts +++ b/packages/coding-agent/test/core/python-kernel-display.test.ts @@ -2,8 +2,8 @@ import { describe, expect, it } from "bun:test"; import { renderKernelDisplay } from "@oh-my-pi/pi-coding-agent/eval/py/kernel"; describe("PythonKernel display rendering", () => { - it("normalizes text/plain output and returns no display outputs", () => { - const { text, outputs } = renderKernelDisplay({ + it("normalizes text/plain output and returns no display outputs", async () => { + const { text, outputs } = await renderKernelDisplay({ data: { "text/plain": "hello" }, }); @@ -11,8 +11,8 @@ describe("PythonKernel display rendering", () => { expect(outputs).toHaveLength(0); }); - it("collects image and json display outputs without text", () => { - const { text, outputs } = renderKernelDisplay({ + it("collects image and json display outputs without text", async () => { + const { text, outputs } = await renderKernelDisplay({ data: { "image/png": "abc", "application/json": { foo: "bar" } }, }); @@ -23,8 +23,8 @@ describe("PythonKernel display rendering", () => { ]); }); - it("converts text/html to markdown", () => { - const { text, outputs } = renderKernelDisplay({ + it("converts text/html to markdown", async () => { + const { text, outputs } = await renderKernelDisplay({ data: { "text/html": "

Hello

" }, }); @@ -32,8 +32,8 @@ describe("PythonKernel display rendering", () => { expect(text).toBe("**Hello**\n"); }); - it("combines text/plain with json output", () => { - const { text, outputs } = renderKernelDisplay({ + it("combines text/plain with json output", async () => { + const { text, outputs } = await renderKernelDisplay({ data: { "text/plain": "value", "application/json": { ok: true } }, }); diff --git a/packages/coding-agent/test/tools/browser-readable.test.ts b/packages/coding-agent/test/tools/browser-readable.test.ts index 7f8d2c5d9..b3fa0305e 100644 --- a/packages/coding-agent/test/tools/browser-readable.test.ts +++ b/packages/coding-agent/test/tools/browser-readable.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it } from "bun:test"; import { extractReadableFromHtml } from "@oh-my-pi/pi-coding-agent/tools/browser"; describe("browser readable extraction", () => { - it("extracts markdown content from article-style pages", () => { + it("extracts markdown content from article-style pages", async () => { const html = ` Docs @@ -14,7 +14,7 @@ describe("browser readable extraction", () => { `; - const result = extractReadableFromHtml(html, "https://example.com/docs", "markdown"); + const result = await extractReadableFromHtml(html, "https://example.com/docs", "markdown"); expect(result).not.toBeNull(); expect(result?.title).toBe("Docs"); @@ -22,7 +22,7 @@ describe("browser readable extraction", () => { expect(result?.markdown).toContain("stores output only when you opt in"); }); - it("extracts docs-style main content", () => { + it("extracts docs-style main content", async () => { const html = ` Reference @@ -39,7 +39,7 @@ describe("browser readable extraction", () => { `; - const result = extractReadableFromHtml(html, "https://developers.openai.com/apps-sdk/reference", "text"); + const result = await extractReadableFromHtml(html, "https://developers.openai.com/apps-sdk/reference", "text"); expect(result).not.toBeNull(); expect(result?.title).toBe("Reference");