smashimo/oh-my-pi
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf: optimized ai providers with lazy-loading imports and cached init

Browse Source 
- Consolidated AI provider imports through register-builtins and moved Gemini/Antigravity header helpers to a shared module.
- Added lazy loading for heavy providers and SDK-backed modules with cached initialization to trim startup cost.
- Converted markdown conversion helpers to async and awaited htmlToBasicMarkdown in affected scraper and kernel output paths.
- Parsed bundled agent definitions on-demand and moved BrowserTool prompt rendering behind a memoized getter.
- Added cached validation/error handling paths by replacing AJV runtime checks with Value.Check and trimming validation error output.
This commit is contained in:
can1357
2026-05-01 18:52:20 +02:00
parent 74d71e8807
commit ca84ee760d
43 changed files with 419 additions and 322 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/ai/CHANGELOG.md
+7
View File
@@ -1,6 +1,13 @@
# Changelog
## [Unreleased]
### Added
- Added package-level `google-gemini-headers` exports (`getGeminiCliHeaders`, `getGeminiCliUserAgent`, `getAntigravityHeaders`, `extractRetryDelay`, and `ANTIGRAVITY_SYSTEM_INSTRUCTION`) for header and retry handling reuse without importing full Google providers
### Changed
- Changed package exports and streaming/provider wiring to load heavy Google/Kimi/GitLab/synthetic provider modules lazily through `register-builtins`, reducing startup import overhead from optional provider SDKs
### Fixed
packages/ai/src/index.ts
+5 -4
View File
@@ -10,11 +10,12 @@ export * from "./provider-details";
export * from "./provider-models";
export * from "./providers/anthropic";
export * from "./providers/azure-openai-responses";
export * from "./providers/cursor";
export type * from "./providers/cursor";
export * from "./providers/gitlab-duo";
export * from "./providers/google";
export * from "./providers/google-gemini-cli";
export * from "./providers/google-vertex";
export type * from "./providers/google";
export type * from "./providers/google-gemini-cli";
export * from "./providers/google-gemini-headers";
export type * from "./providers/google-vertex";
export * from "./providers/kimi";
export * from "./providers/ollama";
export type { OpenAICodexResponsesOptions } from "./providers/openai-codex-responses";
packages/ai/src/providers/antigravity-user-agent.ts
+19
View File
@@ -0,0 +1,19 @@
/**
* Antigravity / Cloud Code Assist user agent. Lives in its own file so discovery
* and usage code can read it without pulling the heavy google-gemini-cli provider
* (and its @google/genai → google-auth-library dependency chain) into the startup
* parse graph.
*/
export let getAntigravityUserAgent = () => {
const DEFAULT_ANTIGRAVITY_VERSION = "1.104.0";
const version = process.env.PI_AI_ANTIGRAVITY_VERSION || DEFAULT_ANTIGRAVITY_VERSION;
// Map Node.js platform/arch to Antigravity's expected format.
// Verified against Antigravity source: _qn() and wqn() in main.js.
// process.platform: win32→windows, others pass through (darwin, linux)
// process.arch: x64→amd64, ia32→386, others pass through (arm64)
const os = process.platform === "win32" ? "windows" : process.platform;
const arch = process.arch === "x64" ? "amd64" : process.arch === "ia32" ? "386" : process.arch;
const userAgent = `antigravity/${version} ${os}/${arch}`;
getAntigravityUserAgent = () => userAgent;
return userAgent;
};
packages/ai/src/providers/gitlab-duo.ts
+1 -3
View File
@@ -1,11 +1,9 @@
import { ANTHROPIC_THINKING, mapAnthropicToolChoice } from "../stream";
import type { Api, Context, Model, SimpleStreamOptions } from "../types";
import { AssistantMessageEventStream } from "../utils/event-stream";
import { streamAnthropic } from "./anthropic";
import type { OpenAICompletionsOptions } from "./openai-completions";
import { streamOpenAICompletions } from "./openai-completions";
import type { OpenAIResponsesOptions } from "./openai-responses";
import { streamOpenAIResponses } from "./openai-responses";
import { streamAnthropic, streamOpenAICompletions, streamOpenAIResponses } from "./register-builtins";
import { createProviderErrorMessage } from "./shared/error-message";
const GITLAB_COM_URL = "https://gitlab.com";
packages/ai/src/providers/google-gemini-cli.ts
+16 -169
View File
@@ -24,6 +24,12 @@ import { refreshAntigravityToken } from "../utils/oauth/google-antigravity";
import { refreshGoogleCloudToken } from "../utils/oauth/google-gemini-cli";
import { extractHttpStatusFromError } from "../utils/retry";
import { sanitizeSchemaForCCA } from "../utils/schema";
import {
ANTIGRAVITY_SYSTEM_INSTRUCTION,
extractRetryDelay,
getAntigravityHeaders,
getGeminiCliHeaders,
} from "./google-gemini-headers";
import {
convertMessages,
convertTools,
@@ -63,75 +69,16 @@ const ANTIGRAVITY_DAILY_ENDPOINT = "https://daily-cloudcode-pa.googleapis.com";
const ANTIGRAVITY_SANDBOX_ENDPOINT = "https://daily-cloudcode-pa.sandbox.googleapis.com";
const ANTIGRAVITY_ENDPOINT_FALLBACKS = [ANTIGRAVITY_DAILY_ENDPOINT, ANTIGRAVITY_SANDBOX_ENDPOINT] as const;
/**
* Build a User-Agent string that identifies as Gemini CLI to unlock higher rate limits.
* Uses the same format as the official Gemini CLI (v0.35+):
* GeminiCLI/VERSION/MODEL (PLATFORM; ARCH; SURFACE)
*/
export function getGeminiCliUserAgent(modelId = "gemini-3.1-pro-preview"): string {
const version = process.env.PI_AI_GEMINI_CLI_VERSION || "0.35.3";
const platform = process.platform === "win32" ? "win32" : process.platform;
const arch = process.arch === "x64" ? "x64" : process.arch;
return `GeminiCLI/${version}/${modelId} (${platform}; ${arch}; terminal)`;
}
const ANTIGRAVITY_USER_AGENT = (() => {
const DEFAULT_ANTIGRAVITY_VERSION = "1.104.0";
const version = process.env.PI_AI_ANTIGRAVITY_VERSION || DEFAULT_ANTIGRAVITY_VERSION;
// Map Node.js platform/arch to Antigravity's expected format.
// Verified against Antigravity source: _qn() and wqn() in main.js.
// process.platform: win32→windows, others pass through (darwin, linux)
// process.arch: x64→amd64, ia32→386, others pass through (arm64)
const os = process.platform === "win32" ? "windows" : process.platform;
const arch = process.arch === "x64" ? "amd64" : process.arch === "ia32" ? "386" : process.arch;
return `antigravity/${version} ${os}/${arch}`;
})();
const GEMINI_CLI_HEADERS = (modelId?: string) =>
Object.freeze({
"User-Agent": getGeminiCliUserAgent(modelId),
"Client-Metadata": "ideType=IDE_UNSPECIFIED,platform=PLATFORM_UNSPECIFIED,pluginType=GEMINI",
});
// Antigravity auth headers (project discovery/onboarding).
// Verified from binary: kae.w() and kae.y() send only Content-Type + User-Agent.
// X-Goog-Api-Client and Client-Metadata are NOT sent by the real client — product
// identification (ideType, ideName, ideVersion) goes in the protobuf request body.
const ANTIGRAVITY_AUTH_HEADERS = Object.freeze({
"User-Agent": ANTIGRAVITY_USER_AGENT,
});
// Antigravity executor headers (streaming/generation).
// Same header set as auth calls — only User-Agent per binary analysis.
const ANTIGRAVITY_STREAMING_HEADERS = Object.freeze({
"User-Agent": ANTIGRAVITY_USER_AGENT,
});
// Headers for Gemini CLI (prod endpoint)
export function getGeminiCliHeaders(modelId?: string) {
return GEMINI_CLI_HEADERS(modelId);
}
export function getGeminiCliUserAgentValue(modelId?: string) {
return getGeminiCliUserAgent(modelId);
}
// Headers for Antigravity (sandbox endpoint)
export function getAntigravityAuthHeaders() {
return ANTIGRAVITY_AUTH_HEADERS;
}
export function getAntigravityHeaders() {
return ANTIGRAVITY_STREAMING_HEADERS;
}
export function getAntigravityUserAgent() {
return ANTIGRAVITY_USER_AGENT;
}
// Antigravity system instruction (compact version from CLIProxyAPI).
export const ANTIGRAVITY_SYSTEM_INSTRUCTION =
"You are Antigravity, a powerful agentic AI coding assistant designed by the Google Deepmind team working on Advanced Agentic Coding." +
"You are pair programming with a USER to solve their coding task. The task may require creating a new codebase, modifying or debugging an existing codebase, or simply answering a question." +
"**Absolute paths only**" +
"**Proactiveness**";
export { getAntigravityUserAgent } from "./antigravity-user-agent";
export {
ANTIGRAVITY_SYSTEM_INSTRUCTION,
extractRetryDelay,
getAntigravityHeaders as getAntigravityAuthHeaders,
getAntigravityHeaders,
getGeminiCliHeaders,
getGeminiCliUserAgent,
getGeminiCliUserAgentValue,
} from "./google-gemini-headers";
// Counter for generating unique tool call IDs
let toolCallCounter = 0;
@@ -146,106 +93,6 @@ const CLAUDE_THINKING_BETA_HEADER = "interleaved-thinking-2025-05-14";
const GOOGLE_GEMINI_REFRESH_SKEW_MS = 60_000;
const ANTIGRAVITY_REFRESH_SKEW_MS = 60_000;
/**
* Extract retry delay from Gemini error response (in milliseconds).
* Checks headers first (Retry-After, x-ratelimit-reset, x-ratelimit-reset-after),
* then parses body patterns like:
* - "Your quota will reset after 39s"
* - "Your quota will reset after 18h31m10s"
* - "Please retry in Xs" or "Please retry in Xms"
* - "retryDelay": "34.074824224s" (JSON field)
*/
export function extractRetryDelay(errorText: string, response?: Response | Headers): number | undefined {
const normalizeDelay = (ms: number): number | undefined => (ms > 0 ? Math.ceil(ms + 1000) : undefined);
const headers = response instanceof Headers ? response : response?.headers;
if (headers) {
const retryAfter = headers.get("retry-after");
if (retryAfter) {
const retryAfterSeconds = Number(retryAfter);
if (Number.isFinite(retryAfterSeconds)) {
const delay = normalizeDelay(retryAfterSeconds * 1000);
if (delay !== undefined) {
return delay;
}
}
const retryAfterDate = new Date(retryAfter);
const retryAfterMs = retryAfterDate.getTime();
if (!Number.isNaN(retryAfterMs)) {
const delay = normalizeDelay(retryAfterMs - Date.now());
if (delay !== undefined) {
return delay;
}
}
}
const rateLimitReset = headers.get("x-ratelimit-reset");
if (rateLimitReset) {
const resetSeconds = Number.parseInt(rateLimitReset, 10);
if (!Number.isNaN(resetSeconds)) {
const delay = normalizeDelay(resetSeconds * 1000 - Date.now());
if (delay !== undefined) {
return delay;
}
}
}
const rateLimitResetAfter = headers.get("x-ratelimit-reset-after");
if (rateLimitResetAfter) {
const resetAfterSeconds = Number(rateLimitResetAfter);
if (Number.isFinite(resetAfterSeconds)) {
const delay = normalizeDelay(resetAfterSeconds * 1000);
if (delay !== undefined) {
return delay;
}
}
}
}
// Pattern 1: "Your quota will reset after ..." (formats: "18h31m10s", "10m15s", "6s", "39s")
const durationMatch = errorText.match(/reset after (?:(\d+)h)?(?:(\d+)m)?(\d+(?:\.\d+)?)s/i);
if (durationMatch) {
const hours = durationMatch[1] ? parseInt(durationMatch[1], 10) : 0;
const minutes = durationMatch[2] ? parseInt(durationMatch[2], 10) : 0;
const seconds = parseFloat(durationMatch[3]);
if (!Number.isNaN(seconds)) {
const totalMs = ((hours * 60 + minutes) * 60 + seconds) * 1000;
const delay = normalizeDelay(totalMs);
if (delay !== undefined) {
return delay;
}
}
}
// Pattern 2: "Please retry in X[ms|s]"
const retryInMatch = errorText.match(/Please retry in ([0-9.]+)(ms|s)/i);
if (retryInMatch?.[1]) {
const value = parseFloat(retryInMatch[1]);
if (!Number.isNaN(value) && value > 0) {
const ms = retryInMatch[2].toLowerCase() === "ms" ? value : value * 1000;
const delay = normalizeDelay(ms);
if (delay !== undefined) {
return delay;
}
}
}
// Pattern 3: "retryDelay": "34.074824224s" (JSON field in error details)
const retryDelayMatch = errorText.match(/"retryDelay":\s*"([0-9.]+)(ms|s)"/i);
if (retryDelayMatch?.[1]) {
const value = parseFloat(retryDelayMatch[1]);
if (!Number.isNaN(value) && value > 0) {
const ms = retryDelayMatch[2].toLowerCase() === "ms" ? value : value * 1000;
const delay = normalizeDelay(ms);
if (delay !== undefined) {
return delay;
}
}
}
return undefined;
}
function isClaudeModel(modelId: string): boolean {
return modelId.toLowerCase().includes("claude");
}
packages/ai/src/providers/google-gemini-headers.ts
+152
View File
@@ -0,0 +1,152 @@
/**
* Lightweight header helpers for the Gemini CLI / Antigravity providers.
*
* Extracted from `./google-gemini-cli` so consumers (usage tracking, OAuth) can
* import these helpers without dragging the heavy `@google/genai` →
* `google-auth-library` dependency graph into the CLI startup parse path.
*
* Only depends on `node:` builtins, `process.env`, and `./antigravity-user-agent`.
*/
import { getAntigravityUserAgent } from "./antigravity-user-agent";
export { getAntigravityUserAgent } from "./antigravity-user-agent";
/**
* Build a User-Agent string that identifies as Gemini CLI to unlock higher rate limits.
* Uses the same format as the official Gemini CLI (v0.35+):
* GeminiCLI/VERSION/MODEL (PLATFORM; ARCH; SURFACE)
*/
export function getGeminiCliUserAgent(modelId = "gemini-3.1-pro-preview"): string {
const version = process.env.PI_AI_GEMINI_CLI_VERSION || "0.35.3";
const platform = process.platform === "win32" ? "win32" : process.platform;
const arch = process.arch === "x64" ? "x64" : process.arch;
return `GeminiCLI/${version}/${modelId} (${platform}; ${arch}; terminal)`;
}
const GEMINI_CLI_HEADERS = (modelId?: string) =>
Object.freeze({
"User-Agent": getGeminiCliUserAgent(modelId),
"Client-Metadata": "ideType=IDE_UNSPECIFIED,platform=PLATFORM_UNSPECIFIED,pluginType=GEMINI",
});
// Headers for Gemini CLI (prod endpoint)
export function getGeminiCliHeaders(modelId?: string) {
return GEMINI_CLI_HEADERS(modelId);
}
export function getGeminiCliUserAgentValue(modelId?: string) {
return getGeminiCliUserAgent(modelId);
}
// Headers for Antigravity (sandbox endpoint)
export function getAntigravityHeaders() {
return {
"User-Agent": getAntigravityUserAgent(),
};
}
// Antigravity system instruction (compact version from CLIProxyAPI).
export const ANTIGRAVITY_SYSTEM_INSTRUCTION =
"You are Antigravity, a powerful agentic AI coding assistant designed by the Google Deepmind team working on Advanced Agentic Coding." +
"You are pair programming with a USER to solve their coding task. The task may require creating a new codebase, modifying or debugging an existing codebase, or simply answering a question." +
"**Absolute paths only**" +
"**Proactiveness**";
/**
* Extract retry delay from Gemini error response (in milliseconds).
* Checks headers first (Retry-After, x-ratelimit-reset, x-ratelimit-reset-after),
* then parses body patterns like:
* - "Your quota will reset after 39s"
* - "Your quota will reset after 18h31m10s"
* - "Please retry in Xs" or "Please retry in Xms"
* - "retryDelay": "34.074824224s" (JSON field)
*/
export function extractRetryDelay(errorText: string, response?: Response | Headers): number | undefined {
const normalizeDelay = (ms: number): number | undefined => (ms > 0 ? Math.ceil(ms + 1000) : undefined);
const headers = response instanceof Headers ? response : response?.headers;
if (headers) {
const retryAfter = headers.get("retry-after");
if (retryAfter) {
const retryAfterSeconds = Number(retryAfter);
if (Number.isFinite(retryAfterSeconds)) {
const delay = normalizeDelay(retryAfterSeconds * 1000);
if (delay !== undefined) {
return delay;
}
}
const retryAfterDate = new Date(retryAfter);
const retryAfterMs = retryAfterDate.getTime();
if (!Number.isNaN(retryAfterMs)) {
const delay = normalizeDelay(retryAfterMs - Date.now());
if (delay !== undefined) {
return delay;
}
}
}
const rateLimitReset = headers.get("x-ratelimit-reset");
if (rateLimitReset) {
const resetSeconds = Number.parseInt(rateLimitReset, 10);
if (!Number.isNaN(resetSeconds)) {
const delay = normalizeDelay(resetSeconds * 1000 - Date.now());
if (delay !== undefined) {
return delay;
}
}
}
const rateLimitResetAfter = headers.get("x-ratelimit-reset-after");
if (rateLimitResetAfter) {
const resetAfterSeconds = Number(rateLimitResetAfter);
if (Number.isFinite(resetAfterSeconds)) {
const delay = normalizeDelay(resetAfterSeconds * 1000);
if (delay !== undefined) {
return delay;
}
}
}
}
// Pattern 1: "Your quota will reset after ..." (formats: "18h31m10s", "10m15s", "6s", "39s")
const durationMatch = errorText.match(/reset after (?:(\d+)h)?(?:(\d+)m)?(\d+(?:\.\d+)?)s/i);
if (durationMatch) {
const hours = durationMatch[1] ? parseInt(durationMatch[1], 10) : 0;
const minutes = durationMatch[2] ? parseInt(durationMatch[2], 10) : 0;
const seconds = parseFloat(durationMatch[3]);
if (!Number.isNaN(seconds)) {
const totalMs = ((hours * 60 + minutes) * 60 + seconds) * 1000;
const delay = normalizeDelay(totalMs);
if (delay !== undefined) {
return delay;
}
}
}
// Pattern 2: "Please retry in X[ms|s]"
const retryInMatch = errorText.match(/Please retry in ([0-9.]+)(ms|s)/i);
if (retryInMatch?.[1]) {
const value = parseFloat(retryInMatch[1]);
if (!Number.isNaN(value) && value > 0) {
const ms = retryInMatch[2].toLowerCase() === "ms" ? value : value * 1000;
const delay = normalizeDelay(ms);
if (delay !== undefined) {
return delay;
}
}
}
// Pattern 3: "retryDelay": "34.074824224s" (JSON field in error details)
const retryDelayMatch = errorText.match(/"retryDelay":\s*"([0-9.]+)(ms|s)"/i);
if (retryDelayMatch?.[1]) {
const value = parseFloat(retryDelayMatch[1]);
if (!Number.isNaN(value) && value > 0) {
const ms = retryDelayMatch[2].toLowerCase() === "ms" ? value : value * 1000;
const delay = normalizeDelay(ms);
if (delay !== undefined) {
return delay;
}
}
}
return undefined;
}
packages/ai/src/providers/kimi.ts
+1 -2
View File
@@ -13,8 +13,7 @@ import { ANTHROPIC_THINKING } from "../stream";
import type { Api, Context, Model, SimpleStreamOptions } from "../types";
import { AssistantMessageEventStream } from "../utils/event-stream";
import { getKimiCommonHeaders } from "../utils/oauth/kimi";
import { streamAnthropic } from "./anthropic";
import { streamOpenAICompletions } from "./openai-completions";
import { streamAnthropic, streamOpenAICompletions } from "./register-builtins";
import { createProviderErrorMessage } from "./shared/error-message";
export type KimiApiFormat = "openai" | "anthropic";
packages/ai/src/providers/openai-completions-compat.ts
+2 -2
View File
@@ -108,7 +108,7 @@ export function detectOpenAICompat(model: Model<"openai-completions">, resolvedB
high: "default",
xhigh: "default",
} satisfies Partial<Record<OpenAIReasoningEffort, string>>)
: isDeepseekFamily && Boolean(model.reasoning)
: isDeepseekFamily && model.reasoning
? { xhigh: "max" }
: {};
@@ -145,7 +145,7 @@ export function detectOpenAICompat(model: Model<"openai-completions">, resolvedB
((provider === "openrouter" || baseUrl.includes("openrouter.ai")) && Boolean(model.reasoning)),
// DeepSeek V4 rejects synthetic reasoning_content placeholders (".") on tool-call turns.
// Kimi and OpenRouter accept them when actual reasoning is unavailable.
allowsSyntheticReasoningContentForToolCalls: !isDeepseekFamily || !Boolean(model.reasoning),
allowsSyntheticReasoningContentForToolCalls: !isDeepseekFamily || !model.reasoning,
requiresAssistantContentForToolCalls: isKimiModel,
openRouterRouting: undefined,
vercelGatewayRouting: undefined,
packages/ai/src/providers/synthetic.ts
+1 -2
View File
@@ -11,8 +11,7 @@
import { ANTHROPIC_THINKING } from "../stream";
import type { Api, Context, Model, SimpleStreamOptions } from "../types";
import { AssistantMessageEventStream } from "../utils/event-stream";
import { streamAnthropic } from "./anthropic";
import { streamOpenAICompletions } from "./openai-completions";
import { streamAnthropic, streamOpenAICompletions } from "./register-builtins";
import { createProviderErrorMessage } from "./shared/error-message";
export type SyntheticApiFormat = "openai" | "anthropic";
packages/ai/src/stream.ts
+29 -11
View File
@@ -9,19 +9,37 @@ import {
mapEffortToGoogleThinkingLevel,
requireSupportedEffort,
} from "./model-thinking";
import { type BedrockOptions, streamBedrock } from "./providers/amazon-bedrock";
import { type AnthropicOptions, streamAnthropic } from "./providers/anthropic";
import { streamAzureOpenAIResponses } from "./providers/azure-openai-responses";
import { type CursorOptions, streamCursor } from "./providers/cursor";
import type { BedrockOptions } from "./providers/amazon-bedrock";
import type { AnthropicOptions } from "./providers/anthropic";
import type { CursorOptions } from "./providers/cursor";
import { isGitLabDuoModel, streamGitLabDuo } from "./providers/gitlab-duo";
import { type GoogleOptions, streamGoogle } from "./providers/google";
import { type GoogleGeminiCliOptions, streamGoogleGeminiCli } from "./providers/google-gemini-cli";
import { type GoogleVertexOptions, streamGoogleVertex } from "./providers/google-vertex";
import type { GoogleOptions } from "./providers/google";
import type { GoogleGeminiCliOptions } from "./providers/google-gemini-cli";
import type { GoogleVertexOptions } from "./providers/google-vertex";
import { isKimiModel, streamKimi } from "./providers/kimi";
import { type OllamaChatOptions, streamOllama } from "./providers/ollama";
import { streamOpenAICodexResponses } from "./providers/openai-codex-responses";
import { type OpenAICompletionsOptions, streamOpenAICompletions } from "./providers/openai-completions";
import { streamOpenAIResponses } from "./providers/openai-responses";
import type { OllamaChatOptions } from "./providers/ollama";
import type { OpenAICompletionsOptions } from "./providers/openai-completions";
// Heavy provider stream functions are imported lazily via register-builtins,
// which wraps each provider module in a dynamic import. This keeps the
// AWS SDK, google-auth-library, @google/genai, @bufbuild/protobuf, and
// other provider SDKs out of the CLI startup parse graph. The
// gitlab-duo / kimi / synthetic providers stay eager because their modules
// export routing predicates (isGitLabDuoModel, isKimiModel, isSyntheticModel)
// that must be callable synchronously before streaming begins, and their
// modules are thin wrappers with no heavy SDK dependencies.
import {
streamAnthropic,
streamAzureOpenAIResponses,
streamBedrock,
streamCursor,
streamGoogle,
streamGoogleGeminiCli,
streamGoogleVertex,
streamOllama,
streamOpenAICodexResponses,
streamOpenAICompletions,
streamOpenAIResponses,
} from "./providers/register-builtins";
import { isSyntheticModel, streamSynthetic } from "./providers/synthetic";
import type {
Api,
packages/ai/src/utils/oauth/google-antigravity.ts
+2 -2
View File
@@ -2,7 +2,7 @@
* Antigravity OAuth flow (Gemini 3, Claude, GPT-OSS via Google Cloud)
* Uses different OAuth credentials than google-gemini-cli for access to additional models.
*/
import { getAntigravityAuthHeaders } from "../../providers/google-gemini-cli";
import { getAntigravityHeaders } from "../../providers/google-gemini-headers";
import { runGoogleOAuthLogin } from "./google-oauth-shared";
import type { OAuthController, OAuthCredentials } from "./types";
@@ -112,7 +112,7 @@ async function discoverProject(accessToken: string, onProgress?: (message: strin
const headers = {
Authorization: `Bearer ${accessToken}`,
"Content-Type": "application/json",
...getAntigravityAuthHeaders(),
...getAntigravityHeaders(),
};
onProgress?.("Checking for existing project...");
packages/ai/src/utils/oauth/google-gemini-cli.ts
+1 -1
View File
@@ -4,7 +4,7 @@
*/
import { $env } from "@oh-my-pi/pi-utils";
import { getGeminiCliHeaders } from "../../providers/google-gemini-cli";
import { getGeminiCliHeaders } from "../../providers/google-gemini-headers";
import { runGoogleOAuthLogin } from "./google-oauth-shared";
import type { OAuthController, OAuthCredentials } from "./types";
packages/coding-agent/CHANGELOG.md
+7
View File
@@ -1,6 +1,13 @@
# Changelog
## [Unreleased]
### Changed
- Changed markdown conversion and archive tooling to defer loading heavy dependencies (Turndown, fflate, and browser agent content) until first use, reducing startup overhead for CLI startup and command initialization
### Fixed
- Fixed changelog state tracking by flushing `lastChangelogVersion` to settings immediately when showing new entries, so the updated version is persisted across restarts
## [14.5.13] - 2026-05-01
packages/coding-agent/src/commit/pipeline.ts
+4 -3
View File
@@ -25,7 +25,8 @@ import type { CommitCommandArgs, ConventionalAnalysis } from "./types";
const SUMMARY_MAX_CHARS = 72;
const RECENT_COMMITS_COUNT = 8;
const TYPES_DESCRIPTION = prompt.render(typesDescriptionPrompt);
let _typesDescription: string | undefined;
const TYPES_DESCRIPTION = (): string => (_typesDescription ??= prompt.render(typesDescriptionPrompt));
/**
* Execute the omp commit pipeline for staged changes.
@@ -176,7 +177,7 @@ async function generateAnalysis(input: {
diff: input.diff,
stat: input.stat,
scopeCandidates: input.scopeCandidates,
typesDescription: TYPES_DESCRIPTION,
typesDescription: TYPES_DESCRIPTION(),
settings: {
enabled: input.commitSettings.mapReduceEnabled,
minFiles: input.commitSettings.mapReduceMinFiles,
@@ -193,7 +194,7 @@ async function generateAnalysis(input: {
thinkingLevel: input.primaryThinkingLevel,
contextFiles: input.contextFiles,
userContext: input.userContext,
typesDescription: TYPES_DESCRIPTION,
typesDescription: TYPES_DESCRIPTION(),
recentCommits: input.recentCommits,
scopeCandidates: input.scopeCandidates,
stat: input.stat,
packages/coding-agent/src/config.ts
+9 -6
View File
@@ -11,7 +11,7 @@ import {
} from "@oh-my-pi/pi-utils";
import type { TSchema } from "@sinclair/typebox";
import { Value } from "@sinclair/typebox/value";
import { Ajv, type ErrorObject, type ValidateFunction } from "ajv";
import type { ErrorObject } from "ajv";
import { JSONC, YAML } from "bun";
import { expandTilde } from "./tools/path-utils";
@@ -143,7 +143,6 @@ export type LoadResult<T> =
| { value: T; error?: undefined; status: "ok" }
| { value?: null; error?: unknown; status: "not-found" };
const ajv = new Ajv();
export class ConfigFile<T> implements IConfigFile<T> {
readonly #basePath: string;
#cache?: LoadResult<T>;
@@ -221,13 +220,17 @@ export class ConfigFile<T> implements IConfigFile<T> {
throw new Error(`Invalid config file path: ${this.#basePath}`);
}
const validate = ajv.compile(this.schema) as ValidateFunction<T>;
if (!validate(parsed)) {
const error = new ConfigError(this.id, validate.errors);
if (!Value.Check(this.schema, parsed)) {
const schemaErrors: ErrorObject[] = [];
for (const err of Value.Errors(this.schema, parsed)) {
schemaErrors.push({ instancePath: err.path, message: err.message } as ErrorObject);
if (schemaErrors.length >= 50) break;
}
const error = new ConfigError(this.id, schemaErrors);
logger.warn("Failed to parse config file", { path: this.path(), error });
return this.#storeCache({ error, status: "error" });
}
return this.#storeCache({ value: parsed, status: "ok" });
return this.#storeCache({ value: parsed as T, status: "ok" });
} catch (error) {
if (isEnoent(error)) {
return this.#storeCache({ status: "not-found" });
packages/coding-agent/src/eval/py/kernel.ts
+4 -4
View File
@@ -271,10 +271,10 @@ function normalizeDisplayText(text: string): string {
}
/** Renders a Jupyter display_data message into text and structured outputs. */
export function renderKernelDisplay(content: Record<string, unknown>): {
export async function renderKernelDisplay(content: Record<string, unknown>): Promise<{
text: string;
outputs: KernelDisplayOutput[];
} {
}> {
const data = content.data as Record<string, unknown> | undefined;
if (!data) return { text: "", outputs: [] };
@@ -307,7 +307,7 @@ export function renderKernelDisplay(content: Record<string, unknown>): {
return { text: normalizeDisplayText(String(data["text/plain"])), outputs };
}
if (data["text/html"] !== undefined) {
const markdown = htmlToBasicMarkdown(String(data["text/html"])) || "";
const markdown = (await htmlToBasicMarkdown(String(data["text/html"]))) || "";
return { text: markdown ? normalizeDisplayText(markdown) : "", outputs };
}
return { text: "", outputs };
@@ -872,7 +872,7 @@ export class PythonKernel {
}
case "execute_result":
case "display_data": {
const { text, outputs } = renderKernelDisplay(response.content);
const { text, outputs } = await renderKernelDisplay(response.content);
if (text && options?.onChunk) {
await options.onChunk(text);
}
packages/coding-agent/src/main.ts
+10
View File
@@ -253,12 +253,14 @@ async function getChangelogForDisplay(parsed: Args): Promise<string | undefined>
if (!lastVersion) {
if (entries.length > 0) {
settings.set("lastChangelogVersion", VERSION);
await flushChangelogVersion();
return entries.map(e => e.content).join("\n\n");
}
} else {
const newEntries = getNewEntries(entries, lastVersion);
if (newEntries.length > 0) {
settings.set("lastChangelogVersion", VERSION);
await flushChangelogVersion();
return newEntries.map(e => e.content).join("\n\n");
}
}
@@ -266,6 +268,14 @@ async function getChangelogForDisplay(parsed: Args): Promise<string | undefined>
return undefined;
}
async function flushChangelogVersion(): Promise<void> {
try {
await settings.flush();
} catch (error: unknown) {
logger.warn("Failed to persist lastChangelogVersion", { error });
}
}
async function createSessionManager(parsed: Args, cwd: string): Promise<SessionManager | undefined> {
if (parsed.fork) {
if (parsed.noSession) {
packages/coding-agent/src/sdk.ts
-1
View File
@@ -300,7 +300,6 @@ function getDefaultAgentDir(): string {
*/
export async function discoverAuthStorage(agentDir: string = getDefaultAgentDir()): Promise<AuthStorage> {
const dbPath = getAgentDbPath(agentDir);
logger.debug("discoverAuthStorage", { agentDir, dbPath });
const storage = await AuthStorage.create(dbPath, { configValueResolver: resolveConfigValue });
await storage.reload();
packages/coding-agent/src/task/agents.ts
+4 -5
View File
@@ -69,10 +69,7 @@ const EMBEDDED_AGENT_DEFS: EmbeddedAgentDef[] = [
},
];
const EMBEDDED_AGENTS: { name: string; content: string }[] = EMBEDDED_AGENT_DEFS.map(def => ({
name: def.fileName,
content: buildAgentContent(def),
}));
// Computed lazily on first loadBundledAgents() call to avoid eager prompt.render at module load.
export class AgentParsingError extends Error {
constructor(
@@ -133,7 +130,9 @@ export function loadBundledAgents(): AgentDefinition[] {
if (bundledAgentsCache !== null) {
return bundledAgentsCache;
}
bundledAgentsCache = EMBEDDED_AGENTS.map(({ name, content }) => parseAgent(`embedded:${name}`, content, "bundled"));
bundledAgentsCache = EMBEDDED_AGENT_DEFS.map(def =>
parseAgent(`embedded:${def.fileName}`, buildAgentContent(def), "bundled"),
);
return bundledAgentsCache;
}
packages/coding-agent/src/tools/archive-reader.ts
+9 -3
View File
@@ -1,6 +1,11 @@
import { unzipSync } from "fflate";
import { ToolError } from "./tool-errors";
let fflateModulePromise: Promise<typeof import("fflate")> | undefined;
async function loadFflate(): Promise<typeof import("fflate")> {
if (!fflateModulePromise) fflateModulePromise = import("fflate");
return fflateModulePromise;
}
export type ArchiveFormat = "zip" | "tar" | "tar.gz";
export interface ArchivePathCandidate {
@@ -150,7 +155,8 @@ async function readTarEntries(bytes: Uint8Array): Promise<ArchiveIndexEntry[]> {
return entries;
}
function readZipEntries(bytes: Uint8Array): ArchiveIndexEntry[] {
async function readZipEntries(bytes: Uint8Array): Promise<ArchiveIndexEntry[]> {
const { unzipSync } = await loadFflate();
let files: Record<string, Uint8Array>;
try {
files = unzipSync(bytes);
@@ -310,6 +316,6 @@ export async function openArchive(filePath: string): Promise<ArchiveReader> {
}
const bytes = await Bun.file(filePath).bytes();
const entries = format === "zip" ? readZipEntries(bytes) : await readTarEntries(bytes);
const entries = format === "zip" ? await readZipEntries(bytes) : await readTarEntries(bytes);
return new ArchiveReader(format, entries);
}
packages/coding-agent/src/tools/browser.ts
+5 -3
View File
@@ -110,12 +110,14 @@ function resolveBrowserKind(params: BrowserParams, session: ToolSession): Browse
export class BrowserTool implements AgentTool<typeof browserSchema, BrowserToolDetails> {
readonly name = "browser";
readonly label = "Browser";
readonly description: string;
readonly parameters = browserSchema;
readonly strict = true;
constructor(private readonly session: ToolSession) {
this.description = prompt.render(browserDescription, {});
constructor(private readonly session: ToolSession) {}
#description?: string;
get description(): string {
this.#description ??= prompt.render(browserDescription, {});
return this.#description;
}
/** Restart browser to apply mode changes (e.g. headless toggle). Drops only headless browsers. */
packages/coding-agent/src/tools/browser/readable.ts
+11 -6
View File
@@ -26,13 +26,17 @@ function normalize(text: string | null | undefined): string | undefined {
* CSS selector chain over the same pre-parsed DOM. Returns null if neither
* path yields usable content.
*/
export function extractReadableFromHtml(html: string, url: string, format: ReadableFormat): ReadableResult | null {
export async function extractReadableFromHtml(
html: string,
url: string,
format: ReadableFormat,
): Promise<ReadableResult | null> {
const { document } = parseHTML(html);
// --- Primary: Readability article extraction ---
const article = new Readability(document).parse();
if (article) {
const result = toReadableResult(url, format, article.textContent, article.content, {
const result = await toReadableResult(url, format, article.textContent, article.content, {
title: article.title,
byline: article.byline,
excerpt: article.excerpt,
@@ -55,7 +59,7 @@ export function extractReadableFromHtml(html: string, url: string, format: Reada
const innerHTML = el.innerHTML?.trim();
const textContent = el.textContent?.trim();
if (!innerHTML || !textContent) continue;
const result = toReadableResult(url, format, textContent, innerHTML, {
const result = await toReadableResult(url, format, textContent, innerHTML, {
title: document.title,
excerpt: textContent.slice(0, 240),
length: textContent.length,
@@ -67,15 +71,16 @@ export function extractReadableFromHtml(html: string, url: string, format: Reada
}
/** Shared builder for both extraction paths. */
function toReadableResult(
async function toReadableResult(
url: string,
format: ReadableFormat,
textContent: string | null | undefined,
htmlContent: string | null | undefined,
meta: { title?: string | null; byline?: string | null; excerpt?: string | null; length?: number | null },
): ReadableResult | null {
): Promise<ReadableResult | null> {
const text = normalize(textContent);
const markdown = format === "markdown" ? (normalize(htmlToBasicMarkdown(htmlContent ?? "")) ?? text) : undefined;
const markdown =
format === "markdown" ? (normalize(await htmlToBasicMarkdown(htmlContent ?? "")) ?? text) : undefined;
const normalizedText = format === "text" ? text : undefined;
if (!normalizedText && !markdown) return null;
return {
packages/coding-agent/src/tools/browser/tab-supervisor.ts
+2 -2
View File
@@ -16,7 +16,6 @@ import type {
WorkerInitPayload,
WorkerOutbound,
} from "./tab-protocol";
import { WorkerCore } from "./tab-worker";
interface WorkerHandle {
send(msg: WorkerInbound, transferList?: Transferable[]): void;
@@ -398,7 +397,7 @@ function wrapBunWorker(worker: Worker): WorkerHandle {
* entry. This preserves normal browser behavior but cannot interrupt synchronous
* infinite loops because user code runs on the main thread.
*/
function spawnInlineWorker(): WorkerHandle {
async function spawnInlineWorker(): Promise<WorkerHandle> {
const hostListeners = new Set<(message: WorkerOutbound) => void>();
const workerListeners = new Set<(message: WorkerInbound) => void>();
const workerTransport: Transport = {
@@ -413,6 +412,7 @@ function spawnInlineWorker(): WorkerHandle {
},
close: () => {},
};
const { WorkerCore } = await import("./tab-worker");
new WorkerCore(workerTransport);
return {
mode: "inline",
packages/coding-agent/src/tools/write.ts
+8 -1
View File
@@ -6,7 +6,6 @@ import type { Component } from "@oh-my-pi/pi-tui";
import { Text } from "@oh-my-pi/pi-tui";
import { isEnoent, isRecord, prompt, untilAborted } from "@oh-my-pi/pi-utils";
import { type Static, Type } from "@sinclair/typebox";
import { unzipSync, zipSync } from "fflate";
import { stripHashlinePrefixes } from "../edit";
import type { RenderResultOptions } from "../extensibility/custom-tools/types";
import { createLspWritethrough, type FileDiagnosticsResult, type WritethroughCallback, writethroughNoop } from "../lsp";
@@ -44,6 +43,12 @@ import {
import { ToolError } from "./tool-errors";
import { toolResult } from "./tool-result";
let fflateModulePromise: Promise<typeof import("fflate")> | undefined;
async function loadFflate(): Promise<typeof import("fflate")> {
if (!fflateModulePromise) fflateModulePromise = import("fflate");
return fflateModulePromise;
}
const writeSchema = Type.Object({
path: Type.String({ description: "file path", examples: ["src/new.ts"] }),
content: Type.String({ description: "file content" }),
@@ -229,6 +234,7 @@ export class WriteTool implements AgentTool<typeof writeSchema, WriteToolDetails
if (resolvedArchivePath.exists) {
try {
const bytes = await Bun.file(resolvedArchivePath.absolutePath).bytes();
const { unzipSync } = await loadFflate();
const existing = unzipSync(new Uint8Array(bytes));
for (const [entryPath, data] of Object.entries(existing)) {
zipEntries[entryPath.replace(/\\/g, "/")] = data;
@@ -241,6 +247,7 @@ export class WriteTool implements AgentTool<typeof writeSchema, WriteToolDetails
zipEntries[resolvedArchivePath.archiveSubPath] = new TextEncoder().encode(content);
try {
const { zipSync } = await loadFflate();
const zipBuffer = zipSync(zipEntries);
await Bun.write(resolvedArchivePath.absolutePath, zipBuffer);
} catch (error) {
packages/coding-agent/src/web/scrapers/crossref.ts
+3 -3
View File
@@ -66,10 +66,10 @@ function formatDate(date?: CrossrefDate): string | null {
return formatted.join("-");
}
function formatAbstract(abstract?: string): string | null {
async function formatAbstract(abstract?: string): Promise<string | null> {
if (!abstract) return null;
const normalized = abstract.replace(/<\/?jats:p[^>]*>/g, match => (match.startsWith("</") ? "</p>" : "<p>"));
const markdown = htmlToBasicMarkdown(normalized);
const markdown = await htmlToBasicMarkdown(normalized);
return markdown.trim().length > 0 ? markdown : null;
}
@@ -114,7 +114,7 @@ export const handleCrossref: SpecialHandler = async (
formatDate(message.issued) ||
formatDate(message.created);
const doiValue = message.DOI || doi;
const abstract = formatAbstract(message.abstract);
const abstract = await formatAbstract(message.abstract);
const type = message.type?.replace(/-/g, " ");
let md = `# ${title}\n\n`;
packages/coding-agent/src/web/scrapers/devto.ts
+1 -1
View File
@@ -133,7 +133,7 @@ export const handleDevTo: SpecialHandler = async (
if (article.body_markdown) {
md += article.body_markdown;
} else if (article.body_html) {
md += htmlToBasicMarkdown(article.body_html);
md += await htmlToBasicMarkdown(article.body_html);
}
notes.push("Fetched via dev.to API");
packages/coding-agent/src/web/scrapers/discourse.ts
+5 -5
View File
@@ -77,12 +77,12 @@ function formatCategory(topic: DiscourseTopic): string | null {
return parts.length ? parts.join(" ") : null;
}
function formatPostBody(post: DiscoursePost): string {
async function formatPostBody(post: DiscoursePost): Promise<string> {
const raw = post.raw?.trim();
if (raw) return raw;
const cooked = post.cooked?.trim();
if (!cooked) return "";
return htmlToBasicMarkdown(cooked);
return await htmlToBasicMarkdown(cooked);
}
function buildTopicUrl(baseUrl: string, topicId: string): string {
@@ -168,9 +168,9 @@ export const handleDiscourse: SpecialHandler = async (
md += "\n";
const description = topic.excerpt
? htmlToBasicMarkdown(topic.excerpt)
? await htmlToBasicMarkdown(topic.excerpt)
: posts.length
? formatPostBody(posts[0])
? await formatPostBody(posts[0])
: "";
if (description) {
md += `## Description\n\n${description}\n\n`;
@@ -182,7 +182,7 @@ export const handleDiscourse: SpecialHandler = async (
const author = formatAuthor({ name: post.name, username: post.username });
const date = formatIsoDate(post.created_at);
const likes = post.like_count ?? 0;
const content = formatPostBody(post);
const content = await formatPostBody(post);
const postLabel = post.post_number != null ? `Post ${post.post_number}` : `Post ${post.id}`;
md += `### ${postLabel} - ${author} - ${date} - Likes: ${likes}\n\n`;
packages/coding-agent/src/web/scrapers/firefox-addons.ts
+1 -1
View File
@@ -112,7 +112,7 @@ export const handleFirefoxAddons: SpecialHandler = async (
const name = getLocalizedText(data.name, defaultLocale) ?? slug;
const summary = getLocalizedText(data.summary, defaultLocale);
const descriptionRaw = getLocalizedText(data.description, defaultLocale);
const description = descriptionRaw ? htmlToBasicMarkdown(descriptionRaw) : undefined;
const description = descriptionRaw ? await htmlToBasicMarkdown(descriptionRaw) : undefined;
const authors = (data.authors ?? [])
.map(author => author.name ?? "")
packages/coding-agent/src/web/scrapers/flathub.ts
+2 -2
View File
@@ -170,7 +170,7 @@ export const handleFlathub: SpecialHandler = async (
}
if (app.description) {
const description = htmlToBasicMarkdown(app.description);
const description = await htmlToBasicMarkdown(app.description);
if (description) md += `\n## Description\n\n${description}\n`;
}
@@ -204,7 +204,7 @@ export const handleFlathub: SpecialHandler = async (
md += `${line}\n`;
if (release.description) {
const releaseDesc = htmlToBasicMarkdown(release.description).replace(/\n+/g, " ").trim();
const releaseDesc = (await htmlToBasicMarkdown(release.description)).replace(/\n+/g, " ").trim();
if (releaseDesc) md += ` - ${releaseDesc}\n`;
}
}
packages/coding-agent/src/web/scrapers/gitlab.ts
+1 -1
View File
@@ -259,7 +259,7 @@ async function renderGitLabIssue(
}
md += `\n---\n\n## Description\n\n`;
md += issue.description ? htmlToBasicMarkdown(issue.description) : "*No description*";
md += issue.description ? await htmlToBasicMarkdown(issue.description) : "*No description*";
return { content: md, ok: true };
}
packages/coding-agent/src/web/scrapers/go-pkg.ts
+2 -2
View File
@@ -159,7 +159,7 @@ export const handleGoPkg: SpecialHandler = async (
// Get overview paragraph
const overview = docSection.querySelector(".go-Message");
if (overview) {
const overviewMd = htmlToBasicMarkdown(overview.innerHTML);
const overviewMd = await htmlToBasicMarkdown(overview.innerHTML);
sections.push(overviewMd);
sections.push("");
}
@@ -172,7 +172,7 @@ export const handleGoPkg: SpecialHandler = async (
const docParts: string[] = [];
for (let i = 0; i < Math.min(3, paragraphs.length); i++) {
const p = paragraphs[i];
const text = htmlToBasicMarkdown(p.innerHTML).trim();
const text = (await htmlToBasicMarkdown(p.innerHTML)).trim();
if (text) {
docParts.push(text);
}
packages/coding-agent/src/web/scrapers/jetbrains-marketplace.ts
+1 -1
View File
@@ -108,7 +108,7 @@ export const handleJetBrainsMarketplace: SpecialHandler = async (
const vendorName = plugin.vendor?.name ?? plugin.vendor?.publicName;
const descriptionSource = plugin.description ?? plugin.preview ?? "";
const description = descriptionSource ? htmlToBasicMarkdown(descriptionSource) : "";
const description = descriptionSource ? await htmlToBasicMarkdown(descriptionSource) : "";
const tags = (plugin.tags ?? []).map(tag => tag.name).filter(Boolean) as string[];
const rating = extractRating(plugin);
const buildCompatibility = update ? formatBuildCompatibility(update) : null;
packages/coding-agent/src/web/scrapers/mastodon.ts
+9 -9
View File
@@ -89,11 +89,11 @@ function formatDate(isoDate: string): string {
/**
* Format a status/post as markdown
*/
function formatStatus(status: MastodonStatus, isReblog = false): string {
async function formatStatus(status: MastodonStatus, isReblog = false): Promise<string> {
// Handle reblogs (boosts)
if (status.reblog && !isReblog) {
let md = `🔁 **${status.account.display_name || status.account.username}** boosted:\n\n`;
md += formatStatus(status.reblog, true);
md += await formatStatus(status.reblog, true);
return md;
}
@@ -116,7 +116,7 @@ function formatStatus(status: MastodonStatus, isReblog = false): string {
}
// Main content (convert HTML to markdown)
const content = htmlToBasicMarkdown(status.content);
const content = await htmlToBasicMarkdown(status.content);
md += `${content}\n\n`;
// Poll
@@ -152,7 +152,7 @@ function formatStatus(status: MastodonStatus, isReblog = false): string {
/**
* Format an account/profile as markdown
*/
function formatAccount(account: MastodonAccount): string {
async function formatAccount(account: MastodonAccount): Promise<string> {
let md = `# ${account.display_name || account.username}\n\n`;
md += `**@${account.acct}**`;
@@ -161,7 +161,7 @@ function formatAccount(account: MastodonAccount): string {
// Bio
if (account.note) {
const bio = htmlToBasicMarkdown(account.note);
const bio = await htmlToBasicMarkdown(account.note);
if (bio && bio !== account.display_name) {
md += `${bio}\n\n`;
}
@@ -179,7 +179,7 @@ function formatAccount(account: MastodonAccount): string {
if (account.fields && account.fields.length > 0) {
md += "\n**Profile Fields:**\n";
for (const field of account.fields) {
const value = htmlToBasicMarkdown(field.value);
const value = await htmlToBasicMarkdown(field.value);
md += `- **${field.name}:** ${value}\n`;
}
}
@@ -228,7 +228,7 @@ export const handleMastodon: SpecialHandler = async (
const status = tryParseJson<MastodonStatus>(result.content);
if (!status) return null;
const md = formatStatus(status);
const md = await formatStatus(status);
return buildResult(md, {
url,
@@ -263,7 +263,7 @@ export const handleMastodon: SpecialHandler = async (
signal,
});
let md = formatAccount(account);
let md = await formatAccount(account);
if (statusesResult.ok) {
const statuses = tryParseJson<MastodonStatus[]>(statusesResult.content);
@@ -271,7 +271,7 @@ export const handleMastodon: SpecialHandler = async (
md += "\n---\n\n## Recent Posts\n\n";
for (const status of statuses.slice(0, 5)) {
md += `### ${formatDate(status.created_at)}\n\n`;
const content = htmlToBasicMarkdown(status.content);
const content = await htmlToBasicMarkdown(status.content);
md += `${content}\n\n`;
md += `\uD83D\uDCAC ${status.replies_count} \u00B7 \uD83D\uDD01 ${status.reblogs_count} \u00B7 \u2B50 ${status.favourites_count}\n\n`;
}
packages/coding-agent/src/web/scrapers/mdn.ts
+11 -7
View File
@@ -29,7 +29,7 @@ interface MDNDoc {
/**
* Convert MDN body sections to markdown
*/
function convertMDNBody(sections: MDNSection[]): string {
async function convertMDNBody(sections: MDNSection[]): Promise<string> {
const parts: string[] = [];
for (const section of sections) {
@@ -38,7 +38,7 @@ function convertMDNBody(sections: MDNSection[]): string {
switch (type) {
case "prose":
if (value.content) {
const markdown = htmlToBasicMarkdown(value.content);
const markdown = await htmlToBasicMarkdown(value.content);
if (value.title) {
const level = value.isH3 ? "###" : "##";
parts.push(`${level} ${value.title}\n\n${markdown}`);
@@ -74,7 +74,7 @@ function convertMDNBody(sections: MDNSection[]): string {
if (value.items) {
for (const item of value.items) {
parts.push(`**${item.term}**`);
const desc = htmlToBasicMarkdown(item.description);
const desc = await htmlToBasicMarkdown(item.description);
parts.push(desc);
}
}
@@ -83,9 +83,13 @@ function convertMDNBody(sections: MDNSection[]): string {
case "table":
if (value.rows && value.rows.length > 0) {
// Simple markdown table
const header = value.rows[0].map(cell => htmlToBasicMarkdown(cell)).join(" | ");
const header = (await Promise.all(value.rows[0].map(cell => htmlToBasicMarkdown(cell)))).join(" | ");
const separator = value.rows[0].map(() => "---").join(" | ");
const bodyRows = value.rows.slice(1).map(row => row.map(cell => htmlToBasicMarkdown(cell)).join(" | "));
const bodyRows = await Promise.all(
value.rows
.slice(1)
.map(async row => (await Promise.all(row.map(cell => htmlToBasicMarkdown(cell)))).join(" | ")),
);
parts.push(`| ${header} |`);
parts.push(`| ${separator} |`);
@@ -144,12 +148,12 @@ export const handleMDN: SpecialHandler = async (url: string, timeout: number, si
parts.push(`# ${doc.title}`);
if (doc.summary) {
const summary = htmlToBasicMarkdown(doc.summary);
const summary = await htmlToBasicMarkdown(doc.summary);
parts.push(summary);
}
if (doc.body && doc.body.length > 0) {
const bodyMarkdown = convertMDNBody(doc.body);
const bodyMarkdown = await convertMDNBody(doc.body);
parts.push(bodyMarkdown);
}
packages/coding-agent/src/web/scrapers/pub-dev.ts
+1 -1
View File
@@ -125,7 +125,7 @@ export const handlePubDev: SpecialHandler = async (url: string, timeout: number,
/<div[^>]*class="[^"]*markdown-body[^"]*"[^>]*>([\s\S]*?)<\/div>/i,
);
if (readmeMatch) {
const readme = htmlToBasicMarkdown(readmeMatch[1]);
const readme = await htmlToBasicMarkdown(readmeMatch[1]);
if (readme.length > 100) {
md += `## README\n\n${readme}\n`;
packages/coding-agent/src/web/scrapers/rawg.ts
+3 -3
View File
@@ -63,7 +63,7 @@ export const handleRawg: SpecialHandler = async (
md += `**RAWG:** https://rawg.io/games/${encodeURIComponent(slug)}\n`;
md += "\n";
const description = extractDescription(game);
const description = await extractDescription(game);
if (description) {
md += `## Description\n\n${description}\n`;
}
@@ -91,11 +91,11 @@ function requiresApiKey(game: RawgGameResponse): boolean {
return detail.includes("api key") || detail.includes("key is required") || detail.includes("apikey");
}
function extractDescription(game: RawgGameResponse): string | null {
async function extractDescription(game: RawgGameResponse): Promise<string | null> {
if (game.description_raw) return game.description_raw.trim();
if (!game.description) return null;
const markdown = htmlToBasicMarkdown(game.description).trim();
const markdown = (await htmlToBasicMarkdown(game.description)).trim();
return markdown || null;
}
packages/coding-agent/src/web/scrapers/readthedocs.ts
+1 -1
View File
@@ -101,7 +101,7 @@ export const handleReadTheDocs: SpecialHandler = async (
// If no raw source, convert HTML to markdown
if (!content && mainContent) {
const html = mainContent.innerHTML;
content = htmlToBasicMarkdown(html);
content = await htmlToBasicMarkdown(html);
}
if (!content) {
packages/coding-agent/src/web/scrapers/spdx.ts
+1 -1
View File
@@ -94,7 +94,7 @@ export const handleSpdx: SpecialHandler = async (
const licenseText = license.licenseText
? license.licenseText
: license.licenseTextHtml
? htmlToBasicMarkdown(license.licenseTextHtml)
? await htmlToBasicMarkdown(license.licenseTextHtml)
: null;
if (licenseText) {
packages/coding-agent/src/web/scrapers/stackoverflow.ts
+2 -2
View File
@@ -90,7 +90,7 @@ export const handleStackOverflow: SpecialHandler = async (
md += question.is_answered ? " (Answered)" : "";
md += `\n**Tags:** ${question.tags.join(", ")}\n`;
md += `**Asked by:** ${question.owner.display_name} · ${formatIsoDate(question.creation_date * 1000)}\n\n`;
md += `---\n\n## Question\n\n${htmlToBasicMarkdown(question.body)}\n\n`;
md += `---\n\n## Question\n\n${await htmlToBasicMarkdown(question.body)}\n\n`;
// Fetch answers
const aUrl = `https://api.stackexchange.com/2.3/questions/${questionId}/answers?order=desc&sort=votes&site=${site}&filter=withbody`;
@@ -103,7 +103,7 @@ export const handleStackOverflow: SpecialHandler = async (
for (const answer of aData.items.slice(0, 5)) {
const accepted = answer.is_accepted ? " (Accepted)" : "";
md += `### Score: ${answer.score}${accepted} · by ${answer.owner.display_name}\n\n`;
md += `${htmlToBasicMarkdown(answer.body)}\n\n---\n\n`;
md += `${await htmlToBasicMarkdown(answer.body)}\n\n---\n\n`;
}
}
}
packages/coding-agent/src/web/scrapers/types.ts
+53 -39
View File
@@ -2,8 +2,8 @@
* Shared types and utilities for web-fetch handlers
*/
import { ptree } from "@oh-my-pi/pi-utils";
import TurndownService from "turndown";
import { gfm } from "turndown-plugin-gfm";
import type TurndownService from "turndown";
import { ToolAbortError } from "../../tools/tool-errors";
export { formatNumber } from "@oh-my-pi/pi-utils";
@@ -155,28 +155,8 @@ export async function loadPage(url: string, options: LoadPageOptions = {}): Prom
return { content: "", contentType: "", finalUrl: url, ok: false };
}
/** Module-level Turndown instance — matches markit-ai's configuration. */
const turndown = new TurndownService({
headingStyle: "atx",
codeBlockStyle: "fenced",
bulletListMarker: "-",
});
turndown.use(gfm);
turndown.addRule("strikethrough", {
filter: ["del", "s", "strike"],
replacement(content) {
return `~~${content}~~`;
},
});
turndown.addRule("heading", {
filter: ["h1", "h2", "h3", "h4", "h5", "h6"],
replacement(content, node) {
const level = Number(node.nodeName.charAt(1));
const prefix = "#".repeat(level);
const cleaned = content.replace(/\\([.])/g, "$1").trim();
return `\n\n${prefix} ${cleaned}\n\n`;
},
});
/** Module-level Turndown instance — built lazily on first use. */
let turndownPromise: Promise<TurndownService> | undefined;
type TurndownListParent = {
nodeName: string;
@@ -184,27 +164,61 @@ type TurndownListParent = {
children: ArrayLike<unknown>;
};
turndown.addRule("listItem", {
filter: "li",
replacement(content, node, options) {
content = content.replace(/^\n+/, "").replace(/\n+$/, "\n").replace(/\n/gm, "\n ");
const parent = node.parentNode as unknown as TurndownListParent | null;
let prefix = `${options.bulletListMarker} `;
if (parent?.nodeName === "OL") {
const start = parent.getAttribute("start");
const index = Array.prototype.indexOf.call(parent.children, node);
prefix = `${(start ? Number(start) : 1) + index}. `;
}
return prefix + content + (node.nextSibling ? "\n" : "");
},
});
function getTurndown(): Promise<TurndownService> {
turndownPromise ||= initTurndown();
return turndownPromise;
}
async function initTurndown(): Promise<TurndownService> {
const [{ default: TurndownService }, { gfm }] = await Promise.all([
import("turndown"),
import("turndown-plugin-gfm"),
]);
const turndown = new TurndownService({
headingStyle: "atx",
codeBlockStyle: "fenced",
bulletListMarker: "-",
});
turndown.use(gfm);
turndown.addRule("strikethrough", {
filter: ["del", "s", "strike"],
replacement(content) {
return `~~${content}~~`;
},
});
turndown.addRule("heading", {
filter: ["h1", "h2", "h3", "h4", "h5", "h6"],
replacement(content, node) {
const level = Number(node.nodeName.charAt(1));
const prefix = "#".repeat(level);
const cleaned = content.replace(/\\([.])/g, "$1").trim();
return `\n\n${prefix} ${cleaned}\n\n`;
},
});
turndown.addRule("listItem", {
filter: "li",
replacement(content, node, options) {
content = content.replace(/^\n+/, "").replace(/\n+$/, "\n").replace(/\n/gm, "\n ");
const parent = node.parentNode as unknown as TurndownListParent | null;
let prefix = `${options.bulletListMarker} `;
if (parent?.nodeName === "OL") {
const start = parent.getAttribute("start");
const index = Array.prototype.indexOf.call(parent.children, node);
prefix = `${(start ? Number(start) : 1) + index}. `;
}
return prefix + content + (node.nextSibling ? "\n" : "");
},
});
return turndown;
}
/**
* Convert HTML to markdown using Turndown with GFM support.
* Strips script/style tags before conversion.
*/
export function htmlToBasicMarkdown(html: string): string {
export async function htmlToBasicMarkdown(html: string): Promise<string> {
const cleaned = html.replace(/<script[\s\S]*?<\/script>/gi, "").replace(/<style[\s\S]*?<\/style>/gi, "");
const turndown = await getTurndown();
return turndown.turndown(cleaned).trim();
}
packages/coding-agent/src/web/scrapers/w3c.ts
+1 -1
View File
@@ -100,7 +100,7 @@ export const handleW3c: SpecialHandler = async (
const title = getString(specPayload, "title");
const shortnameValue = getString(specPayload, "shortname") ?? shortname;
const description = getString(specPayload, "description") ?? getString(specPayload, "abstract");
const abstract = description ? htmlToBasicMarkdown(description) : undefined;
const abstract = description ? await htmlToBasicMarkdown(description) : undefined;
const latestVersionUrl =
getString(latestPayload, "uri") ??
packages/coding-agent/test/core/python-kernel-display.test.ts
+8 -8
View File
@@ -2,8 +2,8 @@ import { describe, expect, it } from "bun:test";
import { renderKernelDisplay } from "@oh-my-pi/pi-coding-agent/eval/py/kernel";
describe("PythonKernel display rendering", () => {
it("normalizes text/plain output and returns no display outputs", () => {
const { text, outputs } = renderKernelDisplay({
it("normalizes text/plain output and returns no display outputs", async () => {
const { text, outputs } = await renderKernelDisplay({
data: { "text/plain": "hello" },
});
@@ -11,8 +11,8 @@ describe("PythonKernel display rendering", () => {
expect(outputs).toHaveLength(0);
});
it("collects image and json display outputs without text", () => {
const { text, outputs } = renderKernelDisplay({
it("collects image and json display outputs without text", async () => {
const { text, outputs } = await renderKernelDisplay({
data: { "image/png": "abc", "application/json": { foo: "bar" } },
});
@@ -23,8 +23,8 @@ describe("PythonKernel display rendering", () => {
]);
});
it("converts text/html to markdown", () => {
const { text, outputs } = renderKernelDisplay({
it("converts text/html to markdown", async () => {
const { text, outputs } = await renderKernelDisplay({
data: { "text/html": "<p><strong>Hello</strong></p>" },
});
@@ -32,8 +32,8 @@ describe("PythonKernel display rendering", () => {
expect(text).toBe("**Hello**\n");
});
it("combines text/plain with json output", () => {
const { text, outputs } = renderKernelDisplay({
it("combines text/plain with json output", async () => {
const { text, outputs } = await renderKernelDisplay({
data: { "text/plain": "value", "application/json": { ok: true } },
});
packages/coding-agent/test/tools/browser-readable.test.ts
+4 -4
View File
@@ -2,7 +2,7 @@ import { describe, expect, it } from "bun:test";
import { extractReadableFromHtml } from "@oh-my-pi/pi-coding-agent/tools/browser";
describe("browser readable extraction", () => {
it("extracts markdown content from article-style pages", () => {
it("extracts markdown content from article-style pages", async () => {
const html = `<!doctype html>
<html>
<head><title>Docs</title></head>
@@ -14,7 +14,7 @@ describe("browser readable extraction", () => {
</body>
</html>`;
const result = extractReadableFromHtml(html, "https://example.com/docs", "markdown");
const result = await extractReadableFromHtml(html, "https://example.com/docs", "markdown");
expect(result).not.toBeNull();
expect(result?.title).toBe("Docs");
@@ -22,7 +22,7 @@ describe("browser readable extraction", () => {
expect(result?.markdown).toContain("stores output only when you opt in");
});
it("extracts docs-style main content", () => {
it("extracts docs-style main content", async () => {
const html = `<!doctype html>
<html>
<head><title>Reference</title></head>
@@ -39,7 +39,7 @@ describe("browser readable extraction", () => {
</body>
</html>`;
const result = extractReadableFromHtml(html, "https://developers.openai.com/apps-sdk/reference", "text");
const result = await extractReadableFromHtml(html, "https://developers.openai.com/apps-sdk/reference", "text");
expect(result).not.toBeNull();
expect(result?.title).toBe("Reference");