diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index a4d66e8d2..20f1e28fd 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -2,6 +2,13 @@ ## [Unreleased] +### Added + +- Added `searxng.safesearch` setting option for SearXNG searches +- Added support for Cloudflare AI Gateway routing for Gemini search +- Added support for Exa MCP search provider +- Added domain inclusion/exclusion filtering and URL deduplication for TinyFish search + ## [17.2.12] - 2026-08-08 ### Fixed diff --git a/packages/coding-agent/src/config/settings-schema.ts b/packages/coding-agent/src/config/settings-schema.ts index 3083aabd1..bcb0a3190 100644 --- a/packages/coding-agent/src/config/settings-schema.ts +++ b/packages/coding-agent/src/config/settings-schema.ts @@ -5471,6 +5471,11 @@ export const SETTINGS_SCHEMA = { default: undefined, }, + "searxng.safesearch": { + type: "number", + default: undefined, + }, + "commit.mapReduceEnabled": { type: "boolean", default: true }, "commit.mapReduceMinFiles": { type: "number", default: 4 }, diff --git a/packages/coding-agent/src/tools/fetch.ts b/packages/coding-agent/src/tools/fetch.ts index 4638b1b71..b9c7c9472 100644 --- a/packages/coding-agent/src/tools/fetch.ts +++ b/packages/coding-agent/src/tools/fetch.ts @@ -576,6 +576,23 @@ async function parseFeedToMarkdown(content: string, maxItems = 10): Promise { const response = await fetchImpl(`https://r.jina.ai/${url}`, { - headers: { Accept: "text/markdown" }, + headers: { + Accept: "text/markdown", + "X-No-Cache": "true", + }, signal: remoteSignal(), }); - return response.ok ? await response.text() : null; + if (!response.ok) return null; + const contentLength = Number(response.headers.get("content-length")); + if (Number.isFinite(contentLength) && contentLength > JINA_READER_MAX_BYTES) return null; + return parseJinaReaderContent(await response.text()); }, }; diff --git a/packages/coding-agent/src/web/kagi.ts b/packages/coding-agent/src/web/kagi.ts index 0f2c776dd..a07217085 100644 --- a/packages/coding-agent/src/web/kagi.ts +++ b/packages/coding-agent/src/web/kagi.ts @@ -83,6 +83,7 @@ export interface KagiErrorResponse { detail?: string; } + // --------------------------------------------------------------------------- // Error Handling // --------------------------------------------------------------------------- @@ -98,7 +99,7 @@ export class KagiApiError extends Error { } function extractKagiErrorMessage(payload: unknown): string | null { - if (!payload || typeof payload !== "object") return null; + if (!payload || typeof payload !== "object" || Array.isArray(payload)) return null; const record = payload as Record; for (const value of [record.message, record.detail]) { @@ -107,17 +108,20 @@ function extractKagiErrorMessage(payload: unknown): string | null { } } - if (typeof record.error === "string" && record.error.trim().length > 0) { - return record.error.trim(); - } - - if (Array.isArray(record.error)) { - for (const entry of record.error) { + for (const errors of [record.error, record.errors]) { + if (typeof errors === "string" && errors.trim().length > 0) { + return errors.trim(); + } + if (!Array.isArray(errors)) continue; + for (const entry of errors) { if (!entry || typeof entry !== "object") continue; const e = entry as Record; - for (const value of [e.message, e.msg]) { - if (typeof value === "string" && value.trim().length > 0) { - return value.trim(); + for (const value of [e.message, e.msg, e.code]) { + if ( + (typeof value === "string" && value.trim().length > 0) || + (typeof value === "number" && Number.isFinite(value)) + ) { + return String(value).trim(); } } } @@ -147,6 +151,34 @@ function parseKagiErrorResponse(statusCode: number, responseText: string): KagiA } } +function parseKagiSuccessResponse(statusCode: number, responseText: string): KagiSearchResponse { + let payload: unknown; + try { + payload = JSON.parse(responseText); + } catch { + throw new KagiApiError("Kagi API returned an invalid response: invalid JSON", statusCode); + } + if (!payload || typeof payload !== "object" || Array.isArray(payload)) { + throw new KagiApiError("Kagi API returned an invalid response: expected an object envelope", statusCode); + } + + const record = payload as Record; + const errorMessage = extractKagiErrorMessage(payload); + if (errorMessage && (record.error !== undefined || record.errors !== undefined)) { + const errors = Array.isArray(record.error) ? record.error : Array.isArray(record.errors) ? record.errors : []; + const first = errors[0]; + const code = + first && typeof first === "object" && typeof (first as Record).code === "number" + ? ((first as Record).code as number) + : statusCode; + throw createKagiApiError(code, errorMessage); + } + if (record.data !== undefined && (!record.data || typeof record.data !== "object" || Array.isArray(record.data))) { + throw new KagiApiError("Kagi API returned an invalid response: expected data to be an object", statusCode); + } + return payload as KagiSearchResponse; +} + // --------------------------------------------------------------------------- // Public API // --------------------------------------------------------------------------- @@ -216,23 +248,40 @@ function buildRequestBody(query: string, options: KagiSearchOptions): KagiSearch return req; } -/** Push every item in a result bucket as a source, with an optional title tag. */ -function collectSources(sources: KagiSearchSource[], items: KagiSearchResultItem[] | undefined, tag?: string): void { - if (!items) return; - for (const item of items) { +function firstNonEmptyString(...values: unknown[]): string | undefined { + for (const value of values) { + if (typeof value === "string" && value.trim().length > 0) return value.trim(); + } + return undefined; +} + +/** Push every valid item in a result bucket as a source, with an optional title tag. */ +function collectSources(sources: KagiSearchSource[], items: unknown, tag?: string): void { + if (!Array.isArray(items)) return; + for (const value of items) { + if (!value || typeof value !== "object" || Array.isArray(value)) continue; + const item = value as Record; + const url = firstNonEmptyString(item.url, item.href, item.link); + if (!url) continue; + const title = firstNonEmptyString(item.title, item.name) ?? url; sources.push({ - title: tag ? `${tag} ${item.title}` : item.title, - url: item.url, - snippet: item.snippet, - publishedDate: item.time, + title: tag ? `${tag} ${title}` : title, + url, + snippet: firstNonEmptyString(item.snippet, item.description, item.summary), + publishedDate: firstNonEmptyString(item.time), }); } } /** Pull a related/adjacent question from an item's props or fall back to title. */ -function questionOf(item: KagiSearchResultItem): string | undefined { - const q = item.props?.question ?? item.props?.query ?? item.title; - return typeof q === "string" && q.length > 0 ? q : undefined; +function questionOf(value: unknown): string | undefined { + if (!value || typeof value !== "object" || Array.isArray(value)) return undefined; + const item = value as Record; + const props = + item.props && typeof item.props === "object" && !Array.isArray(item.props) + ? (item.props as Record) + : undefined; + return firstNonEmptyString(props?.question, props?.query, item.title); } export async function searchWithKagi( @@ -269,11 +318,7 @@ export async function searchWithKagi( }, ); - const payload = (await response.json()) as KagiSearchResponse; - if (payload.error && payload.error.length > 0) { - const first = payload.error[0]; - throw createKagiApiError(first.code ?? response.status, extractKagiErrorMessage(payload) ?? first.message); - } + const payload = parseKagiSuccessResponse(response.status, await response.text()); const data = payload.data; const sources: KagiSearchSource[] = []; @@ -284,17 +329,30 @@ export async function searchWithKagi( collectSources(sources, data?.news, "[News]"); collectSources(sources, data?.infobox, "[Info]"); - for (const item of data?.adjacent_question ?? []) { - const q = questionOf(item); - if (q) relatedQuestions.push(q); + const adjacentQuestions: unknown = data?.adjacent_question; + if (Array.isArray(adjacentQuestions)) { + for (const item of adjacentQuestions) { + const question = questionOf(item); + if (question) relatedQuestions.push(question); + } } - for (const item of data?.related_search ?? []) { - const q = questionOf(item); - if (q) relatedQuestions.push(q); + const relatedSearches: unknown = data?.related_search; + if (Array.isArray(relatedSearches)) { + for (const item of relatedSearches) { + const question = questionOf(item); + if (question) relatedQuestions.push(question); + } } - const directAnswer = data?.direct_answer?.[0]; - const answer = directAnswer ? (directAnswer.snippet ?? directAnswer.title) : undefined; + const directAnswers: unknown = data?.direct_answer; + const directAnswer = Array.isArray(directAnswers) ? directAnswers[0] : undefined; + const answer = + directAnswer && typeof directAnswer === "object" && !Array.isArray(directAnswer) + ? firstNonEmptyString( + (directAnswer as Record).snippet, + (directAnswer as Record).title, + ) + : undefined; return { requestId: payload.meta?.trace ?? payload.meta?.id ?? "", diff --git a/packages/coding-agent/src/web/parallel.ts b/packages/coding-agent/src/web/parallel.ts index 5811b7122..a321a8f09 100644 --- a/packages/coding-agent/src/web/parallel.ts +++ b/packages/coding-agent/src/web/parallel.ts @@ -146,6 +146,18 @@ export function parseParallelErrorResponse(statusCode: number, responseText: str } } +export async function parseParallelJsonResponse( + response: Response, + operation: "search" | "extract", +): Promise { + try { + return await response.json(); + } catch (err) { + const detail = err instanceof Error ? err.message : String(err); + throw new ParallelApiError(`Parallel ${operation} returned invalid JSON: ${detail}`); + } +} + function getAuthHeaders(apiKey: string): { Accept: string; "Content-Type": string; @@ -316,7 +328,7 @@ export async function searchWithParallel( throw parseParallelErrorResponse(response.status, await response.text()); } - const payload: unknown = await response.json(); + const payload = await parseParallelJsonResponse(response, "search"); return parseParallelSearchPayload(payload); } @@ -349,6 +361,6 @@ export async function extractWithParallel( throw parseParallelErrorResponse(response.status, await response.text()); } - const payload: unknown = await response.json(); + const payload = await parseParallelJsonResponse(response, "extract"); return parseExtractPayload(payload); } diff --git a/packages/coding-agent/src/web/search/providers/brave.ts b/packages/coding-agent/src/web/search/providers/brave.ts index 0b5fe65e3..5ab5a4b44 100644 --- a/packages/coding-agent/src/web/search/providers/brave.ts +++ b/packages/coding-agent/src/web/search/providers/brave.ts @@ -4,7 +4,7 @@ * Calls Brave's web search REST API and maps results into the unified * SearchResponse shape used by the web search tool. */ -import { type AuthStorage, type FetchImpl, getEnvApiKey } from "@oh-my-pi/pi-ai"; +import { type ApiKey, type AuthStorage, type FetchImpl, getEnvApiKey, withAuth } from "@oh-my-pi/pi-ai"; import type { SearchResponse, SearchSource } from "../../../web/search/types"; import { SearchProviderError } from "../../../web/search/types"; import type { QuerySyntax, StructuredQuery } from "../query"; @@ -17,6 +17,9 @@ import { classifyProviderHttpError, withHardTimeout } from "./utils"; const BRAVE_SEARCH_URL = "https://api.search.brave.com/res/v1/web/search"; const DEFAULT_NUM_RESULTS = 10; const MAX_NUM_RESULTS = 20; +const MAX_QUERY_CHARACTERS = 500; +const MAX_RESPONSE_BYTES = 2 * 1024 * 1024; +const MAX_ERROR_BYTES = 8 * 1024; const RECENCY_MAP: Record<"day" | "week" | "month" | "year", "pd" | "pw" | "pm" | "py"> = { day: "pd", @@ -51,62 +54,122 @@ export interface BraveSearchParams { num_results?: number; recency?: "day" | "week" | "month" | "year"; parsedQuery?: StructuredQuery; + /** Two-letter market code, or `ALL`. */ + country?: string; + /** Brave search language code, such as `en` or `zh-hans`. */ + search_lang?: string; + safesearch?: "off" | "moderate" | "strict"; + authStorage: AuthStorage; + sessionId?: string; signal?: AbortSignal; timeoutMs?: number; fetch?: FetchImpl; } -interface BraveSearchResult { - title?: string | null; - url?: string | null; - description?: string | null; - age?: string | null; - extra_snippets?: string[] | null; -} interface BraveSearchResponse { - web?: { - results?: BraveSearchResult[]; - }; + web?: unknown; } -/** Find BRAVE_API_KEY from environment or .env files. */ -export function findApiKey(): string | null { - return getEnvApiKey("brave") ?? null; +function normalizeText(value: unknown, maxLength: number): string | undefined { + if (typeof value !== "string") return undefined; + const text = value + .replace(/<[^>]*>/g, " ") + .replace(/\s+/g, " ") + .trim(); + if (!text) return undefined; + return text.length <= maxLength ? text : `${text.slice(0, maxLength - 1)}…`; } -function buildSnippet(result: BraveSearchResult): string | undefined { - const snippets: string[] = []; +function normalizeUrl(value: unknown): string | undefined { + if (typeof value !== "string" || value.length > 2048) return undefined; + try { + const url = new URL(value); + if (url.protocol !== "http:" && url.protocol !== "https:") return undefined; + return url.toString(); + } catch { + return undefined; + } +} - if (result.description?.trim()) { - snippets.push(result.description.trim()); +function webResults(response: BraveSearchResponse): readonly unknown[] { + if (typeof response.web !== "object" || response.web === null || !("results" in response.web)) return []; + return Array.isArray(response.web.results) ? response.web.results : []; +} + +async function readLimitedText(response: Response, maxBytes: number, truncate = false): Promise { + if (!response.body) return ""; + const reader = response.body.getReader(); + let buffer = new Uint8Array(Math.min(maxBytes, 64 * 1024)); + let bytes = 0; + + try { + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + const accepted = Math.min(value.byteLength, maxBytes - bytes); + const nextBytes = bytes + accepted; + if (nextBytes > buffer.byteLength) { + const grown = new Uint8Array(Math.min(maxBytes, Math.max(nextBytes, buffer.byteLength * 2))); + grown.set(buffer.subarray(0, bytes)); + buffer = grown; + } + buffer.set(value.subarray(0, accepted), bytes); + bytes = nextBytes; + if (accepted < value.byteLength) { + await reader.cancel().catch(() => undefined); + if (!truncate) throw new SearchProviderError("brave", "Brave API response exceeded 2 MiB", 500); + break; + } + } + } finally { + reader.releaseLock(); } - if (Array.isArray(result.extra_snippets)) { - for (const snippet of result.extra_snippets) { - if (!snippet?.trim()) continue; - if (snippets.includes(snippet.trim())) continue; - snippets.push(snippet.trim()); + return new TextDecoder().decode(buffer.subarray(0, bytes)); +} + +function buildSnippet(result: object): string | undefined { + const snippets = new Set(); + const description = normalizeText("description" in result ? result.description : undefined, 8_000); + if (description) snippets.add(description); + + const extras = "extra_snippets" in result ? result.extra_snippets : undefined; + if (Array.isArray(extras)) { + for (const value of extras) { + const snippet = normalizeText(value, 8_000); + if (snippet) snippets.add(snippet); } } - return snippets.length > 0 ? snippets.join("\n") : undefined; + const combined = [...snippets].join("\n"); + return combined ? (combined.length <= 8_000 ? combined : `${combined.slice(0, 7_999)}…`) : undefined; } async function callBraveSearch( apiKey: string, params: BraveSearchParams, ): Promise<{ response: BraveSearchResponse; requestId?: string }> { - const numResults = clampNumResults(params.num_results, DEFAULT_NUM_RESULTS, MAX_NUM_RESULTS); + const numResults = Math.floor(clampNumResults(params.num_results, DEFAULT_NUM_RESULTS, MAX_NUM_RESULTS)); const parsed = params.parsedQuery ?? parseSearchQuery(params.query); + const query = parsed.hasDirectives ? formatQuery(parsed, BRAVE_QUERY_SYNTAX) : params.query; + if (query.length > MAX_QUERY_CHARACTERS) { + throw new SearchProviderError( + "brave", + `Brave search queries cannot exceed ${MAX_QUERY_CHARACTERS} characters`, + 400, + ); + } const url = new URL(BRAVE_SEARCH_URL); - url.searchParams.set("q", parsed.hasDirectives ? formatQuery(parsed, BRAVE_QUERY_SYNTAX) : params.query); + url.searchParams.set("q", query); url.searchParams.set("count", String(numResults)); url.searchParams.set("extra_snippets", "true"); + url.searchParams.set("text_decorations", "false"); + url.searchParams.set("safesearch", params.safesearch ?? "moderate"); + if (params.country) url.searchParams.set("country", params.country.toUpperCase()); + if (params.search_lang) url.searchParams.set("search_lang", params.search_lang); const freshness = braveFreshness(parsed, params.recency); - if (freshness) { - url.searchParams.set("freshness", freshness); - } + if (freshness) url.searchParams.set("freshness", freshness); const fetchImpl = params.fetch ?? fetch; const response = await fetchImpl(url, { @@ -118,36 +181,47 @@ async function callBraveSearch( }); if (!response.ok) { - const errorText = await response.text(); + const errorText = await readLimitedText(response, MAX_ERROR_BYTES, true); const classified = classifyProviderHttpError("brave", response.status, errorText); if (classified) throw classified; throw new SearchProviderError("brave", `Brave API error (${response.status}): ${errorText}`, response.status); } - const data = (await response.json()) as BraveSearchResponse; + const raw = await readLimitedText(response, MAX_RESPONSE_BYTES); + let data: BraveSearchResponse; + try { + data = JSON.parse(raw) as BraveSearchResponse; + } catch { + throw new SearchProviderError("brave", "Brave API returned invalid JSON", 500); + } const requestId = response.headers.get("x-request-id") ?? response.headers.get("request-id") ?? undefined; return { response: data, requestId }; } /** Execute Brave web search. */ export async function searchBrave(params: BraveSearchParams): Promise { - const numResults = clampNumResults(params.num_results, DEFAULT_NUM_RESULTS, MAX_NUM_RESULTS); - const apiKey = findApiKey(); - if (!apiKey) { - throw new Error("BRAVE_API_KEY not found. Set it in environment or .env file."); - } - - const { response, requestId } = await callBraveSearch(apiKey, params); + const numResults = Math.floor(clampNumResults(params.num_results, DEFAULT_NUM_RESULTS, MAX_NUM_RESULTS)); + const keyOrResolver: ApiKey = params.authStorage.resolver("brave", { + sessionId: params.sessionId, + }); + const { response, requestId } = await withAuth(keyOrResolver, key => callBraveSearch(key, params), { + signal: params.signal, + missingKeyMessage: + 'Brave credentials not found. Set BRAVE_API_KEY or configure an API key for provider "brave".', + }); const sources: SearchSource[] = []; - for (const result of response.web?.results ?? []) { - if (!result.url) continue; + for (const result of webResults(response)) { + if (typeof result !== "object" || result === null) continue; + const url = normalizeUrl("url" in result ? result.url : undefined); + if (!url) continue; + const publishedDate = normalizeText("age" in result ? result.age : undefined, 100); sources.push({ - title: result.title ?? result.url, - url: result.url, + title: normalizeText("title" in result ? result.title : undefined, 300) ?? url, + url, snippet: buildSnippet(result), - publishedDate: result.age ?? undefined, - ageSeconds: dateToAgeSeconds(result.age), + publishedDate, + ageSeconds: dateToAgeSeconds(publishedDate), }); } @@ -155,6 +229,7 @@ export async function searchBrave(params: BraveSearchParams): Promise { @@ -173,6 +248,8 @@ export class BraveProvider extends SearchProvider { num_results: params.numSearchResults ?? params.limit, recency: params.recency, parsedQuery: params.parsedQuery, + authStorage: params.authStorage, + sessionId: params.sessionId, signal: params.signal, timeoutMs: params.timeoutMs, fetch: params.fetch, diff --git a/packages/coding-agent/src/web/search/providers/codex.ts b/packages/coding-agent/src/web/search/providers/codex.ts index 2d6971957..3813c02b8 100644 --- a/packages/coding-agent/src/web/search/providers/codex.ts +++ b/packages/coding-agent/src/web/search/providers/codex.ts @@ -151,6 +151,13 @@ export interface CodexSearchParams { } /** Codex API response structure */ +interface CodexWebSearchSource { + url?: string; + source_website_url?: string; + title?: string; + caption?: string; +} + interface CodexResponseItem { type: string; id?: string; @@ -161,6 +168,9 @@ interface CodexResponseItem { arguments?: string; content?: CodexContentPart[]; summary?: Array<{ type: string; text: string }>; + action?: { sources?: CodexWebSearchSource[] }; + sources?: CodexWebSearchSource[]; + results?: CodexWebSearchSource[]; } interface CodexContentPart { @@ -221,12 +231,45 @@ function isImagePlaceholderAnswer(text: string): boolean { return IMAGE_PLACEHOLDER_ANSWERS.has(normalized); } -function addSource(sources: SearchSource[], source: SearchSource): void { - if (!sources.some(existing => existing.url === source.url)) { - sources.push(source); +function cleanSourceUrl(rawUrl: string): string { + try { + const url = new URL(rawUrl); + if (url.searchParams.get("utm_source") === "openai") { + url.searchParams.delete("utm_source"); + } + return url.toString(); + } catch { + return rawUrl.replace(/[?&]utm_source=openai$/u, ""); } } +function addSource(sources: SearchSource[], source: SearchSource): void { + const normalizedSource = { ...source, url: cleanSourceUrl(source.url) }; + const existing = sources.find(candidate => candidate.url === normalizedSource.url); + if (!existing) { + sources.push(normalizedSource); + return; + } + if (existing.title === existing.url && normalizedSource.title !== normalizedSource.url) { + existing.title = normalizedSource.title; + } + if (!existing.snippet && normalizedSource.snippet) { + existing.snippet = normalizedSource.snippet; + } +} + +function extractCitationSnippet(text: string, start: number | undefined, end: number | undefined): string | undefined { + if (start === undefined || end === undefined || !text) return undefined; + const before = Math.max(0, start - 100); + const after = Math.min(text.length, end + 100); + const snippet = text + .slice(before, after) + .replace(/\[([^\]]*)\]\([^)]*\)/g, "$1") + .trim(); + if (!snippet) return undefined; + return snippet.length > 300 ? `${snippet.slice(0, 297)}...` : snippet; +} + function countCharacter(text: string, target: string): number { let count = 0; for (const char of text) { @@ -428,6 +471,15 @@ function extractCodexSseError(rawEvent: Record): { code: string return { code, message }; } +function classifyCodexSseErrorStatus(code: string, message: string): number { + const detail = `${code} ${message}`.toLowerCase(); + if (/rate[- ]?limit|too many requests|quota|\b429\b/u.test(detail)) return 429; + if (/unauthori[sz]ed|\b401\b/u.test(detail)) return 401; + if (/forbidden|\b403\b/u.test(detail)) return 403; + if (/timeout|timed out/u.test(detail)) return 504; + return 500; +} + /** * Calls the Codex Responses API with web search tool enabled. * The caller provides the exact model id to send; retry / fallback policy @@ -455,6 +507,8 @@ async function callCodexSearch( model: requestedModel, stream: true, store: false, + include: ["web_search_call.action.sources"], + parallel_tool_calls: true, input: [ { type: "message", @@ -510,7 +564,11 @@ async function callCodexSearch( webSearchInvoked = true; } - if (eventType === "response.output_text.delta") { + if (eventType === "response.created") { + const resp = (rawEvent as { response?: CodexResponse }).response; + if (resp?.id) requestId = resp.id; + if (resp?.model) model = resp.model; + } else if (eventType === "response.output_text.delta") { const delta = typeof rawEvent.delta === "string" ? rawEvent.delta : ""; if (delta) { streamedAnswerParts.push(delta); @@ -518,7 +576,20 @@ async function callCodexSearch( } else if (eventType === "response.output_item.done") { const item = rawEvent.item as CodexResponseItem | undefined; if (!item) continue; - if (item.type === "web_search_call") webSearchInvoked = true; + if (item.type === "web_search_call") { + webSearchInvoked = true; + const sourceGroups = [item.action?.sources, item.sources, item.results]; + for (const group of sourceGroups) { + for (const source of group ?? []) { + const url = source.url ?? source.source_website_url; + if (!url) continue; + addSource(sources, { + title: source.title ?? source.caption ?? url, + url, + }); + } + } + } // Handle text message content and extract sources from annotations if (item.type === "message" && item.content) { @@ -530,8 +601,11 @@ async function callCodexSearch( if (part.annotations) { for (const annotation of part.annotations) { if (annotation.type === "url_citation" && annotation.url) { - // Deduplicate by URL - addSource(sources, { title: annotation.title ?? annotation.url, url: annotation.url }); + addSource(sources, { + title: annotation.title ?? annotation.url, + url: annotation.url, + snippet: extractCitationSnippet(part.text, annotation.start_index, annotation.end_index), + }); } } } @@ -563,13 +637,17 @@ async function callCodexSearch( } } else if (eventType === "error") { const { code, message } = extractCodexSseError(rawEvent); - throw new SearchProviderError("codex", `Codex error (${code}): ${message || "Unknown error"}`, 500); + throw new SearchProviderError( + "codex", + `Codex error (${code}): ${message || "Unknown error"}`, + classifyCodexSseErrorStatus(code, message), + ); } else if (eventType === "response.failed") { const { code, message } = extractCodexSseError(rawEvent); const detail = code ? `Codex request failed (${code}): ${message || "Request failed"}` : `Codex request failed: ${message || "Request failed"}`; - throw new SearchProviderError("codex", detail, 500); + throw new SearchProviderError("codex", detail, classifyCodexSseErrorStatus(code, message)); } } diff --git a/packages/coding-agent/src/web/search/providers/exa.ts b/packages/coding-agent/src/web/search/providers/exa.ts index 37d628e32..9e52a0d26 100644 --- a/packages/coding-agent/src/web/search/providers/exa.ts +++ b/packages/coding-agent/src/web/search/providers/exa.ts @@ -19,6 +19,9 @@ import { SearchProvider } from "./base"; import { classifyProviderHttpError, withHardTimeout } from "./utils"; const EXA_API_URL = "https://api.exa.ai/search"; +const EXA_MCP_URL = "https://mcp.exa.ai/mcp"; +const EXA_MCP_SOURCE = "oh-my-pi"; +const MAX_EXA_SNIPPET_CHARS = 500; const DEFAULT_EXA_SEARCH_DELAY_MS = getDefault("exa.searchDelayMs"); let nextExaSearchRequestAt = 0; @@ -329,13 +332,20 @@ async function callExaSearch(apiKey: string, params: ExaSearchParams): Promise; } function buildExaMcpArgs(params: ExaSearchParams): Record { - const args: Record = { query: params.query }; - if (params.num_results !== undefined) args.num_results = params.num_results; - if (params.type !== undefined) args.type = params.type; - if (params.include_domains !== undefined) args.include_domains = params.include_domains; - if (params.exclude_domains !== undefined) args.exclude_domains = params.exclude_domains; - if (params.start_published_date !== undefined) args.start_published_date = params.start_published_date; - if (params.end_published_date !== undefined) args.end_published_date = params.end_published_date; + const queryParts = [params.query]; + for (const domain of params.include_domains ?? []) { + const trimmed = domain.trim(); + if (trimmed) queryParts.push(`site:${trimmed}`); + } + for (const domain of params.exclude_domains ?? []) { + const trimmed = domain.trim(); + if (trimmed) queryParts.push(`-site:${trimmed}`); + } + if (params.start_published_date) queryParts.push(`after:${params.start_published_date}`); + if (params.end_published_date) queryParts.push(`before:${params.end_published_date}`); + + const args: Record = { query: queryParts.join(" ") }; + if (params.num_results !== undefined) args.numResults = params.num_results; return args; } @@ -346,11 +356,12 @@ async function callExaMcpSearch(params: ExaSearchParams): Promise; + isError?: boolean; }; error?: { code: number; @@ -381,6 +407,12 @@ async function callExaMcpSearch(params: ExaSearchParams): Promise item.type === "text" && typeof item.text === "string") + ?.text?.trim(); + throw new SearchProviderError("exa", message || "Exa MCP returned an error"); + } const responsePayload = normalizeExaMcpPayload(mcpResponse.result); if (isSearchResponse(responsePayload)) { return responsePayload as ExaSearchResponse; @@ -419,7 +451,10 @@ export async function searchExa(params: ExaSearchParams): Promise, string> = { month: "qdr:m", year: "qdr:y", }; +function resolveSearchUrl(): string { + const configured = process.env.FIRECRAWL_BASE_URL ?? process.env.FIRECRAWL_API_URL; + if (!configured?.trim()) return `${FIRECRAWL_DEFAULT_BASE_URL}/search`; + let url: URL; + try { + url = new URL(configured.trim()); + } catch { + throw new Error("Invalid Firecrawl base URL: expected an HTTP or HTTPS URL"); + } + if (url.protocol !== "http:" && url.protocol !== "https:") { + throw new Error("Invalid Firecrawl base URL: expected an HTTP or HTTPS URL"); + } + if (url.username || url.password) { + throw new Error("Invalid Firecrawl base URL: URL credentials are not allowed"); + } + url.search = ""; + url.hash = ""; + url.pathname = url.pathname.replace(/\/+$/, ""); + if (!/\/v[12]$/i.test(url.pathname)) url.pathname += "/v2"; + url.pathname += "/search"; + return url.toString(); +} export interface FirecrawlSearchParams { query: string; @@ -46,14 +68,23 @@ interface FirecrawlWebResult { title?: string | null; url?: string | null; description?: string | null; + snippet?: string | null; markdown?: string | null; } interface FirecrawlSearchResponse { + success?: boolean; + error?: string | null; id?: string | null; - data?: { - web?: FirecrawlWebResult[] | null; - } | null; + data?: + | FirecrawlWebResult[] + | { + web?: FirecrawlWebResult[] | null; + news?: FirecrawlWebResult[] | null; + images?: FirecrawlWebResult[] | null; + } + | null; + results?: FirecrawlWebResult[] | null; } /** Resolve Firecrawl API key through the shared auth storage pipeline. */ @@ -88,7 +119,7 @@ async function callFirecrawlSearch( if (apiKey) { headers.Authorization = `Bearer ${apiKey}`; } - const response = await (params.fetch ?? fetch)(FIRECRAWL_SEARCH_URL, { + const response = await (params.fetch ?? fetch)(resolveSearchUrl(), { method: "POST", headers, body: JSON.stringify(buildRequestBody(params)), @@ -106,7 +137,11 @@ async function callFirecrawlSearch( ); } - return (await response.json()) as FirecrawlSearchResponse; + const data = (await response.json()) as FirecrawlSearchResponse; + if (data.success === false) { + throw new SearchProviderError("firecrawl", data.error?.trim() || "Firecrawl request failed"); + } + return data; } /** ISO `YYYY-MM-DD` to Google `MM/DD/YYYY` for `tbs=cdr` custom date ranges. */ @@ -128,6 +163,11 @@ function buildDateTbs(parsed: StructuredQuery): string | undefined { return parts.join(","); } +function getWebResults(data: FirecrawlSearchResponse): FirecrawlWebResult[] { + if (Array.isArray(data.data)) return data.data; + if (data.data && Array.isArray(data.data.web)) return data.data.web; + return data.results ?? []; +} /** Execute Firecrawl web search. */ export async function searchFirecrawl(params: SearchParams): Promise { const parsed = params.parsedQuery ?? parseSearchQuery(params.query); @@ -169,12 +209,12 @@ export async function searchFirecrawl(params: SearchParams): Promise { + try { + const response = await (fetchImpl ?? fetch)(proxyUrl, { + method: "HEAD", + redirect: "manual", + signal: withHardTimeout(signal, 5000), + }); + const location = response.headers.get("location"); + if (!location) return proxyUrl; + const resolved = new URL(location, proxyUrl); + return resolved.protocol === "http:" || resolved.protocol === "https:" ? resolved.toString() : proxyUrl; + } catch { + return proxyUrl; + } +} + +async function finalizeGeminiSearchResult( + result: GeminiSearchResult, + fetchImpl: FetchImpl | undefined, + signal: AbortSignal | undefined, +): Promise { + if (!result.answer && result.sources.length === 0) { + throw new SearchProviderError("gemini", "Gemini API returned an empty grounded response", 502); + } + + const redirectUrls = new Set(); + for (const source of result.sources) { + if (isGroundingRedirectUrl(source.url)) redirectUrls.add(source.url); + } + for (const citation of result.citations) { + if (isGroundingRedirectUrl(citation.url)) redirectUrls.add(citation.url); + } + if (redirectUrls.size === 0) return result; + + signal?.throwIfAborted(); + const resolvedEntries = await Promise.all( + [...redirectUrls].map(async url => [url, await resolveGroundingRedirect(url, fetchImpl, signal)] as const), + ); + signal?.throwIfAborted(); + const resolvedUrls = new Map(resolvedEntries); + for (const source of result.sources) { + source.url = resolvedUrls.get(source.url) ?? source.url; + } + for (const citation of result.citations) { + citation.url = resolvedUrls.get(citation.url) ?? citation.url; + } + + const seenUrls = new Set(); + let writeIndex = 0; + for (const source of result.sources) { + if (seenUrls.has(source.url)) continue; + seenUrls.add(source.url); + result.sources[writeIndex++] = source; + } + result.sources.length = writeIndex; + return result; +} + /** * Calls the Cloud Code Assist API with Google Search grounding enabled. * @@ -420,7 +523,10 @@ async function callGeminiSearch( } if (!response?.ok) { - const errorText = response ? await response.text() : "Network error"; + const rawErrorText = response ? await response.text() : "Network error"; + const errorText = auth.accessToken + ? rawErrorText.split(auth.accessToken).join("[redacted]") + : rawErrorText; const status = response?.status ?? 502; const classified = classifyProviderHttpError("gemini", status, errorText); if (classified) throw classified; @@ -431,11 +537,12 @@ async function callGeminiSearch( throw new SearchProviderError("gemini", "Gemini API returned no response body", 500); } - return parseGeminiSearchStream(response.body, model); + return finalizeGeminiSearchResult(await parseGeminiSearchStream(response.body, model), fetchImpl, signal); } async function callGeminiDeveloperSearch( apiKey: string, + endpoint: GeminiDeveloperEndpoint, model: string, query: string, systemPrompt: string | undefined, @@ -474,11 +581,13 @@ async function callGeminiDeveloperSearch( } const response = await fetchWithRetry( - () => `${DEVELOPER_API_ENDPOINT}/models/${model}:streamGenerateContent?alt=sse`, + () => `${endpoint.url}/models/${model}:streamGenerateContent?alt=sse`, { method: "POST", headers: { - "x-goog-api-key": apiKey, + ...(endpoint.isCloudflareGateway + ? { "cf-aig-authorization": `Bearer ${apiKey}` } + : { "x-goog-api-key": apiKey }), "Content-Type": "application/json", Accept: "text/event-stream", }, @@ -492,7 +601,8 @@ async function callGeminiDeveloperSearch( ); if (!response.ok) { - const errorText = await response.text(); + const rawErrorText = await response.text(); + const errorText = apiKey ? rawErrorText.split(apiKey).join("[redacted]") : rawErrorText; const classified = classifyProviderHttpError("gemini", response.status, errorText); if (classified) throw classified; throw new SearchProviderError( @@ -506,7 +616,7 @@ async function callGeminiDeveloperSearch( throw new SearchProviderError("gemini", "Gemini API returned no response body", 500); } - return parseGeminiSearchStream(response.body, model); + return finalizeGeminiSearchResult(await parseGeminiSearchStream(response.body, model), fetchImpl, signal); } /** @@ -557,16 +667,20 @@ export async function searchGemini(params: GeminiSearchParams): Promise { diff --git a/packages/coding-agent/src/web/search/providers/jina.ts b/packages/coding-agent/src/web/search/providers/jina.ts index c3467c454..b9fc9e7d4 100644 --- a/packages/coding-agent/src/web/search/providers/jina.ts +++ b/packages/coding-agent/src/web/search/providers/jina.ts @@ -5,19 +5,24 @@ * cleaned content. */ -import { type AuthStorage, type FetchImpl, getEnvApiKey } from "@oh-my-pi/pi-ai"; +import { type ApiKey, type AuthStorage, type FetchImpl, withAuth } from "@oh-my-pi/pi-ai"; import type { SearchResponse, SearchSource } from "../../../web/search/types"; import { SearchProviderError } from "../../../web/search/types"; import { formatQuery, parseSearchQuery } from "../query"; +import { clampNumResults } from "../utils"; import type { SearchParams } from "./base"; import { SearchProvider } from "./base"; import { classifyProviderHttpError, withHardTimeout } from "./utils"; const JINA_SEARCH_URL = "https://s.jina.ai"; +const DEFAULT_NUM_RESULTS = 5; +const MAX_NUM_RESULTS = 20; type SearchParamsWithFetch = SearchParams & { fetch?: FetchImpl }; export interface JinaSearchParams { query: string; + authStorage: AuthStorage; + sessionId?: string; num_results?: number; /** Single bare host for Jina's `X-Site` in-site search header. */ site?: string; @@ -29,31 +34,37 @@ export interface JinaSearchParams { interface JinaSearchResult { title?: string | null; url?: string | null; + description?: string | null; content?: string | null; } -type JinaSearchResponse = JinaSearchResult[]; - -/** Find JINA_API_KEY from environment or .env files. */ -export function findApiKey(): string | null { - return getEnvApiKey("jina") ?? null; +interface JinaSearchEnvelope { + code?: unknown; + data?: unknown; } +type JinaSearchResponse = JinaSearchResult[]; + /** Call Jina Reader search API. */ async function callJinaSearch( apiKey: string, query: string, + numResults: number, site?: string, signal?: AbortSignal, fetchImpl: FetchImpl = fetch, timeoutMs?: number, ): Promise { - const requestUrl = `${JINA_SEARCH_URL}/${encodeURIComponent(query)}`; + const requestUrl = new URL(`${JINA_SEARCH_URL}/${encodeURIComponent(query)}`); + requestUrl.searchParams.set("count", String(numResults)); + const headers: Record = { Accept: "application/json", Authorization: `Bearer ${apiKey}`, }; if (site) headers["X-Site"] = site; + headers["X-Respond-With"] = "no-content"; + headers["X-Retain-Images"] = "none"; const response = await fetchImpl(requestUrl, { headers, signal: withHardTimeout(signal, timeoutMs), @@ -66,24 +77,43 @@ async function callJinaSearch( throw new SearchProviderError("jina", `Jina API error (${response.status}): ${errorText}`, response.status); } - const payload = (await response.json()) as { data?: JinaSearchResponse } | null; - return Array.isArray(payload?.data) ? payload.data : []; + const payload = (await response.json()) as JinaSearchEnvelope | JinaSearchResponse | null; + if (Array.isArray(payload)) return payload; + if (!payload || typeof payload !== "object") { + throw new SearchProviderError("jina", "Jina API returned invalid response: expected an object or array"); + } + if (typeof payload.code === "number" && payload.code !== 200) { + throw new SearchProviderError("jina", `Jina API response reported failure (${payload.code})`, payload.code); + } + if (!Array.isArray(payload.data)) { + throw new SearchProviderError("jina", "Jina API returned invalid response: expected data array"); + } + return payload.data as JinaSearchResponse; } /** Execute Jina web search. */ export async function searchJina(params: JinaSearchParams): Promise { - const apiKey = findApiKey(); - if (!apiKey) { - throw new Error("JINA_API_KEY not found. Set it in environment or .env file."); - } - - const response = await callJinaSearch( - apiKey, - params.query, - params.site, - params.signal, - params.fetch, - params.timeoutMs, + const numResults = clampNumResults(params.num_results, DEFAULT_NUM_RESULTS, MAX_NUM_RESULTS); + const keyOrResolver: ApiKey = params.authStorage.resolver("jina", { + sessionId: params.sessionId, + }); + const response = await withAuth( + keyOrResolver, + apiKey => + callJinaSearch( + apiKey, + params.query, + numResults, + params.site, + params.signal, + params.fetch, + params.timeoutMs, + ), + { + signal: params.signal, + missingKeyMessage: + 'Jina credentials not found. Set JINA_API_KEY or configure an API key for provider "jina".', + }, ); const sources: SearchSource[] = []; @@ -92,11 +122,11 @@ export async function searchJina(params: JinaSearchParams): Promise { @@ -134,6 +164,8 @@ export class JinaProvider extends SearchProvider { return searchJina({ query, + authStorage: params.authStorage, + sessionId: params.sessionId, num_results: params.numSearchResults ?? params.limit, site, signal: params.signal, diff --git a/packages/coding-agent/src/web/search/providers/parallel.ts b/packages/coding-agent/src/web/search/providers/parallel.ts index c9b02fcee..59da2bdc8 100644 --- a/packages/coding-agent/src/web/search/providers/parallel.ts +++ b/packages/coding-agent/src/web/search/providers/parallel.ts @@ -7,6 +7,7 @@ import { ParallelApiError, type ParallelSearchResult, parseParallelErrorResponse, + parseParallelJsonResponse, parseParallelSearchPayload, } from "../../parallel"; import { formatQuery, parseSearchQuery, type StructuredQuery } from "../query"; @@ -28,6 +29,13 @@ interface ParallelSourcePolicy { after_date?: string; } +const RECENCY_DAYS: Record, number> = { + day: 1, + week: 7, + month: 30, + year: 365, +}; + /** Site values may carry paths (`github.com/anthropics`); Parallel takes bare hosts. */ function toHosts(sites: readonly string[]): string[] { const hosts = new Set(); @@ -39,19 +47,23 @@ function toHosts(sites: readonly string[]): string[] { } /** - * Map parsed `site:`/`-site:`/`after:` directives onto Parallel's - * `source_policy`. Per Parallel docs, `exclude_domains` is ignored when - * `include_domains` is set, so exclusions are only sent without an allow - * list (the central lenient filter enforces them regardless). + * Map parsed `site:`/`-site:`/`after:` directives and the relative recency + * option onto Parallel's `source_policy`. An explicit `after:` bound wins. + * Per Parallel docs, `exclude_domains` is ignored when `include_domains` is + * set, so exclusions are only sent without an allow list (the central lenient + * filter enforces them regardless). */ -function toSourcePolicy(parsed: StructuredQuery): ParallelSourcePolicy | undefined { +function toSourcePolicy(parsed: StructuredQuery, recency?: SearchParams["recency"]): ParallelSourcePolicy | undefined { const policy: ParallelSourcePolicy = {}; const include = toHosts(parsed.sites); const exclude = toHosts(parsed.excludedSites); if (include.length) policy.include_domains = include; else if (exclude.length) policy.exclude_domains = exclude; if (parsed.after) policy.after_date = parsed.after; - return Object.keys(policy).length ? policy : undefined; + else if (recency) { + policy.after_date = new Date(Date.now() - RECENCY_DAYS[recency] * 86_400_000).toISOString().slice(0, 10); + } + return policy.include_domains || policy.exclude_domains || policy.after_date ? policy : undefined; } async function searchWithAuthStorage( @@ -105,7 +117,7 @@ async function searchWithAuthStorage( throw parseParallelErrorResponse(response.status, await response.text()); } - const payload: unknown = await response.json(); + const payload = await parseParallelJsonResponse(response, "search"); return parseParallelSearchPayload(payload, { parseMetadata: false }); }, { signal: params.signal }, @@ -116,6 +128,7 @@ export async function searchParallel( params: { query: string; num_results?: number; + recency?: SearchParams["recency"]; signal?: AbortSignal; timeoutMs?: number; fetch?: FetchImpl; @@ -126,9 +139,9 @@ export async function searchParallel( ): Promise { const numResults = clampNumResults(params.num_results, DEFAULT_NUM_RESULTS, MAX_NUM_RESULTS); const parsed = params.parsedQuery ?? parseSearchQuery(params.query); - // Back-compat: without directives the upstream request is byte-identical. + // Directives are removed only where Parallel has a native equivalent. const query = parsed.hasDirectives ? formatQuery(parsed, PARALLEL_QUERY_SYNTAX) : params.query; - const sourcePolicy = parsed.hasDirectives ? toSourcePolicy(parsed) : undefined; + const sourcePolicy = toSourcePolicy(parsed, params.recency); try { const result = await searchWithAuthStorage( @@ -174,6 +187,7 @@ export class ParallelProvider extends SearchProvider { { query: params.query, num_results: params.numSearchResults ?? params.limit, + recency: params.recency, signal: params.signal, timeoutMs: params.timeoutMs, fetch: params.fetch, diff --git a/packages/coding-agent/src/web/search/providers/perplexity.ts b/packages/coding-agent/src/web/search/providers/perplexity.ts index 1839a52d4..c91b6fd2b 100644 --- a/packages/coding-agent/src/web/search/providers/perplexity.ts +++ b/packages/coding-agent/src/web/search/providers/perplexity.ts @@ -150,6 +150,7 @@ interface PerplexityOAuthStreamEvent { error_code?: string; error_message?: string; display_model?: string; + user_selected_model?: string; uuid?: string; } @@ -327,7 +328,11 @@ export interface PerplexitySearchParams { system_prompt?: string; /** Pre-parsed view of `query` from the search pipeline; parsed locally when absent. */ parsedQuery?: StructuredQuery; + /** Direct API model. Defaults to `PI_PERPLEXITY_API_MODEL`, then `sonar-pro`. */ + api_model?: string; search_recency_filter?: "hour" | "day" | "week" | "month" | "year"; + /** Consumer subscription model preference. Defaults to `PI_PERPLEXITY_MODEL`, then Sonar (`experimental`). */ + subscription_model?: string; num_results?: number; /** Maximum output tokens. Defaults to 8192. */ max_tokens?: number; @@ -539,6 +544,15 @@ async function callPerplexityApi( return parseStreamedApiResponse(message, metadata); } +function oauthSourceKey(url: string): string { + const trimmed = url.trim().replace(/\/$/, ""); + try { + return new URL(trimmed).href.replace(/\/$/, ""); + } catch { + return trimmed.toLowerCase(); + } +} + function buildOAuthSources(event: PerplexityOAuthStreamEvent): SearchSource[] { const results = event.blocks?.find(block => block.intended_usage === "web_results")?.web_result_block?.web_results ?? []; @@ -607,6 +621,7 @@ async function callPerplexityAsk( params: PerplexitySearchParams, filters: PerplexityNativeFilters, ): Promise<{ answer: string; sources: SearchSource[]; model?: string; requestId?: string }> { + const subscriptionModel = params.subscription_model?.trim() || $env.PI_PERPLEXITY_MODEL?.trim() || "experimental"; const requestId = crypto.randomUUID(); // The consumer `perplexity_ask` endpoint is itself a research assistant and // has no system-message slot. Prepending the API-style system prompt to the @@ -645,14 +660,14 @@ async function callPerplexityAsk( query_str: effectiveQuery, search_focus: "internet", mode: "copilot", - model_preference: "experimental", + model_preference: subscriptionModel, sources: ["web"], attachments: [], frontend_uuid: crypto.randomUUID(), frontend_context_uuid: crypto.randomUUID(), version: OAUTH_API_VERSION, language: "en-US", - timezone: Intl.DateTimeFormat().resolvedOptions().timeZone, + timezone: Intl.DateTimeFormat().resolvedOptions().timeZone ?? "UTC", // Recency cannot be combined with absolute date filters; explicit // before:/after: bounds take precedence. search_recency_filter: filters.afterDate || filters.beforeDate ? null : (params.search_recency_filter ?? null), @@ -740,12 +755,14 @@ async function callPerplexityAsk( if (eventAnswer.length > 0) { answer = eventAnswer; } - for (const source of buildOAuthSources(mergedEvent)) { - sourcesByUrl.set(source.url, source); + sourcesByUrl.set(oauthSourceKey(source.url), source); } - if (mergedEvent.display_model) model = mergedEvent.display_model; + const reportedModel = [mergedEvent.user_selected_model, mergedEvent.display_model].find( + candidate => candidate && candidate !== "turbo", + ); + if (reportedModel) model = reportedModel; if (mergedEvent.uuid) finalRequestId = mergedEvent.uuid; if (mergedEvent.final || mergedEvent.status === "COMPLETED") { break; @@ -755,7 +772,7 @@ async function callPerplexityAsk( return { answer, sources: [...sourcesByUrl.values()], - model, + model: model ?? (auth.type === "anonymous" ? mergedEvent.display_model : subscriptionModel), requestId: finalRequestId ?? requestId, }; } @@ -872,7 +889,7 @@ export async function searchPerplexity(params: PerplexitySearchParams): Promise< messages.push({ role: "user", content: filters.query }); const request: PerplexityRequest = { - model: "sonar-pro", + model: params.api_model?.trim() || $env.PI_PERPLEXITY_API_MODEL?.trim() || "sonar-pro", messages, max_tokens: params.max_tokens ?? DEFAULT_MAX_TOKENS, temperature: params.temperature ?? DEFAULT_TEMPERATURE, diff --git a/packages/coding-agent/src/web/search/providers/searxng.ts b/packages/coding-agent/src/web/search/providers/searxng.ts index 9ef4794f3..7082d6a9d 100644 --- a/packages/coding-agent/src/web/search/providers/searxng.ts +++ b/packages/coding-agent/src/web/search/providers/searxng.ts @@ -62,6 +62,7 @@ interface SearXNGResult { title?: string; url?: string; content?: string; + snippet?: string; engine?: string; publishedDate?: string; /** SearXNG sometimes uses publishedDate, sometimes just date */ @@ -76,6 +77,7 @@ interface SearXNGResponse { suggestions?: string[]; corrections?: string[]; unresponsive_engines?: Array<[string, string]>; + answers?: unknown[]; } interface SearXNGAuth { @@ -272,6 +274,61 @@ function stripExternalBangs(query: string): string { .join(" "); } +/** Extract displayable text from both legacy string answers and modern + * structured answer plugins (legacy, translations, weather). */ +function extractAnswerText(answer: unknown): string | undefined { + if (typeof answer === "string") return answer.trim() || undefined; + if (!answer || typeof answer !== "object") return undefined; + + const record = answer as Record; + if (typeof record.answer === "string") return record.answer.trim() || undefined; + + if (Array.isArray(record.translations)) { + const translations: string[] = []; + for (const item of record.translations) { + if (!item || typeof item !== "object") continue; + const text = (item as Record).text; + if (typeof text === "string" && text.trim()) translations.push(text.trim()); + if (translations.length === 3) break; + } + if (translations.length) return translations.join("\n"); + } + + if (record.current && typeof record.current === "object") { + const current = record.current as Record; + if (typeof current.summary === "string" && current.summary.trim()) return current.summary.trim(); + const location = + current.location && typeof current.location === "object" + ? (current.location as Record).name + : undefined; + const temperature = + current.temperature && typeof current.temperature === "object" + ? (current.temperature as Record) + : undefined; + const temperatureText = + temperature && (typeof temperature.val === "string" || typeof temperature.val === "number") + ? `${temperature.val}${typeof temperature.unit === "string" ? temperature.unit : ""}` + : undefined; + const condition = typeof current.condition === "string" ? current.condition : undefined; + const parts = [location, temperatureText, condition].filter( + (part): part is string => typeof part === "string" && part.trim().length > 0, + ); + if (parts.length) return parts.join(": "); + } + + return undefined; +} + +function formatAnswers(answers: unknown[] | undefined): string | undefined { + const texts: string[] = []; + for (const answer of answers ?? []) { + const text = extractAnswerText(answer); + if (text) texts.push(text); + if (texts.length === 3) break; + } + return texts.length ? texts.join("\n\n") : undefined; +} + /** Build the search URL and headers for a SearXNG request */ function buildRequest( endpoint: string, @@ -282,6 +339,7 @@ function buildRequest( categories?: string; engines?: string; language?: string; + safesearch?: 0 | 1 | 2; signal?: AbortSignal; }, auth: SearXNGAuth | null, @@ -308,6 +366,10 @@ function buildRequest( url.searchParams.set("engines", params.engines); } + if (params.safesearch !== undefined) { + url.searchParams.set("safesearch", String(params.safesearch)); + } + if (params.language) { url.searchParams.set("language", params.language); } @@ -326,6 +388,7 @@ async function callSearXNGSearch( categories?: string; engines?: string; language?: string; + safesearch?: 0 | 1 | 2; signal?: AbortSignal; timeoutMs?: number; fetch?: FetchImpl; @@ -372,12 +435,23 @@ export async function searchSearXNG(params: { let categories: string | undefined; let language: string | undefined; + let configuredSafesearch: number | undefined; try { categories = settings.get("searxng.categories") ?? undefined; language = settings.get("searxng.language") ?? undefined; + configuredSafesearch = settings.get("searxng.safesearch"); } catch { // Settings not initialized yet } + if ( + configuredSafesearch !== undefined && + configuredSafesearch !== 0 && + configuredSafesearch !== 1 && + configuredSafesearch !== 2 + ) { + throw new Error("searxng.safesearch must be 0 (off), 1 (moderate), or 2 (strict)."); + } + const safesearch = configuredSafesearch; const configuredEngines = findEngines(); // SearXNG forwards `q` to downstream engines, so build it with the shared @@ -402,6 +476,7 @@ export async function searchSearXNG(params: { categories, engines, language, + safesearch, fetch: params.fetch, }, auth, @@ -415,7 +490,7 @@ export async function searchSearXNG(params: { sources.push({ title: result.title ?? result.url, url: result.url, - snippet: result.content?.trim() || undefined, + snippet: (result.content ?? result.snippet)?.trim() || undefined, publishedDate: publishedDate ?? undefined, ageSeconds: dateToAgeSeconds(publishedDate), }); @@ -435,6 +510,7 @@ export async function searchSearXNG(params: { return { provider: "searxng", + answer: formatAnswers(response.answers), sources: limitedSources, relatedQuestions: response.suggestions?.length ? response.suggestions : undefined, }; diff --git a/packages/coding-agent/src/web/search/providers/tavily.ts b/packages/coding-agent/src/web/search/providers/tavily.ts index a31d6ac9c..3556bbda6 100644 --- a/packages/coding-agent/src/web/search/providers/tavily.ts +++ b/packages/coding-agent/src/web/search/providers/tavily.ts @@ -34,17 +34,10 @@ export interface TavilySearchParams { fetch?: FetchImpl; } -interface TavilySearchResult { - title?: string | null; - url?: string | null; - content?: string | null; - published_date?: string | null; -} - interface TavilySearchResponse { - answer?: string | null; - results?: TavilySearchResult[]; - request_id?: string | null; + answer?: unknown; + results?: unknown; + request_id?: unknown; } function asRecord(value: unknown): Record | null { @@ -141,28 +134,36 @@ async function callTavilySearch(apiKey: string, params: TavilySearchParams): Pro throw new SearchProviderError("tavily", `Tavily API error (${response.status}): ${message}`, response.status); } - return (await response.json()) as TavilySearchResponse; + const payload: unknown = await response.json(); + return asRecord(payload) ?? {}; } function toSearchResponse(response: TavilySearchResponse, numResults: number): SearchResponse { const sources: SearchSource[] = []; - for (const result of response.results ?? []) { - if (!result.url) continue; - sources.push({ - title: result.title ?? result.url, - url: result.url, - snippet: result.content ?? undefined, - publishedDate: result.published_date ?? undefined, - ageSeconds: dateToAgeSeconds(result.published_date ?? undefined), - }); + if (Array.isArray(response.results)) { + for (const value of response.results) { + const result = asRecord(value); + if (!result || typeof result.url !== "string" || !result.url) continue; + const title = typeof result.title === "string" && result.title ? result.title : result.url; + const snippet = typeof result.content === "string" ? result.content : undefined; + const publishedDate = typeof result.published_date === "string" ? result.published_date : undefined; + sources.push({ + title, + url: result.url, + snippet, + publishedDate, + ageSeconds: dateToAgeSeconds(publishedDate), + }); + } } + const answer = typeof response.answer === "string" ? response.answer.trim() || undefined : undefined; return { provider: "tavily", - answer: response.answer?.trim() || undefined, + answer, sources: sources.slice(0, numResults), - requestId: response.request_id ?? undefined, + requestId: typeof response.request_id === "string" ? response.request_id : undefined, authMode: "api_key", }; } diff --git a/packages/coding-agent/src/web/search/providers/tinyfish.ts b/packages/coding-agent/src/web/search/providers/tinyfish.ts index 236a1c0ae..47971e3cc 100644 --- a/packages/coding-agent/src/web/search/providers/tinyfish.ts +++ b/packages/coding-agent/src/web/search/providers/tinyfish.ts @@ -19,7 +19,7 @@ const MAX_NUM_RESULTS = 20; const MAX_PAGE = 10; /** TinyFish is SERP-backed: common Google-style operators pass through. */ -const TINYFISH_QUERY_SYNTAX: QuerySyntax = { phrases: true, negation: true, site: true, filetype: true }; +const TINYFISH_QUERY_SYNTAX: QuerySyntax = { phrases: true, negation: true, filetype: true }; const RECENCY_MINUTES: Record, number> = { day: 1440, @@ -33,6 +33,8 @@ export interface TinyFishSearchParams { num_results?: number; recency?: SearchParams["recency"]; page?: number; + include_domains?: string[]; + exclude_domains?: string[]; signal?: AbortSignal; timeoutMs?: number; fetch?: FetchImpl; @@ -66,6 +68,12 @@ async function callTinyFishSearch(apiKey: string, params: TinyFishSearchParams): if (params.recency) { url.searchParams.set("recency_minutes", String(RECENCY_MINUTES[params.recency])); } + if (params.include_domains?.length) { + url.searchParams.set("include_domains", params.include_domains.join(",")); + } + if (params.exclude_domains?.length) { + url.searchParams.set("exclude_domains", params.exclude_domains.join(",")); + } if (params.num_results !== undefined) { url.searchParams.set("num_results", String(params.num_results)); } @@ -96,18 +104,35 @@ async function callTinyFishSearch(apiKey: string, params: TinyFishSearchParams): return (await response.json()) as TinyFishSearchResponse; } -function appendTinyFishSources(sources: SearchSource[], results: readonly TinyFishSearchResult[]): void { +function appendTinyFishSources( + sources: SearchSource[], + results: readonly TinyFishSearchResult[], + seenUrls: Set, +): void { for (const result of results) { - if (!result.url) continue; + const url = result.url?.trim(); + if (!url || seenUrls.has(url)) continue; + seenUrls.add(url); + const siteName = result.site_name?.trim(); sources.push({ - title: result.title ?? result.site_name ?? result.url, - url: result.url, - snippet: result.snippet ?? undefined, - author: result.site_name ?? undefined, + title: result.title?.trim() || siteName || url, + url, + snippet: result.snippet?.replace(/\s+/g, " ").trim() || undefined, + author: siteName || undefined, }); } } +/** Bare hosts from `site:` values; path constraints remain centrally post-filtered. */ +function siteHosts(sites: readonly string[]): string[] { + const hosts = new Set(); + for (const site of sites) { + const host = site.split("/", 1)[0]; + if (host) hosts.add(host); + } + return [...hosts]; +} + /** Execute TinyFish web search. */ export async function searchTinyFish(params: SearchParams): Promise { const numResults = clampNumResults(params.numSearchResults ?? params.limit, DEFAULT_NUM_RESULTS, MAX_NUM_RESULTS); @@ -121,6 +146,12 @@ export async function searchTinyFish(params: SearchParams): Promise 0) tinyFishParams.include_domains = includeDomains; + if (excludeDomains.length > 0) tinyFishParams.exclude_domains = excludeDomains; + } const keyOrResolver: ApiKey = params.authStorage.resolver("tinyfish", { sessionId: params.sessionId, }); @@ -128,11 +159,14 @@ export async function searchTinyFish(params: SearchParams): Promise { const collected: SearchSource[] = []; + const seenUrls = new Set(); for (let page = 0; page <= MAX_PAGE && collected.length < numResults; page += 1) { const searchPage = await callTinyFishSearch(key, { ...tinyFishParams, page }); - const results = searchPage.results ?? []; - appendTinyFishSources(collected, results); - if (results.length < pageSize) break; + if (!Array.isArray(searchPage.results)) { + throw new Error("TinyFish Search API returned an unexpected response shape"); + } + appendTinyFishSources(collected, searchPage.results, seenUrls); + if (searchPage.results.length < pageSize) break; } return collected.slice(0, numResults); diff --git a/packages/coding-agent/src/web/search/providers/xai.ts b/packages/coding-agent/src/web/search/providers/xai.ts index 393aa3b31..6642aa2f8 100644 --- a/packages/coding-agent/src/web/search/providers/xai.ts +++ b/packages/coding-agent/src/web/search/providers/xai.ts @@ -25,6 +25,8 @@ interface XAIUrlCitationAnnotation { title?: string | null; text?: string | null; cited_text?: string | null; + start_index?: number | null; + end_index?: number | null; } interface XAIResponseContentPart { @@ -34,9 +36,20 @@ interface XAIResponseContentPart { annotations?: XAIUrlCitationAnnotation[] | null; } +interface XAIWebSearchSource { + url?: string | null; + source_website_url?: string | null; + title?: string | null; + caption?: string | null; +} + interface XAIResponseOutputItem { + type?: string; content?: XAIResponseContentPart[] | null; annotations?: XAIUrlCitationAnnotation[] | null; + action?: { sources?: XAIWebSearchSource[] | null } | null; + sources?: XAIWebSearchSource[] | null; + results?: XAIWebSearchSource[] | null; } interface XAIResponsesUsage { @@ -161,7 +174,12 @@ async function callXAIResponses( throwXAIResponsesError(response.status, await response.text()); } - return (await response.json()) as XAIResponsesResponse; + try { + return (await response.json()) as XAIResponsesResponse; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new SearchProviderError("xai", `xAI Responses API returned invalid JSON: ${message}`, response.status); + } } function addCitationSource( @@ -189,38 +207,77 @@ function addCitationSource( citedText: sourceSnippet, }); } +function extractSnippetAround( + text: string | null | undefined, + start: number | null | undefined, + end: number | null | undefined, +): string | undefined { + if (!text || typeof start !== "number" || typeof end !== "number") return undefined; + const before = Math.max(0, start - 100); + const after = Math.min(text.length, end + 100); + const snippet = text + .slice(before, after) + .replace(/\[([^\]]*)\]\([^)]*\)/g, "$1") + .trim(); + if (!snippet) return undefined; + return snippet.length > 300 ? `${snippet.slice(0, 297)}...` : snippet; +} function collectAnnotationSources( annotations: readonly XAIUrlCitationAnnotation[] | null | undefined, sources: SearchSource[], citations: SearchCitation[], seenUrls: Set, + contentText?: string | null, ): void { - if (!annotations) return; + if (!Array.isArray(annotations)) return; for (const annotation of annotations) { - if (annotation.type !== "url_citation" || !annotation.url) continue; + if (!annotation || typeof annotation !== "object") continue; + if (annotation.type !== "url_citation" || typeof annotation.url !== "string") continue; addCitationSource( sources, citations, seenUrls, annotation.url, annotation.title, - annotation.cited_text ?? annotation.text, + annotation.cited_text ?? + annotation.text ?? + extractSnippetAround(contentText, annotation.start_index, annotation.end_index), ); } } +function collectWebSearchSources( + item: XAIResponseOutputItem, + sources: SearchSource[], + citations: SearchCitation[], + seenUrls: Set, +): void { + if (item.type !== "web_search_call") return; + for (const group of [item.action?.sources, item.sources, item.results]) { + if (!Array.isArray(group)) continue; + for (const source of group) { + if (!source || typeof source !== "object") continue; + const url = source.url ?? source.source_website_url; + if (typeof url !== "string") continue; + addCitationSource(sources, citations, seenUrls, url, source.title ?? source.caption); + } + } +} + function parseAnswer(response: XAIResponsesResponse): string | undefined { const topLevelText = response.output_text?.trim(); if (topLevelText) return topLevelText; const answerParts: string[] = []; - for (const item of response.output ?? []) { - for (const part of item.content ?? []) { + const output = Array.isArray(response.output) ? response.output : []; + for (const item of output) { + if (!item || typeof item !== "object") continue; + const content = Array.isArray(item.content) ? item.content : []; + for (const part of content) { + if (!part || typeof part !== "object") continue; const text = part.output_text ?? part.text; - if ((part.type === "output_text" || part.type === "text") && text?.trim()) { - answerParts.push(text.trim()); - } + if (text?.trim()) answerParts.push(text.trim()); } } @@ -259,13 +316,29 @@ function parseResponse(response: XAIResponsesResponse, resultCap: number): Searc const seenUrls = new Set(); collectAnnotationSources(response.annotations, sources, citations, seenUrls); - for (const item of response.output ?? []) { + const output = Array.isArray(response.output) ? response.output : []; + for (const item of output) { + if (!item || typeof item !== "object") continue; collectAnnotationSources(item.annotations, sources, citations, seenUrls); - for (const part of item.content ?? []) { - collectAnnotationSources(part.annotations, sources, citations, seenUrls); + const content = Array.isArray(item.content) ? item.content : []; + for (const part of content) { + if (!part || typeof part !== "object") continue; + collectAnnotationSources( + part.annotations, + sources, + citations, + seenUrls, + part.output_text ?? part.text, + ); } } - for (const url of response.citations ?? []) { + for (const item of output) { + if (!item || typeof item !== "object") continue; + collectWebSearchSources(item, sources, citations, seenUrls); + } + const topLevelCitations = Array.isArray(response.citations) ? response.citations : []; + for (const url of topLevelCitations) { + if (typeof url !== "string") continue; addCitationSource(sources, citations, seenUrls, url); } const limited = applyResultCap(sources, citations, resultCap); @@ -354,7 +427,11 @@ export async function searchXAI(params: SearchParams): Promise { signal: params.signal, missingKeyMessage: 'xAI credentials not found. Set XAI_API_KEY or configure an API key for provider "xai".', }); - return parseResponse(response, resultCap); + const parsed = parseResponse(response, resultCap); + if (!parsed.answer && parsed.sources.length === 0) { + throw new SearchProviderError("xai", "xAI web_search returned no answer or sources", 502); + } + return parsed; } /** Search provider for xAI web search. */ diff --git a/packages/coding-agent/test/tools/fetch-jina-stall.test.ts b/packages/coding-agent/test/tools/fetch-jina-stall.test.ts index b0c767d21..9ec7d059c 100644 --- a/packages/coding-agent/test/tools/fetch-jina-stall.test.ts +++ b/packages/coding-agent/test/tools/fetch-jina-stall.test.ts @@ -94,3 +94,63 @@ describe("renderHtmlToText: jina stall does not starve local fallbacks (#1449)", ).toBe(true); }); }); + +describe("renderHtmlToText: Jina response validation", () => { + it("requests fresh markdown and strips the Jina metadata preamble", async () => { + const settings = Settings.isolated({ "providers.fetch": "jina" }); + const markdown = `# Extracted article\n\n${"Substantive reader content. ".repeat(8)}`.trim(); + let requestHeaders: Headers | undefined; + const fetchMock = asGlobalFetch((_input, init) => { + requestHeaders = new Headers(init?.headers); + return new Response(`Title: Example\nURL Source: https://example.com/article\nMarkdown Content:\n${markdown}`); + }); + + const result = await renderHtmlToText( + "https://example.com/article", + "short", + 1, + settings, + undefined, + null, + fetchMock, + ); + + expect(result).toEqual({ content: markdown, ok: true, method: "jina" }); + expect(requestHeaders?.get("accept")).toBe("text/markdown"); + expect(requestHeaders?.get("x-no-cache")).toBe("true"); + }); + + for (const { label, readerBody, headers } of [ + { label: "missing marker", readerBody: "Plausible but unstructured output. ".repeat(8) }, + { label: "short body", readerBody: "Markdown Content:\nToo short" }, + { label: "loading shell", readerBody: `Markdown Content:\nLoading...${" ".repeat(120)}` }, + { label: "JavaScript gate", readerBody: `Markdown Content:\nPlease enable JavaScript${" ".repeat(120)}` }, + { + label: "declared oversized body", + readerBody: `Markdown Content:\n${"Substantive content. ".repeat(8)}`, + headers: { "Content-Length": String(2 * 1024 * 1024 + 1) }, + }, + ]) { + it(`falls back when Jina returns a ${label}`, async () => { + const settings = Settings.isolated({ "providers.fetch": "jina" }); + const paragraph = + "This locally rendered article contains enough meaningful prose to satisfy the shared reader quality gate. "; + const html = `

Fallback article

${paragraph.repeat(4)}

`; + const fetchMock = asGlobalFetch(() => new Response(readerBody, { headers })); + + const result = await renderHtmlToText( + "https://example.com/article", + html, + 1, + settings, + undefined, + null, + fetchMock, + ); + + expect(result.ok).toBe(true); + expect(result.method).toBe("native"); + expect(result.content).toContain("Fallback article"); + }); + } +}); diff --git a/packages/coding-agent/test/tools/web-search-codex.test.ts b/packages/coding-agent/test/tools/web-search-codex.test.ts index ee084ef8d..7251fdeac 100644 --- a/packages/coding-agent/test/tools/web-search-codex.test.ts +++ b/packages/coding-agent/test/tools/web-search-codex.test.ts @@ -531,6 +531,70 @@ describe("searchCodex model selection", () => { expect(result.sources).toEqual([{ title: "Example Article", url: "https://example.com/article" }]); }); + it("requests and merges web-search action sources with citation metadata", async () => { + process.env.PI_CODEX_WEB_SEARCH_MODEL = "gpt-5.4"; + const answer = "The Responses API supports hosted web search."; + const citationStart = answer.indexOf("hosted web search"); + const sse = [ + `data: ${JSON.stringify({ + type: "response.created", + response: { id: "resp_created_id", model: "gpt-5.4" }, + })}`, + "", + `data: ${JSON.stringify({ + type: "response.output_item.done", + item: { + type: "web_search_call", + action: { + sources: [ + { + url: "https://example.com/article?utm_source=openai", + title: "Search result title", + }, + ], + }, + }, + })}`, + "", + `data: ${JSON.stringify({ + type: "response.output_item.done", + item: { + type: "message", + content: [ + { + type: "output_text", + text: answer, + annotations: [ + { + type: "url_citation", + url: "https://example.com/article?utm_source=openai", + title: "Example Article", + start_index: citationStart, + end_index: citationStart + "hosted web search".length, + }, + ], + }, + ], + }, + })}`, + "", + ].join("\n"); + + const result = await searchCodex( + makeSearchParams("action sources", mockCodexFetch("gpt-5.4", sse)), + ); + + expect(capturedRequest?.body?.include).toEqual(["web_search_call.action.sources"]); + expect(result.requestId).toBe("resp_created_id"); + expect(result.sources).toEqual([ + { + title: "Search result title", + url: "https://example.com/article", + snippet: answer, + }, + ]); + }); + it("extracts plain text URLs when annotations are absent", async () => { process.env.PI_CODEX_WEB_SEARCH_MODEL = "gpt-5.4"; const result = await searchCodex( @@ -769,4 +833,22 @@ describe("searchCodex model selection", () => { "Codex request failed (model_snapshot_unavailable): The requested model snapshot is unavailable.", ); }); + + it("classifies rate-limit failures delivered inside a successful SSE response", async () => { + const sse = [ + `data: ${JSON.stringify({ + type: "response.failed", + response: { + error: { code: "rate_limit_exceeded", message: "Too many requests" }, + }, + })}`, + "", + ].join("\n"); + const fetchMock: FetchImpl = () => + Promise.resolve(new Response(sse, { status: 200, headers: { "Content-Type": "text/event-stream" } })); + + await expect(searchCodex(makeSearchParams("rate-limited search", fetchMock))).rejects.toMatchObject({ + status: 429, + }); + }); }); diff --git a/packages/coding-agent/test/tools/web-search-exa.test.ts b/packages/coding-agent/test/tools/web-search-exa.test.ts index 9e424f15c..0bfaa637e 100644 --- a/packages/coding-agent/test/tools/web-search-exa.test.ts +++ b/packages/coding-agent/test/tools/web-search-exa.test.ts @@ -428,6 +428,19 @@ describe("searchExa", () => { expect(result.sources[0].snippet).toBe("summary here"); }); + it("caps snippets at 500 characters", async () => { + const result = await searchExa({ + query: "bounded snippet", + fetch: mockFetch( + makeMockExaResponse({ + results: [{ title: "Long", url: "https://long.example", summary: "x".repeat(800) }], + }), + ), + }); + + expect(result.sources[0].snippet).toHaveLength(500); + }); + it("falls back to text when summary is null", async () => { const result = await searchExa({ query: "fallback", @@ -535,6 +548,68 @@ describe("searchExa", () => { }); }); + it("encodes MCP filters in the basic query, uses camel-case result count, and tags the request source", async () => { + delete process.env.EXA_API_KEY; + let headers: Record | undefined; + const fetchMock: FetchImpl = (_url, init) => { + headers = init?.headers as Record | undefined; + if (init?.body) capturedRequestBody = JSON.parse(init.body as string); + return Promise.resolve( + new Response(JSON.stringify({ jsonrpc: "2.0", id: "mcp-filtered", result: makeMockExaResponse() }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }), + ); + }; + + await searchExa({ + query: "vector databases", + num_results: 4, + include_domains: [" qdrant.tech "], + exclude_domains: ["spam.example"], + start_published_date: "2024-01-01", + end_published_date: "2025-01-01", + fetch: fetchMock, + }); + + expect(headers?.["x-exa-source"]).toBe("oh-my-pi"); + expect(capturedRequestBody?.params).toEqual({ + name: "web_search_exa", + arguments: { + query: + "vector databases site:qdrant.tech -site:spam.example after:2024-01-01 before:2025-01-01", + numResults: 4, + }, + }); + }); + + it("explains how to escape the keyless MCP rate limit", async () => { + delete process.env.EXA_API_KEY; + const fetchMock: FetchImpl = () => + Promise.resolve(new Response("too many requests", { status: 429, statusText: "Too Many Requests" })); + + await expect(searchExa({ query: "rate limited", fetch: fetchMock })).rejects.toThrow( + "exa: MCP rate limit reached (429); configure an Exa API key for higher limits", + ); + }); + + it("surfaces MCP tool-level errors", async () => { + delete process.env.EXA_API_KEY; + const fetchMock: FetchImpl = () => + Promise.resolve( + new Response( + JSON.stringify({ + jsonrpc: "2.0", + id: "mcp-error", + result: { isError: true, content: [{ type: "text", text: "tool quota exceeded" }] }, + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ), + ); + + await expect(searchExa({ query: "tool error", fetch: fetchMock })).rejects.toThrow("tool quota exceeded"); + }); + it("parses Exa MCP plain-text payloads when API key is missing", async () => { delete process.env.EXA_API_KEY; const fetchMock: FetchImpl = () => { diff --git a/packages/coding-agent/test/tools/web-search-firecrawl.test.ts b/packages/coding-agent/test/tools/web-search-firecrawl.test.ts index 3af554448..da6eda78b 100644 --- a/packages/coding-agent/test/tools/web-search-firecrawl.test.ts +++ b/packages/coding-agent/test/tools/web-search-firecrawl.test.ts @@ -235,18 +235,58 @@ describe("Firecrawl web search provider", () => { } }); - it("keeps keyless Firecrawl out of auto selection while allowing explicit selection", () => { + it("keeps hosted keyless Firecrawl explicit-only but admits configured self-hosting", () => { const originalApiKey = process.env.FIRECRAWL_API_KEY; + const originalBaseUrl = process.env.FIRECRAWL_BASE_URL; + const originalApiUrl = process.env.FIRECRAWL_API_URL; delete process.env.FIRECRAWL_API_KEY; + delete process.env.FIRECRAWL_BASE_URL; + delete process.env.FIRECRAWL_API_URL; try { const provider = new FirecrawlProvider(); const authStorage = makeAuthStorage(undefined); expect(provider.isAvailable(authStorage)).toBe(false); expect(provider.isExplicitlyAvailable(authStorage)).toBe(true); + process.env.FIRECRAWL_BASE_URL = "http://localhost:3002"; + expect(provider.isAvailable(authStorage)).toBe(true); } finally { if (originalApiKey === undefined) delete process.env.FIRECRAWL_API_KEY; else process.env.FIRECRAWL_API_KEY = originalApiKey; + if (originalBaseUrl === undefined) delete process.env.FIRECRAWL_BASE_URL; + else process.env.FIRECRAWL_BASE_URL = originalBaseUrl; + if (originalApiUrl === undefined) delete process.env.FIRECRAWL_API_URL; + else process.env.FIRECRAWL_API_URL = originalApiUrl; + } + }); + + it("uses a self-hosted endpoint and accepts Firecrawl v1 array responses", async () => { + const originalBaseUrl = process.env.FIRECRAWL_BASE_URL; + process.env.FIRECRAWL_BASE_URL = "http://localhost:3002/v1/"; + let requestUrl = ""; + try { + const fetchMock: FetchImpl = async input => { + requestUrl = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url; + return new Response( + JSON.stringify({ + success: true, + data: [{ title: "Legacy result", url: "https://example.com/legacy", snippet: "Legacy snippet" }], + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ); + }; + const response = await searchFirecrawl({ + ...makeParams("legacy query", makeAuthStorage(undefined)), + fetch: fetchMock, + }); + + expect(requestUrl).toBe("http://localhost:3002/v1/search"); + expect(response.sources).toEqual([ + { title: "Legacy result", url: "https://example.com/legacy", snippet: "Legacy snippet" }, + ]); + } finally { + if (originalBaseUrl === undefined) delete process.env.FIRECRAWL_BASE_URL; + else process.env.FIRECRAWL_BASE_URL = originalBaseUrl; } }); diff --git a/packages/coding-agent/test/tools/web-search-gemini.test.ts b/packages/coding-agent/test/tools/web-search-gemini.test.ts index 9da285387..9d0c5ce37 100644 --- a/packages/coding-agent/test/tools/web-search-gemini.test.ts +++ b/packages/coding-agent/test/tools/web-search-gemini.test.ts @@ -10,6 +10,7 @@ const DEVELOPER_SSE_RESPONSE = const DEVELOPER_SSE_RESPONSE_WITHOUT_MODEL = 'data: {"candidates":[{"content":{"role":"model","parts":[{"text":"Developer answer"}]},"groundingMetadata":{"webSearchQueries":["latest Bun version"],"groundingChunks":[{"web":{"uri":"https://bun.sh","title":"Bun"}}],"groundingSupports":[{"segment":{"text":"Developer answer"},"groundingChunkIndices":[0]}]}}],"usageMetadata":{"promptTokenCount":3,"candidatesTokenCount":4,"totalTokenCount":7}}\n\n'; const ORIGINAL_GEMINI_SEARCH_MODEL = Bun.env.GEMINI_SEARCH_MODEL; +const ORIGINAL_GEMINI_BASE_URL = Bun.env.GOOGLE_GEMINI_BASE_URL; type CapturedRequest = { url: string; @@ -72,6 +73,11 @@ describe("searchGemini tools serialization", () => { } else { Bun.env.GEMINI_SEARCH_MODEL = ORIGINAL_GEMINI_SEARCH_MODEL; } + if (ORIGINAL_GEMINI_BASE_URL === undefined) { + delete Bun.env.GOOGLE_GEMINI_BASE_URL; + } else { + Bun.env.GOOGLE_GEMINI_BASE_URL = ORIGINAL_GEMINI_BASE_URL; + } }); function makeParams(query: string) { @@ -111,6 +117,61 @@ describe("searchGemini tools serialization", () => { }); }); + it("routes Cloudflare AI Gateway auth through AuthStorage without leaking a Google API key", async () => { + Bun.env.GOOGLE_GEMINI_BASE_URL = + "https://gateway.ai.cloudflare.com/v1/account/gateway/google-ai-studio"; + const gatewayAuthStorage = { + async getOAuthAccess() { + return undefined; + }, + hasOAuth() { + return false; + }, + hasAuth(provider: string) { + return provider === "cloudflare-ai-gateway"; + }, + async getApiKey(provider: string) { + return provider === "cloudflare-ai-gateway" ? "test-cloudflare-key" : undefined; + }, + } as unknown as AuthStorage; + const fetchMock = mockGeminiFetch(DEVELOPER_SSE_RESPONSE); + + expect(new GeminiProvider().isAvailable(gatewayAuthStorage)).toBe(true); + await searchGemini({ + ...makeParams("gateway"), + authStorage: gatewayAuthStorage, + fetch: fetchMock, + }); + + expect(capturedRequest?.url).toBe( + "https://gateway.ai.cloudflare.com/v1/account/gateway/google-ai-studio/v1beta/models/gemini-2.5-flash:streamGenerateContent?alt=sse", + ); + expect(capturedRequest?.headers["cf-aig-authorization"]).toBe("Bearer test-cloudflare-key"); + expect(capturedRequest?.headers["x-goog-api-key"]).toBeUndefined(); + }); + + it("redacts the active credential from Gemini API errors", async () => { + let thrown: unknown; + try { + await searchGemini({ + ...makeParams("redaction"), + authStorage: apiKeyAuthStorage, + fetch: () => + Promise.resolve( + new Response("upstream echoed test-gemini-api-key", { + status: 418, + }), + ), + }); + } catch (error) { + thrown = error; + } + + expect(thrown).toBeInstanceOf(Error); + expect((thrown as Error).message).toContain("[redacted]"); + expect((thrown as Error).message).not.toContain("test-gemini-api-key"); + }); + it("normalizes query directive aliases to canonical Google forms in the grounding request", async () => { const fetchMock = mockGeminiFetch(); await searchGemini({ @@ -221,4 +282,55 @@ describe("searchGemini tools serialization", () => { tools: [{ googleSearch: {} }, { codeExecution: {} }, { urlContext: { allowedDomains: ["example.com"] } }], }); }); + + it("resolves Google grounding proxy URLs in both sources and citations", async () => { + const proxyUrl = + "https://vertexaisearch.cloud.google.com/grounding-api-redirect/abc"; + const responseText = `data: ${JSON.stringify({ + candidates: [ + { + content: { role: "model", parts: [{ text: "Grounded answer" }] }, + groundingMetadata: { + groundingChunks: [{ web: { uri: proxyUrl, title: "Example" } }], + groundingSupports: [{ segment: { text: "Grounded answer" }, groundingChunkIndices: [0] }], + }, + }, + ], + })}\n\n`; + const methods: string[] = []; + const fetchMock: FetchImpl = (_url, init) => { + methods.push(init?.method ?? "GET"); + if (init?.method === "HEAD") { + return Promise.resolve( + new Response(null, { + status: 302, + headers: { location: "https://example.com/article" }, + }), + ); + } + return Promise.resolve(new Response(responseText, { status: 200 })); + }; + + const response = await searchGemini({ + ...makeParams("grounding redirect"), + authStorage: apiKeyAuthStorage, + fetch: fetchMock, + }); + + expect(methods).toEqual(["POST", "HEAD"]); + expect(response.sources).toEqual([{ title: "Example", url: "https://example.com/article" }]); + expect(response.citations).toEqual([ + { title: "Example", url: "https://example.com/article", citedText: "Grounded answer" }, + ]); + }); + + it("rejects a successful Gemini response with no answer or grounding results", async () => { + await expect( + searchGemini({ + ...makeParams("empty"), + authStorage: apiKeyAuthStorage, + fetch: mockGeminiFetch("data: {}\n\n"), + }), + ).rejects.toThrow("Gemini API returned an empty grounded response"); + }); }); diff --git a/packages/coding-agent/test/tools/web-search-kagi.test.ts b/packages/coding-agent/test/tools/web-search-kagi.test.ts index 1a99835ee..573892282 100644 --- a/packages/coding-agent/test/tools/web-search-kagi.test.ts +++ b/packages/coding-agent/test/tools/web-search-kagi.test.ts @@ -58,6 +58,34 @@ describe("Kagi web search error handling", () => { "Kagi API error (502)", ); }); + + it("reports malformed success responses as Kagi API errors", async () => { + const invalidJsonFetch: FetchImpl = async () => new Response("not json", { status: 200 }); + const invalidEnvelopeFetch: FetchImpl = async () => + new Response(JSON.stringify(["unexpected"]), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + + await expect(searchWithKagi("invalid json", { fetch: invalidJsonFetch }, fakeAuthStorage)).rejects.toThrow( + "Kagi API returned an invalid response: invalid JSON", + ); + await expect(searchWithKagi("invalid envelope", { fetch: invalidEnvelopeFetch }, fakeAuthStorage)).rejects.toThrow( + "Kagi API returned an invalid response: expected an object envelope", + ); + }); + + it("recognizes errors plural in a successful HTTP envelope", async () => { + const fetchMock: FetchImpl = async () => + new Response(JSON.stringify({ errors: [{ code: 429, message: "quota exceeded" }] }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + + await expect(searchWithKagi("envelope error", { fetch: fetchMock }, fakeAuthStorage)).rejects.toThrow( + "Kagi API error (429): quota exceeded", + ); + }); it("applies the configured timeout at the provider fetch boundary", async () => { const timeoutSignal = new AbortController().signal; const timeoutSpy = vi.spyOn(AbortSignal, "timeout").mockReturnValue(timeoutSignal); @@ -158,6 +186,39 @@ describe("Kagi search result parsing", () => { expect(result.answer).toBeUndefined(); }); + it("accepts documented result aliases and skips malformed items", async () => { + const fetchMock: FetchImpl = async () => + new Response( + JSON.stringify({ + data: { + search: [ + null, + { title: "Missing URL" }, + { + href: "https://example.com/alias", + name: "Alias Result", + description: "Alias description", + }, + ], + related_search: { invalid: true }, + }, + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ); + + const result = await searchWithKagi("aliases", { fetch: fetchMock }, fakeAuthStorage); + + expect(result.sources).toEqual([ + { + title: "Alias Result", + url: "https://example.com/alias", + snippet: "Alias description", + publishedDate: undefined, + }, + ]); + expect(result.relatedQuestions).toEqual([]); + }); + it("parses direct_answer into the answer field", async () => { const fetchMock: FetchImpl = async () => new Response( diff --git a/packages/coding-agent/test/tools/web-search-parallel.test.ts b/packages/coding-agent/test/tools/web-search-parallel.test.ts index 1cda9781f..55cb51454 100644 --- a/packages/coding-agent/test/tools/web-search-parallel.test.ts +++ b/packages/coding-agent/test/tools/web-search-parallel.test.ts @@ -1,4 +1,4 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from "bun:test"; +import { afterEach, beforeEach, describe, expect, it, setSystemTime, vi } from "bun:test"; import type { AuthStorage, FetchImpl } from "@oh-my-pi/pi-ai"; import type { AgentStorage } from "@oh-my-pi/pi-coding-agent/session/agent-storage"; import { searchWithParallel } from "@oh-my-pi/pi-coding-agent/web/parallel"; @@ -149,6 +149,29 @@ describe("Parallel web search", () => { }); }); + it("maps recency onto source_policy.after_date", async () => { + setSystemTime(new Date("2026-08-10T12:00:00Z")); + try { + const fetchMock = mockFetch({ + search_id: "search-parallel-recency", + results: [], + warnings: null, + usage: null, + }); + + await searchParallel({ query: "recent api changes", recency: "week", fetch: fetchMock }, fakeAuthStorage); + expect(capturedRequestBody).toEqual({ + objective: "recent api changes", + search_queries: ["recent api changes"], + mode: "fast", + excerpts: { max_chars_per_result: 10_000 }, + source_policy: { after_date: "2026-08-03" }, + }); + } finally { + setSystemTime(); + } + }); + it("maps -site: and after: onto exclude_domains/after_date, keeping phrases and negation", async () => { const fetchMock = mockFetch({ search_id: "search-parallel-4", @@ -158,7 +181,11 @@ describe("Parallel web search", () => { }); await searchParallel( - { query: '"web api" -legacy -site:reddit.com/r/node after:2025-06-01', fetch: fetchMock }, + { + query: '"web api" -legacy -site:reddit.com/r/node after:2025-06-01', + recency: "day", + fetch: fetchMock, + }, fakeAuthStorage, ); expect(capturedRequestBody).toEqual({ @@ -178,4 +205,13 @@ describe("Parallel web search", () => { message: "Parallel API error (503): upstream unavailable", }); }); + + it("classifies malformed successful responses as Parallel errors", async () => { + const fetchMock: FetchImpl = () => + Promise.resolve(new Response("{not-json", { status: 200, headers: { "Content-Type": "application/json" } })); + await expect(searchParallel({ query: "broken", fetch: fetchMock }, fakeAuthStorage)).rejects.toMatchObject({ + provider: "parallel", + message: expect.stringContaining("Parallel search returned invalid JSON:"), + }); + }); }); diff --git a/packages/coding-agent/test/tools/web-search-searxng.test.ts b/packages/coding-agent/test/tools/web-search-searxng.test.ts index 31991306a..dd1625f9d 100644 --- a/packages/coding-agent/test/tools/web-search-searxng.test.ts +++ b/packages/coding-agent/test/tools/web-search-searxng.test.ts @@ -121,6 +121,53 @@ describe("SearXNG web search provider", () => { expect(captured.url?.searchParams.get("language")).toBeNull(); }); + it("sends configured category and safe-search filters and preserves instant answers", async () => { + const agentDir = await fs.mkdtemp(path.join(os.tmpdir(), "searxng-filters-")); + try { + await Bun.write( + path.join(agentDir, "config.yml"), + [ + "searxng:", + " endpoint: https://searx.example.org", + " categories: news", + " safesearch: 2", + "", + ].join("\n"), + ); + await Settings.init({ agentDir }); + + const captured: { url?: URL } = {}; + const fetchMock: FetchImpl = input => { + captured.url = new URL(input.toString()); + return Promise.resolve( + new Response( + JSON.stringify({ + results: [{ title: "r", url: "https://example.com", snippet: "Fallback snippet" }], + answers: [ + " Forty-two ", + { template: "answer/legacy.html", answer: "Legacy answer" }, + { + template: "answer/translations.html", + translations: [{ text: "Hallo" }, { text: "Guten Tag" }], + }, + ], + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ), + ); + }; + + const response = await searchSearXNG({ query: "filtered answers", fetch: fetchMock }); + + expect(captured.url?.searchParams.get("categories")).toBe("news"); + expect(captured.url?.searchParams.get("safesearch")).toBe("2"); + expect(response.answer).toBe("Forty-two\n\nLegacy answer\n\nHallo\nGuten Tag"); + expect(response.sources[0]?.snippet).toBe("Fallback snippet"); + } finally { + await removeWithRetries(agentDir); + } + }); + it("reads Basic auth credentials from nested config.yml settings", async () => { const agentDir = await fs.mkdtemp(path.join(os.tmpdir(), "searxng-settings-")); try { diff --git a/packages/coding-agent/test/tools/web-search-tinyfish.test.ts b/packages/coding-agent/test/tools/web-search-tinyfish.test.ts index efa8e2258..a697c11ae 100644 --- a/packages/coding-agent/test/tools/web-search-tinyfish.test.ts +++ b/packages/coding-agent/test/tools/web-search-tinyfish.test.ts @@ -84,10 +84,16 @@ describe("TinyFish web search provider", () => { }); expect(captured).toHaveLength(1); - expect(captured[0].searchParams.get("query")).toBe( - '"error handling" rust site:github.com -site:gitlab.com filetype:pdf', - ); - expectTinyFishParams(captured[0], ["query", "num_results", "page"]); + expect(captured[0].searchParams.get("query")).toBe('"error handling" rust filetype:pdf'); + expect(captured[0].searchParams.get("include_domains")).toBe("github.com"); + expect(captured[0].searchParams.get("exclude_domains")).toBe("gitlab.com"); + expectTinyFishParams(captured[0], [ + "query", + "num_results", + "page", + "include_domains", + "exclude_domains", + ]); }); it("sends directive-free queries verbatim", async () => { @@ -232,6 +238,43 @@ describe("TinyFish web search provider", () => { expect(response.sources.at(-1)?.url).toBe("https://example.com/raw-page-11"); }); + it("deduplicates and normalizes results across pages", async () => { + const captured: URL[] = []; + const firstPage = tinyFishResults("dedupe", 10); + firstPage[0] = { + title: " Primary title ", + url: " https://example.com/dedupe-0 ", + snippet: " spaced \n snippet ", + site_name: " Example ", + }; + firstPage[1] = { + title: "Duplicate title", + url: "https://example.com/dedupe-0", + snippet: "duplicate snippet", + }; + const fetchMock: FetchImpl = async input => { + const url = input instanceof URL ? input : new URL(typeof input === "string" ? input : input.url); + captured.push(url); + const page = Number(url.searchParams.get("page") ?? 0); + const results = page === 0 ? firstPage : tinyFishResults("dedupe", 1, 10); + return new Response(JSON.stringify(tinyFishPage(results, page, 11)), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + }; + + const response = await searchTinyFish({ ...makeParams("dedupe fish"), limit: 10, fetch: fetchMock }); + + expect(captured.map(url => url.searchParams.get("page"))).toEqual(["0", "1"]); + expect(response.sources).toHaveLength(10); + expect(response.sources[0]).toMatchObject({ + title: "Primary title", + url: "https://example.com/dedupe-0", + snippet: "spaced snippet", + }); + expect(response.sources.at(-1)?.url).toBe("https://example.com/dedupe-10"); + }); + it("stops early for limit 20 when page 0 returns fewer than 10 raw results", async () => { const captured: URL[] = []; const fetchMock: FetchImpl = async input => { diff --git a/packages/coding-agent/test/tools/web-search-xai.test.ts b/packages/coding-agent/test/tools/web-search-xai.test.ts index d62b2e132..3a49fbc27 100644 --- a/packages/coding-agent/test/tools/web-search-xai.test.ts +++ b/packages/coding-agent/test/tools/web-search-xai.test.ts @@ -802,6 +802,67 @@ describe("xAI web search provider", () => { }); }); + it("extracts offset snippets and raw sources from web_search_call output", async () => { + const answer = "Context before [cited source](https://example.com/cited) context after."; + const start = answer.indexOf("[cited source]"); + const capture = captureFetch({ + id: "resp_raw_sources", + output: [ + { + type: "message", + content: [ + { + type: "output_text", + text: answer, + annotations: [ + { + type: "url_citation", + url: "https://example.com/cited", + title: "Cited result", + start_index: start, + end_index: start + "[cited source]".length, + }, + ], + }, + ], + }, + { + type: "web_search_call", + action: { + sources: [ + { url: "https://example.com/raw", title: "Raw result" }, + { source_website_url: "https://example.com/fallback", caption: "Fallback result" }, + ], + }, + results: [{ url: "https://example.com/cited", title: "Duplicate result" }], + }, + ], + }); + + const response = await searchXAI(makeParams(capture.fetchMock)); + + expect(response.answer).toBe(answer); + expect(response.sources).toEqual([ + { + title: "Cited result", + url: "https://example.com/cited", + snippet: "Context before cited source context after.", + }, + { title: "Raw result", url: "https://example.com/raw", snippet: undefined }, + { title: "Fallback result", url: "https://example.com/fallback", snippet: undefined }, + ]); + }); + + it("rejects successful responses with no answer or sources", async () => { + const capture = captureFetch({ id: "resp_empty", output: [] }); + + await expect(searchXAI(makeParams(capture.fetchMock))).rejects.toMatchObject({ + provider: "xai", + status: 502, + message: "xAI web_search returned no answer or sources", + }); + }); + it.each([ [401, "xai: 401 unauthorized"], [402, "xai: 402 credits exhausted"], diff --git a/packages/coding-agent/test/web/search/abort-and-timeout.test.ts b/packages/coding-agent/test/web/search/abort-and-timeout.test.ts index 5193def5b..8928975eb 100644 --- a/packages/coding-agent/test/web/search/abort-and-timeout.test.ts +++ b/packages/coding-agent/test/web/search/abort-and-timeout.test.ts @@ -144,8 +144,6 @@ describe("Brave provider hard-timeout wiring", () => { }); it("hands fetch a composed signal even with no caller signal — confirms the rollout reaches non-Anthropic providers", async () => { - process.env.BRAVE_API_KEY = "brave-test-key"; - let capturedSignal: AbortSignal | null | undefined; const fetchMock: FetchImpl = async (_input, init) => { capturedSignal = init?.signal; @@ -155,7 +153,13 @@ describe("Brave provider hard-timeout wiring", () => { }); }; - await searchBrave({ query: "ping", fetch: fetchMock }); + await searchBrave({ + query: "ping", + fetch: fetchMock, + authStorage: { + resolver: vi.fn(() => async () => "brave-test-key"), + } as unknown as AuthStorage, + }); expect(capturedSignal).toBeInstanceOf(AbortSignal); expect(capturedSignal?.aborted).toBe(false); diff --git a/packages/coding-agent/test/web/search/perplexity.test.ts b/packages/coding-agent/test/web/search/perplexity.test.ts index 437817bf7..adb8521af 100644 --- a/packages/coding-agent/test/web/search/perplexity.test.ts +++ b/packages/coding-agent/test/web/search/perplexity.test.ts @@ -75,11 +75,13 @@ describe("Perplexity API-key request shape", () => { const savedOpenRouterKey = process.env.OPENROUTER_API_KEY; const savedCookies = process.env.PERPLEXITY_COOKIES; const savedResponsesMode = process.env.PI_PERPLEXITY_RESPONSES; + const savedApiModel = process.env.PI_PERPLEXITY_API_MODEL; beforeEach(() => { process.env.PERPLEXITY_API_KEY = "test-key"; delete process.env.PERPLEXITY_COOKIES; delete process.env.PI_PERPLEXITY_RESPONSES; + delete process.env.PI_PERPLEXITY_API_MODEL; }); afterEach(() => { @@ -92,6 +94,8 @@ describe("Perplexity API-key request shape", () => { else process.env.PERPLEXITY_COOKIES = savedCookies; if (savedResponsesMode === undefined) delete process.env.PI_PERPLEXITY_RESPONSES; else process.env.PI_PERPLEXITY_RESPONSES = savedResponsesMode; + if (savedApiModel === undefined) delete process.env.PI_PERPLEXITY_API_MODEL; + else process.env.PI_PERPLEXITY_API_MODEL = savedApiModel; }); it("requests comprehensive defaults: 20 results, high context, related questions", async () => { @@ -104,6 +108,18 @@ describe("Perplexity API-key request shape", () => { expect(body?.return_related_questions).toBe(true); }); + it("accepts a configured direct API model", async () => { + process.env.PI_PERPLEXITY_API_MODEL = "sonar-deep-research"; + let body: Record | undefined; + await searchPerplexity({ + query: "quic vs tcp", + authStorage: apiKeyAuthStorage, + fetch: mockApi(b => (body = b), baseResponse()), + }); + + expect(body?.model).toBe("sonar-deep-research"); + }); + it("honors a caller-supplied num_search_results over the default", async () => { let body: Record | undefined; const fetchMock = mockApi(b => (body = b), baseResponse()); @@ -306,7 +322,10 @@ const anonymousAuthStorage = { }, } as unknown as AuthStorage; -function mockOAuth(capture: (body: Record, headers: Headers) => void): FetchImpl { +function mockOAuth( + capture: (body: Record, headers: Headers) => void, + eventOverrides: Record = {}, +): FetchImpl { const event = { final: true, display_model: "turbo", @@ -318,6 +337,7 @@ function mockOAuth(capture: (body: Record, headers: Headers) => web_result_block: { web_results: [{ name: "T", url: "https://example.com", snippet: "s" }] }, }, ], + ...eventOverrides, }; const sseBody = `data: ${JSON.stringify(event)}\n\n`; return async (input, init) => { @@ -356,15 +376,19 @@ function mockAnonymous(capture: (body: Record, headers: Headers describe("Perplexity OAuth request shape", () => { const savedCookies = process.env.PERPLEXITY_COOKIES; + const savedModel = process.env.PI_PERPLEXITY_MODEL; beforeEach(() => { delete process.env.PERPLEXITY_COOKIES; // cookies take precedence over oauth; keep them out + delete process.env.PI_PERPLEXITY_MODEL; }); afterEach(() => { vi.restoreAllMocks(); if (savedCookies === undefined) delete process.env.PERPLEXITY_COOKIES; else process.env.PERPLEXITY_COOKIES = savedCookies; + if (savedModel === undefined) delete process.env.PI_PERPLEXITY_MODEL; + else process.env.PI_PERPLEXITY_MODEL = savedModel; }); it("sends the bare query, never the API-style system prompt, to the ask endpoint", async () => { @@ -395,6 +419,55 @@ describe("Perplexity OAuth request shape", () => { expect(headers?.has("authorization")).toBe(false); expect(response.authMode).toBe("oauth"); expect(response.answer).toBe("OAuth answer"); + // Authenticated streams sometimes report only the generic `turbo` alias; + // preserve the requested subscription model instead of misreporting it. + expect(response.model).toBe("experimental"); + }); + + it("accepts a subscription model preference and reports it when the stream returns turbo", async () => { + let body: Record | undefined; + const response = await searchPerplexity({ + query: "latest model", + subscription_model: "pplx_reasoning", + authStorage: oauthAuthStorage, + fetch: mockOAuth(b => (body = b)), + }); + + expect((body!.params as Record).model_preference).toBe("pplx_reasoning"); + expect(response.model).toBe("pplx_reasoning"); + }); + + it("prefers the concrete user-selected model over the generic display alias", async () => { + const response = await searchPerplexity({ + query: "latest model", + authStorage: oauthAuthStorage, + fetch: mockOAuth(() => {}, { user_selected_model: "pplx_pro_upgraded" }), + }); + + expect(response.model).toBe("pplx_pro_upgraded"); + }); + + it("deduplicates equivalent subscription source URLs", async () => { + const response = await searchPerplexity({ + query: "latest model", + authStorage: oauthAuthStorage, + fetch: mockOAuth(() => {}, { + blocks: [ + { intended_usage: "ask_text", markdown_block: { answer: "OAuth answer" } }, + { + intended_usage: "web_results", + web_result_block: { + web_results: [ + { name: "First", url: "https://EXAMPLE.com/path/" }, + { name: "Duplicate", url: "https://example.com/path" }, + ], + }, + }, + ], + }), + }); + + expect(response.sources).toHaveLength(1); }); it("maps directives onto ask-endpoint native filters and rewrites query_str", async () => { diff --git a/packages/coding-agent/test/web/search/provider-chain.test.ts b/packages/coding-agent/test/web/search/provider-chain.test.ts index 7a6044ab4..463c3f955 100644 --- a/packages/coding-agent/test/web/search/provider-chain.test.ts +++ b/packages/coding-agent/test/web/search/provider-chain.test.ts @@ -9,7 +9,11 @@ import { } from "@oh-my-pi/pi-coding-agent/web/search/provider"; import { SEARCH_PROVIDER_ORDER } from "@oh-my-pi/pi-coding-agent/web/search/types"; -const authStorage = {} as AuthStorage; +const authStorage = { + hasAuth(provider: string): boolean { + return provider === "jina" && Boolean(process.env.JINA_API_KEY); + }, +} as AuthStorage; const originalBraveApiKey = process.env.BRAVE_API_KEY; const originalJinaApiKey = process.env.JINA_API_KEY; diff --git a/packages/coding-agent/test/web/search/tavily.test.ts b/packages/coding-agent/test/web/search/tavily.test.ts index c08d5f8c0..df8f754b3 100644 --- a/packages/coding-agent/test/web/search/tavily.test.ts +++ b/packages/coding-agent/test/web/search/tavily.test.ts @@ -214,4 +214,35 @@ describe("Tavily searchTavily request shape (integration)", () => { expect(capturedBodies[1]).not.toHaveProperty("end_date"); expect(response.sources).toHaveLength(1); }); + + it("ignores malformed response fields while preserving valid results", async () => { + process.env.TAVILY_API_KEY = "test-key"; + const fetchMock: FetchImpl = async () => + new Response( + JSON.stringify({ + answer: 42, + results: [ + null, + { title: 7, url: "https://example.com/valid", content: 99, published_date: false }, + { title: "Missing URL" }, + ], + request_id: 123, + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ); + + const response = await searchTavily({ ...makeParams("robust parsing"), fetch: fetchMock }); + + expect(response.answer).toBeUndefined(); + expect(response.requestId).toBeUndefined(); + expect(response.sources).toEqual([ + { + title: "https://example.com/valid", + url: "https://example.com/valid", + snippet: undefined, + publishedDate: undefined, + ageSeconds: undefined, + }, + ]); + }); });