feat(coding-agent/web): expanded web search and fetching providers

- Expanded web search and fetching providers with robust parsing, authentication storage integration, and response validation.
- Added support for new configurations including SearXNG safesearch, Cloudflare AI Gateway endpoints, and dynamic Firecrawl base URLs.
- Implemented comprehensive test suites covering error handling, content filtering, and provider-specific response behaviors.
This commit is contained in:
can1357
2026-08-10 11:06:22 +02:00
parent 45e12e5bb7
commit baf8a1df7e
31 changed files with 1660 additions and 224 deletions
+7
View File
@@ -2,6 +2,13 @@
## [Unreleased]
### Added
- Added `searxng.safesearch` setting option for SearXNG searches
- Added support for Cloudflare AI Gateway routing for Gemini search
- Added support for Exa MCP search provider
- Added domain inclusion/exclusion filtering and URL deduplication for TinyFish search
## [17.2.12] - 2026-08-08
### Fixed
@@ -5471,6 +5471,11 @@ export const SETTINGS_SCHEMA = {
default: undefined,
},
"searxng.safesearch": {
type: "number",
default: undefined,
},
"commit.mapReduceEnabled": { type: "boolean", default: true },
"commit.mapReduceMinFiles": { type: "number", default: 4 },
+25 -2
View File
@@ -576,6 +576,23 @@ async function parseFeedToMarkdown(content: string, maxItems = 10): Promise<stri
* local fallback renderers (trafilatura, lynx, native). See #1449.
*/
const REMOTE_READER_MAX_MS = 10_000;
const JINA_MARKDOWN_MARKER = "Markdown Content:";
const JINA_READER_MAX_BYTES = 2 * 1024 * 1024;
function parseJinaReaderContent(responseBody: string): string | null {
const markerStart = responseBody.indexOf(JINA_MARKDOWN_MARKER);
if (markerStart < 0) return null;
const content = responseBody.slice(markerStart + JINA_MARKDOWN_MARKER.length).trim();
if (
content.length < 100 ||
content.startsWith("Loading...") ||
content.startsWith("Please enable JavaScript")
) {
return null;
}
return content;
}
/** Reader backends for {@link renderHtmlToText}, in default priority order. */
export type FetchProvider = "native" | "trafilatura" | "lynx" | "parallel" | "jina";
@@ -653,10 +670,16 @@ export async function renderHtmlToText(
},
jina: async () => {
const response = await fetchImpl(`https://r.jina.ai/${url}`, {
headers: { Accept: "text/markdown" },
headers: {
Accept: "text/markdown",
"X-No-Cache": "true",
},
signal: remoteSignal(),
});
return response.ok ? await response.text() : null;
if (!response.ok) return null;
const contentLength = Number(response.headers.get("content-length"));
if (Number.isFinite(contentLength) && contentLength > JINA_READER_MAX_BYTES) return null;
return parseJinaReaderContent(await response.text());
},
};
+92 -34
View File
@@ -83,6 +83,7 @@ export interface KagiErrorResponse {
detail?: string;
}
// ---------------------------------------------------------------------------
// Error Handling
// ---------------------------------------------------------------------------
@@ -98,7 +99,7 @@ export class KagiApiError extends Error {
}
function extractKagiErrorMessage(payload: unknown): string | null {
if (!payload || typeof payload !== "object") return null;
if (!payload || typeof payload !== "object" || Array.isArray(payload)) return null;
const record = payload as Record<string, unknown>;
for (const value of [record.message, record.detail]) {
@@ -107,17 +108,20 @@ function extractKagiErrorMessage(payload: unknown): string | null {
}
}
if (typeof record.error === "string" && record.error.trim().length > 0) {
return record.error.trim();
}
if (Array.isArray(record.error)) {
for (const entry of record.error) {
for (const errors of [record.error, record.errors]) {
if (typeof errors === "string" && errors.trim().length > 0) {
return errors.trim();
}
if (!Array.isArray(errors)) continue;
for (const entry of errors) {
if (!entry || typeof entry !== "object") continue;
const e = entry as Record<string, unknown>;
for (const value of [e.message, e.msg]) {
if (typeof value === "string" && value.trim().length > 0) {
return value.trim();
for (const value of [e.message, e.msg, e.code]) {
if (
(typeof value === "string" && value.trim().length > 0) ||
(typeof value === "number" && Number.isFinite(value))
) {
return String(value).trim();
}
}
}
@@ -147,6 +151,34 @@ function parseKagiErrorResponse(statusCode: number, responseText: string): KagiA
}
}
function parseKagiSuccessResponse(statusCode: number, responseText: string): KagiSearchResponse {
let payload: unknown;
try {
payload = JSON.parse(responseText);
} catch {
throw new KagiApiError("Kagi API returned an invalid response: invalid JSON", statusCode);
}
if (!payload || typeof payload !== "object" || Array.isArray(payload)) {
throw new KagiApiError("Kagi API returned an invalid response: expected an object envelope", statusCode);
}
const record = payload as Record<string, unknown>;
const errorMessage = extractKagiErrorMessage(payload);
if (errorMessage && (record.error !== undefined || record.errors !== undefined)) {
const errors = Array.isArray(record.error) ? record.error : Array.isArray(record.errors) ? record.errors : [];
const first = errors[0];
const code =
first && typeof first === "object" && typeof (first as Record<string, unknown>).code === "number"
? ((first as Record<string, unknown>).code as number)
: statusCode;
throw createKagiApiError(code, errorMessage);
}
if (record.data !== undefined && (!record.data || typeof record.data !== "object" || Array.isArray(record.data))) {
throw new KagiApiError("Kagi API returned an invalid response: expected data to be an object", statusCode);
}
return payload as KagiSearchResponse;
}
// ---------------------------------------------------------------------------
// Public API
// ---------------------------------------------------------------------------
@@ -216,23 +248,40 @@ function buildRequestBody(query: string, options: KagiSearchOptions): KagiSearch
return req;
}
/** Push every item in a result bucket as a source, with an optional title tag. */
function collectSources(sources: KagiSearchSource[], items: KagiSearchResultItem[] | undefined, tag?: string): void {
if (!items) return;
for (const item of items) {
function firstNonEmptyString(...values: unknown[]): string | undefined {
for (const value of values) {
if (typeof value === "string" && value.trim().length > 0) return value.trim();
}
return undefined;
}
/** Push every valid item in a result bucket as a source, with an optional title tag. */
function collectSources(sources: KagiSearchSource[], items: unknown, tag?: string): void {
if (!Array.isArray(items)) return;
for (const value of items) {
if (!value || typeof value !== "object" || Array.isArray(value)) continue;
const item = value as Record<string, unknown>;
const url = firstNonEmptyString(item.url, item.href, item.link);
if (!url) continue;
const title = firstNonEmptyString(item.title, item.name) ?? url;
sources.push({
title: tag ? `${tag} ${item.title}` : item.title,
url: item.url,
snippet: item.snippet,
publishedDate: item.time,
title: tag ? `${tag} ${title}` : title,
url,
snippet: firstNonEmptyString(item.snippet, item.description, item.summary),
publishedDate: firstNonEmptyString(item.time),
});
}
}
/** Pull a related/adjacent question from an item's props or fall back to title. */
function questionOf(item: KagiSearchResultItem): string | undefined {
const q = item.props?.question ?? item.props?.query ?? item.title;
return typeof q === "string" && q.length > 0 ? q : undefined;
function questionOf(value: unknown): string | undefined {
if (!value || typeof value !== "object" || Array.isArray(value)) return undefined;
const item = value as Record<string, unknown>;
const props =
item.props && typeof item.props === "object" && !Array.isArray(item.props)
? (item.props as Record<string, unknown>)
: undefined;
return firstNonEmptyString(props?.question, props?.query, item.title);
}
export async function searchWithKagi(
@@ -269,11 +318,7 @@ export async function searchWithKagi(
},
);
const payload = (await response.json()) as KagiSearchResponse;
if (payload.error && payload.error.length > 0) {
const first = payload.error[0];
throw createKagiApiError(first.code ?? response.status, extractKagiErrorMessage(payload) ?? first.message);
}
const payload = parseKagiSuccessResponse(response.status, await response.text());
const data = payload.data;
const sources: KagiSearchSource[] = [];
@@ -284,17 +329,30 @@ export async function searchWithKagi(
collectSources(sources, data?.news, "[News]");
collectSources(sources, data?.infobox, "[Info]");
for (const item of data?.adjacent_question ?? []) {
const q = questionOf(item);
if (q) relatedQuestions.push(q);
const adjacentQuestions: unknown = data?.adjacent_question;
if (Array.isArray(adjacentQuestions)) {
for (const item of adjacentQuestions) {
const question = questionOf(item);
if (question) relatedQuestions.push(question);
}
}
for (const item of data?.related_search ?? []) {
const q = questionOf(item);
if (q) relatedQuestions.push(q);
const relatedSearches: unknown = data?.related_search;
if (Array.isArray(relatedSearches)) {
for (const item of relatedSearches) {
const question = questionOf(item);
if (question) relatedQuestions.push(question);
}
}
const directAnswer = data?.direct_answer?.[0];
const answer = directAnswer ? (directAnswer.snippet ?? directAnswer.title) : undefined;
const directAnswers: unknown = data?.direct_answer;
const directAnswer = Array.isArray(directAnswers) ? directAnswers[0] : undefined;
const answer =
directAnswer && typeof directAnswer === "object" && !Array.isArray(directAnswer)
? firstNonEmptyString(
(directAnswer as Record<string, unknown>).snippet,
(directAnswer as Record<string, unknown>).title,
)
: undefined;
return {
requestId: payload.meta?.trace ?? payload.meta?.id ?? "",
+14 -2
View File
@@ -146,6 +146,18 @@ export function parseParallelErrorResponse(statusCode: number, responseText: str
}
}
export async function parseParallelJsonResponse(
response: Response,
operation: "search" | "extract",
): Promise<unknown> {
try {
return await response.json();
} catch (err) {
const detail = err instanceof Error ? err.message : String(err);
throw new ParallelApiError(`Parallel ${operation} returned invalid JSON: ${detail}`);
}
}
function getAuthHeaders(apiKey: string): {
Accept: string;
"Content-Type": string;
@@ -316,7 +328,7 @@ export async function searchWithParallel(
throw parseParallelErrorResponse(response.status, await response.text());
}
const payload: unknown = await response.json();
const payload = await parseParallelJsonResponse(response, "search");
return parseParallelSearchPayload(payload);
}
@@ -349,6 +361,6 @@ export async function extractWithParallel(
throw parseParallelErrorResponse(response.status, await response.text());
}
const payload: unknown = await response.json();
const payload = await parseParallelJsonResponse(response, "extract");
return parseExtractPayload(payload);
}
@@ -4,7 +4,7 @@
* Calls Brave's web search REST API and maps results into the unified
* SearchResponse shape used by the web search tool.
*/
import { type AuthStorage, type FetchImpl, getEnvApiKey } from "@oh-my-pi/pi-ai";
import { type ApiKey, type AuthStorage, type FetchImpl, getEnvApiKey, withAuth } from "@oh-my-pi/pi-ai";
import type { SearchResponse, SearchSource } from "../../../web/search/types";
import { SearchProviderError } from "../../../web/search/types";
import type { QuerySyntax, StructuredQuery } from "../query";
@@ -17,6 +17,9 @@ import { classifyProviderHttpError, withHardTimeout } from "./utils";
const BRAVE_SEARCH_URL = "https://api.search.brave.com/res/v1/web/search";
const DEFAULT_NUM_RESULTS = 10;
const MAX_NUM_RESULTS = 20;
const MAX_QUERY_CHARACTERS = 500;
const MAX_RESPONSE_BYTES = 2 * 1024 * 1024;
const MAX_ERROR_BYTES = 8 * 1024;
const RECENCY_MAP: Record<"day" | "week" | "month" | "year", "pd" | "pw" | "pm" | "py"> = {
day: "pd",
@@ -51,62 +54,122 @@ export interface BraveSearchParams {
num_results?: number;
recency?: "day" | "week" | "month" | "year";
parsedQuery?: StructuredQuery;
/** Two-letter market code, or `ALL`. */
country?: string;
/** Brave search language code, such as `en` or `zh-hans`. */
search_lang?: string;
safesearch?: "off" | "moderate" | "strict";
authStorage: AuthStorage;
sessionId?: string;
signal?: AbortSignal;
timeoutMs?: number;
fetch?: FetchImpl;
}
interface BraveSearchResult {
title?: string | null;
url?: string | null;
description?: string | null;
age?: string | null;
extra_snippets?: string[] | null;
}
interface BraveSearchResponse {
web?: {
results?: BraveSearchResult[];
};
web?: unknown;
}
/** Find BRAVE_API_KEY from environment or .env files. */
export function findApiKey(): string | null {
return getEnvApiKey("brave") ?? null;
function normalizeText(value: unknown, maxLength: number): string | undefined {
if (typeof value !== "string") return undefined;
const text = value
.replace(/<[^>]*>/g, " ")
.replace(/\s+/g, " ")
.trim();
if (!text) return undefined;
return text.length <= maxLength ? text : `${text.slice(0, maxLength - 1)}…`;
}
function buildSnippet(result: BraveSearchResult): string | undefined {
const snippets: string[] = [];
function normalizeUrl(value: unknown): string | undefined {
if (typeof value !== "string" || value.length > 2048) return undefined;
try {
const url = new URL(value);
if (url.protocol !== "http:" && url.protocol !== "https:") return undefined;
return url.toString();
} catch {
return undefined;
}
}
if (result.description?.trim()) {
snippets.push(result.description.trim());
function webResults(response: BraveSearchResponse): readonly unknown[] {
if (typeof response.web !== "object" || response.web === null || !("results" in response.web)) return [];
return Array.isArray(response.web.results) ? response.web.results : [];
}
async function readLimitedText(response: Response, maxBytes: number, truncate = false): Promise<string> {
if (!response.body) return "";
const reader = response.body.getReader();
let buffer = new Uint8Array(Math.min(maxBytes, 64 * 1024));
let bytes = 0;
try {
for (;;) {
const { done, value } = await reader.read();
if (done) break;
const accepted = Math.min(value.byteLength, maxBytes - bytes);
const nextBytes = bytes + accepted;
if (nextBytes > buffer.byteLength) {
const grown = new Uint8Array(Math.min(maxBytes, Math.max(nextBytes, buffer.byteLength * 2)));
grown.set(buffer.subarray(0, bytes));
buffer = grown;
}
buffer.set(value.subarray(0, accepted), bytes);
bytes = nextBytes;
if (accepted < value.byteLength) {
await reader.cancel().catch(() => undefined);
if (!truncate) throw new SearchProviderError("brave", "Brave API response exceeded 2 MiB", 500);
break;
}
}
} finally {
reader.releaseLock();
}
if (Array.isArray(result.extra_snippets)) {
for (const snippet of result.extra_snippets) {
if (!snippet?.trim()) continue;
if (snippets.includes(snippet.trim())) continue;
snippets.push(snippet.trim());
return new TextDecoder().decode(buffer.subarray(0, bytes));
}
function buildSnippet(result: object): string | undefined {
const snippets = new Set<string>();
const description = normalizeText("description" in result ? result.description : undefined, 8_000);
if (description) snippets.add(description);
const extras = "extra_snippets" in result ? result.extra_snippets : undefined;
if (Array.isArray(extras)) {
for (const value of extras) {
const snippet = normalizeText(value, 8_000);
if (snippet) snippets.add(snippet);
}
}
return snippets.length > 0 ? snippets.join("\n") : undefined;
const combined = [...snippets].join("\n");
return combined ? (combined.length <= 8_000 ? combined : `${combined.slice(0, 7_999)}…`) : undefined;
}
async function callBraveSearch(
apiKey: string,
params: BraveSearchParams,
): Promise<{ response: BraveSearchResponse; requestId?: string }> {
const numResults = clampNumResults(params.num_results, DEFAULT_NUM_RESULTS, MAX_NUM_RESULTS);
const numResults = Math.floor(clampNumResults(params.num_results, DEFAULT_NUM_RESULTS, MAX_NUM_RESULTS));
const parsed = params.parsedQuery ?? parseSearchQuery(params.query);
const query = parsed.hasDirectives ? formatQuery(parsed, BRAVE_QUERY_SYNTAX) : params.query;
if (query.length > MAX_QUERY_CHARACTERS) {
throw new SearchProviderError(
"brave",
`Brave search queries cannot exceed ${MAX_QUERY_CHARACTERS} characters`,
400,
);
}
const url = new URL(BRAVE_SEARCH_URL);
url.searchParams.set("q", parsed.hasDirectives ? formatQuery(parsed, BRAVE_QUERY_SYNTAX) : params.query);
url.searchParams.set("q", query);
url.searchParams.set("count", String(numResults));
url.searchParams.set("extra_snippets", "true");
url.searchParams.set("text_decorations", "false");
url.searchParams.set("safesearch", params.safesearch ?? "moderate");
if (params.country) url.searchParams.set("country", params.country.toUpperCase());
if (params.search_lang) url.searchParams.set("search_lang", params.search_lang);
const freshness = braveFreshness(parsed, params.recency);
if (freshness) {
url.searchParams.set("freshness", freshness);
}
if (freshness) url.searchParams.set("freshness", freshness);
const fetchImpl = params.fetch ?? fetch;
const response = await fetchImpl(url, {
@@ -118,36 +181,47 @@ async function callBraveSearch(
});
if (!response.ok) {
const errorText = await response.text();
const errorText = await readLimitedText(response, MAX_ERROR_BYTES, true);
const classified = classifyProviderHttpError("brave", response.status, errorText);
if (classified) throw classified;
throw new SearchProviderError("brave", `Brave API error (${response.status}): ${errorText}`, response.status);
}
const data = (await response.json()) as BraveSearchResponse;
const raw = await readLimitedText(response, MAX_RESPONSE_BYTES);
let data: BraveSearchResponse;
try {
data = JSON.parse(raw) as BraveSearchResponse;
} catch {
throw new SearchProviderError("brave", "Brave API returned invalid JSON", 500);
}
const requestId = response.headers.get("x-request-id") ?? response.headers.get("request-id") ?? undefined;
return { response: data, requestId };
}
/** Execute Brave web search. */
export async function searchBrave(params: BraveSearchParams): Promise<SearchResponse> {
const numResults = clampNumResults(params.num_results, DEFAULT_NUM_RESULTS, MAX_NUM_RESULTS);
const apiKey = findApiKey();
if (!apiKey) {
throw new Error("BRAVE_API_KEY not found. Set it in environment or .env file.");
}
const { response, requestId } = await callBraveSearch(apiKey, params);
const numResults = Math.floor(clampNumResults(params.num_results, DEFAULT_NUM_RESULTS, MAX_NUM_RESULTS));
const keyOrResolver: ApiKey = params.authStorage.resolver("brave", {
sessionId: params.sessionId,
});
const { response, requestId } = await withAuth(keyOrResolver, key => callBraveSearch(key, params), {
signal: params.signal,
missingKeyMessage:
'Brave credentials not found. Set BRAVE_API_KEY or configure an API key for provider "brave".',
});
const sources: SearchSource[] = [];
for (const result of response.web?.results ?? []) {
if (!result.url) continue;
for (const result of webResults(response)) {
if (typeof result !== "object" || result === null) continue;
const url = normalizeUrl("url" in result ? result.url : undefined);
if (!url) continue;
const publishedDate = normalizeText("age" in result ? result.age : undefined, 100);
sources.push({
title: result.title ?? result.url,
url: result.url,
title: normalizeText("title" in result ? result.title : undefined, 300) ?? url,
url,
snippet: buildSnippet(result),
publishedDate: result.age ?? undefined,
ageSeconds: dateToAgeSeconds(result.age),
publishedDate,
ageSeconds: dateToAgeSeconds(publishedDate),
});
}
@@ -155,6 +229,7 @@ export async function searchBrave(params: BraveSearchParams): Promise<SearchResp
provider: "brave",
sources: sources.slice(0, numResults),
requestId,
authMode: "api_key",
};
}
@@ -163,8 +238,8 @@ export class BraveProvider extends SearchProvider {
readonly id = "brave";
readonly label = "Brave";
isAvailable(_authStorage: AuthStorage): boolean {
return !!findApiKey();
isAvailable(authStorage: AuthStorage): boolean {
return authStorage.hasAuth("brave") || !!getEnvApiKey("brave");
}
search(params: SearchParams): Promise<SearchResponse> {
@@ -173,6 +248,8 @@ export class BraveProvider extends SearchProvider {
num_results: params.numSearchResults ?? params.limit,
recency: params.recency,
parsedQuery: params.parsedQuery,
authStorage: params.authStorage,
sessionId: params.sessionId,
signal: params.signal,
timeoutMs: params.timeoutMs,
fetch: params.fetch,
@@ -151,6 +151,13 @@ export interface CodexSearchParams {
}
/** Codex API response structure */
interface CodexWebSearchSource {
url?: string;
source_website_url?: string;
title?: string;
caption?: string;
}
interface CodexResponseItem {
type: string;
id?: string;
@@ -161,6 +168,9 @@ interface CodexResponseItem {
arguments?: string;
content?: CodexContentPart[];
summary?: Array<{ type: string; text: string }>;
action?: { sources?: CodexWebSearchSource[] };
sources?: CodexWebSearchSource[];
results?: CodexWebSearchSource[];
}
interface CodexContentPart {
@@ -221,12 +231,45 @@ function isImagePlaceholderAnswer(text: string): boolean {
return IMAGE_PLACEHOLDER_ANSWERS.has(normalized);
}
function addSource(sources: SearchSource[], source: SearchSource): void {
if (!sources.some(existing => existing.url === source.url)) {
sources.push(source);
function cleanSourceUrl(rawUrl: string): string {
try {
const url = new URL(rawUrl);
if (url.searchParams.get("utm_source") === "openai") {
url.searchParams.delete("utm_source");
}
return url.toString();
} catch {
return rawUrl.replace(/[?&]utm_source=openai$/u, "");
}
}
function addSource(sources: SearchSource[], source: SearchSource): void {
const normalizedSource = { ...source, url: cleanSourceUrl(source.url) };
const existing = sources.find(candidate => candidate.url === normalizedSource.url);
if (!existing) {
sources.push(normalizedSource);
return;
}
if (existing.title === existing.url && normalizedSource.title !== normalizedSource.url) {
existing.title = normalizedSource.title;
}
if (!existing.snippet && normalizedSource.snippet) {
existing.snippet = normalizedSource.snippet;
}
}
function extractCitationSnippet(text: string, start: number | undefined, end: number | undefined): string | undefined {
if (start === undefined || end === undefined || !text) return undefined;
const before = Math.max(0, start - 100);
const after = Math.min(text.length, end + 100);
const snippet = text
.slice(before, after)
.replace(/\[([^\]]*)\]\([^)]*\)/g, "$1")
.trim();
if (!snippet) return undefined;
return snippet.length > 300 ? `${snippet.slice(0, 297)}...` : snippet;
}
function countCharacter(text: string, target: string): number {
let count = 0;
for (const char of text) {
@@ -428,6 +471,15 @@ function extractCodexSseError(rawEvent: Record<string, unknown>): { code: string
return { code, message };
}
function classifyCodexSseErrorStatus(code: string, message: string): number {
const detail = `${code} ${message}`.toLowerCase();
if (/rate[- ]?limit|too many requests|quota|\b429\b/u.test(detail)) return 429;
if (/unauthori[sz]ed|\b401\b/u.test(detail)) return 401;
if (/forbidden|\b403\b/u.test(detail)) return 403;
if (/timeout|timed out/u.test(detail)) return 504;
return 500;
}
/**
* Calls the Codex Responses API with web search tool enabled.
* The caller provides the exact model id to send; retry / fallback policy
@@ -455,6 +507,8 @@ async function callCodexSearch(
model: requestedModel,
stream: true,
store: false,
include: ["web_search_call.action.sources"],
parallel_tool_calls: true,
input: [
{
type: "message",
@@ -510,7 +564,11 @@ async function callCodexSearch(
webSearchInvoked = true;
}
if (eventType === "response.output_text.delta") {
if (eventType === "response.created") {
const resp = (rawEvent as { response?: CodexResponse }).response;
if (resp?.id) requestId = resp.id;
if (resp?.model) model = resp.model;
} else if (eventType === "response.output_text.delta") {
const delta = typeof rawEvent.delta === "string" ? rawEvent.delta : "";
if (delta) {
streamedAnswerParts.push(delta);
@@ -518,7 +576,20 @@ async function callCodexSearch(
} else if (eventType === "response.output_item.done") {
const item = rawEvent.item as CodexResponseItem | undefined;
if (!item) continue;
if (item.type === "web_search_call") webSearchInvoked = true;
if (item.type === "web_search_call") {
webSearchInvoked = true;
const sourceGroups = [item.action?.sources, item.sources, item.results];
for (const group of sourceGroups) {
for (const source of group ?? []) {
const url = source.url ?? source.source_website_url;
if (!url) continue;
addSource(sources, {
title: source.title ?? source.caption ?? url,
url,
});
}
}
}
// Handle text message content and extract sources from annotations
if (item.type === "message" && item.content) {
@@ -530,8 +601,11 @@ async function callCodexSearch(
if (part.annotations) {
for (const annotation of part.annotations) {
if (annotation.type === "url_citation" && annotation.url) {
// Deduplicate by URL
addSource(sources, { title: annotation.title ?? annotation.url, url: annotation.url });
addSource(sources, {
title: annotation.title ?? annotation.url,
url: annotation.url,
snippet: extractCitationSnippet(part.text, annotation.start_index, annotation.end_index),
});
}
}
}
@@ -563,13 +637,17 @@ async function callCodexSearch(
}
} else if (eventType === "error") {
const { code, message } = extractCodexSseError(rawEvent);
throw new SearchProviderError("codex", `Codex error (${code}): ${message || "Unknown error"}`, 500);
throw new SearchProviderError(
"codex",
`Codex error (${code}): ${message || "Unknown error"}`,
classifyCodexSseErrorStatus(code, message),
);
} else if (eventType === "response.failed") {
const { code, message } = extractCodexSseError(rawEvent);
const detail = code
? `Codex request failed (${code}): ${message || "Request failed"}`
: `Codex request failed: ${message || "Request failed"}`;
throw new SearchProviderError("codex", detail, 500);
throw new SearchProviderError("codex", detail, classifyCodexSseErrorStatus(code, message));
}
}
@@ -19,6 +19,9 @@ import { SearchProvider } from "./base";
import { classifyProviderHttpError, withHardTimeout } from "./utils";
const EXA_API_URL = "https://api.exa.ai/search";
const EXA_MCP_URL = "https://mcp.exa.ai/mcp";
const EXA_MCP_SOURCE = "oh-my-pi";
const MAX_EXA_SNIPPET_CHARS = 500;
const DEFAULT_EXA_SEARCH_DELAY_MS = getDefault("exa.searchDelayMs");
let nextExaSearchRequestAt = 0;
@@ -329,13 +332,20 @@ async function callExaSearch(apiKey: string, params: ExaSearchParams): Promise<E
return response.json() as Promise<ExaSearchResponse>;
}
function buildExaMcpArgs(params: ExaSearchParams): Record<string, unknown> {
const args: Record<string, unknown> = { query: params.query };
if (params.num_results !== undefined) args.num_results = params.num_results;
if (params.type !== undefined) args.type = params.type;
if (params.include_domains !== undefined) args.include_domains = params.include_domains;
if (params.exclude_domains !== undefined) args.exclude_domains = params.exclude_domains;
if (params.start_published_date !== undefined) args.start_published_date = params.start_published_date;
if (params.end_published_date !== undefined) args.end_published_date = params.end_published_date;
const queryParts = [params.query];
for (const domain of params.include_domains ?? []) {
const trimmed = domain.trim();
if (trimmed) queryParts.push(`site:${trimmed}`);
}
for (const domain of params.exclude_domains ?? []) {
const trimmed = domain.trim();
if (trimmed) queryParts.push(`-site:${trimmed}`);
}
if (params.start_published_date) queryParts.push(`after:${params.start_published_date}`);
if (params.end_published_date) queryParts.push(`before:${params.end_published_date}`);
const args: Record<string, unknown> = { query: queryParts.join(" ") };
if (params.num_results !== undefined) args.numResults = params.num_results;
return args;
}
@@ -346,11 +356,12 @@ async function callExaMcpSearch(params: ExaSearchParams): Promise<ExaSearchRespo
query.set("tools", "web_search_exa");
const fetchImpl = params.fetch ?? fetch;
await waitForExaSearchSlot(params.signal);
const response = await fetchImpl(`https://mcp.exa.ai/mcp?${query.toString()}`, {
const response = await fetchImpl(`${EXA_MCP_URL}?${query.toString()}`, {
method: "POST",
headers: {
"Content-Type": "application/json",
Accept: "application/json, text/event-stream",
"x-exa-source": EXA_MCP_SOURCE,
},
body: JSON.stringify({
jsonrpc: "2.0",
@@ -364,11 +375,26 @@ async function callExaMcpSearch(params: ExaSearchParams): Promise<ExaSearchRespo
signal: withHardTimeout(params.signal, params.timeoutMs),
});
if (!response.ok) {
throw new Error(`MCP request failed: ${response.status} ${response.statusText}`);
const errorText = await response.text();
const classified = classifyProviderHttpError("exa", response.status, errorText);
if (classified) throw classified;
if (response.status === 429) {
throw new SearchProviderError(
"exa",
"exa: MCP rate limit reached (429); configure an Exa API key for higher limits",
response.status,
);
}
throw new SearchProviderError(
"exa",
`Exa MCP request failed (${response.status}): ${errorText}`,
response.status,
);
}
const mcpResponse = parseSSE(await response.text()) as {
result?: {
content?: Array<{ type: string; text?: string }>;
isError?: boolean;
};
error?: {
code: number;
@@ -381,6 +407,12 @@ async function callExaMcpSearch(params: ExaSearchParams): Promise<ExaSearchRespo
if (mcpResponse.error) {
throw new Error(`MCP error: ${mcpResponse.error.message}`);
}
if (mcpResponse.result?.isError) {
const message = mcpResponse.result.content
?.find(item => item.type === "text" && typeof item.text === "string")
?.text?.trim();
throw new SearchProviderError("exa", message || "Exa MCP returned an error");
}
const responsePayload = normalizeExaMcpPayload(mcpResponse.result);
if (isSearchResponse(responsePayload)) {
return responsePayload as ExaSearchResponse;
@@ -419,7 +451,10 @@ export async function searchExa(params: ExaSearchParams): Promise<SearchResponse
sources.push({
title: result.title ?? result.url,
url: result.url,
snippet: result.summary || result.text || result.highlights?.join(" ") || undefined,
snippet: (result.summary || result.text || result.highlights?.join(" ") || undefined)?.slice(
0,
MAX_EXA_SNIPPET_CHARS,
),
publishedDate: result.publishedDate ?? undefined,
ageSeconds: dateToAgeSeconds(result.publishedDate ?? undefined),
author: result.author ?? undefined,
@@ -20,7 +20,7 @@ import type { SearchParams } from "./base";
import { SearchProvider } from "./base";
import { classifyProviderHttpError, withHardTimeout } from "./utils";
const FIRECRAWL_SEARCH_URL = "https://api.firecrawl.dev/v2/search";
const FIRECRAWL_DEFAULT_BASE_URL = "https://api.firecrawl.dev/v2";
const DEFAULT_NUM_RESULTS = 10;
const MAX_NUM_RESULTS = 100;
@@ -30,6 +30,28 @@ const RECENCY_TBS: Record<NonNullable<SearchParams["recency"]>, string> = {
month: "qdr:m",
year: "qdr:y",
};
function resolveSearchUrl(): string {
const configured = process.env.FIRECRAWL_BASE_URL ?? process.env.FIRECRAWL_API_URL;
if (!configured?.trim()) return `${FIRECRAWL_DEFAULT_BASE_URL}/search`;
let url: URL;
try {
url = new URL(configured.trim());
} catch {
throw new Error("Invalid Firecrawl base URL: expected an HTTP or HTTPS URL");
}
if (url.protocol !== "http:" && url.protocol !== "https:") {
throw new Error("Invalid Firecrawl base URL: expected an HTTP or HTTPS URL");
}
if (url.username || url.password) {
throw new Error("Invalid Firecrawl base URL: URL credentials are not allowed");
}
url.search = "";
url.hash = "";
url.pathname = url.pathname.replace(/\/+$/, "");
if (!/\/v[12]$/i.test(url.pathname)) url.pathname += "/v2";
url.pathname += "/search";
return url.toString();
}
export interface FirecrawlSearchParams {
query: string;
@@ -46,14 +68,23 @@ interface FirecrawlWebResult {
title?: string | null;
url?: string | null;
description?: string | null;
snippet?: string | null;
markdown?: string | null;
}
interface FirecrawlSearchResponse {
success?: boolean;
error?: string | null;
id?: string | null;
data?: {
web?: FirecrawlWebResult[] | null;
} | null;
data?:
| FirecrawlWebResult[]
| {
web?: FirecrawlWebResult[] | null;
news?: FirecrawlWebResult[] | null;
images?: FirecrawlWebResult[] | null;
}
| null;
results?: FirecrawlWebResult[] | null;
}
/** Resolve Firecrawl API key through the shared auth storage pipeline. */
@@ -88,7 +119,7 @@ async function callFirecrawlSearch(
if (apiKey) {
headers.Authorization = `Bearer ${apiKey}`;
}
const response = await (params.fetch ?? fetch)(FIRECRAWL_SEARCH_URL, {
const response = await (params.fetch ?? fetch)(resolveSearchUrl(), {
method: "POST",
headers,
body: JSON.stringify(buildRequestBody(params)),
@@ -106,7 +137,11 @@ async function callFirecrawlSearch(
);
}
return (await response.json()) as FirecrawlSearchResponse;
const data = (await response.json()) as FirecrawlSearchResponse;
if (data.success === false) {
throw new SearchProviderError("firecrawl", data.error?.trim() || "Firecrawl request failed");
}
return data;
}
/** ISO `YYYY-MM-DD` to Google `MM/DD/YYYY` for `tbs=cdr` custom date ranges. */
@@ -128,6 +163,11 @@ function buildDateTbs(parsed: StructuredQuery): string | undefined {
return parts.join(",");
}
function getWebResults(data: FirecrawlSearchResponse): FirecrawlWebResult[] {
if (Array.isArray(data.data)) return data.data;
if (data.data && Array.isArray(data.data.web)) return data.data.web;
return data.results ?? [];
}
/** Execute Firecrawl web search. */
export async function searchFirecrawl(params: SearchParams): Promise<SearchResponse> {
const parsed = params.parsedQuery ?? parseSearchQuery(params.query);
@@ -169,12 +209,12 @@ export async function searchFirecrawl(params: SearchParams): Promise<SearchRespo
const sources: SearchSource[] = [];
for (const result of data.data?.web ?? []) {
for (const result of getWebResults(data)) {
if (!result.url) continue;
sources.push({
title: result.title ?? result.url,
url: result.url,
snippet: result.description ?? result.markdown ?? undefined,
snippet: result.description ?? result.snippet ?? result.markdown ?? undefined,
});
}
@@ -192,11 +232,13 @@ export class FirecrawlProvider extends SearchProvider {
readonly label = "Firecrawl";
/**
* Auto-chain admission: requires a credential so an unconfigured Firecrawl
* doesn't displace other providers that the user has set up with API keys.
* Auto-chain admission requires either a credential or an explicitly
* configured self-hosted endpoint. Hosted keyless mode remains explicit-only
* so it does not displace providers the user configured.
*/
isAvailable(authStorage: AuthStorage): boolean {
return authStorage.hasAuth("firecrawl") || !!getEnvApiKey("firecrawl");
const configuredBaseUrl = process.env.FIRECRAWL_BASE_URL ?? process.env.FIRECRAWL_API_URL;
return !!configuredBaseUrl?.trim() || authStorage.hasAuth("firecrawl") || !!getEnvApiKey("firecrawl");
}
/**
@@ -25,7 +25,9 @@ import { classifyProviderHttpError, withHardTimeout } from "./utils";
const DEFAULT_ENDPOINT = "https://cloudcode-pa.googleapis.com";
const DEVELOPER_API_PROVIDER = "google";
const DEVELOPER_API_ENDPOINT = "https://generativelanguage.googleapis.com/v1beta";
const CLOUDFLARE_GATEWAY_PROVIDER = "cloudflare-ai-gateway";
const DEFAULT_DEVELOPER_API_HOST = "https://generativelanguage.googleapis.com";
const DEVELOPER_API_VERSION = "v1beta";
const ANTIGRAVITY_DAILY_ENDPOINT = "https://daily-cloudcode-pa.googleapis.com";
const ANTIGRAVITY_SANDBOX_ENDPOINT = "https://daily-cloudcode-pa.sandbox.googleapis.com";
const ANTIGRAVITY_ENDPOINT_FALLBACKS = [ANTIGRAVITY_DAILY_ENDPOINT, ANTIGRAVITY_SANDBOX_ENDPOINT] as const;
@@ -41,6 +43,32 @@ function resolveGeminiSearchModel(configuredModel: string | undefined): string {
return model || DEFAULT_MODEL;
}
interface GeminiDeveloperEndpoint {
url: string;
authProvider: typeof DEVELOPER_API_PROVIDER | typeof CLOUDFLARE_GATEWAY_PROVIDER;
isCloudflareGateway: boolean;
}
function resolveGeminiDeveloperEndpoint(): GeminiDeveloperEndpoint {
const configuredHost = Bun.env.GOOGLE_GEMINI_BASE_URL?.trim().replace(/\/+$/, "");
const host = configuredHost || DEFAULT_DEVELOPER_API_HOST;
let parsed: URL;
try {
parsed = new URL(host);
} catch {
throw new SearchProviderError("gemini", "GOOGLE_GEMINI_BASE_URL must be a valid absolute URL", 400);
}
if (parsed.protocol !== "https:" && parsed.protocol !== "http:") {
throw new SearchProviderError("gemini", "GOOGLE_GEMINI_BASE_URL must use HTTP or HTTPS", 400);
}
const isCloudflareGateway = parsed.hostname === "gateway.ai.cloudflare.com";
return {
url: `${host}/${DEVELOPER_API_VERSION}`,
authProvider: isCloudflareGateway ? CLOUDFLARE_GATEWAY_PROVIDER : DEVELOPER_API_PROVIDER,
isCloudflareGateway,
};
}
const GEMINI_PROVIDERS = ["google-gemini-cli", "google-antigravity"] as const;
type GeminiProviderId = (typeof GEMINI_PROVIDERS)[number];
@@ -292,6 +320,81 @@ async function parseGeminiSearchStream(
};
}
function isGroundingRedirectUrl(url: string): boolean {
try {
const parsed = new URL(url);
return (
parsed.hostname === "vertexaisearch.cloud.google.com" &&
parsed.pathname.includes("/grounding-api-redirect")
);
} catch {
return false;
}
}
async function resolveGroundingRedirect(
proxyUrl: string,
fetchImpl: FetchImpl | undefined,
signal: AbortSignal | undefined,
): Promise<string> {
try {
const response = await (fetchImpl ?? fetch)(proxyUrl, {
method: "HEAD",
redirect: "manual",
signal: withHardTimeout(signal, 5000),
});
const location = response.headers.get("location");
if (!location) return proxyUrl;
const resolved = new URL(location, proxyUrl);
return resolved.protocol === "http:" || resolved.protocol === "https:" ? resolved.toString() : proxyUrl;
} catch {
return proxyUrl;
}
}
async function finalizeGeminiSearchResult(
result: GeminiSearchResult,
fetchImpl: FetchImpl | undefined,
signal: AbortSignal | undefined,
): Promise<GeminiSearchResult> {
if (!result.answer && result.sources.length === 0) {
throw new SearchProviderError("gemini", "Gemini API returned an empty grounded response", 502);
}
const redirectUrls = new Set<string>();
for (const source of result.sources) {
if (isGroundingRedirectUrl(source.url)) redirectUrls.add(source.url);
}
for (const citation of result.citations) {
if (isGroundingRedirectUrl(citation.url)) redirectUrls.add(citation.url);
}
if (redirectUrls.size === 0) return result;
signal?.throwIfAborted();
const resolvedEntries = await Promise.all(
[...redirectUrls].map(async url => [url, await resolveGroundingRedirect(url, fetchImpl, signal)] as const),
);
signal?.throwIfAborted();
const resolvedUrls = new Map(resolvedEntries);
for (const source of result.sources) {
source.url = resolvedUrls.get(source.url) ?? source.url;
}
for (const citation of result.citations) {
citation.url = resolvedUrls.get(citation.url) ?? citation.url;
}
const seenUrls = new Set<string>();
let writeIndex = 0;
for (const source of result.sources) {
if (seenUrls.has(source.url)) continue;
seenUrls.add(source.url);
result.sources[writeIndex++] = source;
}
result.sources.length = writeIndex;
return result;
}
/**
* Calls the Cloud Code Assist API with Google Search grounding enabled.
*
@@ -420,7 +523,10 @@ async function callGeminiSearch(
}
if (!response?.ok) {
const errorText = response ? await response.text() : "Network error";
const rawErrorText = response ? await response.text() : "Network error";
const errorText = auth.accessToken
? rawErrorText.split(auth.accessToken).join("[redacted]")
: rawErrorText;
const status = response?.status ?? 502;
const classified = classifyProviderHttpError("gemini", status, errorText);
if (classified) throw classified;
@@ -431,11 +537,12 @@ async function callGeminiSearch(
throw new SearchProviderError("gemini", "Gemini API returned no response body", 500);
}
return parseGeminiSearchStream(response.body, model);
return finalizeGeminiSearchResult(await parseGeminiSearchStream(response.body, model), fetchImpl, signal);
}
async function callGeminiDeveloperSearch(
apiKey: string,
endpoint: GeminiDeveloperEndpoint,
model: string,
query: string,
systemPrompt: string | undefined,
@@ -474,11 +581,13 @@ async function callGeminiDeveloperSearch(
}
const response = await fetchWithRetry(
() => `${DEVELOPER_API_ENDPOINT}/models/${model}:streamGenerateContent?alt=sse`,
() => `${endpoint.url}/models/${model}:streamGenerateContent?alt=sse`,
{
method: "POST",
headers: {
"x-goog-api-key": apiKey,
...(endpoint.isCloudflareGateway
? { "cf-aig-authorization": `Bearer ${apiKey}` }
: { "x-goog-api-key": apiKey }),
"Content-Type": "application/json",
Accept: "text/event-stream",
},
@@ -492,7 +601,8 @@ async function callGeminiDeveloperSearch(
);
if (!response.ok) {
const errorText = await response.text();
const rawErrorText = await response.text();
const errorText = apiKey ? rawErrorText.split(apiKey).join("[redacted]") : rawErrorText;
const classified = classifyProviderHttpError("gemini", response.status, errorText);
if (classified) throw classified;
throw new SearchProviderError(
@@ -506,7 +616,7 @@ async function callGeminiDeveloperSearch(
throw new SearchProviderError("gemini", "Gemini API returned no response body", 500);
}
return parseGeminiSearchStream(response.body, model);
return finalizeGeminiSearchResult(await parseGeminiSearchStream(response.body, model), fetchImpl, signal);
}
/**
@@ -557,16 +667,20 @@ export async function searchGemini(params: GeminiSearchParams): Promise<SearchRe
{ sessionId: params.sessionId, signal: params.signal, seed: seed.access },
);
} else {
const apiKey = await params.authStorage.getApiKey(DEVELOPER_API_PROVIDER, params.sessionId, {
const endpoint = resolveGeminiDeveloperEndpoint();
const apiKey = await params.authStorage.getApiKey(endpoint.authProvider, params.sessionId, {
signal: params.signal,
});
if (!apiKey) {
throw new Error(
"No Gemini credentials found. Set GEMINI_API_KEY, configure an API key for provider \"google\", or login with 'omp /login google-gemini-cli' / 'omp /login google-antigravity' to enable Gemini web search.",
endpoint.isCloudflareGateway
? "No Cloudflare AI Gateway credential found. Configure provider \"cloudflare-ai-gateway\" or set CLOUDFLARE_AI_GATEWAY_API_KEY."
: "No Gemini credentials found. Set GEMINI_API_KEY, configure an API key for provider \"google\", or login with 'omp /login google-gemini-cli' / 'omp /login google-antigravity' to enable Gemini web search.",
);
}
result = await callGeminiDeveloperSearch(
apiKey,
endpoint,
selectedModel,
searchQuery,
params.system_prompt,
@@ -609,7 +723,12 @@ export class GeminiProvider extends SearchProvider {
// Cheap, in-memory check — avoids driving the refresh pipeline during
// the provider-chain probe. `searchGemini` refreshes OAuth lazily on the
// actual request and resolves developer API keys through AuthStorage.
return hasGeminiOAuth(authStorage) || authStorage.hasAuth(DEVELOPER_API_PROVIDER);
if (hasGeminiOAuth(authStorage)) return true;
try {
return authStorage.hasAuth(resolveGeminiDeveloperEndpoint().authProvider);
} catch {
return false;
}
}
search(params: SearchParams): Promise<SearchResponse> {
@@ -5,19 +5,24 @@
* cleaned content.
*/
import { type AuthStorage, type FetchImpl, getEnvApiKey } from "@oh-my-pi/pi-ai";
import { type ApiKey, type AuthStorage, type FetchImpl, withAuth } from "@oh-my-pi/pi-ai";
import type { SearchResponse, SearchSource } from "../../../web/search/types";
import { SearchProviderError } from "../../../web/search/types";
import { formatQuery, parseSearchQuery } from "../query";
import { clampNumResults } from "../utils";
import type { SearchParams } from "./base";
import { SearchProvider } from "./base";
import { classifyProviderHttpError, withHardTimeout } from "./utils";
const JINA_SEARCH_URL = "https://s.jina.ai";
const DEFAULT_NUM_RESULTS = 5;
const MAX_NUM_RESULTS = 20;
type SearchParamsWithFetch = SearchParams & { fetch?: FetchImpl };
export interface JinaSearchParams {
query: string;
authStorage: AuthStorage;
sessionId?: string;
num_results?: number;
/** Single bare host for Jina's `X-Site` in-site search header. */
site?: string;
@@ -29,31 +34,37 @@ export interface JinaSearchParams {
interface JinaSearchResult {
title?: string | null;
url?: string | null;
description?: string | null;
content?: string | null;
}
type JinaSearchResponse = JinaSearchResult[];
/** Find JINA_API_KEY from environment or .env files. */
export function findApiKey(): string | null {
return getEnvApiKey("jina") ?? null;
interface JinaSearchEnvelope {
code?: unknown;
data?: unknown;
}
type JinaSearchResponse = JinaSearchResult[];
/** Call Jina Reader search API. */
async function callJinaSearch(
apiKey: string,
query: string,
numResults: number,
site?: string,
signal?: AbortSignal,
fetchImpl: FetchImpl = fetch,
timeoutMs?: number,
): Promise<JinaSearchResponse> {
const requestUrl = `${JINA_SEARCH_URL}/${encodeURIComponent(query)}`;
const requestUrl = new URL(`${JINA_SEARCH_URL}/${encodeURIComponent(query)}`);
requestUrl.searchParams.set("count", String(numResults));
const headers: Record<string, string> = {
Accept: "application/json",
Authorization: `Bearer ${apiKey}`,
};
if (site) headers["X-Site"] = site;
headers["X-Respond-With"] = "no-content";
headers["X-Retain-Images"] = "none";
const response = await fetchImpl(requestUrl, {
headers,
signal: withHardTimeout(signal, timeoutMs),
@@ -66,24 +77,43 @@ async function callJinaSearch(
throw new SearchProviderError("jina", `Jina API error (${response.status}): ${errorText}`, response.status);
}
const payload = (await response.json()) as { data?: JinaSearchResponse } | null;
return Array.isArray(payload?.data) ? payload.data : [];
const payload = (await response.json()) as JinaSearchEnvelope | JinaSearchResponse | null;
if (Array.isArray(payload)) return payload;
if (!payload || typeof payload !== "object") {
throw new SearchProviderError("jina", "Jina API returned invalid response: expected an object or array");
}
if (typeof payload.code === "number" && payload.code !== 200) {
throw new SearchProviderError("jina", `Jina API response reported failure (${payload.code})`, payload.code);
}
if (!Array.isArray(payload.data)) {
throw new SearchProviderError("jina", "Jina API returned invalid response: expected data array");
}
return payload.data as JinaSearchResponse;
}
/** Execute Jina web search. */
export async function searchJina(params: JinaSearchParams): Promise<SearchResponse> {
const apiKey = findApiKey();
if (!apiKey) {
throw new Error("JINA_API_KEY not found. Set it in environment or .env file.");
}
const response = await callJinaSearch(
apiKey,
params.query,
params.site,
params.signal,
params.fetch,
params.timeoutMs,
const numResults = clampNumResults(params.num_results, DEFAULT_NUM_RESULTS, MAX_NUM_RESULTS);
const keyOrResolver: ApiKey = params.authStorage.resolver("jina", {
sessionId: params.sessionId,
});
const response = await withAuth(
keyOrResolver,
apiKey =>
callJinaSearch(
apiKey,
params.query,
numResults,
params.site,
params.signal,
params.fetch,
params.timeoutMs,
),
{
signal: params.signal,
missingKeyMessage:
'Jina credentials not found. Set JINA_API_KEY or configure an API key for provider "jina".',
},
);
const sources: SearchSource[] = [];
@@ -92,11 +122,11 @@ export async function searchJina(params: JinaSearchParams): Promise<SearchRespon
sources.push({
title: result.title ?? result.url,
url: result.url,
snippet: result.content ?? undefined,
snippet: result.description?.trim() || result.content?.trim() || undefined,
});
}
const limitedSources = params.num_results ? sources.slice(0, params.num_results) : sources;
const limitedSources = sources.slice(0, numResults);
return {
provider: "jina",
@@ -109,8 +139,8 @@ export class JinaProvider extends SearchProvider {
readonly id = "jina";
readonly label = "Jina";
isAvailable(_authStorage: AuthStorage): boolean {
return !!findApiKey();
isAvailable(authStorage: AuthStorage): boolean {
return authStorage.hasAuth("jina");
}
search(params: SearchParamsWithFetch): Promise<SearchResponse> {
@@ -134,6 +164,8 @@ export class JinaProvider extends SearchProvider {
return searchJina({
query,
authStorage: params.authStorage,
sessionId: params.sessionId,
num_results: params.numSearchResults ?? params.limit,
site,
signal: params.signal,
@@ -7,6 +7,7 @@ import {
ParallelApiError,
type ParallelSearchResult,
parseParallelErrorResponse,
parseParallelJsonResponse,
parseParallelSearchPayload,
} from "../../parallel";
import { formatQuery, parseSearchQuery, type StructuredQuery } from "../query";
@@ -28,6 +29,13 @@ interface ParallelSourcePolicy {
after_date?: string;
}
const RECENCY_DAYS: Record<NonNullable<SearchParams["recency"]>, number> = {
day: 1,
week: 7,
month: 30,
year: 365,
};
/** Site values may carry paths (`github.com/anthropics`); Parallel takes bare hosts. */
function toHosts(sites: readonly string[]): string[] {
const hosts = new Set<string>();
@@ -39,19 +47,23 @@ function toHosts(sites: readonly string[]): string[] {
}
/**
* Map parsed `site:`/`-site:`/`after:` directives onto Parallel's
* `source_policy`. Per Parallel docs, `exclude_domains` is ignored when
* `include_domains` is set, so exclusions are only sent without an allow
* list (the central lenient filter enforces them regardless).
* Map parsed `site:`/`-site:`/`after:` directives and the relative recency
* option onto Parallel's `source_policy`. An explicit `after:` bound wins.
* Per Parallel docs, `exclude_domains` is ignored when `include_domains` is
* set, so exclusions are only sent without an allow list (the central lenient
* filter enforces them regardless).
*/
function toSourcePolicy(parsed: StructuredQuery): ParallelSourcePolicy | undefined {
function toSourcePolicy(parsed: StructuredQuery, recency?: SearchParams["recency"]): ParallelSourcePolicy | undefined {
const policy: ParallelSourcePolicy = {};
const include = toHosts(parsed.sites);
const exclude = toHosts(parsed.excludedSites);
if (include.length) policy.include_domains = include;
else if (exclude.length) policy.exclude_domains = exclude;
if (parsed.after) policy.after_date = parsed.after;
return Object.keys(policy).length ? policy : undefined;
else if (recency) {
policy.after_date = new Date(Date.now() - RECENCY_DAYS[recency] * 86_400_000).toISOString().slice(0, 10);
}
return policy.include_domains || policy.exclude_domains || policy.after_date ? policy : undefined;
}
async function searchWithAuthStorage(
@@ -105,7 +117,7 @@ async function searchWithAuthStorage(
throw parseParallelErrorResponse(response.status, await response.text());
}
const payload: unknown = await response.json();
const payload = await parseParallelJsonResponse(response, "search");
return parseParallelSearchPayload(payload, { parseMetadata: false });
},
{ signal: params.signal },
@@ -116,6 +128,7 @@ export async function searchParallel(
params: {
query: string;
num_results?: number;
recency?: SearchParams["recency"];
signal?: AbortSignal;
timeoutMs?: number;
fetch?: FetchImpl;
@@ -126,9 +139,9 @@ export async function searchParallel(
): Promise<SearchResponse> {
const numResults = clampNumResults(params.num_results, DEFAULT_NUM_RESULTS, MAX_NUM_RESULTS);
const parsed = params.parsedQuery ?? parseSearchQuery(params.query);
// Back-compat: without directives the upstream request is byte-identical.
// Directives are removed only where Parallel has a native equivalent.
const query = parsed.hasDirectives ? formatQuery(parsed, PARALLEL_QUERY_SYNTAX) : params.query;
const sourcePolicy = parsed.hasDirectives ? toSourcePolicy(parsed) : undefined;
const sourcePolicy = toSourcePolicy(parsed, params.recency);
try {
const result = await searchWithAuthStorage(
@@ -174,6 +187,7 @@ export class ParallelProvider extends SearchProvider {
{
query: params.query,
num_results: params.numSearchResults ?? params.limit,
recency: params.recency,
signal: params.signal,
timeoutMs: params.timeoutMs,
fetch: params.fetch,
@@ -150,6 +150,7 @@ interface PerplexityOAuthStreamEvent {
error_code?: string;
error_message?: string;
display_model?: string;
user_selected_model?: string;
uuid?: string;
}
@@ -327,7 +328,11 @@ export interface PerplexitySearchParams {
system_prompt?: string;
/** Pre-parsed view of `query` from the search pipeline; parsed locally when absent. */
parsedQuery?: StructuredQuery;
/** Direct API model. Defaults to `PI_PERPLEXITY_API_MODEL`, then `sonar-pro`. */
api_model?: string;
search_recency_filter?: "hour" | "day" | "week" | "month" | "year";
/** Consumer subscription model preference. Defaults to `PI_PERPLEXITY_MODEL`, then Sonar (`experimental`). */
subscription_model?: string;
num_results?: number;
/** Maximum output tokens. Defaults to 8192. */
max_tokens?: number;
@@ -539,6 +544,15 @@ async function callPerplexityApi(
return parseStreamedApiResponse(message, metadata);
}
function oauthSourceKey(url: string): string {
const trimmed = url.trim().replace(/\/$/, "");
try {
return new URL(trimmed).href.replace(/\/$/, "");
} catch {
return trimmed.toLowerCase();
}
}
function buildOAuthSources(event: PerplexityOAuthStreamEvent): SearchSource[] {
const results =
event.blocks?.find(block => block.intended_usage === "web_results")?.web_result_block?.web_results ?? [];
@@ -607,6 +621,7 @@ async function callPerplexityAsk(
params: PerplexitySearchParams,
filters: PerplexityNativeFilters,
): Promise<{ answer: string; sources: SearchSource[]; model?: string; requestId?: string }> {
const subscriptionModel = params.subscription_model?.trim() || $env.PI_PERPLEXITY_MODEL?.trim() || "experimental";
const requestId = crypto.randomUUID();
// The consumer `perplexity_ask` endpoint is itself a research assistant and
// has no system-message slot. Prepending the API-style system prompt to the
@@ -645,14 +660,14 @@ async function callPerplexityAsk(
query_str: effectiveQuery,
search_focus: "internet",
mode: "copilot",
model_preference: "experimental",
model_preference: subscriptionModel,
sources: ["web"],
attachments: [],
frontend_uuid: crypto.randomUUID(),
frontend_context_uuid: crypto.randomUUID(),
version: OAUTH_API_VERSION,
language: "en-US",
timezone: Intl.DateTimeFormat().resolvedOptions().timeZone,
timezone: Intl.DateTimeFormat().resolvedOptions().timeZone ?? "UTC",
// Recency cannot be combined with absolute date filters; explicit
// before:/after: bounds take precedence.
search_recency_filter: filters.afterDate || filters.beforeDate ? null : (params.search_recency_filter ?? null),
@@ -740,12 +755,14 @@ async function callPerplexityAsk(
if (eventAnswer.length > 0) {
answer = eventAnswer;
}
for (const source of buildOAuthSources(mergedEvent)) {
sourcesByUrl.set(source.url, source);
sourcesByUrl.set(oauthSourceKey(source.url), source);
}
if (mergedEvent.display_model) model = mergedEvent.display_model;
const reportedModel = [mergedEvent.user_selected_model, mergedEvent.display_model].find(
candidate => candidate && candidate !== "turbo",
);
if (reportedModel) model = reportedModel;
if (mergedEvent.uuid) finalRequestId = mergedEvent.uuid;
if (mergedEvent.final || mergedEvent.status === "COMPLETED") {
break;
@@ -755,7 +772,7 @@ async function callPerplexityAsk(
return {
answer,
sources: [...sourcesByUrl.values()],
model,
model: model ?? (auth.type === "anonymous" ? mergedEvent.display_model : subscriptionModel),
requestId: finalRequestId ?? requestId,
};
}
@@ -872,7 +889,7 @@ export async function searchPerplexity(params: PerplexitySearchParams): Promise<
messages.push({ role: "user", content: filters.query });
const request: PerplexityRequest = {
model: "sonar-pro",
model: params.api_model?.trim() || $env.PI_PERPLEXITY_API_MODEL?.trim() || "sonar-pro",
messages,
max_tokens: params.max_tokens ?? DEFAULT_MAX_TOKENS,
temperature: params.temperature ?? DEFAULT_TEMPERATURE,
@@ -62,6 +62,7 @@ interface SearXNGResult {
title?: string;
url?: string;
content?: string;
snippet?: string;
engine?: string;
publishedDate?: string;
/** SearXNG sometimes uses publishedDate, sometimes just date */
@@ -76,6 +77,7 @@ interface SearXNGResponse {
suggestions?: string[];
corrections?: string[];
unresponsive_engines?: Array<[string, string]>;
answers?: unknown[];
}
interface SearXNGAuth {
@@ -272,6 +274,61 @@ function stripExternalBangs(query: string): string {
.join(" ");
}
/** Extract displayable text from both legacy string answers and modern
* structured answer plugins (legacy, translations, weather). */
function extractAnswerText(answer: unknown): string | undefined {
if (typeof answer === "string") return answer.trim() || undefined;
if (!answer || typeof answer !== "object") return undefined;
const record = answer as Record<string, unknown>;
if (typeof record.answer === "string") return record.answer.trim() || undefined;
if (Array.isArray(record.translations)) {
const translations: string[] = [];
for (const item of record.translations) {
if (!item || typeof item !== "object") continue;
const text = (item as Record<string, unknown>).text;
if (typeof text === "string" && text.trim()) translations.push(text.trim());
if (translations.length === 3) break;
}
if (translations.length) return translations.join("\n");
}
if (record.current && typeof record.current === "object") {
const current = record.current as Record<string, unknown>;
if (typeof current.summary === "string" && current.summary.trim()) return current.summary.trim();
const location =
current.location && typeof current.location === "object"
? (current.location as Record<string, unknown>).name
: undefined;
const temperature =
current.temperature && typeof current.temperature === "object"
? (current.temperature as Record<string, unknown>)
: undefined;
const temperatureText =
temperature && (typeof temperature.val === "string" || typeof temperature.val === "number")
? `${temperature.val}${typeof temperature.unit === "string" ? temperature.unit : ""}`
: undefined;
const condition = typeof current.condition === "string" ? current.condition : undefined;
const parts = [location, temperatureText, condition].filter(
(part): part is string => typeof part === "string" && part.trim().length > 0,
);
if (parts.length) return parts.join(": ");
}
return undefined;
}
function formatAnswers(answers: unknown[] | undefined): string | undefined {
const texts: string[] = [];
for (const answer of answers ?? []) {
const text = extractAnswerText(answer);
if (text) texts.push(text);
if (texts.length === 3) break;
}
return texts.length ? texts.join("\n\n") : undefined;
}
/** Build the search URL and headers for a SearXNG request */
function buildRequest(
endpoint: string,
@@ -282,6 +339,7 @@ function buildRequest(
categories?: string;
engines?: string;
language?: string;
safesearch?: 0 | 1 | 2;
signal?: AbortSignal;
},
auth: SearXNGAuth | null,
@@ -308,6 +366,10 @@ function buildRequest(
url.searchParams.set("engines", params.engines);
}
if (params.safesearch !== undefined) {
url.searchParams.set("safesearch", String(params.safesearch));
}
if (params.language) {
url.searchParams.set("language", params.language);
}
@@ -326,6 +388,7 @@ async function callSearXNGSearch(
categories?: string;
engines?: string;
language?: string;
safesearch?: 0 | 1 | 2;
signal?: AbortSignal;
timeoutMs?: number;
fetch?: FetchImpl;
@@ -372,12 +435,23 @@ export async function searchSearXNG(params: {
let categories: string | undefined;
let language: string | undefined;
let configuredSafesearch: number | undefined;
try {
categories = settings.get("searxng.categories") ?? undefined;
language = settings.get("searxng.language") ?? undefined;
configuredSafesearch = settings.get("searxng.safesearch");
} catch {
// Settings not initialized yet
}
if (
configuredSafesearch !== undefined &&
configuredSafesearch !== 0 &&
configuredSafesearch !== 1 &&
configuredSafesearch !== 2
) {
throw new Error("searxng.safesearch must be 0 (off), 1 (moderate), or 2 (strict).");
}
const safesearch = configuredSafesearch;
const configuredEngines = findEngines();
// SearXNG forwards `q` to downstream engines, so build it with the shared
@@ -402,6 +476,7 @@ export async function searchSearXNG(params: {
categories,
engines,
language,
safesearch,
fetch: params.fetch,
},
auth,
@@ -415,7 +490,7 @@ export async function searchSearXNG(params: {
sources.push({
title: result.title ?? result.url,
url: result.url,
snippet: result.content?.trim() || undefined,
snippet: (result.content ?? result.snippet)?.trim() || undefined,
publishedDate: publishedDate ?? undefined,
ageSeconds: dateToAgeSeconds(publishedDate),
});
@@ -435,6 +510,7 @@ export async function searchSearXNG(params: {
return {
provider: "searxng",
answer: formatAnswers(response.answers),
sources: limitedSources,
relatedQuestions: response.suggestions?.length ? response.suggestions : undefined,
};
@@ -34,17 +34,10 @@ export interface TavilySearchParams {
fetch?: FetchImpl;
}
interface TavilySearchResult {
title?: string | null;
url?: string | null;
content?: string | null;
published_date?: string | null;
}
interface TavilySearchResponse {
answer?: string | null;
results?: TavilySearchResult[];
request_id?: string | null;
answer?: unknown;
results?: unknown;
request_id?: unknown;
}
function asRecord(value: unknown): Record<string, unknown> | null {
@@ -141,28 +134,36 @@ async function callTavilySearch(apiKey: string, params: TavilySearchParams): Pro
throw new SearchProviderError("tavily", `Tavily API error (${response.status}): ${message}`, response.status);
}
return (await response.json()) as TavilySearchResponse;
const payload: unknown = await response.json();
return asRecord(payload) ?? {};
}
function toSearchResponse(response: TavilySearchResponse, numResults: number): SearchResponse {
const sources: SearchSource[] = [];
for (const result of response.results ?? []) {
if (!result.url) continue;
sources.push({
title: result.title ?? result.url,
url: result.url,
snippet: result.content ?? undefined,
publishedDate: result.published_date ?? undefined,
ageSeconds: dateToAgeSeconds(result.published_date ?? undefined),
});
if (Array.isArray(response.results)) {
for (const value of response.results) {
const result = asRecord(value);
if (!result || typeof result.url !== "string" || !result.url) continue;
const title = typeof result.title === "string" && result.title ? result.title : result.url;
const snippet = typeof result.content === "string" ? result.content : undefined;
const publishedDate = typeof result.published_date === "string" ? result.published_date : undefined;
sources.push({
title,
url: result.url,
snippet,
publishedDate,
ageSeconds: dateToAgeSeconds(publishedDate),
});
}
}
const answer = typeof response.answer === "string" ? response.answer.trim() || undefined : undefined;
return {
provider: "tavily",
answer: response.answer?.trim() || undefined,
answer,
sources: sources.slice(0, numResults),
requestId: response.request_id ?? undefined,
requestId: typeof response.request_id === "string" ? response.request_id : undefined,
authMode: "api_key",
};
}
@@ -19,7 +19,7 @@ const MAX_NUM_RESULTS = 20;
const MAX_PAGE = 10;
/** TinyFish is SERP-backed: common Google-style operators pass through. */
const TINYFISH_QUERY_SYNTAX: QuerySyntax = { phrases: true, negation: true, site: true, filetype: true };
const TINYFISH_QUERY_SYNTAX: QuerySyntax = { phrases: true, negation: true, filetype: true };
const RECENCY_MINUTES: Record<NonNullable<SearchParams["recency"]>, number> = {
day: 1440,
@@ -33,6 +33,8 @@ export interface TinyFishSearchParams {
num_results?: number;
recency?: SearchParams["recency"];
page?: number;
include_domains?: string[];
exclude_domains?: string[];
signal?: AbortSignal;
timeoutMs?: number;
fetch?: FetchImpl;
@@ -66,6 +68,12 @@ async function callTinyFishSearch(apiKey: string, params: TinyFishSearchParams):
if (params.recency) {
url.searchParams.set("recency_minutes", String(RECENCY_MINUTES[params.recency]));
}
if (params.include_domains?.length) {
url.searchParams.set("include_domains", params.include_domains.join(","));
}
if (params.exclude_domains?.length) {
url.searchParams.set("exclude_domains", params.exclude_domains.join(","));
}
if (params.num_results !== undefined) {
url.searchParams.set("num_results", String(params.num_results));
}
@@ -96,18 +104,35 @@ async function callTinyFishSearch(apiKey: string, params: TinyFishSearchParams):
return (await response.json()) as TinyFishSearchResponse;
}
function appendTinyFishSources(sources: SearchSource[], results: readonly TinyFishSearchResult[]): void {
function appendTinyFishSources(
sources: SearchSource[],
results: readonly TinyFishSearchResult[],
seenUrls: Set<string>,
): void {
for (const result of results) {
if (!result.url) continue;
const url = result.url?.trim();
if (!url || seenUrls.has(url)) continue;
seenUrls.add(url);
const siteName = result.site_name?.trim();
sources.push({
title: result.title ?? result.site_name ?? result.url,
url: result.url,
snippet: result.snippet ?? undefined,
author: result.site_name ?? undefined,
title: result.title?.trim() || siteName || url,
url,
snippet: result.snippet?.replace(/\s+/g, " ").trim() || undefined,
author: siteName || undefined,
});
}
}
/** Bare hosts from `site:` values; path constraints remain centrally post-filtered. */
function siteHosts(sites: readonly string[]): string[] {
const hosts = new Set<string>();
for (const site of sites) {
const host = site.split("/", 1)[0];
if (host) hosts.add(host);
}
return [...hosts];
}
/** Execute TinyFish web search. */
export async function searchTinyFish(params: SearchParams): Promise<SearchResponse> {
const numResults = clampNumResults(params.numSearchResults ?? params.limit, DEFAULT_NUM_RESULTS, MAX_NUM_RESULTS);
@@ -121,6 +146,12 @@ export async function searchTinyFish(params: SearchParams): Promise<SearchRespon
timeoutMs: params.timeoutMs,
fetch: params.fetch,
};
if (parsed.hasDirectives) {
const includeDomains = siteHosts(parsed.sites);
const excludeDomains = siteHosts(parsed.excludedSites);
if (includeDomains.length > 0) tinyFishParams.include_domains = includeDomains;
if (excludeDomains.length > 0) tinyFishParams.exclude_domains = excludeDomains;
}
const keyOrResolver: ApiKey = params.authStorage.resolver("tinyfish", {
sessionId: params.sessionId,
});
@@ -128,11 +159,14 @@ export async function searchTinyFish(params: SearchParams): Promise<SearchRespon
keyOrResolver,
async key => {
const collected: SearchSource[] = [];
const seenUrls = new Set<string>();
for (let page = 0; page <= MAX_PAGE && collected.length < numResults; page += 1) {
const searchPage = await callTinyFishSearch(key, { ...tinyFishParams, page });
const results = searchPage.results ?? [];
appendTinyFishSources(collected, results);
if (results.length < pageSize) break;
if (!Array.isArray(searchPage.results)) {
throw new Error("TinyFish Search API returned an unexpected response shape");
}
appendTinyFishSources(collected, searchPage.results, seenUrls);
if (searchPage.results.length < pageSize) break;
}
return collected.slice(0, numResults);
@@ -25,6 +25,8 @@ interface XAIUrlCitationAnnotation {
title?: string | null;
text?: string | null;
cited_text?: string | null;
start_index?: number | null;
end_index?: number | null;
}
interface XAIResponseContentPart {
@@ -34,9 +36,20 @@ interface XAIResponseContentPart {
annotations?: XAIUrlCitationAnnotation[] | null;
}
interface XAIWebSearchSource {
url?: string | null;
source_website_url?: string | null;
title?: string | null;
caption?: string | null;
}
interface XAIResponseOutputItem {
type?: string;
content?: XAIResponseContentPart[] | null;
annotations?: XAIUrlCitationAnnotation[] | null;
action?: { sources?: XAIWebSearchSource[] | null } | null;
sources?: XAIWebSearchSource[] | null;
results?: XAIWebSearchSource[] | null;
}
interface XAIResponsesUsage {
@@ -161,7 +174,12 @@ async function callXAIResponses(
throwXAIResponsesError(response.status, await response.text());
}
return (await response.json()) as XAIResponsesResponse;
try {
return (await response.json()) as XAIResponsesResponse;
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
throw new SearchProviderError("xai", `xAI Responses API returned invalid JSON: ${message}`, response.status);
}
}
function addCitationSource(
@@ -189,38 +207,77 @@ function addCitationSource(
citedText: sourceSnippet,
});
}
function extractSnippetAround(
text: string | null | undefined,
start: number | null | undefined,
end: number | null | undefined,
): string | undefined {
if (!text || typeof start !== "number" || typeof end !== "number") return undefined;
const before = Math.max(0, start - 100);
const after = Math.min(text.length, end + 100);
const snippet = text
.slice(before, after)
.replace(/\[([^\]]*)\]\([^)]*\)/g, "$1")
.trim();
if (!snippet) return undefined;
return snippet.length > 300 ? `${snippet.slice(0, 297)}...` : snippet;
}
function collectAnnotationSources(
annotations: readonly XAIUrlCitationAnnotation[] | null | undefined,
sources: SearchSource[],
citations: SearchCitation[],
seenUrls: Set<string>,
contentText?: string | null,
): void {
if (!annotations) return;
if (!Array.isArray(annotations)) return;
for (const annotation of annotations) {
if (annotation.type !== "url_citation" || !annotation.url) continue;
if (!annotation || typeof annotation !== "object") continue;
if (annotation.type !== "url_citation" || typeof annotation.url !== "string") continue;
addCitationSource(
sources,
citations,
seenUrls,
annotation.url,
annotation.title,
annotation.cited_text ?? annotation.text,
annotation.cited_text ??
annotation.text ??
extractSnippetAround(contentText, annotation.start_index, annotation.end_index),
);
}
}
function collectWebSearchSources(
item: XAIResponseOutputItem,
sources: SearchSource[],
citations: SearchCitation[],
seenUrls: Set<string>,
): void {
if (item.type !== "web_search_call") return;
for (const group of [item.action?.sources, item.sources, item.results]) {
if (!Array.isArray(group)) continue;
for (const source of group) {
if (!source || typeof source !== "object") continue;
const url = source.url ?? source.source_website_url;
if (typeof url !== "string") continue;
addCitationSource(sources, citations, seenUrls, url, source.title ?? source.caption);
}
}
}
function parseAnswer(response: XAIResponsesResponse): string | undefined {
const topLevelText = response.output_text?.trim();
if (topLevelText) return topLevelText;
const answerParts: string[] = [];
for (const item of response.output ?? []) {
for (const part of item.content ?? []) {
const output = Array.isArray(response.output) ? response.output : [];
for (const item of output) {
if (!item || typeof item !== "object") continue;
const content = Array.isArray(item.content) ? item.content : [];
for (const part of content) {
if (!part || typeof part !== "object") continue;
const text = part.output_text ?? part.text;
if ((part.type === "output_text" || part.type === "text") && text?.trim()) {
answerParts.push(text.trim());
}
if (text?.trim()) answerParts.push(text.trim());
}
}
@@ -259,13 +316,29 @@ function parseResponse(response: XAIResponsesResponse, resultCap: number): Searc
const seenUrls = new Set<string>();
collectAnnotationSources(response.annotations, sources, citations, seenUrls);
for (const item of response.output ?? []) {
const output = Array.isArray(response.output) ? response.output : [];
for (const item of output) {
if (!item || typeof item !== "object") continue;
collectAnnotationSources(item.annotations, sources, citations, seenUrls);
for (const part of item.content ?? []) {
collectAnnotationSources(part.annotations, sources, citations, seenUrls);
const content = Array.isArray(item.content) ? item.content : [];
for (const part of content) {
if (!part || typeof part !== "object") continue;
collectAnnotationSources(
part.annotations,
sources,
citations,
seenUrls,
part.output_text ?? part.text,
);
}
}
for (const url of response.citations ?? []) {
for (const item of output) {
if (!item || typeof item !== "object") continue;
collectWebSearchSources(item, sources, citations, seenUrls);
}
const topLevelCitations = Array.isArray(response.citations) ? response.citations : [];
for (const url of topLevelCitations) {
if (typeof url !== "string") continue;
addCitationSource(sources, citations, seenUrls, url);
}
const limited = applyResultCap(sources, citations, resultCap);
@@ -354,7 +427,11 @@ export async function searchXAI(params: SearchParams): Promise<SearchResponse> {
signal: params.signal,
missingKeyMessage: 'xAI credentials not found. Set XAI_API_KEY or configure an API key for provider "xai".',
});
return parseResponse(response, resultCap);
const parsed = parseResponse(response, resultCap);
if (!parsed.answer && parsed.sources.length === 0) {
throw new SearchProviderError("xai", "xAI web_search returned no answer or sources", 502);
}
return parsed;
}
/** Search provider for xAI web search. */
@@ -94,3 +94,63 @@ describe("renderHtmlToText: jina stall does not starve local fallbacks (#1449)",
).toBe(true);
});
});
describe("renderHtmlToText: Jina response validation", () => {
it("requests fresh markdown and strips the Jina metadata preamble", async () => {
const settings = Settings.isolated({ "providers.fetch": "jina" });
const markdown = `# Extracted article\n\n${"Substantive reader content. ".repeat(8)}`.trim();
let requestHeaders: Headers | undefined;
const fetchMock = asGlobalFetch((_input, init) => {
requestHeaders = new Headers(init?.headers);
return new Response(`Title: Example\nURL Source: https://example.com/article\nMarkdown Content:\n${markdown}`);
});
const result = await renderHtmlToText(
"https://example.com/article",
"<html><body>short</body></html>",
1,
settings,
undefined,
null,
fetchMock,
);
expect(result).toEqual({ content: markdown, ok: true, method: "jina" });
expect(requestHeaders?.get("accept")).toBe("text/markdown");
expect(requestHeaders?.get("x-no-cache")).toBe("true");
});
for (const { label, readerBody, headers } of [
{ label: "missing marker", readerBody: "Plausible but unstructured output. ".repeat(8) },
{ label: "short body", readerBody: "Markdown Content:\nToo short" },
{ label: "loading shell", readerBody: `Markdown Content:\nLoading...${" ".repeat(120)}` },
{ label: "JavaScript gate", readerBody: `Markdown Content:\nPlease enable JavaScript${" ".repeat(120)}` },
{
label: "declared oversized body",
readerBody: `Markdown Content:\n${"Substantive content. ".repeat(8)}`,
headers: { "Content-Length": String(2 * 1024 * 1024 + 1) },
},
]) {
it(`falls back when Jina returns a ${label}`, async () => {
const settings = Settings.isolated({ "providers.fetch": "jina" });
const paragraph =
"This locally rendered article contains enough meaningful prose to satisfy the shared reader quality gate. ";
const html = `<html><body><article><h1>Fallback article</h1><p>${paragraph.repeat(4)}</p></article></body></html>`;
const fetchMock = asGlobalFetch(() => new Response(readerBody, { headers }));
const result = await renderHtmlToText(
"https://example.com/article",
html,
1,
settings,
undefined,
null,
fetchMock,
);
expect(result.ok).toBe(true);
expect(result.method).toBe("native");
expect(result.content).toContain("Fallback article");
});
}
});
@@ -531,6 +531,70 @@ describe("searchCodex model selection", () => {
expect(result.sources).toEqual([{ title: "Example Article", url: "https://example.com/article" }]);
});
it("requests and merges web-search action sources with citation metadata", async () => {
process.env.PI_CODEX_WEB_SEARCH_MODEL = "gpt-5.4";
const answer = "The Responses API supports hosted web search.";
const citationStart = answer.indexOf("hosted web search");
const sse = [
`data: ${JSON.stringify({
type: "response.created",
response: { id: "resp_created_id", model: "gpt-5.4" },
})}`,
"",
`data: ${JSON.stringify({
type: "response.output_item.done",
item: {
type: "web_search_call",
action: {
sources: [
{
url: "https://example.com/article?utm_source=openai",
title: "Search result title",
},
],
},
},
})}`,
"",
`data: ${JSON.stringify({
type: "response.output_item.done",
item: {
type: "message",
content: [
{
type: "output_text",
text: answer,
annotations: [
{
type: "url_citation",
url: "https://example.com/article?utm_source=openai",
title: "Example Article",
start_index: citationStart,
end_index: citationStart + "hosted web search".length,
},
],
},
],
},
})}`,
"",
].join("\n");
const result = await searchCodex(
makeSearchParams("action sources", mockCodexFetch("gpt-5.4", sse)),
);
expect(capturedRequest?.body?.include).toEqual(["web_search_call.action.sources"]);
expect(result.requestId).toBe("resp_created_id");
expect(result.sources).toEqual([
{
title: "Search result title",
url: "https://example.com/article",
snippet: answer,
},
]);
});
it("extracts plain text URLs when annotations are absent", async () => {
process.env.PI_CODEX_WEB_SEARCH_MODEL = "gpt-5.4";
const result = await searchCodex(
@@ -769,4 +833,22 @@ describe("searchCodex model selection", () => {
"Codex request failed (model_snapshot_unavailable): The requested model snapshot is unavailable.",
);
});
it("classifies rate-limit failures delivered inside a successful SSE response", async () => {
const sse = [
`data: ${JSON.stringify({
type: "response.failed",
response: {
error: { code: "rate_limit_exceeded", message: "Too many requests" },
},
})}`,
"",
].join("\n");
const fetchMock: FetchImpl = () =>
Promise.resolve(new Response(sse, { status: 200, headers: { "Content-Type": "text/event-stream" } }));
await expect(searchCodex(makeSearchParams("rate-limited search", fetchMock))).rejects.toMatchObject({
status: 429,
});
});
});
@@ -428,6 +428,19 @@ describe("searchExa", () => {
expect(result.sources[0].snippet).toBe("summary here");
});
it("caps snippets at 500 characters", async () => {
const result = await searchExa({
query: "bounded snippet",
fetch: mockFetch(
makeMockExaResponse({
results: [{ title: "Long", url: "https://long.example", summary: "x".repeat(800) }],
}),
),
});
expect(result.sources[0].snippet).toHaveLength(500);
});
it("falls back to text when summary is null", async () => {
const result = await searchExa({
query: "fallback",
@@ -535,6 +548,68 @@ describe("searchExa", () => {
});
});
it("encodes MCP filters in the basic query, uses camel-case result count, and tags the request source", async () => {
delete process.env.EXA_API_KEY;
let headers: Record<string, string> | undefined;
const fetchMock: FetchImpl = (_url, init) => {
headers = init?.headers as Record<string, string> | undefined;
if (init?.body) capturedRequestBody = JSON.parse(init.body as string);
return Promise.resolve(
new Response(JSON.stringify({ jsonrpc: "2.0", id: "mcp-filtered", result: makeMockExaResponse() }), {
status: 200,
headers: { "Content-Type": "application/json" },
}),
);
};
await searchExa({
query: "vector databases",
num_results: 4,
include_domains: [" qdrant.tech "],
exclude_domains: ["spam.example"],
start_published_date: "2024-01-01",
end_published_date: "2025-01-01",
fetch: fetchMock,
});
expect(headers?.["x-exa-source"]).toBe("oh-my-pi");
expect(capturedRequestBody?.params).toEqual({
name: "web_search_exa",
arguments: {
query:
"vector databases site:qdrant.tech -site:spam.example after:2024-01-01 before:2025-01-01",
numResults: 4,
},
});
});
it("explains how to escape the keyless MCP rate limit", async () => {
delete process.env.EXA_API_KEY;
const fetchMock: FetchImpl = () =>
Promise.resolve(new Response("too many requests", { status: 429, statusText: "Too Many Requests" }));
await expect(searchExa({ query: "rate limited", fetch: fetchMock })).rejects.toThrow(
"exa: MCP rate limit reached (429); configure an Exa API key for higher limits",
);
});
it("surfaces MCP tool-level errors", async () => {
delete process.env.EXA_API_KEY;
const fetchMock: FetchImpl = () =>
Promise.resolve(
new Response(
JSON.stringify({
jsonrpc: "2.0",
id: "mcp-error",
result: { isError: true, content: [{ type: "text", text: "tool quota exceeded" }] },
}),
{ status: 200, headers: { "Content-Type": "application/json" } },
),
);
await expect(searchExa({ query: "tool error", fetch: fetchMock })).rejects.toThrow("tool quota exceeded");
});
it("parses Exa MCP plain-text payloads when API key is missing", async () => {
delete process.env.EXA_API_KEY;
const fetchMock: FetchImpl = () => {
@@ -235,18 +235,58 @@ describe("Firecrawl web search provider", () => {
}
});
it("keeps keyless Firecrawl out of auto selection while allowing explicit selection", () => {
it("keeps hosted keyless Firecrawl explicit-only but admits configured self-hosting", () => {
const originalApiKey = process.env.FIRECRAWL_API_KEY;
const originalBaseUrl = process.env.FIRECRAWL_BASE_URL;
const originalApiUrl = process.env.FIRECRAWL_API_URL;
delete process.env.FIRECRAWL_API_KEY;
delete process.env.FIRECRAWL_BASE_URL;
delete process.env.FIRECRAWL_API_URL;
try {
const provider = new FirecrawlProvider();
const authStorage = makeAuthStorage(undefined);
expect(provider.isAvailable(authStorage)).toBe(false);
expect(provider.isExplicitlyAvailable(authStorage)).toBe(true);
process.env.FIRECRAWL_BASE_URL = "http://localhost:3002";
expect(provider.isAvailable(authStorage)).toBe(true);
} finally {
if (originalApiKey === undefined) delete process.env.FIRECRAWL_API_KEY;
else process.env.FIRECRAWL_API_KEY = originalApiKey;
if (originalBaseUrl === undefined) delete process.env.FIRECRAWL_BASE_URL;
else process.env.FIRECRAWL_BASE_URL = originalBaseUrl;
if (originalApiUrl === undefined) delete process.env.FIRECRAWL_API_URL;
else process.env.FIRECRAWL_API_URL = originalApiUrl;
}
});
it("uses a self-hosted endpoint and accepts Firecrawl v1 array responses", async () => {
const originalBaseUrl = process.env.FIRECRAWL_BASE_URL;
process.env.FIRECRAWL_BASE_URL = "http://localhost:3002/v1/";
let requestUrl = "";
try {
const fetchMock: FetchImpl = async input => {
requestUrl = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url;
return new Response(
JSON.stringify({
success: true,
data: [{ title: "Legacy result", url: "https://example.com/legacy", snippet: "Legacy snippet" }],
}),
{ status: 200, headers: { "Content-Type": "application/json" } },
);
};
const response = await searchFirecrawl({
...makeParams("legacy query", makeAuthStorage(undefined)),
fetch: fetchMock,
});
expect(requestUrl).toBe("http://localhost:3002/v1/search");
expect(response.sources).toEqual([
{ title: "Legacy result", url: "https://example.com/legacy", snippet: "Legacy snippet" },
]);
} finally {
if (originalBaseUrl === undefined) delete process.env.FIRECRAWL_BASE_URL;
else process.env.FIRECRAWL_BASE_URL = originalBaseUrl;
}
});
@@ -10,6 +10,7 @@ const DEVELOPER_SSE_RESPONSE =
const DEVELOPER_SSE_RESPONSE_WITHOUT_MODEL =
'data: {"candidates":[{"content":{"role":"model","parts":[{"text":"Developer answer"}]},"groundingMetadata":{"webSearchQueries":["latest Bun version"],"groundingChunks":[{"web":{"uri":"https://bun.sh","title":"Bun"}}],"groundingSupports":[{"segment":{"text":"Developer answer"},"groundingChunkIndices":[0]}]}}],"usageMetadata":{"promptTokenCount":3,"candidatesTokenCount":4,"totalTokenCount":7}}\n\n';
const ORIGINAL_GEMINI_SEARCH_MODEL = Bun.env.GEMINI_SEARCH_MODEL;
const ORIGINAL_GEMINI_BASE_URL = Bun.env.GOOGLE_GEMINI_BASE_URL;
type CapturedRequest = {
url: string;
@@ -72,6 +73,11 @@ describe("searchGemini tools serialization", () => {
} else {
Bun.env.GEMINI_SEARCH_MODEL = ORIGINAL_GEMINI_SEARCH_MODEL;
}
if (ORIGINAL_GEMINI_BASE_URL === undefined) {
delete Bun.env.GOOGLE_GEMINI_BASE_URL;
} else {
Bun.env.GOOGLE_GEMINI_BASE_URL = ORIGINAL_GEMINI_BASE_URL;
}
});
function makeParams(query: string) {
@@ -111,6 +117,61 @@ describe("searchGemini tools serialization", () => {
});
});
it("routes Cloudflare AI Gateway auth through AuthStorage without leaking a Google API key", async () => {
Bun.env.GOOGLE_GEMINI_BASE_URL =
"https://gateway.ai.cloudflare.com/v1/account/gateway/google-ai-studio";
const gatewayAuthStorage = {
async getOAuthAccess() {
return undefined;
},
hasOAuth() {
return false;
},
hasAuth(provider: string) {
return provider === "cloudflare-ai-gateway";
},
async getApiKey(provider: string) {
return provider === "cloudflare-ai-gateway" ? "test-cloudflare-key" : undefined;
},
} as unknown as AuthStorage;
const fetchMock = mockGeminiFetch(DEVELOPER_SSE_RESPONSE);
expect(new GeminiProvider().isAvailable(gatewayAuthStorage)).toBe(true);
await searchGemini({
...makeParams("gateway"),
authStorage: gatewayAuthStorage,
fetch: fetchMock,
});
expect(capturedRequest?.url).toBe(
"https://gateway.ai.cloudflare.com/v1/account/gateway/google-ai-studio/v1beta/models/gemini-2.5-flash:streamGenerateContent?alt=sse",
);
expect(capturedRequest?.headers["cf-aig-authorization"]).toBe("Bearer test-cloudflare-key");
expect(capturedRequest?.headers["x-goog-api-key"]).toBeUndefined();
});
it("redacts the active credential from Gemini API errors", async () => {
let thrown: unknown;
try {
await searchGemini({
...makeParams("redaction"),
authStorage: apiKeyAuthStorage,
fetch: () =>
Promise.resolve(
new Response("upstream echoed test-gemini-api-key", {
status: 418,
}),
),
});
} catch (error) {
thrown = error;
}
expect(thrown).toBeInstanceOf(Error);
expect((thrown as Error).message).toContain("[redacted]");
expect((thrown as Error).message).not.toContain("test-gemini-api-key");
});
it("normalizes query directive aliases to canonical Google forms in the grounding request", async () => {
const fetchMock = mockGeminiFetch();
await searchGemini({
@@ -221,4 +282,55 @@ describe("searchGemini tools serialization", () => {
tools: [{ googleSearch: {} }, { codeExecution: {} }, { urlContext: { allowedDomains: ["example.com"] } }],
});
});
it("resolves Google grounding proxy URLs in both sources and citations", async () => {
const proxyUrl =
"https://vertexaisearch.cloud.google.com/grounding-api-redirect/abc";
const responseText = `data: ${JSON.stringify({
candidates: [
{
content: { role: "model", parts: [{ text: "Grounded answer" }] },
groundingMetadata: {
groundingChunks: [{ web: { uri: proxyUrl, title: "Example" } }],
groundingSupports: [{ segment: { text: "Grounded answer" }, groundingChunkIndices: [0] }],
},
},
],
})}\n\n`;
const methods: string[] = [];
const fetchMock: FetchImpl = (_url, init) => {
methods.push(init?.method ?? "GET");
if (init?.method === "HEAD") {
return Promise.resolve(
new Response(null, {
status: 302,
headers: { location: "https://example.com/article" },
}),
);
}
return Promise.resolve(new Response(responseText, { status: 200 }));
};
const response = await searchGemini({
...makeParams("grounding redirect"),
authStorage: apiKeyAuthStorage,
fetch: fetchMock,
});
expect(methods).toEqual(["POST", "HEAD"]);
expect(response.sources).toEqual([{ title: "Example", url: "https://example.com/article" }]);
expect(response.citations).toEqual([
{ title: "Example", url: "https://example.com/article", citedText: "Grounded answer" },
]);
});
it("rejects a successful Gemini response with no answer or grounding results", async () => {
await expect(
searchGemini({
...makeParams("empty"),
authStorage: apiKeyAuthStorage,
fetch: mockGeminiFetch("data: {}\n\n"),
}),
).rejects.toThrow("Gemini API returned an empty grounded response");
});
});
@@ -58,6 +58,34 @@ describe("Kagi web search error handling", () => {
"Kagi API error (502)",
);
});
it("reports malformed success responses as Kagi API errors", async () => {
const invalidJsonFetch: FetchImpl = async () => new Response("<html>not json</html>", { status: 200 });
const invalidEnvelopeFetch: FetchImpl = async () =>
new Response(JSON.stringify(["unexpected"]), {
status: 200,
headers: { "Content-Type": "application/json" },
});
await expect(searchWithKagi("invalid json", { fetch: invalidJsonFetch }, fakeAuthStorage)).rejects.toThrow(
"Kagi API returned an invalid response: invalid JSON",
);
await expect(searchWithKagi("invalid envelope", { fetch: invalidEnvelopeFetch }, fakeAuthStorage)).rejects.toThrow(
"Kagi API returned an invalid response: expected an object envelope",
);
});
it("recognizes errors plural in a successful HTTP envelope", async () => {
const fetchMock: FetchImpl = async () =>
new Response(JSON.stringify({ errors: [{ code: 429, message: "quota exceeded" }] }), {
status: 200,
headers: { "Content-Type": "application/json" },
});
await expect(searchWithKagi("envelope error", { fetch: fetchMock }, fakeAuthStorage)).rejects.toThrow(
"Kagi API error (429): quota exceeded",
);
});
it("applies the configured timeout at the provider fetch boundary", async () => {
const timeoutSignal = new AbortController().signal;
const timeoutSpy = vi.spyOn(AbortSignal, "timeout").mockReturnValue(timeoutSignal);
@@ -158,6 +186,39 @@ describe("Kagi search result parsing", () => {
expect(result.answer).toBeUndefined();
});
it("accepts documented result aliases and skips malformed items", async () => {
const fetchMock: FetchImpl = async () =>
new Response(
JSON.stringify({
data: {
search: [
null,
{ title: "Missing URL" },
{
href: "https://example.com/alias",
name: "Alias Result",
description: "Alias description",
},
],
related_search: { invalid: true },
},
}),
{ status: 200, headers: { "Content-Type": "application/json" } },
);
const result = await searchWithKagi("aliases", { fetch: fetchMock }, fakeAuthStorage);
expect(result.sources).toEqual([
{
title: "Alias Result",
url: "https://example.com/alias",
snippet: "Alias description",
publishedDate: undefined,
},
]);
expect(result.relatedQuestions).toEqual([]);
});
it("parses direct_answer into the answer field", async () => {
const fetchMock: FetchImpl = async () =>
new Response(
@@ -1,4 +1,4 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "bun:test";
import { afterEach, beforeEach, describe, expect, it, setSystemTime, vi } from "bun:test";
import type { AuthStorage, FetchImpl } from "@oh-my-pi/pi-ai";
import type { AgentStorage } from "@oh-my-pi/pi-coding-agent/session/agent-storage";
import { searchWithParallel } from "@oh-my-pi/pi-coding-agent/web/parallel";
@@ -149,6 +149,29 @@ describe("Parallel web search", () => {
});
});
it("maps recency onto source_policy.after_date", async () => {
setSystemTime(new Date("2026-08-10T12:00:00Z"));
try {
const fetchMock = mockFetch({
search_id: "search-parallel-recency",
results: [],
warnings: null,
usage: null,
});
await searchParallel({ query: "recent api changes", recency: "week", fetch: fetchMock }, fakeAuthStorage);
expect(capturedRequestBody).toEqual({
objective: "recent api changes",
search_queries: ["recent api changes"],
mode: "fast",
excerpts: { max_chars_per_result: 10_000 },
source_policy: { after_date: "2026-08-03" },
});
} finally {
setSystemTime();
}
});
it("maps -site: and after: onto exclude_domains/after_date, keeping phrases and negation", async () => {
const fetchMock = mockFetch({
search_id: "search-parallel-4",
@@ -158,7 +181,11 @@ describe("Parallel web search", () => {
});
await searchParallel(
{ query: '"web api" -legacy -site:reddit.com/r/node after:2025-06-01', fetch: fetchMock },
{
query: '"web api" -legacy -site:reddit.com/r/node after:2025-06-01',
recency: "day",
fetch: fetchMock,
},
fakeAuthStorage,
);
expect(capturedRequestBody).toEqual({
@@ -178,4 +205,13 @@ describe("Parallel web search", () => {
message: "Parallel API error (503): upstream unavailable",
});
});
it("classifies malformed successful responses as Parallel errors", async () => {
const fetchMock: FetchImpl = () =>
Promise.resolve(new Response("{not-json", { status: 200, headers: { "Content-Type": "application/json" } }));
await expect(searchParallel({ query: "broken", fetch: fetchMock }, fakeAuthStorage)).rejects.toMatchObject({
provider: "parallel",
message: expect.stringContaining("Parallel search returned invalid JSON:"),
});
});
});
@@ -121,6 +121,53 @@ describe("SearXNG web search provider", () => {
expect(captured.url?.searchParams.get("language")).toBeNull();
});
it("sends configured category and safe-search filters and preserves instant answers", async () => {
const agentDir = await fs.mkdtemp(path.join(os.tmpdir(), "searxng-filters-"));
try {
await Bun.write(
path.join(agentDir, "config.yml"),
[
"searxng:",
" endpoint: https://searx.example.org",
" categories: news",
" safesearch: 2",
"",
].join("\n"),
);
await Settings.init({ agentDir });
const captured: { url?: URL } = {};
const fetchMock: FetchImpl = input => {
captured.url = new URL(input.toString());
return Promise.resolve(
new Response(
JSON.stringify({
results: [{ title: "r", url: "https://example.com", snippet: "Fallback snippet" }],
answers: [
" Forty-two ",
{ template: "answer/legacy.html", answer: "Legacy answer" },
{
template: "answer/translations.html",
translations: [{ text: "Hallo" }, { text: "Guten Tag" }],
},
],
}),
{ status: 200, headers: { "Content-Type": "application/json" } },
),
);
};
const response = await searchSearXNG({ query: "filtered answers", fetch: fetchMock });
expect(captured.url?.searchParams.get("categories")).toBe("news");
expect(captured.url?.searchParams.get("safesearch")).toBe("2");
expect(response.answer).toBe("Forty-two\n\nLegacy answer\n\nHallo\nGuten Tag");
expect(response.sources[0]?.snippet).toBe("Fallback snippet");
} finally {
await removeWithRetries(agentDir);
}
});
it("reads Basic auth credentials from nested config.yml settings", async () => {
const agentDir = await fs.mkdtemp(path.join(os.tmpdir(), "searxng-settings-"));
try {
@@ -84,10 +84,16 @@ describe("TinyFish web search provider", () => {
});
expect(captured).toHaveLength(1);
expect(captured[0].searchParams.get("query")).toBe(
'"error handling" rust site:github.com -site:gitlab.com filetype:pdf',
);
expectTinyFishParams(captured[0], ["query", "num_results", "page"]);
expect(captured[0].searchParams.get("query")).toBe('"error handling" rust filetype:pdf');
expect(captured[0].searchParams.get("include_domains")).toBe("github.com");
expect(captured[0].searchParams.get("exclude_domains")).toBe("gitlab.com");
expectTinyFishParams(captured[0], [
"query",
"num_results",
"page",
"include_domains",
"exclude_domains",
]);
});
it("sends directive-free queries verbatim", async () => {
@@ -232,6 +238,43 @@ describe("TinyFish web search provider", () => {
expect(response.sources.at(-1)?.url).toBe("https://example.com/raw-page-11");
});
it("deduplicates and normalizes results across pages", async () => {
const captured: URL[] = [];
const firstPage = tinyFishResults("dedupe", 10);
firstPage[0] = {
title: " Primary title ",
url: " https://example.com/dedupe-0 ",
snippet: " spaced \n snippet ",
site_name: " Example ",
};
firstPage[1] = {
title: "Duplicate title",
url: "https://example.com/dedupe-0",
snippet: "duplicate snippet",
};
const fetchMock: FetchImpl = async input => {
const url = input instanceof URL ? input : new URL(typeof input === "string" ? input : input.url);
captured.push(url);
const page = Number(url.searchParams.get("page") ?? 0);
const results = page === 0 ? firstPage : tinyFishResults("dedupe", 1, 10);
return new Response(JSON.stringify(tinyFishPage(results, page, 11)), {
status: 200,
headers: { "Content-Type": "application/json" },
});
};
const response = await searchTinyFish({ ...makeParams("dedupe fish"), limit: 10, fetch: fetchMock });
expect(captured.map(url => url.searchParams.get("page"))).toEqual(["0", "1"]);
expect(response.sources).toHaveLength(10);
expect(response.sources[0]).toMatchObject({
title: "Primary title",
url: "https://example.com/dedupe-0",
snippet: "spaced snippet",
});
expect(response.sources.at(-1)?.url).toBe("https://example.com/dedupe-10");
});
it("stops early for limit 20 when page 0 returns fewer than 10 raw results", async () => {
const captured: URL[] = [];
const fetchMock: FetchImpl = async input => {
@@ -802,6 +802,67 @@ describe("xAI web search provider", () => {
});
});
it("extracts offset snippets and raw sources from web_search_call output", async () => {
const answer = "Context before [cited source](https://example.com/cited) context after.";
const start = answer.indexOf("[cited source]");
const capture = captureFetch({
id: "resp_raw_sources",
output: [
{
type: "message",
content: [
{
type: "output_text",
text: answer,
annotations: [
{
type: "url_citation",
url: "https://example.com/cited",
title: "Cited result",
start_index: start,
end_index: start + "[cited source]".length,
},
],
},
],
},
{
type: "web_search_call",
action: {
sources: [
{ url: "https://example.com/raw", title: "Raw result" },
{ source_website_url: "https://example.com/fallback", caption: "Fallback result" },
],
},
results: [{ url: "https://example.com/cited", title: "Duplicate result" }],
},
],
});
const response = await searchXAI(makeParams(capture.fetchMock));
expect(response.answer).toBe(answer);
expect(response.sources).toEqual([
{
title: "Cited result",
url: "https://example.com/cited",
snippet: "Context before cited source context after.",
},
{ title: "Raw result", url: "https://example.com/raw", snippet: undefined },
{ title: "Fallback result", url: "https://example.com/fallback", snippet: undefined },
]);
});
it("rejects successful responses with no answer or sources", async () => {
const capture = captureFetch({ id: "resp_empty", output: [] });
await expect(searchXAI(makeParams(capture.fetchMock))).rejects.toMatchObject({
provider: "xai",
status: 502,
message: "xAI web_search returned no answer or sources",
});
});
it.each([
[401, "xai: 401 unauthorized"],
[402, "xai: 402 credits exhausted"],
@@ -144,8 +144,6 @@ describe("Brave provider hard-timeout wiring", () => {
});
it("hands fetch a composed signal even with no caller signal — confirms the rollout reaches non-Anthropic providers", async () => {
process.env.BRAVE_API_KEY = "brave-test-key";
let capturedSignal: AbortSignal | null | undefined;
const fetchMock: FetchImpl = async (_input, init) => {
capturedSignal = init?.signal;
@@ -155,7 +153,13 @@ describe("Brave provider hard-timeout wiring", () => {
});
};
await searchBrave({ query: "ping", fetch: fetchMock });
await searchBrave({
query: "ping",
fetch: fetchMock,
authStorage: {
resolver: vi.fn(() => async () => "brave-test-key"),
} as unknown as AuthStorage,
});
expect(capturedSignal).toBeInstanceOf(AbortSignal);
expect(capturedSignal?.aborted).toBe(false);
@@ -75,11 +75,13 @@ describe("Perplexity API-key request shape", () => {
const savedOpenRouterKey = process.env.OPENROUTER_API_KEY;
const savedCookies = process.env.PERPLEXITY_COOKIES;
const savedResponsesMode = process.env.PI_PERPLEXITY_RESPONSES;
const savedApiModel = process.env.PI_PERPLEXITY_API_MODEL;
beforeEach(() => {
process.env.PERPLEXITY_API_KEY = "test-key";
delete process.env.PERPLEXITY_COOKIES;
delete process.env.PI_PERPLEXITY_RESPONSES;
delete process.env.PI_PERPLEXITY_API_MODEL;
});
afterEach(() => {
@@ -92,6 +94,8 @@ describe("Perplexity API-key request shape", () => {
else process.env.PERPLEXITY_COOKIES = savedCookies;
if (savedResponsesMode === undefined) delete process.env.PI_PERPLEXITY_RESPONSES;
else process.env.PI_PERPLEXITY_RESPONSES = savedResponsesMode;
if (savedApiModel === undefined) delete process.env.PI_PERPLEXITY_API_MODEL;
else process.env.PI_PERPLEXITY_API_MODEL = savedApiModel;
});
it("requests comprehensive defaults: 20 results, high context, related questions", async () => {
@@ -104,6 +108,18 @@ describe("Perplexity API-key request shape", () => {
expect(body?.return_related_questions).toBe(true);
});
it("accepts a configured direct API model", async () => {
process.env.PI_PERPLEXITY_API_MODEL = "sonar-deep-research";
let body: Record<string, unknown> | undefined;
await searchPerplexity({
query: "quic vs tcp",
authStorage: apiKeyAuthStorage,
fetch: mockApi(b => (body = b), baseResponse()),
});
expect(body?.model).toBe("sonar-deep-research");
});
it("honors a caller-supplied num_search_results over the default", async () => {
let body: Record<string, unknown> | undefined;
const fetchMock = mockApi(b => (body = b), baseResponse());
@@ -306,7 +322,10 @@ const anonymousAuthStorage = {
},
} as unknown as AuthStorage;
function mockOAuth(capture: (body: Record<string, unknown>, headers: Headers) => void): FetchImpl {
function mockOAuth(
capture: (body: Record<string, unknown>, headers: Headers) => void,
eventOverrides: Record<string, unknown> = {},
): FetchImpl {
const event = {
final: true,
display_model: "turbo",
@@ -318,6 +337,7 @@ function mockOAuth(capture: (body: Record<string, unknown>, headers: Headers) =>
web_result_block: { web_results: [{ name: "T", url: "https://example.com", snippet: "s" }] },
},
],
...eventOverrides,
};
const sseBody = `data: ${JSON.stringify(event)}\n\n`;
return async (input, init) => {
@@ -356,15 +376,19 @@ function mockAnonymous(capture: (body: Record<string, unknown>, headers: Headers
describe("Perplexity OAuth request shape", () => {
const savedCookies = process.env.PERPLEXITY_COOKIES;
const savedModel = process.env.PI_PERPLEXITY_MODEL;
beforeEach(() => {
delete process.env.PERPLEXITY_COOKIES; // cookies take precedence over oauth; keep them out
delete process.env.PI_PERPLEXITY_MODEL;
});
afterEach(() => {
vi.restoreAllMocks();
if (savedCookies === undefined) delete process.env.PERPLEXITY_COOKIES;
else process.env.PERPLEXITY_COOKIES = savedCookies;
if (savedModel === undefined) delete process.env.PI_PERPLEXITY_MODEL;
else process.env.PI_PERPLEXITY_MODEL = savedModel;
});
it("sends the bare query, never the API-style system prompt, to the ask endpoint", async () => {
@@ -395,6 +419,55 @@ describe("Perplexity OAuth request shape", () => {
expect(headers?.has("authorization")).toBe(false);
expect(response.authMode).toBe("oauth");
expect(response.answer).toBe("OAuth answer");
// Authenticated streams sometimes report only the generic `turbo` alias;
// preserve the requested subscription model instead of misreporting it.
expect(response.model).toBe("experimental");
});
it("accepts a subscription model preference and reports it when the stream returns turbo", async () => {
let body: Record<string, unknown> | undefined;
const response = await searchPerplexity({
query: "latest model",
subscription_model: "pplx_reasoning",
authStorage: oauthAuthStorage,
fetch: mockOAuth(b => (body = b)),
});
expect((body!.params as Record<string, unknown>).model_preference).toBe("pplx_reasoning");
expect(response.model).toBe("pplx_reasoning");
});
it("prefers the concrete user-selected model over the generic display alias", async () => {
const response = await searchPerplexity({
query: "latest model",
authStorage: oauthAuthStorage,
fetch: mockOAuth(() => {}, { user_selected_model: "pplx_pro_upgraded" }),
});
expect(response.model).toBe("pplx_pro_upgraded");
});
it("deduplicates equivalent subscription source URLs", async () => {
const response = await searchPerplexity({
query: "latest model",
authStorage: oauthAuthStorage,
fetch: mockOAuth(() => {}, {
blocks: [
{ intended_usage: "ask_text", markdown_block: { answer: "OAuth answer" } },
{
intended_usage: "web_results",
web_result_block: {
web_results: [
{ name: "First", url: "https://EXAMPLE.com/path/" },
{ name: "Duplicate", url: "https://example.com/path" },
],
},
},
],
}),
});
expect(response.sources).toHaveLength(1);
});
it("maps directives onto ask-endpoint native filters and rewrites query_str", async () => {
@@ -9,7 +9,11 @@ import {
} from "@oh-my-pi/pi-coding-agent/web/search/provider";
import { SEARCH_PROVIDER_ORDER } from "@oh-my-pi/pi-coding-agent/web/search/types";
const authStorage = {} as AuthStorage;
const authStorage = {
hasAuth(provider: string): boolean {
return provider === "jina" && Boolean(process.env.JINA_API_KEY);
},
} as AuthStorage;
const originalBraveApiKey = process.env.BRAVE_API_KEY;
const originalJinaApiKey = process.env.JINA_API_KEY;
@@ -214,4 +214,35 @@ describe("Tavily searchTavily request shape (integration)", () => {
expect(capturedBodies[1]).not.toHaveProperty("end_date");
expect(response.sources).toHaveLength(1);
});
it("ignores malformed response fields while preserving valid results", async () => {
process.env.TAVILY_API_KEY = "test-key";
const fetchMock: FetchImpl = async () =>
new Response(
JSON.stringify({
answer: 42,
results: [
null,
{ title: 7, url: "https://example.com/valid", content: 99, published_date: false },
{ title: "Missing URL" },
],
request_id: 123,
}),
{ status: 200, headers: { "Content-Type": "application/json" } },
);
const response = await searchTavily({ ...makeParams("robust parsing"), fetch: fetchMock });
expect(response.answer).toBeUndefined();
expect(response.requestId).toBeUndefined();
expect(response.sources).toEqual([
{
title: "https://example.com/valid",
url: "https://example.com/valid",
snippet: undefined,
publishedDate: undefined,
ageSeconds: undefined,
},
]);
});
});