feat(coding-agent/web): expanded web search and fetching providers
- Expanded web search and fetching providers with robust parsing, authentication storage integration, and response validation. - Added support for new configurations including SearXNG safesearch, Cloudflare AI Gateway endpoints, and dynamic Firecrawl base URLs. - Implemented comprehensive test suites covering error handling, content filtering, and provider-specific response behaviors.
This commit is contained in:
@@ -2,6 +2,13 @@
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
### Added
|
||||
|
||||
- Added `searxng.safesearch` setting option for SearXNG searches
|
||||
- Added support for Cloudflare AI Gateway routing for Gemini search
|
||||
- Added support for Exa MCP search provider
|
||||
- Added domain inclusion/exclusion filtering and URL deduplication for TinyFish search
|
||||
|
||||
## [17.2.12] - 2026-08-08
|
||||
|
||||
### Fixed
|
||||
|
||||
@@ -5471,6 +5471,11 @@ export const SETTINGS_SCHEMA = {
|
||||
default: undefined,
|
||||
},
|
||||
|
||||
"searxng.safesearch": {
|
||||
type: "number",
|
||||
default: undefined,
|
||||
},
|
||||
|
||||
"commit.mapReduceEnabled": { type: "boolean", default: true },
|
||||
|
||||
"commit.mapReduceMinFiles": { type: "number", default: 4 },
|
||||
|
||||
@@ -576,6 +576,23 @@ async function parseFeedToMarkdown(content: string, maxItems = 10): Promise<stri
|
||||
* local fallback renderers (trafilatura, lynx, native). See #1449.
|
||||
*/
|
||||
const REMOTE_READER_MAX_MS = 10_000;
|
||||
const JINA_MARKDOWN_MARKER = "Markdown Content:";
|
||||
const JINA_READER_MAX_BYTES = 2 * 1024 * 1024;
|
||||
|
||||
function parseJinaReaderContent(responseBody: string): string | null {
|
||||
const markerStart = responseBody.indexOf(JINA_MARKDOWN_MARKER);
|
||||
if (markerStart < 0) return null;
|
||||
|
||||
const content = responseBody.slice(markerStart + JINA_MARKDOWN_MARKER.length).trim();
|
||||
if (
|
||||
content.length < 100 ||
|
||||
content.startsWith("Loading...") ||
|
||||
content.startsWith("Please enable JavaScript")
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
return content;
|
||||
}
|
||||
|
||||
/** Reader backends for {@link renderHtmlToText}, in default priority order. */
|
||||
export type FetchProvider = "native" | "trafilatura" | "lynx" | "parallel" | "jina";
|
||||
@@ -653,10 +670,16 @@ export async function renderHtmlToText(
|
||||
},
|
||||
jina: async () => {
|
||||
const response = await fetchImpl(`https://r.jina.ai/${url}`, {
|
||||
headers: { Accept: "text/markdown" },
|
||||
headers: {
|
||||
Accept: "text/markdown",
|
||||
"X-No-Cache": "true",
|
||||
},
|
||||
signal: remoteSignal(),
|
||||
});
|
||||
return response.ok ? await response.text() : null;
|
||||
if (!response.ok) return null;
|
||||
const contentLength = Number(response.headers.get("content-length"));
|
||||
if (Number.isFinite(contentLength) && contentLength > JINA_READER_MAX_BYTES) return null;
|
||||
return parseJinaReaderContent(await response.text());
|
||||
},
|
||||
};
|
||||
|
||||
|
||||
@@ -83,6 +83,7 @@ export interface KagiErrorResponse {
|
||||
detail?: string;
|
||||
}
|
||||
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Error Handling
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -98,7 +99,7 @@ export class KagiApiError extends Error {
|
||||
}
|
||||
|
||||
function extractKagiErrorMessage(payload: unknown): string | null {
|
||||
if (!payload || typeof payload !== "object") return null;
|
||||
if (!payload || typeof payload !== "object" || Array.isArray(payload)) return null;
|
||||
const record = payload as Record<string, unknown>;
|
||||
|
||||
for (const value of [record.message, record.detail]) {
|
||||
@@ -107,17 +108,20 @@ function extractKagiErrorMessage(payload: unknown): string | null {
|
||||
}
|
||||
}
|
||||
|
||||
if (typeof record.error === "string" && record.error.trim().length > 0) {
|
||||
return record.error.trim();
|
||||
}
|
||||
|
||||
if (Array.isArray(record.error)) {
|
||||
for (const entry of record.error) {
|
||||
for (const errors of [record.error, record.errors]) {
|
||||
if (typeof errors === "string" && errors.trim().length > 0) {
|
||||
return errors.trim();
|
||||
}
|
||||
if (!Array.isArray(errors)) continue;
|
||||
for (const entry of errors) {
|
||||
if (!entry || typeof entry !== "object") continue;
|
||||
const e = entry as Record<string, unknown>;
|
||||
for (const value of [e.message, e.msg]) {
|
||||
if (typeof value === "string" && value.trim().length > 0) {
|
||||
return value.trim();
|
||||
for (const value of [e.message, e.msg, e.code]) {
|
||||
if (
|
||||
(typeof value === "string" && value.trim().length > 0) ||
|
||||
(typeof value === "number" && Number.isFinite(value))
|
||||
) {
|
||||
return String(value).trim();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -147,6 +151,34 @@ function parseKagiErrorResponse(statusCode: number, responseText: string): KagiA
|
||||
}
|
||||
}
|
||||
|
||||
function parseKagiSuccessResponse(statusCode: number, responseText: string): KagiSearchResponse {
|
||||
let payload: unknown;
|
||||
try {
|
||||
payload = JSON.parse(responseText);
|
||||
} catch {
|
||||
throw new KagiApiError("Kagi API returned an invalid response: invalid JSON", statusCode);
|
||||
}
|
||||
if (!payload || typeof payload !== "object" || Array.isArray(payload)) {
|
||||
throw new KagiApiError("Kagi API returned an invalid response: expected an object envelope", statusCode);
|
||||
}
|
||||
|
||||
const record = payload as Record<string, unknown>;
|
||||
const errorMessage = extractKagiErrorMessage(payload);
|
||||
if (errorMessage && (record.error !== undefined || record.errors !== undefined)) {
|
||||
const errors = Array.isArray(record.error) ? record.error : Array.isArray(record.errors) ? record.errors : [];
|
||||
const first = errors[0];
|
||||
const code =
|
||||
first && typeof first === "object" && typeof (first as Record<string, unknown>).code === "number"
|
||||
? ((first as Record<string, unknown>).code as number)
|
||||
: statusCode;
|
||||
throw createKagiApiError(code, errorMessage);
|
||||
}
|
||||
if (record.data !== undefined && (!record.data || typeof record.data !== "object" || Array.isArray(record.data))) {
|
||||
throw new KagiApiError("Kagi API returned an invalid response: expected data to be an object", statusCode);
|
||||
}
|
||||
return payload as KagiSearchResponse;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Public API
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -216,23 +248,40 @@ function buildRequestBody(query: string, options: KagiSearchOptions): KagiSearch
|
||||
return req;
|
||||
}
|
||||
|
||||
/** Push every item in a result bucket as a source, with an optional title tag. */
|
||||
function collectSources(sources: KagiSearchSource[], items: KagiSearchResultItem[] | undefined, tag?: string): void {
|
||||
if (!items) return;
|
||||
for (const item of items) {
|
||||
function firstNonEmptyString(...values: unknown[]): string | undefined {
|
||||
for (const value of values) {
|
||||
if (typeof value === "string" && value.trim().length > 0) return value.trim();
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/** Push every valid item in a result bucket as a source, with an optional title tag. */
|
||||
function collectSources(sources: KagiSearchSource[], items: unknown, tag?: string): void {
|
||||
if (!Array.isArray(items)) return;
|
||||
for (const value of items) {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) continue;
|
||||
const item = value as Record<string, unknown>;
|
||||
const url = firstNonEmptyString(item.url, item.href, item.link);
|
||||
if (!url) continue;
|
||||
const title = firstNonEmptyString(item.title, item.name) ?? url;
|
||||
sources.push({
|
||||
title: tag ? `${tag} ${item.title}` : item.title,
|
||||
url: item.url,
|
||||
snippet: item.snippet,
|
||||
publishedDate: item.time,
|
||||
title: tag ? `${tag} ${title}` : title,
|
||||
url,
|
||||
snippet: firstNonEmptyString(item.snippet, item.description, item.summary),
|
||||
publishedDate: firstNonEmptyString(item.time),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/** Pull a related/adjacent question from an item's props or fall back to title. */
|
||||
function questionOf(item: KagiSearchResultItem): string | undefined {
|
||||
const q = item.props?.question ?? item.props?.query ?? item.title;
|
||||
return typeof q === "string" && q.length > 0 ? q : undefined;
|
||||
function questionOf(value: unknown): string | undefined {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) return undefined;
|
||||
const item = value as Record<string, unknown>;
|
||||
const props =
|
||||
item.props && typeof item.props === "object" && !Array.isArray(item.props)
|
||||
? (item.props as Record<string, unknown>)
|
||||
: undefined;
|
||||
return firstNonEmptyString(props?.question, props?.query, item.title);
|
||||
}
|
||||
|
||||
export async function searchWithKagi(
|
||||
@@ -269,11 +318,7 @@ export async function searchWithKagi(
|
||||
},
|
||||
);
|
||||
|
||||
const payload = (await response.json()) as KagiSearchResponse;
|
||||
if (payload.error && payload.error.length > 0) {
|
||||
const first = payload.error[0];
|
||||
throw createKagiApiError(first.code ?? response.status, extractKagiErrorMessage(payload) ?? first.message);
|
||||
}
|
||||
const payload = parseKagiSuccessResponse(response.status, await response.text());
|
||||
|
||||
const data = payload.data;
|
||||
const sources: KagiSearchSource[] = [];
|
||||
@@ -284,17 +329,30 @@ export async function searchWithKagi(
|
||||
collectSources(sources, data?.news, "[News]");
|
||||
collectSources(sources, data?.infobox, "[Info]");
|
||||
|
||||
for (const item of data?.adjacent_question ?? []) {
|
||||
const q = questionOf(item);
|
||||
if (q) relatedQuestions.push(q);
|
||||
const adjacentQuestions: unknown = data?.adjacent_question;
|
||||
if (Array.isArray(adjacentQuestions)) {
|
||||
for (const item of adjacentQuestions) {
|
||||
const question = questionOf(item);
|
||||
if (question) relatedQuestions.push(question);
|
||||
}
|
||||
}
|
||||
for (const item of data?.related_search ?? []) {
|
||||
const q = questionOf(item);
|
||||
if (q) relatedQuestions.push(q);
|
||||
const relatedSearches: unknown = data?.related_search;
|
||||
if (Array.isArray(relatedSearches)) {
|
||||
for (const item of relatedSearches) {
|
||||
const question = questionOf(item);
|
||||
if (question) relatedQuestions.push(question);
|
||||
}
|
||||
}
|
||||
|
||||
const directAnswer = data?.direct_answer?.[0];
|
||||
const answer = directAnswer ? (directAnswer.snippet ?? directAnswer.title) : undefined;
|
||||
const directAnswers: unknown = data?.direct_answer;
|
||||
const directAnswer = Array.isArray(directAnswers) ? directAnswers[0] : undefined;
|
||||
const answer =
|
||||
directAnswer && typeof directAnswer === "object" && !Array.isArray(directAnswer)
|
||||
? firstNonEmptyString(
|
||||
(directAnswer as Record<string, unknown>).snippet,
|
||||
(directAnswer as Record<string, unknown>).title,
|
||||
)
|
||||
: undefined;
|
||||
|
||||
return {
|
||||
requestId: payload.meta?.trace ?? payload.meta?.id ?? "",
|
||||
|
||||
@@ -146,6 +146,18 @@ export function parseParallelErrorResponse(statusCode: number, responseText: str
|
||||
}
|
||||
}
|
||||
|
||||
export async function parseParallelJsonResponse(
|
||||
response: Response,
|
||||
operation: "search" | "extract",
|
||||
): Promise<unknown> {
|
||||
try {
|
||||
return await response.json();
|
||||
} catch (err) {
|
||||
const detail = err instanceof Error ? err.message : String(err);
|
||||
throw new ParallelApiError(`Parallel ${operation} returned invalid JSON: ${detail}`);
|
||||
}
|
||||
}
|
||||
|
||||
function getAuthHeaders(apiKey: string): {
|
||||
Accept: string;
|
||||
"Content-Type": string;
|
||||
@@ -316,7 +328,7 @@ export async function searchWithParallel(
|
||||
throw parseParallelErrorResponse(response.status, await response.text());
|
||||
}
|
||||
|
||||
const payload: unknown = await response.json();
|
||||
const payload = await parseParallelJsonResponse(response, "search");
|
||||
return parseParallelSearchPayload(payload);
|
||||
}
|
||||
|
||||
@@ -349,6 +361,6 @@ export async function extractWithParallel(
|
||||
throw parseParallelErrorResponse(response.status, await response.text());
|
||||
}
|
||||
|
||||
const payload: unknown = await response.json();
|
||||
const payload = await parseParallelJsonResponse(response, "extract");
|
||||
return parseExtractPayload(payload);
|
||||
}
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
* Calls Brave's web search REST API and maps results into the unified
|
||||
* SearchResponse shape used by the web search tool.
|
||||
*/
|
||||
import { type AuthStorage, type FetchImpl, getEnvApiKey } from "@oh-my-pi/pi-ai";
|
||||
import { type ApiKey, type AuthStorage, type FetchImpl, getEnvApiKey, withAuth } from "@oh-my-pi/pi-ai";
|
||||
import type { SearchResponse, SearchSource } from "../../../web/search/types";
|
||||
import { SearchProviderError } from "../../../web/search/types";
|
||||
import type { QuerySyntax, StructuredQuery } from "../query";
|
||||
@@ -17,6 +17,9 @@ import { classifyProviderHttpError, withHardTimeout } from "./utils";
|
||||
const BRAVE_SEARCH_URL = "https://api.search.brave.com/res/v1/web/search";
|
||||
const DEFAULT_NUM_RESULTS = 10;
|
||||
const MAX_NUM_RESULTS = 20;
|
||||
const MAX_QUERY_CHARACTERS = 500;
|
||||
const MAX_RESPONSE_BYTES = 2 * 1024 * 1024;
|
||||
const MAX_ERROR_BYTES = 8 * 1024;
|
||||
|
||||
const RECENCY_MAP: Record<"day" | "week" | "month" | "year", "pd" | "pw" | "pm" | "py"> = {
|
||||
day: "pd",
|
||||
@@ -51,62 +54,122 @@ export interface BraveSearchParams {
|
||||
num_results?: number;
|
||||
recency?: "day" | "week" | "month" | "year";
|
||||
parsedQuery?: StructuredQuery;
|
||||
/** Two-letter market code, or `ALL`. */
|
||||
country?: string;
|
||||
/** Brave search language code, such as `en` or `zh-hans`. */
|
||||
search_lang?: string;
|
||||
safesearch?: "off" | "moderate" | "strict";
|
||||
authStorage: AuthStorage;
|
||||
sessionId?: string;
|
||||
signal?: AbortSignal;
|
||||
timeoutMs?: number;
|
||||
fetch?: FetchImpl;
|
||||
}
|
||||
|
||||
interface BraveSearchResult {
|
||||
title?: string | null;
|
||||
url?: string | null;
|
||||
description?: string | null;
|
||||
age?: string | null;
|
||||
extra_snippets?: string[] | null;
|
||||
}
|
||||
|
||||
interface BraveSearchResponse {
|
||||
web?: {
|
||||
results?: BraveSearchResult[];
|
||||
};
|
||||
web?: unknown;
|
||||
}
|
||||
|
||||
/** Find BRAVE_API_KEY from environment or .env files. */
|
||||
export function findApiKey(): string | null {
|
||||
return getEnvApiKey("brave") ?? null;
|
||||
function normalizeText(value: unknown, maxLength: number): string | undefined {
|
||||
if (typeof value !== "string") return undefined;
|
||||
const text = value
|
||||
.replace(/<[^>]*>/g, " ")
|
||||
.replace(/\s+/g, " ")
|
||||
.trim();
|
||||
if (!text) return undefined;
|
||||
return text.length <= maxLength ? text : `${text.slice(0, maxLength - 1)}…`;
|
||||
}
|
||||
|
||||
function buildSnippet(result: BraveSearchResult): string | undefined {
|
||||
const snippets: string[] = [];
|
||||
function normalizeUrl(value: unknown): string | undefined {
|
||||
if (typeof value !== "string" || value.length > 2048) return undefined;
|
||||
try {
|
||||
const url = new URL(value);
|
||||
if (url.protocol !== "http:" && url.protocol !== "https:") return undefined;
|
||||
return url.toString();
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
if (result.description?.trim()) {
|
||||
snippets.push(result.description.trim());
|
||||
function webResults(response: BraveSearchResponse): readonly unknown[] {
|
||||
if (typeof response.web !== "object" || response.web === null || !("results" in response.web)) return [];
|
||||
return Array.isArray(response.web.results) ? response.web.results : [];
|
||||
}
|
||||
|
||||
async function readLimitedText(response: Response, maxBytes: number, truncate = false): Promise<string> {
|
||||
if (!response.body) return "";
|
||||
const reader = response.body.getReader();
|
||||
let buffer = new Uint8Array(Math.min(maxBytes, 64 * 1024));
|
||||
let bytes = 0;
|
||||
|
||||
try {
|
||||
for (;;) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
const accepted = Math.min(value.byteLength, maxBytes - bytes);
|
||||
const nextBytes = bytes + accepted;
|
||||
if (nextBytes > buffer.byteLength) {
|
||||
const grown = new Uint8Array(Math.min(maxBytes, Math.max(nextBytes, buffer.byteLength * 2)));
|
||||
grown.set(buffer.subarray(0, bytes));
|
||||
buffer = grown;
|
||||
}
|
||||
buffer.set(value.subarray(0, accepted), bytes);
|
||||
bytes = nextBytes;
|
||||
if (accepted < value.byteLength) {
|
||||
await reader.cancel().catch(() => undefined);
|
||||
if (!truncate) throw new SearchProviderError("brave", "Brave API response exceeded 2 MiB", 500);
|
||||
break;
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
reader.releaseLock();
|
||||
}
|
||||
|
||||
if (Array.isArray(result.extra_snippets)) {
|
||||
for (const snippet of result.extra_snippets) {
|
||||
if (!snippet?.trim()) continue;
|
||||
if (snippets.includes(snippet.trim())) continue;
|
||||
snippets.push(snippet.trim());
|
||||
return new TextDecoder().decode(buffer.subarray(0, bytes));
|
||||
}
|
||||
|
||||
function buildSnippet(result: object): string | undefined {
|
||||
const snippets = new Set<string>();
|
||||
const description = normalizeText("description" in result ? result.description : undefined, 8_000);
|
||||
if (description) snippets.add(description);
|
||||
|
||||
const extras = "extra_snippets" in result ? result.extra_snippets : undefined;
|
||||
if (Array.isArray(extras)) {
|
||||
for (const value of extras) {
|
||||
const snippet = normalizeText(value, 8_000);
|
||||
if (snippet) snippets.add(snippet);
|
||||
}
|
||||
}
|
||||
|
||||
return snippets.length > 0 ? snippets.join("\n") : undefined;
|
||||
const combined = [...snippets].join("\n");
|
||||
return combined ? (combined.length <= 8_000 ? combined : `${combined.slice(0, 7_999)}…`) : undefined;
|
||||
}
|
||||
|
||||
async function callBraveSearch(
|
||||
apiKey: string,
|
||||
params: BraveSearchParams,
|
||||
): Promise<{ response: BraveSearchResponse; requestId?: string }> {
|
||||
const numResults = clampNumResults(params.num_results, DEFAULT_NUM_RESULTS, MAX_NUM_RESULTS);
|
||||
const numResults = Math.floor(clampNumResults(params.num_results, DEFAULT_NUM_RESULTS, MAX_NUM_RESULTS));
|
||||
const parsed = params.parsedQuery ?? parseSearchQuery(params.query);
|
||||
const query = parsed.hasDirectives ? formatQuery(parsed, BRAVE_QUERY_SYNTAX) : params.query;
|
||||
if (query.length > MAX_QUERY_CHARACTERS) {
|
||||
throw new SearchProviderError(
|
||||
"brave",
|
||||
`Brave search queries cannot exceed ${MAX_QUERY_CHARACTERS} characters`,
|
||||
400,
|
||||
);
|
||||
}
|
||||
const url = new URL(BRAVE_SEARCH_URL);
|
||||
url.searchParams.set("q", parsed.hasDirectives ? formatQuery(parsed, BRAVE_QUERY_SYNTAX) : params.query);
|
||||
url.searchParams.set("q", query);
|
||||
url.searchParams.set("count", String(numResults));
|
||||
url.searchParams.set("extra_snippets", "true");
|
||||
url.searchParams.set("text_decorations", "false");
|
||||
url.searchParams.set("safesearch", params.safesearch ?? "moderate");
|
||||
if (params.country) url.searchParams.set("country", params.country.toUpperCase());
|
||||
if (params.search_lang) url.searchParams.set("search_lang", params.search_lang);
|
||||
const freshness = braveFreshness(parsed, params.recency);
|
||||
if (freshness) {
|
||||
url.searchParams.set("freshness", freshness);
|
||||
}
|
||||
if (freshness) url.searchParams.set("freshness", freshness);
|
||||
|
||||
const fetchImpl = params.fetch ?? fetch;
|
||||
const response = await fetchImpl(url, {
|
||||
@@ -118,36 +181,47 @@ async function callBraveSearch(
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
const errorText = await readLimitedText(response, MAX_ERROR_BYTES, true);
|
||||
const classified = classifyProviderHttpError("brave", response.status, errorText);
|
||||
if (classified) throw classified;
|
||||
throw new SearchProviderError("brave", `Brave API error (${response.status}): ${errorText}`, response.status);
|
||||
}
|
||||
|
||||
const data = (await response.json()) as BraveSearchResponse;
|
||||
const raw = await readLimitedText(response, MAX_RESPONSE_BYTES);
|
||||
let data: BraveSearchResponse;
|
||||
try {
|
||||
data = JSON.parse(raw) as BraveSearchResponse;
|
||||
} catch {
|
||||
throw new SearchProviderError("brave", "Brave API returned invalid JSON", 500);
|
||||
}
|
||||
const requestId = response.headers.get("x-request-id") ?? response.headers.get("request-id") ?? undefined;
|
||||
return { response: data, requestId };
|
||||
}
|
||||
|
||||
/** Execute Brave web search. */
|
||||
export async function searchBrave(params: BraveSearchParams): Promise<SearchResponse> {
|
||||
const numResults = clampNumResults(params.num_results, DEFAULT_NUM_RESULTS, MAX_NUM_RESULTS);
|
||||
const apiKey = findApiKey();
|
||||
if (!apiKey) {
|
||||
throw new Error("BRAVE_API_KEY not found. Set it in environment or .env file.");
|
||||
}
|
||||
|
||||
const { response, requestId } = await callBraveSearch(apiKey, params);
|
||||
const numResults = Math.floor(clampNumResults(params.num_results, DEFAULT_NUM_RESULTS, MAX_NUM_RESULTS));
|
||||
const keyOrResolver: ApiKey = params.authStorage.resolver("brave", {
|
||||
sessionId: params.sessionId,
|
||||
});
|
||||
const { response, requestId } = await withAuth(keyOrResolver, key => callBraveSearch(key, params), {
|
||||
signal: params.signal,
|
||||
missingKeyMessage:
|
||||
'Brave credentials not found. Set BRAVE_API_KEY or configure an API key for provider "brave".',
|
||||
});
|
||||
const sources: SearchSource[] = [];
|
||||
|
||||
for (const result of response.web?.results ?? []) {
|
||||
if (!result.url) continue;
|
||||
for (const result of webResults(response)) {
|
||||
if (typeof result !== "object" || result === null) continue;
|
||||
const url = normalizeUrl("url" in result ? result.url : undefined);
|
||||
if (!url) continue;
|
||||
const publishedDate = normalizeText("age" in result ? result.age : undefined, 100);
|
||||
sources.push({
|
||||
title: result.title ?? result.url,
|
||||
url: result.url,
|
||||
title: normalizeText("title" in result ? result.title : undefined, 300) ?? url,
|
||||
url,
|
||||
snippet: buildSnippet(result),
|
||||
publishedDate: result.age ?? undefined,
|
||||
ageSeconds: dateToAgeSeconds(result.age),
|
||||
publishedDate,
|
||||
ageSeconds: dateToAgeSeconds(publishedDate),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -155,6 +229,7 @@ export async function searchBrave(params: BraveSearchParams): Promise<SearchResp
|
||||
provider: "brave",
|
||||
sources: sources.slice(0, numResults),
|
||||
requestId,
|
||||
authMode: "api_key",
|
||||
};
|
||||
}
|
||||
|
||||
@@ -163,8 +238,8 @@ export class BraveProvider extends SearchProvider {
|
||||
readonly id = "brave";
|
||||
readonly label = "Brave";
|
||||
|
||||
isAvailable(_authStorage: AuthStorage): boolean {
|
||||
return !!findApiKey();
|
||||
isAvailable(authStorage: AuthStorage): boolean {
|
||||
return authStorage.hasAuth("brave") || !!getEnvApiKey("brave");
|
||||
}
|
||||
|
||||
search(params: SearchParams): Promise<SearchResponse> {
|
||||
@@ -173,6 +248,8 @@ export class BraveProvider extends SearchProvider {
|
||||
num_results: params.numSearchResults ?? params.limit,
|
||||
recency: params.recency,
|
||||
parsedQuery: params.parsedQuery,
|
||||
authStorage: params.authStorage,
|
||||
sessionId: params.sessionId,
|
||||
signal: params.signal,
|
||||
timeoutMs: params.timeoutMs,
|
||||
fetch: params.fetch,
|
||||
|
||||
@@ -151,6 +151,13 @@ export interface CodexSearchParams {
|
||||
}
|
||||
|
||||
/** Codex API response structure */
|
||||
interface CodexWebSearchSource {
|
||||
url?: string;
|
||||
source_website_url?: string;
|
||||
title?: string;
|
||||
caption?: string;
|
||||
}
|
||||
|
||||
interface CodexResponseItem {
|
||||
type: string;
|
||||
id?: string;
|
||||
@@ -161,6 +168,9 @@ interface CodexResponseItem {
|
||||
arguments?: string;
|
||||
content?: CodexContentPart[];
|
||||
summary?: Array<{ type: string; text: string }>;
|
||||
action?: { sources?: CodexWebSearchSource[] };
|
||||
sources?: CodexWebSearchSource[];
|
||||
results?: CodexWebSearchSource[];
|
||||
}
|
||||
|
||||
interface CodexContentPart {
|
||||
@@ -221,12 +231,45 @@ function isImagePlaceholderAnswer(text: string): boolean {
|
||||
return IMAGE_PLACEHOLDER_ANSWERS.has(normalized);
|
||||
}
|
||||
|
||||
function addSource(sources: SearchSource[], source: SearchSource): void {
|
||||
if (!sources.some(existing => existing.url === source.url)) {
|
||||
sources.push(source);
|
||||
function cleanSourceUrl(rawUrl: string): string {
|
||||
try {
|
||||
const url = new URL(rawUrl);
|
||||
if (url.searchParams.get("utm_source") === "openai") {
|
||||
url.searchParams.delete("utm_source");
|
||||
}
|
||||
return url.toString();
|
||||
} catch {
|
||||
return rawUrl.replace(/[?&]utm_source=openai$/u, "");
|
||||
}
|
||||
}
|
||||
|
||||
function addSource(sources: SearchSource[], source: SearchSource): void {
|
||||
const normalizedSource = { ...source, url: cleanSourceUrl(source.url) };
|
||||
const existing = sources.find(candidate => candidate.url === normalizedSource.url);
|
||||
if (!existing) {
|
||||
sources.push(normalizedSource);
|
||||
return;
|
||||
}
|
||||
if (existing.title === existing.url && normalizedSource.title !== normalizedSource.url) {
|
||||
existing.title = normalizedSource.title;
|
||||
}
|
||||
if (!existing.snippet && normalizedSource.snippet) {
|
||||
existing.snippet = normalizedSource.snippet;
|
||||
}
|
||||
}
|
||||
|
||||
function extractCitationSnippet(text: string, start: number | undefined, end: number | undefined): string | undefined {
|
||||
if (start === undefined || end === undefined || !text) return undefined;
|
||||
const before = Math.max(0, start - 100);
|
||||
const after = Math.min(text.length, end + 100);
|
||||
const snippet = text
|
||||
.slice(before, after)
|
||||
.replace(/\[([^\]]*)\]\([^)]*\)/g, "$1")
|
||||
.trim();
|
||||
if (!snippet) return undefined;
|
||||
return snippet.length > 300 ? `${snippet.slice(0, 297)}...` : snippet;
|
||||
}
|
||||
|
||||
function countCharacter(text: string, target: string): number {
|
||||
let count = 0;
|
||||
for (const char of text) {
|
||||
@@ -428,6 +471,15 @@ function extractCodexSseError(rawEvent: Record<string, unknown>): { code: string
|
||||
return { code, message };
|
||||
}
|
||||
|
||||
function classifyCodexSseErrorStatus(code: string, message: string): number {
|
||||
const detail = `${code} ${message}`.toLowerCase();
|
||||
if (/rate[- ]?limit|too many requests|quota|\b429\b/u.test(detail)) return 429;
|
||||
if (/unauthori[sz]ed|\b401\b/u.test(detail)) return 401;
|
||||
if (/forbidden|\b403\b/u.test(detail)) return 403;
|
||||
if (/timeout|timed out/u.test(detail)) return 504;
|
||||
return 500;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calls the Codex Responses API with web search tool enabled.
|
||||
* The caller provides the exact model id to send; retry / fallback policy
|
||||
@@ -455,6 +507,8 @@ async function callCodexSearch(
|
||||
model: requestedModel,
|
||||
stream: true,
|
||||
store: false,
|
||||
include: ["web_search_call.action.sources"],
|
||||
parallel_tool_calls: true,
|
||||
input: [
|
||||
{
|
||||
type: "message",
|
||||
@@ -510,7 +564,11 @@ async function callCodexSearch(
|
||||
webSearchInvoked = true;
|
||||
}
|
||||
|
||||
if (eventType === "response.output_text.delta") {
|
||||
if (eventType === "response.created") {
|
||||
const resp = (rawEvent as { response?: CodexResponse }).response;
|
||||
if (resp?.id) requestId = resp.id;
|
||||
if (resp?.model) model = resp.model;
|
||||
} else if (eventType === "response.output_text.delta") {
|
||||
const delta = typeof rawEvent.delta === "string" ? rawEvent.delta : "";
|
||||
if (delta) {
|
||||
streamedAnswerParts.push(delta);
|
||||
@@ -518,7 +576,20 @@ async function callCodexSearch(
|
||||
} else if (eventType === "response.output_item.done") {
|
||||
const item = rawEvent.item as CodexResponseItem | undefined;
|
||||
if (!item) continue;
|
||||
if (item.type === "web_search_call") webSearchInvoked = true;
|
||||
if (item.type === "web_search_call") {
|
||||
webSearchInvoked = true;
|
||||
const sourceGroups = [item.action?.sources, item.sources, item.results];
|
||||
for (const group of sourceGroups) {
|
||||
for (const source of group ?? []) {
|
||||
const url = source.url ?? source.source_website_url;
|
||||
if (!url) continue;
|
||||
addSource(sources, {
|
||||
title: source.title ?? source.caption ?? url,
|
||||
url,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Handle text message content and extract sources from annotations
|
||||
if (item.type === "message" && item.content) {
|
||||
@@ -530,8 +601,11 @@ async function callCodexSearch(
|
||||
if (part.annotations) {
|
||||
for (const annotation of part.annotations) {
|
||||
if (annotation.type === "url_citation" && annotation.url) {
|
||||
// Deduplicate by URL
|
||||
addSource(sources, { title: annotation.title ?? annotation.url, url: annotation.url });
|
||||
addSource(sources, {
|
||||
title: annotation.title ?? annotation.url,
|
||||
url: annotation.url,
|
||||
snippet: extractCitationSnippet(part.text, annotation.start_index, annotation.end_index),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -563,13 +637,17 @@ async function callCodexSearch(
|
||||
}
|
||||
} else if (eventType === "error") {
|
||||
const { code, message } = extractCodexSseError(rawEvent);
|
||||
throw new SearchProviderError("codex", `Codex error (${code}): ${message || "Unknown error"}`, 500);
|
||||
throw new SearchProviderError(
|
||||
"codex",
|
||||
`Codex error (${code}): ${message || "Unknown error"}`,
|
||||
classifyCodexSseErrorStatus(code, message),
|
||||
);
|
||||
} else if (eventType === "response.failed") {
|
||||
const { code, message } = extractCodexSseError(rawEvent);
|
||||
const detail = code
|
||||
? `Codex request failed (${code}): ${message || "Request failed"}`
|
||||
: `Codex request failed: ${message || "Request failed"}`;
|
||||
throw new SearchProviderError("codex", detail, 500);
|
||||
throw new SearchProviderError("codex", detail, classifyCodexSseErrorStatus(code, message));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -19,6 +19,9 @@ import { SearchProvider } from "./base";
|
||||
import { classifyProviderHttpError, withHardTimeout } from "./utils";
|
||||
|
||||
const EXA_API_URL = "https://api.exa.ai/search";
|
||||
const EXA_MCP_URL = "https://mcp.exa.ai/mcp";
|
||||
const EXA_MCP_SOURCE = "oh-my-pi";
|
||||
const MAX_EXA_SNIPPET_CHARS = 500;
|
||||
const DEFAULT_EXA_SEARCH_DELAY_MS = getDefault("exa.searchDelayMs");
|
||||
|
||||
let nextExaSearchRequestAt = 0;
|
||||
@@ -329,13 +332,20 @@ async function callExaSearch(apiKey: string, params: ExaSearchParams): Promise<E
|
||||
return response.json() as Promise<ExaSearchResponse>;
|
||||
}
|
||||
function buildExaMcpArgs(params: ExaSearchParams): Record<string, unknown> {
|
||||
const args: Record<string, unknown> = { query: params.query };
|
||||
if (params.num_results !== undefined) args.num_results = params.num_results;
|
||||
if (params.type !== undefined) args.type = params.type;
|
||||
if (params.include_domains !== undefined) args.include_domains = params.include_domains;
|
||||
if (params.exclude_domains !== undefined) args.exclude_domains = params.exclude_domains;
|
||||
if (params.start_published_date !== undefined) args.start_published_date = params.start_published_date;
|
||||
if (params.end_published_date !== undefined) args.end_published_date = params.end_published_date;
|
||||
const queryParts = [params.query];
|
||||
for (const domain of params.include_domains ?? []) {
|
||||
const trimmed = domain.trim();
|
||||
if (trimmed) queryParts.push(`site:${trimmed}`);
|
||||
}
|
||||
for (const domain of params.exclude_domains ?? []) {
|
||||
const trimmed = domain.trim();
|
||||
if (trimmed) queryParts.push(`-site:${trimmed}`);
|
||||
}
|
||||
if (params.start_published_date) queryParts.push(`after:${params.start_published_date}`);
|
||||
if (params.end_published_date) queryParts.push(`before:${params.end_published_date}`);
|
||||
|
||||
const args: Record<string, unknown> = { query: queryParts.join(" ") };
|
||||
if (params.num_results !== undefined) args.numResults = params.num_results;
|
||||
return args;
|
||||
}
|
||||
|
||||
@@ -346,11 +356,12 @@ async function callExaMcpSearch(params: ExaSearchParams): Promise<ExaSearchRespo
|
||||
query.set("tools", "web_search_exa");
|
||||
const fetchImpl = params.fetch ?? fetch;
|
||||
await waitForExaSearchSlot(params.signal);
|
||||
const response = await fetchImpl(`https://mcp.exa.ai/mcp?${query.toString()}`, {
|
||||
const response = await fetchImpl(`${EXA_MCP_URL}?${query.toString()}`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
Accept: "application/json, text/event-stream",
|
||||
"x-exa-source": EXA_MCP_SOURCE,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
jsonrpc: "2.0",
|
||||
@@ -364,11 +375,26 @@ async function callExaMcpSearch(params: ExaSearchParams): Promise<ExaSearchRespo
|
||||
signal: withHardTimeout(params.signal, params.timeoutMs),
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`MCP request failed: ${response.status} ${response.statusText}`);
|
||||
const errorText = await response.text();
|
||||
const classified = classifyProviderHttpError("exa", response.status, errorText);
|
||||
if (classified) throw classified;
|
||||
if (response.status === 429) {
|
||||
throw new SearchProviderError(
|
||||
"exa",
|
||||
"exa: MCP rate limit reached (429); configure an Exa API key for higher limits",
|
||||
response.status,
|
||||
);
|
||||
}
|
||||
throw new SearchProviderError(
|
||||
"exa",
|
||||
`Exa MCP request failed (${response.status}): ${errorText}`,
|
||||
response.status,
|
||||
);
|
||||
}
|
||||
const mcpResponse = parseSSE(await response.text()) as {
|
||||
result?: {
|
||||
content?: Array<{ type: string; text?: string }>;
|
||||
isError?: boolean;
|
||||
};
|
||||
error?: {
|
||||
code: number;
|
||||
@@ -381,6 +407,12 @@ async function callExaMcpSearch(params: ExaSearchParams): Promise<ExaSearchRespo
|
||||
if (mcpResponse.error) {
|
||||
throw new Error(`MCP error: ${mcpResponse.error.message}`);
|
||||
}
|
||||
if (mcpResponse.result?.isError) {
|
||||
const message = mcpResponse.result.content
|
||||
?.find(item => item.type === "text" && typeof item.text === "string")
|
||||
?.text?.trim();
|
||||
throw new SearchProviderError("exa", message || "Exa MCP returned an error");
|
||||
}
|
||||
const responsePayload = normalizeExaMcpPayload(mcpResponse.result);
|
||||
if (isSearchResponse(responsePayload)) {
|
||||
return responsePayload as ExaSearchResponse;
|
||||
@@ -419,7 +451,10 @@ export async function searchExa(params: ExaSearchParams): Promise<SearchResponse
|
||||
sources.push({
|
||||
title: result.title ?? result.url,
|
||||
url: result.url,
|
||||
snippet: result.summary || result.text || result.highlights?.join(" ") || undefined,
|
||||
snippet: (result.summary || result.text || result.highlights?.join(" ") || undefined)?.slice(
|
||||
0,
|
||||
MAX_EXA_SNIPPET_CHARS,
|
||||
),
|
||||
publishedDate: result.publishedDate ?? undefined,
|
||||
ageSeconds: dateToAgeSeconds(result.publishedDate ?? undefined),
|
||||
author: result.author ?? undefined,
|
||||
|
||||
@@ -20,7 +20,7 @@ import type { SearchParams } from "./base";
|
||||
import { SearchProvider } from "./base";
|
||||
import { classifyProviderHttpError, withHardTimeout } from "./utils";
|
||||
|
||||
const FIRECRAWL_SEARCH_URL = "https://api.firecrawl.dev/v2/search";
|
||||
const FIRECRAWL_DEFAULT_BASE_URL = "https://api.firecrawl.dev/v2";
|
||||
const DEFAULT_NUM_RESULTS = 10;
|
||||
const MAX_NUM_RESULTS = 100;
|
||||
|
||||
@@ -30,6 +30,28 @@ const RECENCY_TBS: Record<NonNullable<SearchParams["recency"]>, string> = {
|
||||
month: "qdr:m",
|
||||
year: "qdr:y",
|
||||
};
|
||||
function resolveSearchUrl(): string {
|
||||
const configured = process.env.FIRECRAWL_BASE_URL ?? process.env.FIRECRAWL_API_URL;
|
||||
if (!configured?.trim()) return `${FIRECRAWL_DEFAULT_BASE_URL}/search`;
|
||||
let url: URL;
|
||||
try {
|
||||
url = new URL(configured.trim());
|
||||
} catch {
|
||||
throw new Error("Invalid Firecrawl base URL: expected an HTTP or HTTPS URL");
|
||||
}
|
||||
if (url.protocol !== "http:" && url.protocol !== "https:") {
|
||||
throw new Error("Invalid Firecrawl base URL: expected an HTTP or HTTPS URL");
|
||||
}
|
||||
if (url.username || url.password) {
|
||||
throw new Error("Invalid Firecrawl base URL: URL credentials are not allowed");
|
||||
}
|
||||
url.search = "";
|
||||
url.hash = "";
|
||||
url.pathname = url.pathname.replace(/\/+$/, "");
|
||||
if (!/\/v[12]$/i.test(url.pathname)) url.pathname += "/v2";
|
||||
url.pathname += "/search";
|
||||
return url.toString();
|
||||
}
|
||||
|
||||
export interface FirecrawlSearchParams {
|
||||
query: string;
|
||||
@@ -46,14 +68,23 @@ interface FirecrawlWebResult {
|
||||
title?: string | null;
|
||||
url?: string | null;
|
||||
description?: string | null;
|
||||
snippet?: string | null;
|
||||
markdown?: string | null;
|
||||
}
|
||||
|
||||
interface FirecrawlSearchResponse {
|
||||
success?: boolean;
|
||||
error?: string | null;
|
||||
id?: string | null;
|
||||
data?: {
|
||||
web?: FirecrawlWebResult[] | null;
|
||||
} | null;
|
||||
data?:
|
||||
| FirecrawlWebResult[]
|
||||
| {
|
||||
web?: FirecrawlWebResult[] | null;
|
||||
news?: FirecrawlWebResult[] | null;
|
||||
images?: FirecrawlWebResult[] | null;
|
||||
}
|
||||
| null;
|
||||
results?: FirecrawlWebResult[] | null;
|
||||
}
|
||||
|
||||
/** Resolve Firecrawl API key through the shared auth storage pipeline. */
|
||||
@@ -88,7 +119,7 @@ async function callFirecrawlSearch(
|
||||
if (apiKey) {
|
||||
headers.Authorization = `Bearer ${apiKey}`;
|
||||
}
|
||||
const response = await (params.fetch ?? fetch)(FIRECRAWL_SEARCH_URL, {
|
||||
const response = await (params.fetch ?? fetch)(resolveSearchUrl(), {
|
||||
method: "POST",
|
||||
headers,
|
||||
body: JSON.stringify(buildRequestBody(params)),
|
||||
@@ -106,7 +137,11 @@ async function callFirecrawlSearch(
|
||||
);
|
||||
}
|
||||
|
||||
return (await response.json()) as FirecrawlSearchResponse;
|
||||
const data = (await response.json()) as FirecrawlSearchResponse;
|
||||
if (data.success === false) {
|
||||
throw new SearchProviderError("firecrawl", data.error?.trim() || "Firecrawl request failed");
|
||||
}
|
||||
return data;
|
||||
}
|
||||
|
||||
/** ISO `YYYY-MM-DD` to Google `MM/DD/YYYY` for `tbs=cdr` custom date ranges. */
|
||||
@@ -128,6 +163,11 @@ function buildDateTbs(parsed: StructuredQuery): string | undefined {
|
||||
return parts.join(",");
|
||||
}
|
||||
|
||||
function getWebResults(data: FirecrawlSearchResponse): FirecrawlWebResult[] {
|
||||
if (Array.isArray(data.data)) return data.data;
|
||||
if (data.data && Array.isArray(data.data.web)) return data.data.web;
|
||||
return data.results ?? [];
|
||||
}
|
||||
/** Execute Firecrawl web search. */
|
||||
export async function searchFirecrawl(params: SearchParams): Promise<SearchResponse> {
|
||||
const parsed = params.parsedQuery ?? parseSearchQuery(params.query);
|
||||
@@ -169,12 +209,12 @@ export async function searchFirecrawl(params: SearchParams): Promise<SearchRespo
|
||||
|
||||
const sources: SearchSource[] = [];
|
||||
|
||||
for (const result of data.data?.web ?? []) {
|
||||
for (const result of getWebResults(data)) {
|
||||
if (!result.url) continue;
|
||||
sources.push({
|
||||
title: result.title ?? result.url,
|
||||
url: result.url,
|
||||
snippet: result.description ?? result.markdown ?? undefined,
|
||||
snippet: result.description ?? result.snippet ?? result.markdown ?? undefined,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -192,11 +232,13 @@ export class FirecrawlProvider extends SearchProvider {
|
||||
readonly label = "Firecrawl";
|
||||
|
||||
/**
|
||||
* Auto-chain admission: requires a credential so an unconfigured Firecrawl
|
||||
* doesn't displace other providers that the user has set up with API keys.
|
||||
* Auto-chain admission requires either a credential or an explicitly
|
||||
* configured self-hosted endpoint. Hosted keyless mode remains explicit-only
|
||||
* so it does not displace providers the user configured.
|
||||
*/
|
||||
isAvailable(authStorage: AuthStorage): boolean {
|
||||
return authStorage.hasAuth("firecrawl") || !!getEnvApiKey("firecrawl");
|
||||
const configuredBaseUrl = process.env.FIRECRAWL_BASE_URL ?? process.env.FIRECRAWL_API_URL;
|
||||
return !!configuredBaseUrl?.trim() || authStorage.hasAuth("firecrawl") || !!getEnvApiKey("firecrawl");
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -25,7 +25,9 @@ import { classifyProviderHttpError, withHardTimeout } from "./utils";
|
||||
|
||||
const DEFAULT_ENDPOINT = "https://cloudcode-pa.googleapis.com";
|
||||
const DEVELOPER_API_PROVIDER = "google";
|
||||
const DEVELOPER_API_ENDPOINT = "https://generativelanguage.googleapis.com/v1beta";
|
||||
const CLOUDFLARE_GATEWAY_PROVIDER = "cloudflare-ai-gateway";
|
||||
const DEFAULT_DEVELOPER_API_HOST = "https://generativelanguage.googleapis.com";
|
||||
const DEVELOPER_API_VERSION = "v1beta";
|
||||
const ANTIGRAVITY_DAILY_ENDPOINT = "https://daily-cloudcode-pa.googleapis.com";
|
||||
const ANTIGRAVITY_SANDBOX_ENDPOINT = "https://daily-cloudcode-pa.sandbox.googleapis.com";
|
||||
const ANTIGRAVITY_ENDPOINT_FALLBACKS = [ANTIGRAVITY_DAILY_ENDPOINT, ANTIGRAVITY_SANDBOX_ENDPOINT] as const;
|
||||
@@ -41,6 +43,32 @@ function resolveGeminiSearchModel(configuredModel: string | undefined): string {
|
||||
return model || DEFAULT_MODEL;
|
||||
}
|
||||
|
||||
interface GeminiDeveloperEndpoint {
|
||||
url: string;
|
||||
authProvider: typeof DEVELOPER_API_PROVIDER | typeof CLOUDFLARE_GATEWAY_PROVIDER;
|
||||
isCloudflareGateway: boolean;
|
||||
}
|
||||
|
||||
function resolveGeminiDeveloperEndpoint(): GeminiDeveloperEndpoint {
|
||||
const configuredHost = Bun.env.GOOGLE_GEMINI_BASE_URL?.trim().replace(/\/+$/, "");
|
||||
const host = configuredHost || DEFAULT_DEVELOPER_API_HOST;
|
||||
let parsed: URL;
|
||||
try {
|
||||
parsed = new URL(host);
|
||||
} catch {
|
||||
throw new SearchProviderError("gemini", "GOOGLE_GEMINI_BASE_URL must be a valid absolute URL", 400);
|
||||
}
|
||||
if (parsed.protocol !== "https:" && parsed.protocol !== "http:") {
|
||||
throw new SearchProviderError("gemini", "GOOGLE_GEMINI_BASE_URL must use HTTP or HTTPS", 400);
|
||||
}
|
||||
const isCloudflareGateway = parsed.hostname === "gateway.ai.cloudflare.com";
|
||||
return {
|
||||
url: `${host}/${DEVELOPER_API_VERSION}`,
|
||||
authProvider: isCloudflareGateway ? CLOUDFLARE_GATEWAY_PROVIDER : DEVELOPER_API_PROVIDER,
|
||||
isCloudflareGateway,
|
||||
};
|
||||
}
|
||||
|
||||
const GEMINI_PROVIDERS = ["google-gemini-cli", "google-antigravity"] as const;
|
||||
type GeminiProviderId = (typeof GEMINI_PROVIDERS)[number];
|
||||
|
||||
@@ -292,6 +320,81 @@ async function parseGeminiSearchStream(
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
function isGroundingRedirectUrl(url: string): boolean {
|
||||
try {
|
||||
const parsed = new URL(url);
|
||||
return (
|
||||
parsed.hostname === "vertexaisearch.cloud.google.com" &&
|
||||
parsed.pathname.includes("/grounding-api-redirect")
|
||||
);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function resolveGroundingRedirect(
|
||||
proxyUrl: string,
|
||||
fetchImpl: FetchImpl | undefined,
|
||||
signal: AbortSignal | undefined,
|
||||
): Promise<string> {
|
||||
try {
|
||||
const response = await (fetchImpl ?? fetch)(proxyUrl, {
|
||||
method: "HEAD",
|
||||
redirect: "manual",
|
||||
signal: withHardTimeout(signal, 5000),
|
||||
});
|
||||
const location = response.headers.get("location");
|
||||
if (!location) return proxyUrl;
|
||||
const resolved = new URL(location, proxyUrl);
|
||||
return resolved.protocol === "http:" || resolved.protocol === "https:" ? resolved.toString() : proxyUrl;
|
||||
} catch {
|
||||
return proxyUrl;
|
||||
}
|
||||
}
|
||||
|
||||
async function finalizeGeminiSearchResult(
|
||||
result: GeminiSearchResult,
|
||||
fetchImpl: FetchImpl | undefined,
|
||||
signal: AbortSignal | undefined,
|
||||
): Promise<GeminiSearchResult> {
|
||||
if (!result.answer && result.sources.length === 0) {
|
||||
throw new SearchProviderError("gemini", "Gemini API returned an empty grounded response", 502);
|
||||
}
|
||||
|
||||
const redirectUrls = new Set<string>();
|
||||
for (const source of result.sources) {
|
||||
if (isGroundingRedirectUrl(source.url)) redirectUrls.add(source.url);
|
||||
}
|
||||
for (const citation of result.citations) {
|
||||
if (isGroundingRedirectUrl(citation.url)) redirectUrls.add(citation.url);
|
||||
}
|
||||
if (redirectUrls.size === 0) return result;
|
||||
|
||||
signal?.throwIfAborted();
|
||||
const resolvedEntries = await Promise.all(
|
||||
[...redirectUrls].map(async url => [url, await resolveGroundingRedirect(url, fetchImpl, signal)] as const),
|
||||
);
|
||||
signal?.throwIfAborted();
|
||||
const resolvedUrls = new Map(resolvedEntries);
|
||||
for (const source of result.sources) {
|
||||
source.url = resolvedUrls.get(source.url) ?? source.url;
|
||||
}
|
||||
for (const citation of result.citations) {
|
||||
citation.url = resolvedUrls.get(citation.url) ?? citation.url;
|
||||
}
|
||||
|
||||
const seenUrls = new Set<string>();
|
||||
let writeIndex = 0;
|
||||
for (const source of result.sources) {
|
||||
if (seenUrls.has(source.url)) continue;
|
||||
seenUrls.add(source.url);
|
||||
result.sources[writeIndex++] = source;
|
||||
}
|
||||
result.sources.length = writeIndex;
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calls the Cloud Code Assist API with Google Search grounding enabled.
|
||||
*
|
||||
@@ -420,7 +523,10 @@ async function callGeminiSearch(
|
||||
}
|
||||
|
||||
if (!response?.ok) {
|
||||
const errorText = response ? await response.text() : "Network error";
|
||||
const rawErrorText = response ? await response.text() : "Network error";
|
||||
const errorText = auth.accessToken
|
||||
? rawErrorText.split(auth.accessToken).join("[redacted]")
|
||||
: rawErrorText;
|
||||
const status = response?.status ?? 502;
|
||||
const classified = classifyProviderHttpError("gemini", status, errorText);
|
||||
if (classified) throw classified;
|
||||
@@ -431,11 +537,12 @@ async function callGeminiSearch(
|
||||
throw new SearchProviderError("gemini", "Gemini API returned no response body", 500);
|
||||
}
|
||||
|
||||
return parseGeminiSearchStream(response.body, model);
|
||||
return finalizeGeminiSearchResult(await parseGeminiSearchStream(response.body, model), fetchImpl, signal);
|
||||
}
|
||||
|
||||
async function callGeminiDeveloperSearch(
|
||||
apiKey: string,
|
||||
endpoint: GeminiDeveloperEndpoint,
|
||||
model: string,
|
||||
query: string,
|
||||
systemPrompt: string | undefined,
|
||||
@@ -474,11 +581,13 @@ async function callGeminiDeveloperSearch(
|
||||
}
|
||||
|
||||
const response = await fetchWithRetry(
|
||||
() => `${DEVELOPER_API_ENDPOINT}/models/${model}:streamGenerateContent?alt=sse`,
|
||||
() => `${endpoint.url}/models/${model}:streamGenerateContent?alt=sse`,
|
||||
{
|
||||
method: "POST",
|
||||
headers: {
|
||||
"x-goog-api-key": apiKey,
|
||||
...(endpoint.isCloudflareGateway
|
||||
? { "cf-aig-authorization": `Bearer ${apiKey}` }
|
||||
: { "x-goog-api-key": apiKey }),
|
||||
"Content-Type": "application/json",
|
||||
Accept: "text/event-stream",
|
||||
},
|
||||
@@ -492,7 +601,8 @@ async function callGeminiDeveloperSearch(
|
||||
);
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
const rawErrorText = await response.text();
|
||||
const errorText = apiKey ? rawErrorText.split(apiKey).join("[redacted]") : rawErrorText;
|
||||
const classified = classifyProviderHttpError("gemini", response.status, errorText);
|
||||
if (classified) throw classified;
|
||||
throw new SearchProviderError(
|
||||
@@ -506,7 +616,7 @@ async function callGeminiDeveloperSearch(
|
||||
throw new SearchProviderError("gemini", "Gemini API returned no response body", 500);
|
||||
}
|
||||
|
||||
return parseGeminiSearchStream(response.body, model);
|
||||
return finalizeGeminiSearchResult(await parseGeminiSearchStream(response.body, model), fetchImpl, signal);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -557,16 +667,20 @@ export async function searchGemini(params: GeminiSearchParams): Promise<SearchRe
|
||||
{ sessionId: params.sessionId, signal: params.signal, seed: seed.access },
|
||||
);
|
||||
} else {
|
||||
const apiKey = await params.authStorage.getApiKey(DEVELOPER_API_PROVIDER, params.sessionId, {
|
||||
const endpoint = resolveGeminiDeveloperEndpoint();
|
||||
const apiKey = await params.authStorage.getApiKey(endpoint.authProvider, params.sessionId, {
|
||||
signal: params.signal,
|
||||
});
|
||||
if (!apiKey) {
|
||||
throw new Error(
|
||||
"No Gemini credentials found. Set GEMINI_API_KEY, configure an API key for provider \"google\", or login with 'omp /login google-gemini-cli' / 'omp /login google-antigravity' to enable Gemini web search.",
|
||||
endpoint.isCloudflareGateway
|
||||
? "No Cloudflare AI Gateway credential found. Configure provider \"cloudflare-ai-gateway\" or set CLOUDFLARE_AI_GATEWAY_API_KEY."
|
||||
: "No Gemini credentials found. Set GEMINI_API_KEY, configure an API key for provider \"google\", or login with 'omp /login google-gemini-cli' / 'omp /login google-antigravity' to enable Gemini web search.",
|
||||
);
|
||||
}
|
||||
result = await callGeminiDeveloperSearch(
|
||||
apiKey,
|
||||
endpoint,
|
||||
selectedModel,
|
||||
searchQuery,
|
||||
params.system_prompt,
|
||||
@@ -609,7 +723,12 @@ export class GeminiProvider extends SearchProvider {
|
||||
// Cheap, in-memory check — avoids driving the refresh pipeline during
|
||||
// the provider-chain probe. `searchGemini` refreshes OAuth lazily on the
|
||||
// actual request and resolves developer API keys through AuthStorage.
|
||||
return hasGeminiOAuth(authStorage) || authStorage.hasAuth(DEVELOPER_API_PROVIDER);
|
||||
if (hasGeminiOAuth(authStorage)) return true;
|
||||
try {
|
||||
return authStorage.hasAuth(resolveGeminiDeveloperEndpoint().authProvider);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
search(params: SearchParams): Promise<SearchResponse> {
|
||||
|
||||
@@ -5,19 +5,24 @@
|
||||
* cleaned content.
|
||||
*/
|
||||
|
||||
import { type AuthStorage, type FetchImpl, getEnvApiKey } from "@oh-my-pi/pi-ai";
|
||||
import { type ApiKey, type AuthStorage, type FetchImpl, withAuth } from "@oh-my-pi/pi-ai";
|
||||
import type { SearchResponse, SearchSource } from "../../../web/search/types";
|
||||
import { SearchProviderError } from "../../../web/search/types";
|
||||
import { formatQuery, parseSearchQuery } from "../query";
|
||||
import { clampNumResults } from "../utils";
|
||||
import type { SearchParams } from "./base";
|
||||
import { SearchProvider } from "./base";
|
||||
import { classifyProviderHttpError, withHardTimeout } from "./utils";
|
||||
|
||||
const JINA_SEARCH_URL = "https://s.jina.ai";
|
||||
const DEFAULT_NUM_RESULTS = 5;
|
||||
const MAX_NUM_RESULTS = 20;
|
||||
type SearchParamsWithFetch = SearchParams & { fetch?: FetchImpl };
|
||||
|
||||
export interface JinaSearchParams {
|
||||
query: string;
|
||||
authStorage: AuthStorage;
|
||||
sessionId?: string;
|
||||
num_results?: number;
|
||||
/** Single bare host for Jina's `X-Site` in-site search header. */
|
||||
site?: string;
|
||||
@@ -29,31 +34,37 @@ export interface JinaSearchParams {
|
||||
interface JinaSearchResult {
|
||||
title?: string | null;
|
||||
url?: string | null;
|
||||
description?: string | null;
|
||||
content?: string | null;
|
||||
}
|
||||
|
||||
type JinaSearchResponse = JinaSearchResult[];
|
||||
|
||||
/** Find JINA_API_KEY from environment or .env files. */
|
||||
export function findApiKey(): string | null {
|
||||
return getEnvApiKey("jina") ?? null;
|
||||
interface JinaSearchEnvelope {
|
||||
code?: unknown;
|
||||
data?: unknown;
|
||||
}
|
||||
|
||||
type JinaSearchResponse = JinaSearchResult[];
|
||||
|
||||
/** Call Jina Reader search API. */
|
||||
async function callJinaSearch(
|
||||
apiKey: string,
|
||||
query: string,
|
||||
numResults: number,
|
||||
site?: string,
|
||||
signal?: AbortSignal,
|
||||
fetchImpl: FetchImpl = fetch,
|
||||
timeoutMs?: number,
|
||||
): Promise<JinaSearchResponse> {
|
||||
const requestUrl = `${JINA_SEARCH_URL}/${encodeURIComponent(query)}`;
|
||||
const requestUrl = new URL(`${JINA_SEARCH_URL}/${encodeURIComponent(query)}`);
|
||||
requestUrl.searchParams.set("count", String(numResults));
|
||||
|
||||
const headers: Record<string, string> = {
|
||||
Accept: "application/json",
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
};
|
||||
if (site) headers["X-Site"] = site;
|
||||
headers["X-Respond-With"] = "no-content";
|
||||
headers["X-Retain-Images"] = "none";
|
||||
const response = await fetchImpl(requestUrl, {
|
||||
headers,
|
||||
signal: withHardTimeout(signal, timeoutMs),
|
||||
@@ -66,24 +77,43 @@ async function callJinaSearch(
|
||||
throw new SearchProviderError("jina", `Jina API error (${response.status}): ${errorText}`, response.status);
|
||||
}
|
||||
|
||||
const payload = (await response.json()) as { data?: JinaSearchResponse } | null;
|
||||
return Array.isArray(payload?.data) ? payload.data : [];
|
||||
const payload = (await response.json()) as JinaSearchEnvelope | JinaSearchResponse | null;
|
||||
if (Array.isArray(payload)) return payload;
|
||||
if (!payload || typeof payload !== "object") {
|
||||
throw new SearchProviderError("jina", "Jina API returned invalid response: expected an object or array");
|
||||
}
|
||||
if (typeof payload.code === "number" && payload.code !== 200) {
|
||||
throw new SearchProviderError("jina", `Jina API response reported failure (${payload.code})`, payload.code);
|
||||
}
|
||||
if (!Array.isArray(payload.data)) {
|
||||
throw new SearchProviderError("jina", "Jina API returned invalid response: expected data array");
|
||||
}
|
||||
return payload.data as JinaSearchResponse;
|
||||
}
|
||||
|
||||
/** Execute Jina web search. */
|
||||
export async function searchJina(params: JinaSearchParams): Promise<SearchResponse> {
|
||||
const apiKey = findApiKey();
|
||||
if (!apiKey) {
|
||||
throw new Error("JINA_API_KEY not found. Set it in environment or .env file.");
|
||||
}
|
||||
|
||||
const response = await callJinaSearch(
|
||||
apiKey,
|
||||
params.query,
|
||||
params.site,
|
||||
params.signal,
|
||||
params.fetch,
|
||||
params.timeoutMs,
|
||||
const numResults = clampNumResults(params.num_results, DEFAULT_NUM_RESULTS, MAX_NUM_RESULTS);
|
||||
const keyOrResolver: ApiKey = params.authStorage.resolver("jina", {
|
||||
sessionId: params.sessionId,
|
||||
});
|
||||
const response = await withAuth(
|
||||
keyOrResolver,
|
||||
apiKey =>
|
||||
callJinaSearch(
|
||||
apiKey,
|
||||
params.query,
|
||||
numResults,
|
||||
params.site,
|
||||
params.signal,
|
||||
params.fetch,
|
||||
params.timeoutMs,
|
||||
),
|
||||
{
|
||||
signal: params.signal,
|
||||
missingKeyMessage:
|
||||
'Jina credentials not found. Set JINA_API_KEY or configure an API key for provider "jina".',
|
||||
},
|
||||
);
|
||||
const sources: SearchSource[] = [];
|
||||
|
||||
@@ -92,11 +122,11 @@ export async function searchJina(params: JinaSearchParams): Promise<SearchRespon
|
||||
sources.push({
|
||||
title: result.title ?? result.url,
|
||||
url: result.url,
|
||||
snippet: result.content ?? undefined,
|
||||
snippet: result.description?.trim() || result.content?.trim() || undefined,
|
||||
});
|
||||
}
|
||||
|
||||
const limitedSources = params.num_results ? sources.slice(0, params.num_results) : sources;
|
||||
const limitedSources = sources.slice(0, numResults);
|
||||
|
||||
return {
|
||||
provider: "jina",
|
||||
@@ -109,8 +139,8 @@ export class JinaProvider extends SearchProvider {
|
||||
readonly id = "jina";
|
||||
readonly label = "Jina";
|
||||
|
||||
isAvailable(_authStorage: AuthStorage): boolean {
|
||||
return !!findApiKey();
|
||||
isAvailable(authStorage: AuthStorage): boolean {
|
||||
return authStorage.hasAuth("jina");
|
||||
}
|
||||
|
||||
search(params: SearchParamsWithFetch): Promise<SearchResponse> {
|
||||
@@ -134,6 +164,8 @@ export class JinaProvider extends SearchProvider {
|
||||
|
||||
return searchJina({
|
||||
query,
|
||||
authStorage: params.authStorage,
|
||||
sessionId: params.sessionId,
|
||||
num_results: params.numSearchResults ?? params.limit,
|
||||
site,
|
||||
signal: params.signal,
|
||||
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
ParallelApiError,
|
||||
type ParallelSearchResult,
|
||||
parseParallelErrorResponse,
|
||||
parseParallelJsonResponse,
|
||||
parseParallelSearchPayload,
|
||||
} from "../../parallel";
|
||||
import { formatQuery, parseSearchQuery, type StructuredQuery } from "../query";
|
||||
@@ -28,6 +29,13 @@ interface ParallelSourcePolicy {
|
||||
after_date?: string;
|
||||
}
|
||||
|
||||
const RECENCY_DAYS: Record<NonNullable<SearchParams["recency"]>, number> = {
|
||||
day: 1,
|
||||
week: 7,
|
||||
month: 30,
|
||||
year: 365,
|
||||
};
|
||||
|
||||
/** Site values may carry paths (`github.com/anthropics`); Parallel takes bare hosts. */
|
||||
function toHosts(sites: readonly string[]): string[] {
|
||||
const hosts = new Set<string>();
|
||||
@@ -39,19 +47,23 @@ function toHosts(sites: readonly string[]): string[] {
|
||||
}
|
||||
|
||||
/**
|
||||
* Map parsed `site:`/`-site:`/`after:` directives onto Parallel's
|
||||
* `source_policy`. Per Parallel docs, `exclude_domains` is ignored when
|
||||
* `include_domains` is set, so exclusions are only sent without an allow
|
||||
* list (the central lenient filter enforces them regardless).
|
||||
* Map parsed `site:`/`-site:`/`after:` directives and the relative recency
|
||||
* option onto Parallel's `source_policy`. An explicit `after:` bound wins.
|
||||
* Per Parallel docs, `exclude_domains` is ignored when `include_domains` is
|
||||
* set, so exclusions are only sent without an allow list (the central lenient
|
||||
* filter enforces them regardless).
|
||||
*/
|
||||
function toSourcePolicy(parsed: StructuredQuery): ParallelSourcePolicy | undefined {
|
||||
function toSourcePolicy(parsed: StructuredQuery, recency?: SearchParams["recency"]): ParallelSourcePolicy | undefined {
|
||||
const policy: ParallelSourcePolicy = {};
|
||||
const include = toHosts(parsed.sites);
|
||||
const exclude = toHosts(parsed.excludedSites);
|
||||
if (include.length) policy.include_domains = include;
|
||||
else if (exclude.length) policy.exclude_domains = exclude;
|
||||
if (parsed.after) policy.after_date = parsed.after;
|
||||
return Object.keys(policy).length ? policy : undefined;
|
||||
else if (recency) {
|
||||
policy.after_date = new Date(Date.now() - RECENCY_DAYS[recency] * 86_400_000).toISOString().slice(0, 10);
|
||||
}
|
||||
return policy.include_domains || policy.exclude_domains || policy.after_date ? policy : undefined;
|
||||
}
|
||||
|
||||
async function searchWithAuthStorage(
|
||||
@@ -105,7 +117,7 @@ async function searchWithAuthStorage(
|
||||
throw parseParallelErrorResponse(response.status, await response.text());
|
||||
}
|
||||
|
||||
const payload: unknown = await response.json();
|
||||
const payload = await parseParallelJsonResponse(response, "search");
|
||||
return parseParallelSearchPayload(payload, { parseMetadata: false });
|
||||
},
|
||||
{ signal: params.signal },
|
||||
@@ -116,6 +128,7 @@ export async function searchParallel(
|
||||
params: {
|
||||
query: string;
|
||||
num_results?: number;
|
||||
recency?: SearchParams["recency"];
|
||||
signal?: AbortSignal;
|
||||
timeoutMs?: number;
|
||||
fetch?: FetchImpl;
|
||||
@@ -126,9 +139,9 @@ export async function searchParallel(
|
||||
): Promise<SearchResponse> {
|
||||
const numResults = clampNumResults(params.num_results, DEFAULT_NUM_RESULTS, MAX_NUM_RESULTS);
|
||||
const parsed = params.parsedQuery ?? parseSearchQuery(params.query);
|
||||
// Back-compat: without directives the upstream request is byte-identical.
|
||||
// Directives are removed only where Parallel has a native equivalent.
|
||||
const query = parsed.hasDirectives ? formatQuery(parsed, PARALLEL_QUERY_SYNTAX) : params.query;
|
||||
const sourcePolicy = parsed.hasDirectives ? toSourcePolicy(parsed) : undefined;
|
||||
const sourcePolicy = toSourcePolicy(parsed, params.recency);
|
||||
|
||||
try {
|
||||
const result = await searchWithAuthStorage(
|
||||
@@ -174,6 +187,7 @@ export class ParallelProvider extends SearchProvider {
|
||||
{
|
||||
query: params.query,
|
||||
num_results: params.numSearchResults ?? params.limit,
|
||||
recency: params.recency,
|
||||
signal: params.signal,
|
||||
timeoutMs: params.timeoutMs,
|
||||
fetch: params.fetch,
|
||||
|
||||
@@ -150,6 +150,7 @@ interface PerplexityOAuthStreamEvent {
|
||||
error_code?: string;
|
||||
error_message?: string;
|
||||
display_model?: string;
|
||||
user_selected_model?: string;
|
||||
uuid?: string;
|
||||
}
|
||||
|
||||
@@ -327,7 +328,11 @@ export interface PerplexitySearchParams {
|
||||
system_prompt?: string;
|
||||
/** Pre-parsed view of `query` from the search pipeline; parsed locally when absent. */
|
||||
parsedQuery?: StructuredQuery;
|
||||
/** Direct API model. Defaults to `PI_PERPLEXITY_API_MODEL`, then `sonar-pro`. */
|
||||
api_model?: string;
|
||||
search_recency_filter?: "hour" | "day" | "week" | "month" | "year";
|
||||
/** Consumer subscription model preference. Defaults to `PI_PERPLEXITY_MODEL`, then Sonar (`experimental`). */
|
||||
subscription_model?: string;
|
||||
num_results?: number;
|
||||
/** Maximum output tokens. Defaults to 8192. */
|
||||
max_tokens?: number;
|
||||
@@ -539,6 +544,15 @@ async function callPerplexityApi(
|
||||
return parseStreamedApiResponse(message, metadata);
|
||||
}
|
||||
|
||||
function oauthSourceKey(url: string): string {
|
||||
const trimmed = url.trim().replace(/\/$/, "");
|
||||
try {
|
||||
return new URL(trimmed).href.replace(/\/$/, "");
|
||||
} catch {
|
||||
return trimmed.toLowerCase();
|
||||
}
|
||||
}
|
||||
|
||||
function buildOAuthSources(event: PerplexityOAuthStreamEvent): SearchSource[] {
|
||||
const results =
|
||||
event.blocks?.find(block => block.intended_usage === "web_results")?.web_result_block?.web_results ?? [];
|
||||
@@ -607,6 +621,7 @@ async function callPerplexityAsk(
|
||||
params: PerplexitySearchParams,
|
||||
filters: PerplexityNativeFilters,
|
||||
): Promise<{ answer: string; sources: SearchSource[]; model?: string; requestId?: string }> {
|
||||
const subscriptionModel = params.subscription_model?.trim() || $env.PI_PERPLEXITY_MODEL?.trim() || "experimental";
|
||||
const requestId = crypto.randomUUID();
|
||||
// The consumer `perplexity_ask` endpoint is itself a research assistant and
|
||||
// has no system-message slot. Prepending the API-style system prompt to the
|
||||
@@ -645,14 +660,14 @@ async function callPerplexityAsk(
|
||||
query_str: effectiveQuery,
|
||||
search_focus: "internet",
|
||||
mode: "copilot",
|
||||
model_preference: "experimental",
|
||||
model_preference: subscriptionModel,
|
||||
sources: ["web"],
|
||||
attachments: [],
|
||||
frontend_uuid: crypto.randomUUID(),
|
||||
frontend_context_uuid: crypto.randomUUID(),
|
||||
version: OAUTH_API_VERSION,
|
||||
language: "en-US",
|
||||
timezone: Intl.DateTimeFormat().resolvedOptions().timeZone,
|
||||
timezone: Intl.DateTimeFormat().resolvedOptions().timeZone ?? "UTC",
|
||||
// Recency cannot be combined with absolute date filters; explicit
|
||||
// before:/after: bounds take precedence.
|
||||
search_recency_filter: filters.afterDate || filters.beforeDate ? null : (params.search_recency_filter ?? null),
|
||||
@@ -740,12 +755,14 @@ async function callPerplexityAsk(
|
||||
if (eventAnswer.length > 0) {
|
||||
answer = eventAnswer;
|
||||
}
|
||||
|
||||
for (const source of buildOAuthSources(mergedEvent)) {
|
||||
sourcesByUrl.set(source.url, source);
|
||||
sourcesByUrl.set(oauthSourceKey(source.url), source);
|
||||
}
|
||||
|
||||
if (mergedEvent.display_model) model = mergedEvent.display_model;
|
||||
const reportedModel = [mergedEvent.user_selected_model, mergedEvent.display_model].find(
|
||||
candidate => candidate && candidate !== "turbo",
|
||||
);
|
||||
if (reportedModel) model = reportedModel;
|
||||
if (mergedEvent.uuid) finalRequestId = mergedEvent.uuid;
|
||||
if (mergedEvent.final || mergedEvent.status === "COMPLETED") {
|
||||
break;
|
||||
@@ -755,7 +772,7 @@ async function callPerplexityAsk(
|
||||
return {
|
||||
answer,
|
||||
sources: [...sourcesByUrl.values()],
|
||||
model,
|
||||
model: model ?? (auth.type === "anonymous" ? mergedEvent.display_model : subscriptionModel),
|
||||
requestId: finalRequestId ?? requestId,
|
||||
};
|
||||
}
|
||||
@@ -872,7 +889,7 @@ export async function searchPerplexity(params: PerplexitySearchParams): Promise<
|
||||
messages.push({ role: "user", content: filters.query });
|
||||
|
||||
const request: PerplexityRequest = {
|
||||
model: "sonar-pro",
|
||||
model: params.api_model?.trim() || $env.PI_PERPLEXITY_API_MODEL?.trim() || "sonar-pro",
|
||||
messages,
|
||||
max_tokens: params.max_tokens ?? DEFAULT_MAX_TOKENS,
|
||||
temperature: params.temperature ?? DEFAULT_TEMPERATURE,
|
||||
|
||||
@@ -62,6 +62,7 @@ interface SearXNGResult {
|
||||
title?: string;
|
||||
url?: string;
|
||||
content?: string;
|
||||
snippet?: string;
|
||||
engine?: string;
|
||||
publishedDate?: string;
|
||||
/** SearXNG sometimes uses publishedDate, sometimes just date */
|
||||
@@ -76,6 +77,7 @@ interface SearXNGResponse {
|
||||
suggestions?: string[];
|
||||
corrections?: string[];
|
||||
unresponsive_engines?: Array<[string, string]>;
|
||||
answers?: unknown[];
|
||||
}
|
||||
|
||||
interface SearXNGAuth {
|
||||
@@ -272,6 +274,61 @@ function stripExternalBangs(query: string): string {
|
||||
.join(" ");
|
||||
}
|
||||
|
||||
/** Extract displayable text from both legacy string answers and modern
|
||||
* structured answer plugins (legacy, translations, weather). */
|
||||
function extractAnswerText(answer: unknown): string | undefined {
|
||||
if (typeof answer === "string") return answer.trim() || undefined;
|
||||
if (!answer || typeof answer !== "object") return undefined;
|
||||
|
||||
const record = answer as Record<string, unknown>;
|
||||
if (typeof record.answer === "string") return record.answer.trim() || undefined;
|
||||
|
||||
if (Array.isArray(record.translations)) {
|
||||
const translations: string[] = [];
|
||||
for (const item of record.translations) {
|
||||
if (!item || typeof item !== "object") continue;
|
||||
const text = (item as Record<string, unknown>).text;
|
||||
if (typeof text === "string" && text.trim()) translations.push(text.trim());
|
||||
if (translations.length === 3) break;
|
||||
}
|
||||
if (translations.length) return translations.join("\n");
|
||||
}
|
||||
|
||||
if (record.current && typeof record.current === "object") {
|
||||
const current = record.current as Record<string, unknown>;
|
||||
if (typeof current.summary === "string" && current.summary.trim()) return current.summary.trim();
|
||||
const location =
|
||||
current.location && typeof current.location === "object"
|
||||
? (current.location as Record<string, unknown>).name
|
||||
: undefined;
|
||||
const temperature =
|
||||
current.temperature && typeof current.temperature === "object"
|
||||
? (current.temperature as Record<string, unknown>)
|
||||
: undefined;
|
||||
const temperatureText =
|
||||
temperature && (typeof temperature.val === "string" || typeof temperature.val === "number")
|
||||
? `${temperature.val}${typeof temperature.unit === "string" ? temperature.unit : ""}`
|
||||
: undefined;
|
||||
const condition = typeof current.condition === "string" ? current.condition : undefined;
|
||||
const parts = [location, temperatureText, condition].filter(
|
||||
(part): part is string => typeof part === "string" && part.trim().length > 0,
|
||||
);
|
||||
if (parts.length) return parts.join(": ");
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function formatAnswers(answers: unknown[] | undefined): string | undefined {
|
||||
const texts: string[] = [];
|
||||
for (const answer of answers ?? []) {
|
||||
const text = extractAnswerText(answer);
|
||||
if (text) texts.push(text);
|
||||
if (texts.length === 3) break;
|
||||
}
|
||||
return texts.length ? texts.join("\n\n") : undefined;
|
||||
}
|
||||
|
||||
/** Build the search URL and headers for a SearXNG request */
|
||||
function buildRequest(
|
||||
endpoint: string,
|
||||
@@ -282,6 +339,7 @@ function buildRequest(
|
||||
categories?: string;
|
||||
engines?: string;
|
||||
language?: string;
|
||||
safesearch?: 0 | 1 | 2;
|
||||
signal?: AbortSignal;
|
||||
},
|
||||
auth: SearXNGAuth | null,
|
||||
@@ -308,6 +366,10 @@ function buildRequest(
|
||||
url.searchParams.set("engines", params.engines);
|
||||
}
|
||||
|
||||
if (params.safesearch !== undefined) {
|
||||
url.searchParams.set("safesearch", String(params.safesearch));
|
||||
}
|
||||
|
||||
if (params.language) {
|
||||
url.searchParams.set("language", params.language);
|
||||
}
|
||||
@@ -326,6 +388,7 @@ async function callSearXNGSearch(
|
||||
categories?: string;
|
||||
engines?: string;
|
||||
language?: string;
|
||||
safesearch?: 0 | 1 | 2;
|
||||
signal?: AbortSignal;
|
||||
timeoutMs?: number;
|
||||
fetch?: FetchImpl;
|
||||
@@ -372,12 +435,23 @@ export async function searchSearXNG(params: {
|
||||
|
||||
let categories: string | undefined;
|
||||
let language: string | undefined;
|
||||
let configuredSafesearch: number | undefined;
|
||||
try {
|
||||
categories = settings.get("searxng.categories") ?? undefined;
|
||||
language = settings.get("searxng.language") ?? undefined;
|
||||
configuredSafesearch = settings.get("searxng.safesearch");
|
||||
} catch {
|
||||
// Settings not initialized yet
|
||||
}
|
||||
if (
|
||||
configuredSafesearch !== undefined &&
|
||||
configuredSafesearch !== 0 &&
|
||||
configuredSafesearch !== 1 &&
|
||||
configuredSafesearch !== 2
|
||||
) {
|
||||
throw new Error("searxng.safesearch must be 0 (off), 1 (moderate), or 2 (strict).");
|
||||
}
|
||||
const safesearch = configuredSafesearch;
|
||||
const configuredEngines = findEngines();
|
||||
|
||||
// SearXNG forwards `q` to downstream engines, so build it with the shared
|
||||
@@ -402,6 +476,7 @@ export async function searchSearXNG(params: {
|
||||
categories,
|
||||
engines,
|
||||
language,
|
||||
safesearch,
|
||||
fetch: params.fetch,
|
||||
},
|
||||
auth,
|
||||
@@ -415,7 +490,7 @@ export async function searchSearXNG(params: {
|
||||
sources.push({
|
||||
title: result.title ?? result.url,
|
||||
url: result.url,
|
||||
snippet: result.content?.trim() || undefined,
|
||||
snippet: (result.content ?? result.snippet)?.trim() || undefined,
|
||||
publishedDate: publishedDate ?? undefined,
|
||||
ageSeconds: dateToAgeSeconds(publishedDate),
|
||||
});
|
||||
@@ -435,6 +510,7 @@ export async function searchSearXNG(params: {
|
||||
|
||||
return {
|
||||
provider: "searxng",
|
||||
answer: formatAnswers(response.answers),
|
||||
sources: limitedSources,
|
||||
relatedQuestions: response.suggestions?.length ? response.suggestions : undefined,
|
||||
};
|
||||
|
||||
@@ -34,17 +34,10 @@ export interface TavilySearchParams {
|
||||
fetch?: FetchImpl;
|
||||
}
|
||||
|
||||
interface TavilySearchResult {
|
||||
title?: string | null;
|
||||
url?: string | null;
|
||||
content?: string | null;
|
||||
published_date?: string | null;
|
||||
}
|
||||
|
||||
interface TavilySearchResponse {
|
||||
answer?: string | null;
|
||||
results?: TavilySearchResult[];
|
||||
request_id?: string | null;
|
||||
answer?: unknown;
|
||||
results?: unknown;
|
||||
request_id?: unknown;
|
||||
}
|
||||
|
||||
function asRecord(value: unknown): Record<string, unknown> | null {
|
||||
@@ -141,28 +134,36 @@ async function callTavilySearch(apiKey: string, params: TavilySearchParams): Pro
|
||||
throw new SearchProviderError("tavily", `Tavily API error (${response.status}): ${message}`, response.status);
|
||||
}
|
||||
|
||||
return (await response.json()) as TavilySearchResponse;
|
||||
const payload: unknown = await response.json();
|
||||
return asRecord(payload) ?? {};
|
||||
}
|
||||
|
||||
function toSearchResponse(response: TavilySearchResponse, numResults: number): SearchResponse {
|
||||
const sources: SearchSource[] = [];
|
||||
|
||||
for (const result of response.results ?? []) {
|
||||
if (!result.url) continue;
|
||||
sources.push({
|
||||
title: result.title ?? result.url,
|
||||
url: result.url,
|
||||
snippet: result.content ?? undefined,
|
||||
publishedDate: result.published_date ?? undefined,
|
||||
ageSeconds: dateToAgeSeconds(result.published_date ?? undefined),
|
||||
});
|
||||
if (Array.isArray(response.results)) {
|
||||
for (const value of response.results) {
|
||||
const result = asRecord(value);
|
||||
if (!result || typeof result.url !== "string" || !result.url) continue;
|
||||
const title = typeof result.title === "string" && result.title ? result.title : result.url;
|
||||
const snippet = typeof result.content === "string" ? result.content : undefined;
|
||||
const publishedDate = typeof result.published_date === "string" ? result.published_date : undefined;
|
||||
sources.push({
|
||||
title,
|
||||
url: result.url,
|
||||
snippet,
|
||||
publishedDate,
|
||||
ageSeconds: dateToAgeSeconds(publishedDate),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const answer = typeof response.answer === "string" ? response.answer.trim() || undefined : undefined;
|
||||
return {
|
||||
provider: "tavily",
|
||||
answer: response.answer?.trim() || undefined,
|
||||
answer,
|
||||
sources: sources.slice(0, numResults),
|
||||
requestId: response.request_id ?? undefined,
|
||||
requestId: typeof response.request_id === "string" ? response.request_id : undefined,
|
||||
authMode: "api_key",
|
||||
};
|
||||
}
|
||||
|
||||
@@ -19,7 +19,7 @@ const MAX_NUM_RESULTS = 20;
|
||||
const MAX_PAGE = 10;
|
||||
|
||||
/** TinyFish is SERP-backed: common Google-style operators pass through. */
|
||||
const TINYFISH_QUERY_SYNTAX: QuerySyntax = { phrases: true, negation: true, site: true, filetype: true };
|
||||
const TINYFISH_QUERY_SYNTAX: QuerySyntax = { phrases: true, negation: true, filetype: true };
|
||||
|
||||
const RECENCY_MINUTES: Record<NonNullable<SearchParams["recency"]>, number> = {
|
||||
day: 1440,
|
||||
@@ -33,6 +33,8 @@ export interface TinyFishSearchParams {
|
||||
num_results?: number;
|
||||
recency?: SearchParams["recency"];
|
||||
page?: number;
|
||||
include_domains?: string[];
|
||||
exclude_domains?: string[];
|
||||
signal?: AbortSignal;
|
||||
timeoutMs?: number;
|
||||
fetch?: FetchImpl;
|
||||
@@ -66,6 +68,12 @@ async function callTinyFishSearch(apiKey: string, params: TinyFishSearchParams):
|
||||
if (params.recency) {
|
||||
url.searchParams.set("recency_minutes", String(RECENCY_MINUTES[params.recency]));
|
||||
}
|
||||
if (params.include_domains?.length) {
|
||||
url.searchParams.set("include_domains", params.include_domains.join(","));
|
||||
}
|
||||
if (params.exclude_domains?.length) {
|
||||
url.searchParams.set("exclude_domains", params.exclude_domains.join(","));
|
||||
}
|
||||
if (params.num_results !== undefined) {
|
||||
url.searchParams.set("num_results", String(params.num_results));
|
||||
}
|
||||
@@ -96,18 +104,35 @@ async function callTinyFishSearch(apiKey: string, params: TinyFishSearchParams):
|
||||
return (await response.json()) as TinyFishSearchResponse;
|
||||
}
|
||||
|
||||
function appendTinyFishSources(sources: SearchSource[], results: readonly TinyFishSearchResult[]): void {
|
||||
function appendTinyFishSources(
|
||||
sources: SearchSource[],
|
||||
results: readonly TinyFishSearchResult[],
|
||||
seenUrls: Set<string>,
|
||||
): void {
|
||||
for (const result of results) {
|
||||
if (!result.url) continue;
|
||||
const url = result.url?.trim();
|
||||
if (!url || seenUrls.has(url)) continue;
|
||||
seenUrls.add(url);
|
||||
const siteName = result.site_name?.trim();
|
||||
sources.push({
|
||||
title: result.title ?? result.site_name ?? result.url,
|
||||
url: result.url,
|
||||
snippet: result.snippet ?? undefined,
|
||||
author: result.site_name ?? undefined,
|
||||
title: result.title?.trim() || siteName || url,
|
||||
url,
|
||||
snippet: result.snippet?.replace(/\s+/g, " ").trim() || undefined,
|
||||
author: siteName || undefined,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/** Bare hosts from `site:` values; path constraints remain centrally post-filtered. */
|
||||
function siteHosts(sites: readonly string[]): string[] {
|
||||
const hosts = new Set<string>();
|
||||
for (const site of sites) {
|
||||
const host = site.split("/", 1)[0];
|
||||
if (host) hosts.add(host);
|
||||
}
|
||||
return [...hosts];
|
||||
}
|
||||
|
||||
/** Execute TinyFish web search. */
|
||||
export async function searchTinyFish(params: SearchParams): Promise<SearchResponse> {
|
||||
const numResults = clampNumResults(params.numSearchResults ?? params.limit, DEFAULT_NUM_RESULTS, MAX_NUM_RESULTS);
|
||||
@@ -121,6 +146,12 @@ export async function searchTinyFish(params: SearchParams): Promise<SearchRespon
|
||||
timeoutMs: params.timeoutMs,
|
||||
fetch: params.fetch,
|
||||
};
|
||||
if (parsed.hasDirectives) {
|
||||
const includeDomains = siteHosts(parsed.sites);
|
||||
const excludeDomains = siteHosts(parsed.excludedSites);
|
||||
if (includeDomains.length > 0) tinyFishParams.include_domains = includeDomains;
|
||||
if (excludeDomains.length > 0) tinyFishParams.exclude_domains = excludeDomains;
|
||||
}
|
||||
const keyOrResolver: ApiKey = params.authStorage.resolver("tinyfish", {
|
||||
sessionId: params.sessionId,
|
||||
});
|
||||
@@ -128,11 +159,14 @@ export async function searchTinyFish(params: SearchParams): Promise<SearchRespon
|
||||
keyOrResolver,
|
||||
async key => {
|
||||
const collected: SearchSource[] = [];
|
||||
const seenUrls = new Set<string>();
|
||||
for (let page = 0; page <= MAX_PAGE && collected.length < numResults; page += 1) {
|
||||
const searchPage = await callTinyFishSearch(key, { ...tinyFishParams, page });
|
||||
const results = searchPage.results ?? [];
|
||||
appendTinyFishSources(collected, results);
|
||||
if (results.length < pageSize) break;
|
||||
if (!Array.isArray(searchPage.results)) {
|
||||
throw new Error("TinyFish Search API returned an unexpected response shape");
|
||||
}
|
||||
appendTinyFishSources(collected, searchPage.results, seenUrls);
|
||||
if (searchPage.results.length < pageSize) break;
|
||||
}
|
||||
|
||||
return collected.slice(0, numResults);
|
||||
|
||||
@@ -25,6 +25,8 @@ interface XAIUrlCitationAnnotation {
|
||||
title?: string | null;
|
||||
text?: string | null;
|
||||
cited_text?: string | null;
|
||||
start_index?: number | null;
|
||||
end_index?: number | null;
|
||||
}
|
||||
|
||||
interface XAIResponseContentPart {
|
||||
@@ -34,9 +36,20 @@ interface XAIResponseContentPart {
|
||||
annotations?: XAIUrlCitationAnnotation[] | null;
|
||||
}
|
||||
|
||||
interface XAIWebSearchSource {
|
||||
url?: string | null;
|
||||
source_website_url?: string | null;
|
||||
title?: string | null;
|
||||
caption?: string | null;
|
||||
}
|
||||
|
||||
interface XAIResponseOutputItem {
|
||||
type?: string;
|
||||
content?: XAIResponseContentPart[] | null;
|
||||
annotations?: XAIUrlCitationAnnotation[] | null;
|
||||
action?: { sources?: XAIWebSearchSource[] | null } | null;
|
||||
sources?: XAIWebSearchSource[] | null;
|
||||
results?: XAIWebSearchSource[] | null;
|
||||
}
|
||||
|
||||
interface XAIResponsesUsage {
|
||||
@@ -161,7 +174,12 @@ async function callXAIResponses(
|
||||
throwXAIResponsesError(response.status, await response.text());
|
||||
}
|
||||
|
||||
return (await response.json()) as XAIResponsesResponse;
|
||||
try {
|
||||
return (await response.json()) as XAIResponsesResponse;
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
throw new SearchProviderError("xai", `xAI Responses API returned invalid JSON: ${message}`, response.status);
|
||||
}
|
||||
}
|
||||
|
||||
function addCitationSource(
|
||||
@@ -189,38 +207,77 @@ function addCitationSource(
|
||||
citedText: sourceSnippet,
|
||||
});
|
||||
}
|
||||
function extractSnippetAround(
|
||||
text: string | null | undefined,
|
||||
start: number | null | undefined,
|
||||
end: number | null | undefined,
|
||||
): string | undefined {
|
||||
if (!text || typeof start !== "number" || typeof end !== "number") return undefined;
|
||||
const before = Math.max(0, start - 100);
|
||||
const after = Math.min(text.length, end + 100);
|
||||
const snippet = text
|
||||
.slice(before, after)
|
||||
.replace(/\[([^\]]*)\]\([^)]*\)/g, "$1")
|
||||
.trim();
|
||||
if (!snippet) return undefined;
|
||||
return snippet.length > 300 ? `${snippet.slice(0, 297)}...` : snippet;
|
||||
}
|
||||
|
||||
function collectAnnotationSources(
|
||||
annotations: readonly XAIUrlCitationAnnotation[] | null | undefined,
|
||||
sources: SearchSource[],
|
||||
citations: SearchCitation[],
|
||||
seenUrls: Set<string>,
|
||||
contentText?: string | null,
|
||||
): void {
|
||||
if (!annotations) return;
|
||||
if (!Array.isArray(annotations)) return;
|
||||
for (const annotation of annotations) {
|
||||
if (annotation.type !== "url_citation" || !annotation.url) continue;
|
||||
if (!annotation || typeof annotation !== "object") continue;
|
||||
if (annotation.type !== "url_citation" || typeof annotation.url !== "string") continue;
|
||||
addCitationSource(
|
||||
sources,
|
||||
citations,
|
||||
seenUrls,
|
||||
annotation.url,
|
||||
annotation.title,
|
||||
annotation.cited_text ?? annotation.text,
|
||||
annotation.cited_text ??
|
||||
annotation.text ??
|
||||
extractSnippetAround(contentText, annotation.start_index, annotation.end_index),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function collectWebSearchSources(
|
||||
item: XAIResponseOutputItem,
|
||||
sources: SearchSource[],
|
||||
citations: SearchCitation[],
|
||||
seenUrls: Set<string>,
|
||||
): void {
|
||||
if (item.type !== "web_search_call") return;
|
||||
for (const group of [item.action?.sources, item.sources, item.results]) {
|
||||
if (!Array.isArray(group)) continue;
|
||||
for (const source of group) {
|
||||
if (!source || typeof source !== "object") continue;
|
||||
const url = source.url ?? source.source_website_url;
|
||||
if (typeof url !== "string") continue;
|
||||
addCitationSource(sources, citations, seenUrls, url, source.title ?? source.caption);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function parseAnswer(response: XAIResponsesResponse): string | undefined {
|
||||
const topLevelText = response.output_text?.trim();
|
||||
if (topLevelText) return topLevelText;
|
||||
|
||||
const answerParts: string[] = [];
|
||||
for (const item of response.output ?? []) {
|
||||
for (const part of item.content ?? []) {
|
||||
const output = Array.isArray(response.output) ? response.output : [];
|
||||
for (const item of output) {
|
||||
if (!item || typeof item !== "object") continue;
|
||||
const content = Array.isArray(item.content) ? item.content : [];
|
||||
for (const part of content) {
|
||||
if (!part || typeof part !== "object") continue;
|
||||
const text = part.output_text ?? part.text;
|
||||
if ((part.type === "output_text" || part.type === "text") && text?.trim()) {
|
||||
answerParts.push(text.trim());
|
||||
}
|
||||
if (text?.trim()) answerParts.push(text.trim());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -259,13 +316,29 @@ function parseResponse(response: XAIResponsesResponse, resultCap: number): Searc
|
||||
const seenUrls = new Set<string>();
|
||||
|
||||
collectAnnotationSources(response.annotations, sources, citations, seenUrls);
|
||||
for (const item of response.output ?? []) {
|
||||
const output = Array.isArray(response.output) ? response.output : [];
|
||||
for (const item of output) {
|
||||
if (!item || typeof item !== "object") continue;
|
||||
collectAnnotationSources(item.annotations, sources, citations, seenUrls);
|
||||
for (const part of item.content ?? []) {
|
||||
collectAnnotationSources(part.annotations, sources, citations, seenUrls);
|
||||
const content = Array.isArray(item.content) ? item.content : [];
|
||||
for (const part of content) {
|
||||
if (!part || typeof part !== "object") continue;
|
||||
collectAnnotationSources(
|
||||
part.annotations,
|
||||
sources,
|
||||
citations,
|
||||
seenUrls,
|
||||
part.output_text ?? part.text,
|
||||
);
|
||||
}
|
||||
}
|
||||
for (const url of response.citations ?? []) {
|
||||
for (const item of output) {
|
||||
if (!item || typeof item !== "object") continue;
|
||||
collectWebSearchSources(item, sources, citations, seenUrls);
|
||||
}
|
||||
const topLevelCitations = Array.isArray(response.citations) ? response.citations : [];
|
||||
for (const url of topLevelCitations) {
|
||||
if (typeof url !== "string") continue;
|
||||
addCitationSource(sources, citations, seenUrls, url);
|
||||
}
|
||||
const limited = applyResultCap(sources, citations, resultCap);
|
||||
@@ -354,7 +427,11 @@ export async function searchXAI(params: SearchParams): Promise<SearchResponse> {
|
||||
signal: params.signal,
|
||||
missingKeyMessage: 'xAI credentials not found. Set XAI_API_KEY or configure an API key for provider "xai".',
|
||||
});
|
||||
return parseResponse(response, resultCap);
|
||||
const parsed = parseResponse(response, resultCap);
|
||||
if (!parsed.answer && parsed.sources.length === 0) {
|
||||
throw new SearchProviderError("xai", "xAI web_search returned no answer or sources", 502);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
/** Search provider for xAI web search. */
|
||||
|
||||
@@ -94,3 +94,63 @@ describe("renderHtmlToText: jina stall does not starve local fallbacks (#1449)",
|
||||
).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("renderHtmlToText: Jina response validation", () => {
|
||||
it("requests fresh markdown and strips the Jina metadata preamble", async () => {
|
||||
const settings = Settings.isolated({ "providers.fetch": "jina" });
|
||||
const markdown = `# Extracted article\n\n${"Substantive reader content. ".repeat(8)}`.trim();
|
||||
let requestHeaders: Headers | undefined;
|
||||
const fetchMock = asGlobalFetch((_input, init) => {
|
||||
requestHeaders = new Headers(init?.headers);
|
||||
return new Response(`Title: Example\nURL Source: https://example.com/article\nMarkdown Content:\n${markdown}`);
|
||||
});
|
||||
|
||||
const result = await renderHtmlToText(
|
||||
"https://example.com/article",
|
||||
"<html><body>short</body></html>",
|
||||
1,
|
||||
settings,
|
||||
undefined,
|
||||
null,
|
||||
fetchMock,
|
||||
);
|
||||
|
||||
expect(result).toEqual({ content: markdown, ok: true, method: "jina" });
|
||||
expect(requestHeaders?.get("accept")).toBe("text/markdown");
|
||||
expect(requestHeaders?.get("x-no-cache")).toBe("true");
|
||||
});
|
||||
|
||||
for (const { label, readerBody, headers } of [
|
||||
{ label: "missing marker", readerBody: "Plausible but unstructured output. ".repeat(8) },
|
||||
{ label: "short body", readerBody: "Markdown Content:\nToo short" },
|
||||
{ label: "loading shell", readerBody: `Markdown Content:\nLoading...${" ".repeat(120)}` },
|
||||
{ label: "JavaScript gate", readerBody: `Markdown Content:\nPlease enable JavaScript${" ".repeat(120)}` },
|
||||
{
|
||||
label: "declared oversized body",
|
||||
readerBody: `Markdown Content:\n${"Substantive content. ".repeat(8)}`,
|
||||
headers: { "Content-Length": String(2 * 1024 * 1024 + 1) },
|
||||
},
|
||||
]) {
|
||||
it(`falls back when Jina returns a ${label}`, async () => {
|
||||
const settings = Settings.isolated({ "providers.fetch": "jina" });
|
||||
const paragraph =
|
||||
"This locally rendered article contains enough meaningful prose to satisfy the shared reader quality gate. ";
|
||||
const html = `<html><body><article><h1>Fallback article</h1><p>${paragraph.repeat(4)}</p></article></body></html>`;
|
||||
const fetchMock = asGlobalFetch(() => new Response(readerBody, { headers }));
|
||||
|
||||
const result = await renderHtmlToText(
|
||||
"https://example.com/article",
|
||||
html,
|
||||
1,
|
||||
settings,
|
||||
undefined,
|
||||
null,
|
||||
fetchMock,
|
||||
);
|
||||
|
||||
expect(result.ok).toBe(true);
|
||||
expect(result.method).toBe("native");
|
||||
expect(result.content).toContain("Fallback article");
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
@@ -531,6 +531,70 @@ describe("searchCodex model selection", () => {
|
||||
expect(result.sources).toEqual([{ title: "Example Article", url: "https://example.com/article" }]);
|
||||
});
|
||||
|
||||
it("requests and merges web-search action sources with citation metadata", async () => {
|
||||
process.env.PI_CODEX_WEB_SEARCH_MODEL = "gpt-5.4";
|
||||
const answer = "The Responses API supports hosted web search.";
|
||||
const citationStart = answer.indexOf("hosted web search");
|
||||
const sse = [
|
||||
`data: ${JSON.stringify({
|
||||
type: "response.created",
|
||||
response: { id: "resp_created_id", model: "gpt-5.4" },
|
||||
})}`,
|
||||
"",
|
||||
`data: ${JSON.stringify({
|
||||
type: "response.output_item.done",
|
||||
item: {
|
||||
type: "web_search_call",
|
||||
action: {
|
||||
sources: [
|
||||
{
|
||||
url: "https://example.com/article?utm_source=openai",
|
||||
title: "Search result title",
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
})}`,
|
||||
"",
|
||||
`data: ${JSON.stringify({
|
||||
type: "response.output_item.done",
|
||||
item: {
|
||||
type: "message",
|
||||
content: [
|
||||
{
|
||||
type: "output_text",
|
||||
text: answer,
|
||||
annotations: [
|
||||
{
|
||||
type: "url_citation",
|
||||
url: "https://example.com/article?utm_source=openai",
|
||||
title: "Example Article",
|
||||
start_index: citationStart,
|
||||
end_index: citationStart + "hosted web search".length,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
})}`,
|
||||
"",
|
||||
].join("\n");
|
||||
|
||||
const result = await searchCodex(
|
||||
makeSearchParams("action sources", mockCodexFetch("gpt-5.4", sse)),
|
||||
);
|
||||
|
||||
expect(capturedRequest?.body?.include).toEqual(["web_search_call.action.sources"]);
|
||||
expect(result.requestId).toBe("resp_created_id");
|
||||
expect(result.sources).toEqual([
|
||||
{
|
||||
title: "Search result title",
|
||||
url: "https://example.com/article",
|
||||
snippet: answer,
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("extracts plain text URLs when annotations are absent", async () => {
|
||||
process.env.PI_CODEX_WEB_SEARCH_MODEL = "gpt-5.4";
|
||||
const result = await searchCodex(
|
||||
@@ -769,4 +833,22 @@ describe("searchCodex model selection", () => {
|
||||
"Codex request failed (model_snapshot_unavailable): The requested model snapshot is unavailable.",
|
||||
);
|
||||
});
|
||||
|
||||
it("classifies rate-limit failures delivered inside a successful SSE response", async () => {
|
||||
const sse = [
|
||||
`data: ${JSON.stringify({
|
||||
type: "response.failed",
|
||||
response: {
|
||||
error: { code: "rate_limit_exceeded", message: "Too many requests" },
|
||||
},
|
||||
})}`,
|
||||
"",
|
||||
].join("\n");
|
||||
const fetchMock: FetchImpl = () =>
|
||||
Promise.resolve(new Response(sse, { status: 200, headers: { "Content-Type": "text/event-stream" } }));
|
||||
|
||||
await expect(searchCodex(makeSearchParams("rate-limited search", fetchMock))).rejects.toMatchObject({
|
||||
status: 429,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -428,6 +428,19 @@ describe("searchExa", () => {
|
||||
expect(result.sources[0].snippet).toBe("summary here");
|
||||
});
|
||||
|
||||
it("caps snippets at 500 characters", async () => {
|
||||
const result = await searchExa({
|
||||
query: "bounded snippet",
|
||||
fetch: mockFetch(
|
||||
makeMockExaResponse({
|
||||
results: [{ title: "Long", url: "https://long.example", summary: "x".repeat(800) }],
|
||||
}),
|
||||
),
|
||||
});
|
||||
|
||||
expect(result.sources[0].snippet).toHaveLength(500);
|
||||
});
|
||||
|
||||
it("falls back to text when summary is null", async () => {
|
||||
const result = await searchExa({
|
||||
query: "fallback",
|
||||
@@ -535,6 +548,68 @@ describe("searchExa", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("encodes MCP filters in the basic query, uses camel-case result count, and tags the request source", async () => {
|
||||
delete process.env.EXA_API_KEY;
|
||||
let headers: Record<string, string> | undefined;
|
||||
const fetchMock: FetchImpl = (_url, init) => {
|
||||
headers = init?.headers as Record<string, string> | undefined;
|
||||
if (init?.body) capturedRequestBody = JSON.parse(init.body as string);
|
||||
return Promise.resolve(
|
||||
new Response(JSON.stringify({ jsonrpc: "2.0", id: "mcp-filtered", result: makeMockExaResponse() }), {
|
||||
status: 200,
|
||||
headers: { "Content-Type": "application/json" },
|
||||
}),
|
||||
);
|
||||
};
|
||||
|
||||
await searchExa({
|
||||
query: "vector databases",
|
||||
num_results: 4,
|
||||
include_domains: [" qdrant.tech "],
|
||||
exclude_domains: ["spam.example"],
|
||||
start_published_date: "2024-01-01",
|
||||
end_published_date: "2025-01-01",
|
||||
fetch: fetchMock,
|
||||
});
|
||||
|
||||
expect(headers?.["x-exa-source"]).toBe("oh-my-pi");
|
||||
expect(capturedRequestBody?.params).toEqual({
|
||||
name: "web_search_exa",
|
||||
arguments: {
|
||||
query:
|
||||
"vector databases site:qdrant.tech -site:spam.example after:2024-01-01 before:2025-01-01",
|
||||
numResults: 4,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("explains how to escape the keyless MCP rate limit", async () => {
|
||||
delete process.env.EXA_API_KEY;
|
||||
const fetchMock: FetchImpl = () =>
|
||||
Promise.resolve(new Response("too many requests", { status: 429, statusText: "Too Many Requests" }));
|
||||
|
||||
await expect(searchExa({ query: "rate limited", fetch: fetchMock })).rejects.toThrow(
|
||||
"exa: MCP rate limit reached (429); configure an Exa API key for higher limits",
|
||||
);
|
||||
});
|
||||
|
||||
it("surfaces MCP tool-level errors", async () => {
|
||||
delete process.env.EXA_API_KEY;
|
||||
const fetchMock: FetchImpl = () =>
|
||||
Promise.resolve(
|
||||
new Response(
|
||||
JSON.stringify({
|
||||
jsonrpc: "2.0",
|
||||
id: "mcp-error",
|
||||
result: { isError: true, content: [{ type: "text", text: "tool quota exceeded" }] },
|
||||
}),
|
||||
{ status: 200, headers: { "Content-Type": "application/json" } },
|
||||
),
|
||||
);
|
||||
|
||||
await expect(searchExa({ query: "tool error", fetch: fetchMock })).rejects.toThrow("tool quota exceeded");
|
||||
});
|
||||
|
||||
it("parses Exa MCP plain-text payloads when API key is missing", async () => {
|
||||
delete process.env.EXA_API_KEY;
|
||||
const fetchMock: FetchImpl = () => {
|
||||
|
||||
@@ -235,18 +235,58 @@ describe("Firecrawl web search provider", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps keyless Firecrawl out of auto selection while allowing explicit selection", () => {
|
||||
it("keeps hosted keyless Firecrawl explicit-only but admits configured self-hosting", () => {
|
||||
const originalApiKey = process.env.FIRECRAWL_API_KEY;
|
||||
const originalBaseUrl = process.env.FIRECRAWL_BASE_URL;
|
||||
const originalApiUrl = process.env.FIRECRAWL_API_URL;
|
||||
delete process.env.FIRECRAWL_API_KEY;
|
||||
delete process.env.FIRECRAWL_BASE_URL;
|
||||
delete process.env.FIRECRAWL_API_URL;
|
||||
try {
|
||||
const provider = new FirecrawlProvider();
|
||||
const authStorage = makeAuthStorage(undefined);
|
||||
|
||||
expect(provider.isAvailable(authStorage)).toBe(false);
|
||||
expect(provider.isExplicitlyAvailable(authStorage)).toBe(true);
|
||||
process.env.FIRECRAWL_BASE_URL = "http://localhost:3002";
|
||||
expect(provider.isAvailable(authStorage)).toBe(true);
|
||||
} finally {
|
||||
if (originalApiKey === undefined) delete process.env.FIRECRAWL_API_KEY;
|
||||
else process.env.FIRECRAWL_API_KEY = originalApiKey;
|
||||
if (originalBaseUrl === undefined) delete process.env.FIRECRAWL_BASE_URL;
|
||||
else process.env.FIRECRAWL_BASE_URL = originalBaseUrl;
|
||||
if (originalApiUrl === undefined) delete process.env.FIRECRAWL_API_URL;
|
||||
else process.env.FIRECRAWL_API_URL = originalApiUrl;
|
||||
}
|
||||
});
|
||||
|
||||
it("uses a self-hosted endpoint and accepts Firecrawl v1 array responses", async () => {
|
||||
const originalBaseUrl = process.env.FIRECRAWL_BASE_URL;
|
||||
process.env.FIRECRAWL_BASE_URL = "http://localhost:3002/v1/";
|
||||
let requestUrl = "";
|
||||
try {
|
||||
const fetchMock: FetchImpl = async input => {
|
||||
requestUrl = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url;
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
success: true,
|
||||
data: [{ title: "Legacy result", url: "https://example.com/legacy", snippet: "Legacy snippet" }],
|
||||
}),
|
||||
{ status: 200, headers: { "Content-Type": "application/json" } },
|
||||
);
|
||||
};
|
||||
const response = await searchFirecrawl({
|
||||
...makeParams("legacy query", makeAuthStorage(undefined)),
|
||||
fetch: fetchMock,
|
||||
});
|
||||
|
||||
expect(requestUrl).toBe("http://localhost:3002/v1/search");
|
||||
expect(response.sources).toEqual([
|
||||
{ title: "Legacy result", url: "https://example.com/legacy", snippet: "Legacy snippet" },
|
||||
]);
|
||||
} finally {
|
||||
if (originalBaseUrl === undefined) delete process.env.FIRECRAWL_BASE_URL;
|
||||
else process.env.FIRECRAWL_BASE_URL = originalBaseUrl;
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@ const DEVELOPER_SSE_RESPONSE =
|
||||
const DEVELOPER_SSE_RESPONSE_WITHOUT_MODEL =
|
||||
'data: {"candidates":[{"content":{"role":"model","parts":[{"text":"Developer answer"}]},"groundingMetadata":{"webSearchQueries":["latest Bun version"],"groundingChunks":[{"web":{"uri":"https://bun.sh","title":"Bun"}}],"groundingSupports":[{"segment":{"text":"Developer answer"},"groundingChunkIndices":[0]}]}}],"usageMetadata":{"promptTokenCount":3,"candidatesTokenCount":4,"totalTokenCount":7}}\n\n';
|
||||
const ORIGINAL_GEMINI_SEARCH_MODEL = Bun.env.GEMINI_SEARCH_MODEL;
|
||||
const ORIGINAL_GEMINI_BASE_URL = Bun.env.GOOGLE_GEMINI_BASE_URL;
|
||||
|
||||
type CapturedRequest = {
|
||||
url: string;
|
||||
@@ -72,6 +73,11 @@ describe("searchGemini tools serialization", () => {
|
||||
} else {
|
||||
Bun.env.GEMINI_SEARCH_MODEL = ORIGINAL_GEMINI_SEARCH_MODEL;
|
||||
}
|
||||
if (ORIGINAL_GEMINI_BASE_URL === undefined) {
|
||||
delete Bun.env.GOOGLE_GEMINI_BASE_URL;
|
||||
} else {
|
||||
Bun.env.GOOGLE_GEMINI_BASE_URL = ORIGINAL_GEMINI_BASE_URL;
|
||||
}
|
||||
});
|
||||
|
||||
function makeParams(query: string) {
|
||||
@@ -111,6 +117,61 @@ describe("searchGemini tools serialization", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("routes Cloudflare AI Gateway auth through AuthStorage without leaking a Google API key", async () => {
|
||||
Bun.env.GOOGLE_GEMINI_BASE_URL =
|
||||
"https://gateway.ai.cloudflare.com/v1/account/gateway/google-ai-studio";
|
||||
const gatewayAuthStorage = {
|
||||
async getOAuthAccess() {
|
||||
return undefined;
|
||||
},
|
||||
hasOAuth() {
|
||||
return false;
|
||||
},
|
||||
hasAuth(provider: string) {
|
||||
return provider === "cloudflare-ai-gateway";
|
||||
},
|
||||
async getApiKey(provider: string) {
|
||||
return provider === "cloudflare-ai-gateway" ? "test-cloudflare-key" : undefined;
|
||||
},
|
||||
} as unknown as AuthStorage;
|
||||
const fetchMock = mockGeminiFetch(DEVELOPER_SSE_RESPONSE);
|
||||
|
||||
expect(new GeminiProvider().isAvailable(gatewayAuthStorage)).toBe(true);
|
||||
await searchGemini({
|
||||
...makeParams("gateway"),
|
||||
authStorage: gatewayAuthStorage,
|
||||
fetch: fetchMock,
|
||||
});
|
||||
|
||||
expect(capturedRequest?.url).toBe(
|
||||
"https://gateway.ai.cloudflare.com/v1/account/gateway/google-ai-studio/v1beta/models/gemini-2.5-flash:streamGenerateContent?alt=sse",
|
||||
);
|
||||
expect(capturedRequest?.headers["cf-aig-authorization"]).toBe("Bearer test-cloudflare-key");
|
||||
expect(capturedRequest?.headers["x-goog-api-key"]).toBeUndefined();
|
||||
});
|
||||
|
||||
it("redacts the active credential from Gemini API errors", async () => {
|
||||
let thrown: unknown;
|
||||
try {
|
||||
await searchGemini({
|
||||
...makeParams("redaction"),
|
||||
authStorage: apiKeyAuthStorage,
|
||||
fetch: () =>
|
||||
Promise.resolve(
|
||||
new Response("upstream echoed test-gemini-api-key", {
|
||||
status: 418,
|
||||
}),
|
||||
),
|
||||
});
|
||||
} catch (error) {
|
||||
thrown = error;
|
||||
}
|
||||
|
||||
expect(thrown).toBeInstanceOf(Error);
|
||||
expect((thrown as Error).message).toContain("[redacted]");
|
||||
expect((thrown as Error).message).not.toContain("test-gemini-api-key");
|
||||
});
|
||||
|
||||
it("normalizes query directive aliases to canonical Google forms in the grounding request", async () => {
|
||||
const fetchMock = mockGeminiFetch();
|
||||
await searchGemini({
|
||||
@@ -221,4 +282,55 @@ describe("searchGemini tools serialization", () => {
|
||||
tools: [{ googleSearch: {} }, { codeExecution: {} }, { urlContext: { allowedDomains: ["example.com"] } }],
|
||||
});
|
||||
});
|
||||
|
||||
it("resolves Google grounding proxy URLs in both sources and citations", async () => {
|
||||
const proxyUrl =
|
||||
"https://vertexaisearch.cloud.google.com/grounding-api-redirect/abc";
|
||||
const responseText = `data: ${JSON.stringify({
|
||||
candidates: [
|
||||
{
|
||||
content: { role: "model", parts: [{ text: "Grounded answer" }] },
|
||||
groundingMetadata: {
|
||||
groundingChunks: [{ web: { uri: proxyUrl, title: "Example" } }],
|
||||
groundingSupports: [{ segment: { text: "Grounded answer" }, groundingChunkIndices: [0] }],
|
||||
},
|
||||
},
|
||||
],
|
||||
})}\n\n`;
|
||||
const methods: string[] = [];
|
||||
const fetchMock: FetchImpl = (_url, init) => {
|
||||
methods.push(init?.method ?? "GET");
|
||||
if (init?.method === "HEAD") {
|
||||
return Promise.resolve(
|
||||
new Response(null, {
|
||||
status: 302,
|
||||
headers: { location: "https://example.com/article" },
|
||||
}),
|
||||
);
|
||||
}
|
||||
return Promise.resolve(new Response(responseText, { status: 200 }));
|
||||
};
|
||||
|
||||
const response = await searchGemini({
|
||||
...makeParams("grounding redirect"),
|
||||
authStorage: apiKeyAuthStorage,
|
||||
fetch: fetchMock,
|
||||
});
|
||||
|
||||
expect(methods).toEqual(["POST", "HEAD"]);
|
||||
expect(response.sources).toEqual([{ title: "Example", url: "https://example.com/article" }]);
|
||||
expect(response.citations).toEqual([
|
||||
{ title: "Example", url: "https://example.com/article", citedText: "Grounded answer" },
|
||||
]);
|
||||
});
|
||||
|
||||
it("rejects a successful Gemini response with no answer or grounding results", async () => {
|
||||
await expect(
|
||||
searchGemini({
|
||||
...makeParams("empty"),
|
||||
authStorage: apiKeyAuthStorage,
|
||||
fetch: mockGeminiFetch("data: {}\n\n"),
|
||||
}),
|
||||
).rejects.toThrow("Gemini API returned an empty grounded response");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -58,6 +58,34 @@ describe("Kagi web search error handling", () => {
|
||||
"Kagi API error (502)",
|
||||
);
|
||||
});
|
||||
|
||||
it("reports malformed success responses as Kagi API errors", async () => {
|
||||
const invalidJsonFetch: FetchImpl = async () => new Response("<html>not json</html>", { status: 200 });
|
||||
const invalidEnvelopeFetch: FetchImpl = async () =>
|
||||
new Response(JSON.stringify(["unexpected"]), {
|
||||
status: 200,
|
||||
headers: { "Content-Type": "application/json" },
|
||||
});
|
||||
|
||||
await expect(searchWithKagi("invalid json", { fetch: invalidJsonFetch }, fakeAuthStorage)).rejects.toThrow(
|
||||
"Kagi API returned an invalid response: invalid JSON",
|
||||
);
|
||||
await expect(searchWithKagi("invalid envelope", { fetch: invalidEnvelopeFetch }, fakeAuthStorage)).rejects.toThrow(
|
||||
"Kagi API returned an invalid response: expected an object envelope",
|
||||
);
|
||||
});
|
||||
|
||||
it("recognizes errors plural in a successful HTTP envelope", async () => {
|
||||
const fetchMock: FetchImpl = async () =>
|
||||
new Response(JSON.stringify({ errors: [{ code: 429, message: "quota exceeded" }] }), {
|
||||
status: 200,
|
||||
headers: { "Content-Type": "application/json" },
|
||||
});
|
||||
|
||||
await expect(searchWithKagi("envelope error", { fetch: fetchMock }, fakeAuthStorage)).rejects.toThrow(
|
||||
"Kagi API error (429): quota exceeded",
|
||||
);
|
||||
});
|
||||
it("applies the configured timeout at the provider fetch boundary", async () => {
|
||||
const timeoutSignal = new AbortController().signal;
|
||||
const timeoutSpy = vi.spyOn(AbortSignal, "timeout").mockReturnValue(timeoutSignal);
|
||||
@@ -158,6 +186,39 @@ describe("Kagi search result parsing", () => {
|
||||
expect(result.answer).toBeUndefined();
|
||||
});
|
||||
|
||||
it("accepts documented result aliases and skips malformed items", async () => {
|
||||
const fetchMock: FetchImpl = async () =>
|
||||
new Response(
|
||||
JSON.stringify({
|
||||
data: {
|
||||
search: [
|
||||
null,
|
||||
{ title: "Missing URL" },
|
||||
{
|
||||
href: "https://example.com/alias",
|
||||
name: "Alias Result",
|
||||
description: "Alias description",
|
||||
},
|
||||
],
|
||||
related_search: { invalid: true },
|
||||
},
|
||||
}),
|
||||
{ status: 200, headers: { "Content-Type": "application/json" } },
|
||||
);
|
||||
|
||||
const result = await searchWithKagi("aliases", { fetch: fetchMock }, fakeAuthStorage);
|
||||
|
||||
expect(result.sources).toEqual([
|
||||
{
|
||||
title: "Alias Result",
|
||||
url: "https://example.com/alias",
|
||||
snippet: "Alias description",
|
||||
publishedDate: undefined,
|
||||
},
|
||||
]);
|
||||
expect(result.relatedQuestions).toEqual([]);
|
||||
});
|
||||
|
||||
it("parses direct_answer into the answer field", async () => {
|
||||
const fetchMock: FetchImpl = async () =>
|
||||
new Response(
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "bun:test";
|
||||
import { afterEach, beforeEach, describe, expect, it, setSystemTime, vi } from "bun:test";
|
||||
import type { AuthStorage, FetchImpl } from "@oh-my-pi/pi-ai";
|
||||
import type { AgentStorage } from "@oh-my-pi/pi-coding-agent/session/agent-storage";
|
||||
import { searchWithParallel } from "@oh-my-pi/pi-coding-agent/web/parallel";
|
||||
@@ -149,6 +149,29 @@ describe("Parallel web search", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("maps recency onto source_policy.after_date", async () => {
|
||||
setSystemTime(new Date("2026-08-10T12:00:00Z"));
|
||||
try {
|
||||
const fetchMock = mockFetch({
|
||||
search_id: "search-parallel-recency",
|
||||
results: [],
|
||||
warnings: null,
|
||||
usage: null,
|
||||
});
|
||||
|
||||
await searchParallel({ query: "recent api changes", recency: "week", fetch: fetchMock }, fakeAuthStorage);
|
||||
expect(capturedRequestBody).toEqual({
|
||||
objective: "recent api changes",
|
||||
search_queries: ["recent api changes"],
|
||||
mode: "fast",
|
||||
excerpts: { max_chars_per_result: 10_000 },
|
||||
source_policy: { after_date: "2026-08-03" },
|
||||
});
|
||||
} finally {
|
||||
setSystemTime();
|
||||
}
|
||||
});
|
||||
|
||||
it("maps -site: and after: onto exclude_domains/after_date, keeping phrases and negation", async () => {
|
||||
const fetchMock = mockFetch({
|
||||
search_id: "search-parallel-4",
|
||||
@@ -158,7 +181,11 @@ describe("Parallel web search", () => {
|
||||
});
|
||||
|
||||
await searchParallel(
|
||||
{ query: '"web api" -legacy -site:reddit.com/r/node after:2025-06-01', fetch: fetchMock },
|
||||
{
|
||||
query: '"web api" -legacy -site:reddit.com/r/node after:2025-06-01',
|
||||
recency: "day",
|
||||
fetch: fetchMock,
|
||||
},
|
||||
fakeAuthStorage,
|
||||
);
|
||||
expect(capturedRequestBody).toEqual({
|
||||
@@ -178,4 +205,13 @@ describe("Parallel web search", () => {
|
||||
message: "Parallel API error (503): upstream unavailable",
|
||||
});
|
||||
});
|
||||
|
||||
it("classifies malformed successful responses as Parallel errors", async () => {
|
||||
const fetchMock: FetchImpl = () =>
|
||||
Promise.resolve(new Response("{not-json", { status: 200, headers: { "Content-Type": "application/json" } }));
|
||||
await expect(searchParallel({ query: "broken", fetch: fetchMock }, fakeAuthStorage)).rejects.toMatchObject({
|
||||
provider: "parallel",
|
||||
message: expect.stringContaining("Parallel search returned invalid JSON:"),
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -121,6 +121,53 @@ describe("SearXNG web search provider", () => {
|
||||
expect(captured.url?.searchParams.get("language")).toBeNull();
|
||||
});
|
||||
|
||||
it("sends configured category and safe-search filters and preserves instant answers", async () => {
|
||||
const agentDir = await fs.mkdtemp(path.join(os.tmpdir(), "searxng-filters-"));
|
||||
try {
|
||||
await Bun.write(
|
||||
path.join(agentDir, "config.yml"),
|
||||
[
|
||||
"searxng:",
|
||||
" endpoint: https://searx.example.org",
|
||||
" categories: news",
|
||||
" safesearch: 2",
|
||||
"",
|
||||
].join("\n"),
|
||||
);
|
||||
await Settings.init({ agentDir });
|
||||
|
||||
const captured: { url?: URL } = {};
|
||||
const fetchMock: FetchImpl = input => {
|
||||
captured.url = new URL(input.toString());
|
||||
return Promise.resolve(
|
||||
new Response(
|
||||
JSON.stringify({
|
||||
results: [{ title: "r", url: "https://example.com", snippet: "Fallback snippet" }],
|
||||
answers: [
|
||||
" Forty-two ",
|
||||
{ template: "answer/legacy.html", answer: "Legacy answer" },
|
||||
{
|
||||
template: "answer/translations.html",
|
||||
translations: [{ text: "Hallo" }, { text: "Guten Tag" }],
|
||||
},
|
||||
],
|
||||
}),
|
||||
{ status: 200, headers: { "Content-Type": "application/json" } },
|
||||
),
|
||||
);
|
||||
};
|
||||
|
||||
const response = await searchSearXNG({ query: "filtered answers", fetch: fetchMock });
|
||||
|
||||
expect(captured.url?.searchParams.get("categories")).toBe("news");
|
||||
expect(captured.url?.searchParams.get("safesearch")).toBe("2");
|
||||
expect(response.answer).toBe("Forty-two\n\nLegacy answer\n\nHallo\nGuten Tag");
|
||||
expect(response.sources[0]?.snippet).toBe("Fallback snippet");
|
||||
} finally {
|
||||
await removeWithRetries(agentDir);
|
||||
}
|
||||
});
|
||||
|
||||
it("reads Basic auth credentials from nested config.yml settings", async () => {
|
||||
const agentDir = await fs.mkdtemp(path.join(os.tmpdir(), "searxng-settings-"));
|
||||
try {
|
||||
|
||||
@@ -84,10 +84,16 @@ describe("TinyFish web search provider", () => {
|
||||
});
|
||||
|
||||
expect(captured).toHaveLength(1);
|
||||
expect(captured[0].searchParams.get("query")).toBe(
|
||||
'"error handling" rust site:github.com -site:gitlab.com filetype:pdf',
|
||||
);
|
||||
expectTinyFishParams(captured[0], ["query", "num_results", "page"]);
|
||||
expect(captured[0].searchParams.get("query")).toBe('"error handling" rust filetype:pdf');
|
||||
expect(captured[0].searchParams.get("include_domains")).toBe("github.com");
|
||||
expect(captured[0].searchParams.get("exclude_domains")).toBe("gitlab.com");
|
||||
expectTinyFishParams(captured[0], [
|
||||
"query",
|
||||
"num_results",
|
||||
"page",
|
||||
"include_domains",
|
||||
"exclude_domains",
|
||||
]);
|
||||
});
|
||||
|
||||
it("sends directive-free queries verbatim", async () => {
|
||||
@@ -232,6 +238,43 @@ describe("TinyFish web search provider", () => {
|
||||
expect(response.sources.at(-1)?.url).toBe("https://example.com/raw-page-11");
|
||||
});
|
||||
|
||||
it("deduplicates and normalizes results across pages", async () => {
|
||||
const captured: URL[] = [];
|
||||
const firstPage = tinyFishResults("dedupe", 10);
|
||||
firstPage[0] = {
|
||||
title: " Primary title ",
|
||||
url: " https://example.com/dedupe-0 ",
|
||||
snippet: " spaced \n snippet ",
|
||||
site_name: " Example ",
|
||||
};
|
||||
firstPage[1] = {
|
||||
title: "Duplicate title",
|
||||
url: "https://example.com/dedupe-0",
|
||||
snippet: "duplicate snippet",
|
||||
};
|
||||
const fetchMock: FetchImpl = async input => {
|
||||
const url = input instanceof URL ? input : new URL(typeof input === "string" ? input : input.url);
|
||||
captured.push(url);
|
||||
const page = Number(url.searchParams.get("page") ?? 0);
|
||||
const results = page === 0 ? firstPage : tinyFishResults("dedupe", 1, 10);
|
||||
return new Response(JSON.stringify(tinyFishPage(results, page, 11)), {
|
||||
status: 200,
|
||||
headers: { "Content-Type": "application/json" },
|
||||
});
|
||||
};
|
||||
|
||||
const response = await searchTinyFish({ ...makeParams("dedupe fish"), limit: 10, fetch: fetchMock });
|
||||
|
||||
expect(captured.map(url => url.searchParams.get("page"))).toEqual(["0", "1"]);
|
||||
expect(response.sources).toHaveLength(10);
|
||||
expect(response.sources[0]).toMatchObject({
|
||||
title: "Primary title",
|
||||
url: "https://example.com/dedupe-0",
|
||||
snippet: "spaced snippet",
|
||||
});
|
||||
expect(response.sources.at(-1)?.url).toBe("https://example.com/dedupe-10");
|
||||
});
|
||||
|
||||
it("stops early for limit 20 when page 0 returns fewer than 10 raw results", async () => {
|
||||
const captured: URL[] = [];
|
||||
const fetchMock: FetchImpl = async input => {
|
||||
|
||||
@@ -802,6 +802,67 @@ describe("xAI web search provider", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("extracts offset snippets and raw sources from web_search_call output", async () => {
|
||||
const answer = "Context before [cited source](https://example.com/cited) context after.";
|
||||
const start = answer.indexOf("[cited source]");
|
||||
const capture = captureFetch({
|
||||
id: "resp_raw_sources",
|
||||
output: [
|
||||
{
|
||||
type: "message",
|
||||
content: [
|
||||
{
|
||||
type: "output_text",
|
||||
text: answer,
|
||||
annotations: [
|
||||
{
|
||||
type: "url_citation",
|
||||
url: "https://example.com/cited",
|
||||
title: "Cited result",
|
||||
start_index: start,
|
||||
end_index: start + "[cited source]".length,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
type: "web_search_call",
|
||||
action: {
|
||||
sources: [
|
||||
{ url: "https://example.com/raw", title: "Raw result" },
|
||||
{ source_website_url: "https://example.com/fallback", caption: "Fallback result" },
|
||||
],
|
||||
},
|
||||
results: [{ url: "https://example.com/cited", title: "Duplicate result" }],
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const response = await searchXAI(makeParams(capture.fetchMock));
|
||||
|
||||
expect(response.answer).toBe(answer);
|
||||
expect(response.sources).toEqual([
|
||||
{
|
||||
title: "Cited result",
|
||||
url: "https://example.com/cited",
|
||||
snippet: "Context before cited source context after.",
|
||||
},
|
||||
{ title: "Raw result", url: "https://example.com/raw", snippet: undefined },
|
||||
{ title: "Fallback result", url: "https://example.com/fallback", snippet: undefined },
|
||||
]);
|
||||
});
|
||||
|
||||
it("rejects successful responses with no answer or sources", async () => {
|
||||
const capture = captureFetch({ id: "resp_empty", output: [] });
|
||||
|
||||
await expect(searchXAI(makeParams(capture.fetchMock))).rejects.toMatchObject({
|
||||
provider: "xai",
|
||||
status: 502,
|
||||
message: "xAI web_search returned no answer or sources",
|
||||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
[401, "xai: 401 unauthorized"],
|
||||
[402, "xai: 402 credits exhausted"],
|
||||
|
||||
@@ -144,8 +144,6 @@ describe("Brave provider hard-timeout wiring", () => {
|
||||
});
|
||||
|
||||
it("hands fetch a composed signal even with no caller signal — confirms the rollout reaches non-Anthropic providers", async () => {
|
||||
process.env.BRAVE_API_KEY = "brave-test-key";
|
||||
|
||||
let capturedSignal: AbortSignal | null | undefined;
|
||||
const fetchMock: FetchImpl = async (_input, init) => {
|
||||
capturedSignal = init?.signal;
|
||||
@@ -155,7 +153,13 @@ describe("Brave provider hard-timeout wiring", () => {
|
||||
});
|
||||
};
|
||||
|
||||
await searchBrave({ query: "ping", fetch: fetchMock });
|
||||
await searchBrave({
|
||||
query: "ping",
|
||||
fetch: fetchMock,
|
||||
authStorage: {
|
||||
resolver: vi.fn(() => async () => "brave-test-key"),
|
||||
} as unknown as AuthStorage,
|
||||
});
|
||||
|
||||
expect(capturedSignal).toBeInstanceOf(AbortSignal);
|
||||
expect(capturedSignal?.aborted).toBe(false);
|
||||
|
||||
@@ -75,11 +75,13 @@ describe("Perplexity API-key request shape", () => {
|
||||
const savedOpenRouterKey = process.env.OPENROUTER_API_KEY;
|
||||
const savedCookies = process.env.PERPLEXITY_COOKIES;
|
||||
const savedResponsesMode = process.env.PI_PERPLEXITY_RESPONSES;
|
||||
const savedApiModel = process.env.PI_PERPLEXITY_API_MODEL;
|
||||
|
||||
beforeEach(() => {
|
||||
process.env.PERPLEXITY_API_KEY = "test-key";
|
||||
delete process.env.PERPLEXITY_COOKIES;
|
||||
delete process.env.PI_PERPLEXITY_RESPONSES;
|
||||
delete process.env.PI_PERPLEXITY_API_MODEL;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
@@ -92,6 +94,8 @@ describe("Perplexity API-key request shape", () => {
|
||||
else process.env.PERPLEXITY_COOKIES = savedCookies;
|
||||
if (savedResponsesMode === undefined) delete process.env.PI_PERPLEXITY_RESPONSES;
|
||||
else process.env.PI_PERPLEXITY_RESPONSES = savedResponsesMode;
|
||||
if (savedApiModel === undefined) delete process.env.PI_PERPLEXITY_API_MODEL;
|
||||
else process.env.PI_PERPLEXITY_API_MODEL = savedApiModel;
|
||||
});
|
||||
|
||||
it("requests comprehensive defaults: 20 results, high context, related questions", async () => {
|
||||
@@ -104,6 +108,18 @@ describe("Perplexity API-key request shape", () => {
|
||||
expect(body?.return_related_questions).toBe(true);
|
||||
});
|
||||
|
||||
it("accepts a configured direct API model", async () => {
|
||||
process.env.PI_PERPLEXITY_API_MODEL = "sonar-deep-research";
|
||||
let body: Record<string, unknown> | undefined;
|
||||
await searchPerplexity({
|
||||
query: "quic vs tcp",
|
||||
authStorage: apiKeyAuthStorage,
|
||||
fetch: mockApi(b => (body = b), baseResponse()),
|
||||
});
|
||||
|
||||
expect(body?.model).toBe("sonar-deep-research");
|
||||
});
|
||||
|
||||
it("honors a caller-supplied num_search_results over the default", async () => {
|
||||
let body: Record<string, unknown> | undefined;
|
||||
const fetchMock = mockApi(b => (body = b), baseResponse());
|
||||
@@ -306,7 +322,10 @@ const anonymousAuthStorage = {
|
||||
},
|
||||
} as unknown as AuthStorage;
|
||||
|
||||
function mockOAuth(capture: (body: Record<string, unknown>, headers: Headers) => void): FetchImpl {
|
||||
function mockOAuth(
|
||||
capture: (body: Record<string, unknown>, headers: Headers) => void,
|
||||
eventOverrides: Record<string, unknown> = {},
|
||||
): FetchImpl {
|
||||
const event = {
|
||||
final: true,
|
||||
display_model: "turbo",
|
||||
@@ -318,6 +337,7 @@ function mockOAuth(capture: (body: Record<string, unknown>, headers: Headers) =>
|
||||
web_result_block: { web_results: [{ name: "T", url: "https://example.com", snippet: "s" }] },
|
||||
},
|
||||
],
|
||||
...eventOverrides,
|
||||
};
|
||||
const sseBody = `data: ${JSON.stringify(event)}\n\n`;
|
||||
return async (input, init) => {
|
||||
@@ -356,15 +376,19 @@ function mockAnonymous(capture: (body: Record<string, unknown>, headers: Headers
|
||||
|
||||
describe("Perplexity OAuth request shape", () => {
|
||||
const savedCookies = process.env.PERPLEXITY_COOKIES;
|
||||
const savedModel = process.env.PI_PERPLEXITY_MODEL;
|
||||
|
||||
beforeEach(() => {
|
||||
delete process.env.PERPLEXITY_COOKIES; // cookies take precedence over oauth; keep them out
|
||||
delete process.env.PI_PERPLEXITY_MODEL;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
if (savedCookies === undefined) delete process.env.PERPLEXITY_COOKIES;
|
||||
else process.env.PERPLEXITY_COOKIES = savedCookies;
|
||||
if (savedModel === undefined) delete process.env.PI_PERPLEXITY_MODEL;
|
||||
else process.env.PI_PERPLEXITY_MODEL = savedModel;
|
||||
});
|
||||
|
||||
it("sends the bare query, never the API-style system prompt, to the ask endpoint", async () => {
|
||||
@@ -395,6 +419,55 @@ describe("Perplexity OAuth request shape", () => {
|
||||
expect(headers?.has("authorization")).toBe(false);
|
||||
expect(response.authMode).toBe("oauth");
|
||||
expect(response.answer).toBe("OAuth answer");
|
||||
// Authenticated streams sometimes report only the generic `turbo` alias;
|
||||
// preserve the requested subscription model instead of misreporting it.
|
||||
expect(response.model).toBe("experimental");
|
||||
});
|
||||
|
||||
it("accepts a subscription model preference and reports it when the stream returns turbo", async () => {
|
||||
let body: Record<string, unknown> | undefined;
|
||||
const response = await searchPerplexity({
|
||||
query: "latest model",
|
||||
subscription_model: "pplx_reasoning",
|
||||
authStorage: oauthAuthStorage,
|
||||
fetch: mockOAuth(b => (body = b)),
|
||||
});
|
||||
|
||||
expect((body!.params as Record<string, unknown>).model_preference).toBe("pplx_reasoning");
|
||||
expect(response.model).toBe("pplx_reasoning");
|
||||
});
|
||||
|
||||
it("prefers the concrete user-selected model over the generic display alias", async () => {
|
||||
const response = await searchPerplexity({
|
||||
query: "latest model",
|
||||
authStorage: oauthAuthStorage,
|
||||
fetch: mockOAuth(() => {}, { user_selected_model: "pplx_pro_upgraded" }),
|
||||
});
|
||||
|
||||
expect(response.model).toBe("pplx_pro_upgraded");
|
||||
});
|
||||
|
||||
it("deduplicates equivalent subscription source URLs", async () => {
|
||||
const response = await searchPerplexity({
|
||||
query: "latest model",
|
||||
authStorage: oauthAuthStorage,
|
||||
fetch: mockOAuth(() => {}, {
|
||||
blocks: [
|
||||
{ intended_usage: "ask_text", markdown_block: { answer: "OAuth answer" } },
|
||||
{
|
||||
intended_usage: "web_results",
|
||||
web_result_block: {
|
||||
web_results: [
|
||||
{ name: "First", url: "https://EXAMPLE.com/path/" },
|
||||
{ name: "Duplicate", url: "https://example.com/path" },
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
}),
|
||||
});
|
||||
|
||||
expect(response.sources).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("maps directives onto ask-endpoint native filters and rewrites query_str", async () => {
|
||||
|
||||
@@ -9,7 +9,11 @@ import {
|
||||
} from "@oh-my-pi/pi-coding-agent/web/search/provider";
|
||||
import { SEARCH_PROVIDER_ORDER } from "@oh-my-pi/pi-coding-agent/web/search/types";
|
||||
|
||||
const authStorage = {} as AuthStorage;
|
||||
const authStorage = {
|
||||
hasAuth(provider: string): boolean {
|
||||
return provider === "jina" && Boolean(process.env.JINA_API_KEY);
|
||||
},
|
||||
} as AuthStorage;
|
||||
const originalBraveApiKey = process.env.BRAVE_API_KEY;
|
||||
const originalJinaApiKey = process.env.JINA_API_KEY;
|
||||
|
||||
|
||||
@@ -214,4 +214,35 @@ describe("Tavily searchTavily request shape (integration)", () => {
|
||||
expect(capturedBodies[1]).not.toHaveProperty("end_date");
|
||||
expect(response.sources).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("ignores malformed response fields while preserving valid results", async () => {
|
||||
process.env.TAVILY_API_KEY = "test-key";
|
||||
const fetchMock: FetchImpl = async () =>
|
||||
new Response(
|
||||
JSON.stringify({
|
||||
answer: 42,
|
||||
results: [
|
||||
null,
|
||||
{ title: 7, url: "https://example.com/valid", content: 99, published_date: false },
|
||||
{ title: "Missing URL" },
|
||||
],
|
||||
request_id: 123,
|
||||
}),
|
||||
{ status: 200, headers: { "Content-Type": "application/json" } },
|
||||
);
|
||||
|
||||
const response = await searchTavily({ ...makeParams("robust parsing"), fetch: fetchMock });
|
||||
|
||||
expect(response.answer).toBeUndefined();
|
||||
expect(response.requestId).toBeUndefined();
|
||||
expect(response.sources).toEqual([
|
||||
{
|
||||
title: "https://example.com/valid",
|
||||
url: "https://example.com/valid",
|
||||
snippet: undefined,
|
||||
publishedDate: undefined,
|
||||
ageSeconds: undefined,
|
||||
},
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user