smashimo/oh-my-pi
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(coding-agent/web): expanded web search and fetching providers

Browse Source 
- Expanded web search and fetching providers with robust parsing, authentication storage integration, and response validation.
- Added support for new configurations including SearXNG safesearch, Cloudflare AI Gateway endpoints, and dynamic Firecrawl base URLs.
- Implemented comprehensive test suites covering error handling, content filtering, and provider-specific response behaviors.
This commit is contained in:
can1357
2026-08-10 11:06:22 +02:00
parent 45e12e5bb7
commit baf8a1df7e
31 changed files with 1660 additions and 224 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/coding-agent/src/config/settings-schema.ts
+5
View File
@@ -5471,6 +5471,11 @@ export const SETTINGS_SCHEMA = {
default: undefined,
},
"searxng.safesearch": {
type: "number",
default: undefined,
},
"commit.mapReduceEnabled": { type: "boolean", default: true },
"commit.mapReduceMinFiles": { type: "number", default: 4 },
packages/coding-agent/src/tools/fetch.ts
+25 -2
View File
@@ -576,6 +576,23 @@ async function parseFeedToMarkdown(content: string, maxItems = 10): Promise<stri
* local fallback renderers (trafilatura, lynx, native). See #1449.
*/
const REMOTE_READER_MAX_MS = 10_000;
const JINA_MARKDOWN_MARKER = "Markdown Content:";
const JINA_READER_MAX_BYTES = 2 * 1024 * 1024;
function parseJinaReaderContent(responseBody: string): string | null {
const markerStart = responseBody.indexOf(JINA_MARKDOWN_MARKER);
if (markerStart < 0) return null;
const content = responseBody.slice(markerStart + JINA_MARKDOWN_MARKER.length).trim();
if (
content.length < 100 ||
content.startsWith("Loading...") ||
content.startsWith("Please enable JavaScript")
) {
return null;
}
return content;
}
/** Reader backends for {@link renderHtmlToText}, in default priority order. */
export type FetchProvider = "native" | "trafilatura" | "lynx" | "parallel" | "jina";
@@ -653,10 +670,16 @@ export async function renderHtmlToText(
},
jina: async () => {
const response = await fetchImpl(`https://r.jina.ai/${url}`, {
headers: { Accept: "text/markdown" },
headers: {
Accept: "text/markdown",
"X-No-Cache": "true",
},
signal: remoteSignal(),
});
return response.ok ? await response.text() : null;
if (!response.ok) return null;
const contentLength = Number(response.headers.get("content-length"));
if (Number.isFinite(contentLength) && contentLength > JINA_READER_MAX_BYTES) return null;
return parseJinaReaderContent(await response.text());
},
};
packages/coding-agent/src/web/kagi.ts
+92 -34
View File
@@ -83,6 +83,7 @@ export interface KagiErrorResponse {
detail?: string;
}
// ---------------------------------------------------------------------------
// Error Handling
// ---------------------------------------------------------------------------
@@ -98,7 +99,7 @@ export class KagiApiError extends Error {
}
function extractKagiErrorMessage(payload: unknown): string | null {
if (!payload || typeof payload !== "object") return null;
if (!payload || typeof payload !== "object" || Array.isArray(payload)) return null;
const record = payload as Record<string, unknown>;
for (const value of [record.message, record.detail]) {
@@ -107,17 +108,20 @@ function extractKagiErrorMessage(payload: unknown): string | null {
}
}
if (typeof record.error === "string" && record.error.trim().length > 0) {
return record.error.trim();
}
if (Array.isArray(record.error)) {
for (const entry of record.error) {
for (const errors of [record.error, record.errors]) {
if (typeof errors === "string" && errors.trim().length > 0) {
return errors.trim();
}
if (!Array.isArray(errors)) continue;
for (const entry of errors) {
if (!entry || typeof entry !== "object") continue;
const e = entry as Record<string, unknown>;
for (const value of [e.message, e.msg]) {
if (typeof value === "string" && value.trim().length > 0) {
return value.trim();
for (const value of [e.message, e.msg, e.code]) {
if (
(typeof value === "string" && value.trim().length > 0) ||
(typeof value === "number" && Number.isFinite(value))
) {
return String(value).trim();
}
}
}
@@ -147,6 +151,34 @@ function parseKagiErrorResponse(statusCode: number, responseText: string): KagiA
}
}
function parseKagiSuccessResponse(statusCode: number, responseText: string): KagiSearchResponse {
let payload: unknown;
try {
payload = JSON.parse(responseText);
} catch {
throw new KagiApiError("Kagi API returned an invalid response: invalid JSON", statusCode);
}
if (!payload || typeof payload !== "object" || Array.isArray(payload)) {
throw new KagiApiError("Kagi API returned an invalid response: expected an object envelope", statusCode);
}
const record = payload as Record<string, unknown>;
const errorMessage = extractKagiErrorMessage(payload);
if (errorMessage && (record.error !== undefined || record.errors !== undefined)) {
const errors = Array.isArray(record.error) ? record.error : Array.isArray(record.errors) ? record.errors : [];
const first = errors[0];
const code =
first && typeof first === "object" && typeof (first as Record<string, unknown>).code === "number"
? ((first as Record<string, unknown>).code as number)
: statusCode;
throw createKagiApiError(code, errorMessage);
}
if (record.data !== undefined && (!record.data || typeof record.data !== "object" || Array.isArray(record.data))) {
throw new KagiApiError("Kagi API returned an invalid response: expected data to be an object", statusCode);
}
return payload as KagiSearchResponse;
}
// ---------------------------------------------------------------------------
// Public API
// ---------------------------------------------------------------------------
@@ -216,23 +248,40 @@ function buildRequestBody(query: string, options: KagiSearchOptions): KagiSearch
return req;
}
/** Push every item in a result bucket as a source, with an optional title tag. */
function collectSources(sources: KagiSearchSource[], items: KagiSearchResultItem[] | undefined, tag?: string): void {
if (!items) return;
for (const item of items) {
function firstNonEmptyString(...values: unknown[]): string | undefined {
for (const value of values) {
if (typeof value === "string" && value.trim().length > 0) return value.trim();
}
return undefined;
}
/** Push every valid item in a result bucket as a source, with an optional title tag. */
function collectSources(sources: KagiSearchSource[], items: unknown, tag?: string): void {
if (!Array.isArray(items)) return;
for (const value of items) {
if (!value || typeof value !== "object" || Array.isArray(value)) continue;
const item = value as Record<string, unknown>;
const url = firstNonEmptyString(item.url, item.href, item.link);
if (!url) continue;
const title = firstNonEmptyString(item.title, item.name) ?? url;
sources.push({
title: tag ? `${tag} ${item.title}` : item.title,
url: item.url,
snippet: item.snippet,
publishedDate: item.time,
title: tag ? `${tag} ${title}` : title,
url,
snippet: firstNonEmptyString(item.snippet, item.description, item.summary),
publishedDate: firstNonEmptyString(item.time),
});
}
}
/** Pull a related/adjacent question from an item's props or fall back to title. */
function questionOf(item: KagiSearchResultItem): string | undefined {
const q = item.props?.question ?? item.props?.query ?? item.title;
return typeof q === "string" && q.length > 0 ? q : undefined;
function questionOf(value: unknown): string | undefined {
if (!value || typeof value !== "object" || Array.isArray(value)) return undefined;
const item = value as Record<string, unknown>;
const props =
item.props && typeof item.props === "object" && !Array.isArray(item.props)
? (item.props as Record<string, unknown>)
: undefined;
return firstNonEmptyString(props?.question, props?.query, item.title);
}
export async function searchWithKagi(
@@ -269,11 +318,7 @@ export async function searchWithKagi(
},
);
const payload = (await response.json()) as KagiSearchResponse;
if (payload.error && payload.error.length > 0) {
const first = payload.error[0];
throw createKagiApiError(first.code ?? response.status, extractKagiErrorMessage(payload) ?? first.message);
}
const payload = parseKagiSuccessResponse(response.status, await response.text());
const data = payload.data;
const sources: KagiSearchSource[] = [];
@@ -284,17 +329,30 @@ export async function searchWithKagi(
collectSources(sources, data?.news, "[News]");
collectSources(sources, data?.infobox, "[Info]");
for (const item of data?.adjacent_question ?? []) {
const q = questionOf(item);
if (q) relatedQuestions.push(q);
const adjacentQuestions: unknown = data?.adjacent_question;
if (Array.isArray(adjacentQuestions)) {
for (const item of adjacentQuestions) {
const question = questionOf(item);
if (question) relatedQuestions.push(question);
}
}
for (const item of data?.related_search ?? []) {
const q = questionOf(item);
if (q) relatedQuestions.push(q);
const relatedSearches: unknown = data?.related_search;
if (Array.isArray(relatedSearches)) {
for (const item of relatedSearches) {
const question = questionOf(item);
if (question) relatedQuestions.push(question);
}
}
const directAnswer = data?.direct_answer?.[0];
const answer = directAnswer ? (directAnswer.snippet ?? directAnswer.title) : undefined;
const directAnswers: unknown = data?.direct_answer;
const directAnswer = Array.isArray(directAnswers) ? directAnswers[0] : undefined;
const answer =
directAnswer && typeof directAnswer === "object" && !Array.isArray(directAnswer)
? firstNonEmptyString(
(directAnswer as Record<string, unknown>).snippet,
(directAnswer as Record<string, unknown>).title,
)
: undefined;
return {
requestId: payload.meta?.trace ?? payload.meta?.id ?? "",
packages/coding-agent/src/web/parallel.ts
+14 -2
View File
@@ -146,6 +146,18 @@ export function parseParallelErrorResponse(statusCode: number, responseText: str
}
}
export async function parseParallelJsonResponse(
response: Response,
operation: "search" | "extract",
): Promise<unknown> {
try {
return await response.json();
} catch (err) {
const detail = err instanceof Error ? err.message : String(err);
throw new ParallelApiError(`Parallel ${operation} returned invalid JSON: ${detail}`);
}
}
function getAuthHeaders(apiKey: string): {
Accept: string;
"Content-Type": string;
@@ -316,7 +328,7 @@ export async function searchWithParallel(
throw parseParallelErrorResponse(response.status, await response.text());
}
const payload: unknown = await response.json();
const payload = await parseParallelJsonResponse(response, "search");
return parseParallelSearchPayload(payload);
}
@@ -349,6 +361,6 @@ export async function extractWithParallel(
throw parseParallelErrorResponse(response.status, await response.text());
}
const payload: unknown = await response.json();
const payload = await parseParallelJsonResponse(response, "extract");
return parseExtractPayload(payload);
}
packages/coding-agent/src/web/search/providers/brave.ts
+123 -46
View File
@@ -4,7 +4,7 @@
* Calls Brave's web search REST API and maps results into the unified
* SearchResponse shape used by the web search tool.
*/
import { type AuthStorage, type FetchImpl, getEnvApiKey } from "@oh-my-pi/pi-ai";
import { type ApiKey, type AuthStorage, type FetchImpl, getEnvApiKey, withAuth } from "@oh-my-pi/pi-ai";
import type { SearchResponse, SearchSource } from "../../../web/search/types";
import { SearchProviderError } from "../../../web/search/types";
import type { QuerySyntax, StructuredQuery } from "../query";
@@ -17,6 +17,9 @@ import { classifyProviderHttpError, withHardTimeout } from "./utils";
const BRAVE_SEARCH_URL = "https://api.search.brave.com/res/v1/web/search";
const DEFAULT_NUM_RESULTS = 10;
const MAX_NUM_RESULTS = 20;
const MAX_QUERY_CHARACTERS = 500;
const MAX_RESPONSE_BYTES = 2 * 1024 * 1024;
const MAX_ERROR_BYTES = 8 * 1024;
const RECENCY_MAP: Record<"day" | "week" | "month" | "year", "pd" | "pw" | "pm" | "py"> = {
day: "pd",
@@ -51,62 +54,122 @@ export interface BraveSearchParams {
num_results?: number;
recency?: "day" | "week" | "month" | "year";
parsedQuery?: StructuredQuery;
/** Two-letter market code, or `ALL`. */
country?: string;
/** Brave search language code, such as `en` or `zh-hans`. */
search_lang?: string;
safesearch?: "off" | "moderate" | "strict";
authStorage: AuthStorage;
sessionId?: string;
signal?: AbortSignal;
timeoutMs?: number;
fetch?: FetchImpl;
}
interface BraveSearchResult {
title?: string | null;
url?: string | null;
description?: string | null;
age?: string | null;
extra_snippets?: string[] | null;
}
interface BraveSearchResponse {
web?: {
results?: BraveSearchResult[];
};
web?: unknown;
}
/** Find BRAVE_API_KEY from environment or .env files. */
export function findApiKey(): string | null {
return getEnvApiKey("brave") ?? null;
function normalizeText(value: unknown, maxLength: number): string | undefined {
if (typeof value !== "string") return undefined;
const text = value
.replace(/<[^>]*>/g, " ")
.replace(/\s+/g, " ")
.trim();
if (!text) return undefined;
return text.length <= maxLength ? text : `${text.slice(0, maxLength - 1)}…`;
}
function buildSnippet(result: BraveSearchResult): string | undefined {
const snippets: string[] = [];
function normalizeUrl(value: unknown): string | undefined {
if (typeof value !== "string" || value.length > 2048) return undefined;
try {
const url = new URL(value);
if (url.protocol !== "http:" && url.protocol !== "https:") return undefined;
return url.toString();
} catch {
return undefined;
}
}
if (result.description?.trim()) {
snippets.push(result.description.trim());
function webResults(response: BraveSearchResponse): readonly unknown[] {
if (typeof response.web !== "object" || response.web === null || !("results" in response.web)) return [];
return Array.isArray(response.web.results) ? response.web.results : [];
}
async function readLimitedText(response: Response, maxBytes: number, truncate = false): Promise<string> {
if (!response.body) return "";
const reader = response.body.getReader();
let buffer = new Uint8Array(Math.min(maxBytes, 64 * 1024));
let bytes = 0;
try {
for (;;) {
const { done, value } = await reader.read();
if (done) break;
const accepted = Math.min(value.byteLength, maxBytes - bytes);
const nextBytes = bytes + accepted;
if (nextBytes > buffer.byteLength) {
const grown = new Uint8Array(Math.min(maxBytes, Math.max(nextBytes, buffer.byteLength * 2)));
grown.set(buffer.subarray(0, bytes));
buffer = grown;
}
buffer.set(value.subarray(0, accepted), bytes);
bytes = nextBytes;
if (accepted < value.byteLength) {
await reader.cancel().catch(() => undefined);
if (!truncate) throw new SearchProviderError("brave", "Brave API response exceeded 2 MiB", 500);
break;
}
}
} finally {
reader.releaseLock();
}
if (Array.isArray(result.extra_snippets)) {
for (const snippet of result.extra_snippets) {
if (!snippet?.trim()) continue;
if (snippets.includes(snippet.trim())) continue;
snippets.push(snippet.trim());
return new TextDecoder().decode(buffer.subarray(0, bytes));
}
function buildSnippet(result: object): string | undefined {
const snippets = new Set<string>();
const description = normalizeText("description" in result ? result.description : undefined, 8_000);
if (description) snippets.add(description);
const extras = "extra_snippets" in result ? result.extra_snippets : undefined;
if (Array.isArray(extras)) {
for (const value of extras) {
const snippet = normalizeText(value, 8_000);
if (snippet) snippets.add(snippet);
}
}
return snippets.length > 0 ? snippets.join("\n") : undefined;
const combined = [...snippets].join("\n");
return combined ? (combined.length <= 8_000 ? combined : `${combined.slice(0, 7_999)}…`) : undefined;
}
async function callBraveSearch(
apiKey: string,
params: BraveSearchParams,
): Promise<{ response: BraveSearchResponse; requestId?: string }> {
const numResults = clampNumResults(params.num_results, DEFAULT_NUM_RESULTS, MAX_NUM_RESULTS);
const numResults = Math.floor(clampNumResults(params.num_results, DEFAULT_NUM_RESULTS, MAX_NUM_RESULTS));
const parsed = params.parsedQuery ?? parseSearchQuery(params.query);
const query = parsed.hasDirectives ? formatQuery(parsed, BRAVE_QUERY_SYNTAX) : params.query;
if (query.length > MAX_QUERY_CHARACTERS) {
throw new SearchProviderError(
"brave",
`Brave search queries cannot exceed ${MAX_QUERY_CHARACTERS} characters`,
400,
);
}
const url = new URL(BRAVE_SEARCH_URL);
url.searchParams.set("q", parsed.hasDirectives ? formatQuery(parsed, BRAVE_QUERY_SYNTAX) : params.query);
url.searchParams.set("q", query);
url.searchParams.set("count", String(numResults));
url.searchParams.set("extra_snippets", "true");
url.searchParams.set("text_decorations", "false");
url.searchParams.set("safesearch", params.safesearch ?? "moderate");
if (params.country) url.searchParams.set("country", params.country.toUpperCase());
if (params.search_lang) url.searchParams.set("search_lang", params.search_lang);
const freshness = braveFreshness(parsed, params.recency);
if (freshness) {
url.searchParams.set("freshness", freshness);
}
if (freshness) url.searchParams.set("freshness", freshness);
const fetchImpl = params.fetch ?? fetch;
const response = await fetchImpl(url, {
@@ -118,36 +181,47 @@ async function callBraveSearch(
});
if (!response.ok) {
const errorText = await response.text();
const errorText = await readLimitedText(response, MAX_ERROR_BYTES, true);
const classified = classifyProviderHttpError("brave", response.status, errorText);
if (classified) throw classified;
throw new SearchProviderError("brave", `Brave API error (${response.status}): ${errorText}`, response.status);
}
const data = (await response.json()) as BraveSearchResponse;
const raw = await readLimitedText(response, MAX_RESPONSE_BYTES);
let data: BraveSearchResponse;
try {
data = JSON.parse(raw) as BraveSearchResponse;
} catch {
throw new SearchProviderError("brave", "Brave API returned invalid JSON", 500);
}
const requestId = response.headers.get("x-request-id") ?? response.headers.get("request-id") ?? undefined;
return { response: data, requestId };
}
/** Execute Brave web search. */
export async function searchBrave(params: BraveSearchParams): Promise<SearchResponse> {
const numResults = clampNumResults(params.num_results, DEFAULT_NUM_RESULTS, MAX_NUM_RESULTS);
const apiKey = findApiKey();
if (!apiKey) {
throw new Error("BRAVE_API_KEY not found. Set it in environment or .env file.");
}
const { response, requestId } = await callBraveSearch(apiKey, params);
const numResults = Math.floor(clampNumResults(params.num_results, DEFAULT_NUM_RESULTS, MAX_NUM_RESULTS));
const keyOrResolver: ApiKey = params.authStorage.resolver("brave", {
sessionId: params.sessionId,
});
const { response, requestId } = await withAuth(keyOrResolver, key => callBraveSearch(key, params), {
signal: params.signal,
missingKeyMessage:
'Brave credentials not found. Set BRAVE_API_KEY or configure an API key for provider "brave".',
});
const sources: SearchSource[] = [];
for (const result of response.web?.results ?? []) {
if (!result.url) continue;
for (const result of webResults(response)) {
if (typeof result !== "object" || result === null) continue;
const url = normalizeUrl("url" in result ? result.url : undefined);
if (!url) continue;
const publishedDate = normalizeText("age" in result ? result.age : undefined, 100);
sources.push({
title: result.title ?? result.url,
url: result.url,
title: normalizeText("title" in result ? result.title : undefined, 300) ?? url,
url,
snippet: buildSnippet(result),
publishedDate: result.age ?? undefined,
ageSeconds: dateToAgeSeconds(result.age),
publishedDate,
ageSeconds: dateToAgeSeconds(publishedDate),
});
}
@@ -155,6 +229,7 @@ export async function searchBrave(params: BraveSearchParams): Promise<SearchResp
provider: "brave",
sources: sources.slice(0, numResults),
requestId,
authMode: "api_key",
};
}
@@ -163,8 +238,8 @@ export class BraveProvider extends SearchProvider {
readonly id = "brave";
readonly label = "Brave";
isAvailable(_authStorage: AuthStorage): boolean {
return !!findApiKey();
isAvailable(authStorage: AuthStorage): boolean {
return authStorage.hasAuth("brave") || !!getEnvApiKey("brave");
}
search(params: SearchParams): Promise<SearchResponse> {
@@ -173,6 +248,8 @@ export class BraveProvider extends SearchProvider {
num_results: params.numSearchResults ?? params.limit,
recency: params.recency,
parsedQuery: params.parsedQuery,
authStorage: params.authStorage,
sessionId: params.sessionId,
signal: params.signal,
timeoutMs: params.timeoutMs,
fetch: params.fetch,
packages/coding-agent/src/web/search/providers/codex.ts
+87 -9
View File
@@ -151,6 +151,13 @@ export interface CodexSearchParams {
}
/** Codex API response structure */
interface CodexWebSearchSource {
url?: string;
source_website_url?: string;
title?: string;
caption?: string;
}
interface CodexResponseItem {
type: string;
id?: string;
@@ -161,6 +168,9 @@ interface CodexResponseItem {
arguments?: string;
content?: CodexContentPart[];
summary?: Array<{ type: string; text: string }>;
action?: { sources?: CodexWebSearchSource[] };
sources?: CodexWebSearchSource[];
results?: CodexWebSearchSource[];
}
interface CodexContentPart {
@@ -221,12 +231,45 @@ function isImagePlaceholderAnswer(text: string): boolean {
return IMAGE_PLACEHOLDER_ANSWERS.has(normalized);
}
function addSource(sources: SearchSource[], source: SearchSource): void {
if (!sources.some(existing => existing.url === source.url)) {
sources.push(source);
function cleanSourceUrl(rawUrl: string): string {
try {
const url = new URL(rawUrl);
if (url.searchParams.get("utm_source") === "openai") {
url.searchParams.delete("utm_source");
}
return url.toString();
} catch {
return rawUrl.replace(/[?&]utm_source=openai$/u, "");
}
}
function addSource(sources: SearchSource[], source: SearchSource): void {
const normalizedSource = { ...source, url: cleanSourceUrl(source.url) };
const existing = sources.find(candidate => candidate.url === normalizedSource.url);
if (!existing) {
sources.push(normalizedSource);
return;
}
if (existing.title === existing.url && normalizedSource.title !== normalizedSource.url) {
existing.title = normalizedSource.title;
}
if (!existing.snippet && normalizedSource.snippet) {
existing.snippet = normalizedSource.snippet;
}
}
function extractCitationSnippet(text: string, start: number | undefined, end: number | undefined): string | undefined {
if (start === undefined || end === undefined || !text) return undefined;
const before = Math.max(0, start - 100);
const after = Math.min(text.length, end + 100);
const snippet = text
.slice(before, after)
.replace(/\[([^\]]*)\]\([^)]*\)/g, "$1")
.trim();
if (!snippet) return undefined;
return snippet.length > 300 ? `${snippet.slice(0, 297)}...` : snippet;
}
function countCharacter(text: string, target: string): number {
let count = 0;
for (const char of text) {
@@ -428,6 +471,15 @@ function extractCodexSseError(rawEvent: Record<string, unknown>): { code: string
return { code, message };
}
function classifyCodexSseErrorStatus(code: string, message: string): number {
const detail = `${code} ${message}`.toLowerCase();
if (/rate[- ]?limit|too many requests|quota|\b429\b/u.test(detail)) return 429;
if (/unauthori[sz]ed|\b401\b/u.test(detail)) return 401;
if (/forbidden|\b403\b/u.test(detail)) return 403;
if (/timeout|timed out/u.test(detail)) return 504;
return 500;
}
/**
* Calls the Codex Responses API with web search tool enabled.
* The caller provides the exact model id to send; retry / fallback policy
@@ -455,6 +507,8 @@ async function callCodexSearch(
model: requestedModel,
stream: true,
store: false,
include: ["web_search_call.action.sources"],
parallel_tool_calls: true,
input: [
{
type: "message",
@@ -510,7 +564,11 @@ async function callCodexSearch(
webSearchInvoked = true;
}
if (eventType === "response.output_text.delta") {
if (eventType === "response.created") {
const resp = (rawEvent as { response?: CodexResponse }).response;
if (resp?.id) requestId = resp.id;
if (resp?.model) model = resp.model;
} else if (eventType === "response.output_text.delta") {
const delta = typeof rawEvent.delta === "string" ? rawEvent.delta : "";
if (delta) {
streamedAnswerParts.push(delta);
@@ -518,7 +576,20 @@ async function callCodexSearch(
} else if (eventType === "response.output_item.done") {
const item = rawEvent.item as CodexResponseItem | undefined;
if (!item) continue;
if (item.type === "web_search_call") webSearchInvoked = true;
if (item.type === "web_search_call") {
webSearchInvoked = true;
const sourceGroups = [item.action?.sources, item.sources, item.results];
for (const group of sourceGroups) {
for (const source of group ?? []) {
const url = source.url ?? source.source_website_url;
if (!url) continue;
addSource(sources, {
title: source.title ?? source.caption ?? url,
url,
});
}
}
}
// Handle text message content and extract sources from annotations
if (item.type === "message" && item.content) {
@@ -530,8 +601,11 @@ async function callCodexSearch(
if (part.annotations) {
for (const annotation of part.annotations) {
if (annotation.type === "url_citation" && annotation.url) {
// Deduplicate by URL
addSource(sources, { title: annotation.title ?? annotation.url, url: annotation.url });
addSource(sources, {
title: annotation.title ?? annotation.url,
url: annotation.url,
snippet: extractCitationSnippet(part.text, annotation.start_index, annotation.end_index),
});
}
}
}
@@ -563,13 +637,17 @@ async function callCodexSearch(
}
} else if (eventType === "error") {
const { code, message } = extractCodexSseError(rawEvent);
throw new SearchProviderError("codex", `Codex error (${code}): ${message || "Unknown error"}`, 500);
throw new SearchProviderError(
"codex",
`Codex error (${code}): ${message || "Unknown error"}`,
classifyCodexSseErrorStatus(code, message),
);
} else if (eventType === "response.failed") {
const { code, message } = extractCodexSseError(rawEvent);
const detail = code
? `Codex request failed (${code}): ${message || "Request failed"}`
: `Codex request failed: ${message || "Request failed"}`;
throw new SearchProviderError("codex", detail, 500);
throw new SearchProviderError("codex", detail, classifyCodexSseErrorStatus(code, message));
}
}
packages/coding-agent/src/web/search/providers/exa.ts
+45 -10
View File
@@ -19,6 +19,9 @@ import { SearchProvider } from "./base";
import { classifyProviderHttpError, withHardTimeout } from "./utils";
const EXA_API_URL = "https://api.exa.ai/search";
const EXA_MCP_URL = "https://mcp.exa.ai/mcp";
const EXA_MCP_SOURCE = "oh-my-pi";
const MAX_EXA_SNIPPET_CHARS = 500;
const DEFAULT_EXA_SEARCH_DELAY_MS = getDefault("exa.searchDelayMs");
let nextExaSearchRequestAt = 0;
@@ -329,13 +332,20 @@ async function callExaSearch(apiKey: string, params: ExaSearchParams): Promise<E
return response.json() as Promise<ExaSearchResponse>;
}
function buildExaMcpArgs(params: ExaSearchParams): Record<string, unknown> {
const args: Record<string, unknown> = { query: params.query };
if (params.num_results !== undefined) args.num_results = params.num_results;
if (params.type !== undefined) args.type = params.type;
if (params.include_domains !== undefined) args.include_domains = params.include_domains;
if (params.exclude_domains !== undefined) args.exclude_domains = params.exclude_domains;
if (params.start_published_date !== undefined) args.start_published_date = params.start_published_date;
if (params.end_published_date !== undefined) args.end_published_date = params.end_published_date;
const queryParts = [params.query];
for (const domain of params.include_domains ?? []) {
const trimmed = domain.trim();
if (trimmed) queryParts.push(`site:${trimmed}`);
}
for (const domain of params.exclude_domains ?? []) {
const trimmed = domain.trim();
if (trimmed) queryParts.push(`-site:${trimmed}`);
}
if (params.start_published_date) queryParts.push(`after:${params.start_published_date}`);
if (params.end_published_date) queryParts.push(`before:${params.end_published_date}`);
const args: Record<string, unknown> = { query: queryParts.join(" ") };
if (params.num_results !== undefined) args.numResults = params.num_results;
return args;
}
@@ -346,11 +356,12 @@ async function callExaMcpSearch(params: ExaSearchParams): Promise<ExaSearchRespo
query.set("tools", "web_search_exa");
const fetchImpl = params.fetch ?? fetch;
await waitForExaSearchSlot(params.signal);
const response = await fetchImpl(`https://mcp.exa.ai/mcp?${query.toString()}`, {
const response = await fetchImpl(`${EXA_MCP_URL}?${query.toString()}`, {
method: "POST",
headers: {
"Content-Type": "application/json",
Accept: "application/json, text/event-stream",
"x-exa-source": EXA_MCP_SOURCE,
},
body: JSON.stringify({
jsonrpc: "2.0",
@@ -364,11 +375,26 @@ async function callExaMcpSearch(params: ExaSearchParams): Promise<ExaSearchRespo
signal: withHardTimeout(params.signal, params.timeoutMs),
});
if (!response.ok) {
throw new Error(`MCP request failed: ${response.status} ${response.statusText}`);
const errorText = await response.text();
const classified = classifyProviderHttpError("exa", response.status, errorText);
if (classified) throw classified;
if (response.status === 429) {
throw new SearchProviderError(
"exa",
"exa: MCP rate limit reached (429); configure an Exa API key for higher limits",
response.status,
);
}
throw new SearchProviderError(
"exa",
`Exa MCP request failed (${response.status}): ${errorText}`,
response.status,
);
}
const mcpResponse = parseSSE(await response.text()) as {
result?: {
content?: Array<{ type: string; text?: string }>;
isError?: boolean;
};
error?: {
code: number;
@@ -381,6 +407,12 @@ async function callExaMcpSearch(params: ExaSearchParams): Promise<ExaSearchRespo
if (mcpResponse.error) {
throw new Error(`MCP error: ${mcpResponse.error.message}`);
}
if (mcpResponse.result?.isError) {
const message = mcpResponse.result.content
?.find(item => item.type === "text" && typeof item.text === "string")
?.text?.trim();
throw new SearchProviderError("exa", message || "Exa MCP returned an error");
}
const responsePayload = normalizeExaMcpPayload(mcpResponse.result);
if (isSearchResponse(responsePayload)) {
return responsePayload as ExaSearchResponse;
@@ -419,7 +451,10 @@ export async function searchExa(params: ExaSearchParams): Promise<SearchResponse
sources.push({
title: result.title ?? result.url,
url: result.url,
snippet: result.summary || result.text || result.highlights?.join(" ") || undefined,
snippet: (result.summary || result.text || result.highlights?.join(" ") || undefined)?.slice(
0,
MAX_EXA_SNIPPET_CHARS,
),
publishedDate: result.publishedDate ?? undefined,
ageSeconds: dateToAgeSeconds(result.publishedDate ?? undefined),
author: result.author ?? undefined,
packages/coding-agent/src/web/search/providers/firecrawl.ts
+53 -11
View File
@@ -20,7 +20,7 @@ import type { SearchParams } from "./base";
import { SearchProvider } from "./base";
import { classifyProviderHttpError, withHardTimeout } from "./utils";
const FIRECRAWL_SEARCH_URL = "https://api.firecrawl.dev/v2/search";
const FIRECRAWL_DEFAULT_BASE_URL = "https://api.firecrawl.dev/v2";
const DEFAULT_NUM_RESULTS = 10;
const MAX_NUM_RESULTS = 100;
@@ -30,6 +30,28 @@ const RECENCY_TBS: Record<NonNullable<SearchParams["recency"]>, string> = {
month: "qdr:m",
year: "qdr:y",
};
function resolveSearchUrl(): string {
const configured = process.env.FIRECRAWL_BASE_URL ?? process.env.FIRECRAWL_API_URL;
if (!configured?.trim()) return `${FIRECRAWL_DEFAULT_BASE_URL}/search`;
let url: URL;
try {
url = new URL(configured.trim());
} catch {
throw new Error("Invalid Firecrawl base URL: expected an HTTP or HTTPS URL");
}
if (url.protocol !== "http:" && url.protocol !== "https:") {
throw new Error("Invalid Firecrawl base URL: expected an HTTP or HTTPS URL");
}
if (url.username || url.password) {
throw new Error("Invalid Firecrawl base URL: URL credentials are not allowed");
}
url.search = "";
url.hash = "";
url.pathname = url.pathname.replace(/\/+$/, "");
if (!/\/v[12]$/i.test(url.pathname)) url.pathname += "/v2";
url.pathname += "/search";
return url.toString();
}
export interface FirecrawlSearchParams {
query: string;
@@ -46,14 +68,23 @@ interface FirecrawlWebResult {
title?: string | null;
url?: string | null;
description?: string | null;
snippet?: string | null;
markdown?: string | null;
}
interface FirecrawlSearchResponse {
success?: boolean;
error?: string | null;
id?: string | null;
data?: {
web?: FirecrawlWebResult[] | null;
} | null;
data?:
| FirecrawlWebResult[]
| {
web?: FirecrawlWebResult[] | null;
news?: FirecrawlWebResult[] | null;
images?: FirecrawlWebResult[] | null;
}
| null;
results?: FirecrawlWebResult[] | null;
}
/** Resolve Firecrawl API key through the shared auth storage pipeline. */
@@ -88,7 +119,7 @@ async function callFirecrawlSearch(
if (apiKey) {
headers.Authorization = `Bearer ${apiKey}`;
}
const response = await (params.fetch ?? fetch)(FIRECRAWL_SEARCH_URL, {
const response = await (params.fetch ?? fetch)(resolveSearchUrl(), {
method: "POST",
headers,
body: JSON.stringify(buildRequestBody(params)),
@@ -106,7 +137,11 @@ async function callFirecrawlSearch(
);
}
return (await response.json()) as FirecrawlSearchResponse;
const data = (await response.json()) as FirecrawlSearchResponse;
if (data.success === false) {
throw new SearchProviderError("firecrawl", data.error?.trim() || "Firecrawl request failed");
}
return data;
}
/** ISO `YYYY-MM-DD` to Google `MM/DD/YYYY` for `tbs=cdr` custom date ranges. */
@@ -128,6 +163,11 @@ function buildDateTbs(parsed: StructuredQuery): string | undefined {
return parts.join(",");
}
function getWebResults(data: FirecrawlSearchResponse): FirecrawlWebResult[] {
if (Array.isArray(data.data)) return data.data;
if (data.data && Array.isArray(data.data.web)) return data.data.web;
return data.results ?? [];
}
/** Execute Firecrawl web search. */
export async function searchFirecrawl(params: SearchParams): Promise<SearchResponse> {
const parsed = params.parsedQuery ?? parseSearchQuery(params.query);
@@ -169,12 +209,12 @@ export async function searchFirecrawl(params: SearchParams): Promise<SearchRespo
const sources: SearchSource[] = [];
for (const result of data.data?.web ?? []) {
for (const result of getWebResults(data)) {
if (!result.url) continue;
sources.push({
title: result.title ?? result.url,
url: result.url,
snippet: result.description ?? result.markdown ?? undefined,
snippet: result.description ?? result.snippet ?? result.markdown ?? undefined,
});
}
@@ -192,11 +232,13 @@ export class FirecrawlProvider extends SearchProvider {
readonly label = "Firecrawl";
/**
* Auto-chain admission: requires a credential so an unconfigured Firecrawl
* doesn't displace other providers that the user has set up with API keys.
* Auto-chain admission requires either a credential or an explicitly
* configured self-hosted endpoint. Hosted keyless mode remains explicit-only
* so it does not displace providers the user configured.
*/
isAvailable(authStorage: AuthStorage): boolean {
return authStorage.hasAuth("firecrawl") || !!getEnvApiKey("firecrawl");
const configuredBaseUrl = process.env.FIRECRAWL_BASE_URL ?? process.env.FIRECRAWL_API_URL;
return !!configuredBaseUrl?.trim() || authStorage.hasAuth("firecrawl") || !!getEnvApiKey("firecrawl");
}
/**
packages/coding-agent/src/web/search/providers/gemini.ts
+129 -10
View File
@@ -25,7 +25,9 @@ import { classifyProviderHttpError, withHardTimeout } from "./utils";
const DEFAULT_ENDPOINT = "https://cloudcode-pa.googleapis.com";
const DEVELOPER_API_PROVIDER = "google";
const DEVELOPER_API_ENDPOINT = "https://generativelanguage.googleapis.com/v1beta";
const CLOUDFLARE_GATEWAY_PROVIDER = "cloudflare-ai-gateway";
const DEFAULT_DEVELOPER_API_HOST = "https://generativelanguage.googleapis.com";
const DEVELOPER_API_VERSION = "v1beta";
const ANTIGRAVITY_DAILY_ENDPOINT = "https://daily-cloudcode-pa.googleapis.com";
const ANTIGRAVITY_SANDBOX_ENDPOINT = "https://daily-cloudcode-pa.sandbox.googleapis.com";
const ANTIGRAVITY_ENDPOINT_FALLBACKS = [ANTIGRAVITY_DAILY_ENDPOINT, ANTIGRAVITY_SANDBOX_ENDPOINT] as const;
@@ -41,6 +43,32 @@ function resolveGeminiSearchModel(configuredModel: string | undefined): string {
return model || DEFAULT_MODEL;
}
interface GeminiDeveloperEndpoint {
url: string;
authProvider: typeof DEVELOPER_API_PROVIDER | typeof CLOUDFLARE_GATEWAY_PROVIDER;
isCloudflareGateway: boolean;
}
function resolveGeminiDeveloperEndpoint(): GeminiDeveloperEndpoint {
const configuredHost = Bun.env.GOOGLE_GEMINI_BASE_URL?.trim().replace(/\/+$/, "");
const host = configuredHost || DEFAULT_DEVELOPER_API_HOST;
let parsed: URL;
try {
parsed = new URL(host);
} catch {
throw new SearchProviderError("gemini", "GOOGLE_GEMINI_BASE_URL must be a valid absolute URL", 400);
}
if (parsed.protocol !== "https:" && parsed.protocol !== "http:") {
throw new SearchProviderError("gemini", "GOOGLE_GEMINI_BASE_URL must use HTTP or HTTPS", 400);
}
const isCloudflareGateway = parsed.hostname === "gateway.ai.cloudflare.com";
return {
url: `${host}/${DEVELOPER_API_VERSION}`,
authProvider: isCloudflareGateway ? CLOUDFLARE_GATEWAY_PROVIDER : DEVELOPER_API_PROVIDER,
isCloudflareGateway,
};
}
const GEMINI_PROVIDERS = ["google-gemini-cli", "google-antigravity"] as const;
type GeminiProviderId = (typeof GEMINI_PROVIDERS)[number];
@@ -292,6 +320,81 @@ async function parseGeminiSearchStream(
};
}
function isGroundingRedirectUrl(url: string): boolean {
try {
const parsed = new URL(url);
return (
parsed.hostname === "vertexaisearch.cloud.google.com" &&
parsed.pathname.includes("/grounding-api-redirect")
);
} catch {
return false;
}
}
async function resolveGroundingRedirect(
proxyUrl: string,
fetchImpl: FetchImpl | undefined,
signal: AbortSignal | undefined,
): Promise<string> {
try {
const response = await (fetchImpl ?? fetch)(proxyUrl, {
method: "HEAD",
redirect: "manual",
signal: withHardTimeout(signal, 5000),
});
const location = response.headers.get("location");
if (!location) return proxyUrl;
const resolved = new URL(location, proxyUrl);
return resolved.protocol === "http:" || resolved.protocol === "https:" ? resolved.toString() : proxyUrl;
} catch {
return proxyUrl;
}
}
async function finalizeGeminiSearchResult(
result: GeminiSearchResult,
fetchImpl: FetchImpl | undefined,
signal: AbortSignal | undefined,
): Promise<GeminiSearchResult> {
if (!result.answer && result.sources.length === 0) {
throw new SearchProviderError("gemini", "Gemini API returned an empty grounded response", 502);
}
const redirectUrls = new Set<string>();
for (const source of result.sources) {
if (isGroundingRedirectUrl(source.url)) redirectUrls.add(source.url);
}
for (const citation of result.citations) {
if (isGroundingRedirectUrl(citation.url)) redirectUrls.add(citation.url);
}
if (redirectUrls.size === 0) return result;
signal?.throwIfAborted();
const resolvedEntries = await Promise.all(
[...redirectUrls].map(async url => [url, await resolveGroundingRedirect(url, fetchImpl, signal)] as const),
);
signal?.throwIfAborted();
const resolvedUrls = new Map(resolvedEntries);
for (const source of result.sources) {
source.url = resolvedUrls.get(source.url) ?? source.url;
}
for (const citation of result.citations) {
citation.url = resolvedUrls.get(citation.url) ?? citation.url;
}
const seenUrls = new Set<string>();
let writeIndex = 0;
for (const source of result.sources) {
if (seenUrls.has(source.url)) continue;
seenUrls.add(source.url);
result.sources[writeIndex++] = source;
}
result.sources.length = writeIndex;
return result;
}
/**
* Calls the Cloud Code Assist API with Google Search grounding enabled.
*
@@ -420,7 +523,10 @@ async function callGeminiSearch(
}
if (!response?.ok) {
const errorText = response ? await response.text() : "Network error";
const rawErrorText = response ? await response.text() : "Network error";
const errorText = auth.accessToken
? rawErrorText.split(auth.accessToken).join("[redacted]")
: rawErrorText;
const status = response?.status ?? 502;
const classified = classifyProviderHttpError("gemini", status, errorText);
if (classified) throw classified;
@@ -431,11 +537,12 @@ async function callGeminiSearch(
throw new SearchProviderError("gemini", "Gemini API returned no response body", 500);
}
return parseGeminiSearchStream(response.body, model);
return finalizeGeminiSearchResult(await parseGeminiSearchStream(response.body, model), fetchImpl, signal);
}
async function callGeminiDeveloperSearch(
apiKey: string,
endpoint: GeminiDeveloperEndpoint,
model: string,
query: string,
systemPrompt: string | undefined,
@@ -474,11 +581,13 @@ async function callGeminiDeveloperSearch(
}
const response = await fetchWithRetry(
() => `${DEVELOPER_API_ENDPOINT}/models/${model}:streamGenerateContent?alt=sse`,
() => `${endpoint.url}/models/${model}:streamGenerateContent?alt=sse`,
{
method: "POST",
headers: {
"x-goog-api-key": apiKey,
...(endpoint.isCloudflareGateway
? { "cf-aig-authorization": `Bearer ${apiKey}` }
: { "x-goog-api-key": apiKey }),
"Content-Type": "application/json",
Accept: "text/event-stream",
},
@@ -492,7 +601,8 @@ async function callGeminiDeveloperSearch(
);
if (!response.ok) {
const errorText = await response.text();
const rawErrorText = await response.text();
const errorText = apiKey ? rawErrorText.split(apiKey).join("[redacted]") : rawErrorText;
const classified = classifyProviderHttpError("gemini", response.status, errorText);
if (classified) throw classified;
throw new SearchProviderError(
@@ -506,7 +616,7 @@ async function callGeminiDeveloperSearch(
throw new SearchProviderError("gemini", "Gemini API returned no response body", 500);
}
return parseGeminiSearchStream(response.body, model);
return finalizeGeminiSearchResult(await parseGeminiSearchStream(response.body, model), fetchImpl, signal);
}
/**
@@ -557,16 +667,20 @@ export async function searchGemini(params: GeminiSearchParams): Promise<SearchRe
{ sessionId: params.sessionId, signal: params.signal, seed: seed.access },
);
} else {
const apiKey = await params.authStorage.getApiKey(DEVELOPER_API_PROVIDER, params.sessionId, {
const endpoint = resolveGeminiDeveloperEndpoint();
const apiKey = await params.authStorage.getApiKey(endpoint.authProvider, params.sessionId, {
signal: params.signal,
});
if (!apiKey) {
throw new Error(
"No Gemini credentials found. Set GEMINI_API_KEY, configure an API key for provider \"google\", or login with 'omp /login google-gemini-cli' / 'omp /login google-antigravity' to enable Gemini web search.",
endpoint.isCloudflareGateway
? "No Cloudflare AI Gateway credential found. Configure provider \"cloudflare-ai-gateway\" or set CLOUDFLARE_AI_GATEWAY_API_KEY."
: "No Gemini credentials found. Set GEMINI_API_KEY, configure an API key for provider \"google\", or login with 'omp /login google-gemini-cli' / 'omp /login google-antigravity' to enable Gemini web search.",
);
}
result = await callGeminiDeveloperSearch(
apiKey,
endpoint,
selectedModel,
searchQuery,
params.system_prompt,
@@ -609,7 +723,12 @@ export class GeminiProvider extends SearchProvider {
// Cheap, in-memory check — avoids driving the refresh pipeline during
// the provider-chain probe. `searchGemini` refreshes OAuth lazily on the
// actual request and resolves developer API keys through AuthStorage.
return hasGeminiOAuth(authStorage) || authStorage.hasAuth(DEVELOPER_API_PROVIDER);
if (hasGeminiOAuth(authStorage)) return true;
try {
return authStorage.hasAuth(resolveGeminiDeveloperEndpoint().authProvider);
} catch {
return false;
}
}
search(params: SearchParams): Promise<SearchResponse> {
packages/coding-agent/src/web/search/providers/jina.ts
+57 -25
View File
@@ -5,19 +5,24 @@
* cleaned content.
*/
import { type AuthStorage, type FetchImpl, getEnvApiKey } from "@oh-my-pi/pi-ai";
import { type ApiKey, type AuthStorage, type FetchImpl, withAuth } from "@oh-my-pi/pi-ai";
import type { SearchResponse, SearchSource } from "../../../web/search/types";
import { SearchProviderError } from "../../../web/search/types";
import { formatQuery, parseSearchQuery } from "../query";
import { clampNumResults } from "../utils";
import type { SearchParams } from "./base";
import { SearchProvider } from "./base";
import { classifyProviderHttpError, withHardTimeout } from "./utils";
const JINA_SEARCH_URL = "https://s.jina.ai";
const DEFAULT_NUM_RESULTS = 5;
const MAX_NUM_RESULTS = 20;
type SearchParamsWithFetch = SearchParams & { fetch?: FetchImpl };
export interface JinaSearchParams {
query: string;
authStorage: AuthStorage;
sessionId?: string;
num_results?: number;
/** Single bare host for Jina's `X-Site` in-site search header. */
site?: string;
@@ -29,31 +34,37 @@ export interface JinaSearchParams {
interface JinaSearchResult {
title?: string | null;
url?: string | null;
description?: string | null;
content?: string | null;
}
type JinaSearchResponse = JinaSearchResult[];
/** Find JINA_API_KEY from environment or .env files. */
export function findApiKey(): string | null {
return getEnvApiKey("jina") ?? null;
interface JinaSearchEnvelope {
code?: unknown;
data?: unknown;
}
type JinaSearchResponse = JinaSearchResult[];
/** Call Jina Reader search API. */
async function callJinaSearch(
apiKey: string,
query: string,
numResults: number,
site?: string,
signal?: AbortSignal,
fetchImpl: FetchImpl = fetch,
timeoutMs?: number,
): Promise<JinaSearchResponse> {
const requestUrl = `${JINA_SEARCH_URL}/${encodeURIComponent(query)}`;
const requestUrl = new URL(`${JINA_SEARCH_URL}/${encodeURIComponent(query)}`);
requestUrl.searchParams.set("count", String(numResults));
const headers: Record<string, string> = {
Accept: "application/json",
Authorization: `Bearer ${apiKey}`,
};
if (site) headers["X-Site"] = site;
headers["X-Respond-With"] = "no-content";
headers["X-Retain-Images"] = "none";
const response = await fetchImpl(requestUrl, {
headers,
signal: withHardTimeout(signal, timeoutMs),
@@ -66,24 +77,43 @@ async function callJinaSearch(
throw new SearchProviderError("jina", `Jina API error (${response.status}): ${errorText}`, response.status);
}
const payload = (await response.json()) as { data?: JinaSearchResponse } | null;
return Array.isArray(payload?.data) ? payload.data : [];
const payload = (await response.json()) as JinaSearchEnvelope | JinaSearchResponse | null;
if (Array.isArray(payload)) return payload;
if (!payload || typeof payload !== "object") {
throw new SearchProviderError("jina", "Jina API returned invalid response: expected an object or array");
}
if (typeof payload.code === "number" && payload.code !== 200) {
throw new SearchProviderError("jina", `Jina API response reported failure (${payload.code})`, payload.code);
}
if (!Array.isArray(payload.data)) {
throw new SearchProviderError("jina", "Jina API returned invalid response: expected data array");
}
return payload.data as JinaSearchResponse;
}
/** Execute Jina web search. */
export async function searchJina(params: JinaSearchParams): Promise<SearchResponse> {
const apiKey = findApiKey();
if (!apiKey) {
throw new Error("JINA_API_KEY not found. Set it in environment or .env file.");
}
const response = await callJinaSearch(
apiKey,
params.query,
params.site,
params.signal,
params.fetch,
params.timeoutMs,
const numResults = clampNumResults(params.num_results, DEFAULT_NUM_RESULTS, MAX_NUM_RESULTS);
const keyOrResolver: ApiKey = params.authStorage.resolver("jina", {
sessionId: params.sessionId,
});
const response = await withAuth(
keyOrResolver,
apiKey =>
callJinaSearch(
apiKey,
params.query,
numResults,
params.site,
params.signal,
params.fetch,
params.timeoutMs,
),
{
signal: params.signal,
missingKeyMessage:
'Jina credentials not found. Set JINA_API_KEY or configure an API key for provider "jina".',
},
);
const sources: SearchSource[] = [];
@@ -92,11 +122,11 @@ export async function searchJina(params: JinaSearchParams): Promise<SearchRespon
sources.push({
title: result.title ?? result.url,
url: result.url,
snippet: result.content ?? undefined,
snippet: result.description?.trim() || result.content?.trim() || undefined,
});
}
const limitedSources = params.num_results ? sources.slice(0, params.num_results) : sources;
const limitedSources = sources.slice(0, numResults);
return {
provider: "jina",
@@ -109,8 +139,8 @@ export class JinaProvider extends SearchProvider {
readonly id = "jina";
readonly label = "Jina";
isAvailable(_authStorage: AuthStorage): boolean {
return !!findApiKey();
isAvailable(authStorage: AuthStorage): boolean {
return authStorage.hasAuth("jina");
}
search(params: SearchParamsWithFetch): Promise<SearchResponse> {
@@ -134,6 +164,8 @@ export class JinaProvider extends SearchProvider {
return searchJina({
query,
authStorage: params.authStorage,
sessionId: params.sessionId,
num_results: params.numSearchResults ?? params.limit,
site,
signal: params.signal,
packages/coding-agent/src/web/search/providers/parallel.ts
+23 -9
View File
@@ -7,6 +7,7 @@ import {
ParallelApiError,
type ParallelSearchResult,
parseParallelErrorResponse,
parseParallelJsonResponse,
parseParallelSearchPayload,
} from "../../parallel";
import { formatQuery, parseSearchQuery, type StructuredQuery } from "../query";
@@ -28,6 +29,13 @@ interface ParallelSourcePolicy {
after_date?: string;
}
const RECENCY_DAYS: Record<NonNullable<SearchParams["recency"]>, number> = {
day: 1,
week: 7,
month: 30,
year: 365,
};
/** Site values may carry paths (`github.com/anthropics`); Parallel takes bare hosts. */
function toHosts(sites: readonly string[]): string[] {
const hosts = new Set<string>();
@@ -39,19 +47,23 @@ function toHosts(sites: readonly string[]): string[] {
}
/**
* Map parsed `site:`/`-site:`/`after:` directives onto Parallel's
* `source_policy`. Per Parallel docs, `exclude_domains` is ignored when
* `include_domains` is set, so exclusions are only sent without an allow
* list (the central lenient filter enforces them regardless).
* Map parsed `site:`/`-site:`/`after:` directives and the relative recency
* option onto Parallel's `source_policy`. An explicit `after:` bound wins.
* Per Parallel docs, `exclude_domains` is ignored when `include_domains` is
* set, so exclusions are only sent without an allow list (the central lenient
* filter enforces them regardless).
*/
function toSourcePolicy(parsed: StructuredQuery): ParallelSourcePolicy | undefined {
function toSourcePolicy(parsed: StructuredQuery, recency?: SearchParams["recency"]): ParallelSourcePolicy | undefined {
const policy: ParallelSourcePolicy = {};
const include = toHosts(parsed.sites);
const exclude = toHosts(parsed.excludedSites);
if (include.length) policy.include_domains = include;
else if (exclude.length) policy.exclude_domains = exclude;
if (parsed.after) policy.after_date = parsed.after;
return Object.keys(policy).length ? policy : undefined;
else if (recency) {
policy.after_date = new Date(Date.now() - RECENCY_DAYS[recency] * 86_400_000).toISOString().slice(0, 10);
}
return policy.include_domains || policy.exclude_domains || policy.after_date ? policy : undefined;
}
async function searchWithAuthStorage(
@@ -105,7 +117,7 @@ async function searchWithAuthStorage(
throw parseParallelErrorResponse(response.status, await response.text());
}
const payload: unknown = await response.json();
const payload = await parseParallelJsonResponse(response, "search");
return parseParallelSearchPayload(payload, { parseMetadata: false });
},
{ signal: params.signal },
@@ -116,6 +128,7 @@ export async function searchParallel(
params: {
query: string;
num_results?: number;
recency?: SearchParams["recency"];
signal?: AbortSignal;
timeoutMs?: number;
fetch?: FetchImpl;
@@ -126,9 +139,9 @@ export async function searchParallel(
): Promise<SearchResponse> {
const numResults = clampNumResults(params.num_results, DEFAULT_NUM_RESULTS, MAX_NUM_RESULTS);
const parsed = params.parsedQuery ?? parseSearchQuery(params.query);
// Back-compat: without directives the upstream request is byte-identical.
// Directives are removed only where Parallel has a native equivalent.
const query = parsed.hasDirectives ? formatQuery(parsed, PARALLEL_QUERY_SYNTAX) : params.query;
const sourcePolicy = parsed.hasDirectives ? toSourcePolicy(parsed) : undefined;
const sourcePolicy = toSourcePolicy(parsed, params.recency);
try {
const result = await searchWithAuthStorage(
@@ -174,6 +187,7 @@ export class ParallelProvider extends SearchProvider {
{
query: params.query,
num_results: params.numSearchResults ?? params.limit,
recency: params.recency,
signal: params.signal,
timeoutMs: params.timeoutMs,
fetch: params.fetch,
packages/coding-agent/src/web/search/providers/perplexity.ts
+24 -7
View File
@@ -150,6 +150,7 @@ interface PerplexityOAuthStreamEvent {
error_code?: string;
error_message?: string;
display_model?: string;
user_selected_model?: string;
uuid?: string;
}
@@ -327,7 +328,11 @@ export interface PerplexitySearchParams {
system_prompt?: string;
/** Pre-parsed view of `query` from the search pipeline; parsed locally when absent. */
parsedQuery?: StructuredQuery;
/** Direct API model. Defaults to `PI_PERPLEXITY_API_MODEL`, then `sonar-pro`. */
api_model?: string;
search_recency_filter?: "hour" | "day" | "week" | "month" | "year";
/** Consumer subscription model preference. Defaults to `PI_PERPLEXITY_MODEL`, then Sonar (`experimental`). */
subscription_model?: string;
num_results?: number;
/** Maximum output tokens. Defaults to 8192. */
max_tokens?: number;
@@ -539,6 +544,15 @@ async function callPerplexityApi(
return parseStreamedApiResponse(message, metadata);
}
function oauthSourceKey(url: string): string {
const trimmed = url.trim().replace(/\/$/, "");
try {
return new URL(trimmed).href.replace(/\/$/, "");
} catch {
return trimmed.toLowerCase();
}
}
function buildOAuthSources(event: PerplexityOAuthStreamEvent): SearchSource[] {
const results =
event.blocks?.find(block => block.intended_usage === "web_results")?.web_result_block?.web_results ?? [];
@@ -607,6 +621,7 @@ async function callPerplexityAsk(
params: PerplexitySearchParams,
filters: PerplexityNativeFilters,
): Promise<{ answer: string; sources: SearchSource[]; model?: string; requestId?: string }> {
const subscriptionModel = params.subscription_model?.trim() || $env.PI_PERPLEXITY_MODEL?.trim() || "experimental";
const requestId = crypto.randomUUID();
// The consumer `perplexity_ask` endpoint is itself a research assistant and
// has no system-message slot. Prepending the API-style system prompt to the
@@ -645,14 +660,14 @@ async function callPerplexityAsk(
query_str: effectiveQuery,
search_focus: "internet",
mode: "copilot",
model_preference: "experimental",
model_preference: subscriptionModel,
sources: ["web"],
attachments: [],
frontend_uuid: crypto.randomUUID(),
frontend_context_uuid: crypto.randomUUID(),
version: OAUTH_API_VERSION,
language: "en-US",
timezone: Intl.DateTimeFormat().resolvedOptions().timeZone,
timezone: Intl.DateTimeFormat().resolvedOptions().timeZone ?? "UTC",
// Recency cannot be combined with absolute date filters; explicit
// before:/after: bounds take precedence.
search_recency_filter: filters.afterDate || filters.beforeDate ? null : (params.search_recency_filter ?? null),
@@ -740,12 +755,14 @@ async function callPerplexityAsk(
if (eventAnswer.length > 0) {
answer = eventAnswer;
}
for (const source of buildOAuthSources(mergedEvent)) {
sourcesByUrl.set(source.url, source);
sourcesByUrl.set(oauthSourceKey(source.url), source);
}
if (mergedEvent.display_model) model = mergedEvent.display_model;
const reportedModel = [mergedEvent.user_selected_model, mergedEvent.display_model].find(
candidate => candidate && candidate !== "turbo",
);
if (reportedModel) model = reportedModel;
if (mergedEvent.uuid) finalRequestId = mergedEvent.uuid;
if (mergedEvent.final || mergedEvent.status === "COMPLETED") {
break;
@@ -755,7 +772,7 @@ async function callPerplexityAsk(
return {
answer,
sources: [...sourcesByUrl.values()],
model,
model: model ?? (auth.type === "anonymous" ? mergedEvent.display_model : subscriptionModel),
requestId: finalRequestId ?? requestId,
};
}
@@ -872,7 +889,7 @@ export async function searchPerplexity(params: PerplexitySearchParams): Promise<
messages.push({ role: "user", content: filters.query });
const request: PerplexityRequest = {
model: "sonar-pro",
model: params.api_model?.trim() || $env.PI_PERPLEXITY_API_MODEL?.trim() || "sonar-pro",
messages,
max_tokens: params.max_tokens ?? DEFAULT_MAX_TOKENS,
temperature: params.temperature ?? DEFAULT_TEMPERATURE,
packages/coding-agent/src/web/search/providers/searxng.ts
+77 -1
View File
@@ -62,6 +62,7 @@ interface SearXNGResult {
title?: string;
url?: string;
content?: string;
snippet?: string;
engine?: string;
publishedDate?: string;
/** SearXNG sometimes uses publishedDate, sometimes just date */
@@ -76,6 +77,7 @@ interface SearXNGResponse {
suggestions?: string[];
corrections?: string[];
unresponsive_engines?: Array<[string, string]>;
answers?: unknown[];
}
interface SearXNGAuth {
@@ -272,6 +274,61 @@ function stripExternalBangs(query: string): string {
.join(" ");
}
/** Extract displayable text from both legacy string answers and modern
* structured answer plugins (legacy, translations, weather). */
function extractAnswerText(answer: unknown): string | undefined {
if (typeof answer === "string") return answer.trim() || undefined;
if (!answer || typeof answer !== "object") return undefined;
const record = answer as Record<string, unknown>;
if (typeof record.answer === "string") return record.answer.trim() || undefined;
if (Array.isArray(record.translations)) {
const translations: string[] = [];
for (const item of record.translations) {
if (!item || typeof item !== "object") continue;
const text = (item as Record<string, unknown>).text;
if (typeof text === "string" && text.trim()) translations.push(text.trim());
if (translations.length === 3) break;
}
if (translations.length) return translations.join("\n");
}
if (record.current && typeof record.current === "object") {
const current = record.current as Record<string, unknown>;
if (typeof current.summary === "string" && current.summary.trim()) return current.summary.trim();
const location =
current.location && typeof current.location === "object"
? (current.location as Record<string, unknown>).name
: undefined;
const temperature =
current.temperature && typeof current.temperature === "object"
? (current.temperature as Record<string, unknown>)
: undefined;
const temperatureText =
temperature && (typeof temperature.val === "string" || typeof temperature.val === "number")
? `${temperature.val}${typeof temperature.unit === "string" ? temperature.unit : ""}`
: undefined;
const condition = typeof current.condition === "string" ? current.condition : undefined;
const parts = [location, temperatureText, condition].filter(
(part): part is string => typeof part === "string" && part.trim().length > 0,
);
if (parts.length) return parts.join(": ");
}
return undefined;
}
function formatAnswers(answers: unknown[] | undefined): string | undefined {
const texts: string[] = [];
for (const answer of answers ?? []) {
const text = extractAnswerText(answer);
if (text) texts.push(text);
if (texts.length === 3) break;
}
return texts.length ? texts.join("\n\n") : undefined;
}
/** Build the search URL and headers for a SearXNG request */
function buildRequest(
endpoint: string,
@@ -282,6 +339,7 @@ function buildRequest(
categories?: string;
engines?: string;
language?: string;
safesearch?: 0 | 1 | 2;
signal?: AbortSignal;
},
auth: SearXNGAuth | null,
@@ -308,6 +366,10 @@ function buildRequest(
url.searchParams.set("engines", params.engines);
}
if (params.safesearch !== undefined) {
url.searchParams.set("safesearch", String(params.safesearch));
}
if (params.language) {
url.searchParams.set("language", params.language);
}
@@ -326,6 +388,7 @@ async function callSearXNGSearch(
categories?: string;
engines?: string;
language?: string;
safesearch?: 0 | 1 | 2;
signal?: AbortSignal;
timeoutMs?: number;
fetch?: FetchImpl;
@@ -372,12 +435,23 @@ export async function searchSearXNG(params: {
let categories: string | undefined;
let language: string | undefined;
let configuredSafesearch: number | undefined;
try {
categories = settings.get("searxng.categories") ?? undefined;
language = settings.get("searxng.language") ?? undefined;
configuredSafesearch = settings.get("searxng.safesearch");
} catch {
// Settings not initialized yet
}
if (
configuredSafesearch !== undefined &&
configuredSafesearch !== 0 &&
configuredSafesearch !== 1 &&
configuredSafesearch !== 2
) {
throw new Error("searxng.safesearch must be 0 (off), 1 (moderate), or 2 (strict).");
}
const safesearch = configuredSafesearch;
const configuredEngines = findEngines();
// SearXNG forwards `q` to downstream engines, so build it with the shared
@@ -402,6 +476,7 @@ export async function searchSearXNG(params: {
categories,
engines,
language,
safesearch,
fetch: params.fetch,
},
auth,
@@ -415,7 +490,7 @@ export async function searchSearXNG(params: {
sources.push({
title: result.title ?? result.url,
url: result.url,
snippet: result.content?.trim() || undefined,
snippet: (result.content ?? result.snippet)?.trim() || undefined,
publishedDate: publishedDate ?? undefined,
ageSeconds: dateToAgeSeconds(publishedDate),
});
@@ -435,6 +510,7 @@ export async function searchSearXNG(params: {
return {
provider: "searxng",
answer: formatAnswers(response.answers),
sources: limitedSources,
relatedQuestions: response.suggestions?.length ? response.suggestions : undefined,
};
packages/coding-agent/src/web/search/providers/tavily.ts
+23 -22
View File
@@ -34,17 +34,10 @@ export interface TavilySearchParams {
fetch?: FetchImpl;
}
interface TavilySearchResult {
title?: string | null;
url?: string | null;
content?: string | null;
published_date?: string | null;
}
interface TavilySearchResponse {
answer?: string | null;
results?: TavilySearchResult[];
request_id?: string | null;
answer?: unknown;
results?: unknown;
request_id?: unknown;
}
function asRecord(value: unknown): Record<string, unknown> | null {
@@ -141,28 +134,36 @@ async function callTavilySearch(apiKey: string, params: TavilySearchParams): Pro
throw new SearchProviderError("tavily", `Tavily API error (${response.status}): ${message}`, response.status);
}
return (await response.json()) as TavilySearchResponse;
const payload: unknown = await response.json();
return asRecord(payload) ?? {};
}
function toSearchResponse(response: TavilySearchResponse, numResults: number): SearchResponse {
const sources: SearchSource[] = [];
for (const result of response.results ?? []) {
if (!result.url) continue;
sources.push({
title: result.title ?? result.url,
url: result.url,
snippet: result.content ?? undefined,
publishedDate: result.published_date ?? undefined,
ageSeconds: dateToAgeSeconds(result.published_date ?? undefined),
});
if (Array.isArray(response.results)) {
for (const value of response.results) {
const result = asRecord(value);
if (!result || typeof result.url !== "string" || !result.url) continue;
const title = typeof result.title === "string" && result.title ? result.title : result.url;
const snippet = typeof result.content === "string" ? result.content : undefined;
const publishedDate = typeof result.published_date === "string" ? result.published_date : undefined;
sources.push({
title,
url: result.url,
snippet,
publishedDate,
ageSeconds: dateToAgeSeconds(publishedDate),
});
}
}
const answer = typeof response.answer === "string" ? response.answer.trim() || undefined : undefined;
return {
provider: "tavily",
answer: response.answer?.trim() || undefined,
answer,
sources: sources.slice(0, numResults),
requestId: response.request_id ?? undefined,
requestId: typeof response.request_id === "string" ? response.request_id : undefined,
authMode: "api_key",
};
}
packages/coding-agent/src/web/search/providers/tinyfish.ts
+44 -10
View File
@@ -19,7 +19,7 @@ const MAX_NUM_RESULTS = 20;
const MAX_PAGE = 10;
/** TinyFish is SERP-backed: common Google-style operators pass through. */
const TINYFISH_QUERY_SYNTAX: QuerySyntax = { phrases: true, negation: true, site: true, filetype: true };
const TINYFISH_QUERY_SYNTAX: QuerySyntax = { phrases: true, negation: true, filetype: true };
const RECENCY_MINUTES: Record<NonNullable<SearchParams["recency"]>, number> = {
day: 1440,
@@ -33,6 +33,8 @@ export interface TinyFishSearchParams {
num_results?: number;
recency?: SearchParams["recency"];
page?: number;
include_domains?: string[];
exclude_domains?: string[];
signal?: AbortSignal;
timeoutMs?: number;
fetch?: FetchImpl;
@@ -66,6 +68,12 @@ async function callTinyFishSearch(apiKey: string, params: TinyFishSearchParams):
if (params.recency) {
url.searchParams.set("recency_minutes", String(RECENCY_MINUTES[params.recency]));
}
if (params.include_domains?.length) {
url.searchParams.set("include_domains", params.include_domains.join(","));
}
if (params.exclude_domains?.length) {
url.searchParams.set("exclude_domains", params.exclude_domains.join(","));
}
if (params.num_results !== undefined) {
url.searchParams.set("num_results", String(params.num_results));
}
@@ -96,18 +104,35 @@ async function callTinyFishSearch(apiKey: string, params: TinyFishSearchParams):
return (await response.json()) as TinyFishSearchResponse;
}
function appendTinyFishSources(sources: SearchSource[], results: readonly TinyFishSearchResult[]): void {
function appendTinyFishSources(
sources: SearchSource[],
results: readonly TinyFishSearchResult[],
seenUrls: Set<string>,
): void {
for (const result of results) {
if (!result.url) continue;
const url = result.url?.trim();
if (!url || seenUrls.has(url)) continue;
seenUrls.add(url);
const siteName = result.site_name?.trim();
sources.push({
title: result.title ?? result.site_name ?? result.url,
url: result.url,
snippet: result.snippet ?? undefined,
author: result.site_name ?? undefined,
title: result.title?.trim() || siteName || url,
url,
snippet: result.snippet?.replace(/\s+/g, " ").trim() || undefined,
author: siteName || undefined,
});
}
}
/** Bare hosts from `site:` values; path constraints remain centrally post-filtered. */
function siteHosts(sites: readonly string[]): string[] {
const hosts = new Set<string>();
for (const site of sites) {
const host = site.split("/", 1)[0];
if (host) hosts.add(host);
}
return [...hosts];
}
/** Execute TinyFish web search. */
export async function searchTinyFish(params: SearchParams): Promise<SearchResponse> {
const numResults = clampNumResults(params.numSearchResults ?? params.limit, DEFAULT_NUM_RESULTS, MAX_NUM_RESULTS);
@@ -121,6 +146,12 @@ export async function searchTinyFish(params: SearchParams): Promise<SearchRespon
timeoutMs: params.timeoutMs,
fetch: params.fetch,
};
if (parsed.hasDirectives) {
const includeDomains = siteHosts(parsed.sites);
const excludeDomains = siteHosts(parsed.excludedSites);
if (includeDomains.length > 0) tinyFishParams.include_domains = includeDomains;
if (excludeDomains.length > 0) tinyFishParams.exclude_domains = excludeDomains;
}
const keyOrResolver: ApiKey = params.authStorage.resolver("tinyfish", {
sessionId: params.sessionId,
});
@@ -128,11 +159,14 @@ export async function searchTinyFish(params: SearchParams): Promise<SearchRespon
keyOrResolver,
async key => {
const collected: SearchSource[] = [];
const seenUrls = new Set<string>();
for (let page = 0; page <= MAX_PAGE && collected.length < numResults; page += 1) {
const searchPage = await callTinyFishSearch(key, { ...tinyFishParams, page });
const results = searchPage.results ?? [];
appendTinyFishSources(collected, results);
if (results.length < pageSize) break;
if (!Array.isArray(searchPage.results)) {
throw new Error("TinyFish Search API returned an unexpected response shape");
}
appendTinyFishSources(collected, searchPage.results, seenUrls);
if (searchPage.results.length < pageSize) break;
}
return collected.slice(0, numResults);
packages/coding-agent/src/web/search/providers/xai.ts
+91 -14
View File
@@ -25,6 +25,8 @@ interface XAIUrlCitationAnnotation {
title?: string | null;
text?: string | null;
cited_text?: string | null;
start_index?: number | null;
end_index?: number | null;
}
interface XAIResponseContentPart {
@@ -34,9 +36,20 @@ interface XAIResponseContentPart {
annotations?: XAIUrlCitationAnnotation[] | null;
}
interface XAIWebSearchSource {
url?: string | null;
source_website_url?: string | null;
title?: string | null;
caption?: string | null;
}
interface XAIResponseOutputItem {
type?: string;
content?: XAIResponseContentPart[] | null;
annotations?: XAIUrlCitationAnnotation[] | null;
action?: { sources?: XAIWebSearchSource[] | null } | null;
sources?: XAIWebSearchSource[] | null;
results?: XAIWebSearchSource[] | null;
}
interface XAIResponsesUsage {
@@ -161,7 +174,12 @@ async function callXAIResponses(
throwXAIResponsesError(response.status, await response.text());
}
return (await response.json()) as XAIResponsesResponse;
try {
return (await response.json()) as XAIResponsesResponse;
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
throw new SearchProviderError("xai", `xAI Responses API returned invalid JSON: ${message}`, response.status);
}
}
function addCitationSource(
@@ -189,38 +207,77 @@ function addCitationSource(
citedText: sourceSnippet,
});
}
function extractSnippetAround(
text: string | null | undefined,
start: number | null | undefined,
end: number | null | undefined,
): string | undefined {
if (!text || typeof start !== "number" || typeof end !== "number") return undefined;
const before = Math.max(0, start - 100);
const after = Math.min(text.length, end + 100);
const snippet = text
.slice(before, after)
.replace(/\[([^\]]*)\]\([^)]*\)/g, "$1")
.trim();
if (!snippet) return undefined;
return snippet.length > 300 ? `${snippet.slice(0, 297)}...` : snippet;
}
function collectAnnotationSources(
annotations: readonly XAIUrlCitationAnnotation[] | null | undefined,
sources: SearchSource[],
citations: SearchCitation[],
seenUrls: Set<string>,
contentText?: string | null,
): void {
if (!annotations) return;
if (!Array.isArray(annotations)) return;
for (const annotation of annotations) {
if (annotation.type !== "url_citation" || !annotation.url) continue;
if (!annotation || typeof annotation !== "object") continue;
if (annotation.type !== "url_citation" || typeof annotation.url !== "string") continue;
addCitationSource(
sources,
citations,
seenUrls,
annotation.url,
annotation.title,
annotation.cited_text ?? annotation.text,
annotation.cited_text ??
annotation.text ??
extractSnippetAround(contentText, annotation.start_index, annotation.end_index),
);
}
}
function collectWebSearchSources(
item: XAIResponseOutputItem,
sources: SearchSource[],
citations: SearchCitation[],
seenUrls: Set<string>,
): void {
if (item.type !== "web_search_call") return;
for (const group of [item.action?.sources, item.sources, item.results]) {
if (!Array.isArray(group)) continue;
for (const source of group) {
if (!source || typeof source !== "object") continue;
const url = source.url ?? source.source_website_url;
if (typeof url !== "string") continue;
addCitationSource(sources, citations, seenUrls, url, source.title ?? source.caption);
}
}
}
function parseAnswer(response: XAIResponsesResponse): string | undefined {
const topLevelText = response.output_text?.trim();
if (topLevelText) return topLevelText;
const answerParts: string[] = [];
for (const item of response.output ?? []) {
for (const part of item.content ?? []) {
const output = Array.isArray(response.output) ? response.output : [];
for (const item of output) {
if (!item || typeof item !== "object") continue;
const content = Array.isArray(item.content) ? item.content : [];
for (const part of content) {
if (!part || typeof part !== "object") continue;
const text = part.output_text ?? part.text;
if ((part.type === "output_text" || part.type === "text") && text?.trim()) {
answerParts.push(text.trim());
}
if (text?.trim()) answerParts.push(text.trim());
}
}
@@ -259,13 +316,29 @@ function parseResponse(response: XAIResponsesResponse, resultCap: number): Searc
const seenUrls = new Set<string>();
collectAnnotationSources(response.annotations, sources, citations, seenUrls);
for (const item of response.output ?? []) {
const output = Array.isArray(response.output) ? response.output : [];
for (const item of output) {
if (!item || typeof item !== "object") continue;
collectAnnotationSources(item.annotations, sources, citations, seenUrls);
for (const part of item.content ?? []) {
collectAnnotationSources(part.annotations, sources, citations, seenUrls);
const content = Array.isArray(item.content) ? item.content : [];
for (const part of content) {
if (!part || typeof part !== "object") continue;
collectAnnotationSources(
part.annotations,
sources,
citations,
seenUrls,
part.output_text ?? part.text,
);
}
}
for (const url of response.citations ?? []) {
for (const item of output) {
if (!item || typeof item !== "object") continue;
collectWebSearchSources(item, sources, citations, seenUrls);
}
const topLevelCitations = Array.isArray(response.citations) ? response.citations : [];
for (const url of topLevelCitations) {
if (typeof url !== "string") continue;
addCitationSource(sources, citations, seenUrls, url);
}
const limited = applyResultCap(sources, citations, resultCap);
@@ -354,7 +427,11 @@ export async function searchXAI(params: SearchParams): Promise<SearchResponse> {
signal: params.signal,
missingKeyMessage: 'xAI credentials not found. Set XAI_API_KEY or configure an API key for provider "xai".',
});
return parseResponse(response, resultCap);
const parsed = parseResponse(response, resultCap);
if (!parsed.answer && parsed.sources.length === 0) {
throw new SearchProviderError("xai", "xAI web_search returned no answer or sources", 502);
}
return parsed;
}
/** Search provider for xAI web search. */