feat(coding-agent/web): expanded web search and fetching providers

- Expanded web search and fetching providers with robust parsing, authentication storage integration, and response validation.
- Added support for new configurations including SearXNG safesearch, Cloudflare AI Gateway endpoints, and dynamic Firecrawl base URLs.
- Implemented comprehensive test suites covering error handling, content filtering, and provider-specific response behaviors.

packages/coding-agent/test/tools/fetch-jina-stall.test.ts

@@ -94,3 +94,63 @@ describe("renderHtmlToText: jina stall does not starve local fallbacks (#1449)",
 		).toBe(true);
 	});
 });
+
+describe("renderHtmlToText: Jina response validation", () => {
+	it("requests fresh markdown and strips the Jina metadata preamble", async () => {
+		const settings = Settings.isolated({ "providers.fetch": "jina" });
+		const markdown = `# Extracted article\n\n${"Substantive reader content. ".repeat(8)}`.trim();
+		let requestHeaders: Headers | undefined;
+		const fetchMock = asGlobalFetch((_input, init) => {
+			requestHeaders = new Headers(init?.headers);
+			return new Response(`Title: Example\nURL Source: https://example.com/article\nMarkdown Content:\n${markdown}`);
+		});
+
+		const result = await renderHtmlToText(
+			"https://example.com/article",
+			"<html><body>short</body></html>",
+			1,
+			settings,
+			undefined,
+			null,
+			fetchMock,
+		);
+
+		expect(result).toEqual({ content: markdown, ok: true, method: "jina" });
+		expect(requestHeaders?.get("accept")).toBe("text/markdown");
+		expect(requestHeaders?.get("x-no-cache")).toBe("true");
+	});
+
+	for (const { label, readerBody, headers } of [
+		{ label: "missing marker", readerBody: "Plausible but unstructured output. ".repeat(8) },
+		{ label: "short body", readerBody: "Markdown Content:\nToo short" },
+		{ label: "loading shell", readerBody: `Markdown Content:\nLoading...${" ".repeat(120)}` },
+		{ label: "JavaScript gate", readerBody: `Markdown Content:\nPlease enable JavaScript${" ".repeat(120)}` },
+		{
+			label: "declared oversized body",
+			readerBody: `Markdown Content:\n${"Substantive content. ".repeat(8)}`,
+			headers: { "Content-Length": String(2 * 1024 * 1024 + 1) },
+		},
+	]) {
+		it(`falls back when Jina returns a ${label}`, async () => {
+			const settings = Settings.isolated({ "providers.fetch": "jina" });
+			const paragraph =
+				"This locally rendered article contains enough meaningful prose to satisfy the shared reader quality gate. ";
+			const html = `<html><body><article><h1>Fallback article</h1><p>${paragraph.repeat(4)}</p></article></body></html>`;
+			const fetchMock = asGlobalFetch(() => new Response(readerBody, { headers }));
+
+			const result = await renderHtmlToText(
+				"https://example.com/article",
+				html,
+				1,
+				settings,
+				undefined,
+				null,
+				fetchMock,
+			);
+
+			expect(result.ok).toBe(true);
+			expect(result.method).toBe("native");
+			expect(result.content).toContain("Fallback article");
+		});
+	}
+});

packages/coding-agent/test/tools/web-search-codex.test.ts

@@ -531,6 +531,70 @@ describe("searchCodex model selection", () => {
 		expect(result.sources).toEqual([{ title: "Example Article", url: "https://example.com/article" }]);
 	});
 
+	it("requests and merges web-search action sources with citation metadata", async () => {
+		process.env.PI_CODEX_WEB_SEARCH_MODEL = "gpt-5.4";
+		const answer = "The Responses API supports hosted web search.";
+		const citationStart = answer.indexOf("hosted web search");
+		const sse = [
+			`data: ${JSON.stringify({
+				type: "response.created",
+				response: { id: "resp_created_id", model: "gpt-5.4" },
+			})}`,
+			"",
+			`data: ${JSON.stringify({
+				type: "response.output_item.done",
+				item: {
+					type: "web_search_call",
+					action: {
+						sources: [
+							{
+								url: "https://example.com/article?utm_source=openai",
+								title: "Search result title",
+							},
+						],
+					},
+				},
+			})}`,
+			"",
+			`data: ${JSON.stringify({
+				type: "response.output_item.done",
+				item: {
+					type: "message",
+					content: [
+						{
+							type: "output_text",
+							text: answer,
+							annotations: [
+								{
+									type: "url_citation",
+									url: "https://example.com/article?utm_source=openai",
+									title: "Example Article",
+									start_index: citationStart,
+									end_index: citationStart + "hosted web search".length,
+								},
+							],
+						},
+					],
+				},
+			})}`,
+			"",
+		].join("\n");
+
+		const result = await searchCodex(
+			makeSearchParams("action sources", mockCodexFetch("gpt-5.4", sse)),
+		);
+
+		expect(capturedRequest?.body?.include).toEqual(["web_search_call.action.sources"]);
+		expect(result.requestId).toBe("resp_created_id");
+		expect(result.sources).toEqual([
+			{
+				title: "Search result title",
+				url: "https://example.com/article",
+				snippet: answer,
+			},
+		]);
+	});
+
 	it("extracts plain text URLs when annotations are absent", async () => {
 		process.env.PI_CODEX_WEB_SEARCH_MODEL = "gpt-5.4";
 		const result = await searchCodex(
@@ -769,4 +833,22 @@ describe("searchCodex model selection", () => {
 			"Codex request failed (model_snapshot_unavailable): The requested model snapshot is unavailable.",
 		);
 	});
+
+	it("classifies rate-limit failures delivered inside a successful SSE response", async () => {
+		const sse = [
+			`data: ${JSON.stringify({
+				type: "response.failed",
+				response: {
+					error: { code: "rate_limit_exceeded", message: "Too many requests" },
+				},
+			})}`,
+			"",
+		].join("\n");
+		const fetchMock: FetchImpl = () =>
+			Promise.resolve(new Response(sse, { status: 200, headers: { "Content-Type": "text/event-stream" } }));
+
+		await expect(searchCodex(makeSearchParams("rate-limited search", fetchMock))).rejects.toMatchObject({
+			status: 429,
+		});
+	});
 });

packages/coding-agent/test/tools/web-search-exa.test.ts

@@ -428,6 +428,19 @@ describe("searchExa", () => {
 		expect(result.sources[0].snippet).toBe("summary here");
 	});
 
+	it("caps snippets at 500 characters", async () => {
+		const result = await searchExa({
+			query: "bounded snippet",
+			fetch: mockFetch(
+				makeMockExaResponse({
+					results: [{ title: "Long", url: "https://long.example", summary: "x".repeat(800) }],
+				}),
+			),
+		});
+
+		expect(result.sources[0].snippet).toHaveLength(500);
+	});
+
 	it("falls back to text when summary is null", async () => {
 		const result = await searchExa({
 			query: "fallback",
@@ -535,6 +548,68 @@ describe("searchExa", () => {
 		});
 	});
 
+	it("encodes MCP filters in the basic query, uses camel-case result count, and tags the request source", async () => {
+		delete process.env.EXA_API_KEY;
+		let headers: Record<string, string> | undefined;
+		const fetchMock: FetchImpl = (_url, init) => {
+			headers = init?.headers as Record<string, string> | undefined;
+			if (init?.body) capturedRequestBody = JSON.parse(init.body as string);
+			return Promise.resolve(
+				new Response(JSON.stringify({ jsonrpc: "2.0", id: "mcp-filtered", result: makeMockExaResponse() }), {
+					status: 200,
+					headers: { "Content-Type": "application/json" },
+				}),
+			);
+		};
+
+		await searchExa({
+			query: "vector databases",
+			num_results: 4,
+			include_domains: [" qdrant.tech "],
+			exclude_domains: ["spam.example"],
+			start_published_date: "2024-01-01",
+			end_published_date: "2025-01-01",
+			fetch: fetchMock,
+		});
+
+		expect(headers?.["x-exa-source"]).toBe("oh-my-pi");
+		expect(capturedRequestBody?.params).toEqual({
+			name: "web_search_exa",
+			arguments: {
+				query:
+					"vector databases site:qdrant.tech -site:spam.example after:2024-01-01 before:2025-01-01",
+				numResults: 4,
+			},
+		});
+	});
+
+	it("explains how to escape the keyless MCP rate limit", async () => {
+		delete process.env.EXA_API_KEY;
+		const fetchMock: FetchImpl = () =>
+			Promise.resolve(new Response("too many requests", { status: 429, statusText: "Too Many Requests" }));
+
+		await expect(searchExa({ query: "rate limited", fetch: fetchMock })).rejects.toThrow(
+			"exa: MCP rate limit reached (429); configure an Exa API key for higher limits",
+		);
+	});
+
+	it("surfaces MCP tool-level errors", async () => {
+		delete process.env.EXA_API_KEY;
+		const fetchMock: FetchImpl = () =>
+			Promise.resolve(
+				new Response(
+					JSON.stringify({
+						jsonrpc: "2.0",
+						id: "mcp-error",
+						result: { isError: true, content: [{ type: "text", text: "tool quota exceeded" }] },
+					}),
+					{ status: 200, headers: { "Content-Type": "application/json" } },
+				),
+			);
+
+		await expect(searchExa({ query: "tool error", fetch: fetchMock })).rejects.toThrow("tool quota exceeded");
+	});
+
 	it("parses Exa MCP plain-text payloads when API key is missing", async () => {
 		delete process.env.EXA_API_KEY;
 		const fetchMock: FetchImpl = () => {

packages/coding-agent/test/tools/web-search-firecrawl.test.ts

@@ -235,18 +235,58 @@ describe("Firecrawl web search provider", () => {
 		}
 	});
 
-	it("keeps keyless Firecrawl out of auto selection while allowing explicit selection", () => {
+	it("keeps hosted keyless Firecrawl explicit-only but admits configured self-hosting", () => {
 		const originalApiKey = process.env.FIRECRAWL_API_KEY;
+		const originalBaseUrl = process.env.FIRECRAWL_BASE_URL;
+		const originalApiUrl = process.env.FIRECRAWL_API_URL;
 		delete process.env.FIRECRAWL_API_KEY;
+		delete process.env.FIRECRAWL_BASE_URL;
+		delete process.env.FIRECRAWL_API_URL;
 		try {
 			const provider = new FirecrawlProvider();
 			const authStorage = makeAuthStorage(undefined);
 
 			expect(provider.isAvailable(authStorage)).toBe(false);
 			expect(provider.isExplicitlyAvailable(authStorage)).toBe(true);
+			process.env.FIRECRAWL_BASE_URL = "http://localhost:3002";
+			expect(provider.isAvailable(authStorage)).toBe(true);
 		} finally {
 			if (originalApiKey === undefined) delete process.env.FIRECRAWL_API_KEY;
 			else process.env.FIRECRAWL_API_KEY = originalApiKey;
+			if (originalBaseUrl === undefined) delete process.env.FIRECRAWL_BASE_URL;
+			else process.env.FIRECRAWL_BASE_URL = originalBaseUrl;
+			if (originalApiUrl === undefined) delete process.env.FIRECRAWL_API_URL;
+			else process.env.FIRECRAWL_API_URL = originalApiUrl;
+		}
+	});
+
+	it("uses a self-hosted endpoint and accepts Firecrawl v1 array responses", async () => {
+		const originalBaseUrl = process.env.FIRECRAWL_BASE_URL;
+		process.env.FIRECRAWL_BASE_URL = "http://localhost:3002/v1/";
+		let requestUrl = "";
+		try {
+			const fetchMock: FetchImpl = async input => {
+				requestUrl = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url;
+				return new Response(
+					JSON.stringify({
+						success: true,
+						data: [{ title: "Legacy result", url: "https://example.com/legacy", snippet: "Legacy snippet" }],
+					}),
+					{ status: 200, headers: { "Content-Type": "application/json" } },
+				);
+			};
+			const response = await searchFirecrawl({
+				...makeParams("legacy query", makeAuthStorage(undefined)),
+				fetch: fetchMock,
+			});
+
+			expect(requestUrl).toBe("http://localhost:3002/v1/search");
+			expect(response.sources).toEqual([
+				{ title: "Legacy result", url: "https://example.com/legacy", snippet: "Legacy snippet" },
+			]);
+		} finally {
+			if (originalBaseUrl === undefined) delete process.env.FIRECRAWL_BASE_URL;
+			else process.env.FIRECRAWL_BASE_URL = originalBaseUrl;
 		}
 	});
 

packages/coding-agent/test/tools/web-search-gemini.test.ts

@@ -10,6 +10,7 @@ const DEVELOPER_SSE_RESPONSE =
 const DEVELOPER_SSE_RESPONSE_WITHOUT_MODEL =
 	'data: {"candidates":[{"content":{"role":"model","parts":[{"text":"Developer answer"}]},"groundingMetadata":{"webSearchQueries":["latest Bun version"],"groundingChunks":[{"web":{"uri":"https://bun.sh","title":"Bun"}}],"groundingSupports":[{"segment":{"text":"Developer answer"},"groundingChunkIndices":[0]}]}}],"usageMetadata":{"promptTokenCount":3,"candidatesTokenCount":4,"totalTokenCount":7}}\n\n';
 const ORIGINAL_GEMINI_SEARCH_MODEL = Bun.env.GEMINI_SEARCH_MODEL;
+const ORIGINAL_GEMINI_BASE_URL = Bun.env.GOOGLE_GEMINI_BASE_URL;
 
 type CapturedRequest = {
 	url: string;
@@ -72,6 +73,11 @@ describe("searchGemini tools serialization", () => {
 		} else {
 			Bun.env.GEMINI_SEARCH_MODEL = ORIGINAL_GEMINI_SEARCH_MODEL;
 		}
+		if (ORIGINAL_GEMINI_BASE_URL === undefined) {
+			delete Bun.env.GOOGLE_GEMINI_BASE_URL;
+		} else {
+			Bun.env.GOOGLE_GEMINI_BASE_URL = ORIGINAL_GEMINI_BASE_URL;
+		}
 	});
 
 	function makeParams(query: string) {
@@ -111,6 +117,61 @@ describe("searchGemini tools serialization", () => {
 		});
 	});
 
+	it("routes Cloudflare AI Gateway auth through AuthStorage without leaking a Google API key", async () => {
+		Bun.env.GOOGLE_GEMINI_BASE_URL =
+			"https://gateway.ai.cloudflare.com/v1/account/gateway/google-ai-studio";
+		const gatewayAuthStorage = {
+			async getOAuthAccess() {
+				return undefined;
+			},
+			hasOAuth() {
+				return false;
+			},
+			hasAuth(provider: string) {
+				return provider === "cloudflare-ai-gateway";
+			},
+			async getApiKey(provider: string) {
+				return provider === "cloudflare-ai-gateway" ? "test-cloudflare-key" : undefined;
+			},
+		} as unknown as AuthStorage;
+		const fetchMock = mockGeminiFetch(DEVELOPER_SSE_RESPONSE);
+
+		expect(new GeminiProvider().isAvailable(gatewayAuthStorage)).toBe(true);
+		await searchGemini({
+			...makeParams("gateway"),
+			authStorage: gatewayAuthStorage,
+			fetch: fetchMock,
+		});
+
+		expect(capturedRequest?.url).toBe(
+			"https://gateway.ai.cloudflare.com/v1/account/gateway/google-ai-studio/v1beta/models/gemini-2.5-flash:streamGenerateContent?alt=sse",
+		);
+		expect(capturedRequest?.headers["cf-aig-authorization"]).toBe("Bearer test-cloudflare-key");
+		expect(capturedRequest?.headers["x-goog-api-key"]).toBeUndefined();
+	});
+
+	it("redacts the active credential from Gemini API errors", async () => {
+		let thrown: unknown;
+		try {
+			await searchGemini({
+				...makeParams("redaction"),
+				authStorage: apiKeyAuthStorage,
+				fetch: () =>
+					Promise.resolve(
+						new Response("upstream echoed test-gemini-api-key", {
+							status: 418,
+						}),
+					),
+			});
+		} catch (error) {
+			thrown = error;
+		}
+
+		expect(thrown).toBeInstanceOf(Error);
+		expect((thrown as Error).message).toContain("[redacted]");
+		expect((thrown as Error).message).not.toContain("test-gemini-api-key");
+	});
+
 	it("normalizes query directive aliases to canonical Google forms in the grounding request", async () => {
 		const fetchMock = mockGeminiFetch();
 		await searchGemini({
@@ -221,4 +282,55 @@ describe("searchGemini tools serialization", () => {
 			tools: [{ googleSearch: {} }, { codeExecution: {} }, { urlContext: { allowedDomains: ["example.com"] } }],
 		});
 	});
+
+	it("resolves Google grounding proxy URLs in both sources and citations", async () => {
+		const proxyUrl =
+			"https://vertexaisearch.cloud.google.com/grounding-api-redirect/abc";
+		const responseText = `data: ${JSON.stringify({
+			candidates: [
+				{
+					content: { role: "model", parts: [{ text: "Grounded answer" }] },
+					groundingMetadata: {
+						groundingChunks: [{ web: { uri: proxyUrl, title: "Example" } }],
+						groundingSupports: [{ segment: { text: "Grounded answer" }, groundingChunkIndices: [0] }],
+					},
+				},
+			],
+		})}\n\n`;
+		const methods: string[] = [];
+		const fetchMock: FetchImpl = (_url, init) => {
+			methods.push(init?.method ?? "GET");
+			if (init?.method === "HEAD") {
+				return Promise.resolve(
+					new Response(null, {
+						status: 302,
+						headers: { location: "https://example.com/article" },
+					}),
+				);
+			}
+			return Promise.resolve(new Response(responseText, { status: 200 }));
+		};
+
+		const response = await searchGemini({
+			...makeParams("grounding redirect"),
+			authStorage: apiKeyAuthStorage,
+			fetch: fetchMock,
+		});
+
+		expect(methods).toEqual(["POST", "HEAD"]);
+		expect(response.sources).toEqual([{ title: "Example", url: "https://example.com/article" }]);
+		expect(response.citations).toEqual([
+			{ title: "Example", url: "https://example.com/article", citedText: "Grounded answer" },
+		]);
+	});
+
+	it("rejects a successful Gemini response with no answer or grounding results", async () => {
+		await expect(
+			searchGemini({
+				...makeParams("empty"),
+				authStorage: apiKeyAuthStorage,
+				fetch: mockGeminiFetch("data: {}\n\n"),
+			}),
+		).rejects.toThrow("Gemini API returned an empty grounded response");
+	});
 });

packages/coding-agent/test/tools/web-search-kagi.test.ts

@@ -58,6 +58,34 @@ describe("Kagi web search error handling", () => {
 			"Kagi API error (502)",
 		);
 	});
+
+	it("reports malformed success responses as Kagi API errors", async () => {
+		const invalidJsonFetch: FetchImpl = async () => new Response("<html>not json</html>", { status: 200 });
+		const invalidEnvelopeFetch: FetchImpl = async () =>
+			new Response(JSON.stringify(["unexpected"]), {
+				status: 200,
+				headers: { "Content-Type": "application/json" },
+			});
+
+		await expect(searchWithKagi("invalid json", { fetch: invalidJsonFetch }, fakeAuthStorage)).rejects.toThrow(
+			"Kagi API returned an invalid response: invalid JSON",
+		);
+		await expect(searchWithKagi("invalid envelope", { fetch: invalidEnvelopeFetch }, fakeAuthStorage)).rejects.toThrow(
+			"Kagi API returned an invalid response: expected an object envelope",
+		);
+	});
+
+	it("recognizes errors plural in a successful HTTP envelope", async () => {
+		const fetchMock: FetchImpl = async () =>
+			new Response(JSON.stringify({ errors: [{ code: 429, message: "quota exceeded" }] }), {
+				status: 200,
+				headers: { "Content-Type": "application/json" },
+			});
+
+		await expect(searchWithKagi("envelope error", { fetch: fetchMock }, fakeAuthStorage)).rejects.toThrow(
+			"Kagi API error (429): quota exceeded",
+		);
+	});
 	it("applies the configured timeout at the provider fetch boundary", async () => {
 		const timeoutSignal = new AbortController().signal;
 		const timeoutSpy = vi.spyOn(AbortSignal, "timeout").mockReturnValue(timeoutSignal);
@@ -158,6 +186,39 @@ describe("Kagi search result parsing", () => {
 		expect(result.answer).toBeUndefined();
 	});
 
+	it("accepts documented result aliases and skips malformed items", async () => {
+		const fetchMock: FetchImpl = async () =>
+			new Response(
+				JSON.stringify({
+					data: {
+						search: [
+							null,
+							{ title: "Missing URL" },
+							{
+								href: "https://example.com/alias",
+								name: "Alias Result",
+								description: "Alias description",
+							},
+						],
+						related_search: { invalid: true },
+					},
+				}),
+				{ status: 200, headers: { "Content-Type": "application/json" } },
+			);
+
+		const result = await searchWithKagi("aliases", { fetch: fetchMock }, fakeAuthStorage);
+
+		expect(result.sources).toEqual([
+			{
+				title: "Alias Result",
+				url: "https://example.com/alias",
+				snippet: "Alias description",
+				publishedDate: undefined,
+			},
+		]);
+		expect(result.relatedQuestions).toEqual([]);
+	});
+
 	it("parses direct_answer into the answer field", async () => {
 		const fetchMock: FetchImpl = async () =>
 			new Response(

packages/coding-agent/test/tools/web-search-parallel.test.ts

@@ -1,4 +1,4 @@
-import { afterEach, beforeEach, describe, expect, it, vi } from "bun:test";
+import { afterEach, beforeEach, describe, expect, it, setSystemTime, vi } from "bun:test";
 import type { AuthStorage, FetchImpl } from "@oh-my-pi/pi-ai";
 import type { AgentStorage } from "@oh-my-pi/pi-coding-agent/session/agent-storage";
 import { searchWithParallel } from "@oh-my-pi/pi-coding-agent/web/parallel";
@@ -149,6 +149,29 @@ describe("Parallel web search", () => {
 		});
 	});
 
+	it("maps recency onto source_policy.after_date", async () => {
+		setSystemTime(new Date("2026-08-10T12:00:00Z"));
+		try {
+			const fetchMock = mockFetch({
+				search_id: "search-parallel-recency",
+				results: [],
+				warnings: null,
+				usage: null,
+			});
+
+			await searchParallel({ query: "recent api changes", recency: "week", fetch: fetchMock }, fakeAuthStorage);
+			expect(capturedRequestBody).toEqual({
+				objective: "recent api changes",
+				search_queries: ["recent api changes"],
+				mode: "fast",
+				excerpts: { max_chars_per_result: 10_000 },
+				source_policy: { after_date: "2026-08-03" },
+			});
+		} finally {
+			setSystemTime();
+		}
+	});
+
 	it("maps -site: and after: onto exclude_domains/after_date, keeping phrases and negation", async () => {
 		const fetchMock = mockFetch({
 			search_id: "search-parallel-4",
@@ -158,7 +181,11 @@ describe("Parallel web search", () => {
 		});
 
 		await searchParallel(
-			{ query: '"web api" -legacy -site:reddit.com/r/node after:2025-06-01', fetch: fetchMock },
+			{
+				query: '"web api" -legacy -site:reddit.com/r/node after:2025-06-01',
+				recency: "day",
+				fetch: fetchMock,
+			},
 			fakeAuthStorage,
 		);
 		expect(capturedRequestBody).toEqual({
@@ -178,4 +205,13 @@ describe("Parallel web search", () => {
 			message: "Parallel API error (503): upstream unavailable",
 		});
 	});
+
+	it("classifies malformed successful responses as Parallel errors", async () => {
+		const fetchMock: FetchImpl = () =>
+			Promise.resolve(new Response("{not-json", { status: 200, headers: { "Content-Type": "application/json" } }));
+		await expect(searchParallel({ query: "broken", fetch: fetchMock }, fakeAuthStorage)).rejects.toMatchObject({
+			provider: "parallel",
+			message: expect.stringContaining("Parallel search returned invalid JSON:"),
+		});
+	});
 });

packages/coding-agent/test/tools/web-search-searxng.test.ts

@@ -121,6 +121,53 @@ describe("SearXNG web search provider", () => {
 		expect(captured.url?.searchParams.get("language")).toBeNull();
 	});
 
+	it("sends configured category and safe-search filters and preserves instant answers", async () => {
+		const agentDir = await fs.mkdtemp(path.join(os.tmpdir(), "searxng-filters-"));
+		try {
+			await Bun.write(
+				path.join(agentDir, "config.yml"),
+				[
+					"searxng:",
+					"  endpoint: https://searx.example.org",
+					"  categories: news",
+					"  safesearch: 2",
+					"",
+				].join("\n"),
+			);
+			await Settings.init({ agentDir });
+
+			const captured: { url?: URL } = {};
+			const fetchMock: FetchImpl = input => {
+				captured.url = new URL(input.toString());
+				return Promise.resolve(
+					new Response(
+						JSON.stringify({
+							results: [{ title: "r", url: "https://example.com", snippet: "Fallback snippet" }],
+							answers: [
+								"  Forty-two  ",
+								{ template: "answer/legacy.html", answer: "Legacy answer" },
+								{
+									template: "answer/translations.html",
+									translations: [{ text: "Hallo" }, { text: "Guten Tag" }],
+								},
+							],
+						}),
+						{ status: 200, headers: { "Content-Type": "application/json" } },
+					),
+				);
+			};
+
+			const response = await searchSearXNG({ query: "filtered answers", fetch: fetchMock });
+
+			expect(captured.url?.searchParams.get("categories")).toBe("news");
+			expect(captured.url?.searchParams.get("safesearch")).toBe("2");
+			expect(response.answer).toBe("Forty-two\n\nLegacy answer\n\nHallo\nGuten Tag");
+			expect(response.sources[0]?.snippet).toBe("Fallback snippet");
+		} finally {
+			await removeWithRetries(agentDir);
+		}
+	});
+
 	it("reads Basic auth credentials from nested config.yml settings", async () => {
 		const agentDir = await fs.mkdtemp(path.join(os.tmpdir(), "searxng-settings-"));
 		try {

packages/coding-agent/test/tools/web-search-tinyfish.test.ts

@@ -84,10 +84,16 @@ describe("TinyFish web search provider", () => {
 		});
 
 		expect(captured).toHaveLength(1);
-		expect(captured[0].searchParams.get("query")).toBe(
-			'"error handling" rust site:github.com -site:gitlab.com filetype:pdf',
-		);
-		expectTinyFishParams(captured[0], ["query", "num_results", "page"]);
+		expect(captured[0].searchParams.get("query")).toBe('"error handling" rust filetype:pdf');
+		expect(captured[0].searchParams.get("include_domains")).toBe("github.com");
+		expect(captured[0].searchParams.get("exclude_domains")).toBe("gitlab.com");
+		expectTinyFishParams(captured[0], [
+			"query",
+			"num_results",
+			"page",
+			"include_domains",
+			"exclude_domains",
+		]);
 	});
 
 	it("sends directive-free queries verbatim", async () => {
@@ -232,6 +238,43 @@ describe("TinyFish web search provider", () => {
 		expect(response.sources.at(-1)?.url).toBe("https://example.com/raw-page-11");
 	});
 
+	it("deduplicates and normalizes results across pages", async () => {
+		const captured: URL[] = [];
+		const firstPage = tinyFishResults("dedupe", 10);
+		firstPage[0] = {
+			title: "  Primary title  ",
+			url: "  https://example.com/dedupe-0  ",
+			snippet: "  spaced \n snippet  ",
+			site_name: "  Example  ",
+		};
+		firstPage[1] = {
+			title: "Duplicate title",
+			url: "https://example.com/dedupe-0",
+			snippet: "duplicate snippet",
+		};
+		const fetchMock: FetchImpl = async input => {
+			const url = input instanceof URL ? input : new URL(typeof input === "string" ? input : input.url);
+			captured.push(url);
+			const page = Number(url.searchParams.get("page") ?? 0);
+			const results = page === 0 ? firstPage : tinyFishResults("dedupe", 1, 10);
+			return new Response(JSON.stringify(tinyFishPage(results, page, 11)), {
+				status: 200,
+				headers: { "Content-Type": "application/json" },
+			});
+		};
+
+		const response = await searchTinyFish({ ...makeParams("dedupe fish"), limit: 10, fetch: fetchMock });
+
+		expect(captured.map(url => url.searchParams.get("page"))).toEqual(["0", "1"]);
+		expect(response.sources).toHaveLength(10);
+		expect(response.sources[0]).toMatchObject({
+			title: "Primary title",
+			url: "https://example.com/dedupe-0",
+			snippet: "spaced snippet",
+		});
+		expect(response.sources.at(-1)?.url).toBe("https://example.com/dedupe-10");
+	});
+
 	it("stops early for limit 20 when page 0 returns fewer than 10 raw results", async () => {
 		const captured: URL[] = [];
 		const fetchMock: FetchImpl = async input => {

packages/coding-agent/test/tools/web-search-xai.test.ts

@@ -802,6 +802,67 @@ describe("xAI web search provider", () => {
 		});
 	});
 
+	it("extracts offset snippets and raw sources from web_search_call output", async () => {
+		const answer = "Context before [cited source](https://example.com/cited) context after.";
+		const start = answer.indexOf("[cited source]");
+		const capture = captureFetch({
+			id: "resp_raw_sources",
+			output: [
+				{
+					type: "message",
+					content: [
+						{
+							type: "output_text",
+							text: answer,
+							annotations: [
+								{
+									type: "url_citation",
+									url: "https://example.com/cited",
+									title: "Cited result",
+									start_index: start,
+									end_index: start + "[cited source]".length,
+								},
+							],
+						},
+					],
+				},
+				{
+					type: "web_search_call",
+					action: {
+						sources: [
+							{ url: "https://example.com/raw", title: "Raw result" },
+							{ source_website_url: "https://example.com/fallback", caption: "Fallback result" },
+						],
+					},
+					results: [{ url: "https://example.com/cited", title: "Duplicate result" }],
+				},
+			],
+		});
+
+		const response = await searchXAI(makeParams(capture.fetchMock));
+
+		expect(response.answer).toBe(answer);
+		expect(response.sources).toEqual([
+			{
+				title: "Cited result",
+				url: "https://example.com/cited",
+				snippet: "Context before cited source context after.",
+			},
+			{ title: "Raw result", url: "https://example.com/raw", snippet: undefined },
+			{ title: "Fallback result", url: "https://example.com/fallback", snippet: undefined },
+		]);
+	});
+
+	it("rejects successful responses with no answer or sources", async () => {
+		const capture = captureFetch({ id: "resp_empty", output: [] });
+
+		await expect(searchXAI(makeParams(capture.fetchMock))).rejects.toMatchObject({
+			provider: "xai",
+			status: 502,
+			message: "xAI web_search returned no answer or sources",
+		});
+	});
+
 	it.each([
 		[401, "xai: 401 unauthorized"],
 		[402, "xai: 402 credits exhausted"],