From a07d7820583925e8b9efd28458f96e49deeb00af Mon Sep 17 00:00:00 2001 From: roboomp Date: Thu, 30 Jul 2026 22:09:44 +0000 Subject: [PATCH] fix(search): paginated duckduckgo results Followed DuckDuckGo's returned continuation form until the requested result limit is satisfied, preserving deduplication across pages and the existing search deadline. Added regression coverage for continuation field submission and 20-result collection. Fixes #7116 --- packages/coding-agent/CHANGELOG.md | 4 + .../src/web/search/providers/duckduckgo.ts | 49 +++++++++--- .../test/tools/web-search-duckduckgo.test.ts | 75 +++++++++++++++++++ 3 files changed, 118 insertions(+), 10 deletions(-) create mode 100644 packages/coding-agent/test/tools/web-search-duckduckgo.test.ts diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 25c809467..b6e849a7e 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Fixed + +- Fixed DuckDuckGo web search under-returning requests above the first-page result count by submitting the returned continuation form until the requested limit is reached ([#7116](https://github.com/can1357/oh-my-pi/issues/7116)). + ## [17.2.1] - 2026-07-30 ### Added diff --git a/packages/coding-agent/src/web/search/providers/duckduckgo.ts b/packages/coding-agent/src/web/search/providers/duckduckgo.ts index f8343d7fb..173d35156 100644 --- a/packages/coding-agent/src/web/search/providers/duckduckgo.ts +++ b/packages/coding-agent/src/web/search/providers/duckduckgo.ts @@ -110,6 +110,25 @@ function parseHtmlResults(html: string): ParsedResult[] { } return results; } +/** + * Extract the hidden fields from DDG's next-page form. + * + * Attribute order varies across responses, so each input tag is parsed + * independently instead of matching one fixed HTML layout. + */ +function parseContinuationForm(html: string): URLSearchParams | undefined { + for (const formMatch of html.matchAll(/]*>([\s\S]*?)<\/form>/gi)) { + const form = new URLSearchParams(); + for (const inputMatch of formMatch[1].matchAll(/]*>/gi)) { + const input = inputMatch[0]; + const name = /\bname\s*=\s*(["'])(.*?)\1/i.exec(input)?.[2]; + const value = /\bvalue\s*=\s*(["'])(.*?)\1/i.exec(input)?.[2]; + if (name && value !== undefined) form.append(decodeHtmlText(name), decodeHtmlText(value)); + } + if (form.has("s") && form.has("vqd")) return form; + } + return undefined; +} /** * `true` when the page DDG returned is the bot-challenge modal instead of @@ -137,7 +156,7 @@ const DDG_QUERY_SYNTAX: QuerySyntax = { filetype: true, }; -async function callDuckDuckGoHtml(params: SearchParams): Promise { +function createDuckDuckGoForm(params: SearchParams): URLSearchParams { const form = new URLSearchParams({ q: formatScraperQuery(params.query, params.parsedQuery, DDG_QUERY_SYNTAX), kl: "us-en", @@ -146,10 +165,13 @@ async function callDuckDuckGoHtml(params: SearchParams): Promise { if (df) form.set("df", df); // Add b: "" parameter as specified in the browser fetch template to match real browser form submission form.set("b", ""); + return form; +} +async function callDuckDuckGoHtml(params: SearchParams, form: URLSearchParams, signal: AbortSignal): Promise { const page = await browserFetch(DUCKDUCKGO_HTML_URL, { fetch: params.fetch ?? fetch, - signal: withHardTimeout(params.signal), + signal, referer: "https://html.duckduckgo.com/", init: { method: "POST", @@ -179,16 +201,23 @@ async function callDuckDuckGoHtml(params: SearchParams): Promise { /** Execute a DuckDuckGo web search via the no-JS HTML frontend. */ export async function searchDuckDuckGo(params: SearchParams): Promise { const numResults = clampNumResults(params.numSearchResults ?? params.limit, DEFAULT_NUM_RESULTS, MAX_NUM_RESULTS); - const html = await callDuckDuckGoHtml(params); - const parsed = parseHtmlResults(html); - + const signal = withHardTimeout(params.signal); const sources: SearchSource[] = []; const seen = new Set(); - for (const result of parsed) { - if (seen.has(result.url)) continue; - seen.add(result.url); - sources.push({ title: result.title, url: result.url, snippet: result.snippet }); - if (sources.length >= numResults) break; + let form: URLSearchParams | undefined = createDuckDuckGoForm(params); + + while (form && sources.length < numResults) { + const html = await callDuckDuckGoHtml(params, form, signal); + const sourceCount = sources.length; + for (const result of parseHtmlResults(html)) { + if (seen.has(result.url)) continue; + seen.add(result.url); + sources.push({ title: result.title, url: result.url, snippet: result.snippet }); + if (sources.length >= numResults) break; + } + + if (sources.length === sourceCount) break; + form = parseContinuationForm(html); } return { provider: "duckduckgo", sources }; diff --git a/packages/coding-agent/test/tools/web-search-duckduckgo.test.ts b/packages/coding-agent/test/tools/web-search-duckduckgo.test.ts new file mode 100644 index 000000000..60cf37c12 --- /dev/null +++ b/packages/coding-agent/test/tools/web-search-duckduckgo.test.ts @@ -0,0 +1,75 @@ +import { Database } from "bun:sqlite"; +import { describe, expect, it } from "bun:test"; +import { AuthStorage, type FetchImpl, SqliteAuthCredentialStore } from "@oh-my-pi/pi-ai"; +import { searchDuckDuckGo } from "@oh-my-pi/pi-coding-agent/web/search/providers/duckduckgo"; + +function duckResult(index: number): string { + return `
Result ${index}Snippet ${index}
`; +} + +function duckPage(indices: readonly number[], continuation = false): string { + const results = indices.map(duckResult).join("\n"); + if (!continuation) return results; + return `${results} +
+
+ + + + + + + + + + +
+
`; +} + +describe("DuckDuckGo web search provider", () => { + it("submits the returned continuation form to satisfy a 20-result limit", async () => { + const authStorage = new AuthStorage(new SqliteAuthCredentialStore(new Database(":memory:"))); + const requests: URLSearchParams[] = []; + const fetchMock: FetchImpl = async (_input, init) => { + expect(init?.method).toBe("POST"); + const body = new URLSearchParams(String(init?.body)); + requests.push(body); + const html = + requests.length === 1 + ? duckPage([0, 1, 2, 3, 4, 5, 6, 7, 8, 9], true) + : duckPage([9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19]); + return new Response(html, { status: 200 }); + }; + + try { + const response = await searchDuckDuckGo({ + query: "open source software", + limit: 20, + systemPrompt: "Test DuckDuckGo search", + authStorage, + fetch: fetchMock, + }); + + expect(requests).toHaveLength(2); + expect(Object.fromEntries(requests[0])).toEqual({ q: "open source software", kl: "us-en", b: "" }); + expect(Object.fromEntries(requests[1])).toEqual({ + q: "open source software", + s: "10", + nextParams: "", + v: "l", + o: "json", + dc: "11", + api: "d.js", + vqd: "test-vqd", + kl: "us-en", + }); + expect(response.provider).toBe("duckduckgo"); + expect(response.sources).toHaveLength(20); + expect(response.sources[0]?.url).toBe("https://example.com/0"); + expect(response.sources.at(-1)?.url).toBe("https://example.com/19"); + } finally { + authStorage.close(); + } + }); +});