diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index d94287d59..c13afa8fe 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -40,6 +40,9 @@ ### Fixed - Removed the `read` exemption from the centralized artifact spill wrapper. Oversized read results now persist to a recoverable session artifact and return the configured inline head/tail instead of bypassing the threshold shared by other tools. +### Fixed + +- Fixed DuckDuckGo web search under-returning requests above the first-page result count by submitting the returned continuation form until the requested limit is reached ([#7116](https://github.com/can1357/oh-my-pi/issues/7116)). ## [17.2.1] - 2026-07-30 diff --git a/packages/coding-agent/src/web/search/providers/duckduckgo.ts b/packages/coding-agent/src/web/search/providers/duckduckgo.ts index f8343d7fb..173d35156 100644 --- a/packages/coding-agent/src/web/search/providers/duckduckgo.ts +++ b/packages/coding-agent/src/web/search/providers/duckduckgo.ts @@ -110,6 +110,25 @@ function parseHtmlResults(html: string): ParsedResult[] { } return results; } +/** + * Extract the hidden fields from DDG's next-page form. + * + * Attribute order varies across responses, so each input tag is parsed + * independently instead of matching one fixed HTML layout. + */ +function parseContinuationForm(html: string): URLSearchParams | undefined { + for (const formMatch of html.matchAll(/]*>([\s\S]*?)<\/form>/gi)) { + const form = new URLSearchParams(); + for (const inputMatch of formMatch[1].matchAll(/]*>/gi)) { + const input = inputMatch[0]; + const name = /\bname\s*=\s*(["'])(.*?)\1/i.exec(input)?.[2]; + const value = /\bvalue\s*=\s*(["'])(.*?)\1/i.exec(input)?.[2]; + if (name && value !== undefined) form.append(decodeHtmlText(name), decodeHtmlText(value)); + } + if (form.has("s") && form.has("vqd")) return form; + } + return undefined; +} /** * `true` when the page DDG returned is the bot-challenge modal instead of @@ -137,7 +156,7 @@ const DDG_QUERY_SYNTAX: QuerySyntax = { filetype: true, }; -async function callDuckDuckGoHtml(params: SearchParams): Promise { +function createDuckDuckGoForm(params: SearchParams): URLSearchParams { const form = new URLSearchParams({ q: formatScraperQuery(params.query, params.parsedQuery, DDG_QUERY_SYNTAX), kl: "us-en", @@ -146,10 +165,13 @@ async function callDuckDuckGoHtml(params: SearchParams): Promise { if (df) form.set("df", df); // Add b: "" parameter as specified in the browser fetch template to match real browser form submission form.set("b", ""); + return form; +} +async function callDuckDuckGoHtml(params: SearchParams, form: URLSearchParams, signal: AbortSignal): Promise { const page = await browserFetch(DUCKDUCKGO_HTML_URL, { fetch: params.fetch ?? fetch, - signal: withHardTimeout(params.signal), + signal, referer: "https://html.duckduckgo.com/", init: { method: "POST", @@ -179,16 +201,23 @@ async function callDuckDuckGoHtml(params: SearchParams): Promise { /** Execute a DuckDuckGo web search via the no-JS HTML frontend. */ export async function searchDuckDuckGo(params: SearchParams): Promise { const numResults = clampNumResults(params.numSearchResults ?? params.limit, DEFAULT_NUM_RESULTS, MAX_NUM_RESULTS); - const html = await callDuckDuckGoHtml(params); - const parsed = parseHtmlResults(html); - + const signal = withHardTimeout(params.signal); const sources: SearchSource[] = []; const seen = new Set(); - for (const result of parsed) { - if (seen.has(result.url)) continue; - seen.add(result.url); - sources.push({ title: result.title, url: result.url, snippet: result.snippet }); - if (sources.length >= numResults) break; + let form: URLSearchParams | undefined = createDuckDuckGoForm(params); + + while (form && sources.length < numResults) { + const html = await callDuckDuckGoHtml(params, form, signal); + const sourceCount = sources.length; + for (const result of parseHtmlResults(html)) { + if (seen.has(result.url)) continue; + seen.add(result.url); + sources.push({ title: result.title, url: result.url, snippet: result.snippet }); + if (sources.length >= numResults) break; + } + + if (sources.length === sourceCount) break; + form = parseContinuationForm(html); } return { provider: "duckduckgo", sources }; diff --git a/packages/coding-agent/test/tools/web-search-duckduckgo.test.ts b/packages/coding-agent/test/tools/web-search-duckduckgo.test.ts new file mode 100644 index 000000000..60cf37c12 --- /dev/null +++ b/packages/coding-agent/test/tools/web-search-duckduckgo.test.ts @@ -0,0 +1,75 @@ +import { Database } from "bun:sqlite"; +import { describe, expect, it } from "bun:test"; +import { AuthStorage, type FetchImpl, SqliteAuthCredentialStore } from "@oh-my-pi/pi-ai"; +import { searchDuckDuckGo } from "@oh-my-pi/pi-coding-agent/web/search/providers/duckduckgo"; + +function duckResult(index: number): string { + return `
Result ${index}Snippet ${index}
`; +} + +function duckPage(indices: readonly number[], continuation = false): string { + const results = indices.map(duckResult).join("\n"); + if (!continuation) return results; + return `${results} +
+
+ + + + + + + + + + +
+
`; +} + +describe("DuckDuckGo web search provider", () => { + it("submits the returned continuation form to satisfy a 20-result limit", async () => { + const authStorage = new AuthStorage(new SqliteAuthCredentialStore(new Database(":memory:"))); + const requests: URLSearchParams[] = []; + const fetchMock: FetchImpl = async (_input, init) => { + expect(init?.method).toBe("POST"); + const body = new URLSearchParams(String(init?.body)); + requests.push(body); + const html = + requests.length === 1 + ? duckPage([0, 1, 2, 3, 4, 5, 6, 7, 8, 9], true) + : duckPage([9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19]); + return new Response(html, { status: 200 }); + }; + + try { + const response = await searchDuckDuckGo({ + query: "open source software", + limit: 20, + systemPrompt: "Test DuckDuckGo search", + authStorage, + fetch: fetchMock, + }); + + expect(requests).toHaveLength(2); + expect(Object.fromEntries(requests[0])).toEqual({ q: "open source software", kl: "us-en", b: "" }); + expect(Object.fromEntries(requests[1])).toEqual({ + q: "open source software", + s: "10", + nextParams: "", + v: "l", + o: "json", + dc: "11", + api: "d.js", + vqd: "test-vqd", + kl: "us-en", + }); + expect(response.provider).toBe("duckduckgo"); + expect(response.sources).toHaveLength(20); + expect(response.sources[0]?.url).toBe("https://example.com/0"); + expect(response.sources.at(-1)?.url).toBe("https://example.com/19"); + } finally { + authStorage.close(); + } + }); +});