Merge PR #7121: fix(search): paginate DuckDuckGo HTML results (@roboomp)
This commit is contained in:
can1357
@@ -40,6 +40,9 @@
|
||||
### Fixed
|
||||
|
||||
- Removed the `read` exemption from the centralized artifact spill wrapper. Oversized read results now persist to a recoverable session artifact and return the configured inline head/tail instead of bypassing the threshold shared by other tools.
|
||||
### Fixed
|
||||
|
||||
- Fixed DuckDuckGo web search under-returning requests above the first-page result count by submitting the returned continuation form until the requested limit is reached ([#7116](https://github.com/can1357/oh-my-pi/issues/7116)).
|
||||
|
||||
## [17.2.1] - 2026-07-30
|
||||
|
||||
|
||||
@@ -110,6 +110,25 @@ function parseHtmlResults(html: string): ParsedResult[] {
|
||||
}
|
||||
return results;
|
||||
}
|
||||
/**
|
||||
* Extract the hidden fields from DDG's next-page form.
|
||||
*
|
||||
* Attribute order varies across responses, so each input tag is parsed
|
||||
* independently instead of matching one fixed HTML layout.
|
||||
*/
|
||||
function parseContinuationForm(html: string): URLSearchParams | undefined {
|
||||
for (const formMatch of html.matchAll(/<form\b[^>]*>([\s\S]*?)<\/form>/gi)) {
|
||||
const form = new URLSearchParams();
|
||||
for (const inputMatch of formMatch[1].matchAll(/<input\b[^>]*>/gi)) {
|
||||
const input = inputMatch[0];
|
||||
const name = /\bname\s*=\s*(["'])(.*?)\1/i.exec(input)?.[2];
|
||||
const value = /\bvalue\s*=\s*(["'])(.*?)\1/i.exec(input)?.[2];
|
||||
if (name && value !== undefined) form.append(decodeHtmlText(name), decodeHtmlText(value));
|
||||
}
|
||||
if (form.has("s") && form.has("vqd")) return form;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* `true` when the page DDG returned is the bot-challenge modal instead of
|
||||
@@ -137,7 +156,7 @@ const DDG_QUERY_SYNTAX: QuerySyntax = {
|
||||
filetype: true,
|
||||
};
|
||||
|
||||
async function callDuckDuckGoHtml(params: SearchParams): Promise<string> {
|
||||
function createDuckDuckGoForm(params: SearchParams): URLSearchParams {
|
||||
const form = new URLSearchParams({
|
||||
q: formatScraperQuery(params.query, params.parsedQuery, DDG_QUERY_SYNTAX),
|
||||
kl: "us-en",
|
||||
@@ -146,10 +165,13 @@ async function callDuckDuckGoHtml(params: SearchParams): Promise<string> {
|
||||
if (df) form.set("df", df);
|
||||
// Add b: "" parameter as specified in the browser fetch template to match real browser form submission
|
||||
form.set("b", "");
|
||||
return form;
|
||||
}
|
||||
|
||||
async function callDuckDuckGoHtml(params: SearchParams, form: URLSearchParams, signal: AbortSignal): Promise<string> {
|
||||
const page = await browserFetch(DUCKDUCKGO_HTML_URL, {
|
||||
fetch: params.fetch ?? fetch,
|
||||
signal: withHardTimeout(params.signal),
|
||||
signal,
|
||||
referer: "https://html.duckduckgo.com/",
|
||||
init: {
|
||||
method: "POST",
|
||||
@@ -179,16 +201,23 @@ async function callDuckDuckGoHtml(params: SearchParams): Promise<string> {
|
||||
/** Execute a DuckDuckGo web search via the no-JS HTML frontend. */
|
||||
export async function searchDuckDuckGo(params: SearchParams): Promise<SearchResponse> {
|
||||
const numResults = clampNumResults(params.numSearchResults ?? params.limit, DEFAULT_NUM_RESULTS, MAX_NUM_RESULTS);
|
||||
const html = await callDuckDuckGoHtml(params);
|
||||
const parsed = parseHtmlResults(html);
|
||||
|
||||
const signal = withHardTimeout(params.signal);
|
||||
const sources: SearchSource[] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const result of parsed) {
|
||||
if (seen.has(result.url)) continue;
|
||||
seen.add(result.url);
|
||||
sources.push({ title: result.title, url: result.url, snippet: result.snippet });
|
||||
if (sources.length >= numResults) break;
|
||||
let form: URLSearchParams | undefined = createDuckDuckGoForm(params);
|
||||
|
||||
while (form && sources.length < numResults) {
|
||||
const html = await callDuckDuckGoHtml(params, form, signal);
|
||||
const sourceCount = sources.length;
|
||||
for (const result of parseHtmlResults(html)) {
|
||||
if (seen.has(result.url)) continue;
|
||||
seen.add(result.url);
|
||||
sources.push({ title: result.title, url: result.url, snippet: result.snippet });
|
||||
if (sources.length >= numResults) break;
|
||||
}
|
||||
|
||||
if (sources.length === sourceCount) break;
|
||||
form = parseContinuationForm(html);
|
||||
}
|
||||
|
||||
return { provider: "duckduckgo", sources };
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
import { Database } from "bun:sqlite";
|
||||
import { describe, expect, it } from "bun:test";
|
||||
import { AuthStorage, type FetchImpl, SqliteAuthCredentialStore } from "@oh-my-pi/pi-ai";
|
||||
import { searchDuckDuckGo } from "@oh-my-pi/pi-coding-agent/web/search/providers/duckduckgo";
|
||||
|
||||
function duckResult(index: number): string {
|
||||
return `<div class="result results_links"><a class="result__a" href="https://example.com/${index}">Result ${index}</a><a class="result__snippet">Snippet ${index}</a></div>`;
|
||||
}
|
||||
|
||||
function duckPage(indices: readonly number[], continuation = false): string {
|
||||
const results = indices.map(duckResult).join("\n");
|
||||
if (!continuation) return results;
|
||||
return `${results}
|
||||
<div class="nav-link">
|
||||
<form action="/html/" method="post">
|
||||
<input type="submit" class="btn btn--alt" value="Next" />
|
||||
<input type="hidden" name="q" value="open source software" />
|
||||
<input value="10" type="hidden" name="s" />
|
||||
<input type="hidden" name="nextParams" value="" />
|
||||
<input type="hidden" name="v" value="l" />
|
||||
<input type="hidden" name="o" value="json" />
|
||||
<input type="hidden" name="dc" value="11" />
|
||||
<input type="hidden" name="api" value="d.js" />
|
||||
<input value="test-vqd" name="vqd" type="hidden" />
|
||||
<input name="kl" value="us-en" type="hidden" />
|
||||
</form>
|
||||
</div>`;
|
||||
}
|
||||
|
||||
describe("DuckDuckGo web search provider", () => {
|
||||
it("submits the returned continuation form to satisfy a 20-result limit", async () => {
|
||||
const authStorage = new AuthStorage(new SqliteAuthCredentialStore(new Database(":memory:")));
|
||||
const requests: URLSearchParams[] = [];
|
||||
const fetchMock: FetchImpl = async (_input, init) => {
|
||||
expect(init?.method).toBe("POST");
|
||||
const body = new URLSearchParams(String(init?.body));
|
||||
requests.push(body);
|
||||
const html =
|
||||
requests.length === 1
|
||||
? duckPage([0, 1, 2, 3, 4, 5, 6, 7, 8, 9], true)
|
||||
: duckPage([9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19]);
|
||||
return new Response(html, { status: 200 });
|
||||
};
|
||||
|
||||
try {
|
||||
const response = await searchDuckDuckGo({
|
||||
query: "open source software",
|
||||
limit: 20,
|
||||
systemPrompt: "Test DuckDuckGo search",
|
||||
authStorage,
|
||||
fetch: fetchMock,
|
||||
});
|
||||
|
||||
expect(requests).toHaveLength(2);
|
||||
expect(Object.fromEntries(requests[0])).toEqual({ q: "open source software", kl: "us-en", b: "" });
|
||||
expect(Object.fromEntries(requests[1])).toEqual({
|
||||
q: "open source software",
|
||||
s: "10",
|
||||
nextParams: "",
|
||||
v: "l",
|
||||
o: "json",
|
||||
dc: "11",
|
||||
api: "d.js",
|
||||
vqd: "test-vqd",
|
||||
kl: "us-en",
|
||||
});
|
||||
expect(response.provider).toBe("duckduckgo");
|
||||
expect(response.sources).toHaveLength(20);
|
||||
expect(response.sources[0]?.url).toBe("https://example.com/0");
|
||||
expect(response.sources.at(-1)?.url).toBe("https://example.com/19");
|
||||
} finally {
|
||||
authStorage.close();
|
||||
}
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user