Merge PR #7111: fix(web-search): honor lang: directive in DuckDuckGo kl param (@roboomp)
# Conflicts: # packages/coding-agent/src/web/search/providers/duckduckgo.ts
This commit is contained in:
can1357
@@ -2,7 +2,7 @@ import type { AuthStorage } from "@oh-my-pi/pi-ai";
|
||||
import type { SearchResponse, SearchSource } from "../../../web/search/types";
|
||||
import { SearchProviderError } from "../../../web/search/types";
|
||||
import type { QuerySyntax } from "../query";
|
||||
import { formatScraperQuery } from "../query";
|
||||
import { formatScraperQuery, parseSearchQuery } from "../query";
|
||||
import { clampNumResults, dateToAgeSeconds } from "../utils";
|
||||
import type { SearchParams } from "./base";
|
||||
import { SearchProvider } from "./base";
|
||||
@@ -181,10 +181,119 @@ const DDG_QUERY_SYNTAX: QuerySyntax = {
|
||||
filetype: true,
|
||||
};
|
||||
|
||||
/**
|
||||
* DuckDuckGo's documented `kl` values.
|
||||
*
|
||||
* The codes resemble `region-language` locales but contain provider-specific
|
||||
* identifiers (`jp-jp`, `tw-tzh`, `uk-en`) that cannot be derived mechanically.
|
||||
*/
|
||||
const DDG_KL_CODES = new Set([
|
||||
"xa-ar",
|
||||
"xa-en",
|
||||
"ar-es",
|
||||
"au-en",
|
||||
"at-de",
|
||||
"be-fr",
|
||||
"be-nl",
|
||||
"br-pt",
|
||||
"bg-bg",
|
||||
"ca-en",
|
||||
"ca-fr",
|
||||
"ct-ca",
|
||||
"cl-es",
|
||||
"cn-zh",
|
||||
"co-es",
|
||||
"hr-hr",
|
||||
"cz-cs",
|
||||
"dk-da",
|
||||
"ee-et",
|
||||
"fi-fi",
|
||||
"fr-fr",
|
||||
"de-de",
|
||||
"gr-el",
|
||||
"hk-tzh",
|
||||
"hu-hu",
|
||||
"in-en",
|
||||
"id-id",
|
||||
"id-en",
|
||||
"ie-en",
|
||||
"il-he",
|
||||
"it-it",
|
||||
"jp-jp",
|
||||
"kr-kr",
|
||||
"lv-lv",
|
||||
"lt-lt",
|
||||
"xl-es",
|
||||
"my-ms",
|
||||
"my-en",
|
||||
"mx-es",
|
||||
"nl-nl",
|
||||
"nz-en",
|
||||
"no-no",
|
||||
"pe-es",
|
||||
"ph-en",
|
||||
"ph-tl",
|
||||
"pl-pl",
|
||||
"pt-pt",
|
||||
"ro-ro",
|
||||
"ru-ru",
|
||||
"sg-en",
|
||||
"sk-sk",
|
||||
"sl-sl",
|
||||
"za-en",
|
||||
"es-es",
|
||||
"se-sv",
|
||||
"ch-de",
|
||||
"ch-fr",
|
||||
"ch-it",
|
||||
"tw-tzh",
|
||||
"th-th",
|
||||
"tr-tr",
|
||||
"ua-uk",
|
||||
"uk-en",
|
||||
"us-en",
|
||||
"ue-es",
|
||||
"ve-es",
|
||||
"vn-vi",
|
||||
"wt-wt",
|
||||
]);
|
||||
|
||||
/** BCP 47 locales whose DDG code does not follow a simple component swap. */
|
||||
const DDG_LOCALE_ALIASES: Record<string, string> = {
|
||||
"ca-es": "ct-ca",
|
||||
"en-gb": "uk-en",
|
||||
"es-419": "xl-es",
|
||||
"es-us": "ue-es",
|
||||
"ja-jp": "jp-jp",
|
||||
"ko-kr": "kr-kr",
|
||||
"zh-hk": "hk-tzh",
|
||||
"zh-tw": "tw-tzh",
|
||||
};
|
||||
|
||||
/**
|
||||
* Map a parsed `lang:` locale onto DuckDuckGo's documented `kl` values.
|
||||
*
|
||||
* Shared queries use `language-region` order while DDG generally uses
|
||||
* `region-language`. Provider-specific exceptions resolve through
|
||||
* {@link DDG_LOCALE_ALIASES}; all other values must survive the documented
|
||||
* allowlist after swapping or the caller keeps its default region.
|
||||
*/
|
||||
export function localeToKl(lang: string | undefined): string | undefined {
|
||||
if (!lang) return undefined;
|
||||
const locale = lang.toLowerCase().replaceAll("_", "-");
|
||||
const alias = DDG_LOCALE_ALIASES[locale];
|
||||
if (alias) return alias;
|
||||
const match = /^([a-z]{2})-([a-z]{2})$/.exec(locale);
|
||||
if (!match) return undefined;
|
||||
const candidate = `${match[2]}-${match[1]}`;
|
||||
return DDG_KL_CODES.has(candidate) ? candidate : undefined;
|
||||
}
|
||||
|
||||
function createDuckDuckGoForm(params: SearchParams): URLSearchParams {
|
||||
const parsed = params.parsedQuery ?? parseSearchQuery(params.query);
|
||||
const form = new URLSearchParams({
|
||||
q: formatScraperQuery(params.query, params.parsedQuery, DDG_QUERY_SYNTAX),
|
||||
kl: "us-en",
|
||||
q: formatScraperQuery(params.query, parsed, DDG_QUERY_SYNTAX),
|
||||
kl: localeToKl(parsed.lang) ?? "us-en",
|
||||
});
|
||||
const df = params.recency ? RECENCY_TO_DDG_DF[params.recency] : undefined;
|
||||
if (df) form.set("df", df);
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
import { describe, expect, it } from "bun:test";
|
||||
import type { AuthStorage, FetchImpl } from "@oh-my-pi/pi-ai";
|
||||
import { localeToKl, searchDuckDuckGo } from "@oh-my-pi/pi-coding-agent/web/search/providers/duckduckgo";
|
||||
import { parseSearchQuery } from "@oh-my-pi/pi-coding-agent/web/search/query";
|
||||
|
||||
describe("localeToKl", () => {
|
||||
it("maps standard region-qualified locales to documented DDG codes", () => {
|
||||
expect(localeToKl("de-de")).toBe("de-de");
|
||||
expect(localeToKl("fr-fr")).toBe("fr-fr");
|
||||
expect(localeToKl("en-us")).toBe("us-en");
|
||||
expect(localeToKl("pt-br")).toBe("br-pt");
|
||||
expect(localeToKl("zh-cn")).toBe("cn-zh");
|
||||
});
|
||||
|
||||
it("maps locales with provider-specific DDG codes", () => {
|
||||
expect(localeToKl("en-gb")).toBe("uk-en");
|
||||
expect(localeToKl("ja-jp")).toBe("jp-jp");
|
||||
expect(localeToKl("ko-kr")).toBe("kr-kr");
|
||||
expect(localeToKl("zh-hk")).toBe("hk-tzh");
|
||||
expect(localeToKl("zh-tw")).toBe("tw-tzh");
|
||||
});
|
||||
|
||||
it("normalizes case and underscore separators", () => {
|
||||
expect(localeToKl("EN_US")).toBe("us-en");
|
||||
expect(localeToKl("De-DE")).toBe("de-de");
|
||||
});
|
||||
|
||||
it("returns undefined for unsupported, language-only, empty, or malformed values", () => {
|
||||
expect(localeToKl(undefined)).toBeUndefined();
|
||||
expect(localeToKl("de")).toBeUndefined();
|
||||
expect(localeToKl("en")).toBeUndefined();
|
||||
expect(localeToKl("en-jp")).toBeUndefined();
|
||||
expect(localeToKl("zz-zz")).toBeUndefined();
|
||||
expect(localeToKl("english")).toBeUndefined();
|
||||
expect(localeToKl("")).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe("searchDuckDuckGo kl parameter (integration)", () => {
|
||||
const fakeAuthStorage = {} as unknown as AuthStorage;
|
||||
|
||||
async function effectiveForm(query: string, opts: { withParsedQuery?: boolean } = {}): Promise<URLSearchParams> {
|
||||
const withParsedQuery = opts.withParsedQuery ?? true;
|
||||
let body: string | undefined;
|
||||
const fetchMock: FetchImpl = async (_input, init) => {
|
||||
body = init?.body as string;
|
||||
return new Response("<html><body></body></html>", {
|
||||
status: 200,
|
||||
headers: { "Content-Type": "text/html" },
|
||||
});
|
||||
};
|
||||
await searchDuckDuckGo({
|
||||
query,
|
||||
parsedQuery: withParsedQuery ? parseSearchQuery(query) : undefined,
|
||||
systemPrompt: "",
|
||||
authStorage: fakeAuthStorage,
|
||||
fetch: fetchMock,
|
||||
});
|
||||
return new URLSearchParams(body ?? "");
|
||||
}
|
||||
|
||||
it("honors the lang: directive so distinct locales produce distinct requests", async () => {
|
||||
const de = await effectiveForm("weather lang:de-de");
|
||||
const fr = await effectiveForm("weather lang:fr-fr");
|
||||
expect(de.get("q")).toBe("weather");
|
||||
expect(de.get("kl")).toBe("de-de");
|
||||
expect(fr.get("kl")).toBe("fr-fr");
|
||||
});
|
||||
|
||||
it("uses DDG's provider-specific locale codes", async () => {
|
||||
const ja = await effectiveForm("news lang:ja-jp");
|
||||
const ko = await effectiveForm("news lang:ko-kr");
|
||||
const tw = await effectiveForm("news lang:zh-tw");
|
||||
expect(ja.get("kl")).toBe("jp-jp");
|
||||
expect(ko.get("kl")).toBe("kr-kr");
|
||||
expect(tw.get("kl")).toBe("tw-tzh");
|
||||
});
|
||||
|
||||
it("falls back to us-en when no lang: directive is supplied", async () => {
|
||||
const form = await effectiveForm("weather");
|
||||
expect(form.get("kl")).toBe("us-en");
|
||||
});
|
||||
|
||||
it("falls back to us-en for language-only locales", async () => {
|
||||
const form = await effectiveForm("weather lang:de");
|
||||
expect(form.get("kl")).toBe("us-en");
|
||||
});
|
||||
|
||||
it("parses lang: from the raw query when parsedQuery is omitted (direct call)", async () => {
|
||||
const form = await effectiveForm("weather lang:de-de", { withParsedQuery: false });
|
||||
expect(form.get("q")).toBe("weather");
|
||||
expect(form.get("kl")).toBe("de-de");
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user