From 9cd42dd8f9634eb72d60ba6f9df0ff97b0553c24 Mon Sep 17 00:00:00 2001 From: roboomp Date: Thu, 30 Jul 2026 21:47:03 +0000 Subject: [PATCH 1/3] fix(web-search): honored lang: directive in duckduckgo kl param MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit callDuckDuckGoHtml hardcoded kl=us-en and never read parsedQuery.lang, so the shared lang: directive was silently discarded — unlike the Perplexity and SearXNG providers, which map it. Distinct locales collapsed to the same request. Added localeToKl mapping the parsed language-region locale onto DDG's region-language kl code (swapping components, gb->uk alias), falling back to us-en for language-only, malformed, or absent locales. Fixes #7110 --- .../src/web/search/providers/duckduckgo.ts | 25 +++++- .../test/web/search/duckduckgo.test.ts | 78 +++++++++++++++++++ 2 files changed, 102 insertions(+), 1 deletion(-) create mode 100644 packages/coding-agent/test/web/search/duckduckgo.test.ts diff --git a/packages/coding-agent/src/web/search/providers/duckduckgo.ts b/packages/coding-agent/src/web/search/providers/duckduckgo.ts index f8343d7fb..c4a3d3b59 100644 --- a/packages/coding-agent/src/web/search/providers/duckduckgo.ts +++ b/packages/coding-agent/src/web/search/providers/duckduckgo.ts @@ -137,10 +137,33 @@ const DDG_QUERY_SYNTAX: QuerySyntax = { filetype: true, }; +/** + * DuckDuckGo names the United Kingdom region `uk` where ISO 3166 uses `gb`. + * Every other locale reuses its country code verbatim. + */ +const DDG_REGION_ALIASES: Record = { gb: "uk" }; + +/** + * Map a parsed `lang:` locale onto DuckDuckGo's `kl` region parameter. + * + * DDG orders `kl` as `region-language` (`us-en`, `de-de`) — the reverse of the + * shared `language-region` locale convention parsed into `StructuredQuery.lang` + * (`en-us`, `de-de`). Region-qualified locales are swapped into DDG order and + * run through {@link DDG_REGION_ALIASES}; language-only or malformed values + * return undefined so the caller keeps its default region. + */ +export function localeToKl(lang: string | undefined): string | undefined { + if (!lang) return undefined; + const match = /^([a-z]{2})[-_]([a-z]{2})$/.exec(lang.toLowerCase()); + if (!match) return undefined; + const [, language, country] = match; + return `${DDG_REGION_ALIASES[country] ?? country}-${language}`; +} + async function callDuckDuckGoHtml(params: SearchParams): Promise { const form = new URLSearchParams({ q: formatScraperQuery(params.query, params.parsedQuery, DDG_QUERY_SYNTAX), - kl: "us-en", + kl: localeToKl(params.parsedQuery?.lang) ?? "us-en", }); const df = params.recency ? RECENCY_TO_DDG_DF[params.recency] : undefined; if (df) form.set("df", df); diff --git a/packages/coding-agent/test/web/search/duckduckgo.test.ts b/packages/coding-agent/test/web/search/duckduckgo.test.ts new file mode 100644 index 000000000..0999aca4e --- /dev/null +++ b/packages/coding-agent/test/web/search/duckduckgo.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, it } from "bun:test"; +import type { AuthStorage, FetchImpl } from "@oh-my-pi/pi-ai"; +import { localeToKl, searchDuckDuckGo } from "@oh-my-pi/pi-coding-agent/web/search/providers/duckduckgo"; +import { parseSearchQuery } from "@oh-my-pi/pi-coding-agent/web/search/query"; + +describe("localeToKl", () => { + it("maps region-qualified locales into DDG region-language order", () => { + expect(localeToKl("de-de")).toBe("de-de"); + expect(localeToKl("fr-fr")).toBe("fr-fr"); + // asymmetric locales swap: our language-region -> DDG region-language + expect(localeToKl("en-us")).toBe("us-en"); + expect(localeToKl("pt-br")).toBe("br-pt"); + expect(localeToKl("zh-cn")).toBe("cn-zh"); + }); + + it("applies the gb->uk region alias DDG uses for the United Kingdom", () => { + expect(localeToKl("en-gb")).toBe("uk-en"); + }); + + it("normalizes case and underscore separators", () => { + expect(localeToKl("EN_US")).toBe("us-en"); + expect(localeToKl("De-DE")).toBe("de-de"); + }); + + it("returns undefined for language-only, empty, or malformed values", () => { + expect(localeToKl(undefined)).toBeUndefined(); + expect(localeToKl("de")).toBeUndefined(); + expect(localeToKl("en")).toBeUndefined(); + expect(localeToKl("english")).toBeUndefined(); + expect(localeToKl("")).toBeUndefined(); + }); +}); + +describe("searchDuckDuckGo kl parameter (integration)", () => { + const fakeAuthStorage = {} as unknown as AuthStorage; + + async function effectiveForm(query: string): Promise { + let body: string | undefined; + const fetchMock: FetchImpl = async (_input, init) => { + body = init?.body as string; + return new Response("", { + status: 200, + headers: { "Content-Type": "text/html" }, + }); + }; + await searchDuckDuckGo({ + query, + parsedQuery: parseSearchQuery(query), + systemPrompt: "", + authStorage: fakeAuthStorage, + fetch: fetchMock, + }); + return new URLSearchParams(body ?? ""); + } + + it("honors the lang: directive so distinct locales produce distinct requests", async () => { + const de = await effectiveForm("weather lang:de-de"); + const fr = await effectiveForm("weather lang:fr-fr"); + expect(de.get("q")).toBe("weather"); + expect(de.get("kl")).toBe("de-de"); + expect(fr.get("kl")).toBe("fr-fr"); + }); + + it("swaps language-region into DDG region-language order", async () => { + const form = await effectiveForm("news lang:en-us"); + expect(form.get("kl")).toBe("us-en"); + }); + + it("falls back to us-en when no lang: directive is supplied", async () => { + const form = await effectiveForm("weather"); + expect(form.get("kl")).toBe("us-en"); + }); + + it("falls back to us-en for language-only locales", async () => { + const form = await effectiveForm("weather lang:de"); + expect(form.get("kl")).toBe("us-en"); + }); +}); From 8cf4e30a61b265a333f81d3caa3977cb78c76467 Mon Sep 17 00:00:00 2001 From: roboomp Date: Thu, 30 Jul 2026 21:53:44 +0000 Subject: [PATCH 2/3] fix(web-search): validated duckduckgo locale codes Replaced unrestricted locale component swapping with DuckDuckGo's documented kl allowlist and explicit aliases for provider-specific codes such as jp-jp, kr-kr, tw-tzh, and uk-en. Unsupported locale combinations now fall back to the existing us-en default instead of sending invalid kl values. Expanded regression coverage for Japanese, Korean, Traditional Chinese, and unsupported region-language combinations. Fixes #7110 --- .../src/web/search/providers/duckduckgo.ts | 109 ++++++++++++++++-- .../test/web/search/duckduckgo.test.ts | 23 ++-- 2 files changed, 113 insertions(+), 19 deletions(-) diff --git a/packages/coding-agent/src/web/search/providers/duckduckgo.ts b/packages/coding-agent/src/web/search/providers/duckduckgo.ts index c4a3d3b59..aeb411cbe 100644 --- a/packages/coding-agent/src/web/search/providers/duckduckgo.ts +++ b/packages/coding-agent/src/web/search/providers/duckduckgo.ts @@ -138,26 +138,111 @@ const DDG_QUERY_SYNTAX: QuerySyntax = { }; /** - * DuckDuckGo names the United Kingdom region `uk` where ISO 3166 uses `gb`. - * Every other locale reuses its country code verbatim. + * DuckDuckGo's documented `kl` values. + * + * The codes resemble `region-language` locales but contain provider-specific + * identifiers (`jp-jp`, `tw-tzh`, `uk-en`) that cannot be derived mechanically. */ -const DDG_REGION_ALIASES: Record = { gb: "uk" }; +const DDG_KL_CODES = new Set([ + "xa-ar", + "xa-en", + "ar-es", + "au-en", + "at-de", + "be-fr", + "be-nl", + "br-pt", + "bg-bg", + "ca-en", + "ca-fr", + "ct-ca", + "cl-es", + "cn-zh", + "co-es", + "hr-hr", + "cz-cs", + "dk-da", + "ee-et", + "fi-fi", + "fr-fr", + "de-de", + "gr-el", + "hk-tzh", + "hu-hu", + "in-en", + "id-id", + "id-en", + "ie-en", + "il-he", + "it-it", + "jp-jp", + "kr-kr", + "lv-lv", + "lt-lt", + "xl-es", + "my-ms", + "my-en", + "mx-es", + "nl-nl", + "nz-en", + "no-no", + "pe-es", + "ph-en", + "ph-tl", + "pl-pl", + "pt-pt", + "ro-ro", + "ru-ru", + "sg-en", + "sk-sk", + "sl-sl", + "za-en", + "es-es", + "se-sv", + "ch-de", + "ch-fr", + "ch-it", + "tw-tzh", + "th-th", + "tr-tr", + "ua-uk", + "uk-en", + "us-en", + "ue-es", + "ve-es", + "vn-vi", + "wt-wt", +]); + +/** BCP 47 locales whose DDG code does not follow a simple component swap. */ +const DDG_LOCALE_ALIASES: Record = { + "ca-es": "ct-ca", + "en-gb": "uk-en", + "es-419": "xl-es", + "es-us": "ue-es", + "ja-jp": "jp-jp", + "ko-kr": "kr-kr", + "zh-hk": "hk-tzh", + "zh-tw": "tw-tzh", +}; /** - * Map a parsed `lang:` locale onto DuckDuckGo's `kl` region parameter. + * Map a parsed `lang:` locale onto DuckDuckGo's documented `kl` values. * - * DDG orders `kl` as `region-language` (`us-en`, `de-de`) — the reverse of the - * shared `language-region` locale convention parsed into `StructuredQuery.lang` - * (`en-us`, `de-de`). Region-qualified locales are swapped into DDG order and - * run through {@link DDG_REGION_ALIASES}; language-only or malformed values - * return undefined so the caller keeps its default region. + * Shared queries use `language-region` order while DDG generally uses + * `region-language`. Provider-specific exceptions resolve through + * {@link DDG_LOCALE_ALIASES}; all other values must survive the documented + * allowlist after swapping or the caller keeps its default region. */ export function localeToKl(lang: string | undefined): string | undefined { if (!lang) return undefined; - const match = /^([a-z]{2})[-_]([a-z]{2})$/.exec(lang.toLowerCase()); + const locale = lang.toLowerCase().replaceAll("_", "-"); + const alias = DDG_LOCALE_ALIASES[locale]; + if (alias) return alias; + const match = /^([a-z]{2})-([a-z]{2})$/.exec(locale); if (!match) return undefined; - const [, language, country] = match; - return `${DDG_REGION_ALIASES[country] ?? country}-${language}`; + const candidate = `${match[2]}-${match[1]}`; + return DDG_KL_CODES.has(candidate) ? candidate : undefined; } async function callDuckDuckGoHtml(params: SearchParams): Promise { diff --git a/packages/coding-agent/test/web/search/duckduckgo.test.ts b/packages/coding-agent/test/web/search/duckduckgo.test.ts index 0999aca4e..071c46632 100644 --- a/packages/coding-agent/test/web/search/duckduckgo.test.ts +++ b/packages/coding-agent/test/web/search/duckduckgo.test.ts @@ -4,17 +4,20 @@ import { localeToKl, searchDuckDuckGo } from "@oh-my-pi/pi-coding-agent/web/sear import { parseSearchQuery } from "@oh-my-pi/pi-coding-agent/web/search/query"; describe("localeToKl", () => { - it("maps region-qualified locales into DDG region-language order", () => { + it("maps standard region-qualified locales to documented DDG codes", () => { expect(localeToKl("de-de")).toBe("de-de"); expect(localeToKl("fr-fr")).toBe("fr-fr"); - // asymmetric locales swap: our language-region -> DDG region-language expect(localeToKl("en-us")).toBe("us-en"); expect(localeToKl("pt-br")).toBe("br-pt"); expect(localeToKl("zh-cn")).toBe("cn-zh"); }); - it("applies the gb->uk region alias DDG uses for the United Kingdom", () => { + it("maps locales with provider-specific DDG codes", () => { expect(localeToKl("en-gb")).toBe("uk-en"); + expect(localeToKl("ja-jp")).toBe("jp-jp"); + expect(localeToKl("ko-kr")).toBe("kr-kr"); + expect(localeToKl("zh-hk")).toBe("hk-tzh"); + expect(localeToKl("zh-tw")).toBe("tw-tzh"); }); it("normalizes case and underscore separators", () => { @@ -22,10 +25,12 @@ describe("localeToKl", () => { expect(localeToKl("De-DE")).toBe("de-de"); }); - it("returns undefined for language-only, empty, or malformed values", () => { + it("returns undefined for unsupported, language-only, empty, or malformed values", () => { expect(localeToKl(undefined)).toBeUndefined(); expect(localeToKl("de")).toBeUndefined(); expect(localeToKl("en")).toBeUndefined(); + expect(localeToKl("en-jp")).toBeUndefined(); + expect(localeToKl("zz-zz")).toBeUndefined(); expect(localeToKl("english")).toBeUndefined(); expect(localeToKl("")).toBeUndefined(); }); @@ -61,9 +66,13 @@ describe("searchDuckDuckGo kl parameter (integration)", () => { expect(fr.get("kl")).toBe("fr-fr"); }); - it("swaps language-region into DDG region-language order", async () => { - const form = await effectiveForm("news lang:en-us"); - expect(form.get("kl")).toBe("us-en"); + it("uses DDG's provider-specific locale codes", async () => { + const ja = await effectiveForm("news lang:ja-jp"); + const ko = await effectiveForm("news lang:ko-kr"); + const tw = await effectiveForm("news lang:zh-tw"); + expect(ja.get("kl")).toBe("jp-jp"); + expect(ko.get("kl")).toBe("kr-kr"); + expect(tw.get("kl")).toBe("tw-tzh"); }); it("falls back to us-en when no lang: directive is supplied", async () => { From 942f856b296a8a35e886ddc2ba2ecbe330604d96 Mon Sep 17 00:00:00 2001 From: roboomp Date: Thu, 30 Jul 2026 21:55:47 +0000 Subject: [PATCH 3/3] fix(web-search): derived duckduckgo lang from raw query callDuckDuckGoHtml now parses params.query once when parsedQuery is absent and uses that structured view for both q and kl. Direct searchDuckDuckGo/DuckDuckGoProvider.search calls no longer strip lang: from q while defaulting kl to us-en. Fixes #7110 --- .../src/web/search/providers/duckduckgo.ts | 7 ++++--- .../coding-agent/test/web/search/duckduckgo.test.ts | 11 +++++++++-- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/packages/coding-agent/src/web/search/providers/duckduckgo.ts b/packages/coding-agent/src/web/search/providers/duckduckgo.ts index aeb411cbe..72756fe68 100644 --- a/packages/coding-agent/src/web/search/providers/duckduckgo.ts +++ b/packages/coding-agent/src/web/search/providers/duckduckgo.ts @@ -2,7 +2,7 @@ import type { AuthStorage } from "@oh-my-pi/pi-ai"; import type { SearchResponse, SearchSource } from "../../../web/search/types"; import { SearchProviderError } from "../../../web/search/types"; import type { QuerySyntax } from "../query"; -import { formatScraperQuery } from "../query"; +import { formatScraperQuery, parseSearchQuery } from "../query"; import { clampNumResults } from "../utils"; import type { SearchParams } from "./base"; import { SearchProvider } from "./base"; @@ -246,9 +246,10 @@ export function localeToKl(lang: string | undefined): string | undefined { } async function callDuckDuckGoHtml(params: SearchParams): Promise { + const parsed = params.parsedQuery ?? parseSearchQuery(params.query); const form = new URLSearchParams({ - q: formatScraperQuery(params.query, params.parsedQuery, DDG_QUERY_SYNTAX), - kl: localeToKl(params.parsedQuery?.lang) ?? "us-en", + q: formatScraperQuery(params.query, parsed, DDG_QUERY_SYNTAX), + kl: localeToKl(parsed.lang) ?? "us-en", }); const df = params.recency ? RECENCY_TO_DDG_DF[params.recency] : undefined; if (df) form.set("df", df); diff --git a/packages/coding-agent/test/web/search/duckduckgo.test.ts b/packages/coding-agent/test/web/search/duckduckgo.test.ts index 071c46632..6ab1ad4ec 100644 --- a/packages/coding-agent/test/web/search/duckduckgo.test.ts +++ b/packages/coding-agent/test/web/search/duckduckgo.test.ts @@ -39,7 +39,8 @@ describe("localeToKl", () => { describe("searchDuckDuckGo kl parameter (integration)", () => { const fakeAuthStorage = {} as unknown as AuthStorage; - async function effectiveForm(query: string): Promise { + async function effectiveForm(query: string, opts: { withParsedQuery?: boolean } = {}): Promise { + const withParsedQuery = opts.withParsedQuery ?? true; let body: string | undefined; const fetchMock: FetchImpl = async (_input, init) => { body = init?.body as string; @@ -50,7 +51,7 @@ describe("searchDuckDuckGo kl parameter (integration)", () => { }; await searchDuckDuckGo({ query, - parsedQuery: parseSearchQuery(query), + parsedQuery: withParsedQuery ? parseSearchQuery(query) : undefined, systemPrompt: "", authStorage: fakeAuthStorage, fetch: fetchMock, @@ -84,4 +85,10 @@ describe("searchDuckDuckGo kl parameter (integration)", () => { const form = await effectiveForm("weather lang:de"); expect(form.get("kl")).toBe("us-en"); }); + + it("parses lang: from the raw query when parsedQuery is omitted (direct call)", async () => { + const form = await effectiveForm("weather lang:de-de", { withParsedQuery: false }); + expect(form.get("q")).toBe("weather"); + expect(form.get("kl")).toBe("de-de"); + }); });