diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index abeb200e5..e0b1b1608 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -10,6 +10,10 @@ - Updated status event log to prioritize the most recent entries in the display window +### Removed + +- Removed the unreliable Bing and Yahoo HTML-scraping web search providers + ## [16.4.8] - 2026-07-12 ### Added @@ -17,6 +21,7 @@ ### Changed +- Replaced the Alt+P / `/switch` temporary model selector's fullscreen /models hub with a compact full-width floating overlay anchored above the editor (~40% of the terminal height): just the searchable model list — no provider sidebar or role management — with the session's active model highlighted and preselected - Improved tab recovery after timeouts by automatically clearing pending navigation and JS dialogs - Made `tab.goto` navigation failures catchable with a named error instead of triggering a whole-cell timeout - Made `tab.evaluate` run in the page's main JavaScript world so page-defined globals are available without a directive @@ -26,6 +31,7 @@ ### Fixed +- Fixed the eval tool's status-event tree truncating from the bottom: the newest `log()` progress lines were hidden behind an `… N more` marker while the oldest stayed visible; the tree now shows a tail window behind an `… N earlier` marker, and the expanded view widens to the viewport instead of a fixed 10 events - Fixed the `//!world=main` directive being silently ignored for string expressions passed to raw Puppeteer evaluation APIs - Fixed tab reuse issues where hung navigation or unhandled modals would cause initialization to stall and trigger a force-kill - Improved search reliability for Perplexity provider by forcing retrieval for all queries diff --git a/packages/coding-agent/src/web/search/provider.ts b/packages/coding-agent/src/web/search/provider.ts index 4c2ee6f28..3a4a33c2a 100644 --- a/packages/coding-agent/src/web/search/provider.ts +++ b/packages/coding-agent/src/web/search/provider.ts @@ -119,16 +119,6 @@ const PROVIDER_META: Record = { label: SEARCH_PROVIDER_LABELS.google, load: async () => new (await import("./providers/google")).GoogleProvider(), }, - bing: { - id: "bing", - label: SEARCH_PROVIDER_LABELS.bing, - load: async () => new (await import("./providers/bing")).BingProvider(), - }, - yahoo: { - id: "yahoo", - label: SEARCH_PROVIDER_LABELS.yahoo, - load: async () => new (await import("./providers/yahoo")).YahooProvider(), - }, ecosia: { id: "ecosia", label: SEARCH_PROVIDER_LABELS.ecosia, diff --git a/packages/coding-agent/src/web/search/providers/bing.ts b/packages/coding-agent/src/web/search/providers/bing.ts deleted file mode 100644 index aaad2d427..000000000 --- a/packages/coding-agent/src/web/search/providers/bing.ts +++ /dev/null @@ -1,197 +0,0 @@ -import type { AuthStorage } from "@oh-my-pi/pi-ai"; -import { parseHTML } from "linkedom"; -import type { SearchResponse, SearchSource } from "../../../web/search/types"; -import { SearchProviderError } from "../../../web/search/types"; -import { clampNumResults } from "../utils"; -import type { SearchParams } from "./base"; -import { SearchProvider } from "./base"; -import { browserFetch } from "./browser-page"; -import { classifyProviderHttpError, withHardTimeout } from "./utils"; - -/** - * Bing's HTML search frontend. A plain GET with browser navigation headers - * returns a fully server-rendered results page — no JavaScript challenge on - * the organic path — so we parse it directly without a real browser. - */ -const BING_HOME_URL = "https://www.bing.com/"; -const BING_SEARCH_URL = "https://www.bing.com/search"; -const DEFAULT_NUM_RESULTS = 10; -const MAX_NUM_RESULTS = 20; -const MS_PER_DAY = 86_400_000; - -/** - * Recency → Bing `filters=ex1:"…"` freshness codes, as emitted by Bing's own - * "Any time" dropdown. `year` has no fixed code; the dropdown emits a custom - * epoch-day range (`ez5__`, days since 1970-01-01) which - * {@link recencyToFilters} computes. Bing parses the parameter (the SERP - * filter UI reflects it) but enforcement is server-side and vantage-dependent. - */ -const RECENCY_TO_BING_EZ: Record, "year">, string> = { - day: "ez1", - week: "ez2", - month: "ez3", -}; - -/** Snippet containers observed on Bing result blocks, in preference order. */ -const BING_SNIPPET_SELECTORS: readonly string[] = [".b_caption p", "p[class*='b_lineclamp']", ".b_algoSlug"]; - -interface ParsedResult { - title: string; - url: string; - snippet?: string; -} - -/** Build the `filters` value for a recency window, mirroring Bing's dropdown URLs. */ -function recencyToFilters(recency: NonNullable): string { - if (recency === "year") { - const epochDay = Math.floor(Date.now() / MS_PER_DAY); - return `ex1:"ez5_${epochDay - 365}_${epochDay}"`; - } - return `ex1:"${RECENCY_TO_BING_EZ[recency]}"`; -} - -/** - * Resolve a Bing result href to the underlying target URL. - * - * Organic hrefs are usually wrapped as `https://www.bing.com/ck/a?…&u=a1` - * where the payload after the literal `a1` prefix is the unpadded base64url - * encoding of the target URL. Direct external hrefs also occur; Bing-internal - * links (vertical tabs, ads plumbing) and non-http(s) schemes are rejected. - */ -function unwrapResultUrl(href: string): string | undefined { - let url: URL; - try { - url = new URL(href, BING_HOME_URL); - } catch { - return undefined; - } - - if (url.hostname === "bing.com" || url.hostname.endsWith(".bing.com")) { - if (url.pathname !== "/ck/a") return undefined; - const wrapped = url.searchParams.get("u"); - if (!wrapped?.startsWith("a1")) return undefined; - try { - url = new URL(Buffer.from(wrapped.slice(2), "base64url").toString("utf-8")); - } catch { - return undefined; - } - } - - if (url.protocol !== "http:" && url.protocol !== "https:") return undefined; - return url.href; -} - -function findSnippet(item: Element): string | undefined { - for (const selector of BING_SNIPPET_SELECTORS) { - const text = (item.querySelector(selector)?.textContent ?? "").replace(/\s+/g, " ").trim(); - if (text) return text; - } - return undefined; -} - -/** - * Pull organic result blocks out of the page in document order. - * - * Each organic hit is an `
  • ` with the title link in - * `h2 > a[href]` (sitelink/attribution anchors live outside the `h2`) and the - * preview text in one of {@link BING_SNIPPET_SELECTORS}. Ads, answer cards, - * and the "no results" row use other classes and fall out naturally. - */ -function parseHtmlResults(html: string): ParsedResult[] { - const { document } = parseHTML(html); - const results: ParsedResult[] = []; - for (const item of document.querySelectorAll("li.b_algo")) { - const anchor = item.querySelector("h2 a[href]"); - const href = anchor?.getAttribute("href"); - if (!href) continue; - const url = unwrapResultUrl(href); - if (!url) continue; - const title = (anchor?.textContent ?? "").replace(/\s+/g, " ").trim(); - if (!title) continue; - results.push({ title, url, snippet: findSnippet(item) }); - } - return results; -} - -/** - * `true` when Bing answered with its CAPTCHA/consent interstitial instead of - * a results page. The challenge redirects to `/turing/captcha/…`; body - * markers are only trusted when no organic result block is present so a - * search *about* CAPTCHAs never trips the detector. - */ -function isChallengeResponse(html: string, finalUrl: string): boolean { - if (finalUrl.includes("/turing/captcha")) return true; - if (html.includes('class="b_algo"')) return false; - return /turing\/captcha|b_captcha|px-captcha|verify (?:that )?you are (?:a )?human/i.test(html); -} - -function buildSearchUrl(params: SearchParams, numResults: number): string { - const url = new URL(BING_SEARCH_URL); - url.searchParams.set("q", params.query); - url.searchParams.set("count", String(numResults)); - url.searchParams.set("mkt", "en-US"); - url.searchParams.set("setlang", "en"); - if (params.recency) url.searchParams.set("filters", recencyToFilters(params.recency)); - return url.href; -} - -async function callBingHtml(params: SearchParams, numResults: number): Promise { - const url = buildSearchUrl(params, numResults); - const page = await browserFetch(url, { - fetch: params.fetch ?? fetch, - signal: withHardTimeout(params.signal), - referer: BING_HOME_URL, - }); - - const body = page.html; - if (isChallengeResponse(body, page.url)) { - throw new SearchProviderError( - "bing", - "Bing blocked the request with a CAPTCHA challenge. Bing throttles automated searches from datacenter/shared-egress IPs; try the duckduckgo or mojeek provider, or configure a credentialed provider such as Brave, Tavily, Exa, or Kagi.", - 429, - ); - } - if (page.status < 200 || page.status >= 300) { - const classified = classifyProviderHttpError("bing", page.status, body); - if (classified) throw classified; - throw new SearchProviderError("bing", `Bing HTML error (${page.status})`, page.status); - } - - return body; -} - -/** Execute a Bing web search via the server-rendered HTML results page. */ -export async function searchBing(params: SearchParams): Promise { - const numResults = clampNumResults(params.numSearchResults ?? params.limit, DEFAULT_NUM_RESULTS, MAX_NUM_RESULTS); - const html = await callBingHtml(params, numResults); - const parsed = parseHtmlResults(html); - - const sources: SearchSource[] = []; - const seen = new Set(); - for (const result of parsed) { - if (seen.has(result.url)) continue; - seen.add(result.url); - sources.push({ title: result.title, url: result.url, snippet: result.snippet }); - if (sources.length >= numResults) break; - } - - return { provider: "bing", sources }; -} - -/** Search provider for Bing (no API key required). */ -export class BingProvider extends SearchProvider { - readonly id = "bing"; - readonly label = "Bing"; - - isAvailable(_authStorage: AuthStorage): boolean { - return true; - } - - isExplicitlyAvailable(_authStorage: AuthStorage): boolean { - return true; - } - - search(params: SearchParams): Promise { - return searchBing(params); - } -} diff --git a/packages/coding-agent/src/web/search/providers/public.ts b/packages/coding-agent/src/web/search/providers/public.ts index 4e0fb7ce8..34c5a5617 100644 --- a/packages/coding-agent/src/web/search/providers/public.ts +++ b/packages/coding-agent/src/web/search/providers/public.ts @@ -11,15 +11,13 @@ import { withHardTimeout } from "./utils"; * Credential-free engines the Public Web aggregate fans out to. Order is the * tiebreak for merged ranking (earlier engines win equal consensus/rank), so * engines with the best ranking quality when they answer come first: - * Google-index engines (startpage, google) lead, Bing-backed scrapers follow, - * and Mojeek's independent index breaks remaining ties (measured 2026-07). + * Google-index engines (startpage, google) lead, and Mojeek's independent + * index breaks remaining ties (measured 2026-07). */ const PUBLIC_ENGINE_IDS = [ "startpage", "google", "duckduckgo", - "bing", - "yahoo", "ecosia", "mojeek", ] as const satisfies readonly SearchProviderId[]; diff --git a/packages/coding-agent/src/web/search/providers/yahoo.ts b/packages/coding-agent/src/web/search/providers/yahoo.ts deleted file mode 100644 index bdc1e312f..000000000 --- a/packages/coding-agent/src/web/search/providers/yahoo.ts +++ /dev/null @@ -1,179 +0,0 @@ -import type { AuthStorage } from "@oh-my-pi/pi-ai"; -import { parseHTML } from "linkedom"; -import type { SearchResponse, SearchSource } from "../../../web/search/types"; -import { SearchProviderError } from "../../../web/search/types"; -import { clampNumResults } from "../utils"; -import type { SearchParams } from "./base"; -import { SearchProvider } from "./base"; -import { browserFetch } from "./browser-page"; -import { classifyProviderHttpError, withHardTimeout } from "./utils"; - -/** - * Yahoo Search's server-rendered results page. A plain GET with browser - * navigation headers returns the full SERP without any JavaScript challenge, - * so no headless-browser fallback is needed (verified live 2026-07). - */ -const YAHOO_HOME_URL = "https://search.yahoo.com/"; -const YAHOO_SEARCH_URL = "https://search.yahoo.com/search"; -const DEFAULT_NUM_RESULTS = 10; -const MAX_NUM_RESULTS = 20; - -/** - * Recency → Yahoo `btf` query param. Yahoo's time filter only offers - * day/week/month; `year` has no equivalent and is silently dropped per the - * {@link SearchParams.recency} contract. - */ -const RECENCY_TO_YAHOO_BTF: Partial, string>> = { - day: "d", - week: "w", - month: "m", -}; - -interface ParsedResult { - title: string; - url: string; - snippet?: string; -} - -/** - * Resolve a Yahoo result href back to the underlying target URL. - * - * Organic hrefs are wrapped through the click tracker - * `https://r.search.yahoo.com/_ylt=…/RU=/RK=…/RS=…`; - * the `/RU=` path segment carries the destination. Older layouts emit plain - * absolute hrefs, so both shapes are handled. Tracker links without a - * recoverable target and Yahoo-internal navigation are rejected. - */ -function unwrapResultUrl(href: string): string | undefined { - let url: URL; - try { - url = new URL(href, YAHOO_HOME_URL); - } catch { - return undefined; - } - if (url.protocol !== "http:" && url.protocol !== "https:") return undefined; - - const wrapped = /\/RU=([^/]+)/.exec(url.pathname); - if (wrapped) { - let target: string; - try { - target = decodeURIComponent(wrapped[1]); - } catch { - return undefined; - } - return target.startsWith("http://") || target.startsWith("https://") ? target : undefined; - } - // A tracker link without an RU segment has no recoverable destination. - if (url.hostname === "r.search.yahoo.com") return undefined; - // Relative hrefs resolve against the search host: internal navigation. - if (url.hostname === "search.yahoo.com") return undefined; - return url.href; -} - -/** - * Walk the SERP and pull organic result blocks in document order. - * - * Organics render as `
    ` blocks (inside `#web`'s - * `
      `): the title `

      ` sits inside the tracker `` in the current - * layout, while legacy layouts nested the `` inside `

      ` - * — both are handled. The preview text lives in a sibling - * `
      `. Module headers ("Videos", "People also ask") - * carry `

      `s outside `.algo` blocks and are excluded by construction. - */ -function parseHtmlResults(html: string): ParsedResult[] { - const { document } = parseHTML(html); - const results: ParsedResult[] = []; - for (const block of document.querySelectorAll("div.algo")) { - const heading = block.querySelector("h3"); - if (!heading) continue; - const anchor = heading.querySelector("a") ?? heading.closest("a"); - const href = anchor?.getAttribute("href"); - if (!href) continue; - const url = unwrapResultUrl(href); - if (!url) continue; - const title = (heading.textContent ?? "").replace(/\s+/g, " ").trim(); - if (!title) continue; - const snippet = (block.querySelector(".compText")?.textContent ?? "").replace(/\s+/g, " ").trim() || undefined; - results.push({ title, url, snippet }); - } - return results; -} - -/** - * `true` when Yahoo answered with its EU consent interstitial instead of - * results: either the request was redirected to consent.yahoo.com / - * guce.yahoo.com, or the body carries the consent form. The normal SERP - * mentions guce.yahoo.com only in a meta tag, so detection keys on the - * consent-host redirect and the `collectConsent` form action. - */ -function isConsentInterstitial(finalUrl: string, html: string): boolean { - if (/^https?:\/\/(?:[^/]*\.)?(?:consent|guce)\.yahoo\.com\//i.test(finalUrl)) return true; - return html.includes("consent.yahoo.com") || html.includes("collectConsent"); -} - -async function callYahooHtml(params: SearchParams, numResults: number): Promise { - const url = new URL(YAHOO_SEARCH_URL); - url.searchParams.set("p", params.query); - url.searchParams.set("n", String(numResults)); - const btf = params.recency ? RECENCY_TO_YAHOO_BTF[params.recency] : undefined; - if (btf) url.searchParams.set("btf", btf); - - const page = await browserFetch(url.href, { - fetch: params.fetch ?? fetch, - signal: withHardTimeout(params.signal), - referer: YAHOO_HOME_URL, - }); - - const body = page.html; - if (page.status < 200 || page.status >= 300) { - const classified = classifyProviderHttpError("yahoo", page.status, body); - if (classified) throw classified; - throw new SearchProviderError("yahoo", `Yahoo HTML error (${page.status})`, page.status); - } - - if (isConsentInterstitial(page.url, body)) { - throw new SearchProviderError( - "yahoo", - "Yahoo served its GDPR consent interstitial instead of search results. This typically affects EU egress IPs; use another web search provider such as DuckDuckGo, Brave, or Mojeek.", - 429, - ); - } - - return body; -} - -/** Execute a Yahoo web search via the server-rendered HTML results page. */ -export async function searchYahoo(params: SearchParams): Promise { - const numResults = clampNumResults(params.numSearchResults ?? params.limit, DEFAULT_NUM_RESULTS, MAX_NUM_RESULTS); - const html = await callYahooHtml(params, numResults); - const parsed = parseHtmlResults(html); - - const sources: SearchSource[] = []; - const seen = new Set(); - for (const result of parsed) { - if (seen.has(result.url)) continue; - seen.add(result.url); - sources.push({ title: result.title, url: result.url, snippet: result.snippet }); - if (sources.length >= numResults) break; - } - - return { provider: "yahoo", sources }; -} - -/** Search provider for Yahoo (no API key required). */ -export class YahooProvider extends SearchProvider { - readonly id = "yahoo"; - readonly label = "Yahoo"; - - isAvailable(_authStorage: AuthStorage): boolean { - return true; - } - - isExplicitlyAvailable(_authStorage: AuthStorage): boolean { - return true; - } - - search(params: SearchParams): Promise { - return searchYahoo(params); - } -} diff --git a/packages/coding-agent/src/web/search/types.ts b/packages/coding-agent/src/web/search/types.ts index 477525ea1..86ac098fa 100644 --- a/packages/coding-agent/src/web/search/types.ts +++ b/packages/coding-agent/src/web/search/types.ts @@ -53,16 +53,6 @@ export const SEARCH_PROVIDER_OPTIONS = [ label: "DuckDuckGo", description: "Credential-free best-effort fallback; may be bot-challenged on datacenter/shared-egress IPs", }, - { - value: "bing", - label: "Bing", - description: "Credential-free HTML scrape of Bing results; may be bot-challenged", - }, - { - value: "yahoo", - label: "Yahoo", - description: "Credential-free HTML scrape of Yahoo (Bing-backed) results", - }, { value: "ecosia", label: "Ecosia", diff --git a/packages/coding-agent/test/tools/web-search-public.test.ts b/packages/coding-agent/test/tools/web-search-public.test.ts index a3c58b53f..681f5318a 100644 --- a/packages/coding-agent/test/tools/web-search-public.test.ts +++ b/packages/coding-agent/test/tools/web-search-public.test.ts @@ -18,7 +18,7 @@ const fakeAuthStorage = { } as unknown as AuthStorage; /** Restrict the fan-out to the two engines these tests provide fixtures for. */ -const NON_TEST_ENGINES: readonly SearchProviderId[] = ["bing", "yahoo", "ecosia", "startpage", "mojeek"]; +const NON_TEST_ENGINES: readonly SearchProviderId[] = ["ecosia", "startpage", "mojeek"]; function makeParams(query: string, fetch: FetchImpl): SearchParams { return {