refactor(coding-agent): removed unreliable web search providers
- Removed unreliable Bing and Yahoo HTML-scraping search providers. - Deleted `src/web/search/providers/bing.ts` and `src/web/search/providers/yahoo.ts` implementation files. - Updated `provider.ts`, `types.ts`, and `public.ts` to prune provider registration and configuration. - Adjusted `web-search-public.test.ts` to exclude removed engines from test coverage.
This commit is contained in:
can1357
@@ -10,6 +10,10 @@
|
||||
|
||||
- Updated status event log to prioritize the most recent entries in the display window
|
||||
|
||||
### Removed
|
||||
|
||||
- Removed the unreliable Bing and Yahoo HTML-scraping web search providers
|
||||
|
||||
## [16.4.8] - 2026-07-12
|
||||
### Added
|
||||
|
||||
@@ -17,6 +21,7 @@
|
||||
|
||||
### Changed
|
||||
|
||||
- Replaced the Alt+P / `/switch` temporary model selector's fullscreen /models hub with a compact full-width floating overlay anchored above the editor (~40% of the terminal height): just the searchable model list — no provider sidebar or role management — with the session's active model highlighted and preselected
|
||||
- Improved tab recovery after timeouts by automatically clearing pending navigation and JS dialogs
|
||||
- Made `tab.goto` navigation failures catchable with a named error instead of triggering a whole-cell timeout
|
||||
- Made `tab.evaluate` run in the page's main JavaScript world so page-defined globals are available without a directive
|
||||
@@ -26,6 +31,7 @@
|
||||
|
||||
### Fixed
|
||||
|
||||
- Fixed the eval tool's status-event tree truncating from the bottom: the newest `log()` progress lines were hidden behind an `… N more` marker while the oldest stayed visible; the tree now shows a tail window behind an `… N earlier` marker, and the expanded view widens to the viewport instead of a fixed 10 events
|
||||
- Fixed the `//!world=main` directive being silently ignored for string expressions passed to raw Puppeteer evaluation APIs
|
||||
- Fixed tab reuse issues where hung navigation or unhandled modals would cause initialization to stall and trigger a force-kill
|
||||
- Improved search reliability for Perplexity provider by forcing retrieval for all queries
|
||||
|
||||
@@ -119,16 +119,6 @@ const PROVIDER_META: Record<SearchProviderId, ProviderMeta> = {
|
||||
label: SEARCH_PROVIDER_LABELS.google,
|
||||
load: async () => new (await import("./providers/google")).GoogleProvider(),
|
||||
},
|
||||
bing: {
|
||||
id: "bing",
|
||||
label: SEARCH_PROVIDER_LABELS.bing,
|
||||
load: async () => new (await import("./providers/bing")).BingProvider(),
|
||||
},
|
||||
yahoo: {
|
||||
id: "yahoo",
|
||||
label: SEARCH_PROVIDER_LABELS.yahoo,
|
||||
load: async () => new (await import("./providers/yahoo")).YahooProvider(),
|
||||
},
|
||||
ecosia: {
|
||||
id: "ecosia",
|
||||
label: SEARCH_PROVIDER_LABELS.ecosia,
|
||||
|
||||
@@ -1,197 +0,0 @@
|
||||
import type { AuthStorage } from "@oh-my-pi/pi-ai";
|
||||
import { parseHTML } from "linkedom";
|
||||
import type { SearchResponse, SearchSource } from "../../../web/search/types";
|
||||
import { SearchProviderError } from "../../../web/search/types";
|
||||
import { clampNumResults } from "../utils";
|
||||
import type { SearchParams } from "./base";
|
||||
import { SearchProvider } from "./base";
|
||||
import { browserFetch } from "./browser-page";
|
||||
import { classifyProviderHttpError, withHardTimeout } from "./utils";
|
||||
|
||||
/**
|
||||
* Bing's HTML search frontend. A plain GET with browser navigation headers
|
||||
* returns a fully server-rendered results page — no JavaScript challenge on
|
||||
* the organic path — so we parse it directly without a real browser.
|
||||
*/
|
||||
const BING_HOME_URL = "https://www.bing.com/";
|
||||
const BING_SEARCH_URL = "https://www.bing.com/search";
|
||||
const DEFAULT_NUM_RESULTS = 10;
|
||||
const MAX_NUM_RESULTS = 20;
|
||||
const MS_PER_DAY = 86_400_000;
|
||||
|
||||
/**
|
||||
* Recency → Bing `filters=ex1:"…"` freshness codes, as emitted by Bing's own
|
||||
* "Any time" dropdown. `year` has no fixed code; the dropdown emits a custom
|
||||
* epoch-day range (`ez5_<start>_<end>`, days since 1970-01-01) which
|
||||
* {@link recencyToFilters} computes. Bing parses the parameter (the SERP
|
||||
* filter UI reflects it) but enforcement is server-side and vantage-dependent.
|
||||
*/
|
||||
const RECENCY_TO_BING_EZ: Record<Exclude<NonNullable<SearchParams["recency"]>, "year">, string> = {
|
||||
day: "ez1",
|
||||
week: "ez2",
|
||||
month: "ez3",
|
||||
};
|
||||
|
||||
/** Snippet containers observed on Bing result blocks, in preference order. */
|
||||
const BING_SNIPPET_SELECTORS: readonly string[] = [".b_caption p", "p[class*='b_lineclamp']", ".b_algoSlug"];
|
||||
|
||||
interface ParsedResult {
|
||||
title: string;
|
||||
url: string;
|
||||
snippet?: string;
|
||||
}
|
||||
|
||||
/** Build the `filters` value for a recency window, mirroring Bing's dropdown URLs. */
|
||||
function recencyToFilters(recency: NonNullable<SearchParams["recency"]>): string {
|
||||
if (recency === "year") {
|
||||
const epochDay = Math.floor(Date.now() / MS_PER_DAY);
|
||||
return `ex1:"ez5_${epochDay - 365}_${epochDay}"`;
|
||||
}
|
||||
return `ex1:"${RECENCY_TO_BING_EZ[recency]}"`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve a Bing result href to the underlying target URL.
|
||||
*
|
||||
* Organic hrefs are usually wrapped as `https://www.bing.com/ck/a?…&u=a1<payload>`
|
||||
* where the payload after the literal `a1` prefix is the unpadded base64url
|
||||
* encoding of the target URL. Direct external hrefs also occur; Bing-internal
|
||||
* links (vertical tabs, ads plumbing) and non-http(s) schemes are rejected.
|
||||
*/
|
||||
function unwrapResultUrl(href: string): string | undefined {
|
||||
let url: URL;
|
||||
try {
|
||||
url = new URL(href, BING_HOME_URL);
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
if (url.hostname === "bing.com" || url.hostname.endsWith(".bing.com")) {
|
||||
if (url.pathname !== "/ck/a") return undefined;
|
||||
const wrapped = url.searchParams.get("u");
|
||||
if (!wrapped?.startsWith("a1")) return undefined;
|
||||
try {
|
||||
url = new URL(Buffer.from(wrapped.slice(2), "base64url").toString("utf-8"));
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
if (url.protocol !== "http:" && url.protocol !== "https:") return undefined;
|
||||
return url.href;
|
||||
}
|
||||
|
||||
function findSnippet(item: Element): string | undefined {
|
||||
for (const selector of BING_SNIPPET_SELECTORS) {
|
||||
const text = (item.querySelector(selector)?.textContent ?? "").replace(/\s+/g, " ").trim();
|
||||
if (text) return text;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pull organic result blocks out of the page in document order.
|
||||
*
|
||||
* Each organic hit is an `<li class="b_algo">` with the title link in
|
||||
* `h2 > a[href]` (sitelink/attribution anchors live outside the `h2`) and the
|
||||
* preview text in one of {@link BING_SNIPPET_SELECTORS}. Ads, answer cards,
|
||||
* and the "no results" row use other classes and fall out naturally.
|
||||
*/
|
||||
function parseHtmlResults(html: string): ParsedResult[] {
|
||||
const { document } = parseHTML(html);
|
||||
const results: ParsedResult[] = [];
|
||||
for (const item of document.querySelectorAll("li.b_algo")) {
|
||||
const anchor = item.querySelector("h2 a[href]");
|
||||
const href = anchor?.getAttribute("href");
|
||||
if (!href) continue;
|
||||
const url = unwrapResultUrl(href);
|
||||
if (!url) continue;
|
||||
const title = (anchor?.textContent ?? "").replace(/\s+/g, " ").trim();
|
||||
if (!title) continue;
|
||||
results.push({ title, url, snippet: findSnippet(item) });
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* `true` when Bing answered with its CAPTCHA/consent interstitial instead of
|
||||
* a results page. The challenge redirects to `/turing/captcha/…`; body
|
||||
* markers are only trusted when no organic result block is present so a
|
||||
* search *about* CAPTCHAs never trips the detector.
|
||||
*/
|
||||
function isChallengeResponse(html: string, finalUrl: string): boolean {
|
||||
if (finalUrl.includes("/turing/captcha")) return true;
|
||||
if (html.includes('class="b_algo"')) return false;
|
||||
return /turing\/captcha|b_captcha|px-captcha|verify (?:that )?you are (?:a )?human/i.test(html);
|
||||
}
|
||||
|
||||
function buildSearchUrl(params: SearchParams, numResults: number): string {
|
||||
const url = new URL(BING_SEARCH_URL);
|
||||
url.searchParams.set("q", params.query);
|
||||
url.searchParams.set("count", String(numResults));
|
||||
url.searchParams.set("mkt", "en-US");
|
||||
url.searchParams.set("setlang", "en");
|
||||
if (params.recency) url.searchParams.set("filters", recencyToFilters(params.recency));
|
||||
return url.href;
|
||||
}
|
||||
|
||||
async function callBingHtml(params: SearchParams, numResults: number): Promise<string> {
|
||||
const url = buildSearchUrl(params, numResults);
|
||||
const page = await browserFetch(url, {
|
||||
fetch: params.fetch ?? fetch,
|
||||
signal: withHardTimeout(params.signal),
|
||||
referer: BING_HOME_URL,
|
||||
});
|
||||
|
||||
const body = page.html;
|
||||
if (isChallengeResponse(body, page.url)) {
|
||||
throw new SearchProviderError(
|
||||
"bing",
|
||||
"Bing blocked the request with a CAPTCHA challenge. Bing throttles automated searches from datacenter/shared-egress IPs; try the duckduckgo or mojeek provider, or configure a credentialed provider such as Brave, Tavily, Exa, or Kagi.",
|
||||
429,
|
||||
);
|
||||
}
|
||||
if (page.status < 200 || page.status >= 300) {
|
||||
const classified = classifyProviderHttpError("bing", page.status, body);
|
||||
if (classified) throw classified;
|
||||
throw new SearchProviderError("bing", `Bing HTML error (${page.status})`, page.status);
|
||||
}
|
||||
|
||||
return body;
|
||||
}
|
||||
|
||||
/** Execute a Bing web search via the server-rendered HTML results page. */
|
||||
export async function searchBing(params: SearchParams): Promise<SearchResponse> {
|
||||
const numResults = clampNumResults(params.numSearchResults ?? params.limit, DEFAULT_NUM_RESULTS, MAX_NUM_RESULTS);
|
||||
const html = await callBingHtml(params, numResults);
|
||||
const parsed = parseHtmlResults(html);
|
||||
|
||||
const sources: SearchSource[] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const result of parsed) {
|
||||
if (seen.has(result.url)) continue;
|
||||
seen.add(result.url);
|
||||
sources.push({ title: result.title, url: result.url, snippet: result.snippet });
|
||||
if (sources.length >= numResults) break;
|
||||
}
|
||||
|
||||
return { provider: "bing", sources };
|
||||
}
|
||||
|
||||
/** Search provider for Bing (no API key required). */
|
||||
export class BingProvider extends SearchProvider {
|
||||
readonly id = "bing";
|
||||
readonly label = "Bing";
|
||||
|
||||
isAvailable(_authStorage: AuthStorage): boolean {
|
||||
return true;
|
||||
}
|
||||
|
||||
isExplicitlyAvailable(_authStorage: AuthStorage): boolean {
|
||||
return true;
|
||||
}
|
||||
|
||||
search(params: SearchParams): Promise<SearchResponse> {
|
||||
return searchBing(params);
|
||||
}
|
||||
}
|
||||
@@ -11,15 +11,13 @@ import { withHardTimeout } from "./utils";
|
||||
* Credential-free engines the Public Web aggregate fans out to. Order is the
|
||||
* tiebreak for merged ranking (earlier engines win equal consensus/rank), so
|
||||
* engines with the best ranking quality when they answer come first:
|
||||
* Google-index engines (startpage, google) lead, Bing-backed scrapers follow,
|
||||
* and Mojeek's independent index breaks remaining ties (measured 2026-07).
|
||||
* Google-index engines (startpage, google) lead, and Mojeek's independent
|
||||
* index breaks remaining ties (measured 2026-07).
|
||||
*/
|
||||
const PUBLIC_ENGINE_IDS = [
|
||||
"startpage",
|
||||
"google",
|
||||
"duckduckgo",
|
||||
"bing",
|
||||
"yahoo",
|
||||
"ecosia",
|
||||
"mojeek",
|
||||
] as const satisfies readonly SearchProviderId[];
|
||||
|
||||
@@ -1,179 +0,0 @@
|
||||
import type { AuthStorage } from "@oh-my-pi/pi-ai";
|
||||
import { parseHTML } from "linkedom";
|
||||
import type { SearchResponse, SearchSource } from "../../../web/search/types";
|
||||
import { SearchProviderError } from "../../../web/search/types";
|
||||
import { clampNumResults } from "../utils";
|
||||
import type { SearchParams } from "./base";
|
||||
import { SearchProvider } from "./base";
|
||||
import { browserFetch } from "./browser-page";
|
||||
import { classifyProviderHttpError, withHardTimeout } from "./utils";
|
||||
|
||||
/**
|
||||
* Yahoo Search's server-rendered results page. A plain GET with browser
|
||||
* navigation headers returns the full SERP without any JavaScript challenge,
|
||||
* so no headless-browser fallback is needed (verified live 2026-07).
|
||||
*/
|
||||
const YAHOO_HOME_URL = "https://search.yahoo.com/";
|
||||
const YAHOO_SEARCH_URL = "https://search.yahoo.com/search";
|
||||
const DEFAULT_NUM_RESULTS = 10;
|
||||
const MAX_NUM_RESULTS = 20;
|
||||
|
||||
/**
|
||||
* Recency → Yahoo `btf` query param. Yahoo's time filter only offers
|
||||
* day/week/month; `year` has no equivalent and is silently dropped per the
|
||||
* {@link SearchParams.recency} contract.
|
||||
*/
|
||||
const RECENCY_TO_YAHOO_BTF: Partial<Record<NonNullable<SearchParams["recency"]>, string>> = {
|
||||
day: "d",
|
||||
week: "w",
|
||||
month: "m",
|
||||
};
|
||||
|
||||
interface ParsedResult {
|
||||
title: string;
|
||||
url: string;
|
||||
snippet?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve a Yahoo result href back to the underlying target URL.
|
||||
*
|
||||
* Organic hrefs are wrapped through the click tracker
|
||||
* `https://r.search.yahoo.com/_ylt=…/RU=<percent-encoded-target>/RK=…/RS=…`;
|
||||
* the `/RU=` path segment carries the destination. Older layouts emit plain
|
||||
* absolute hrefs, so both shapes are handled. Tracker links without a
|
||||
* recoverable target and Yahoo-internal navigation are rejected.
|
||||
*/
|
||||
function unwrapResultUrl(href: string): string | undefined {
|
||||
let url: URL;
|
||||
try {
|
||||
url = new URL(href, YAHOO_HOME_URL);
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
if (url.protocol !== "http:" && url.protocol !== "https:") return undefined;
|
||||
|
||||
const wrapped = /\/RU=([^/]+)/.exec(url.pathname);
|
||||
if (wrapped) {
|
||||
let target: string;
|
||||
try {
|
||||
target = decodeURIComponent(wrapped[1]);
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
return target.startsWith("http://") || target.startsWith("https://") ? target : undefined;
|
||||
}
|
||||
// A tracker link without an RU segment has no recoverable destination.
|
||||
if (url.hostname === "r.search.yahoo.com") return undefined;
|
||||
// Relative hrefs resolve against the search host: internal navigation.
|
||||
if (url.hostname === "search.yahoo.com") return undefined;
|
||||
return url.href;
|
||||
}
|
||||
|
||||
/**
|
||||
* Walk the SERP and pull organic result blocks in document order.
|
||||
*
|
||||
* Organics render as `<div class="… algo …">` blocks (inside `#web`'s
|
||||
* `<ol>`): the title `<h3>` sits inside the tracker `<a>` in the current
|
||||
* layout, while legacy layouts nested the `<a>` inside `<h3 class="title">`
|
||||
* — both are handled. The preview text lives in a sibling
|
||||
* `<div class="compText">`. Module headers ("Videos", "People also ask")
|
||||
* carry `<h3>`s outside `.algo` blocks and are excluded by construction.
|
||||
*/
|
||||
function parseHtmlResults(html: string): ParsedResult[] {
|
||||
const { document } = parseHTML(html);
|
||||
const results: ParsedResult[] = [];
|
||||
for (const block of document.querySelectorAll("div.algo")) {
|
||||
const heading = block.querySelector("h3");
|
||||
if (!heading) continue;
|
||||
const anchor = heading.querySelector("a") ?? heading.closest("a");
|
||||
const href = anchor?.getAttribute("href");
|
||||
if (!href) continue;
|
||||
const url = unwrapResultUrl(href);
|
||||
if (!url) continue;
|
||||
const title = (heading.textContent ?? "").replace(/\s+/g, " ").trim();
|
||||
if (!title) continue;
|
||||
const snippet = (block.querySelector(".compText")?.textContent ?? "").replace(/\s+/g, " ").trim() || undefined;
|
||||
results.push({ title, url, snippet });
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* `true` when Yahoo answered with its EU consent interstitial instead of
|
||||
* results: either the request was redirected to consent.yahoo.com /
|
||||
* guce.yahoo.com, or the body carries the consent form. The normal SERP
|
||||
* mentions guce.yahoo.com only in a meta tag, so detection keys on the
|
||||
* consent-host redirect and the `collectConsent` form action.
|
||||
*/
|
||||
function isConsentInterstitial(finalUrl: string, html: string): boolean {
|
||||
if (/^https?:\/\/(?:[^/]*\.)?(?:consent|guce)\.yahoo\.com\//i.test(finalUrl)) return true;
|
||||
return html.includes("consent.yahoo.com") || html.includes("collectConsent");
|
||||
}
|
||||
|
||||
async function callYahooHtml(params: SearchParams, numResults: number): Promise<string> {
|
||||
const url = new URL(YAHOO_SEARCH_URL);
|
||||
url.searchParams.set("p", params.query);
|
||||
url.searchParams.set("n", String(numResults));
|
||||
const btf = params.recency ? RECENCY_TO_YAHOO_BTF[params.recency] : undefined;
|
||||
if (btf) url.searchParams.set("btf", btf);
|
||||
|
||||
const page = await browserFetch(url.href, {
|
||||
fetch: params.fetch ?? fetch,
|
||||
signal: withHardTimeout(params.signal),
|
||||
referer: YAHOO_HOME_URL,
|
||||
});
|
||||
|
||||
const body = page.html;
|
||||
if (page.status < 200 || page.status >= 300) {
|
||||
const classified = classifyProviderHttpError("yahoo", page.status, body);
|
||||
if (classified) throw classified;
|
||||
throw new SearchProviderError("yahoo", `Yahoo HTML error (${page.status})`, page.status);
|
||||
}
|
||||
|
||||
if (isConsentInterstitial(page.url, body)) {
|
||||
throw new SearchProviderError(
|
||||
"yahoo",
|
||||
"Yahoo served its GDPR consent interstitial instead of search results. This typically affects EU egress IPs; use another web search provider such as DuckDuckGo, Brave, or Mojeek.",
|
||||
429,
|
||||
);
|
||||
}
|
||||
|
||||
return body;
|
||||
}
|
||||
|
||||
/** Execute a Yahoo web search via the server-rendered HTML results page. */
|
||||
export async function searchYahoo(params: SearchParams): Promise<SearchResponse> {
|
||||
const numResults = clampNumResults(params.numSearchResults ?? params.limit, DEFAULT_NUM_RESULTS, MAX_NUM_RESULTS);
|
||||
const html = await callYahooHtml(params, numResults);
|
||||
const parsed = parseHtmlResults(html);
|
||||
|
||||
const sources: SearchSource[] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const result of parsed) {
|
||||
if (seen.has(result.url)) continue;
|
||||
seen.add(result.url);
|
||||
sources.push({ title: result.title, url: result.url, snippet: result.snippet });
|
||||
if (sources.length >= numResults) break;
|
||||
}
|
||||
|
||||
return { provider: "yahoo", sources };
|
||||
}
|
||||
|
||||
/** Search provider for Yahoo (no API key required). */
|
||||
export class YahooProvider extends SearchProvider {
|
||||
readonly id = "yahoo";
|
||||
readonly label = "Yahoo";
|
||||
|
||||
isAvailable(_authStorage: AuthStorage): boolean {
|
||||
return true;
|
||||
}
|
||||
|
||||
isExplicitlyAvailable(_authStorage: AuthStorage): boolean {
|
||||
return true;
|
||||
}
|
||||
|
||||
search(params: SearchParams): Promise<SearchResponse> {
|
||||
return searchYahoo(params);
|
||||
}
|
||||
}
|
||||
@@ -53,16 +53,6 @@ export const SEARCH_PROVIDER_OPTIONS = [
|
||||
label: "DuckDuckGo",
|
||||
description: "Credential-free best-effort fallback; may be bot-challenged on datacenter/shared-egress IPs",
|
||||
},
|
||||
{
|
||||
value: "bing",
|
||||
label: "Bing",
|
||||
description: "Credential-free HTML scrape of Bing results; may be bot-challenged",
|
||||
},
|
||||
{
|
||||
value: "yahoo",
|
||||
label: "Yahoo",
|
||||
description: "Credential-free HTML scrape of Yahoo (Bing-backed) results",
|
||||
},
|
||||
{
|
||||
value: "ecosia",
|
||||
label: "Ecosia",
|
||||
|
||||
@@ -18,7 +18,7 @@ const fakeAuthStorage = {
|
||||
} as unknown as AuthStorage;
|
||||
|
||||
/** Restrict the fan-out to the two engines these tests provide fixtures for. */
|
||||
const NON_TEST_ENGINES: readonly SearchProviderId[] = ["bing", "yahoo", "ecosia", "startpage", "mojeek"];
|
||||
const NON_TEST_ENGINES: readonly SearchProviderId[] = ["ecosia", "startpage", "mojeek"];
|
||||
|
||||
function makeParams(query: string, fetch: FetchImpl): SearchParams {
|
||||
return {
|
||||
|
||||
Reference in New Issue
Block a user