car-inference 0.63.0

//! Web search capability — the `search` tool, fulfilled natively.
//!
//! CAR ships a `search` ToolSchema (car-ir builtins) that agents can declare,
//! but until now nothing implemented it. This module backs it from two sources,
//! resolved automatically (see [`resolve_provider`]):
//!
//! - **Parslee** — when a Parslee session is present (`PARSLEE_ACCESS_TOKEN`).
//!   Calls Parslee's hosted search (`POST {base}/api/v1/orgs/{org}/search`),
//!   which is Tavily under the hood with org/billing wrapping. No user key
//!   needed; Parslee users get search through their account.
//! - **Tavily (bring-your-own-key)** — when `TAVILY_API_KEY` is set. Calls
//!   `POST https://api.tavily.com/search` directly.
//!
//! Both return the same [`WebSearchResponse`] shape (mirrors Parslee's
//! `WebSearchResult`: title/url/snippet/score/published_date). The HTTP calls
//! are the only impure part; the JSON→struct mapping is pure and unit-tested.

use crate::http_body::{read_capped, MAX_REMOTE_BODY_BYTES};
use crate::search_gate::{
    is_ad_link, is_challenge_page, is_throttle_status, time_left, Refusal, SearchGate,
    SEARCH_DEADLINE, THROTTLE_HINT,
};
use crate::web_markup::{self, web_address};
use serde::{Deserialize, Serialize};
use std::sync::{Mutex, OnceLock};
use tokio::time::Instant;

/// Tavily's hard cap on results per query (search_depth=basic).
const MAX_RESULTS_CAP: u32 = 20;
const DEFAULT_MAX_RESULTS: u32 = 5;
const TAVILY_ENDPOINT: &str = "https://api.tavily.com/search";
pub(crate) const TAVILY_API_KEY_ENV: &str = "TAVILY_API_KEY";
const HTTP_TIMEOUT_SECS: u64 = 20;

/// One web-search hit. Shape mirrors Parslee's `WebSearchResult` so the two
/// sources are interchangeable to callers.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct WebSearchResult {
    pub title: String,
    pub url: String,
    /// The result excerpt (Tavily calls this `content`; Parslee `snippet`).
    pub snippet: String,
    /// Relevance score in `[0,1]` when the provider reports it.
    #[serde(default)]
    pub score: f64,
    /// Publication date as the provider reports it (ISO-ish string), if any.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub published_date: Option<String>,
}

/// A web-search response: the echoed query, the results, and which backend
/// served it (`"tavily"` or `"parslee"`).
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct WebSearchResponse {
    pub query: String,
    pub results: Vec<WebSearchResult>,
    pub source: String,
}

/// Parameters for the `search` capability (matches the `search` ToolSchema).
#[derive(Debug, Clone, Serialize, Deserialize, schemars::JsonSchema)]
pub struct SearchRequest {
    pub query: String,
    /// 1..=20; defaults to 5 when omitted.
    #[serde(default)]
    pub max_results: Option<u32>,
}

#[derive(Debug, thiserror::Error)]
pub enum SearchError {
    #[error("web search HTTP error: {0}")]
    Http(String),
    #[error("web search parse error: {0}")]
    Parse(String),
}

/// Which backend serves a search, resolved from the environment.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum SearchProvider {
    /// Parslee hosted search (bearer token; no user key needed).
    Parslee { base: String, token: String },
    /// Tavily direct, with the user's own key.
    Tavily { api_key: String },
    /// Keyless fallback: DuckDuckGo's no-JS HTML endpoint, scraped over plain
    /// HTTP. Zero-config (no account, no key) so search always works; lower
    /// quality + more fragile than the keyed backends. (car-releases#59 asked
    /// for a no-key `browser_run` fallback; this is the same goal via a plain
    /// HTTP fetch — no Chromium — which is lighter and lives in this layer.)
    DuckDuckGo,
}

/// Pick the search backend: Parslee session first (no key needed for the
/// signed-in user), else a bring-your-own Tavily key, else the keyless
/// DuckDuckGo fallback so search works with zero configuration.
pub async fn resolve_provider() -> SearchProvider {
    // Mint a proactively-refreshed bearer (#318) — the sibling chat path was
    // already fixed in #313 but the search path still read the raw token and
    // 401'd on a lapsed session. `access_token_refreshing` is env-first, so
    // the `PARSLEE_ACCESS_TOKEN` override still wins and is never refreshed.
    if let Some(token) = car_auth::access_token_refreshing()
        .await
        .filter(|t| !t.is_empty())
    {
        let base = car_auth::api_base(None);
        return SearchProvider::Parslee { base, token };
    }
    if let Some(api_key) =
        car_secrets::resolve_env_or_keychain(TAVILY_API_KEY_ENV).filter(|k| !k.is_empty())
    {
        return SearchProvider::Tavily { api_key };
    }
    SearchProvider::DuckDuckGo
}

fn clamp_max(max_results: Option<u32>) -> u32 {
    max_results
        .unwrap_or(DEFAULT_MAX_RESULTS)
        .clamp(1, MAX_RESULTS_CAP)
}

/// The process-wide search/fetch client.
///
/// Every `ClientBuilder::build()` re-reads and re-parses the whole OS trust
/// store (`rustls_native_certs::load_native_certs()` runs inside it, with no
/// memoization anywhere below us), so building one per tool call was a real
/// per-call cost. `reqwest::Client` is internally `Arc`'d, so handing out
/// clones is cheap and incidentally gives search and fetch the cross-call
/// connection pooling they never had.
///
/// Deliberately keyed on nothing: the client carries no host affinity, only
/// the URL differs per request, and keying it on host or provider would
/// reintroduce the per-call build this exists to remove. The trade-off that
/// comes with it: proxy settings are frozen at first build, so a change to the
/// machine's proxy environment is not picked up until the daemon restarts.
///
/// `Mutex<Option<_>>` rather than a bare `OnceLock<Client>` because the cell
/// has to be emptiable — see [`reset_http_client_cache_for_test`]. Shape
/// mirrors `cache()` in `car-external-agents/src/health.rs`.
fn client_cache() -> &'static Mutex<Option<reqwest::Client>> {
    static CACHE: OnceLock<Mutex<Option<reqwest::Client>>> = OnceLock::new();
    CACHE.get_or_init(|| Mutex::new(None))
}

/// Counts trips through the builder, so a test can prove the second call did
/// not rebuild.
#[cfg(test)]
fn build_count() -> &'static std::sync::atomic::AtomicUsize {
    static COUNT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
    &COUNT
}

/// The shared search/fetch client. Builds at most once per process.
///
/// Successes only are cached: a build failure returns to the caller and leaves
/// the cell empty, so a machine whose trust store gets repaired recovers on the
/// next search with no restart. (Inference makes no such promise — its
/// degraded client persists for the life of the daemon by design.)
fn http_client() -> Result<reqwest::Client, SearchError> {
    // Scoped so the guard is released before the (slow) build below.
    {
        let cached = client_cache().lock().unwrap_or_else(|p| p.into_inner());
        if let Some(client) = cached.as_ref() {
            return Ok(client.clone());
        }
    }

    #[cfg(test)]
    build_count().fetch_add(1, std::sync::atomic::Ordering::SeqCst);

    let client = reqwest::Client::builder()
        .timeout(std::time::Duration::from_secs(HTTP_TIMEOUT_SECS))
        .build()
        .map_err(|e| SearchError::Http(e.to_string()))?;

    // Losing the race is harmless: the winner's client is kept and used, ours
    // is dropped.
    Ok(client_cache()
        .lock()
        .unwrap_or_else(|p| p.into_inner())
        .get_or_insert(client)
        .clone())
}

/// Test-only: empty the cache and zero the build counter. Compiled out of
/// release builds, so nothing can clear the cache at runtime.
#[cfg(test)]
pub(crate) fn reset_http_client_cache_for_test() {
    *client_cache().lock().unwrap_or_else(|p| p.into_inner()) = None;
    build_count().store(0, std::sync::atomic::Ordering::SeqCst);
}

#[cfg(test)]
pub(crate) fn http_client_is_cached() -> bool {
    client_cache()
        .lock()
        .unwrap_or_else(|p| p.into_inner())
        .is_some()
}

#[cfg(test)]
pub(crate) fn http_client_build_count() -> usize {
    build_count().load(std::sync::atomic::Ordering::SeqCst)
}

/// Run a web search, auto-resolving the backend. The entry point the `search`
/// capability calls.
pub async fn web_search(
    query: &str,
    max_results: Option<u32>,
) -> Result<WebSearchResponse, SearchError> {
    let max = clamp_max(max_results);
    match resolve_provider().await {
        SearchProvider::Parslee { base, token } => parslee_search(query, max, &base, &token).await,
        SearchProvider::Tavily { api_key } => tavily_search(query, max, &api_key).await,
        SearchProvider::DuckDuckGo => ddg_search(query, max).await,
    }
}

/// Tavily direct (BYO key). `POST https://api.tavily.com/search`.
pub async fn tavily_search(
    query: &str,
    max_results: u32,
    api_key: &str,
) -> Result<WebSearchResponse, SearchError> {
    let body = serde_json::json!({
        "api_key": api_key,
        "query": query,
        "max_results": max_results,
        "search_depth": "basic",
        "include_answer": false,
        "include_raw_content": false,
        "include_images": false,
    });
    let resp = http_client()?
        .post(TAVILY_ENDPOINT)
        .json(&body)
        .send()
        .await
        .map_err(|e| SearchError::Http(format!("tavily: {e}")))?;
    if !resp.status().is_success() {
        return Err(SearchError::Http(format!("tavily: HTTP {}", resp.status())));
    }
    let json: serde_json::Value = resp
        .json()
        .await
        .map_err(|e| SearchError::Parse(format!("tavily: {e}")))?;
    Ok(parse_tavily(query, &json))
}

/// Parslee hosted search. Resolves the org for the bearer, then
/// `POST {base}/api/v1/orgs/{org}/search`.
pub async fn parslee_search(
    query: &str,
    max_results: u32,
    base: &str,
    token: &str,
) -> Result<WebSearchResponse, SearchError> {
    let client = http_client()?;
    let base = base.trim_end_matches('/');
    let org_id = parslee_org_id(&client, base, token).await?;
    let url = format!("{base}/api/v1/orgs/{org_id}/search");
    let body = serde_json::json!({ "query": query, "maxResults": max_results });
    let resp = client
        .post(&url)
        .bearer_auth(token)
        .json(&body)
        .send()
        .await
        .map_err(|e| SearchError::Http(format!("parslee search: {e}")))?;
    if !resp.status().is_success() {
        return Err(SearchError::Http(format!(
            "parslee search: HTTP {}",
            resp.status()
        )));
    }
    let json: serde_json::Value = resp
        .json()
        .await
        .map_err(|e| SearchError::Parse(format!("parslee search: {e}")))?;
    Ok(parse_parslee(query, &json))
}

/// Resolve the Parslee org id for a bearer via `/api/v1/organizations/me`
/// (mirrors the inference path in `remote.rs`).
async fn parslee_org_id(
    client: &reqwest::Client,
    base: &str,
    token: &str,
) -> Result<String, SearchError> {
    let resp = client
        .get(format!("{base}/api/v1/organizations/me"))
        .bearer_auth(token)
        .send()
        .await
        .map_err(|e| SearchError::Http(format!("parslee org lookup: {e}")))?;
    if !resp.status().is_success() {
        return Err(SearchError::Http(format!(
            "parslee org lookup: HTTP {}",
            resp.status()
        )));
    }
    let json: serde_json::Value = resp
        .json()
        .await
        .map_err(|e| SearchError::Parse(format!("parslee org: {e}")))?;
    json.get("organizationId")
        .or_else(|| json.get("OrganizationId"))
        .and_then(|v| v.as_str())
        .map(String::from)
        .ok_or_else(|| {
            SearchError::Parse(
                "parslee org response has no organizationId (finish onboarding)".to_string(),
            )
        })
}

/// Pure: map a Tavily `/search` JSON body to [`WebSearchResponse`]. Tavily's
/// `content` becomes our `snippet`; results without a url are dropped.
fn parse_tavily(query: &str, json: &serde_json::Value) -> WebSearchResponse {
    let results = json
        .get("results")
        .and_then(|r| r.as_array())
        .map(|arr| {
            arr.iter()
                .filter_map(|r| {
                    // Only a web address is a result's url.
                    let url = r
                        .get("url")
                        .and_then(|v| v.as_str())
                        .filter(|url| web_address(url).is_some())?
                        .to_string();
                    Some(WebSearchResult {
                        title: r
                            .get("title")
                            .and_then(|v| v.as_str())
                            .unwrap_or("")
                            .to_string(),
                        url,
                        snippet: r
                            .get("content")
                            .and_then(|v| v.as_str())
                            .unwrap_or("")
                            .to_string(),
                        score: r.get("score").and_then(|v| v.as_f64()).unwrap_or(0.0),
                        published_date: r
                            .get("published_date")
                            .and_then(|v| v.as_str())
                            .map(String::from),
                    })
                })
                .collect()
        })
        .unwrap_or_default();
    WebSearchResponse {
        query: query.to_string(),
        results,
        source: "tavily".to_string(),
    }
}

/// Pure: map Parslee's `/search` JSON body to [`WebSearchResponse`]. Parslee
/// already returns `snippet`/`publishedDate`; results without a url are dropped.
fn parse_parslee(query: &str, json: &serde_json::Value) -> WebSearchResponse {
    let results = json
        .get("results")
        .and_then(|r| r.as_array())
        .map(|arr| {
            arr.iter()
                .filter_map(|r| {
                    // Only a web address is a result's url.
                    let url = r
                        .get("url")
                        .and_then(|v| v.as_str())
                        .filter(|url| web_address(url).is_some())?
                        .to_string();
                    Some(WebSearchResult {
                        title: r
                            .get("title")
                            .and_then(|v| v.as_str())
                            .unwrap_or("")
                            .to_string(),
                        url,
                        snippet: r
                            .get("snippet")
                            .and_then(|v| v.as_str())
                            .unwrap_or("")
                            .to_string(),
                        score: r.get("score").and_then(|v| v.as_f64()).unwrap_or(0.0),
                        published_date: r
                            .get("publishedDate")
                            .and_then(|v| v.as_str())
                            .map(String::from),
                    })
                })
                .collect()
        })
        .unwrap_or_default();
    WebSearchResponse {
        query: query.to_string(),
        results,
        source: "parslee".to_string(),
    }
}

const DDG_HTML_ENDPOINT: &str = "https://html.duckduckgo.com/html/";
const USER_AGENT: &str = "Mozilla/5.0 (compatible; CAR/1.0; +https://parslee.ai)";

/// Keyless search via DuckDuckGo's no-JS HTML endpoint, scraped over plain
/// HTTP. The zero-config fallback (no key, no account). Best-effort: the SERP
/// markup can change and there's no relevance score.
///
/// Asked the way the provider's own search form asks, with a form POST: it
/// answers a GET of the page with its challenge page after the first search
/// or two. Requests are spaced and held off after a throttle by the
/// process-wide [`SearchGate::duckduckgo`], which Parslee Core's `web_search`
/// shares, since the provider throttles by address.
pub async fn ddg_search(query: &str, max_results: u32) -> Result<WebSearchResponse, SearchError> {
    ddg_search_at(
        &SearchGate::duckduckgo(),
        DDG_HTML_ENDPOINT,
        query,
        max_results,
    )
    .await
}

/// [`ddg_search`] against the result page at `endpoint`, paced by `gate`
/// within one [`SEARCH_DEADLINE`].
async fn ddg_search_at(
    gate: &SearchGate,
    endpoint: &str,
    query: &str,
    max_results: u32,
) -> Result<WebSearchResponse, SearchError> {
    let deadline = Instant::now() + SEARCH_DEADLINE;
    match gate.take_turn(deadline).await {
        Ok(()) => {}
        Err(Refusal::BackingOff(left)) => {
            return Err(SearchError::Http(format!(
                "duckduckgo: it throttled an earlier search, so CAR is not asking it again \
                 for another {}s; try again after that",
                left.as_secs().max(1)
            )))
        }
        Err(Refusal::PastDeadline) => {
            return Err(SearchError::Http(format!(
                "duckduckgo: other searches are queued for it, and this one's turn would \
                 come after its {}s budget",
                SEARCH_DEADLINE.as_secs()
            )))
        }
    }
    let resp = http_client()?
        .post(endpoint)
        .header(reqwest::header::USER_AGENT, USER_AGENT)
        .form(&[("q", query), ("b", "")])
        .timeout(time_left(deadline, SEARCH_DEADLINE))
        .send()
        .await
        .map_err(|e| SearchError::Http(format!("duckduckgo: {e}")))?;
    // The provider answers a client it is throttling with 202 and its
    // challenge page — a success status, so it is checked by value — and may
    // refuse one outright with 403 or 429.
    let status = resp.status();
    if status != reqwest::StatusCode::OK {
        let throttled = is_throttle_status(status.as_u16());
        if throttled {
            gate.throttled();
        }
        return Err(SearchError::Http(format!(
            "duckduckgo: HTTP {status}{}",
            if throttled { THROTTLE_HINT } else { "" }
        )));
    }
    // The result page is a few tens of kilobytes with its results at the
    // top, so what fits in the cap is parsed even when the rest is not read.
    let html = read_capped(resp, MAX_REMOTE_BODY_BYTES)
        .await
        .map_err(|e| SearchError::Parse(format!("duckduckgo: {e}")))?
        .text();
    let found = parse_ddg(query, &html, max_results)?;
    if found.results.is_empty() && is_challenge_page(&html) {
        gate.throttled();
        return Err(SearchError::Http(format!(
            "duckduckgo: it served a challenge page instead of results{THROTTLE_HINT}"
        )));
    }
    Ok(found)
}

/// Pure: parse a DuckDuckGo HTML SERP into results. Result links are DDG
/// redirects (`//duckduckgo.com/l/?uddg=<real-url>`); we unwrap the real url.
/// A result block, link or snippet that is hidden ([`ddg_hidden`]) is not
/// used, and hidden text inside a title or snippet is left out. A page whose
/// result blocks hold result links that give a web address, every one of
/// them hidden, is a parse error, not a search that found nothing.
fn parse_ddg(query: &str, html: &str, max_results: u32) -> Result<WebSearchResponse, SearchError> {
    use scraper::{Html, Selector};
    let doc = Html::parse_document(html);
    let result_sel = Selector::parse("div.result, div.web-result").unwrap();
    let a_sel = Selector::parse("a.result__a").unwrap();
    let snippet_sel = Selector::parse("a.result__snippet, .result__snippet").unwrap();
    let mut results = Vec::new();
    for el in doc.select(&result_sel) {
        if results.len() >= max_results as usize {
            break;
        }
        if ddg_hidden(el) {
            continue;
        }
        let Some(a) = el.select(&a_sel).find(|a| !ddg_hidden(*a)) else {
            continue;
        };
        let url = ddg_unwrap_href(a.value().attr("href").unwrap_or(""));
        if url.is_empty() || is_ad_link(&url) {
            continue;
        }
        let title = ddg_visible_text(a)
            .split_whitespace()
            .collect::<Vec<_>>()
            .join(" ");
        let snippet = el
            .select(&snippet_sel)
            .find(|s| !ddg_hidden(*s))
            .map(|s| {
                ddg_visible_text(s)
                    .split_whitespace()
                    .collect::<Vec<_>>()
                    .join(" ")
            })
            .unwrap_or_default();
        results.push(WebSearchResult {
            title,
            url,
            snippet,
            score: 0.0,
            published_date: None,
        });
    }
    if results.is_empty()
        && doc
            .select(&result_sel)
            .flat_map(|el| el.select(&a_sel))
            .any(|a| {
                ddg_hidden(a) && !ddg_unwrap_href(a.value().attr("href").unwrap_or("")).is_empty()
            })
    {
        return Err(SearchError::Parse(
            "duckduckgo: every result link was hidden".to_string(),
        ));
    }
    Ok(WebSearchResponse {
        query: query.to_string(),
        results,
        source: "duckduckgo".to_string(),
    })
}

/// Whether `el` is hidden: it or an element it sits in is a `template` or
/// is marked hidden ([`ddg_marked_hidden`]).
fn ddg_hidden(el: scraper::ElementRef<'_>) -> bool {
    std::iter::once(*el).chain(el.ancestors()).any(|node| {
        node.value()
            .as_element()
            .is_some_and(|e| e.name() == "template" || ddg_marked_hidden(e))
    })
}

/// Whether an element's attributes hide it
/// ([`web_markup::hides_by_attributes`], the reader Parslee Core's
/// `web_search` uses too).
fn ddg_marked_hidden(el: &scraper::node::Element) -> bool {
    web_markup::hides_by_attributes(el.attrs())
}

/// The text inside `el`, leaving out what hidden elements in it hold.
fn ddg_visible_text(el: scraper::ElementRef<'_>) -> String {
    let mut text = String::new();
    for child in el.children() {
        if let Some(piece) = child.value().as_text() {
            text.push_str(piece);
        } else if let Some(child) = scraper::ElementRef::wrap(child) {
            if child.value().name() != "template" && !ddg_marked_hidden(child.value()) {
                text.push_str(&ddg_visible_text(child));
            }
        }
    }
    text
}

/// Unwrap a DuckDuckGo result href to the real destination url. A direct
/// url passes through; `//duckduckgo.com/l/?uddg=<encoded>` carries the
/// target in the (percent-decoded) `uddg` query param. Either is kept only
/// as a web address ([`web_address`]: an `http` or `https` url, serialized),
/// so a `javascript:` or `data:` destination, or a relative one, is empty.
fn ddg_unwrap_href(href: &str) -> String {
    if let Some(url) = web_address(href) {
        return url;
    }
    let Some(rest) = href.strip_prefix("//") else {
        return String::new();
    };
    reqwest::Url::parse(&format!("https://{rest}"))
        .ok()
        .and_then(|u| {
            u.query_pairs()
                .find(|(k, _)| k == "uddg")
                .map(|(_, v)| v.into_owned())
        })
        .and_then(|target| web_address(&target))
        .unwrap_or_default()
}

/// A fetched web page: final url, status, content type, extracted title + text.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct WebFetchResponse {
    pub url: String,
    pub status: u16,
    pub content_type: String,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub title: Option<String>,
    pub text: String,
}

/// Parameters for the `web_fetch` capability.
#[derive(Debug, Clone, Serialize, Deserialize, schemars::JsonSchema)]
pub struct FetchRequest {
    pub url: String,
}

/// Fetch a URL and extract readable text. Keyless (plain HTTP GET). HTML is
/// reduced to title + visible text (scripts/styles dropped); non-HTML bodies
/// are returned verbatim. Best-effort extraction — good enough for an LLM to
/// read, not a full readability engine. Only the first
/// [`MAX_REMOTE_BODY_BYTES`] of the body are read; when it is longer, the
/// text is of that part and ends in [`TRUNCATED`].
pub async fn web_fetch(url: &str) -> Result<WebFetchResponse, SearchError> {
    let resp = http_client()?
        .get(url)
        .header(reqwest::header::USER_AGENT, USER_AGENT)
        .send()
        .await
        .map_err(|e| SearchError::Http(format!("fetch: {e}")))?;
    let status = resp.status().as_u16();
    let final_url = resp.url().to_string();
    let content_type = resp
        .headers()
        .get(reqwest::header::CONTENT_TYPE)
        .and_then(|v| v.to_str().ok())
        .unwrap_or("")
        .to_string();
    let read = read_capped(resp, MAX_REMOTE_BODY_BYTES)
        .await
        .map_err(|e| SearchError::Parse(format!("fetch body: {e}")))?;
    let body = read.text();
    let (title, mut text) = if content_type.contains("html") || looks_like_html(&body) {
        (extract_title(&body), html_to_text(&body))
    } else {
        (None, body)
    };
    if read.cut {
        text.push_str(TRUNCATED);
    }
    Ok(WebFetchResponse {
        url: final_url,
        status,
        content_type,
        title,
        text,
    })
}

/// The marker ending the text of a page whose body was longer than the
/// [`MAX_REMOTE_BODY_BYTES`] read.
pub const TRUNCATED: &str = "…[truncated]…";

fn looks_like_html(body: &str) -> bool {
    let head = body.get(..512).unwrap_or(body).to_ascii_lowercase();
    head.contains("<!doctype html") || head.contains("<html") || head.contains("<body")
}

/// Pure: extract the `<title>` text from HTML, if any.
fn extract_title(html: &str) -> Option<String> {
    use scraper::{Html, Selector};
    let doc = Html::parse_document(html);
    let sel = Selector::parse("title").ok()?;
    doc.select(&sel)
        .next()
        .map(|t| {
            t.text()
                .collect::<String>()
                .split_whitespace()
                .collect::<Vec<_>>()
                .join(" ")
        })
        .filter(|s| !s.is_empty())
}

/// Pure: reduce HTML to readable text. Collects text from content-bearing
/// elements (so `<script>`/`<style>` are excluded), falling back to body text
/// for div-only pages.
fn html_to_text(html: &str) -> String {
    use scraper::{Html, Selector};
    let doc = Html::parse_document(html);
    let content = Selector::parse(
        "p, li, h1, h2, h3, h4, h5, h6, blockquote, td, th, caption, figcaption, dd, dt, pre",
    )
    .unwrap();
    let parts: Vec<String> = doc
        .select(&content)
        .filter_map(|el| {
            let t = el
                .text()
                .collect::<String>()
                .split_whitespace()
                .collect::<Vec<_>>()
                .join(" ");
            (!t.is_empty()).then_some(t)
        })
        .collect();
    let joined = parts.join("\n");
    if !joined.is_empty() {
        // Content extraction excludes <script>/<style>; prefer it whenever it
        // found anything.
        return joined;
    }
    // Fallback only for pages with no semantic content elements at all (rare,
    // div-only markup). Best-effort — may include some non-content text.
    Selector::parse("body")
        .ok()
        .and_then(|b| doc.select(&b).next())
        .map(|b| {
            b.text()
                .collect::<String>()
                .split_whitespace()
                .collect::<Vec<_>>()
                .join(" ")
        })
        .unwrap_or(joined)
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn clamp_max_results() {
        assert_eq!(clamp_max(None), DEFAULT_MAX_RESULTS);
        assert_eq!(clamp_max(Some(0)), 1);
        assert_eq!(clamp_max(Some(7)), 7);
        assert_eq!(clamp_max(Some(999)), MAX_RESULTS_CAP);
    }

    #[test]
    fn parses_tavily_shape_content_to_snippet() {
        let json = serde_json::json!({
            "query": "rust async",
            "results": [
                { "title": "Async Rust", "url": "https://ex.com/a", "content": "tokio…",
                  "score": 0.92, "published_date": "2026-01-02" },
                { "title": "no url dropped", "url": "", "content": "x" }
            ],
            "response_time": 1.2
        });
        let r = parse_tavily("rust async", &json);
        assert_eq!(r.source, "tavily");
        assert_eq!(r.results.len(), 1, "empty-url result dropped");
        assert_eq!(r.results[0].url, "https://ex.com/a");
        assert_eq!(r.results[0].snippet, "tokio…", "content → snippet");
        assert_eq!(r.results[0].published_date.as_deref(), Some("2026-01-02"));
        assert!((r.results[0].score - 0.92).abs() < 1e-9);
    }

    #[test]
    fn parses_parslee_shape() {
        let json = serde_json::json!({
            "query": "q",
            "results": [
                { "title": "T", "url": "https://ex.com/p", "snippet": "snip",
                  "score": 0.5, "publishedDate": "2026-06-13T00:00:00" }
            ],
            "source": "Tavily"
        });
        let r = parse_parslee("q", &json);
        assert_eq!(r.source, "parslee");
        assert_eq!(r.results.len(), 1);
        assert_eq!(r.results[0].snippet, "snip");
        assert_eq!(
            r.results[0].published_date.as_deref(),
            Some("2026-06-13T00:00:00")
        );
    }

    #[test]
    fn tavily_and_parslee_results_need_a_web_address() {
        let json = serde_json::json!({
            "results": [
                { "title": "Evil", "url": "javascript:alert(1)", "content": "x", "snippet": "x" },
                { "title": "Data", "url": "data:text/html,hi", "content": "x", "snippet": "x" },
                { "title": "Relative", "url": "/page", "content": "x", "snippet": "x" },
                { "title": "Real", "url": "https://ex.com/a", "content": "x", "snippet": "x" }
            ]
        });
        for r in [parse_tavily("q", &json), parse_parslee("q", &json)] {
            let urls: Vec<_> = r.results.iter().map(|x| x.url.as_str()).collect();
            assert_eq!(urls, ["https://ex.com/a"], "{}", r.source);
        }
    }

    #[test]
    fn missing_results_is_empty_not_error() {
        let r = parse_tavily("q", &serde_json::json!({}));
        assert!(r.results.is_empty());
    }

    #[test]
    fn resolver_floors_at_duckduckgo() {
        // With no Parslee token and no Tavily key, search still works via the
        // keyless DDG fallback (zero-config). We can't guarantee the env is
        // clean here, but the fallback variant must be reachable.
        // (Just assert the type is constructible / matchable.)
        let p = SearchProvider::DuckDuckGo;
        assert!(matches!(p, SearchProvider::DuckDuckGo));
    }

    #[test]
    fn ddg_unwrap_redirect_and_direct() {
        // DDG redirect → real url (percent-decoded uddg param).
        let real = ddg_unwrap_href(
            "//duckduckgo.com/l/?uddg=https%3A%2F%2Fexample.com%2Fa%3Fx%3D1&rut=abc",
        );
        assert_eq!(real, "https://example.com/a?x=1");
        // Direct url passes through.
        assert_eq!(ddg_unwrap_href("https://ex.com/b"), "https://ex.com/b");
        // Junk → empty (dropped by the caller).
        assert_eq!(ddg_unwrap_href("/about"), "");
        assert_eq!(ddg_unwrap_href("javascript:alert(1)"), "");
        // A redirect whose destination is not a web url is junk too.
        for target in [
            "javascript%3Aalert(1)",
            "data%3Atext%2Fhtml%2Chi",
            "%2Frelative",
        ] {
            assert_eq!(
                ddg_unwrap_href(&format!("//duckduckgo.com/l/?uddg={target}")),
                "",
                "{target}"
            );
        }
        assert_eq!(
            ddg_unwrap_href("//duckduckgo.com/l/?uddg=HTTP%3A%2F%2Fex.com%2F"),
            "http://ex.com/"
        );
        // A direct url is read as a url, whatever the case of its scheme.
        assert_eq!(ddg_unwrap_href("HTTPS://Ex.com/b"), "https://ex.com/b");
        assert_eq!(ddg_unwrap_href("data:text/html,hi"), "");
    }

    #[test]
    fn parse_ddg_extracts_results() {
        let html = r#"
        <div class="result web-result">
          <a class="result__a" href="//duckduckgo.com/l/?uddg=https%3A%2F%2Fr.com%2F1">First &amp; Best</a>
          <a class="result__snippet">A snippet here.</a>
        </div>
        <div class="result">
          <a class="result__a" href="//duckduckgo.com/l/?uddg=https%3A%2F%2Fr.com%2F2">Second</a>
        </div>"#;
        let r = parse_ddg("q", html, 10).unwrap();
        assert_eq!(r.source, "duckduckgo");
        assert_eq!(r.results.len(), 2);
        assert_eq!(r.results[0].url, "https://r.com/1");
        assert_eq!(r.results[0].title, "First & Best");
        assert_eq!(r.results[0].snippet, "A snippet here.");
        assert_eq!(r.results[1].url, "https://r.com/2");
        // max_results caps.
        assert_eq!(parse_ddg("q", html, 1).unwrap().results.len(), 1);
    }

    #[test]
    fn parse_ddg_skips_hidden_results_links_and_text() {
        let real = r#"<div class="result"><a class="result__a" href="https://real.example/">Real</a><a class="result__snippet">Snippet</a></div>"#;
        for marked in [
            "hidden",
            r#"aria-hidden="true""#,
            r#"style="display: none""#,
            r#"style="visibility:hidden""#,
            r#"style="display:none !important; display:block""#,
            // Ways past a weaker CSS reader: a comment, an escape, and a
            // keyword the browser rejects, which leaves `none` in force.
            r#"style="display/**/:none""#,
            r#"style="display:\6e one""#,
            r#"style="display:none;display:inherit""#,
        ] {
            for page in [
                // A hidden result block, or one inside a hidden element.
                format!(
                    r#"<div class="result" {marked}><a class="result__a" href="https://evil.example/">Fake</a></div>{real}"#
                ),
                format!(
                    r#"<section {marked}><div class="result"><a class="result__a" href="https://evil.example/">Fake</a></div></section>{real}"#
                ),
                // A hidden link, snippet and text inside the real result.
                format!(
                    r#"<div class="result"><div {marked}><a class="result__a" href="https://evil.example/">Fake</a><a class="result__snippet">Planted</a></div><a class="result__a" href="https://real.example/">Re<span {marked}>planted</span>al</a><a class="result__snippet">Snip<b {marked}>planted</b>pet</a></div>"#
                ),
                format!(
                    r#"<template>{}</template>{real}"#,
                    real.replace("real.example", "evil.example")
                ),
            ] {
                let r = parse_ddg("q", &page, 10).unwrap();
                let found: Vec<_> = r
                    .results
                    .iter()
                    .map(|x| (x.url.as_str(), x.title.as_str(), x.snippet.as_str()))
                    .collect();
                assert_eq!(
                    found,
                    [("https://real.example/", "Real", "Snippet")],
                    "{page}"
                );
            }
        }
        // A style that shows the element in the end hides nothing.
        let page = r#"<div class="result" style="display:none;display:block"><a class="result__a" href="https://one.example/">One</a></div>"#;
        assert_eq!(parse_ddg("q", page, 10).unwrap().results.len(), 1);
    }

    #[test]
    fn parse_ddg_reports_a_page_whose_results_are_all_hidden() {
        for page in [
            r#"<div class="result" hidden><a class="result__a" href="https://one.example/">One</a></div>"#,
            r#"<div style="display:none"><div class="result"><a class="result__a" href="//duckduckgo.com/l/?uddg=https%3A%2F%2Fone.example%2F">One</a></div></div>"#,
            r#"<div class="result"><a class="result__a" aria-hidden="true" href="https://one.example/">One</a></div>"#,
        ] {
            let err = parse_ddg("q", page, 10).unwrap_err();
            assert!(
                err.to_string().contains("every result link was hidden"),
                "{page}: {err}"
            );
        }
        // No result links at all, or only hidden links with no web address,
        // is a search that found nothing.
        for page in [
            "<p>No results.</p>",
            r#"<div class="result"></div>"#,
            r#"<div class="result" hidden><a class="result__a" href="javascript:x">One</a></div>"#,
        ] {
            assert!(
                parse_ddg("q", page, 10).unwrap().results.is_empty(),
                "{page}"
            );
        }
    }

    #[test]
    fn html_to_text_drops_script_keeps_content() {
        let html = r#"<html><head><title> Hi — Page </title><style>.x{}</style></head>
            <body><script>var leak='SECRET';</script>
            <h1>Heading</h1><p>Para one.</p><p>Para two.</p></body></html>"#;
        assert_eq!(extract_title(html).as_deref(), Some("Hi — Page"));
        let text = html_to_text(html);
        assert!(text.contains("Heading"));
        assert!(text.contains("Para one."));
        assert!(text.contains("Para two."));
        assert!(
            !text.contains("SECRET"),
            "script text must be excluded: {text}"
        );
    }

    #[test]
    fn looks_like_html_detects() {
        assert!(looks_like_html("<!DOCTYPE html><html>…"));
        assert!(looks_like_html("<html lang=en>"));
        assert!(!looks_like_html("{\"json\": true}"));
    }

    /// A page whose body never ends is read up to the cap: the text of what
    /// was read comes back ending in the truncation marker, instead of the
    /// read running until the request timeout.
    #[tokio::test]
    async fn web_fetch_stops_reading_an_endless_body() {
        use crate::tls_client::test_seam::{TrustStoreScope, VALID_CERT_PEM};
        use tokio::io::{AsyncReadExt, AsyncWriteExt};

        let _scope = TrustStoreScope::acquire_async(VALID_CERT_PEM).await;
        reset_http_client_cache_for_test();
        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
        let address = listener.local_addr().unwrap();
        let sent = tokio::spawn(async move {
            let (mut socket, _) = listener.accept().await.unwrap();
            let mut request = [0u8; 2048];
            let _ = socket.read(&mut request).await;
            socket
                .write_all(
                    b"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nConnection: close\r\n\r\n",
                )
                .await
                .unwrap();
            let fill = [b'x'; 16 * 1024];
            let mut sent = 0usize;
            while socket.write_all(&fill).await.is_ok() {
                sent += fill.len();
            }
            sent
        });
        // short-wait-ok: must stop at the cap before the 20 s request timeout, the outcome it is told apart from
        let page = tokio::time::timeout(
            std::time::Duration::from_secs(15),
            web_fetch(&format!("http://{address}/")),
        )
        .await
        .expect("the read stops at the cap, before the 20 s request timeout")
        .unwrap();
        assert!(page.text.ends_with(TRUNCATED));
        assert_eq!(page.text.len(), MAX_REMOTE_BODY_BYTES + TRUNCATED.len());
        let sent = sent.await.unwrap();
        assert!(
            sent < 2 * MAX_REMOTE_BODY_BYTES,
            "the server sent {sent} bytes"
        );
    }

    // --- DuckDuckGo search HTTP path (wiremock), on a gate of its own that
    // does not space requests. ---

    /// A one-result DuckDuckGo result page.
    const DDG_PAGE: &str = r#"<div class="result"><a class="result__a" href="//duckduckgo.com/l/?uddg=https%3A%2F%2Fexample.com%2Fadams">John Adams</a><a class="result__snippet">Second president.</a></div>"#;

    /// The challenge page DuckDuckGo serves a client it is throttling.
    const DDG_CHALLENGE: &str = r#"<html><form id="challenge-form"><div class="anomaly-modal__title">Unfortunately, bots use DuckDuckGo too.</div></form></html>"#;

    fn unpaced_gate() -> SearchGate {
        SearchGate::new(std::time::Duration::ZERO, crate::search_gate::BACKOFF)
    }

    /// The provider's ad links (its `/y.js` click tracker) are not results.
    #[test]
    fn parse_ddg_drops_the_providers_ad_links() {
        let html = r#"<div class="result"><a class="result__a" href="https://duckduckgo.com/y.js?ad_domain=abebooks.com&amp;u3=x">Buy John Adams</a><a class="result__snippet">Compare prices.</a></div><div class="result"><a class="result__a" href="//duckduckgo.com/l/?uddg=https%3A%2F%2Fen.wikipedia.org%2Fwiki%2FJohn_Adams">John Adams - Wikipedia</a></div>"#;
        let found = parse_ddg("John Adams", html, 5).unwrap();
        let urls: Vec<_> = found.results.iter().map(|r| r.url.as_str()).collect();
        assert_eq!(urls, ["https://en.wikipedia.org/wiki/John_Adams"]);
    }

    /// The search asks for the result page the way the provider's own form
    /// does, with a form POST carrying the query, and reads its results.
    #[tokio::test]
    async fn ddg_search_posts_the_form_and_reads_its_results() {
        use wiremock::matchers::{body_string, header, method, path};
        use wiremock::{Mock, MockServer, ResponseTemplate};
        let _env = crate::openrouter::test_environment_scope_async().await;
        let server = MockServer::start().await;
        Mock::given(method("POST"))
            .and(path("/html/"))
            .and(header("content-type", "application/x-www-form-urlencoded"))
            .and(body_string("q=John+Adams+%26+co&b="))
            .respond_with(ResponseTemplate::new(200).set_body_string(DDG_PAGE))
            .expect(1)
            .mount(&server)
            .await;
        let found = ddg_search_at(
            &unpaced_gate(),
            &format!("{}/html/", server.uri()),
            "John Adams & co",
            5,
        )
        .await
        .unwrap();
        assert_eq!(found.results.len(), 1);
        assert_eq!(found.results[0].url, "https://example.com/adams");
    }

    /// A throttled result page — the challenge page with the 202 the
    /// provider sends it with, or with 200 — is an error naming the
    /// throttle, never a search that found nothing; and the provider is then
    /// left alone for the back-off window.
    #[tokio::test]
    async fn a_throttled_ddg_search_is_an_error_and_backs_off() {
        use wiremock::matchers::method;
        use wiremock::{Mock, MockServer, ResponseTemplate};
        let _env = crate::openrouter::test_environment_scope_async().await;
        for status in [202, 200] {
            let server = MockServer::start().await;
            Mock::given(method("POST"))
                .respond_with(ResponseTemplate::new(status).set_body_string(DDG_CHALLENGE))
                .expect(1)
                .mount(&server)
                .await;
            let gate = unpaced_gate();
            let endpoint = format!("{}/html/", server.uri());
            let first = ddg_search_at(&gate, &endpoint, "John Adams", 5)
                .await
                .unwrap_err()
                .to_string();
            assert!(first.contains("throttl"), "{status}: {first}");
            if status == 202 {
                assert!(first.contains("HTTP 202"), "{first}");
            } else {
                assert!(first.contains("challenge page"), "{first}");
            }
            let second = ddg_search_at(&gate, &endpoint, "Thomas Jefferson", 5)
                .await
                .unwrap_err()
                .to_string();
            assert!(second.contains("not asking it again"), "{status}: {second}");
        }
    }

    // --- Parslee search HTTP path (wiremock). Exercises the real
    // org-lookup → search request/parse chain, including the 401 surface
    // that #318 was about (a lapsed token). ---

    #[tokio::test]
    async fn parslee_search_resolves_org_then_returns_results() {
        use wiremock::matchers::{method, path};
        use wiremock::{Mock, MockServer, ResponseTemplate};
        // `parslee_search` builds (or reuses) the process-wide client, and
        // client construction reads the trust store. Every `TrustStoreScope`
        // test points `SSL_CERT_FILE` at an unparseable fixture for its
        // duration, so without this lock a concurrent one makes the build fail
        // here as `Http("builder error")` — a failure about certificates
        // masquerading as a search bug. Any test that constructs a reqwest
        // client needs this guard.
        let _env = crate::openrouter::test_environment_scope_async().await;
        let server = MockServer::start().await;
        Mock::given(method("GET"))
            .and(path("/api/v1/organizations/me"))
            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
                "organizationId": "org_test"
            })))
            .mount(&server)
            .await;
        Mock::given(method("POST"))
            .and(path("/api/v1/orgs/org_test/search"))
            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
                "results": [
                    { "title": "T", "url": "https://example.com", "snippet": "S", "score": 0.9 }
                ]
            })))
            .mount(&server)
            .await;

        let resp = parslee_search("q", 5, &server.uri(), "bearer-xyz")
            .await
            .expect("search should succeed");
        assert_eq!(resp.source, "parslee");
        assert_eq!(resp.results.len(), 1);
        assert_eq!(resp.results[0].url, "https://example.com");
    }

    #[tokio::test]
    async fn parslee_search_surfaces_401_on_org_lookup() {
        use wiremock::matchers::{method, path};
        use wiremock::{Mock, MockServer, ResponseTemplate};
        // Same reason as the sibling above: this builds the shared client.
        let _env = crate::openrouter::test_environment_scope_async().await;
        let server = MockServer::start().await;
        Mock::given(method("GET"))
            .and(path("/api/v1/organizations/me"))
            .respond_with(ResponseTemplate::new(401))
            .mount(&server)
            .await;

        let err = parslee_search("q", 5, &server.uri(), "stale-token")
            .await
            .expect_err("a 401 org lookup must surface as an error");
        match err {
            SearchError::Http(m) => assert!(m.contains("HTTP 401"), "got: {m}"),
            other => panic!("expected Http(401), got {other:?}"),
        }
    }

    // --- One trust-store parse per process ---------------------------------

    #[tokio::test]
    async fn http_client_builds_once_and_is_reused() {
        use crate::tls_client::test_seam::{TrustStoreScope, VALID_CERT_PEM};

        // A controlled valid fixture rather than the host's own store, so the
        // result does not depend on the machine this runs on.
        let _scope = TrustStoreScope::acquire_async(VALID_CERT_PEM).await;
        reset_http_client_cache_for_test();

        let _first = http_client().expect("first call builds");
        let _second = http_client().expect("second call is served from the cache");
        assert_eq!(
            http_client_build_count(),
            1,
            "the second call must not re-read the OS trust store"
        );
        assert!(http_client_is_cached());
        reset_http_client_cache_for_test();
    }

    #[tokio::test]
    async fn http_client_surfaces_trust_failure_without_caching_it() {
        use crate::tls_client::test_seam::{TrustStoreScope, UNPARSEABLE_CERT_PEM, VALID_CERT_PEM};

        // Depends on `cargo nextest run -p car-inference` (process per test)
        // plus the cache reset below. Under a plain `cargo test` the whole
        // crate shares one process, where either wiremock test above could
        // populate the cache first and hand this one a cache hit that never
        // observes the failure. nextest is the gate.
        let mut scope = TrustStoreScope::acquire_async(UNPARSEABLE_CERT_PEM).await;
        scope.assert_breaks_client_construction();
        reset_http_client_cache_for_test();

        let err = http_client().expect_err("a broken trust store must surface, not panic");
        assert!(matches!(err, SearchError::Http(_)), "got: {err:?}");
        assert!(
            !http_client_is_cached(),
            "a construction failure must never be cached — the next search has to retry"
        );

        // A repaired machine recovers on the very next call, with no restart.
        // Proved by pointing the override at a *second* controlled file, never
        // by unsetting it: unsetting resolves the host's real store, which is
        // exactly the machine coupling these tests exist to avoid.
        scope.repoint(VALID_CERT_PEM);
        let _client = http_client().expect("the next call after a repair must succeed");
        assert!(http_client_is_cached());
        assert_eq!(
            http_client_build_count(),
            2,
            "one failed build plus one successful rebuild"
        );
        reset_http_client_cache_for_test();
    }
}