thndrs_lib/core/

search.rs

//! Application-owned web search and article extraction.
//!
//! Search backends are deliberately kept outside provider adapters. DuckDuckGo
//! is the default public backend; an explicitly configured SearXNG instance is
//! also supported, while `none` disables this boundary. Search results are
//! normalized before their public pages are fetched through the same bounded
//! Lectito-backed extraction path used by `read_url`.
//!
//! - Search: sync HTTP via [`ureq`] against `html.duckduckgo.com/html/`.
//! - Parsing: ported from lectito's mcp bin, using [`scraper`] for CSS
//!   selectors.
//! - Extraction: [`lectito::extract`] for HTML → Markdown/text.
//! - Bot-challenge detection: checks for known DDG anomaly markers.
//! - Result limits: kept small (default 5, hard maximum 10).

use std::io;
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
use std::time::{Duration, Instant};

use crate::cli::WebSearchMode;
use scraper::{Html, Selector};
use ureq::unversioned::resolver::{DefaultResolver, ResolvedSocketAddrs, Resolver};
use ureq::unversioned::transport::{DefaultConnector, NextTimeout};

/// Maximum number of local search results returned by default.
pub const DEFAULT_SEARCH_LIMIT: usize = 5;
/// Hard maximum number of search results returned by one tool call.
pub const MAX_SEARCH_LIMIT: usize = 10;

/// DuckDuckGo's form-backed HTML search endpoint.
pub const DUCKDUCKGO_HTML_URL: &str = "https://html.duckduckgo.com/html/";

/// Maximum article content length before truncation.
const MAX_ARTICLE_CONTENT_LEN: usize = 65_536;

/// Maximum response body size for fetched URLs (1 MiB).
const MAX_RESPONSE_BYTES: usize = 1_048_576;

/// Maximum response body size accepted from a search backend.
const MAX_SEARCH_RESPONSE_BYTES: usize = 512 * 1024;

/// Maximum combined output emitted by one `web_search` call.
pub const MAX_SEARCH_OUTPUT_BYTES: usize = 65_536;

/// Maximum number of HTTP redirects to follow. ureq's default is 10; we tighten
/// this to keep redirect chains short and bounded.
const MAX_REDIRECTS: u32 = 5;

/// Hard timeout (seconds) for the entire `read_url` fetch: DNS, connect, TLS,
/// redirects, and body read. Prevents a slow or malicious server from hanging
/// the agent loop.
const FETCH_TIMEOUT_SECS: u64 = 15;

/// Marker used to distinguish a resolver-level public-network rejection from
/// unrelated I/O permission failures.
const PRIVATE_RESOLUTION_ERROR: &str = "resolved host includes a non-public network address";

/// User agent string for DuckDuckGo requests.
const USER_AGENT: &str = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)  \
     AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36";

type Result<T> = std::result::Result<T, SearchError>;

/// Errors from local search or extraction.
#[derive(Debug, thiserror::Error)]
pub enum SearchError {
    /// HTTP transport error.
    #[error("http error: {0}")]
    Http(String),
    /// Non-success HTTP status.
    #[error("http {status}: {body}")]
    HttpStatus { status: u16, body: String },
    /// DuckDuckGo returned an anti-bot page instead of search results.
    #[error("bot challenge: {0}")]
    Blocked(String),
    /// A search backend returned malformed JSON.
    #[error("invalid search JSON: {0}")]
    Json(String),
    /// The configured search backend is incomplete or invalid.
    #[error("invalid search configuration: {0}")]
    InvalidConfiguration(String),
    /// The application-owned web search backend is disabled.
    #[error("web search is disabled")]
    Disabled,
    /// Lectito extraction failed.
    #[error("extraction error: {0}")]
    Extraction(String),
    /// A hard-coded selector failed to parse.
    #[error("invalid CSS selector: {0}")]
    InvalidSelector(&'static str),
    /// The URL scheme is not `http` or `https`.
    #[error("unsupported URL scheme: {0}")]
    UnsupportedScheme(String),
    /// The URL points to a private/loopback network address.
    #[error("private network target rejected: {0}")]
    PrivateNetwork(String),
    /// The redirect chain exceeded the configured limit.
    #[error("too many redirects (max {max})")]
    TooManyRedirects { max: u32 },
    /// The request did not complete within the timeout.
    #[error("request timed out after {secs}s")]
    Timeout { secs: u64 },
    /// The response exceeded the maximum allowed size.
    #[error("response too large (>{max} bytes)")]
    Oversized { max: usize },
    /// The response content type is not HTML.
    #[error("unexpected content type: {0}")]
    BadContentType(String),
}

/// Resolver wrapper that validates the exact addresses handed to the socket
/// connector. Rejecting the whole answer when any address is non-public avoids
/// DNS rebinding and mixed public/private answer ambiguity.
#[derive(Debug)]
struct PublicResolver<R> {
    inner: R,
}

impl<R> PublicResolver<R> {
    fn new(inner: R) -> Self {
        Self { inner }
    }
}

impl<R: Resolver> Resolver for PublicResolver<R> {
    fn resolve(
        &self, uri: &ureq::http::Uri, config: &ureq::config::Config, timeout: NextTimeout,
    ) -> std::result::Result<ResolvedSocketAddrs, ureq::Error> {
        let addresses = self.inner.resolve(uri, config, timeout)?;
        if addresses.iter().any(|address| !is_public_ip(address.ip())) {
            return Err(ureq::Error::Io(io::Error::new(
                io::ErrorKind::PermissionDenied,
                PRIVATE_RESOLUTION_ERROR,
            )));
        }
        Ok(addresses)
    }
}

/// One result from DuckDuckGo's HTML search page.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct SearchResult {
    /// The result title as displayed by DuckDuckGo.
    pub title: String,
    /// The normalized target URL.
    pub url: String,
    /// DuckDuckGo's result snippet, when present.
    pub snippet: Option<String>,
}

/// Configuration consumed by the application-owned search tool.
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct SearchConfig {
    /// Selected application-owned backend.
    pub backend: WebSearchMode,
    /// Base URL for SearXNG, when that backend is selected.
    pub searxng_url: Option<String>,
}

impl Default for SearchConfig {
    fn default() -> Self {
        Self { backend: WebSearchMode::DuckDuckGo, searxng_url: None }
    }
}

impl SearchConfig {
    /// Construct and validate a backend configuration.
    pub fn new(backend: WebSearchMode, searxng_url: Option<String>) -> Result<Self> {
        let config = Self { backend, searxng_url: searxng_url.map(|url| url.trim().to_string()) };
        config.validate()?;
        Ok(config)
    }

    /// Construct a configuration from already merged application settings.
    /// Runtime validation remains at the tool boundary so malformed CLI-only
    /// combinations still become structured tool errors.
    pub fn from_parts(backend: WebSearchMode, searxng_url: Option<String>) -> Self {
        Self { backend, searxng_url: searxng_url.map(|url| url.trim().to_string()) }
    }

    /// Validate the selected backend and any required SearXNG base URL.
    pub fn validate(&self) -> Result<()> {
        if self.backend != WebSearchMode::Searxng {
            return Ok(());
        }

        let Some(base_url) = self.searxng_url.as_deref().filter(|url| !url.is_empty()) else {
            return Err(SearchError::InvalidConfiguration(
                "searxng requires an HTTP(S) base URL".to_string(),
            ));
        };
        validate_search_base_url(base_url)
    }
}

/// One normalized result plus the best-effort extracted page content.
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct SearchResultContent {
    /// Normalized search result metadata.
    pub result: SearchResult,
    /// Extracted page content when the result URL could be fetched safely.
    pub content: Option<FetchedContent>,
    /// Per-result extraction error, retained so later results can succeed.
    pub error: Option<String>,
}

/// Extracted article content.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct ArticleContent {
    /// Article title, if detected.
    pub title: String,
    /// Markdown-formatted content.
    pub markdown: String,
    /// Plain text content.
    pub text_content: String,
    /// Whether the content was truncated to fit the size cap.
    pub truncated: bool,
}

/// Content fetched from a public URL.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct FetchedContent {
    /// The final URL after redirects.
    pub final_url: String,
    /// HTTP status code of the final response.
    pub status: u16,
    /// The page title (from Lectito extraction, if available).
    pub title: String,
    /// Markdown-formatted content.
    pub markdown: String,
    /// Plain text content.
    pub text_content: String,
    /// Whether the content was truncated.
    pub truncated: bool,
    /// Diagnostics: redirects followed, content-type seen, limits applied.
    pub diagnostics: Vec<String>,
}

/// Check whether a URL string uses a public scheme (`http` or `https`).
pub fn is_public_scheme(url_str: &str) -> bool {
    url_str.starts_with("http://") || url_str.starts_with("https://")
}

/// Check whether a URL has a literal non-public network address or a localhost
/// hostname. Domain names are resolved and checked at connection time by the
/// fetcher's public-address resolver.
pub fn is_private_url(url_str: &str) -> bool {
    let Ok(parsed) = url::Url::parse(url_str) else {
        return true;
    };

    let scheme = parsed.scheme();
    if scheme != "http" && scheme != "https" {
        return true;
    }

    let host = match parsed.host_str() {
        Some(h) => h,
        None => return true,
    };

    if host.eq_ignore_ascii_case("localhost") || host.eq_ignore_ascii_case("localhost.") {
        return true;
    }

    match parsed.host() {
        Some(url::Host::Ipv4(v4)) => !is_public_ipv4(v4),
        Some(url::Host::Ipv6(v6)) => !is_public_ipv6(v6),
        Some(url::Host::Domain(_)) => false,
        None => true,
    }
}

/// Fetch a public URL and extract readable content.
///
/// ## Safety guards
///
/// - Only `http`/`https` schemes are allowed.
/// - Literal and DNS-resolved non-public addresses are rejected before a
///   connection is opened.
/// - Redirects are followed one at a time and every target is validated before
///   its request is sent.
/// - At most `MAX_REDIRECTS` redirects are followed; the chain errors on excess.
/// - The entire request is bounded by a `FETCH_TIMEOUT_SECS` global timeout.
/// - Response size is capped at `MAX_RESPONSE_BYTES`, enforced *while streaming*
///   so a large body cannot exhaust memory before the cap triggers.
/// - Content type must be on the [`allowed_content_kind`] allow-list: HTML/XHTML
///   is extracted via Lectito; other text types (JSON, XML, plain text, feeds,
///   YAML, CSV, JS) are returned as raw text. Binary types are rejected.
pub fn fetch_url(url_str: &str) -> Result<FetchedContent> {
    fetch_url_with_agent_factory(url_str, public_fetch_agent)
}

fn fetch_url_with_agent_factory(
    url_str: &str, mut agent_factory: impl FnMut(Duration) -> ureq::Agent,
) -> Result<FetchedContent> {
    if !is_public_scheme(url_str) {
        return Err(SearchError::UnsupportedScheme(url_str.to_string()));
    }
    if is_private_url(url_str) {
        return Err(SearchError::PrivateNetwork(url_str.to_string()));
    }

    let started = Instant::now();
    let timeout = Duration::from_secs(FETCH_TIMEOUT_SECS);
    let mut current = url::Url::parse(url_str).map_err(|error| SearchError::Http(error.to_string()))?;
    let mut redirect_count = 0;

    let response = loop {
        let remaining = timeout
            .checked_sub(started.elapsed())
            .filter(|remaining| !remaining.is_zero())
            .ok_or(SearchError::Timeout { secs: FETCH_TIMEOUT_SECS })?;
        let agent = agent_factory(remaining);
        let response = request_public_url(&agent, current.as_str())?;

        let Some(next) = redirect_target(&current, &response)? else {
            break response;
        };
        if redirect_count >= MAX_REDIRECTS {
            return Err(SearchError::TooManyRedirects { max: MAX_REDIRECTS });
        }
        validate_public_url(next.as_str())?;
        current = next;
        redirect_count += 1;
    };

    let final_url = current.to_string();

    let status = response.status().as_u16();
    let content_type = response
        .headers()
        .get("Content-Type")
        .and_then(|v| v.to_str().ok())
        .unwrap_or("")
        .to_string();

    let kind = allowed_content_kind(&content_type).ok_or_else(|| SearchError::BadContentType(content_type.clone()))?;

    let body_result = response
        .into_body()
        .with_config()
        .limit(MAX_RESPONSE_BYTES as u64)
        .read_to_string();

    let (body, body_truncated) = match body_result {
        Ok(s) => (s, false),
        Err(ureq::Error::BodyExceedsLimit(_)) => (String::new(), true),
        Err(e) => return Err(SearchError::Http(e.to_string())),
    };

    let content = process_body(&body, &final_url, kind)?;

    let truncated = body_truncated || content.truncated;

    let mut diagnostics = vec![
        format!("status: {status}"),
        format!("content_type: {content_type}"),
        format!("redirects_followed: {redirect_count}"),
        format!("max_redirects: {MAX_REDIRECTS}"),
        format!("timeout_secs: {FETCH_TIMEOUT_SECS}"),
        format!("max_bytes: {MAX_RESPONSE_BYTES}"),
    ];
    if truncated {
        diagnostics.push("truncated: true".to_string());
    }
    if url_str != final_url {
        diagnostics.push(format!("redirected: {url_str} -> {final_url}"));
    }

    Ok(FetchedContent {
        final_url,
        status,
        title: content.title,
        markdown: content.markdown,
        text_content: content.text_content,
        truncated,
        diagnostics,
    })
}

fn public_fetch_agent(timeout: Duration) -> ureq::Agent {
    let config = ureq::Agent::config_builder()
        .max_redirects(0)
        // A proxy can resolve the destination itself and bypass the guarded
        // resolver. Public URL fetching therefore always connects directly.
        .proxy(None)
        .timeout_global(Some(timeout))
        .build();
    ureq::Agent::with_parts(
        config,
        DefaultConnector::default(),
        PublicResolver::new(DefaultResolver::default()),
    )
}

fn request_public_url(agent: &ureq::Agent, url: &str) -> Result<ureq::http::Response<ureq::Body>> {
    match agent
        .get(url)
        .header("User-Agent", USER_AGENT)
        .header("Accept", ALLOWED_ACCEPT_HEADER)
        .call()
    {
        Ok(response) => Ok(response),
        Err(ureq::Error::StatusCode(code)) => Err(SearchError::HttpStatus { status: code, body: String::new() }),
        Err(ureq::Error::Timeout(_)) => Err(SearchError::Timeout { secs: FETCH_TIMEOUT_SECS }),
        Err(ureq::Error::BodyExceedsLimit(limit)) => Err(SearchError::Oversized { max: limit as usize }),
        Err(ureq::Error::Io(error)) if is_private_resolution_error(&error) => {
            Err(SearchError::PrivateNetwork(url.to_string()))
        }
        Err(error) => Err(SearchError::Http(error.to_string())),
    }
}

fn redirect_target(current: &url::Url, response: &ureq::http::Response<ureq::Body>) -> Result<Option<url::Url>> {
    if !response.status().is_redirection() {
        return Ok(None);
    }
    let Some(location) = response.headers().get("Location") else {
        return Ok(None);
    };
    let location = location
        .to_str()
        .map_err(|error| SearchError::Http(format!("invalid redirect location: {error}")))?;
    let next = current
        .join(location)
        .map_err(|error| SearchError::Http(format!("invalid redirect location: {error}")))?;
    Ok(Some(next))
}

fn validate_public_url(url_str: &str) -> Result<()> {
    if !is_public_scheme(url_str) {
        return Err(SearchError::UnsupportedScheme(url_str.to_string()));
    }
    if is_private_url(url_str) {
        return Err(SearchError::PrivateNetwork(url_str.to_string()));
    }
    Ok(())
}

fn is_private_resolution_error(error: &io::Error) -> bool {
    error.kind() == io::ErrorKind::PermissionDenied && error.to_string() == PRIVATE_RESOLUTION_ERROR
}

/// Classify a `Content-Type` header value into a [`ContentKind`] on the allow-list.
///
/// Returns `None` for binary types (images, audio, video, archives, octet-stream),
/// unrecognized types, and non-text application types not explicitly listed.
///
/// The check is deliberately permissive about parameters (`; charset=utf-8`) and
/// tolerates `+json` / `+xml` suffixes (`application/feed+json`, `application/atom+xml`).
pub fn allowed_content_kind(content_type: &str) -> Option<ContentKind> {
    let essence = content_type.split(';').next().unwrap_or("").trim().to_ascii_lowercase();
    if essence.is_empty() {
        return None;
    }

    if essence.starts_with("text/") {
        return Some(html_kind(&essence));
    }

    if let Some(sub) = essence.strip_prefix("application/") {
        if sub == "html" || sub == "xhtml+xml" {
            return Some(ContentKind::Html);
        }

        if sub == "json" || sub.ends_with("+json") {
            return Some(ContentKind::Text);
        }

        if sub == "xml" || sub.ends_with("+xml") {
            return Some(ContentKind::Text);
        }

        if matches!(
            sub,
            "javascript" | "x-javascript" | "yaml" | "x-yaml" | "x-www-form-urlencoded"
        ) {
            return Some(ContentKind::Text);
        }
    }

    None
}

/// Map a `text/*` essence to the right kind (HTML vs plain text).
fn html_kind(essence: &str) -> ContentKind {
    match essence {
        "text/html" | "text/xhtml" => ContentKind::Html,
        _ => ContentKind::Text,
    }
}

/// Process a fetched body according to its [`ContentKind`].
///
/// HTML/XHTML is run through Lectito readability extraction; other text types
/// are returned as raw text with the title derived from the URL path. This is
/// the no-network, fixture-testable core of [`fetch_url`].
pub fn process_body(body: &str, final_url: &str, kind: ContentKind) -> Result<ProcessedContent> {
    match kind {
        ContentKind::Html => {
            let article = extract_article(body, Some(final_url))?;
            match article {
                Some(a) => Ok(ProcessedContent {
                    title: a.title,
                    markdown: cap_text(&a.markdown, MAX_ARTICLE_CONTENT_LEN),
                    text_content: cap_text(&a.text_content, MAX_ARTICLE_CONTENT_LEN),
                    truncated: a.truncated,
                }),

                None => Ok(ProcessedContent {
                    title: title_from_url(final_url),
                    markdown: cap_text(body, MAX_ARTICLE_CONTENT_LEN),
                    text_content: cap_text(body, MAX_ARTICLE_CONTENT_LEN),
                    truncated: body.len() > MAX_ARTICLE_CONTENT_LEN,
                }),
            }
        }
        ContentKind::Text => Ok(ProcessedContent {
            title: title_from_url(final_url),
            markdown: cap_text(body, MAX_ARTICLE_CONTENT_LEN),
            text_content: cap_text(body, MAX_ARTICLE_CONTENT_LEN),
            truncated: body.len() > MAX_ARTICLE_CONTENT_LEN,
        }),
    }
}

/// Derive a best-effort title from the final URL path.
fn title_from_url(url_str: &str) -> String {
    let Ok(parsed) = url::Url::parse(url_str) else {
        return String::new();
    };
    let path = parsed.path();
    let last = path.rsplit('/').find(|s| !s.is_empty()).unwrap_or("");
    percent_decode(last).unwrap_or_default()
}

/// Content category after allow-list classification.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum ContentKind {
    /// HTML/XHTML — run through Lectito readability extraction.
    Html,
    /// Other text types (JSON, XML, plain text, feeds, YAML, JS) — raw body.
    Text,
}

/// Processed body content, independent of transport details.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct ProcessedContent {
    /// Derived or extracted title.
    pub title: String,
    /// Markdown-formatted content.
    pub markdown: String,
    /// Plain text content.
    pub text_content: String,
    /// Whether the content was truncated to fit the size cap.
    pub truncated: bool,
}

/// Comma-separated `Accept` header value advertising the allow-listed types.
const ALLOWED_ACCEPT_HEADER: &str = "text/html, application/xhtml+xml, text/plain, \
    text/markdown, text/css, text/csv, text/xml, application/json, application/xml, \
    application/javascript, application/yaml, application/rss+xml, application/atom+xml, \
    application/feed+json, */+json, */+xml;q=0.5";

/// Search DuckDuckGo and return up to `limit` parsed results.
///
/// Uses a sync `ureq` POST to `html.duckduckgo.com/html/`. Empty queries and
/// zero limits return an empty result set without a network request. The
/// request shares the bounded fetch timeout so a stalled search cannot hold an
/// agent worker indefinitely.
pub fn search_duckduckgo(query: &str, limit: usize) -> Result<Vec<SearchResult>> {
    let query = query.trim();
    if query.is_empty() || limit == 0 {
        return Ok(Vec::new());
    }

    let form = url::form_urlencoded::Serializer::new(String::new())
        .append_pair("q", query)
        .append_pair("b", "")
        .append_pair("l", "us-en")
        .finish();

    let agent = duckduckgo_agent();
    let response = agent
        .post(DUCKDUCKGO_HTML_URL)
        .header("User-Agent", USER_AGENT)
        .header("Accept", "text/html,application/xhtml+xml")
        .header("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")
        .send(&form);

    let response = match response {
        Ok(r) => r,
        Err(ureq::Error::Timeout(_)) => return Err(SearchError::Timeout { secs: FETCH_TIMEOUT_SECS }),
        Err(e) => return Err(SearchError::Http(e.to_string())),
    };
    let (status, body) = read_response_body(response, MAX_SEARCH_RESPONSE_BYTES)?;
    if status >= 400 {
        return Err(SearchError::HttpStatus { status, body: body.chars().take(500).collect() });
    }
    parse_duckduckgo_html(&body, cap_search_limit(limit))
}

/// Search with the configured application-owned backend.
pub fn search_with_config(config: &SearchConfig, query: &str, limit: usize) -> Result<Vec<SearchResult>> {
    config.validate()?;
    let limit = cap_search_limit(limit);
    match config.backend {
        WebSearchMode::DuckDuckGo => search_duckduckgo(query, limit),
        WebSearchMode::Searxng => search_searxng(config.searxng_url.as_deref().unwrap_or_default(), query, limit),
        WebSearchMode::None => Err(SearchError::Disabled),
    }
}

// TODO: Add Exa as another application-owned backend when its API contract is
// stable enough to fit this normalized result boundary.
// TODO: Add Tavily as another application-owned backend behind the same caps.
// TODO: Add Brave as another application-owned backend without reintroducing
// provider-specific request payloads or headers.

/// Search and best-effort fetch each selected result page in result order.
pub fn search_and_extract(config: &SearchConfig, query: &str, limit: usize) -> Result<Vec<SearchResultContent>> {
    let results = search_with_config(config, query, limit)?;
    Ok(extract_result_pages(results, fetch_url))
}

/// Extract result pages with a caller-supplied fetcher.
///
/// The real tool passes [`fetch_url`]. Keeping this small boundary injectable
/// makes ordering, partial-success, limit, and SSRF-policy behavior testable
/// without relying on public internet pages.
pub fn extract_result_pages<F>(results: Vec<SearchResult>, mut fetcher: F) -> Vec<SearchResultContent>
where
    F: FnMut(&str) -> Result<FetchedContent>,
{
    results
        .into_iter()
        .map(|result| match fetcher(&result.url) {
            Ok(content) => SearchResultContent { result, content: Some(content), error: None },
            Err(error) => SearchResultContent { result, content: None, error: Some(error.to_string()) },
        })
        .collect()
}

/// Search a SearXNG JSON endpoint and return normalized results.
///
/// SearXNG's configured base URL may be loopback or private because it is an
/// explicit local service. URLs returned in its result list are not trusted;
/// page extraction still goes through [`fetch_url`] and its public-target
/// checks.
pub fn search_searxng(base_url: &str, query: &str, limit: usize) -> Result<Vec<SearchResult>> {
    let query = query.trim();
    if query.is_empty() || limit == 0 {
        return Ok(Vec::new());
    }
    let endpoint = searxng_search_url(base_url)?;
    let mut url = endpoint;
    url.query_pairs_mut()
        .append_pair("q", query)
        .append_pair("format", "json");

    let response = searxng_agent()
        .get(url.as_str())
        .header("User-Agent", USER_AGENT)
        .header("Accept", "application/json")
        .config()
        .http_status_as_error(false)
        .build()
        .call()
        .map_err(|error| match error {
            ureq::Error::TooManyRedirects => SearchError::TooManyRedirects { max: MAX_REDIRECTS },
            ureq::Error::Timeout(_) => SearchError::Timeout { secs: FETCH_TIMEOUT_SECS },
            other => SearchError::Http(other.to_string()),
        })?;
    let (status, body) = read_response_body(response, MAX_SEARCH_RESPONSE_BYTES)?;
    if status >= 400 {
        return Err(SearchError::HttpStatus { status, body: body.chars().take(500).collect() });
    }
    parse_searxng_json(&body, cap_search_limit(limit))
}

/// Parse SearXNG's normalized JSON result envelope.
pub fn parse_searxng_json(json: &str, limit: usize) -> Result<Vec<SearchResult>> {
    if limit == 0 {
        return Ok(Vec::new());
    }
    let value =
        serde_json::from_str::<serde_json::Value>(json).map_err(|error| SearchError::Json(error.to_string()))?;
    let Some(results) = value.get("results").and_then(serde_json::Value::as_array) else {
        return Err(SearchError::Json(
            "response did not contain a results array".to_string(),
        ));
    };

    let mut normalized = Vec::new();
    for result in results {
        let Some(url) = result.get("url").and_then(serde_json::Value::as_str).map(str::trim) else {
            continue;
        };
        let Some(url) = normalize_result_url(url) else {
            continue;
        };
        let title = result
            .get("title")
            .and_then(serde_json::Value::as_str)
            .map(clean_text)
            .filter(|title| !title.is_empty())
            .unwrap_or_else(|| url.clone());
        let snippet = result
            .get("content")
            .or_else(|| result.get("snippet"))
            .and_then(serde_json::Value::as_str)
            .map(clean_text)
            .filter(|snippet| !snippet.is_empty());
        normalized.push(SearchResult { title, url, snippet });
        if normalized.len() >= limit {
            break;
        }
    }
    Ok(normalized)
}

/// Parse DuckDuckGo HTML search results.
///
/// Detects the common bot-challenge page before returning results. Normalizes
/// DuckDuckGo redirect links (`/l/?uddg=...`) into their target URLs.
pub fn parse_duckduckgo_html(html: &str, limit: usize) -> Result<Vec<SearchResult>> {
    if limit == 0 {
        return Ok(Vec::new());
    }

    if is_bot_challenge(html) {
        return Err(SearchError::Blocked(
            "DuckDuckGo returned a bot challenge instead of search results".to_string(),
        ));
    }

    let document = Html::parse_document(html);
    let result_selector = selector(".result")?;
    let title_selector = selector(".result__title a, a.result__a")?;
    let snippet_selector = selector(".result__snippet")?;
    let url_selector = selector(".result__url")?;
    let mut results = Vec::new();

    for result in document.select(&result_selector) {
        let Some(link) = result.select(&title_selector).next() else {
            continue;
        };
        let Some(href) = link.value().attr("href") else {
            continue;
        };

        let title = clean_text(&link.text().collect::<Vec<_>>().join(" "));
        if title.is_empty() {
            continue;
        }

        let snippet = result
            .select(&snippet_selector)
            .next()
            .map(|node| clean_text(&node.text().collect::<Vec<_>>().join(" ")))
            .filter(|text| !text.is_empty());
        let fallback_url = result
            .select(&url_selector)
            .next()
            .map(|node| clean_text(&node.text().collect::<Vec<_>>().join(" ")))
            .filter(|text| !text.is_empty());

        match normalize_duckduckgo_url(href).or(fallback_url) {
            Some(url) => {
                results.push(SearchResult { title, url, snippet });
                if results.len() >= limit {
                    break;
                }
            }
            None => continue,
        }
    }

    Ok(results)
}

/// Detect DuckDuckGo bot-challenge / anomaly pages.
///
/// Checks for known markers that DDG uses when it suspects automated traffic.
pub fn is_bot_challenge(html: &str) -> bool {
    html.contains("anomaly-modal")
        || html.contains("Unfortunately, bots use DuckDuckGo too")
        || html.contains("/anomaly.js")
}

/// Extract readable content from already-fetched HTML using Lectito.
///
/// Returns the article title, Markdown content, and a truncation flag.
/// Returns `None` if the page is not probably readable.
pub fn extract_article(html: &str, base_url: Option<&str>) -> Result<Option<ArticleContent>> {
    let options = lectito::ReadabilityOptions::default();
    let article = lectito::extract(html, base_url, &options).map_err(|e| SearchError::Extraction(e.to_string()))?;

    match article {
        Some(a) => Ok(Some(ArticleContent {
            title: a.title.unwrap_or_default(),
            markdown: cap_text(&a.markdown, MAX_ARTICLE_CONTENT_LEN),
            text_content: cap_text(&a.text_content, MAX_ARTICLE_CONTENT_LEN),
            truncated: a.markdown.len() > MAX_ARTICLE_CONTENT_LEN,
        })),
        None => Ok(None),
    }
}

/// Format search results as transcript output lines.
pub fn format_search_results(results: &[SearchResult]) -> Vec<String> {
    let content = results
        .iter()
        .cloned()
        .map(|result| SearchResultContent { result, content: None, error: None })
        .collect::<Vec<_>>();
    format_search_results_with_content(&content)
}

/// Format normalized results and best-effort page extraction with a total cap.
pub fn format_search_results_with_content(results: &[SearchResultContent]) -> Vec<String> {
    let mut output = String::new();
    for (index, result) in results.iter().enumerate() {
        let snippet = result.result.snippet.as_deref().unwrap_or("(no snippet)");
        let mut lines = vec![
            format!("{}. {} — {}", index + 1, result.result.title, snippet),
            format!("   url: {}", result.result.url),
        ];
        if let Some(content) = &result.content {
            lines.push(format!("   extracted_title: {}", content.title));
            lines.push(format!("   final_url: {}", content.final_url));
            if content.truncated {
                lines.push("   extracted_content: (truncated)".to_string());
            }
            lines.extend(content.markdown.lines().map(|line| format!("   {line}")));
            if !content.diagnostics.is_empty() {
                lines.push(format!("   diagnostics: {}", content.diagnostics.join(", ")));
            }
        } else if let Some(error) = &result.error {
            lines.push(format!("   extraction_error: {error}"));
        }

        for line in lines {
            let line = cap_text(&line, MAX_SEARCH_OUTPUT_BYTES);
            if output.len() + line.len() + 1 > MAX_SEARCH_OUTPUT_BYTES {
                output.push_str("[search output truncated]\n");
                return output.lines().map(str::to_string).collect();
            }
            output.push_str(&line);
            output.push('\n');
        }
    }
    output.lines().map(str::to_string).collect()
}

/// Create the bounded transport used by the local DuckDuckGo fallback.
fn duckduckgo_agent() -> ureq::Agent {
    let config = ureq::Agent::config_builder()
        .max_redirects(MAX_REDIRECTS)
        .max_redirects_will_error(true)
        .timeout_global(Some(Duration::from_secs(FETCH_TIMEOUT_SECS)))
        .build();
    ureq::Agent::new_with_config(config)
}

/// Create the bounded transport used by a configured SearXNG backend.
fn searxng_agent() -> ureq::Agent {
    ureq::Agent::config_builder()
        .max_redirects(MAX_REDIRECTS)
        .max_redirects_will_error(true)
        .timeout_global(Some(Duration::from_secs(FETCH_TIMEOUT_SECS)))
        .build()
        .new_agent()
}

fn searxng_search_url(base_url: &str) -> Result<url::Url> {
    validate_search_base_url(base_url)?;
    let mut base = url::Url::parse(base_url.trim_end_matches('/'))
        .map_err(|error| SearchError::InvalidConfiguration(error.to_string()))?;
    let path = format!("{}/search", base.path().trim_end_matches('/'));
    base.set_path(if path == "/search" { "/search" } else { &path });
    Ok(base)
}

fn validate_search_base_url(base_url: &str) -> Result<()> {
    let parsed = url::Url::parse(base_url)
        .map_err(|error| SearchError::InvalidConfiguration(format!("invalid base URL: {error}")))?;
    if !matches!(parsed.scheme(), "http" | "https") {
        return Err(SearchError::InvalidConfiguration(
            "SearXNG base URL must use HTTP or HTTPS".to_string(),
        ));
    }
    if parsed.host_str().is_none() {
        return Err(SearchError::InvalidConfiguration(
            "SearXNG base URL must include a host".to_string(),
        ));
    }
    if parsed.query().is_some() || parsed.fragment().is_some() {
        return Err(SearchError::InvalidConfiguration(
            "SearXNG base URL must not include a query or fragment".to_string(),
        ));
    }
    Ok(())
}

fn normalize_result_url(value: &str) -> Option<String> {
    let parsed = url::Url::parse(value).ok()?;
    if !matches!(parsed.scheme(), "http" | "https") || parsed.host_str().is_none() {
        return None;
    }
    Some(parsed.to_string())
}

fn cap_search_limit(limit: usize) -> usize {
    limit.min(MAX_SEARCH_LIMIT)
}

fn read_response_body(response: ureq::http::Response<ureq::Body>, limit: usize) -> Result<(u16, String)> {
    let status = response.status().as_u16();
    let body = response
        .into_body()
        .with_config()
        .limit(limit as u64)
        .read_to_string()
        .map_err(|error| match error {
            ureq::Error::BodyExceedsLimit(_) => SearchError::Oversized { max: limit },
            ureq::Error::Timeout(_) => SearchError::Timeout { secs: FETCH_TIMEOUT_SECS },
            other => SearchError::Http(other.to_string()),
        })?;
    Ok((status, body))
}

fn cap_text(text: &str, max_bytes: usize) -> String {
    if text.len() <= max_bytes {
        return text.to_string();
    }
    let mut end = max_bytes.saturating_sub("…".len());
    while !text.is_char_boundary(end) {
        end = end.saturating_sub(1);
    }
    format!("{}…", &text[..end])
}

fn is_public_ip(ip: IpAddr) -> bool {
    match ip {
        IpAddr::V4(ip) => is_public_ipv4(ip),
        IpAddr::V6(ip) => is_public_ipv6(ip),
    }
}

/// Return whether an IPv4 address is globally routable. The explicit special
/// ranges keep this compatible with the project's Rust 1.88 MSRV.
fn is_public_ipv4(ip: Ipv4Addr) -> bool {
    let [a, b, c, _] = ip.octets();
    !(a == 0
        || a == 10
        || a == 127
        || (a == 100 && (64..=127).contains(&b))
        || (a == 169 && b == 254)
        || (a == 172 && (16..=31).contains(&b))
        || (a == 192 && b == 0 && c == 0)
        || (a == 192 && b == 0 && c == 2)
        || (a == 192 && b == 88 && c == 99)
        || (a == 192 && b == 168)
        || (a == 198 && (b == 18 || b == 19))
        || (a == 198 && b == 51 && c == 100)
        || (a == 203 && b == 0 && c == 113)
        || a >= 224)
}

/// Return whether an IPv6 address is globally routable. IPv4-mapped addresses
/// inherit the IPv4 classification; local, documentation, benchmarking, and
/// transition ranges are rejected conservatively.
fn is_public_ipv6(ip: Ipv6Addr) -> bool {
    if let Some(ipv4) = ip.to_ipv4_mapped() {
        return is_public_ipv4(ipv4);
    }

    let segments = ip.segments();
    let first = segments[0];
    let second = segments[1];
    let global_unicast = (first & 0xe000) == 0x2000;
    let teredo = first == 0x2001 && second == 0;
    let benchmarking = first == 0x2001 && second == 2 && segments[2] == 0;
    let orchid = first == 0x2001 && (second & 0xfff0 == 0x0010 || second & 0xfff0 == 0x0020);
    let documentation = (first == 0x2001 && second == 0x0db8) || (first == 0x3fff && second & 0xf000 == 0);

    global_unicast && !(teredo || benchmarking || orchid || documentation || first == 0x2002)
}

/// Minimal percent-decoding without an extra dependency.
fn percent_decode(s: &str) -> Option<String> {
    let mut result = String::with_capacity(s.len());
    let bytes = s.as_bytes();
    let mut i = 0;
    while i < bytes.len() {
        if bytes[i] == b'%'
            && i + 2 < bytes.len()
            && let (Some(hi), Some(lo)) = (hex_val(bytes[i + 1]), hex_val(bytes[i + 2]))
        {
            result.push(char::from_u32(hi * 16 + lo).unwrap_or('?'));
            i += 3;
            continue;
        }
        if bytes[i] == b'+' {
            result.push(' ');
        } else {
            result.push(s[i..].chars().next().unwrap_or('?'));
        }
        i += 1;
    }
    Some(result)
}

fn selector(css: &'static str) -> Result<Selector> {
    Selector::parse(css).map_err(|_| SearchError::InvalidSelector(css))
}

fn clean_text(text: &str) -> String {
    text.split_whitespace().collect::<Vec<_>>().join(" ")
}

fn normalize_duckduckgo_url(href: &str) -> Option<String> {
    let decoded = html_unescape(href);
    if decoded.starts_with("http://") || decoded.starts_with("https://") {
        return Some(decoded);
    }

    if let Some(query_start) = decoded.find("uddg=") {
        let encoded = &decoded[query_start + 5..];
        let end = encoded.find('&').unwrap_or(encoded.len());
        return percent_decode(&encoded[..end]);
    }

    let base = url::Url::parse(DUCKDUCKGO_HTML_URL).ok()?;
    Some(base.join(&decoded).ok()?.to_string())
}

fn hex_val(b: u8) -> Option<u32> {
    match b {
        b'0'..=b'9' => Some((b - b'0') as u32),
        b'a'..=b'f' => Some((b - b'a' + 10) as u32),
        b'A'..=b'F' => Some((b - b'A' + 10) as u32),
        _ => None,
    }
}

fn html_unescape(text: &str) -> String {
    text.replace("&amp;", "&")
        .replace("&lt;", "<")
        .replace("&gt;", ">")
        .replace("&quot;", "\"")
        .replace("&#x27;", "'")
}

#[cfg(test)]
mod tests {
    use std::io::{Read, Write};
    use std::net::{SocketAddr, TcpListener};
    use std::sync::Arc;
    use std::sync::atomic::{AtomicUsize, Ordering};

    use super::*;

    #[derive(Clone, Debug)]
    struct FixedResolver {
        address: SocketAddr,
    }

    impl Resolver for FixedResolver {
        fn resolve(
            &self, _uri: &ureq::http::Uri, _config: &ureq::config::Config, _timeout: NextTimeout,
        ) -> std::result::Result<ResolvedSocketAddrs, ureq::Error> {
            let mut addresses = self.empty();
            addresses.push(self.address);
            Ok(addresses)
        }
    }

    fn test_agent(timeout: Duration, resolver: impl Resolver) -> ureq::Agent {
        let config = ureq::Agent::config_builder()
            .max_redirects(0)
            .timeout_global(Some(timeout))
            .build();
        ureq::Agent::with_parts(config, DefaultConnector::default(), resolver)
    }

    fn spawn_http_response(response: String) -> (String, std::thread::JoinHandle<()>) {
        let listener = TcpListener::bind(("127.0.0.1", 0)).expect("bind test server");
        let address = listener.local_addr().expect("test server address");
        let handle = std::thread::spawn(move || {
            let Ok((mut stream, _)) = listener.accept() else {
                return;
            };
            let mut request = [0_u8; 1024];
            let _ = stream.read(&mut request);
            let _ = stream.write_all(response.as_bytes());
        });
        (format!("http://{address}"), handle)
    }

    #[test]
    fn parse_duckduckgo_html_extracts_results() {
        let html = r#"
            <div class="result">
              <h2 class="result__title">
                <a class="result__a" href="//duckduckgo.com/l/?uddg=https%3A%2F%2Fexample.com%2Fpost%3Fx%3D1&amp;rut=abc">
                  Example Result
                </a>
              </h2>
              <a class="result__url">example.com/post</a>
              <a class="result__snippet">A compact result snippet.</a>
            </div>
        "#;

        let results = parse_duckduckgo_html(html, 10).expect("html parses");
        assert_eq!(results.len(), 1);
        assert_eq!(results[0].title, "Example Result");
        assert_eq!(results[0].url, "https://example.com/post?x=1");
        assert_eq!(results[0].snippet.as_deref(), Some("A compact result snippet."));
    }

    #[test]
    fn parse_duckduckgo_html_respects_limit() {
        let html = r#"
            <div class="result"><a class="result__a" href="https://a.test">A</a></div>
            <div class="result"><a class="result__a" href="https://b.test">B</a></div>
        "#;

        let results = parse_duckduckgo_html(html, 1).expect("html parses");
        assert_eq!(results.len(), 1);
        assert_eq!(results[0].title, "A");
    }

    #[test]
    fn parse_duckduckgo_html_detects_bot_challenge() {
        let html = r#"
            <form id="challenge-form" action="//duckduckgo.com/anomaly.js">
              <div class="anomaly-modal__title">
                Unfortunately, bots use DuckDuckGo too.
              </div>
            </form>
        "#;

        let error = parse_duckduckgo_html(html, 10).expect_err("challenge is an error");
        assert!(matches!(error, SearchError::Blocked(_)));
    }

    #[test]
    fn is_bot_challenge_detects_anomaly_markers() {
        assert!(is_bot_challenge("anomaly-modal test"));
        assert!(is_bot_challenge("Unfortunately, bots use DuckDuckGo too"));
        assert!(is_bot_challenge("/anomaly.js"));
        assert!(!is_bot_challenge("normal search results"));
    }

    #[test]
    fn is_bot_challenge_false_for_normal_html() {
        let html = r#"<div class="result"><a href="https://example.com">Normal</a></div>"#;
        assert!(!is_bot_challenge(html));
    }

    #[test]
    fn empty_query_returns_empty_results() {
        let results = parse_duckduckgo_html("", 10).expect("empty html");
        assert!(results.is_empty());
    }

    #[test]
    fn zero_limit_returns_empty_results() {
        let html = r#"<div class="result"><a href="https://example.com">Test</a></div>"#;
        let results = parse_duckduckgo_html(html, 0).expect("zero limit");
        assert!(results.is_empty());
    }

    #[test]
    fn normalize_duckduckgo_url_resolves_redirect() {
        let url = normalize_duckduckgo_url("//duckduckgo.com/l/?uddg=https%3A%2F%2Fexample.com%2Fpost");
        assert_eq!(url.as_deref(), Some("https://example.com/post"));
    }

    #[test]
    fn normalize_duckduckgo_url_passes_through_absolute() {
        let url = normalize_duckduckgo_url("https://example.com/direct");
        assert_eq!(url.as_deref(), Some("https://example.com/direct"));
    }

    #[test]
    fn default_search_limit_is_small() {
        let limit = DEFAULT_SEARCH_LIMIT;
        assert!(limit <= 10, "search limit should be small");
        assert!(limit >= 3, "search limit should be useful");
    }

    #[test]
    fn format_search_results_produces_lines() {
        let lines = format_search_results(&[
            SearchResult {
                title: "Rust Async".to_string(),
                url: "https://tokio.rs".to_string(),
                snippet: Some("Async runtime".to_string()),
            },
            SearchResult {
                title: "Async Book".to_string(),
                url: "https://rust-lang.org/async".to_string(),
                snippet: None,
            },
        ]);
        assert_eq!(lines.len(), 4);
        assert!(lines[0].contains("Rust Async"));
        assert!(lines[1].contains("tokio.rs"));
        assert!(lines[2].contains("Async Book"));
    }

    #[test]
    fn extract_article_returns_none_for_empty_html() {
        assert!(
            extract_article("<html><body></body></html>", None)
                .expect("should not error")
                .is_none()
        );
    }

    #[test]
    fn extract_article_returns_content_for_readable_html() {
        let html = r#"
            <article>
                <h1>Test Article</h1>
                <p>This is a readable article with enough content to pass readability checks.
                It has multiple sentences and proper structure for extraction.</p>
                <p>Second paragraph with more content to ensure the article is detected
                as readable by the Lectito algorithm.</p>
            </article>
        "#;
        let result = extract_article(html, Some("https://example.com/post")).expect("should extract");
        assert!(result.is_some(), "should extract readable article");

        let article = result.unwrap();
        assert!(!article.markdown.is_empty());
        assert!(!article.text_content.is_empty());
    }

    #[test]
    fn is_private_url_table() {
        let rejected: &[(&str, &str)] = &[
            ("localhost", "http://localhost:8080/test"),
            ("localhost dot", "https://localhost./path"),
            ("loopback ipv4", "http://127.0.0.1/test"),
            ("loopback ipv4 high", "http://127.255.255.255/test"),
            ("private 10.x", "http://10.0.0.1/test"),
            ("private 10.x high", "http://10.255.255.255/test"),
            ("private 172.16.x", "http://172.16.0.1/test"),
            ("private 172.31.x high", "http://172.31.255.255/test"),
            ("private 192.168.x", "http://192.168.1.1/test"),
            ("private 192.168.0", "http://192.168.0.0/test"),
            ("link-local", "http://169.254.1.1/test"),
            ("link-local metadata", "http://169.254.169.254/latest/meta-data"),
            ("shared address space", "http://100.64.0.1/test"),
            ("benchmarking", "http://198.18.0.1/test"),
            ("documentation", "http://203.0.113.10/test"),
            ("multicast", "http://224.0.0.1/test"),
            ("zero address", "http://0.0.0.0/test"),
            ("ipv6 loopback", "http://[::1]/test"),
            ("ipv6 mapped loopback", "http://[::ffff:127.0.0.1]/test"),
            ("ipv6 documentation", "http://[2001:db8::1]/test"),
            ("ipv6 documentation 3fff", "http://[3fff::1]/test"),
            ("ipv6 unique local", "http://[fd00::1]/test"),
            ("ipv6 reserved", "http://[4000::1]/test"),
            ("file scheme", "file:///etc/passwd"),
            ("ftp scheme", "ftp://example.com/file"),
            ("javascript scheme", "javascript:alert(1)"),
            ("unparseable", "not a url"),
            ("empty", ""),
        ];
        for (label, url) in rejected {
            assert!(
                is_private_url(url),
                "{label}: expected private/rejected, got allowed: {url}"
            );
        }

        let allowed: &[(&str, &str)] = &[
            ("public domain", "https://example.com/article"),
            ("public ipv4", "http://93.184.216.34/test"),
            ("public ipv6", "https://[2606:2800:220:1:248:1893:25c8:1946]/test"),
            ("public blog", "https://blog.rust-lang.org/2024/01/01/post"),
        ];
        for (label, url) in allowed {
            assert!(!is_private_url(url), "{label}: expected allowed, got rejected: {url}");
        }
    }

    #[test]
    fn is_public_scheme_checks_prefix() {
        assert!(is_public_scheme("http://example.com"));
        assert!(is_public_scheme("https://example.com"));
        assert!(!is_public_scheme("file:///etc/passwd"));
        assert!(!is_public_scheme("ftp://example.com"));
    }

    #[test]
    fn fetch_url_rejects_non_public_scheme() {
        let result = fetch_url("file:///etc/passwd");
        assert!(matches!(result, Err(SearchError::UnsupportedScheme(_))));
    }

    #[test]
    fn fetch_url_rejects_private_network() {
        let result = fetch_url("http://127.0.0.1:8080/secret");
        assert!(matches!(result, Err(SearchError::PrivateNetwork(_))));

        let result = fetch_url("http://localhost/admin");
        assert!(matches!(result, Err(SearchError::PrivateNetwork(_))));
    }

    #[test]
    fn fetch_url_rejects_domain_that_resolves_to_private_address_before_connecting() {
        let listener = TcpListener::bind(("127.0.0.1", 0)).expect("bind test server");
        listener.set_nonblocking(true).expect("nonblocking test server");
        let address = listener.local_addr().expect("test server address");
        let url = format!("http://public.test:{}/secret", address.port());

        let result = fetch_url_with_agent_factory(&url, |timeout| {
            test_agent(timeout, PublicResolver::new(FixedResolver { address }))
        });

        assert!(matches!(result, Err(SearchError::PrivateNetwork(target)) if target == url));
        let error = listener
            .accept()
            .expect_err("private destination must not receive a connection");
        assert_eq!(error.kind(), io::ErrorKind::WouldBlock);
    }

    #[test]
    fn fetch_url_rejects_private_redirect_before_second_request() {
        let listener = TcpListener::bind(("127.0.0.1", 0)).expect("bind test server");
        let address = listener.local_addr().expect("test server address");
        let requests = Arc::new(AtomicUsize::new(0));
        let requests_for_server = requests.clone();
        let handle = std::thread::spawn(move || {
            let (mut stream, _) = listener.accept().expect("first request");
            requests_for_server.fetch_add(1, Ordering::SeqCst);
            let mut request = [0_u8; 1024];
            let _ = stream.read(&mut request);
            let response = format!(
                "HTTP/1.1 302 Found\r\nLocation: http://127.0.0.1:{}/secret\r\nContent-Length: 0\r\nConnection: close\r\n\r\n",
                address.port()
            );
            stream.write_all(response.as_bytes()).expect("redirect response");
            listener.set_nonblocking(true).expect("nonblocking test server");
            for _ in 0..20 {
                match listener.accept() {
                    Ok((_stream, _)) => {
                        requests_for_server.fetch_add(1, Ordering::SeqCst);
                        break;
                    }
                    Err(error) if error.kind() == io::ErrorKind::WouldBlock => {
                        std::thread::sleep(Duration::from_millis(5));
                    }
                    Err(_) => break,
                }
            }
        });
        let url = format!("http://public.test:{}/start", address.port());

        let result = fetch_url_with_agent_factory(&url, |timeout| test_agent(timeout, FixedResolver { address }));

        assert!(matches!(result, Err(SearchError::PrivateNetwork(target)) if target.contains("127.0.0.1")));
        handle.join().expect("test server");
        assert_eq!(
            requests.load(Ordering::SeqCst),
            1,
            "redirect target must not receive a request"
        );
    }

    #[test]
    fn max_response_bytes_is_reasonable() {
        let max = MAX_RESPONSE_BYTES;
        assert!(max >= 65_536, "should allow at least 64 KiB");
        assert!(max <= 2_097_152, "should cap at 2 MiB");
    }

    #[test]
    fn max_redirects_is_bounded_and_small() {
        let max = MAX_REDIRECTS;
        assert!(max <= 5, "redirect limit should be small");
        assert!(max >= 1, "should follow at least one redirect");
    }

    #[test]
    fn fetch_timeout_is_bounded() {
        let secs = FETCH_TIMEOUT_SECS;
        assert!(secs <= 60, "timeout should be at most 60s");
        assert!(secs >= 5, "timeout should allow at least 5s");
    }

    #[test]
    fn duckduckgo_search_uses_the_bounded_fetch_timeout() {
        let timeouts = duckduckgo_agent().config().timeouts();
        assert_eq!(timeouts.global, Some(Duration::from_secs(FETCH_TIMEOUT_SECS)));
    }

    #[test]
    fn search_error_too_many_redirects_displays_message() {
        let max = MAX_REDIRECTS;
        let err = SearchError::TooManyRedirects { max };
        assert!(err.to_string().contains("too many redirects"));
        assert!(err.to_string().contains(&max.to_string()));
    }

    #[test]
    fn search_error_timeout_displays_message() {
        let secs = FETCH_TIMEOUT_SECS;
        let err = SearchError::Timeout { secs };
        assert!(err.to_string().contains("timed out"));
        assert!(err.to_string().contains(&secs.to_string()));
    }

    #[test]
    fn search_error_oversized_displays_message() {
        let err = SearchError::Oversized { max: 1024 };
        assert!(err.to_string().contains("too large"));
        assert!(err.to_string().contains("1024"));
    }

    #[test]
    fn allowed_content_kind_html() {
        assert_eq!(allowed_content_kind("text/html"), Some(ContentKind::Html));
        assert_eq!(
            allowed_content_kind("text/html; charset=utf-8"),
            Some(ContentKind::Html)
        );
        assert_eq!(allowed_content_kind("application/xhtml+xml"), Some(ContentKind::Html));
        assert_eq!(allowed_content_kind("TEXT/HTML"), Some(ContentKind::Html));
    }

    #[test]
    fn allowed_content_kind_text_family() {
        assert_eq!(allowed_content_kind("text/plain"), Some(ContentKind::Text));
        assert_eq!(
            allowed_content_kind("text/plain; charset=iso-8859-1"),
            Some(ContentKind::Text)
        );
        assert_eq!(allowed_content_kind("text/csv"), Some(ContentKind::Text));
        assert_eq!(allowed_content_kind("text/markdown"), Some(ContentKind::Text));
        assert_eq!(allowed_content_kind("text/css"), Some(ContentKind::Text));
        assert_eq!(allowed_content_kind("text/xml"), Some(ContentKind::Text));
    }

    #[test]
    fn allowed_content_kind_application_text_types() {
        assert_eq!(allowed_content_kind("application/json"), Some(ContentKind::Text));
        assert_eq!(
            allowed_content_kind("application/json; charset=utf-8"),
            Some(ContentKind::Text)
        );
        assert_eq!(allowed_content_kind("application/xml"), Some(ContentKind::Text));
        assert_eq!(allowed_content_kind("application/javascript"), Some(ContentKind::Text));
        assert_eq!(allowed_content_kind("application/yaml"), Some(ContentKind::Text));
        assert_eq!(allowed_content_kind("application/x-yaml"), Some(ContentKind::Text));
    }

    #[test]
    fn allowed_content_kind_suffixes() {
        assert_eq!(allowed_content_kind("application/feed+json"), Some(ContentKind::Text));
        assert_eq!(
            allowed_content_kind("application/vnd.api+json"),
            Some(ContentKind::Text)
        );
        assert_eq!(allowed_content_kind("application/atom+xml"), Some(ContentKind::Text));
        assert_eq!(allowed_content_kind("application/rss+xml"), Some(ContentKind::Text));
    }

    #[test]
    fn allowed_content_kind_rejects_binary_and_unknown() {
        assert_eq!(allowed_content_kind("image/png"), None);
        assert_eq!(allowed_content_kind("application/octet-stream"), None);
        assert_eq!(allowed_content_kind("application/pdf"), None);
        assert_eq!(allowed_content_kind("application/zip"), None);
        assert_eq!(allowed_content_kind("audio/mpeg"), None);
        assert_eq!(allowed_content_kind("video/mp4"), None);
        assert_eq!(allowed_content_kind("application/octet-stream; charset=binary"), None);
        assert_eq!(allowed_content_kind(""), None);
        assert_eq!(allowed_content_kind("garbage"), None);
    }

    #[test]
    fn process_body_html_uses_lectito_extraction() {
        let html = r#"
            <article>
                <h1>Test Article</h1>
                <p>This is a readable article with enough content to pass readability checks.
                It has multiple sentences and proper structure for extraction.</p>
                <p>Second paragraph with more content to ensure the article is detected
                as readable by the Lectito algorithm.</p>
            </article>
        "#;
        let result = process_body(html, "https://example.com/post", ContentKind::Html).expect("html should process");
        assert!(!result.markdown.is_empty());
        assert!(!result.text_content.is_empty());
        assert!(!result.truncated);
    }

    #[test]
    fn process_body_html_unreadable_falls_back_to_raw() {
        let html = "<html><body></body></html>";
        let result = process_body(html, "https://example.com/empty", ContentKind::Html)
            .expect("unreadable html should not error");
        assert_eq!(result.markdown, html);
        assert_eq!(result.text_content, html);
    }

    #[test]
    fn process_body_text_returns_raw_with_url_title() {
        let body = r#"{"name": "thndrs", "version": "0.1.0"}"#;
        let result = process_body(body, "https://example.com/package.json", ContentKind::Text)
            .expect("json should process as text");
        assert_eq!(result.markdown, body);
        assert_eq!(result.text_content, body);
        assert_eq!(result.title, "package.json");
        assert!(!result.truncated);
    }

    #[test]
    fn process_body_text_title_from_url_path() {
        let result = process_body("plain", "https://example.com/docs/readme.txt", ContentKind::Text)
            .expect("text should process");
        assert_eq!(result.title, "readme.txt");
    }

    #[test]
    fn process_body_text_truncation_flag() {
        let body = "a".repeat(MAX_ARTICLE_CONTENT_LEN + 100);
        let result =
            process_body(&body, "https://example.com/big.txt", ContentKind::Text).expect("large text should process");
        assert!(result.truncated);
    }

    #[test]
    fn process_body_text_title_percent_decoded() {
        let result = process_body("body", "https://example.com/path/my%20file.txt", ContentKind::Text)
            .expect("text should process");
        assert_eq!(result.title, "my file.txt");
    }

    #[test]
    fn parse_searxng_json_normalizes_results_and_caps_limit() {
        let json = r#"{
            "results": [
                {"title":"First","url":"https://example.com/one","content":" first   snippet "},
                {"title":"Private","url":"http://127.0.0.1:8080/secret","content":"local"},
                {"title":"Third","url":"https://example.com/three","content":"third"}
            ]
        }"#;

        let results = parse_searxng_json(json, 2).expect("SearXNG JSON parses");
        assert_eq!(results.len(), 2);
        assert_eq!(results[0].title, "First");
        assert_eq!(results[0].snippet.as_deref(), Some("first snippet"));
        assert_eq!(results[1].url, "http://127.0.0.1:8080/secret");
    }

    #[test]
    fn parse_searxng_json_rejects_malformed_and_missing_results() {
        assert!(matches!(parse_searxng_json("{not json}", 5), Err(SearchError::Json(_))));
        assert!(matches!(parse_searxng_json("{}", 5), Err(SearchError::Json(_))));
    }

    #[test]
    fn parse_searxng_json_accepts_empty_results() {
        assert!(
            parse_searxng_json(r#"{"results":[]}"#, 5)
                .expect("empty response parses")
                .is_empty()
        );
    }

    #[test]
    fn searxng_configuration_requires_http_base_url_but_allows_loopback() {
        assert!(SearchConfig::new(WebSearchMode::Searxng, None).is_err());
        assert!(SearchConfig::new(WebSearchMode::Searxng, Some("ftp://localhost".to_string())).is_err());
        assert!(SearchConfig::new(WebSearchMode::Searxng, Some("http://127.0.0.1:8080".to_string())).is_ok());
    }

    #[test]
    fn searxng_url_uses_search_path_and_query_parameters() {
        let url = searxng_search_url("http://127.0.0.1:8080/searxng/").expect("base URL");
        assert_eq!(url.as_str(), "http://127.0.0.1:8080/searxng/search");
        let config =
            SearchConfig::new(WebSearchMode::Searxng, Some("http://127.0.0.1:8080".to_string())).expect("config");
        assert!(matches!(
            search_with_config(&config, "", 5),
            Ok(results) if results.is_empty()
        ));
    }

    #[test]
    fn extract_result_pages_preserves_order_and_partial_failures() {
        let results = vec![
            SearchResult { title: "one".to_string(), url: "https://example.com/one".to_string(), snippet: None },
            SearchResult { title: "private".to_string(), url: "http://127.0.0.1/secret".to_string(), snippet: None },
            SearchResult { title: "three".to_string(), url: "https://example.com/three".to_string(), snippet: None },
        ];
        let extracted = extract_result_pages(results, |url| {
            if is_private_url(url) {
                Err(SearchError::PrivateNetwork(url.to_string()))
            } else {
                Ok(FetchedContent {
                    final_url: url.to_string(),
                    status: 200,
                    title: "page".to_string(),
                    markdown: "content".to_string(),
                    text_content: "content".to_string(),
                    truncated: false,
                    diagnostics: vec!["fake".to_string()],
                })
            }
        });
        assert_eq!(extracted.len(), 3);
        assert_eq!(extracted[0].result.title, "one");
        assert!(extracted[0].content.is_some());
        assert!(extracted[1].content.is_none());
        assert!(
            extracted[1]
                .error
                .as_deref()
                .is_some_and(|error| error.contains("private network"))
        );
        assert!(extracted[2].content.is_some());
    }

    #[test]
    fn formatted_search_output_is_capped() {
        let result = SearchResultContent {
            result: SearchResult {
                title: "large".to_string(),
                url: "https://example.com/large".to_string(),
                snippet: None,
            },
            content: Some(FetchedContent {
                final_url: "https://example.com/large".to_string(),
                status: 200,
                title: "large".to_string(),
                markdown: "x".repeat(MAX_SEARCH_OUTPUT_BYTES * 2),
                text_content: String::new(),
                truncated: true,
                diagnostics: Vec::new(),
            }),
            error: None,
        };
        let lines = format_search_results_with_content(&[result]);
        let bytes: usize = lines.iter().map(|line| line.len() + 1).sum();
        assert!(bytes <= MAX_SEARCH_OUTPUT_BYTES + "[search output truncated]\n".len());
        assert!(lines.iter().any(|line| line.contains("truncated")));
    }

    #[test]
    fn search_backend_errors_are_structured() {
        let config =
            SearchConfig::new(WebSearchMode::Searxng, Some("http://127.0.0.1:1".to_string())).expect("loopback config");
        let error = search_with_config(&config, "rust", 1).expect_err("unreachable instance");
        assert!(matches!(error, SearchError::Http(_) | SearchError::Timeout { .. }));
        assert!(
            SearchError::Timeout { secs: FETCH_TIMEOUT_SECS }
                .to_string()
                .contains("timed out")
        );
    }

    #[test]
    fn searxng_http_errors_preserve_status_and_bounded_body() {
        let body = r#"{"error":"temporarily unavailable"}"#;
        let response = format!(
            "HTTP/1.1 503 Service Unavailable\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
            body.len(),
            body
        );
        let (base_url, handle) = spawn_http_response(response);
        let error = search_searxng(&base_url, "rust", 5).expect_err("HTTP error should be structured");
        handle.join().expect("test server thread");
        assert!(matches!(
            error,
            SearchError::HttpStatus { status: 503, body } if body.contains("temporarily unavailable")
        ));
    }

    #[test]
    fn searxng_local_server_returns_normalized_results() {
        let body = r#"{"results":[{"title":"Rust","url":"https://www.rust-lang.org/","content":"A language"},{"title":"Tokio","url":"https://tokio.rs/","snippet":"Async runtime"}]}"#;
        let response = format!(
            "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
            body.len(),
            body
        );
        let (base_url, handle) = spawn_http_response(response);
        let results = search_searxng(&base_url, "rust", 20).expect("local SearXNG response");
        handle.join().expect("test server thread");
        assert_eq!(results.len(), 2);
        assert_eq!(results[0].title, "Rust");
        assert_eq!(results[0].snippet.as_deref(), Some("A language"));
        assert_eq!(results[1].title, "Tokio");
    }

    #[test]
    #[ignore = "requires public network access"]
    fn live_duckduckgo_smoke() {
        match search_duckduckgo("Rust programming language", 1) {
            Ok(results) => {
                assert!(!results.is_empty(), "DuckDuckGo should return at least one result");
                assert!(is_public_scheme(&results[0].url));
            }
            Err(SearchError::Blocked(message)) => {
                assert!(message.contains("bot challenge"));
            }
            Err(error) => panic!("unexpected DuckDuckGo smoke-test error: {error}"),
        }
    }
}