rust_scrapling 0.2.3

//! Helpers for scraping search-engine result pages.
//!
//! Currently this covers DuckDuckGo's HTML endpoint
//! (`html.duckduckgo.com/html/`), which has two quirks that silently break
//! naive scrapers:
//!
//! 1. On bot detection it returns **HTTP 202 with its homepage** instead of an
//!    error or CAPTCHA, so a plain `status()`/`is_blocked()` check misses it
//!    (see [`is_duckduckgo_blocked`]).
//! 2. Every organic result link is wrapped in an internal redirect
//!    (`//duckduckgo.com/l/?uddg=<percent-encoded-target>`); the real URL must
//!    be decoded from the `uddg` query parameter (see [`decode_duckduckgo_href`]).
//!
//! For reliable automated search, prefer Startpage or the DuckDuckGo Instant
//! Answers API — DuckDuckGo's HTML endpoint rate-limits aggressively even with
//! stealth headers.

use crate::fetchers::response::Response;

/// Whether a host is `duckduckgo.com` or one of its subdomains. An exact
/// host comparison — unlike a URL substring check, this cannot be fooled by
/// `evil.com/?ref=duckduckgo.com` or `duckduckgo.com.evil.com`.
fn is_duckduckgo_host(host: &str) -> bool {
    host == "duckduckgo.com" || host.ends_with(".duckduckgo.com")
}

/// Detect DuckDuckGo's soft-block, where it answers a search query with HTTP
/// 202 and its homepage HTML rather than results — a case the generic
/// [`Response::is_blocked`](crate::fetchers::response::Response::is_blocked)
/// cannot catch because 202 is nominally a success code.
///
/// Returns `false` for non-DuckDuckGo responses (exact host comparison, so a
/// URL that merely mentions duckduckgo.com is not treated as DuckDuckGo), so
/// it is safe to call on any response.
#[must_use]
pub fn is_duckduckgo_blocked(response: &Response) -> bool {
    let is_ddg = url::Url::parse(response.url())
        .ok()
        .and_then(|u| u.host_str().map(is_duckduckgo_host))
        .unwrap_or(false);
    if !is_ddg {
        return false;
    }
    // The documented soft-block signal: a 202 carrying the homepage.
    if response.status() == 202 {
        return true;
    }
    // Or a 2xx body that has no result links — i.e. the homepage was served
    // in place of results.
    response.is_success() && !response.text().contains("result__a")
}

/// Decode a DuckDuckGo result `href` to its real target URL.
///
/// DuckDuckGo wraps result links as `//duckduckgo.com/l/?uddg=<encoded>&rut=…`.
/// This prepends a scheme to protocol-relative hrefs, then extracts and
/// percent-decodes the `uddg` parameter. Any href that is not a DuckDuckGo
/// redirect is returned unchanged.
#[must_use]
pub fn decode_duckduckgo_href(href: &str) -> String {
    let full = if let Some(stripped) = href.strip_prefix("//") {
        format!("https://{}", stripped)
    } else {
        href.to_string()
    };

    if let Ok(parsed) = url::Url::parse(&full) {
        let is_ddg_redirect =
            parsed.host_str().map(is_duckduckgo_host).unwrap_or(false) && parsed.path() == "/l/";
        if is_ddg_redirect {
            if let Some((_, target)) = parsed.query_pairs().find(|(k, _)| k == "uddg") {
                return target.into_owned();
            }
        }
    }
    href.to_string()
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::collections::HashMap;

    fn resp(status: u16, url: &str, body: &str) -> Response {
        Response::new(
            status,
            "text/html".to_string(),
            body.to_string(),
            url.to_string(),
            HashMap::new(),
        )
    }

    #[test]
    fn decodes_ddg_redirect_href() {
        let href = "//duckduckgo.com/l/?uddg=https%3A%2F%2Fwww.andritz.com%2F&rut=abc";
        assert_eq!(decode_duckduckgo_href(href), "https://www.andritz.com/");
    }

    #[test]
    fn passes_through_non_ddg_href() {
        assert_eq!(
            decode_duckduckgo_href("https://example.com/page"),
            "https://example.com/page"
        );
    }

    #[test]
    fn passes_through_ddg_non_redirect_path() {
        // Not the /l/ redirect endpoint — leave it alone.
        let href = "//duckduckgo.com/about";
        assert_eq!(decode_duckduckgo_href(href), href);
    }

    #[test]
    fn detects_202_soft_block() {
        let r = resp(
            202,
            "https://html.duckduckgo.com/html/?q=x",
            "<html>home</html>",
        );
        assert!(is_duckduckgo_blocked(&r));
    }

    #[test]
    fn detects_homepage_without_results() {
        let r = resp(
            200,
            "https://html.duckduckgo.com/html/?q=x",
            "<html>no results here</html>",
        );
        assert!(is_duckduckgo_blocked(&r));
    }

    #[test]
    fn real_ddg_results_are_not_blocked() {
        let r = resp(
            200,
            "https://html.duckduckgo.com/html/?q=x",
            "<a class=\"result__a\" href=\"//duckduckgo.com/l/?uddg=https%3A%2F%2Fx\">X</a>",
        );
        assert!(!is_duckduckgo_blocked(&r));
    }

    #[test]
    fn non_ddg_response_is_never_ddg_blocked() {
        let r = resp(202, "https://example.com", "");
        assert!(!is_duckduckgo_blocked(&r));
    }
    #[test]
    fn url_substring_cannot_spoof_ddg_detection() {
        // Regression: the check used to be a URL substring match, so ANY
        // response whose URL merely mentioned duckduckgo.com (query param,
        // path, or a hostile registrable domain) was treated as DuckDuckGo
        // and could be misclassified as soft-blocked.
        for url in [
            "https://evil.com/?ref=duckduckgo.com",
            "https://duckduckgo.com.evil.com/html/?q=x",
            "https://example.com/duckduckgo.com/page",
        ] {
            let r = resp(202, url, "<html>home</html>");
            assert!(
                !is_duckduckgo_blocked(&r),
                "{url} must not be treated as DuckDuckGo"
            );
        }
        // Subdomains of the real host still are.
        let r = resp(
            202,
            "https://html.duckduckgo.com/html/?q=x",
            "<html></html>",
        );
        assert!(is_duckduckgo_blocked(&r));
    }
}