servo-fetch 0.15.1

//! URL discovery via sitemap parsing — no rendering required.

use std::collections::{HashSet, VecDeque};
use std::io::Read as _;
use std::ops::ControlFlow;
use std::time::Duration;

use tokio::time::Instant;
use url::Url;

use crate::robots::RobotsPolicy;
use crate::scope::{is_same_site, matches_scope, normalize_url};
use crate::{bridge, net};

const MAP_SITEMAP_MAX_BYTES: u64 = 50 * 1024 * 1024;
const MAP_SITEMAP_MAX_DECOMPRESSED: u64 = 10 * 1024 * 1024;
const MAP_GZIP_MAX_RATIO: u64 = 100;
const MAP_HTML_MAX_BYTES: u64 = 2 * 1024 * 1024;
const MAP_MAX_REDIRECTS: u8 = 5;
const MAP_MAX_SITEMAPS: usize = 200;
const MAP_MAX_INDEX_DEPTH: u8 = 5;
const MAP_MIN_FETCH_INTERVAL: Duration = Duration::from_millis(500);
const MAP_URL_MAX_LEN: usize = 2048;
const HTML_SNIFF_LEN: usize = 100;

/// Options for URL discovery (sitemap + link extraction, no rendering).
#[must_use = "options do nothing until passed to map()"]
#[derive(Debug, Clone)]
pub struct MapOptions {
    url: String,
    limit: usize,
    include: Vec<String>,
    exclude: Vec<String>,
    user_agent: Option<String>,
    timeout: u64,
    no_fallback: bool,
    headers: http::HeaderMap,
}

impl MapOptions {
    /// Create map options for the given URL.
    pub fn new(url: impl Into<String>) -> Self {
        Self {
            url: url.into(),
            limit: 5000,
            include: Vec::new(),
            exclude: Vec::new(),
            user_agent: None,
            timeout: 30,
            no_fallback: false,
            headers: http::HeaderMap::new(),
        }
    }

    /// Maximum number of URLs to discover.
    pub fn limit(mut self, n: usize) -> Self {
        self.limit = n;
        self
    }

    /// URL path glob patterns to include.
    pub fn include(mut self, patterns: &[&str]) -> Self {
        self.include = patterns.iter().map(|s| (*s).to_string()).collect();
        self
    }

    /// URL path glob patterns to exclude.
    pub fn exclude(mut self, patterns: &[&str]) -> Self {
        self.exclude = patterns.iter().map(|s| (*s).to_string()).collect();
        self
    }

    /// Override the User-Agent string.
    pub fn user_agent(mut self, ua: impl Into<String>) -> Self {
        self.user_agent = Some(ua.into());
        self
    }

    /// Timeout in seconds per HTTP request.
    pub fn timeout(mut self, secs: u64) -> Self {
        self.timeout = secs;
        self
    }

    /// Skip HTML link fallback if no sitemap is found.
    pub fn no_fallback(mut self, yes: bool) -> Self {
        self.no_fallback = yes;
        self
    }

    /// Custom request headers sent with every discovery request.
    pub fn headers(mut self, headers: http::HeaderMap) -> Self {
        self.headers = headers;
        self
    }
}

/// A discovered URL from sitemap or link extraction.
#[derive(Debug, Clone, serde::Serialize)]
pub struct MappedUrl {
    /// The discovered URL.
    pub url: String,
    /// Last modification date from sitemap, if available.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub lastmod: Option<String>,
}

/// Discover URLs on a site via sitemaps and link extraction (blocking).
pub fn map_blocking(opts: &MapOptions) -> crate::error::Result<Vec<MappedUrl>> {
    crate::runtime::block_on(map(opts)).map_err(|e| crate::error::Error::engine(e, None))?
}

/// Discover URLs on a site via sitemaps and link extraction.
pub async fn map(opts: &MapOptions) -> crate::error::Result<Vec<MappedUrl>> {
    let seed = net::validate_url(&opts.url)?;
    let include = if opts.include.is_empty() {
        None
    } else {
        Some(crate::scope::build_globset(&opts.include)?)
    };
    let exclude = if opts.exclude.is_empty() {
        None
    } else {
        Some(crate::scope::build_globset(&opts.exclude)?)
    };
    let config = MapConfig {
        seed,
        limit: opts.limit,
        include,
        exclude,
        user_agent: opts.user_agent.clone(),
        timeout: Duration::from_secs(opts.timeout),
        no_fallback: opts.no_fallback,
        headers: opts.headers.clone(),
    };
    let mut traversal = MapTraversal::new(config, crate::transfer::client()?, crate::transfer::raw_client()?);
    let mut urls = Vec::new();
    while let Some(url) = traversal.next_url().await {
        urls.push(url);
    }
    Ok(urls)
}

/// Options for URL discovery.
struct MapConfig {
    seed: Url,
    limit: usize,
    include: Option<globset::GlobSet>,
    exclude: Option<globset::GlobSet>,
    user_agent: Option<String>,
    timeout: Duration,
    no_fallback: bool,
    headers: http::HeaderMap,
}

impl MapConfig {
    fn headers(&self) -> crate::transfer::Headers {
        crate::transfer::Headers::new(&self.headers, self.user_agent.as_deref())
    }
}

enum MapPhase {
    Robots,
    Sitemaps(SitemapWalk),
    Fallback { robots: RobotsPolicy, links: VecDeque<Url> },
    Done,
}

/// Sitemap-phase state: the parsed robots policy plus the index walk.
struct SitemapWalk {
    robots: RobotsPolicy,
    queue: VecDeque<(Url, u8)>,
    entries: VecDeque<SitemapEntry>,
    depth: u8,
    fetched: usize,
}

impl SitemapWalk {
    fn budget_reached(&self) -> bool {
        self.fetched >= MAP_MAX_SITEMAPS
    }
}

struct MapTraversal {
    opts: MapConfig,
    client: reqwest::Client,
    sitemap_client: reqwest::Client,
    phase: MapPhase,
    visited: HashSet<String>,
    count: usize,
    last_fetch: Instant,
}

impl MapTraversal {
    fn new(opts: MapConfig, client: reqwest::Client, sitemap_client: reqwest::Client) -> Self {
        let now = Instant::now();
        Self {
            opts,
            client,
            sitemap_client,
            phase: MapPhase::Robots,
            visited: HashSet::new(),
            count: 0,
            last_fetch: now.checked_sub(MAP_MIN_FETCH_INTERVAL).unwrap_or(now),
        }
    }

    fn url_budget_reached(&self) -> bool {
        self.count >= self.opts.limit
    }

    fn admits_sitemap(&self, url: &Url, depth: u8) -> bool {
        depth <= MAP_MAX_INDEX_DEPTH && is_same_site(&self.opts.seed, url)
    }

    fn fallback_applies(&self) -> bool {
        self.count == 0 && !self.opts.no_fallback
    }

    /// Leave the sitemap walk: gather fallback links or finish outright.
    async fn enter_fallback(&mut self, robots: RobotsPolicy) -> MapPhase {
        if !self.fallback_applies() {
            return MapPhase::Done;
        }
        throttle(&mut self.last_fetch).await;
        let links = fetch_html(&self.client, &self.opts.seed, &self.opts)
            .await
            .map_or_else(VecDeque::new, |html| extract_links(&html, &self.opts.seed).into());
        MapPhase::Fallback { robots, links }
    }

    async fn next_url(&mut self) -> Option<MappedUrl> {
        loop {
            let step = match std::mem::replace(&mut self.phase, MapPhase::Done) {
                MapPhase::Robots => self.bootstrap_robots().await,
                MapPhase::Sitemaps(walk) => self.advance_sitemaps(walk).await,
                MapPhase::Fallback { robots, links } => self.drain_fallback(robots, links),
                MapPhase::Done => return None,
            };
            if let ControlFlow::Break(url) = step {
                return Some(url);
            }
        }
    }

    /// Fetch robots.txt and seed the sitemap queue.
    async fn bootstrap_robots(&mut self) -> ControlFlow<MappedUrl> {
        let robots = crate::robots::fetch(&self.client, &self.opts.seed, &self.opts.headers(), self.opts.timeout).await;
        let queue = discover_sitemaps(&robots, &self.opts.seed)
            .into_iter()
            .map(|url| (url, 0))
            .collect();
        self.phase = MapPhase::Sitemaps(SitemapWalk {
            robots,
            queue,
            entries: VecDeque::new(),
            depth: 0,
            fetched: 0,
        });
        ControlFlow::Continue(())
    }

    /// Drain parsed entries, then fetch the next admissible sitemap.
    async fn advance_sitemaps(&mut self, mut walk: SitemapWalk) -> ControlFlow<MappedUrl> {
        if self.url_budget_reached() {
            self.phase = self.enter_fallback(walk.robots).await;
            return ControlFlow::Continue(());
        }
        if let Some(entry) = walk.entries.pop_front() {
            return self.emit_entry(walk, entry);
        }
        if walk.budget_reached() {
            self.phase = self.enter_fallback(walk.robots).await;
            return ControlFlow::Continue(());
        }
        let Some((sitemap_url, depth)) = walk.queue.pop_front() else {
            self.phase = self.enter_fallback(walk.robots).await;
            return ControlFlow::Continue(());
        };
        if !self.admits_sitemap(&sitemap_url, depth) {
            self.phase = MapPhase::Sitemaps(walk);
            return ControlFlow::Continue(());
        }
        throttle(&mut self.last_fetch).await;
        walk.fetched += 1;
        if let Some(body) = fetch_sitemap(&self.sitemap_client, &sitemap_url, &self.opts).await {
            walk.entries = parse_sitemap(&body).into();
            walk.depth = depth;
        }
        self.phase = MapPhase::Sitemaps(walk);
        ControlFlow::Continue(())
    }

    /// Yield a validated sitemap URL or enqueue a nested sitemap index.
    fn emit_entry(&mut self, mut walk: SitemapWalk, entry: SitemapEntry) -> ControlFlow<MappedUrl> {
        match entry {
            SitemapEntry::Url { loc, lastmod } => {
                let mapped = validate_entry(
                    &loc,
                    lastmod,
                    &self.opts.seed,
                    &walk.robots,
                    &self.opts,
                    &mut self.visited,
                );
                self.phase = MapPhase::Sitemaps(walk);
                match mapped {
                    Some(url) => {
                        self.count += 1;
                        ControlFlow::Break(url)
                    }
                    None => ControlFlow::Continue(()),
                }
            }
            SitemapEntry::Sitemap { loc } => {
                if let Ok(url) = Url::parse(&loc) {
                    walk.queue.push_back((url, walk.depth + 1));
                }
                self.phase = MapPhase::Sitemaps(walk);
                ControlFlow::Continue(())
            }
        }
    }

    /// Pop extracted links until one validates; the phase stays Done once drained.
    fn drain_fallback(&mut self, robots: RobotsPolicy, mut links: VecDeque<Url>) -> ControlFlow<MappedUrl> {
        if self.url_budget_reached() {
            return ControlFlow::Continue(());
        }
        let Some(link) = links.pop_front() else {
            return ControlFlow::Continue(());
        };
        let mapped = validate_entry(
            link.as_str(),
            None,
            &self.opts.seed,
            &robots,
            &self.opts,
            &mut self.visited,
        );
        self.phase = MapPhase::Fallback { robots, links };
        match mapped {
            Some(url) => {
                self.count += 1;
                ControlFlow::Break(url)
            }
            None => ControlFlow::Continue(()),
        }
    }
}

fn discover_sitemaps(robots: &RobotsPolicy, seed: &Url) -> Vec<Url> {
    let mut urls = Vec::new();
    if let RobotsPolicy::Rules(rules) = robots {
        urls.extend(rules.sitemaps.iter().cloned());
    }
    if let Ok(default) = seed.join("/sitemap.xml")
        && !urls.contains(&default)
    {
        urls.push(default);
    }
    urls
}

async fn fetch_following_redirects(
    client: &reqwest::Client,
    url: &Url,
    seed: &Url,
    opts: &MapConfig,
) -> Option<reqwest::Response> {
    let mut current = url.clone();
    for _ in 0..MAP_MAX_REDIRECTS {
        let response = opts.headers().get(client, &current, opts.timeout).send().await.ok()?;
        let status = response.status().as_u16();
        if matches!(status, 301 | 302 | 303 | 307 | 308) {
            let location = response.headers().get("location")?.to_str().ok()?;
            let next = current.join(location).ok()?;
            if net::validate_url_with_policy(next.as_str(), bridge::engine_policy()).is_err()
                || !is_same_site(seed, &next)
            {
                return None;
            }
            current = next;
            continue;
        }
        if status >= 400 {
            return None;
        }
        return Some(response);
    }
    None
}

async fn fetch_sitemap(client: &reqwest::Client, url: &Url, opts: &MapConfig) -> Option<String> {
    let response = fetch_following_redirects(client, url, &opts.seed, opts).await?;
    let content_type = response
        .headers()
        .get("content-type")
        .and_then(|value| value.to_str().ok())
        .unwrap_or("");
    let is_gzip = url
        .path()
        .rsplit('/')
        .next()
        .and_then(|segment| std::path::Path::new(segment).extension())
        .is_some_and(|extension| extension.eq_ignore_ascii_case("gz"))
        || content_type.contains("gzip")
        || response
            .headers()
            .get("content-encoding")
            .and_then(|value| value.to_str().ok())
            .is_some_and(|value| value.contains("gzip"));
    let bytes = crate::transfer::collect_bounded(response, MAP_SITEMAP_MAX_BYTES).await?;

    if is_gzip {
        let mut decoded = Vec::new();
        flate2::read::GzDecoder::new(bytes.as_slice())
            .take(MAP_SITEMAP_MAX_DECOMPRESSED + 1)
            .read_to_end(&mut decoded)
            .ok()?;
        let decoded_len = u64::try_from(decoded.len()).ok()?;
        let compressed_len = u64::try_from(bytes.len()).ok()?;
        if decoded_len > MAP_SITEMAP_MAX_DECOMPRESSED || decoded_len > compressed_len.checked_mul(MAP_GZIP_MAX_RATIO)? {
            return None;
        }
        if looks_like_html(&decoded) {
            return None;
        }
        String::from_utf8(decoded).ok()
    } else {
        if looks_like_html(&bytes) {
            return None;
        }
        String::from_utf8(bytes).ok()
    }
}

fn looks_like_html(bytes: &[u8]) -> bool {
    const DOCTYPE: &[u8] = b"<!doctype";
    const HTML: &[u8] = b"<html";
    const BOM: &[u8] = b"\xef\xbb\xbf";
    let mut prefix = bytes.get(..HTML_SNIFF_LEN).unwrap_or(bytes);
    if prefix.starts_with(BOM) {
        prefix = &prefix[BOM.len()..];
    }
    let prefix = prefix
        .iter()
        .position(|b| !b.is_ascii_whitespace())
        .map_or(&[][..], |i| &prefix[i..]);
    prefix
        .get(..DOCTYPE.len())
        .is_some_and(|p| p.eq_ignore_ascii_case(DOCTYPE))
        || prefix.get(..HTML.len()).is_some_and(|p| p.eq_ignore_ascii_case(HTML))
}

async fn fetch_html(client: &reqwest::Client, url: &Url, opts: &MapConfig) -> Option<String> {
    let response = fetch_following_redirects(client, url, url, opts).await?;
    let bytes = crate::transfer::collect_bounded(response, MAP_HTML_MAX_BYTES).await?;
    Some(String::from_utf8_lossy(&bytes).into_owned())
}

fn extract_links(html: &str, base: &Url) -> Vec<Url> {
    dom_query::Document::from(html)
        .select("a[href]")
        .iter()
        .filter_map(|el| {
            let href = el.attr("href")?;
            let href = href.trim();
            if href.is_empty() {
                return None;
            }
            let resolved = base.join(href).ok()?;
            matches!(resolved.scheme(), "http" | "https").then_some(resolved)
        })
        .collect()
}

enum SitemapEntry {
    Url { loc: String, lastmod: Option<String> },
    Sitemap { loc: String },
}

fn parse_sitemap(body: &str) -> Vec<SitemapEntry> {
    use quick_xml::events::Event;
    use quick_xml::reader::Reader;

    let mut reader = Reader::from_str(body);
    let mut entries = Vec::new();
    let mut capture = Capture::Idle;
    let mut loc = String::new();
    let mut lastmod = String::new();
    let mut in_url = false;
    let mut in_sitemap = false;
    let mut depth: u32 = 0;

    loop {
        match reader.read_event() {
            Ok(Event::Start(e)) => {
                let name = e.local_name();
                match name.as_ref() {
                    "url" => {
                        in_url = true;
                        depth = 0;
                    }
                    "sitemap" => {
                        in_sitemap = true;
                        depth = 0;
                    }
                    "loc" if (in_url || in_sitemap) && depth == 0 => capture = Capture::Loc,
                    "lastmod" if in_url && depth == 0 => capture = Capture::Lastmod,
                    _ if in_url || in_sitemap => depth += 1,
                    _ => {}
                }
            }
            Ok(Event::Text(e)) => {
                let text = e.xml10_content();
                match capture {
                    Capture::Loc => loc.push_str(text.trim()),
                    Capture::Lastmod => lastmod.push_str(text.trim()),
                    Capture::Idle => {}
                }
            }
            Ok(Event::GeneralRef(e)) => {
                let resolved = match &*e {
                    "amp" => "&",
                    "lt" => "<",
                    "gt" => ">",
                    "quot" => "\"",
                    "apos" => "'",
                    _ => "",
                };
                match capture {
                    Capture::Loc => loc.push_str(resolved),
                    Capture::Lastmod => lastmod.push_str(resolved),
                    Capture::Idle => {}
                }
            }
            Ok(Event::End(e)) => {
                let name = e.local_name();
                match name.as_ref() {
                    "url" if in_url => {
                        if !loc.is_empty() {
                            let lm = if lastmod.is_empty() {
                                None
                            } else {
                                Some(std::mem::take(&mut lastmod))
                            };
                            entries.push(SitemapEntry::Url {
                                loc: std::mem::take(&mut loc),
                                lastmod: lm,
                            });
                        }
                        loc.clear();
                        lastmod.clear();
                        in_url = false;
                    }
                    "sitemap" if in_sitemap => {
                        if !loc.is_empty() {
                            entries.push(SitemapEntry::Sitemap {
                                loc: std::mem::take(&mut loc),
                            });
                        }
                        loc.clear();
                        lastmod.clear();
                        in_sitemap = false;
                    }
                    "loc" | "lastmod" if capture != Capture::Idle => capture = Capture::Idle,
                    _ if depth > 0 => depth -= 1,
                    _ => {}
                }
            }
            Ok(Event::Eof) | Err(_) => break,
            _ => {}
        }
    }

    entries
}

#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum Capture {
    Idle,
    Loc,
    Lastmod,
}

fn validate_entry(
    loc: &str,
    lastmod: Option<String>,
    seed: &Url,
    robots: &RobotsPolicy,
    opts: &MapConfig,
    visited: &mut HashSet<String>,
) -> Option<MappedUrl> {
    if loc.len() > MAP_URL_MAX_LEN {
        return None;
    }
    let url = Url::parse(loc)
        .ok()
        .filter(|u| matches!(u.scheme(), "http" | "https"))?;
    if !is_same_site(seed, &url) {
        return None;
    }
    if !robots.is_allowed(&url) {
        return None;
    }
    if !matches_scope(&url, opts.include.as_ref(), opts.exclude.as_ref()) {
        return None;
    }
    let normalized = normalize_url(&url);
    if !visited.insert(normalized.clone()) {
        return None;
    }
    Some(MappedUrl {
        url: normalized,
        lastmod,
    })
}

async fn throttle(last_fetch: &mut Instant) {
    let elapsed = last_fetch.elapsed();
    if elapsed < MAP_MIN_FETCH_INTERVAL {
        tokio::time::sleep(MAP_MIN_FETCH_INTERVAL.saturating_sub(elapsed)).await;
    }
    *last_fetch = Instant::now();
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::robots::RobotsRules;

    fn test_config(seed: &str) -> MapConfig {
        MapConfig {
            seed: Url::parse(seed).unwrap(),
            limit: 100,
            include: None,
            exclude: None,
            user_agent: None,
            timeout: Duration::from_secs(30),
            no_fallback: false,
            headers: http::HeaderMap::new(),
        }
    }

    #[test]
    fn parse_urlset() {
        let xml = r#"<?xml version="1.0"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
  <url><loc>https://example.com/a</loc><lastmod>2026-01-01</lastmod></url>
  <url><loc>https://example.com/b</loc></url>
</urlset>"#;
        let entries = parse_sitemap(xml);
        assert_eq!(entries.len(), 2);
        match &entries[0] {
            SitemapEntry::Url { loc, lastmod } => {
                assert_eq!(loc, "https://example.com/a");
                assert_eq!(lastmod.as_deref(), Some("2026-01-01"));
            }
            SitemapEntry::Sitemap { .. } => panic!("expected Url"),
        }
        match &entries[1] {
            SitemapEntry::Url { loc, lastmod } => {
                assert_eq!(loc, "https://example.com/b");
                assert!(lastmod.is_none());
            }
            SitemapEntry::Sitemap { .. } => panic!("expected Url"),
        }
    }

    #[test]
    fn parse_sitemapindex() {
        let xml = r#"<?xml version="1.0"?>
<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
  <sitemap><loc>https://example.com/sitemap1.xml</loc></sitemap>
  <sitemap><loc>https://example.com/sitemap2.xml</loc></sitemap>
</sitemapindex>"#;
        let entries = parse_sitemap(xml);
        assert_eq!(entries.len(), 2);
        match &entries[0] {
            SitemapEntry::Sitemap { loc } => assert_eq!(loc, "https://example.com/sitemap1.xml"),
            SitemapEntry::Url { .. } => panic!("expected Sitemap"),
        }
    }

    #[test]
    fn parse_handles_xml_entities() {
        let xml = r"<urlset><url><loc>https://example.com/a?b=1&amp;c=2</loc></url></urlset>";
        let entries = parse_sitemap(xml);
        assert_eq!(entries.len(), 1);
        match &entries[0] {
            SitemapEntry::Url { loc, .. } => assert_eq!(loc, "https://example.com/a?b=1&c=2"),
            SitemapEntry::Sitemap { .. } => panic!("expected Url"),
        }
    }

    #[test]
    fn parse_handles_namespaced_tags() {
        let xml = r#"<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
                xmlns:image="http://www.google.com/schemas/sitemap-image/1.1">
  <url>
    <loc>https://example.com/page</loc>
    <image:image><image:loc>https://example.com/img.png</image:loc></image:image>
  </url>
</urlset>"#;
        let entries = parse_sitemap(xml);
        assert_eq!(entries.len(), 1);
        match &entries[0] {
            SitemapEntry::Url { loc, .. } => assert_eq!(loc, "https://example.com/page"),
            SitemapEntry::Sitemap { .. } => panic!("expected Url"),
        }
    }

    #[test]
    fn parse_loc_after_nested_extension() {
        let xml = r#"<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
                xmlns:image="http://www.google.com/schemas/sitemap-image/1.1">
  <url>
    <image:image><image:loc>https://example.com/img.png</image:loc></image:image>
    <loc>https://example.com/page</loc>
    <lastmod>2026-01-01</lastmod>
  </url>
</urlset>"#;
        let entries = parse_sitemap(xml);
        assert_eq!(entries.len(), 1);
        match &entries[0] {
            SitemapEntry::Url { loc, lastmod } => {
                assert_eq!(loc, "https://example.com/page");
                assert_eq!(lastmod.as_deref(), Some("2026-01-01"));
            }
            SitemapEntry::Sitemap { .. } => panic!("expected Url"),
        }
    }

    #[test]
    fn parse_empty_body_returns_empty() {
        assert!(parse_sitemap("").is_empty());
        assert!(parse_sitemap("<html><body>Not Found</body></html>").is_empty());
    }

    #[test]
    fn looks_like_html_detects_variants() {
        assert!(looks_like_html(b"<!DOCTYPE html>"));
        assert!(looks_like_html(b"<!doctype html>"));
        assert!(looks_like_html(b"<html lang=\"en\">"));
        assert!(looks_like_html(b"<HTML>"));
        assert!(looks_like_html(b"\xef\xbb\xbf<!DOCTYPE html>"));
        assert!(looks_like_html(b"  \n<!doctype html>"));
        assert!(looks_like_html(b"\xef\xbb\xbf  <html>"));
        assert!(!looks_like_html(b"<?xml version=\"1.0\"?>"));
        assert!(!looks_like_html(b"<urlset>"));
        assert!(!looks_like_html(b""));
    }

    #[test]
    fn validate_entry_rejects_private_ip() {
        let opts = test_config("https://example.com");

        let mut visited = HashSet::new();
        let result = validate_entry(
            "http://127.0.0.1/secret",
            None,
            &opts.seed,
            &RobotsPolicy::Unavailable,
            &opts,
            &mut visited,
        );
        assert!(result.is_none());
    }

    #[test]
    fn validate_entry_rejects_cross_site() {
        let opts = test_config("https://example.com");
        let robots = RobotsPolicy::Unavailable;
        let mut visited = HashSet::new();
        let result = validate_entry("https://evil.com/page", None, &opts.seed, &robots, &opts, &mut visited);
        assert!(result.is_none());
    }

    #[test]
    fn validate_entry_deduplicates() {
        let opts = test_config("https://example.com");
        let robots = RobotsPolicy::Unavailable;
        let mut visited = HashSet::new();
        let first = validate_entry(
            "https://example.com/page",
            None,
            &opts.seed,
            &robots,
            &opts,
            &mut visited,
        );
        assert!(first.is_some());
        let second = validate_entry(
            "https://example.com/page",
            None,
            &opts.seed,
            &robots,
            &opts,
            &mut visited,
        );
        assert!(second.is_none());
    }

    #[test]
    fn validate_entry_rejects_long_url() {
        let opts = test_config("https://example.com");
        let robots = RobotsPolicy::Unavailable;
        let mut visited = HashSet::new();
        let long_url = format!("https://example.com/{}", "a".repeat(MAP_URL_MAX_LEN));
        let result = validate_entry(&long_url, None, &opts.seed, &robots, &opts, &mut visited);
        assert!(result.is_none());
    }

    #[test]
    fn discover_sitemaps_includes_robots_and_default() {
        let seed = Url::parse("https://example.com").unwrap();
        let robots = RobotsPolicy::Rules(RobotsRules {
            rules: Vec::new(),
            sitemaps: vec![Url::parse("https://example.com/custom-sitemap.xml").unwrap()],
        });
        let sitemaps = discover_sitemaps(&robots, &seed);
        assert_eq!(sitemaps.len(), 2);
        assert_eq!(sitemaps[0].as_str(), "https://example.com/custom-sitemap.xml");
        assert_eq!(sitemaps[1].as_str(), "https://example.com/sitemap.xml");
    }

    #[test]
    fn discover_sitemaps_deduplicates_default() {
        let seed = Url::parse("https://example.com").unwrap();
        let robots = RobotsPolicy::Rules(RobotsRules {
            rules: Vec::new(),
            sitemaps: vec![Url::parse("https://example.com/sitemap.xml").unwrap()],
        });
        let sitemaps = discover_sitemaps(&robots, &seed);
        assert_eq!(sitemaps.len(), 1);
    }

    mod integration {
        use std::io::Write as _;
        use std::time::Duration;

        use flate2::Compression;
        use flate2::write::GzEncoder;
        use url::Url;
        use wiremock::matchers::{method, path};
        use wiremock::{Mock, MockServer, ResponseTemplate};

        use crate::map::{MapConfig, MapTraversal, MappedUrl, extract_links, fetch_html, fetch_sitemap, parse_sitemap};

        fn gzip(body: &[u8]) -> Vec<u8> {
            let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
            encoder.write_all(body).unwrap();
            encoder.finish().unwrap()
        }

        #[tokio::test]
        async fn fetch_sitemap_parses_urlset() {
            let server = MockServer::start().await;
            let xml = r#"<?xml version="1.0"?><urlset><url><loc>https://example.com/a</loc></url></urlset>"#;
            Mock::given(method("GET"))
                .and(path("/sitemap.xml"))
                .respond_with(ResponseTemplate::new(200).set_body_raw(xml.as_bytes().to_vec(), "application/xml"))
                .mount(&server)
                .await;

            let client = crate::transfer::raw_client().unwrap();
            let config = super::test_config(&server.uri());
            let url = Url::parse(&format!("{}/sitemap.xml", server.uri())).unwrap();
            let body = fetch_sitemap(&client, &url, &config).await;

            let entries = parse_sitemap(&body.unwrap());
            assert_eq!(entries.len(), 1);
        }

        #[tokio::test]
        async fn fetch_sitemap_rejects_html_error_page() {
            let server = MockServer::start().await;
            Mock::given(method("GET"))
                .and(path("/sitemap.xml"))
                .respond_with(ResponseTemplate::new(200).set_body_raw(
                    b"<!DOCTYPE html><html><body>Not Found</body></html>".to_vec(),
                    "text/html; charset=utf-8",
                ))
                .mount(&server)
                .await;

            let client = crate::transfer::raw_client().unwrap();
            let config = super::test_config(&server.uri());
            let url = Url::parse(&format!("{}/sitemap.xml", server.uri())).unwrap();
            let body = fetch_sitemap(&client, &url, &config).await;

            assert!(body.is_none());
        }

        #[tokio::test]
        async fn fetch_sitemap_returns_none_on_404() {
            let server = MockServer::start().await;
            Mock::given(method("GET"))
                .and(path("/sitemap.xml"))
                .respond_with(ResponseTemplate::new(404))
                .mount(&server)
                .await;

            let client = crate::transfer::raw_client().unwrap();
            let config = super::test_config(&server.uri());
            let url = Url::parse(&format!("{}/sitemap.xml", server.uri())).unwrap();
            let body = fetch_sitemap(&client, &url, &config).await;

            assert!(body.is_none());
        }

        #[tokio::test]
        async fn fetch_sitemap_handles_gzip() {
            let server = MockServer::start().await;
            let xml = r#"<?xml version="1.0"?><urlset><url><loc>https://example.com/gz</loc></url></urlset>"#;
            let compressed = gzip(xml.as_bytes());

            Mock::given(method("GET"))
                .and(path("/sitemap.xml.gz"))
                .respond_with(ResponseTemplate::new(200).set_body_raw(compressed, "application/gzip"))
                .mount(&server)
                .await;

            let client = crate::transfer::raw_client().unwrap();
            let config = super::test_config(&server.uri());
            let url = Url::parse(&format!("{}/sitemap.xml.gz", server.uri())).unwrap();
            let body = fetch_sitemap(&client, &url, &config).await;

            let entries = parse_sitemap(&body.unwrap());
            assert_eq!(entries.len(), 1);
        }

        #[tokio::test]
        async fn fetch_sitemap_decodes_overlapping_gzip_signals_once() {
            let server = MockServer::start().await;
            let xml = r#"<?xml version="1.0"?><urlset><url><loc>https://example.com/gz</loc></url></urlset>"#;
            Mock::given(method("GET"))
                .and(path("/sitemap.xml.gz"))
                .respond_with(
                    ResponseTemplate::new(200)
                        .insert_header("content-encoding", "gzip")
                        .set_body_raw(gzip(xml.as_bytes()), "application/gzip"),
                )
                .mount(&server)
                .await;

            let client = crate::transfer::raw_client().unwrap();
            let config = super::test_config(&server.uri());
            let url = Url::parse(&format!("{}/sitemap.xml.gz", server.uri())).unwrap();
            let body = fetch_sitemap(&client, &url, &config).await;

            let entries = parse_sitemap(&body.unwrap());
            assert_eq!(entries.len(), 1);
        }

        #[tokio::test]
        async fn fetch_sitemap_rejects_content_encoding_over_decompressed_limit() {
            let server = MockServer::start().await;
            let oversized_len = usize::try_from(super::super::MAP_SITEMAP_MAX_DECOMPRESSED)
                .unwrap()
                .checked_add(1)
                .unwrap();
            let oversized = vec![b' '; oversized_len];
            Mock::given(method("GET"))
                .and(path("/sitemap.xml"))
                .respond_with(
                    ResponseTemplate::new(200)
                        .insert_header("content-encoding", "gzip")
                        .set_body_raw(gzip(&oversized), "application/xml"),
                )
                .mount(&server)
                .await;

            let client = crate::transfer::raw_client().unwrap();
            let config = super::test_config(&server.uri());
            let url = Url::parse(&format!("{}/sitemap.xml", server.uri())).unwrap();
            let body = fetch_sitemap(&client, &url, &config).await;

            assert!(body.is_none());
        }

        #[tokio::test]
        async fn fetch_html_extracts_links() {
            let server = MockServer::start().await;
            Mock::given(method("GET"))
                .and(path("/"))
                .respond_with(ResponseTemplate::new(200).set_body_raw(
                    br#"<html><body><a href="/link">x</a></body></html>"#.to_vec(),
                    "text/html; charset=utf-8",
                ))
                .mount(&server)
                .await;

            let client = crate::transfer::client().unwrap();
            let seed = Url::parse(&server.uri()).unwrap();
            let config = super::test_config(&server.uri());
            let html = fetch_html(&client, &seed, &config).await.unwrap();

            let links = extract_links(&html, &seed);
            assert_eq!(links.len(), 1);
        }

        #[tokio::test]
        async fn dropping_map_mid_robots_body_closes_the_socket() {
            use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _};
            use tokio::net::TcpListener;
            use tokio::sync::oneshot;

            let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind map fixture");
            let address = listener.local_addr().expect("map fixture address");
            let (partial_tx, partial_rx) = oneshot::channel();
            let (closed_tx, closed_rx) = oneshot::channel();
            let fixture = tokio::spawn(async move {
                let (mut socket, _) = listener.accept().await.expect("accept robots request");
                let mut request = Vec::new();
                let mut byte = [0_u8; 1];
                while !request.ends_with(b"\r\n\r\n") {
                    socket.read_exact(&mut byte).await.expect("read robots request");
                    request.push(byte[0]);
                }
                socket
                    .write_all(
                        b"HTTP/1.1 200 OK\r\ncontent-type: text/plain\r\ncontent-length: 1048576\r\n\r\nUser-agent: *\n",
                    )
                    .await
                    .expect("write partial robots response");
                socket.flush().await.expect("flush partial robots response");
                partial_tx.send(()).expect("report partial robots body");

                let observed = socket.read(&mut byte).await;
                closed_tx.send(observed).expect("report client disconnect");
            });

            let seed = Url::parse(&format!("http://{address}/")).expect("fixture seed URL");
            let mapping = tokio::spawn(async move {
                let config = MapConfig {
                    seed,
                    limit: 100,
                    include: None,
                    exclude: None,
                    user_agent: Some("test-bot".into()),
                    timeout: Duration::from_secs(30),
                    no_fallback: false,
                    headers: http::HeaderMap::new(),
                };
                let mut traversal = MapTraversal::new(
                    config,
                    crate::transfer::client().unwrap(),
                    crate::transfer::raw_client().unwrap(),
                );
                traversal.next_url().await
            });
            partial_rx.await.expect("map reaches the partial robots body");
            mapping.abort();
            assert!(mapping.await.expect_err("map task is cancelled").is_cancelled());

            let observed = tokio::time::timeout(Duration::from_secs(1), closed_rx)
                .await
                .expect("cancelled map closes its socket promptly")
                .expect("fixture reports client disconnect");
            match observed {
                Ok(0) => {}
                Err(error)
                    if matches!(
                        error.kind(),
                        std::io::ErrorKind::ConnectionReset | std::io::ErrorKind::BrokenPipe
                    ) => {}
                other => panic!("cancelled map must close its response socket, observed {other:?}"),
            }
            fixture.await.expect("map fixture task completes");
        }

        async fn check_run(server: &MockServer, configure: impl FnOnce(&mut MapConfig)) -> Vec<MappedUrl> {
            let mut config = MapConfig {
                seed: Url::parse(&server.uri()).unwrap(),
                limit: 100,
                include: None,
                exclude: None,
                user_agent: Some("test-bot".into()),
                timeout: Duration::from_secs(5),
                no_fallback: false,
                headers: http::HeaderMap::new(),
            };
            configure(&mut config);
            let mut traversal = MapTraversal::new(
                config,
                crate::transfer::client().unwrap(),
                crate::transfer::raw_client().unwrap(),
            );
            let mut entries = Vec::new();
            while let Some(entry) = traversal.next_url().await {
                entries.push(entry);
            }
            entries
        }

        #[tokio::test]
        async fn run_discovers_urls_from_sitemap() {
            let server = MockServer::start().await;
            Mock::given(method("GET"))
                .and(path("/robots.txt"))
                .respond_with(
                    ResponseTemplate::new(200)
                        .set_body_raw(b"User-agent: *\nAllow: /".to_vec(), "text/plain; charset=utf-8"),
                )
                .mount(&server)
                .await;
            let sitemap = format!(
                "<urlset><url><loc>{}/page1</loc></url><url><loc>{}/page2</loc></url></urlset>",
                server.uri(),
                server.uri()
            );
            Mock::given(method("GET"))
                .and(path("/sitemap.xml"))
                .respond_with(
                    ResponseTemplate::new(200).set_body_raw(sitemap.into_bytes(), "application/xml; charset=utf-8"),
                )
                .mount(&server)
                .await;

            let entries = check_run(&server, |_| {}).await;
            assert_eq!(entries.len(), 2);
            assert!(entries.iter().any(|e| e.url.ends_with("/page1")));
            assert!(entries.iter().any(|e| e.url.ends_with("/page2")));
        }

        #[tokio::test]
        async fn run_decodes_gzip_encoded_robots() {
            let server = MockServer::start().await;
            Mock::given(method("GET"))
                .and(path("/robots.txt"))
                .respond_with(
                    ResponseTemplate::new(200)
                        .insert_header("content-encoding", "gzip")
                        .set_body_raw(gzip(b"User-agent: *\nAllow: /"), "text/plain; charset=utf-8"),
                )
                .mount(&server)
                .await;
            let sitemap = format!("<urlset><url><loc>{}/allowed</loc></url></urlset>", server.uri());
            Mock::given(method("GET"))
                .and(path("/sitemap.xml"))
                .respond_with(
                    ResponseTemplate::new(200).set_body_raw(sitemap.into_bytes(), "application/xml; charset=utf-8"),
                )
                .mount(&server)
                .await;

            let entries = check_run(&server, |config| {
                config
                    .headers
                    .insert("accept-encoding", http::HeaderValue::from_static("gzip"));
            })
            .await;

            assert_eq!(entries.len(), 1);
            assert!(entries[0].url.ends_with("/allowed"));
        }

        #[tokio::test]
        async fn run_decodes_gzip_sitemap_with_overlapping_signals_once() {
            let server = MockServer::start().await;
            let robots = format!("User-agent: *\nAllow: /\nSitemap: {}/site.xml.gz", server.uri());
            Mock::given(method("GET"))
                .and(path("/robots.txt"))
                .respond_with(
                    ResponseTemplate::new(200)
                        .insert_header("content-encoding", "gzip")
                        .set_body_raw(gzip(robots.as_bytes()), "text/plain; charset=utf-8"),
                )
                .mount(&server)
                .await;
            let sitemap = format!("<urlset><url><loc>{}/found</loc></url></urlset>", server.uri());
            Mock::given(method("GET"))
                .and(path("/site.xml.gz"))
                .respond_with(
                    ResponseTemplate::new(200)
                        .insert_header("content-encoding", "gzip")
                        .set_body_raw(gzip(sitemap.as_bytes()), "application/gzip"),
                )
                .mount(&server)
                .await;

            let entries = check_run(&server, |config| {
                config
                    .headers
                    .insert("accept-encoding", http::HeaderValue::from_static("gzip"));
            })
            .await;

            assert_eq!(entries.len(), 1);
            assert!(entries[0].url.ends_with("/found"));
        }

        #[tokio::test]
        async fn run_respects_limit() {
            let server = MockServer::start().await;
            Mock::given(method("GET"))
                .and(path("/robots.txt"))
                .respond_with(ResponseTemplate::new(404))
                .mount(&server)
                .await;
            let sitemap = format!(
                "<urlset><url><loc>{}/a</loc></url><url><loc>{}/b</loc></url><url><loc>{}/c</loc></url></urlset>",
                server.uri(),
                server.uri(),
                server.uri()
            );
            Mock::given(method("GET"))
                .and(path("/sitemap.xml"))
                .respond_with(
                    ResponseTemplate::new(200).set_body_raw(sitemap.into_bytes(), "application/xml; charset=utf-8"),
                )
                .mount(&server)
                .await;

            let entries = check_run(&server, |c| c.limit = 2).await;
            assert_eq!(entries.len(), 2);
        }

        #[tokio::test]
        async fn run_follows_sitemap_index() {
            let server = MockServer::start().await;
            Mock::given(method("GET"))
                .and(path("/robots.txt"))
                .respond_with(ResponseTemplate::new(404))
                .mount(&server)
                .await;
            let index = format!(
                "<sitemapindex><sitemap><loc>{}/sub.xml</loc></sitemap></sitemapindex>",
                server.uri()
            );
            Mock::given(method("GET"))
                .and(path("/sitemap.xml"))
                .respond_with(
                    ResponseTemplate::new(200).set_body_raw(index.into_bytes(), "application/xml; charset=utf-8"),
                )
                .mount(&server)
                .await;
            let sub = format!("<urlset><url><loc>{}/deep</loc></url></urlset>", server.uri());
            Mock::given(method("GET"))
                .and(path("/sub.xml"))
                .respond_with(
                    ResponseTemplate::new(200).set_body_raw(sub.into_bytes(), "application/xml; charset=utf-8"),
                )
                .mount(&server)
                .await;

            let entries = check_run(&server, |_| {}).await;
            assert_eq!(entries.len(), 1);
            assert!(entries[0].url.ends_with("/deep"));
        }

        #[tokio::test]
        async fn run_falls_back_to_html_links() {
            let server = MockServer::start().await;
            Mock::given(method("GET"))
                .and(path("/robots.txt"))
                .respond_with(ResponseTemplate::new(404))
                .mount(&server)
                .await;
            Mock::given(method("GET"))
                .and(path("/sitemap.xml"))
                .respond_with(ResponseTemplate::new(404))
                .mount(&server)
                .await;
            let html = format!(
                r#"<html><body><a href="{}/link1">L1</a><a href="{}/link2">L2</a></body></html>"#,
                server.uri(),
                server.uri()
            );
            Mock::given(method("GET"))
                .and(path("/"))
                .respond_with(ResponseTemplate::new(200).set_body_raw(html.into_bytes(), "text/html; charset=utf-8"))
                .mount(&server)
                .await;

            let entries = check_run(&server, |_| {}).await;
            assert_eq!(entries.len(), 2);
        }

        #[tokio::test]
        async fn run_falls_back_to_html_links_with_invalid_utf8() {
            let server = MockServer::start().await;
            Mock::given(method("GET"))
                .and(path("/robots.txt"))
                .respond_with(ResponseTemplate::new(404))
                .mount(&server)
                .await;
            Mock::given(method("GET"))
                .and(path("/sitemap.xml"))
                .respond_with(ResponseTemplate::new(404))
                .mount(&server)
                .await;
            Mock::given(method("GET"))
                .and(path("/"))
                .respond_with(ResponseTemplate::new(200).set_body_raw(
                    b"<html><body>invalid: \xff<a href=\"/article\">Article</a></body></html>".to_vec(),
                    "text/html; charset=utf-8",
                ))
                .mount(&server)
                .await;

            let entries = check_run(&server, |_| {}).await;

            assert_eq!(entries.len(), 1);
            assert!(entries[0].url.ends_with("/article"));
        }

        #[tokio::test]
        async fn run_decodes_gzip_encoded_html_fallback() {
            let server = MockServer::start().await;
            Mock::given(method("GET"))
                .and(path("/robots.txt"))
                .respond_with(ResponseTemplate::new(404))
                .mount(&server)
                .await;
            Mock::given(method("GET"))
                .and(path("/sitemap.xml"))
                .respond_with(ResponseTemplate::new(404))
                .mount(&server)
                .await;
            let html = format!(
                r#"<html><body><a href="{}/article">Article</a></body></html>"#,
                server.uri()
            );
            Mock::given(method("GET"))
                .and(path("/"))
                .respond_with(
                    ResponseTemplate::new(200)
                        .insert_header("content-encoding", "gzip")
                        .set_body_raw(gzip(html.as_bytes()), "text/html; charset=utf-8"),
                )
                .mount(&server)
                .await;

            let entries = check_run(&server, |config| {
                config
                    .headers
                    .insert("accept-encoding", http::HeaderValue::from_static("gzip"));
            })
            .await;

            assert_eq!(entries.len(), 1);
            assert!(entries[0].url.ends_with("/article"));
        }

        #[tokio::test]
        async fn run_no_fallback_skips_html() {
            let server = MockServer::start().await;
            Mock::given(method("GET"))
                .and(path("/robots.txt"))
                .respond_with(ResponseTemplate::new(404))
                .mount(&server)
                .await;
            Mock::given(method("GET"))
                .and(path("/sitemap.xml"))
                .respond_with(ResponseTemplate::new(404))
                .mount(&server)
                .await;
            Mock::given(method("GET"))
                .and(path("/"))
                .respond_with(ResponseTemplate::new(200).set_body_raw(
                    br#"<html><body><a href="/link">L</a></body></html>"#.to_vec(),
                    "text/html; charset=utf-8",
                ))
                .mount(&server)
                .await;

            let entries = check_run(&server, |c| c.no_fallback = true).await;
            assert_eq!(entries.len(), 0);
        }

        #[tokio::test]
        async fn run_deduplicates_urls() {
            let server = MockServer::start().await;
            Mock::given(method("GET"))
                .and(path("/robots.txt"))
                .respond_with(ResponseTemplate::new(404))
                .mount(&server)
                .await;
            let sitemap = format!(
                "<urlset><url><loc>{}/dup</loc></url><url><loc>{}/dup</loc></url><url><loc>{}/unique</loc></url></urlset>",
                server.uri(),
                server.uri(),
                server.uri()
            );
            Mock::given(method("GET"))
                .and(path("/sitemap.xml"))
                .respond_with(
                    ResponseTemplate::new(200).set_body_raw(sitemap.into_bytes(), "application/xml; charset=utf-8"),
                )
                .mount(&server)
                .await;

            let entries = check_run(&server, |_| {}).await;
            assert_eq!(entries.len(), 2);
        }
    }
}