servo_fetch/
map.rs

1//! URL discovery via sitemap parsing — no rendering required.
2
3use std::collections::{HashSet, VecDeque};
4use std::io::Read as _;
5use std::time::{Duration, Instant};
6
7use tokio::task::spawn_blocking;
8use url::Url;
9
10use crate::robots::{RobotsPolicy, RobotsRules};
11use crate::scope::{is_same_site, matches_scope, normalize_url};
12use crate::{bridge, net};
13
14const MAP_SITEMAP_MAX_BYTES: u64 = 50 * 1024 * 1024;
15const MAP_SITEMAP_MAX_DECOMPRESSED: u64 = 10 * 1024 * 1024;
16const MAP_GZIP_MAX_RATIO: u64 = 100;
17const MAP_HTML_MAX_BYTES: u64 = 2 * 1024 * 1024;
18const MAP_MAX_REDIRECTS: u8 = 5;
19const MAP_MAX_SITEMAPS: usize = 200;
20const MAP_MAX_INDEX_DEPTH: u8 = 5;
21const MAP_MIN_FETCH_INTERVAL: Duration = Duration::from_millis(500);
22const MAP_URL_MAX_LEN: usize = 2048;
23const HTML_SNIFF_LEN: usize = 100;
24
25/// Options for URL discovery (sitemap + link extraction, no rendering).
26#[must_use = "options do nothing until passed to map()"]
27#[derive(Debug, Clone)]
28pub struct MapOptions {
29    url: String,
30    limit: usize,
31    include: Vec<String>,
32    exclude: Vec<String>,
33    user_agent: Option<String>,
34    timeout: u64,
35    no_fallback: bool,
36    headers: http::HeaderMap,
37}
38
39impl MapOptions {
40    /// Create map options for the given URL.
41    pub fn new(url: impl Into<String>) -> Self {
42        Self {
43            url: url.into(),
44            limit: 5000,
45            include: Vec::new(),
46            exclude: Vec::new(),
47            user_agent: None,
48            timeout: 30,
49            no_fallback: false,
50            headers: http::HeaderMap::new(),
51        }
52    }
53
54    /// Maximum number of URLs to discover.
55    pub fn limit(mut self, n: usize) -> Self {
56        self.limit = n;
57        self
58    }
59
60    /// URL path glob patterns to include.
61    pub fn include(mut self, patterns: &[&str]) -> Self {
62        self.include = patterns.iter().map(|s| (*s).to_string()).collect();
63        self
64    }
65
66    /// URL path glob patterns to exclude.
67    pub fn exclude(mut self, patterns: &[&str]) -> Self {
68        self.exclude = patterns.iter().map(|s| (*s).to_string()).collect();
69        self
70    }
71
72    /// Override the User-Agent string.
73    pub fn user_agent(mut self, ua: impl Into<String>) -> Self {
74        self.user_agent = Some(ua.into());
75        self
76    }
77
78    /// Timeout in seconds per HTTP request.
79    pub fn timeout(mut self, secs: u64) -> Self {
80        self.timeout = secs;
81        self
82    }
83
84    /// Skip HTML link fallback if no sitemap is found.
85    pub fn no_fallback(mut self, yes: bool) -> Self {
86        self.no_fallback = yes;
87        self
88    }
89
90    /// Custom request headers sent with every discovery request.
91    pub fn headers(mut self, headers: http::HeaderMap) -> Self {
92        self.headers = headers;
93        self
94    }
95}
96
97/// A discovered URL from sitemap or link extraction.
98#[derive(Debug, Clone, serde::Serialize)]
99pub struct MappedUrl {
100    /// The discovered URL.
101    pub url: String,
102    /// Last modification date from sitemap, if available.
103    #[serde(skip_serializing_if = "Option::is_none")]
104    pub lastmod: Option<String>,
105}
106
107/// Discover URLs on a site via sitemaps and link extraction (blocking).
108pub fn map_blocking(opts: &MapOptions) -> crate::error::Result<Vec<MappedUrl>> {
109    crate::runtime::block_on(map(opts)).map_err(|e| crate::error::Error::engine(e, None))?
110}
111
112/// Discover URLs on a site via sitemaps and link extraction.
113pub async fn map(opts: &MapOptions) -> crate::error::Result<Vec<MappedUrl>> {
114    net::ensure_crypto_provider();
115    let seed = net::validate_url(&opts.url)?;
116
117    let include = if opts.include.is_empty() {
118        None
119    } else {
120        Some(crate::scope::build_globset(&opts.include)?)
121    };
122    let exclude = if opts.exclude.is_empty() {
123        None
124    } else {
125        Some(crate::scope::build_globset(&opts.exclude)?)
126    };
127
128    let internal = MapConfig {
129        seed,
130        limit: opts.limit,
131        include,
132        exclude,
133        user_agent: opts.user_agent.clone(),
134        timeout: Duration::from_secs(opts.timeout),
135        no_fallback: opts.no_fallback,
136        headers: opts.headers.clone(),
137    };
138
139    let mut results = Vec::new();
140    run(&internal, |entry| {
141        results.push(MappedUrl {
142            url: entry.url.clone(),
143            lastmod: entry.lastmod.clone(),
144        });
145    })
146    .await;
147    Ok(results)
148}
149
150/// Options for URL discovery.
151pub(crate) struct MapConfig {
152    pub seed: Url,
153    pub limit: usize,
154    pub include: Option<globset::GlobSet>,
155    pub exclude: Option<globset::GlobSet>,
156    pub user_agent: Option<String>,
157    pub timeout: Duration,
158    pub no_fallback: bool,
159    pub headers: http::HeaderMap,
160}
161
162/// A discovered URL with optional metadata.
163#[derive(serde::Serialize)]
164pub(crate) struct MapEntry {
165    pub url: String,
166    #[serde(skip_serializing_if = "Option::is_none")]
167    pub lastmod: Option<String>,
168}
169
170/// Run URL discovery for a site.
171pub(crate) async fn run(opts: &MapConfig, mut on_url: impl FnMut(&MapEntry)) {
172    let ua = opts
173        .user_agent
174        .as_deref()
175        .unwrap_or_else(|| bridge::default_user_agent());
176    let agent = build_agent(ua, opts.timeout);
177
178    let robots = {
179        let seed = opts.seed.clone();
180        let user_agent = opts.user_agent.clone();
181        let headers = opts.headers.clone();
182        let timeout = opts.timeout;
183        spawn_blocking(move || RobotsRules::fetch(&seed, user_agent.as_deref(), &headers, timeout))
184            .await
185            .unwrap_or(RobotsPolicy::Unreachable)
186    };
187
188    let mut visited = HashSet::new();
189    let mut count = 0;
190    let mut last_fetch = Instant::now()
191        .checked_sub(MAP_MIN_FETCH_INTERVAL)
192        .unwrap_or_else(Instant::now);
193    let mut sitemap_queue: VecDeque<(Url, u8)> = discover_sitemaps(&robots, &opts.seed)
194        .into_iter()
195        .map(|u| (u, 0))
196        .collect();
197    let mut sitemaps_fetched = 0;
198
199    while let Some((sitemap_url, depth)) = sitemap_queue.pop_front() {
200        if sitemaps_fetched >= MAP_MAX_SITEMAPS || count >= opts.limit {
201            break;
202        }
203        if depth > MAP_MAX_INDEX_DEPTH || !is_same_site(&opts.seed, &sitemap_url) {
204            continue;
205        }
206
207        throttle(&mut last_fetch).await;
208        sitemaps_fetched += 1;
209
210        let body = {
211            let agent = agent.clone();
212            spawn_blocking({
213                let seed = opts.seed.clone();
214                let headers = opts.headers.clone();
215                move || fetch_sitemap(&agent, &sitemap_url, &seed, &headers)
216            })
217            .await
218            .ok()
219            .flatten()
220        };
221        let Some(body) = body else { continue };
222
223        for entry in parse_sitemap(&body) {
224            match entry {
225                SitemapEntry::Url { loc, lastmod } => {
226                    if count >= opts.limit {
227                        break;
228                    }
229                    if let Some(e) = validate_entry(&loc, lastmod, &opts.seed, &robots, opts, &mut visited) {
230                        on_url(&e);
231                        count += 1;
232                    }
233                }
234                SitemapEntry::Sitemap { loc } => {
235                    if let Ok(url) = Url::parse(&loc) {
236                        sitemap_queue.push_back((url, depth + 1));
237                    }
238                }
239            }
240        }
241    }
242
243    if count == 0 && !opts.no_fallback {
244        throttle(&mut last_fetch).await;
245        let html = {
246            let agent = agent.clone();
247            let seed = opts.seed.clone();
248            let headers = opts.headers.clone();
249            spawn_blocking(move || fetch_html(&agent, &seed, &headers))
250                .await
251                .ok()
252                .flatten()
253        };
254        if let Some(html) = html {
255            for link in extract_links(&html, &opts.seed) {
256                if count >= opts.limit {
257                    break;
258                }
259                if let Some(e) = validate_entry(link.as_str(), None, &opts.seed, &robots, opts, &mut visited) {
260                    on_url(&e);
261                    count += 1;
262                }
263            }
264        }
265    }
266}
267
268fn discover_sitemaps(robots: &RobotsPolicy, seed: &Url) -> Vec<Url> {
269    let mut urls = Vec::new();
270    if let RobotsPolicy::Rules(rules) = robots {
271        urls.extend(rules.sitemaps.iter().cloned());
272    }
273    if let Ok(default) = seed.join("/sitemap.xml") {
274        if !urls.contains(&default) {
275            urls.push(default);
276        }
277    }
278    urls
279}
280
281fn build_agent(ua: &str, timeout: Duration) -> ureq::Agent {
282    ureq::Agent::new_with_config(
283        ureq::config::Config::builder()
284            .max_redirects(0)
285            .http_status_as_error(false)
286            .timeout_global(Some(timeout))
287            .user_agent(ua)
288            .build(),
289    )
290}
291
292fn fetch_following_redirects(
293    agent: &ureq::Agent,
294    url: &Url,
295    seed: &Url,
296    headers: &http::HeaderMap,
297) -> Option<http::Response<ureq::Body>> {
298    let mut current = url.clone();
299    for _ in 0..MAP_MAX_REDIRECTS {
300        let mut req = agent.get(current.as_str());
301        for (name, value) in headers {
302            req = req.header(name.clone(), value.clone());
303        }
304        let resp = req.call().ok()?;
305        let status = resp.status().as_u16();
306        if matches!(status, 301 | 302 | 303 | 307 | 308) {
307            let location = resp.headers().get("location")?.to_str().ok()?;
308            let next = current.join(location).ok()?;
309            if net::validate_url_with_policy(next.as_str(), bridge::engine_policy()).is_err()
310                || !is_same_site(seed, &next)
311            {
312                return None;
313            }
314            current = next;
315            continue;
316        }
317        if status >= 400 {
318            return None;
319        }
320        return Some(resp);
321    }
322    None
323}
324
325fn fetch_sitemap(agent: &ureq::Agent, url: &Url, seed: &Url, headers: &http::HeaderMap) -> Option<String> {
326    let resp = fetch_following_redirects(agent, url, seed, headers)?;
327    let content_type = resp
328        .headers()
329        .get("content-type")
330        .and_then(|v| v.to_str().ok())
331        .unwrap_or("");
332
333    let is_gzip = url
334        .path()
335        .rsplit('/')
336        .next()
337        .and_then(|seg| std::path::Path::new(seg).extension())
338        .is_some_and(|ext| ext.eq_ignore_ascii_case("gz"))
339        || content_type.contains("gzip")
340        || resp
341            .headers()
342            .get("content-encoding")
343            .and_then(|v| v.to_str().ok())
344            .is_some_and(|v| v.contains("gzip"));
345
346    if is_gzip {
347        let bytes = resp
348            .into_body()
349            .with_config()
350            .limit(MAP_SITEMAP_MAX_BYTES)
351            .read_to_vec()
352            .ok()?;
353        let mut decoded = Vec::new();
354        flate2::read::GzDecoder::new(bytes.as_slice())
355            .take(MAP_SITEMAP_MAX_DECOMPRESSED)
356            .read_to_end(&mut decoded)
357            .ok()?;
358        if decoded.len() as u64 > bytes.len() as u64 * MAP_GZIP_MAX_RATIO {
359            return None;
360        }
361        if looks_like_html(&decoded) {
362            return None;
363        }
364        String::from_utf8(decoded).ok()
365    } else {
366        let body = resp
367            .into_body()
368            .with_config()
369            .limit(MAP_SITEMAP_MAX_BYTES)
370            .read_to_string()
371            .ok()?;
372        if looks_like_html(body.as_bytes()) {
373            return None;
374        }
375        Some(body)
376    }
377}
378
379fn looks_like_html(bytes: &[u8]) -> bool {
380    const DOCTYPE: &[u8] = b"<!doctype";
381    const HTML: &[u8] = b"<html";
382    const BOM: &[u8] = b"\xef\xbb\xbf";
383    let mut prefix = bytes.get(..HTML_SNIFF_LEN).unwrap_or(bytes);
384    if prefix.starts_with(BOM) {
385        prefix = &prefix[BOM.len()..];
386    }
387    let prefix = prefix
388        .iter()
389        .position(|b| !b.is_ascii_whitespace())
390        .map_or(&[][..], |i| &prefix[i..]);
391    prefix
392        .get(..DOCTYPE.len())
393        .is_some_and(|p| p.eq_ignore_ascii_case(DOCTYPE))
394        || prefix.get(..HTML.len()).is_some_and(|p| p.eq_ignore_ascii_case(HTML))
395}
396
397fn fetch_html(agent: &ureq::Agent, url: &Url, headers: &http::HeaderMap) -> Option<String> {
398    let resp = fetch_following_redirects(agent, url, url, headers)?;
399    resp.into_body()
400        .with_config()
401        .limit(MAP_HTML_MAX_BYTES)
402        .read_to_string()
403        .ok()
404}
405
406fn extract_links(html: &str, base: &Url) -> Vec<Url> {
407    dom_query::Document::from(html)
408        .select("a[href]")
409        .iter()
410        .filter_map(|el| {
411            let href = el.attr("href")?;
412            let href = href.trim();
413            if href.is_empty() {
414                return None;
415            }
416            let resolved = base.join(href).ok()?;
417            matches!(resolved.scheme(), "http" | "https").then_some(resolved)
418        })
419        .collect()
420}
421
422enum SitemapEntry {
423    Url { loc: String, lastmod: Option<String> },
424    Sitemap { loc: String },
425}
426
427fn parse_sitemap(body: &str) -> Vec<SitemapEntry> {
428    use quick_xml::events::Event;
429    use quick_xml::reader::Reader;
430
431    let mut reader = Reader::from_str(body);
432    let mut entries = Vec::new();
433    let mut buf = Vec::new();
434    let mut capture = Capture::Idle;
435    let mut loc = String::new();
436    let mut lastmod = String::new();
437    let mut in_url = false;
438    let mut in_sitemap = false;
439    let mut depth: u32 = 0;
440
441    loop {
442        match reader.read_event_into(&mut buf) {
443            Ok(Event::Start(e)) => {
444                let name = e.local_name();
445                match name.as_ref() {
446                    b"url" => {
447                        in_url = true;
448                        depth = 0;
449                    }
450                    b"sitemap" => {
451                        in_sitemap = true;
452                        depth = 0;
453                    }
454                    b"loc" if (in_url || in_sitemap) && depth == 0 => capture = Capture::Loc,
455                    b"lastmod" if in_url && depth == 0 => capture = Capture::Lastmod,
456                    _ if in_url || in_sitemap => depth += 1,
457                    _ => {}
458                }
459            }
460            Ok(Event::Text(e)) => {
461                if let Ok(text) = e.xml10_content() {
462                    match capture {
463                        Capture::Loc => loc.push_str(text.trim()),
464                        Capture::Lastmod => lastmod.push_str(text.trim()),
465                        Capture::Idle => {}
466                    }
467                } else {
468                    loc.clear();
469                    lastmod.clear();
470                    capture = Capture::Idle;
471                }
472            }
473            Ok(Event::GeneralRef(e)) => {
474                let resolved = match &*e {
475                    b"amp" => "&",
476                    b"lt" => "<",
477                    b"gt" => ">",
478                    b"quot" => "\"",
479                    b"apos" => "'",
480                    _ => "",
481                };
482                match capture {
483                    Capture::Loc => loc.push_str(resolved),
484                    Capture::Lastmod => lastmod.push_str(resolved),
485                    Capture::Idle => {}
486                }
487            }
488            Ok(Event::End(e)) => {
489                let name = e.local_name();
490                match name.as_ref() {
491                    b"url" if in_url => {
492                        if !loc.is_empty() {
493                            let lm = if lastmod.is_empty() {
494                                None
495                            } else {
496                                Some(std::mem::take(&mut lastmod))
497                            };
498                            entries.push(SitemapEntry::Url {
499                                loc: std::mem::take(&mut loc),
500                                lastmod: lm,
501                            });
502                        }
503                        loc.clear();
504                        lastmod.clear();
505                        in_url = false;
506                    }
507                    b"sitemap" if in_sitemap => {
508                        if !loc.is_empty() {
509                            entries.push(SitemapEntry::Sitemap {
510                                loc: std::mem::take(&mut loc),
511                            });
512                        }
513                        loc.clear();
514                        lastmod.clear();
515                        in_sitemap = false;
516                    }
517                    b"loc" | b"lastmod" if capture != Capture::Idle => capture = Capture::Idle,
518                    _ if depth > 0 => depth -= 1,
519                    _ => {}
520                }
521            }
522            Ok(Event::Eof) | Err(_) => break,
523            _ => {}
524        }
525        buf.clear();
526    }
527
528    entries
529}
530
531#[derive(Clone, Copy, Debug, PartialEq, Eq)]
532enum Capture {
533    Idle,
534    Loc,
535    Lastmod,
536}
537
538fn validate_entry(
539    loc: &str,
540    lastmod: Option<String>,
541    seed: &Url,
542    robots: &RobotsPolicy,
543    opts: &MapConfig,
544    visited: &mut HashSet<String>,
545) -> Option<MapEntry> {
546    if loc.len() > MAP_URL_MAX_LEN {
547        return None;
548    }
549    let url = Url::parse(loc)
550        .ok()
551        .filter(|u| matches!(u.scheme(), "http" | "https"))?;
552    if !is_same_site(seed, &url) {
553        return None;
554    }
555    if !robots.is_allowed(&url) {
556        return None;
557    }
558    if !matches_scope(&url, opts.include.as_ref(), opts.exclude.as_ref()) {
559        return None;
560    }
561    let normalized = normalize_url(&url);
562    if !visited.insert(normalized.clone()) {
563        return None;
564    }
565    Some(MapEntry {
566        url: normalized,
567        lastmod,
568    })
569}
570
571async fn throttle(last_fetch: &mut Instant) {
572    let elapsed = last_fetch.elapsed();
573    if elapsed < MAP_MIN_FETCH_INTERVAL {
574        tokio::time::sleep(MAP_MIN_FETCH_INTERVAL.saturating_sub(elapsed)).await;
575    }
576    *last_fetch = Instant::now();
577}
578
579#[cfg(test)]
580mod tests {
581    use super::*;
582
583    fn test_config(seed: &str) -> MapConfig {
584        MapConfig {
585            seed: Url::parse(seed).unwrap(),
586            limit: 100,
587            include: None,
588            exclude: None,
589            user_agent: None,
590            timeout: Duration::from_secs(30),
591            no_fallback: false,
592            headers: http::HeaderMap::new(),
593        }
594    }
595
596    #[test]
597    fn parse_urlset() {
598        let xml = r#"<?xml version="1.0"?>
599<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
600  <url><loc>https://example.com/a</loc><lastmod>2026-01-01</lastmod></url>
601  <url><loc>https://example.com/b</loc></url>
602</urlset>"#;
603        let entries = parse_sitemap(xml);
604        assert_eq!(entries.len(), 2);
605        match &entries[0] {
606            SitemapEntry::Url { loc, lastmod } => {
607                assert_eq!(loc, "https://example.com/a");
608                assert_eq!(lastmod.as_deref(), Some("2026-01-01"));
609            }
610            SitemapEntry::Sitemap { .. } => panic!("expected Url"),
611        }
612        match &entries[1] {
613            SitemapEntry::Url { loc, lastmod } => {
614                assert_eq!(loc, "https://example.com/b");
615                assert!(lastmod.is_none());
616            }
617            SitemapEntry::Sitemap { .. } => panic!("expected Url"),
618        }
619    }
620
621    #[test]
622    fn parse_sitemapindex() {
623        let xml = r#"<?xml version="1.0"?>
624<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
625  <sitemap><loc>https://example.com/sitemap1.xml</loc></sitemap>
626  <sitemap><loc>https://example.com/sitemap2.xml</loc></sitemap>
627</sitemapindex>"#;
628        let entries = parse_sitemap(xml);
629        assert_eq!(entries.len(), 2);
630        match &entries[0] {
631            SitemapEntry::Sitemap { loc } => assert_eq!(loc, "https://example.com/sitemap1.xml"),
632            SitemapEntry::Url { .. } => panic!("expected Sitemap"),
633        }
634    }
635
636    #[test]
637    fn parse_handles_xml_entities() {
638        let xml = r"<urlset><url><loc>https://example.com/a?b=1&amp;c=2</loc></url></urlset>";
639        let entries = parse_sitemap(xml);
640        assert_eq!(entries.len(), 1);
641        match &entries[0] {
642            SitemapEntry::Url { loc, .. } => assert_eq!(loc, "https://example.com/a?b=1&c=2"),
643            SitemapEntry::Sitemap { .. } => panic!("expected Url"),
644        }
645    }
646
647    #[test]
648    fn parse_handles_namespaced_tags() {
649        let xml = r#"<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
650                xmlns:image="http://www.google.com/schemas/sitemap-image/1.1">
651  <url>
652    <loc>https://example.com/page</loc>
653    <image:image><image:loc>https://example.com/img.png</image:loc></image:image>
654  </url>
655</urlset>"#;
656        let entries = parse_sitemap(xml);
657        assert_eq!(entries.len(), 1);
658        match &entries[0] {
659            SitemapEntry::Url { loc, .. } => assert_eq!(loc, "https://example.com/page"),
660            SitemapEntry::Sitemap { .. } => panic!("expected Url"),
661        }
662    }
663
664    #[test]
665    fn parse_loc_after_nested_extension() {
666        let xml = r#"<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
667                xmlns:image="http://www.google.com/schemas/sitemap-image/1.1">
668  <url>
669    <image:image><image:loc>https://example.com/img.png</image:loc></image:image>
670    <loc>https://example.com/page</loc>
671    <lastmod>2026-01-01</lastmod>
672  </url>
673</urlset>"#;
674        let entries = parse_sitemap(xml);
675        assert_eq!(entries.len(), 1);
676        match &entries[0] {
677            SitemapEntry::Url { loc, lastmod } => {
678                assert_eq!(loc, "https://example.com/page");
679                assert_eq!(lastmod.as_deref(), Some("2026-01-01"));
680            }
681            SitemapEntry::Sitemap { .. } => panic!("expected Url"),
682        }
683    }
684
685    #[test]
686    fn parse_empty_body_returns_empty() {
687        assert!(parse_sitemap("").is_empty());
688        assert!(parse_sitemap("<html><body>Not Found</body></html>").is_empty());
689    }
690
691    #[test]
692    fn looks_like_html_detects_variants() {
693        assert!(looks_like_html(b"<!DOCTYPE html>"));
694        assert!(looks_like_html(b"<!doctype html>"));
695        assert!(looks_like_html(b"<html lang=\"en\">"));
696        assert!(looks_like_html(b"<HTML>"));
697        assert!(looks_like_html(b"\xef\xbb\xbf<!DOCTYPE html>"));
698        assert!(looks_like_html(b"  \n<!doctype html>"));
699        assert!(looks_like_html(b"\xef\xbb\xbf  <html>"));
700        assert!(!looks_like_html(b"<?xml version=\"1.0\"?>"));
701        assert!(!looks_like_html(b"<urlset>"));
702        assert!(!looks_like_html(b""));
703    }
704
705    #[test]
706    fn validate_entry_rejects_private_ip() {
707        let opts = test_config("https://example.com");
708
709        let mut visited = HashSet::new();
710        let result = validate_entry(
711            "http://127.0.0.1/secret",
712            None,
713            &opts.seed,
714            &RobotsPolicy::Unavailable,
715            &opts,
716            &mut visited,
717        );
718        assert!(result.is_none());
719    }
720
721    #[test]
722    fn validate_entry_rejects_cross_site() {
723        let opts = test_config("https://example.com");
724        let robots = RobotsPolicy::Unavailable;
725        let mut visited = HashSet::new();
726        let result = validate_entry("https://evil.com/page", None, &opts.seed, &robots, &opts, &mut visited);
727        assert!(result.is_none());
728    }
729
730    #[test]
731    fn validate_entry_deduplicates() {
732        let opts = test_config("https://example.com");
733        let robots = RobotsPolicy::Unavailable;
734        let mut visited = HashSet::new();
735        let first = validate_entry(
736            "https://example.com/page",
737            None,
738            &opts.seed,
739            &robots,
740            &opts,
741            &mut visited,
742        );
743        assert!(first.is_some());
744        let second = validate_entry(
745            "https://example.com/page",
746            None,
747            &opts.seed,
748            &robots,
749            &opts,
750            &mut visited,
751        );
752        assert!(second.is_none());
753    }
754
755    #[test]
756    fn validate_entry_rejects_long_url() {
757        let opts = test_config("https://example.com");
758        let robots = RobotsPolicy::Unavailable;
759        let mut visited = HashSet::new();
760        let long_url = format!("https://example.com/{}", "a".repeat(MAP_URL_MAX_LEN));
761        let result = validate_entry(&long_url, None, &opts.seed, &robots, &opts, &mut visited);
762        assert!(result.is_none());
763    }
764
765    #[test]
766    fn discover_sitemaps_includes_robots_and_default() {
767        let seed = Url::parse("https://example.com").unwrap();
768        let robots = RobotsPolicy::Rules(RobotsRules {
769            rules: Vec::new(),
770            sitemaps: vec![Url::parse("https://example.com/custom-sitemap.xml").unwrap()],
771        });
772        let sitemaps = discover_sitemaps(&robots, &seed);
773        assert_eq!(sitemaps.len(), 2);
774        assert_eq!(sitemaps[0].as_str(), "https://example.com/custom-sitemap.xml");
775        assert_eq!(sitemaps[1].as_str(), "https://example.com/sitemap.xml");
776    }
777
778    #[test]
779    fn discover_sitemaps_deduplicates_default() {
780        let seed = Url::parse("https://example.com").unwrap();
781        let robots = RobotsPolicy::Rules(RobotsRules {
782            rules: Vec::new(),
783            sitemaps: vec![Url::parse("https://example.com/sitemap.xml").unwrap()],
784        });
785        let sitemaps = discover_sitemaps(&robots, &seed);
786        assert_eq!(sitemaps.len(), 1);
787    }
788
789    mod integration {
790        use std::time::Duration;
791
792        use tokio::task::spawn_blocking;
793        use url::Url;
794        use wiremock::matchers::{method, path};
795        use wiremock::{Mock, MockServer, ResponseTemplate};
796
797        use crate::map::{
798            MapConfig, MapEntry, build_agent, extract_links, fetch_html, fetch_sitemap, parse_sitemap, run,
799        };
800
801        #[tokio::test]
802        async fn fetch_sitemap_parses_urlset() {
803            let server = MockServer::start().await;
804            let xml = r#"<?xml version="1.0"?><urlset><url><loc>https://example.com/a</loc></url></urlset>"#;
805            Mock::given(method("GET"))
806                .and(path("/sitemap.xml"))
807                .respond_with(ResponseTemplate::new(200).set_body_raw(xml.as_bytes().to_vec(), "application/xml"))
808                .mount(&server)
809                .await;
810
811            let agent = build_agent("test/1.0", Duration::from_secs(5));
812            let url = Url::parse(&format!("{}/sitemap.xml", server.uri())).unwrap();
813            let body = spawn_blocking(move || fetch_sitemap(&agent, &url, &url, &http::HeaderMap::new()))
814                .await
815                .unwrap();
816
817            let entries = parse_sitemap(&body.unwrap());
818            assert_eq!(entries.len(), 1);
819        }
820
821        #[tokio::test]
822        async fn fetch_sitemap_rejects_html_error_page() {
823            let server = MockServer::start().await;
824            Mock::given(method("GET"))
825                .and(path("/sitemap.xml"))
826                .respond_with(ResponseTemplate::new(200).set_body_raw(
827                    b"<!DOCTYPE html><html><body>Not Found</body></html>".to_vec(),
828                    "text/html; charset=utf-8",
829                ))
830                .mount(&server)
831                .await;
832
833            let agent = build_agent("test/1.0", Duration::from_secs(5));
834            let url = Url::parse(&format!("{}/sitemap.xml", server.uri())).unwrap();
835            let body = spawn_blocking(move || fetch_sitemap(&agent, &url, &url, &http::HeaderMap::new()))
836                .await
837                .unwrap();
838
839            assert!(body.is_none());
840        }
841
842        #[tokio::test]
843        async fn fetch_sitemap_returns_none_on_404() {
844            let server = MockServer::start().await;
845            Mock::given(method("GET"))
846                .and(path("/sitemap.xml"))
847                .respond_with(ResponseTemplate::new(404))
848                .mount(&server)
849                .await;
850
851            let agent = build_agent("test/1.0", Duration::from_secs(5));
852            let url = Url::parse(&format!("{}/sitemap.xml", server.uri())).unwrap();
853            let body = spawn_blocking(move || fetch_sitemap(&agent, &url, &url, &http::HeaderMap::new()))
854                .await
855                .unwrap();
856
857            assert!(body.is_none());
858        }
859
860        #[tokio::test]
861        async fn fetch_sitemap_handles_gzip() {
862            use std::io::Write as _;
863
864            use flate2::Compression;
865            use flate2::write::GzEncoder;
866
867            let server = MockServer::start().await;
868            let xml = r#"<?xml version="1.0"?><urlset><url><loc>https://example.com/gz</loc></url></urlset>"#;
869            let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
870            encoder.write_all(xml.as_bytes()).unwrap();
871            let compressed = encoder.finish().unwrap();
872
873            Mock::given(method("GET"))
874                .and(path("/sitemap.xml.gz"))
875                .respond_with(ResponseTemplate::new(200).set_body_raw(compressed, "application/gzip"))
876                .mount(&server)
877                .await;
878
879            let agent = build_agent("test/1.0", Duration::from_secs(5));
880            let url = Url::parse(&format!("{}/sitemap.xml.gz", server.uri())).unwrap();
881            let body = spawn_blocking(move || fetch_sitemap(&agent, &url, &url, &http::HeaderMap::new()))
882                .await
883                .unwrap();
884
885            let entries = parse_sitemap(&body.unwrap());
886            assert_eq!(entries.len(), 1);
887        }
888
889        #[tokio::test]
890        async fn fetch_html_extracts_links() {
891            let server = MockServer::start().await;
892            Mock::given(method("GET"))
893                .and(path("/"))
894                .respond_with(ResponseTemplate::new(200).set_body_raw(
895                    br#"<html><body><a href="/link">x</a></body></html>"#.to_vec(),
896                    "text/html; charset=utf-8",
897                ))
898                .mount(&server)
899                .await;
900
901            let agent = build_agent("test/1.0", Duration::from_secs(5));
902            let seed = Url::parse(&server.uri()).unwrap();
903            let html = spawn_blocking({
904                let seed = seed.clone();
905                move || fetch_html(&agent, &seed, &http::HeaderMap::new())
906            })
907            .await
908            .unwrap()
909            .unwrap();
910
911            let links = extract_links(&html, &seed);
912            assert_eq!(links.len(), 1);
913        }
914
915        async fn check_run(server: &MockServer, configure: impl FnOnce(&mut MapConfig)) -> Vec<MapEntry> {
916            let mut config = MapConfig {
917                seed: Url::parse(&server.uri()).unwrap(),
918                limit: 100,
919                include: None,
920                exclude: None,
921                user_agent: Some("test-bot".into()),
922                timeout: Duration::from_secs(5),
923                no_fallback: false,
924                headers: http::HeaderMap::new(),
925            };
926            configure(&mut config);
927            let mut entries = Vec::new();
928            run(&config, |e| {
929                entries.push(MapEntry {
930                    url: e.url.clone(),
931                    lastmod: e.lastmod.clone(),
932                });
933            })
934            .await;
935            entries
936        }
937
938        #[tokio::test]
939        async fn run_discovers_urls_from_sitemap() {
940            let server = MockServer::start().await;
941            Mock::given(method("GET"))
942                .and(path("/robots.txt"))
943                .respond_with(ResponseTemplate::new(200).set_body_string("User-agent: *\nAllow: /"))
944                .mount(&server)
945                .await;
946            let sitemap = format!(
947                "<urlset><url><loc>{}/page1</loc></url><url><loc>{}/page2</loc></url></urlset>",
948                server.uri(),
949                server.uri()
950            );
951            Mock::given(method("GET"))
952                .and(path("/sitemap.xml"))
953                .respond_with(ResponseTemplate::new(200).set_body_string(sitemap))
954                .mount(&server)
955                .await;
956
957            let entries = check_run(&server, |_| {}).await;
958            assert_eq!(entries.len(), 2);
959            assert!(entries.iter().any(|e| e.url.ends_with("/page1")));
960            assert!(entries.iter().any(|e| e.url.ends_with("/page2")));
961        }
962
963        #[tokio::test]
964        async fn run_respects_limit() {
965            let server = MockServer::start().await;
966            Mock::given(method("GET"))
967                .and(path("/robots.txt"))
968                .respond_with(ResponseTemplate::new(404))
969                .mount(&server)
970                .await;
971            let sitemap = format!(
972                "<urlset><url><loc>{}/a</loc></url><url><loc>{}/b</loc></url><url><loc>{}/c</loc></url></urlset>",
973                server.uri(),
974                server.uri(),
975                server.uri()
976            );
977            Mock::given(method("GET"))
978                .and(path("/sitemap.xml"))
979                .respond_with(ResponseTemplate::new(200).set_body_string(sitemap))
980                .mount(&server)
981                .await;
982
983            let entries = check_run(&server, |c| c.limit = 2).await;
984            assert_eq!(entries.len(), 2);
985        }
986
987        #[tokio::test]
988        async fn run_follows_sitemap_index() {
989            let server = MockServer::start().await;
990            Mock::given(method("GET"))
991                .and(path("/robots.txt"))
992                .respond_with(ResponseTemplate::new(404))
993                .mount(&server)
994                .await;
995            let index = format!(
996                "<sitemapindex><sitemap><loc>{}/sub.xml</loc></sitemap></sitemapindex>",
997                server.uri()
998            );
999            Mock::given(method("GET"))
1000                .and(path("/sitemap.xml"))
1001                .respond_with(ResponseTemplate::new(200).set_body_string(index))
1002                .mount(&server)
1003                .await;
1004            let sub = format!("<urlset><url><loc>{}/deep</loc></url></urlset>", server.uri());
1005            Mock::given(method("GET"))
1006                .and(path("/sub.xml"))
1007                .respond_with(ResponseTemplate::new(200).set_body_string(sub))
1008                .mount(&server)
1009                .await;
1010
1011            let entries = check_run(&server, |_| {}).await;
1012            assert_eq!(entries.len(), 1);
1013            assert!(entries[0].url.ends_with("/deep"));
1014        }
1015
1016        #[tokio::test]
1017        async fn run_falls_back_to_html_links() {
1018            let server = MockServer::start().await;
1019            Mock::given(method("GET"))
1020                .and(path("/robots.txt"))
1021                .respond_with(ResponseTemplate::new(404))
1022                .mount(&server)
1023                .await;
1024            Mock::given(method("GET"))
1025                .and(path("/sitemap.xml"))
1026                .respond_with(ResponseTemplate::new(404))
1027                .mount(&server)
1028                .await;
1029            let html = format!(
1030                r#"<html><body><a href="{}/link1">L1</a><a href="{}/link2">L2</a></body></html>"#,
1031                server.uri(),
1032                server.uri()
1033            );
1034            Mock::given(method("GET"))
1035                .and(path("/"))
1036                .respond_with(ResponseTemplate::new(200).set_body_string(html))
1037                .mount(&server)
1038                .await;
1039
1040            let entries = check_run(&server, |_| {}).await;
1041            assert_eq!(entries.len(), 2);
1042        }
1043
1044        #[tokio::test]
1045        async fn run_no_fallback_skips_html() {
1046            let server = MockServer::start().await;
1047            Mock::given(method("GET"))
1048                .and(path("/robots.txt"))
1049                .respond_with(ResponseTemplate::new(404))
1050                .mount(&server)
1051                .await;
1052            Mock::given(method("GET"))
1053                .and(path("/sitemap.xml"))
1054                .respond_with(ResponseTemplate::new(404))
1055                .mount(&server)
1056                .await;
1057            Mock::given(method("GET"))
1058                .and(path("/"))
1059                .respond_with(
1060                    ResponseTemplate::new(200).set_body_string(r#"<html><body><a href="/link">L</a></body></html>"#),
1061                )
1062                .mount(&server)
1063                .await;
1064
1065            let entries = check_run(&server, |c| c.no_fallback = true).await;
1066            assert_eq!(entries.len(), 0);
1067        }
1068
1069        #[tokio::test]
1070        async fn run_deduplicates_urls() {
1071            let server = MockServer::start().await;
1072            Mock::given(method("GET"))
1073                .and(path("/robots.txt"))
1074                .respond_with(ResponseTemplate::new(404))
1075                .mount(&server)
1076                .await;
1077            let sitemap = format!(
1078                "<urlset><url><loc>{}/dup</loc></url><url><loc>{}/dup</loc></url><url><loc>{}/unique</loc></url></urlset>",
1079                server.uri(),
1080                server.uri(),
1081                server.uri()
1082            );
1083            Mock::given(method("GET"))
1084                .and(path("/sitemap.xml"))
1085                .respond_with(ResponseTemplate::new(200).set_body_string(sitemap))
1086                .mount(&server)
1087                .await;
1088
1089            let entries = check_run(&server, |_| {}).await;
1090            assert_eq!(entries.len(), 2);
1091        }
1092    }
1093}
servo_fetch/map.rs

servo_fetch/
map.rs
