spider 2.52.0

A web crawler and scraper, building blocks for data curation workloads.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147

//! End-to-end DOI multi-hop redirect: crawl_smart and crawl (chrome).
//!
//! `https://doi.org/10.1007/978-981-96-6596-9_10` redirects through ~5 hops
//! to `link.springer.com/chapter/10.1007/...`. Asserts both crawl paths:
//!   - report a non-empty body for the seed,
//!   - record `final_redirect_destination` pointing at the springer chapter,
//!   - extract secondary links that resolve against the redirect host.
//!
//! Run: RUN_LIVE_TESTS=1 cargo test -p spider --test doi_smart_repro \
//!      --features "smart chrome chrome_intercept" --release -- --ignored --nocapture

#[cfg(all(feature = "chrome", feature = "smart"))]
mod repro {
    use spider::page::Page;
    use spider::tokio;
    use spider::website::Website;
    use std::time::{Duration, Instant};

    const DOI_URL: &str = "https://doi.org/10.1007/978-981-96-6596-9_10";
    const SPRINGER_CHAPTER: &str = "https://link.springer.com/chapter/10.1007/978-981-96-6596-9_10";

    fn run_live_tests() -> bool {
        matches!(
            std::env::var("RUN_LIVE_TESTS")
                .unwrap_or_default()
                .trim()
                .to_ascii_lowercase()
                .as_str(),
            "1" | "true" | "yes" | "on"
        )
    }

    fn build(url: &str) -> Website {
        let mut w = Website::new(url);
        w.with_limit(2)
            .with_request_timeout(Some(Duration::from_secs(45)))
            .with_crawl_timeout(Some(Duration::from_secs(120)))
            .with_redirect_limit(15)
            .with_respect_robots_txt(false);
        w
    }

    async fn drain(label: &'static str, mut w: Website, chrome: bool) -> (Vec<Page>, Duration) {
        let mut rx = w.subscribe(64);
        let collector = tokio::spawn(async move {
            let mut pages = Vec::new();
            while let Ok(p) = rx.recv().await {
                eprintln!(
                    "[{label}] url={} status={} bytes={} final_redirect={:?}",
                    p.get_url(),
                    p.status_code.as_u16(),
                    p.get_html_bytes_u8().len(),
                    p.final_redirect_destination,
                );
                pages.push(p);
            }
            pages
        });
        let start = Instant::now();
        let _ = tokio::time::timeout(Duration::from_secs(180), async {
            if chrome {
                w.crawl().await;
            } else {
                w.crawl_smart().await;
            }
        })
        .await;
        let elapsed = start.elapsed();
        w.unsubscribe();
        let pages = collector.await.unwrap_or_default();
        (pages, elapsed)
    }

    fn assert_secondary_on_springer(label: &str, pages: &[Page]) {
        for p in pages.iter().filter(|p| p.get_url() != DOI_URL) {
            let url = p.get_url().to_string();
            assert!(
                !url.starts_with("https://doi.org/") && !url.starts_with("https://www.doi.org/"),
                "[{label}] secondary leaked back to doi.org: {url}"
            );
        }
    }

    #[tokio::test]
    #[ignore]
    async fn doi_smart_path() {
        if !run_live_tests() {
            eprintln!("SKIP");
            return;
        }
        let _ = env_logger::try_init();
        let (pages, elapsed) = drain("smart", build(DOI_URL), false).await;
        eprintln!("[smart] elapsed={:?} pages={}", elapsed, pages.len());

        let seed = pages
            .iter()
            .find(|p| p.get_url() == DOI_URL)
            .expect("seed page");
        assert_eq!(seed.status_code.as_u16(), 200, "seed status");
        assert!(
            seed.get_html_bytes_u8().len() > 100_000,
            "seed body too small"
        );
        assert_eq!(
            seed.final_redirect_destination.as_deref(),
            Some(SPRINGER_CHAPTER),
            "seed final_redirect mismatch"
        );
        assert_secondary_on_springer("smart", &pages);
    }

    #[tokio::test]
    #[ignore]
    async fn doi_chrome_path() {
        if !run_live_tests() {
            eprintln!("SKIP");
            return;
        }
        let _ = env_logger::try_init();
        let (pages, elapsed) = drain("chrome", build(DOI_URL), true).await;
        eprintln!("[chrome] elapsed={:?} pages={}", elapsed, pages.len());

        if pages.is_empty() {
            eprintln!("[chrome] SKIP — no pages emitted (chrome unavailable)");
            return;
        }

        let seed = pages
            .iter()
            .find(|p| p.get_url() == DOI_URL)
            .expect("seed page");
        assert_eq!(seed.status_code.as_u16(), 200, "[chrome] seed status");
        assert!(
            seed.get_html_bytes_u8().len() > 100_000,
            "[chrome] seed body too small"
        );
        let final_url = seed
            .final_redirect_destination
            .as_deref()
            .unwrap_or_default();
        assert!(
            final_url.starts_with("https://link.springer.com/chapter/10.1007/"),
            "[chrome] seed final_redirect mismatch: {final_url:?}"
        );
        assert_secondary_on_springer("chrome", &pages);
    }
}