crawlberg 1.0.5

High-performance web crawling engine
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95

//! Tests that streaming APIs emit CrawlEvent::Error when a seed URL fails with a 5xx response.
//!
//! Regression coverage for: batch_crawl_stream and crawl_stream silently swallowing seed-level
//! HTTP 500 errors — they returned Ok+Complete without emitting Error because the 5xx was
//! caught in Phase 1 (resolve_initial_redirects) before reaching process_fetch_result.

use crawlberg::{CrawlConfig, CrawlEvent, batch_crawl_stream, crawl_stream, create_engine};
use futures::StreamExt;
use wiremock::matchers::{method, path};
use wiremock::{Mock, MockServer, ResponseTemplate};

/// batch_crawl_stream must emit CrawlEvent::Error for each seed URL that returns 500.
#[tokio::test]
async fn test_batch_crawl_stream_seed_500_emits_error_event() {
    let mock = MockServer::start().await;

    Mock::given(method("GET"))
        .and(path("/ok"))
        .respond_with(
            ResponseTemplate::new(200)
                .set_body_string("<html><body>OK</body></html>")
                .append_header("content-type", "text/html"),
        )
        .mount(&mock)
        .await;

    Mock::given(method("GET"))
        .and(path("/fail"))
        .respond_with(ResponseTemplate::new(500).set_body_string("Internal Server Error"))
        .mount(&mock)
        .await;

    let config = CrawlConfig {
        max_depth: Some(0),
        respect_robots_txt: false,
        ..Default::default()
    };
    let engine = create_engine(Some(config)).unwrap();

    let urls = vec![format!("{}/ok", mock.uri()), format!("{}/fail", mock.uri())];

    let stream = batch_crawl_stream(&engine, urls)
        .await
        .expect("batch_crawl_stream should not fail to initialise");
    let events: Vec<CrawlEvent> = stream
        .map(|r| r.expect("stream item should not be Err"))
        .collect()
        .await;

    assert!(
        events.iter().any(|e| matches!(e, CrawlEvent::Page { .. })),
        "stream should contain at least one Page event for the 200 seed"
    );
    assert!(
        events.iter().any(|e| matches!(e, CrawlEvent::Error { .. })),
        "stream must emit Error event for the 500 seed; got events: {events:?}"
    );
    assert!(
        events.iter().any(|e| matches!(e, CrawlEvent::Complete { .. })),
        "stream must emit Complete event"
    );
}

/// crawl_stream must emit CrawlEvent::Error when the single seed URL returns 500.
#[tokio::test]
async fn test_crawl_stream_seed_500_emits_error_event() {
    let mock = MockServer::start().await;

    Mock::given(method("GET"))
        .and(path("/fail"))
        .respond_with(ResponseTemplate::new(500).set_body_string("Internal Server Error"))
        .mount(&mock)
        .await;

    let config = CrawlConfig {
        max_depth: Some(0),
        respect_robots_txt: false,
        ..Default::default()
    };
    let engine = create_engine(Some(config)).unwrap();
    let url = format!("{}/fail", mock.uri());

    let stream = crawl_stream(&engine, &url)
        .await
        .expect("crawl_stream should not fail to initialise");
    let events: Vec<CrawlEvent> = stream
        .map(|r| r.expect("stream item should not be Err"))
        .collect()
        .await;

    assert!(
        events.iter().any(|e| matches!(e, CrawlEvent::Error { .. })),
        "crawl_stream must emit Error event for 500 seed; got events: {events:?}"
    );
}