crawlberg 1.0.5

High-performance web crawling engine
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149

//! Integration tests for the `soft_http_errors` configuration flag.
//!
//! Covers the five scenarios from the task spec:
//! 1. Direct 404 raises when `soft_http_errors` is `false` (default).
//! 2. Direct 404 returns `Ok(ScrapeResult { status_code: 404 })` when enabled.
//! 3. Redirected 404 (302→404) returns `Ok` regardless of the flag.
//! 4. Direct 403 raises when `soft_http_errors` is `false` (default).
//! 5. Direct 403 returns `Ok(ScrapeResult { status_code: 403 })` when enabled.

use crawlberg::{BrowserMode, CrawlConfig, CrawlError, create_engine, scrape};
use wiremock::matchers::{method, path};
use wiremock::{Mock, MockServer, ResponseTemplate};

fn engine_with_config(mut config: CrawlConfig) -> crawlberg::CrawlEngineHandle {
    config.browser.mode = BrowserMode::Never;
    create_engine(Some(config)).expect("engine build must not fail")
}

/// With the default config (`soft_http_errors = false`), a bare 404 response
/// must propagate as `Err(CrawlError::NotFound)`.
#[tokio::test]
async fn direct_404_raises_when_soft_errors_disabled() {
    let mock = MockServer::start().await;

    Mock::given(method("GET"))
        .and(path("/not-found"))
        .respond_with(ResponseTemplate::new(404))
        .mount(&mock)
        .await;

    let handle = engine_with_config(CrawlConfig::default());
    let url = format!("{}/not-found", mock.uri());
    let result = scrape(&handle, &url).await;

    assert!(result.is_err(), "expected Err, got Ok: {result:?}");
    assert!(
        matches!(result.unwrap_err(), CrawlError::NotFound(_)),
        "expected CrawlError::NotFound"
    );
}

/// With `soft_http_errors = true`, a bare 404 must be returned as
/// `Ok(ScrapeResult { status_code: 404, .. })` rather than an error.
#[tokio::test]
async fn direct_404_returns_result_when_soft_errors_enabled() {
    let mock = MockServer::start().await;

    Mock::given(method("GET"))
        .and(path("/not-found"))
        .respond_with(ResponseTemplate::new(404))
        .mount(&mock)
        .await;

    let handle = engine_with_config(CrawlConfig {
        soft_http_errors: true,
        ..CrawlConfig::default()
    });
    let url = format!("{}/not-found", mock.uri());
    let result = scrape(&handle, &url).await;

    assert!(result.is_ok(), "expected Ok, got Err: {:?}", result.err());
    let page = result.unwrap();
    assert_eq!(page.status_code, 404, "status_code must be 404");
    assert!(page.html.is_empty(), "body must be empty for synthesised 404");
}

/// A 302→404 chain must always surface as `Ok(ScrapeResult { status_code: 404 })`
/// regardless of the `soft_http_errors` setting, because the caller opted into
/// redirect-following.
#[tokio::test]
async fn redirected_404_returns_result_regardless_of_soft_errors() {
    let mock = MockServer::start().await;

    Mock::given(method("GET"))
        .and(path("/start"))
        .respond_with(
            ResponseTemplate::new(302)
                .append_header("location", "/not-found")
                .append_header("content-type", "text/html"),
        )
        .mount(&mock)
        .await;

    Mock::given(method("GET"))
        .and(path("/not-found"))
        .respond_with(ResponseTemplate::new(404))
        .mount(&mock)
        .await;

    let handle = engine_with_config(CrawlConfig::default());
    let url = format!("{}/start", mock.uri());
    let result = scrape(&handle, &url).await;

    assert!(
        result.is_ok(),
        "redirected 404 must return Ok regardless of soft_http_errors: {:?}",
        result.err()
    );
    let page = result.unwrap();
    assert_eq!(page.status_code, 404, "status_code must be 404 after redirect chain");
}

/// With the default config (`soft_http_errors = false`), a bare 403 response
/// must propagate as `Err(CrawlError::Forbidden)`.
#[tokio::test]
async fn direct_403_raises_when_soft_errors_disabled() {
    let mock = MockServer::start().await;

    Mock::given(method("GET"))
        .and(path("/forbidden"))
        .respond_with(ResponseTemplate::new(403))
        .mount(&mock)
        .await;

    let handle = engine_with_config(CrawlConfig::default());
    let url = format!("{}/forbidden", mock.uri());
    let result = scrape(&handle, &url).await;

    assert!(result.is_err(), "expected Err, got Ok: {result:?}");
    assert!(
        matches!(result.unwrap_err(), CrawlError::Forbidden(_)),
        "expected CrawlError::Forbidden"
    );
}

/// With `soft_http_errors = true`, a bare 403 must be returned as
/// `Ok(ScrapeResult { status_code: 403, .. })` rather than an error.
#[tokio::test]
async fn direct_403_returns_result_when_soft_errors_enabled() {
    let mock = MockServer::start().await;

    Mock::given(method("GET"))
        .and(path("/forbidden"))
        .respond_with(ResponseTemplate::new(403))
        .mount(&mock)
        .await;

    let handle = engine_with_config(CrawlConfig {
        soft_http_errors: true,
        ..CrawlConfig::default()
    });
    let url = format!("{}/forbidden", mock.uri());
    let result = scrape(&handle, &url).await;

    assert!(result.is_ok(), "expected Ok, got Err: {:?}", result.err());
    let page = result.unwrap();
    assert_eq!(page.status_code, 403, "status_code must be 403");
    assert!(page.html.is_empty(), "body must be empty for synthesised 403");
}