crawlberg 1.0.5

High-performance web crawling engine
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160

//! Integration tests for Bug 1 — escalation with no browser target.
//!
//! When `BrowserMode::Never` is set (or `escalation_strategy = None`), the
//! engine must surface HTTP 403 / WAF-block responses as `Err(CrawlError::...)`
//! rather than silently returning `Ok`.
//!
//! Tests are written RED-first: they prove the bug exists before the fix lands.

use crawlberg::{BrowserMode, CrawlConfig, CrawlError, DispatchProfile, EscalationStrategy, create_engine, scrape};
use wiremock::matchers::{method, path};
use wiremock::{Mock, MockServer, ResponseTemplate};

fn engine_with(config: CrawlConfig) -> crawlberg::CrawlEngineHandle {
    create_engine(Some(config)).expect("engine build must not fail")
}

/// `BrowserOnly` strategy with `BrowserMode::Never`: the policy wants to escalate
/// to Browser, but no browser is available. The engine must surface the 403 as
/// `Err(CrawlError::Forbidden)` rather than returning `Ok` or using browser.
#[tokio::test]
async fn browser_only_strategy_browser_never_returns_forbidden_error() {
    let mock = MockServer::start().await;
    Mock::given(method("GET"))
        .and(path("/forbidden"))
        .respond_with(ResponseTemplate::new(403).set_body_string("nope"))
        .mount(&mock)
        .await;

    let handle = engine_with(CrawlConfig {
        dispatch: Some(DispatchProfile {
            strategy: EscalationStrategy::BrowserOnly,
            ..DispatchProfile::default()
        }),
        browser: crawlberg::BrowserConfig {
            mode: BrowserMode::Never,
            ..Default::default()
        },
        ..CrawlConfig::default()
    });

    let url = format!("{}/forbidden", mock.uri());
    let result = scrape(&handle, &url).await;

    assert!(result.is_err(), "expected Err, got Ok: {result:?}");
    let err = result.unwrap_err();
    assert!(
        matches!(err, CrawlError::Forbidden(_) | CrawlError::WafBlocked { .. }),
        "expected Forbidden or WafBlocked, got: {err:?}"
    );
}

/// Same constraint with a Cloudflare-fingerprinted 403. Engine must return
/// `Err(CrawlError::WafBlocked { .. })` rather than escalating to browser.
#[tokio::test]
async fn browser_only_strategy_browser_never_returns_waf_blocked_error() {
    let mock = MockServer::start().await;
    Mock::given(method("GET"))
        .and(path("/cf"))
        .respond_with(
            ResponseTemplate::new(403)
                .insert_header("server", "cloudflare")
                .set_body_string("<html><head><title>Just a moment...</title></head></html>"),
        )
        .mount(&mock)
        .await;

    let handle = engine_with(CrawlConfig {
        dispatch: Some(DispatchProfile {
            strategy: EscalationStrategy::BrowserOnly,
            ..DispatchProfile::default()
        }),
        browser: crawlberg::BrowserConfig {
            mode: BrowserMode::Never,
            ..Default::default()
        },
        ..CrawlConfig::default()
    });

    let url = format!("{}/cf", mock.uri());
    let result = scrape(&handle, &url).await;

    assert!(result.is_err(), "expected Err, got Ok: {result:?}");
    let err = result.unwrap_err();
    assert!(
        matches!(err, CrawlError::Forbidden(_) | CrawlError::WafBlocked { .. }),
        "expected Forbidden or WafBlocked, got: {err:?}"
    );
}

/// `None` strategy must surface errors unconditionally — even without `BrowserMode::Never`.
#[tokio::test]
async fn none_strategy_browser_auto_returns_forbidden_error() {
    let mock = MockServer::start().await;
    Mock::given(method("GET"))
        .and(path("/forbidden"))
        .respond_with(ResponseTemplate::new(403).set_body_string("access denied"))
        .mount(&mock)
        .await;

    let handle = engine_with(CrawlConfig {
        dispatch: Some(DispatchProfile {
            strategy: EscalationStrategy::None,
            ..DispatchProfile::default()
        }),
        browser: crawlberg::BrowserConfig {
            mode: BrowserMode::Never,
            ..Default::default()
        },
        ..CrawlConfig::default()
    });

    let url = format!("{}/forbidden", mock.uri());
    let result = scrape(&handle, &url).await;

    assert!(result.is_err(), "expected Err, got Ok: {result:?}");
    assert!(
        matches!(
            result.unwrap_err(),
            CrawlError::Forbidden(_) | CrawlError::WafBlocked { .. }
        ),
        "expected Forbidden or WafBlocked"
    );
}

/// None strategy + WAF-fingerprinted 403 must return the original WAF error.
#[tokio::test]
async fn none_strategy_returns_waf_blocked_not_ok() {
    let mock = MockServer::start().await;
    Mock::given(method("GET"))
        .and(path("/akamai"))
        .respond_with(
            ResponseTemplate::new(403)
                .insert_header("server", "AkamaiGHost")
                .set_body_string("<html><body>Access Denied</body></html>"),
        )
        .mount(&mock)
        .await;

    let handle = engine_with(CrawlConfig {
        dispatch: Some(DispatchProfile {
            strategy: EscalationStrategy::None,
            ..DispatchProfile::default()
        }),
        browser: crawlberg::BrowserConfig {
            mode: BrowserMode::Never,
            ..Default::default()
        },
        ..CrawlConfig::default()
    });

    let url = format!("{}/akamai", mock.uri());
    let result = scrape(&handle, &url).await;

    assert!(result.is_err(), "expected Err, got Ok: {result:?}");
    let err = result.unwrap_err();
    assert!(
        matches!(err, CrawlError::Forbidden(_) | CrawlError::WafBlocked { .. }),
        "expected Forbidden or WafBlocked, got: {err:?}"
    );
}