crawlex 1.0.4

Stealth crawler with Chrome-perfect TLS/H2 fingerprint, render pool, hooks, persistent queue
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77

//! Fase 6 — session scope policy wiring.
//!
//! Verifies `decide_scope` returns the demotion/force scope decisions
//! the crawler uses when `session_scope_auto` is enabled. The crawler
//! hot-path test is in `session_registry.rs`; this file isolates the
//! policy matrix for scope transitions.

use crawlex::antibot::{ChallengeLevel, ChallengeVendor};
use crawlex::config::RenderSessionScope;
use crawlex::policy::{decide_scope, ScopeDecision, ScopeSignal};

#[test]
fn login_page_demotes_registrable_domain_to_origin() {
    assert_eq!(
        decide_scope(
            RenderSessionScope::RegistrableDomain,
            &ScopeSignal::LoginPageDetected,
        ),
        ScopeDecision::DemoteTo(RenderSessionScope::Origin)
    );
}

#[test]
fn login_page_is_noop_when_scope_already_origin() {
    assert_eq!(
        decide_scope(RenderSessionScope::Origin, &ScopeSignal::LoginPageDetected),
        ScopeDecision::Keep
    );
}

#[test]
fn hard_block_demotes_to_url() {
    let decision = decide_scope(
        RenderSessionScope::Origin,
        &ScopeSignal::AntibotHostility(ChallengeVendor::Akamai, ChallengeLevel::HardBlock),
    );
    assert_eq!(decision, ScopeDecision::DemoteTo(RenderSessionScope::Url));
}

#[test]
fn challenge_page_demotes_to_origin() {
    let decision = decide_scope(
        RenderSessionScope::Host,
        &ScopeSignal::AntibotHostility(
            ChallengeVendor::CloudflareJsChallenge,
            ChallengeLevel::ChallengePage,
        ),
    );
    assert_eq!(
        decision,
        ScopeDecision::DemoteTo(RenderSessionScope::Origin)
    );
}

#[test]
fn quarantine_is_forced_regardless_of_current_scope() {
    for scope in [
        RenderSessionScope::RegistrableDomain,
        RenderSessionScope::Host,
        RenderSessionScope::Origin,
        RenderSessionScope::Url,
    ] {
        assert_eq!(
            decide_scope(scope, &ScopeSignal::HostQuarantined),
            ScopeDecision::Force(RenderSessionScope::Url)
        );
    }
}

#[test]
fn suspected_is_not_aggressive_enough_to_demote() {
    let decision = decide_scope(
        RenderSessionScope::RegistrableDomain,
        &ScopeSignal::AntibotHostility(ChallengeVendor::Recaptcha, ChallengeLevel::Suspected),
    );
    assert_eq!(decision, ScopeDecision::Keep);
}