crawlex 1.0.6

Stealth crawler with Chrome-perfect TLS/H2 fingerprint, render pool, hooks, persistent queue
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111

//! StatusPattern source — Hot tier (B3).
//!
//! Block-shape heuristic: a 403 or 503 response with a very small body
//! and no `Server` header is the shape of "we got blocked, vendor
//! refuses to identify itself". Emits a low-confidence
//! `Antibot::Unknown` detection — `BlockPattern` in B6 will provide
//! the deeper variant; this source is the cheap heuristic.

use crate::fingerprint::detection::{Category, Detection, Evidence, EvidenceSource, Tier, Vendor};
use crate::fingerprint::target::TargetContext;

use super::Source;

#[derive(Default)]
pub struct StatusPatternSource;

impl StatusPatternSource {
    pub fn new() -> Self {
        Self
    }
}

const SMALL_BODY_THRESHOLD: usize = 1024;

impl Source for StatusPatternSource {
    fn name(&self) -> &'static str {
        "status_pattern"
    }

    fn tier(&self) -> Tier {
        Tier::Hot
    }

    fn analyze(&self, ctx: &TargetContext<'_>) -> Vec<Detection> {
        if !matches!(ctx.status, 403 | 503) {
            return Vec::new();
        }
        if ctx.body.len() >= SMALL_BODY_THRESHOLD {
            return Vec::new();
        }
        if ctx.headers.get("server").is_some() {
            // A normal origin returned 403/503 with a small body —
            // doesn't fit the "vendor hidden" pattern.
            return Vec::new();
        }
        vec![Detection::from_single(
            Category::Antibot,
            Vendor::Unknown,
            Evidence::new(
                EvidenceSource::StatusPattern,
                format!(
                    "status={} small body ({}B) no server header",
                    ctx.status,
                    ctx.body.len()
                ),
                4,
            ),
        )]
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use http::HeaderMap;
    use url::Url;

    fn ctx_with<'a>(
        headers: &'a HeaderMap,
        url: &'a Url,
        status: u16,
        body: &'a [u8],
    ) -> TargetContext<'a> {
        TargetContext::http_only(url, status, headers, body)
    }

    #[test]
    fn fires_on_403_small_no_server() {
        let h = HeaderMap::new();
        let u: Url = "https://example.com/".parse().unwrap();
        let dets = StatusPatternSource::new().analyze(&ctx_with(&h, &u, 403, b"blocked"));
        assert_eq!(dets.len(), 1);
        assert_eq!(dets[0].vendor, Vendor::Unknown);
    }

    #[test]
    fn skips_on_200() {
        let h = HeaderMap::new();
        let u: Url = "https://example.com/".parse().unwrap();
        let dets = StatusPatternSource::new().analyze(&ctx_with(&h, &u, 200, b""));
        assert!(dets.is_empty());
    }

    #[test]
    fn skips_on_403_large_body() {
        let h = HeaderMap::new();
        let u: Url = "https://example.com/".parse().unwrap();
        let big = vec![b'x'; 2048];
        let dets = StatusPatternSource::new().analyze(&ctx_with(&h, &u, 403, &big));
        assert!(dets.is_empty());
    }

    #[test]
    fn skips_when_server_header_present() {
        let mut h = HeaderMap::new();
        h.insert("server", "nginx".parse().unwrap());
        let u: Url = "https://example.com/".parse().unwrap();
        let dets = StatusPatternSource::new().analyze(&ctx_with(&h, &u, 403, b"x"));
        assert!(dets.is_empty());
    }
}