stealthreq 0.1.0

Trait-driven, human-like request mutation primitives for crawlers and scrapers.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117

//! `stealthreq` generates human-like request behavior for scraping and crawler clients.
//!
//! It intentionally avoids hard-coupling to any single HTTP implementation.

mod config;
mod headers;
mod policy;
mod timing;
mod tls;
pub mod waf;

pub use crate::config::StealthProfileConfig;
pub use crate::headers::{HeaderPolicy, HeaderPolicyConfig};
pub use crate::policy::{AppliedRequestProfile, MutableRequest, RequestModifier, StealthPolicy};
pub use crate::timing::{TimingJitter, TimingJitterConfig};
pub use crate::tls::{TlsFingerprint, TlsRotationConfig, TlsRotationPolicy};

use thiserror::Error;

pub type Result<T> = std::result::Result<T, StealthError>;

/// Public error type.
#[derive(Debug, Error)]
pub enum StealthError {
    /// Invalid TOML/config input.
    #[error("configuration error: {0}")]
    Config(String),
    /// Internal policy construction issue.
    #[error("policy error: {0}")]
    Internal(&'static str),
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::policy::RequestModifier;
    use rand::rngs::StdRng;
    use rand::SeedableRng;

    #[derive(Default)]
    struct MockReq {
        headers: Vec<(String, String)>,
    }

    impl MutableRequest for MockReq {
        fn set_header(&mut self, name: &str, value: &str) {
            self.headers.push((name.to_string(), value.to_string()));
        }
    }

    #[test]
    fn toml_config_parse() {
        let toml = r#"
jitter_ms_min = 50
jitter_ms_max = 100
header_budget = 7
rotate_tls = true
seed = 123

[headers]
referer_hosts = ["https://example.com"]
include_pragmas = true

[tls]
enabled = true
"#;
        let cfg = StealthProfileConfig::from_toml(toml).unwrap();
        assert_eq!(cfg.jitter_ms_min, 50);
        assert_eq!(cfg.jitter_ms_max, 100);
        assert_eq!(cfg.header_budget, 7);
    }

    #[test]
    fn policy_runs_end_to_end() {
        let mut req = MockReq::default();
        let profile = StealthPolicy::default();
        let applied = profile.apply(&mut req).unwrap();
        assert!(!req.headers.is_empty());
        assert!(!applied.user_agent.is_empty());
        assert!(!applied.tls_profile.name.is_empty());
        assert!(applied.jitter > std::time::Duration::from_millis(0));
    }

    #[test]
    fn profile_with_seed_matches_applied_headers() {
        let mut req1 = MockReq::default();
        let mut req2 = MockReq::default();
        let mut rng1 = StdRng::seed_from_u64(100);
        let mut rng2 = StdRng::seed_from_u64(100);
        let policy = StealthPolicy::default().with_seed(Some(100));

        let a = policy.apply_with_rng(&mut req1, &mut rng1).unwrap();
        let b = policy.apply_with_rng(&mut req2, &mut rng2).unwrap();

        assert_eq!(a.user_agent, b.user_agent);
        assert_eq!(a.applied_headers, b.applied_headers);
        assert_eq!(a.tls_profile.name, b.tls_profile.name);
    }

    #[test]
    fn custom_config_build() {
        let cfg = StealthProfileConfig {
            jitter_ms_min: 10,
            jitter_ms_max: 20,
            header_budget: 3,
            seed: Some(44),
            rotate_tls: false,
            headers: HeaderPolicyConfig::default(),
            tls: TlsRotationConfig { enabled: false },
        };
        let policy = cfg.build();
        let mut req = MockReq::default();
        let applied = policy.apply(&mut req).unwrap();
        assert!(!applied.applied_headers.is_empty());
        assert!(!applied.user_agent.is_empty());
    }
}