cloudscraper_rs/modules/anti_detection/
mod.rs

1//! Traffic pattern and anti-detection utilities.
2//!
3//! Provides request obfuscation, burst control, and adaptive cooldowns for the
4//! layer that prepares requests before they hit the network.
5
6use http::{HeaderMap, HeaderName, HeaderValue, Method};
7use rand::Rng;
8use std::collections::{HashMap, VecDeque};
9use std::time::{Duration, Instant};
10use url::Url;
11
12/// Configuration toggles for anti-detection behaviour.
13#[derive(Debug, Clone)]
14pub struct AntiDetectionConfig {
15    pub randomize_headers: bool,
16    pub inject_noise_headers: bool,
17    pub header_noise_range: (usize, usize),
18    pub burst_window: Duration,
19    pub max_requests_per_window: usize,
20    pub cooldown: Duration,
21    pub failure_cooldown: Duration,
22    pub jitter_range: (f32, f32),
23}
24
25impl Default for AntiDetectionConfig {
26    fn default() -> Self {
27        Self {
28            randomize_headers: true,
29            inject_noise_headers: true,
30            header_noise_range: (1, 3),
31            burst_window: Duration::from_secs(30),
32            max_requests_per_window: 10,
33            cooldown: Duration::from_secs(3),
34            failure_cooldown: Duration::from_secs(20),
35            jitter_range: (0.85, 1.25),
36        }
37    }
38}
39
40/// Context object mutated by anti-detection strategies before dispatch.
41#[derive(Debug, Clone)]
42pub struct AntiDetectionContext {
43    pub url: Url,
44    pub method: Method,
45    pub headers: HeaderMap,
46    pub body_size: usize,
47    pub user_agent: Option<String>,
48    pub delay_hint: Option<Duration>,
49    pub metadata: HashMap<String, String>,
50}
51
52impl AntiDetectionContext {
53    pub fn new(url: Url, method: Method) -> Self {
54        Self {
55            url,
56            method,
57            headers: HeaderMap::new(),
58            body_size: 0,
59            user_agent: None,
60            delay_hint: None,
61            metadata: HashMap::new(),
62        }
63    }
64
65    pub fn with_headers(mut self, headers: HeaderMap) -> Self {
66        self.headers = headers;
67        self
68    }
69
70    pub fn set_body_size(&mut self, size: usize) {
71        self.body_size = size;
72    }
73
74    pub fn set_user_agent(&mut self, value: impl Into<String>) {
75        self.user_agent = Some(value.into());
76    }
77
78    pub fn delay_hint(&self) -> Option<Duration> {
79        self.delay_hint
80    }
81}
82
83/// Trait describing an anti detection step.
84pub trait AntiDetectionStrategy: Send + Sync {
85    fn prepare_request(&mut self, domain: &str, ctx: &mut AntiDetectionContext);
86    fn record_response(&mut self, domain: &str, status: u16, latency: Duration);
87}
88
89/// Default anti-detection layer combining header jitter, burst throttling, and
90/// cooldown management.
91#[derive(Debug)]
92pub struct DefaultAntiDetection {
93    config: AntiDetectionConfig,
94    per_domain: HashMap<String, DomainAntiDetection>,
95}
96
97#[derive(Debug)]
98struct DomainAntiDetection {
99    recent_requests: VecDeque<Instant>,
100    failure_streak: u8,
101    cooldown_until: Option<Instant>,
102    rolling_latency: VecDeque<f32>,
103    fingerprint_salt: u32,
104}
105
106impl Default for DomainAntiDetection {
107    fn default() -> Self {
108        Self {
109            recent_requests: VecDeque::with_capacity(32),
110            failure_streak: 0,
111            cooldown_until: None,
112            rolling_latency: VecDeque::with_capacity(32),
113            fingerprint_salt: rand::thread_rng().r#gen(),
114        }
115    }
116}
117
118impl DefaultAntiDetection {
119    pub fn new(config: AntiDetectionConfig) -> Self {
120        Self {
121            config,
122            per_domain: HashMap::new(),
123        }
124    }
125
126    pub fn config(&self) -> &AntiDetectionConfig {
127        &self.config
128    }
129
130    fn state_mut(&mut self, domain: &str) -> &mut DomainAntiDetection {
131        self.per_domain.entry(domain.to_string()).or_default()
132    }
133
134    fn prune_old_requests(state: &mut DomainAntiDetection, window: Duration) {
135        let cutoff = Instant::now() - window;
136        while matches!(state.recent_requests.front(), Some(ts) if *ts < cutoff) {
137            state.recent_requests.pop_front();
138        }
139    }
140
141    fn enforce_burst_limits(
142        config: &AntiDetectionConfig,
143        state: &mut DomainAntiDetection,
144        ctx: &mut AntiDetectionContext,
145    ) {
146        Self::prune_old_requests(state, config.burst_window);
147        if state.recent_requests.len() > config.max_requests_per_window && ctx.delay_hint.is_none()
148        {
149            ctx.delay_hint = Some(config.cooldown);
150        }
151    }
152
153    fn maybe_apply_cooldown(state: &mut DomainAntiDetection, ctx: &mut AntiDetectionContext) {
154        if let Some(until) = state.cooldown_until {
155            let now = Instant::now();
156            if now < until {
157                let remaining = until - now;
158                ctx.delay_hint = Some(ctx.delay_hint.map_or(remaining, |hint| hint.max(remaining)));
159            } else {
160                state.cooldown_until = None;
161            }
162        }
163    }
164
165    fn randomize_headers(
166        config: &AntiDetectionConfig,
167        state: &DomainAntiDetection,
168        ctx: &mut AntiDetectionContext,
169    ) {
170        if !config.randomize_headers {
171            return;
172        }
173
174        let mut rng = rand::thread_rng();
175        // Rotate a few headers that commonly trigger fingerprinting.
176        static TARGET_HEADERS: &[&str] = &[
177            "accept-language",
178            "sec-fetch-site",
179            "sec-fetch-mode",
180            "sec-fetch-dest",
181        ];
182
183        for header in TARGET_HEADERS {
184            if let Ok(name) = HeaderName::from_lowercase(header.as_bytes())
185                && rng.gen_bool(0.3)
186            {
187                let value = random_header_value(&mut rng, state.fingerprint_salt);
188                ctx.headers.insert(name, value);
189            }
190        }
191
192        if let Some(agent) = &ctx.user_agent {
193            let name = HeaderName::from_static("user-agent");
194            let value = HeaderValue::from_str(agent)
195                .unwrap_or_else(|_| HeaderValue::from_static("Mozilla/5.0"));
196            ctx.headers.insert(name, value);
197        }
198    }
199
200    fn inject_noise_headers(config: &AntiDetectionConfig, ctx: &mut AntiDetectionContext) {
201        if !config.inject_noise_headers {
202            return;
203        }
204
205        let mut rng = rand::thread_rng();
206        let (min, max) = config.header_noise_range;
207        let upper = max.max(min);
208        let count = rng.gen_range(min..=upper);
209
210        for _ in 0..count {
211            let token: String = (0..8)
212                .map(|_| format!("{:x}", rng.r#gen::<u16>()))
213                .collect();
214            let name = format!("x-cf-client-{}", token);
215            if let Ok(header_name) = HeaderName::from_bytes(name.as_bytes())
216                && let Ok(header_value) =
217                    HeaderValue::from_str(&format!("{}-{}", rng.r#gen::<u32>(), ctx.body_size))
218            {
219                ctx.headers.insert(header_name, header_value);
220            }
221        }
222    }
223}
224
225impl AntiDetectionStrategy for DefaultAntiDetection {
226    fn prepare_request(&mut self, domain: &str, ctx: &mut AntiDetectionContext) {
227        let config = self.config.clone();
228        {
229            let state = self.state_mut(domain);
230            state.recent_requests.push_back(Instant::now());
231            Self::enforce_burst_limits(&config, state, ctx);
232            Self::maybe_apply_cooldown(state, ctx);
233            Self::randomize_headers(&config, state, ctx);
234        }
235
236        Self::inject_noise_headers(&config, ctx);
237
238        // Apply jitter hint so that timing layer can increase randomness.
239        let jitter = {
240            let mut rng = rand::thread_rng();
241            rng.gen_range(config.jitter_range.0..=config.jitter_range.1)
242        };
243        ctx.metadata
244            .insert("anti_detection_jitter".into(), format!("{:.3}", jitter));
245    }
246
247    fn record_response(&mut self, domain: &str, status: u16, latency: Duration) {
248        let failure_cooldown = self.config.failure_cooldown;
249        let state = self.state_mut(domain);
250        let success = status < 500;
251
252        if !success {
253            state.failure_streak = state.failure_streak.saturating_add(1);
254            state.cooldown_until = Some(Instant::now() + failure_cooldown);
255        } else {
256            state.failure_streak = 0;
257        }
258
259        if state.rolling_latency.len() == 32 {
260            state.rolling_latency.pop_front();
261        }
262        state
263            .rolling_latency
264            .push_back(latency.as_secs_f32().min(30.0));
265    }
266}
267
268fn random_header_value<R: Rng + ?Sized>(rng: &mut R, salt: u32) -> HeaderValue {
269    let seed = rng.r#gen::<u32>() ^ salt;
270    let choices = [
271        format!("same-origin;sid={:x}", seed),
272        format!("cross-site;hash={:x}", seed.rotate_left(5)),
273        format!("none;trace={:x}", seed.rotate_right(7)),
274    ];
275    HeaderValue::from_str(&choices[rng.gen_range(0..choices.len())])
276        .unwrap_or_else(|_| HeaderValue::from_static("same-origin"))
277}
278
279#[cfg(test)]
280mod tests {
281    use super::*;
282
283    #[test]
284    fn applies_delay_hint_when_bursting() {
285        let mut strategy = DefaultAntiDetection::new(AntiDetectionConfig {
286            max_requests_per_window: 2,
287            burst_window: Duration::from_secs(60),
288            cooldown: Duration::from_secs(5),
289            ..Default::default()
290        });
291
292        let url = Url::parse("https://example.com").unwrap();
293        let method = Method::GET;
294
295        let mut ctx1 = AntiDetectionContext::new(url.clone(), method.clone());
296        strategy.prepare_request("example.com", &mut ctx1);
297        assert!(ctx1.delay_hint.is_none());
298
299        let mut ctx2 = AntiDetectionContext::new(url.clone(), method.clone());
300        strategy.prepare_request("example.com", &mut ctx2);
301        assert!(ctx2.delay_hint.is_none());
302
303        let mut ctx3 = AntiDetectionContext::new(url, method);
304        strategy.prepare_request("example.com", &mut ctx3);
305        assert!(ctx3.delay_hint.is_some());
306    }
307}
cloudscraper_rs/modules/anti_detection/mod.rs

cloudscraper_rs/modules/anti_detection/
mod.rs
