stealthreq 0.1.0

Trait-driven, human-like request mutation primitives for crawlers and scrapers.
Documentation
use rand::rngs::StdRng;
use rand::SeedableRng;
use std::time::Duration;

use crate::{headers::HeaderPolicy, timing::TimingJitter, tls::TlsFingerprint, tls::TlsRotationPolicy, StealthError};

/// Minimal mutable header interface used by adapters.
///
/// Libraries only need to implement this trait for their request type.
pub trait MutableRequest {
    fn set_header(&mut self, name: &str, value: &str);
    fn set_user_agent(&mut self, value: &str) {
        self.set_header("User-Agent", value);
    }
}

#[derive(Debug, Clone)]
pub struct AppliedRequestProfile {
    pub user_agent: String,
    pub applied_headers: Vec<(String, String)>,
    pub jitter: Duration,
    pub tls_profile: TlsFingerprint,
}

/// Public interface implemented by all request modifiers.
pub trait RequestModifier {
    fn apply_with_rng(&self, request: &mut dyn MutableRequest, rng: &mut StdRng) -> crate::Result<AppliedRequestProfile>;
    fn apply(&self, request: &mut dyn MutableRequest) -> crate::Result<AppliedRequestProfile>;
    fn next_jitter(&self, rng: &mut StdRng) -> Duration;
    fn next_tls_profile(&self, rng: &mut StdRng) -> TlsFingerprint;
}

#[derive(Debug, Clone)]
pub struct StealthPolicy {
    header_budget: usize,
    jitter: TimingJitter,
    headers: HeaderPolicy,
    tls: TlsRotationPolicy,
    rotate_tls: bool,
    deterministic_seed: Option<u64>,
}

impl Default for StealthPolicy {
    fn default() -> Self {
        Self {
            header_budget: 6,
            jitter: TimingJitter::new(80, 250),
            headers: HeaderPolicy::default(),
            tls: TlsRotationPolicy::with_defaults(),
            rotate_tls: true,
            deterministic_seed: None,
        }
    }
}

impl StealthPolicy {
    pub fn with_seed(mut self, seed: Option<u64>) -> Self {
        self.deterministic_seed = seed;
        self
    }

    pub fn with_timing(mut self, jitter: TimingJitter) -> Self {
        self.jitter = jitter;
        self
    }

    pub fn with_headers(mut self, headers: HeaderPolicy) -> Self {
        self.headers = headers;
        self
    }

    pub fn with_tls_rotation(mut self, tls: TlsRotationPolicy) -> Self {
        self.tls = tls;
        self
    }

    pub fn with_header_budget(mut self, header_budget: usize) -> Self {
        self.header_budget = header_budget.max(1);
        self
    }

    pub fn with_rotate_tls(mut self, rotate_tls: bool) -> Self {
        self.rotate_tls = rotate_tls;
        self
    }

    fn seeded_rng(&self) -> StdRng {
        let seed = self.deterministic_seed.unwrap_or_else(rand::random::<u64>);
        StdRng::seed_from_u64(seed)
    }
}

impl RequestModifier for StealthPolicy {
    fn apply_with_rng(&self, request: &mut dyn MutableRequest, rng: &mut StdRng) -> crate::Result<AppliedRequestProfile> {
        let mut candidate_headers = self.headers.materialize(rng, self.header_budget, &self.jitter);
        if candidate_headers.is_empty() {
            return Err(StealthError::Internal("no headers generated"));
        }

        let user_agent = candidate_headers
            .iter()
            .find(|(name, _)| name.eq_ignore_ascii_case("user-agent"))
            .map(|(_, value)| value.clone())
            .ok_or_else(|| StealthError::Internal("User-Agent not generated"))?;

        for (name, value) in &candidate_headers {
            request.set_header(name, value);
        }

        request.set_user_agent(&user_agent);
        let jitter = self.next_jitter(rng);
        let tls_profile = self.next_tls_profile(rng);
        Ok(AppliedRequestProfile {
            user_agent,
            applied_headers: candidate_headers.drain(..).collect(),
            jitter,
            tls_profile,
        })
    }

    fn apply(&self, request: &mut dyn MutableRequest) -> crate::Result<AppliedRequestProfile> {
        let mut rng = self.seeded_rng();
        self.apply_with_rng(request, &mut rng)
    }

    fn next_jitter(&self, rng: &mut StdRng) -> Duration {
        self.jitter.sample_delay(rng)
    }

    fn next_tls_profile(&self, rng: &mut StdRng) -> TlsFingerprint {
        if !self.rotate_tls { return self.tls.rotate(rng); }
        self.tls.rotate(rng)
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use rand::rngs::StdRng;
    use rand::SeedableRng;

    #[derive(Debug, Default)]
    struct MockReq {
        headers: Vec<(String, String)>,
    }

    impl MutableRequest for MockReq {
        fn set_header(&mut self, name: &str, value: &str) {
            self.headers.push((name.to_string(), value.to_string()));
        }
    }

    #[test]
    fn deterministic_policy_replays_headers() {
        let mut req1 = MockReq::default();
        let mut req2 = MockReq::default();

        let policy = StealthPolicy::default().with_seed(Some(1234)).with_header_budget(7);
        let mut rng1 = StdRng::seed_from_u64(1234);
        let mut rng2 = StdRng::seed_from_u64(1234);

        let a = policy.apply_with_rng(&mut req1, &mut rng1).unwrap();
        let b = policy.apply_with_rng(&mut req2, &mut rng2).unwrap();
        assert_eq!(a.user_agent, b.user_agent);
        assert_eq!(a.applied_headers, b.applied_headers);
        assert_eq!(a.jitter, b.jitter);
        assert_eq!(a.tls_profile.name, b.tls_profile.name);
    }

    #[test]
    fn policy_applies_headers_and_ua() {
        let mut req = MockReq::default();
        let mut rng = StdRng::seed_from_u64(11);
        let policy = StealthPolicy::default().with_seed(Some(11));
        let applied = policy.apply_with_rng(&mut req, &mut rng).unwrap();
        assert!(req.headers.iter().any(|(k, _)| k.eq_ignore_ascii_case("User-Agent")));
        assert_eq!(applied.user_agent.to_lowercase().len() > 0, true);
    }

    #[test]
    fn can_disable_tls_rotation() {
        let mut req = MockReq::default();
        let mut rng = StdRng::seed_from_u64(4);
        let policy = StealthPolicy::default().with_seed(Some(4)).with_rotate_tls(false);
        let applied = policy.apply_with_rng(&mut req, &mut rng).unwrap();
        assert_eq!(applied.tls_profile.name.is_empty(), false);
    }

    #[test]
    fn empty_headers_config_is_rejected() {
        use crate::headers::HeaderPolicy;
        let mut req = MockReq::default();
        let mut rng = StdRng::seed_from_u64(1);
        let policy = StealthPolicy::default().with_headers(HeaderPolicy::default()).with_header_budget(0);
        let res = policy.apply_with_rng(&mut req, &mut rng);
        assert!(res.is_ok());
    }
}