mara 0.2.0

A high-performance scraper that clears challenges over a rotating pool of egress IPs.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99

use std::time::{SystemTime, UNIX_EPOCH};

use serde::{Deserialize, Serialize};

const FALLBACK_TTL_SECS: f64 = 25.0 * 60.0;

#[derive(Clone, Serialize, Deserialize)]
pub struct Clearance {
    cookies: Vec<(String, String)>,
    pub user_agent: String,
    pub captured_unix: f64,
    pub expires_unix: Option<f64>,
    pub host: String,
}

impl Clearance {
    pub fn new(
        cookies: Vec<(String, String)>,
        user_agent: String,
        expires_unix: Option<f64>,
        host: String,
    ) -> Clearance {
        Clearance {
            cookies,
            user_agent,
            captured_unix: now_unix(),
            expires_unix,
            host,
        }
    }

    pub fn cookie_header(&self) -> String {
        self.cookies
            .iter()
            .map(|(n, v)| format!("{n}={v}"))
            .collect::<Vec<_>>()
            .join("; ")
    }

    pub fn is_stale(&self) -> bool {
        let now = now_unix();
        match self.expires_unix {
            Some(exp) => now >= exp,
            None => now - self.captured_unix > FALLBACK_TTL_SECS,
        }
    }

    pub fn age_secs(&self) -> u64 {
        (now_unix() - self.captured_unix).max(0.0) as u64
    }
}

pub fn now_unix() -> f64 {
    SystemTime::now()
        .duration_since(UNIX_EPOCH)
        .map(|d| d.as_secs_f64())
        .unwrap_or(0.0)
}

#[cfg(test)]
mod tests {
    use super::*;

    fn fixed(cookies: &[(&str, &str)]) -> Clearance {
        Clearance {
            cookies: cookies
                .iter()
                .map(|(n, v)| (n.to_string(), v.to_string()))
                .collect(),
            user_agent: "Mozilla/5.0 … Chrome/147.0.0.0 Safari/537.36".into(),
            captured_unix: now_unix(),
            expires_unix: None,
            host: String::new(),
        }
    }

    #[test]
    fn cookie_header_joins_with_semicolons() {
        let c = fixed(&[("cf_clearance", "abc"), ("__cf_bm", "xyz")]);
        assert_eq!(c.cookie_header(), "cf_clearance=abc; __cf_bm=xyz");
    }

    #[test]
    fn single_cookie_has_no_separator() {
        assert_eq!(
            fixed(&[("cf_clearance", "abc")]).cookie_header(),
            "cf_clearance=abc"
        );
    }

    #[test]
    fn expiry_drives_staleness() {
        let mut c = fixed(&[("cf_clearance", "x")]);
        c.expires_unix = Some(now_unix() - 1.0);
        assert!(c.is_stale());
        c.expires_unix = Some(now_unix() + 3600.0);
        assert!(!c.is_stale());
    }
}