stealthscraper_rs/
geo.rs

1//! Geo/locale consistency: country codes, a curated country -> locale table, and
2//! a resolver port for discovering a proxy's exit country.
3//!
4//! Modern anti-bot systems cross-check the egress IP's geolocation against the
5//! browser's declared locale (Accept-Language, `navigator.languages`, timezone).
6//! A mismatch — e.g. a German IP with a US English, `America/New_York` browser —
7//! is a strong bot signal. This module supplies the building blocks the scraper
8//! uses to keep those layers coherent, **proxy-led**: the egress proxy's country
9//! is the source of truth and the browser locale is derived to match.
10
11/// An ISO 3166-1 alpha-2 country code (stored uppercase, e.g. `DE`).
12#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
13pub struct CountryCode([u8; 2]);
14
15impl CountryCode {
16    /// Parse a two-letter country code, normalising to uppercase.
17    ///
18    /// Returns `None` unless the input is exactly two ASCII letters.
19    pub fn new(code: &str) -> Option<Self> {
20        let bytes = code.as_bytes();
21        if bytes.len() == 2 && bytes.iter().all(u8::is_ascii_alphabetic) {
22            Some(Self([
23                bytes[0].to_ascii_uppercase(),
24                bytes[1].to_ascii_uppercase(),
25            ]))
26        } else {
27            None
28        }
29    }
30
31    /// The uppercase two-letter code as a string slice.
32    pub fn as_str(&self) -> &str {
33        // SAFETY-equivalent: bytes are ASCII letters by construction in `new`.
34        std::str::from_utf8(&self.0).expect("country code is valid ASCII by construction")
35    }
36}
37
38impl std::fmt::Display for CountryCode {
39    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
40        f.write_str(self.as_str())
41    }
42}
43
44/// A coherent locale bundle for a country: the values that must agree with the
45/// egress IP across the HTTP, JS, and timezone layers.
46#[derive(Debug, Clone, PartialEq, Eq)]
47pub struct Locale {
48    /// The country this locale represents.
49    pub country: CountryCode,
50    /// `Accept-Language` header value (e.g. `de-DE,de;q=0.9,en;q=0.8`).
51    pub accept_language: String,
52    /// `navigator.languages` list (e.g. `["de-DE", "de", "en"]`).
53    pub languages: Vec<String>,
54    /// IANA timezone id (e.g. `Europe/Berlin`).
55    pub timezone: String,
56}
57
58impl Locale {
59    /// The curated locale for a country, or `None` if we have no data for it.
60    ///
61    /// The table is intentionally a representative subset; unknown countries
62    /// return `None` so the caller can fall back rather than apply a wrong locale.
63    pub fn for_country(country: CountryCode) -> Option<Self> {
64        let (accept_language, languages, timezone): (&str, &[&str], &str) = match country.as_str() {
65            "US" => ("en-US,en;q=0.9", &["en-US", "en"], "America/New_York"),
66            "GB" => ("en-GB,en;q=0.9", &["en-GB", "en"], "Europe/London"),
67            "CA" => (
68                "en-CA,en;q=0.9,fr-CA;q=0.8",
69                &["en-CA", "en", "fr-CA"],
70                "America/Toronto",
71            ),
72            "AU" => ("en-AU,en;q=0.9", &["en-AU", "en"], "Australia/Sydney"),
73            "DE" => (
74                "de-DE,de;q=0.9,en;q=0.8",
75                &["de-DE", "de", "en"],
76                "Europe/Berlin",
77            ),
78            "FR" => (
79                "fr-FR,fr;q=0.9,en;q=0.8",
80                &["fr-FR", "fr", "en"],
81                "Europe/Paris",
82            ),
83            "ES" => (
84                "es-ES,es;q=0.9,en;q=0.8",
85                &["es-ES", "es", "en"],
86                "Europe/Madrid",
87            ),
88            "IT" => (
89                "it-IT,it;q=0.9,en;q=0.8",
90                &["it-IT", "it", "en"],
91                "Europe/Rome",
92            ),
93            "NL" => (
94                "nl-NL,nl;q=0.9,en;q=0.8",
95                &["nl-NL", "nl", "en"],
96                "Europe/Amsterdam",
97            ),
98            "PL" => (
99                "pl-PL,pl;q=0.9,en;q=0.8",
100                &["pl-PL", "pl", "en"],
101                "Europe/Warsaw",
102            ),
103            "SE" => (
104                "sv-SE,sv;q=0.9,en;q=0.8",
105                &["sv-SE", "sv", "en"],
106                "Europe/Stockholm",
107            ),
108            "BR" => (
109                "pt-BR,pt;q=0.9,en;q=0.8",
110                &["pt-BR", "pt", "en"],
111                "America/Sao_Paulo",
112            ),
113            "MX" => (
114                "es-MX,es;q=0.9,en;q=0.8",
115                &["es-MX", "es", "en"],
116                "America/Mexico_City",
117            ),
118            "JP" => (
119                "ja-JP,ja;q=0.9,en;q=0.8",
120                &["ja-JP", "ja", "en"],
121                "Asia/Tokyo",
122            ),
123            "IN" => (
124                "en-IN,en;q=0.9,hi;q=0.8",
125                &["en-IN", "en", "hi"],
126                "Asia/Kolkata",
127            ),
128            _ => return None,
129        };
130        Some(Self {
131            country,
132            accept_language: accept_language.to_string(),
133            languages: languages.iter().map(|s| (*s).to_string()).collect(),
134            timezone: timezone.to_string(),
135        })
136    }
137
138    /// The primary BCP-47 language tag (first in [`Self::languages`]), used for
139    /// CDP `Emulation.setLocaleOverride`.
140    pub fn primary_language(&self) -> &str {
141        self.languages.first().map_or("en-US", String::as_str)
142    }
143}
144
145/// Output port for discovering the exit country of a proxy URL.
146///
147/// Implementations may consult a local GeoIP database or a remote API. None ship
148/// by default (to avoid bundling data files or making network calls); explicit
149/// per-proxy country tags are the dependency-free path, with this port as the
150/// pluggable fallback for dynamic resolution.
151pub trait GeoResolver: Send + Sync {
152    /// Best-effort country for the given proxy URL, or `None` if unknown.
153    fn country_of(&self, proxy_url: &str) -> Option<CountryCode>;
154}
155
156/// Parse an `Accept-Language` header into an ordered `navigator.languages` list,
157/// dropping the `;q=` quality weights (e.g. `de-DE,de;q=0.9` -> `["de-DE","de"]`).
158pub fn languages_from_accept_language(accept_language: &str) -> Vec<String> {
159    accept_language
160        .split(',')
161        .filter_map(|part| {
162            let tag = part.split(';').next()?.trim();
163            (!tag.is_empty()).then(|| tag.to_string())
164        })
165        .collect()
166}
167
168#[cfg(test)]
169mod tests {
170    use super::*;
171
172    #[test]
173    fn country_code_parses_and_normalises() {
174        assert_eq!(CountryCode::new("de").unwrap().as_str(), "DE");
175        assert_eq!(CountryCode::new("US").unwrap().to_string(), "US");
176        assert!(CountryCode::new("USA").is_none());
177        assert!(CountryCode::new("1").is_none());
178        assert!(CountryCode::new("u1").is_none());
179    }
180
181    #[test]
182    fn locale_for_known_country_is_coherent() {
183        let de = Locale::for_country(CountryCode::new("DE").unwrap()).unwrap();
184        assert_eq!(de.timezone, "Europe/Berlin");
185        assert_eq!(de.languages, vec!["de-DE", "de", "en"]);
186        assert!(de.accept_language.starts_with("de-DE"));
187        assert_eq!(de.primary_language(), "de-DE");
188    }
189
190    #[test]
191    fn locale_for_unknown_country_is_none() {
192        assert!(Locale::for_country(CountryCode::new("ZZ").unwrap()).is_none());
193    }
194
195    #[test]
196    fn locale_table_is_coherent_for_every_supported_country() {
197        let supported = [
198            "US", "GB", "CA", "AU", "DE", "FR", "ES", "IT", "NL", "PL", "SE", "BR", "MX", "JP",
199            "IN",
200        ];
201        for code in supported {
202            let cc = CountryCode::new(code).unwrap();
203            let loc =
204                Locale::for_country(cc).unwrap_or_else(|| panic!("missing locale for {code}"));
205            assert_eq!(loc.country, cc);
206            assert!(!loc.accept_language.is_empty(), "{code} accept_language");
207            assert!(!loc.languages.is_empty(), "{code} languages");
208            assert!(
209                loc.timezone.contains('/'),
210                "{code} timezone looks like IANA"
211            );
212            assert_eq!(loc.primary_language(), loc.languages[0]);
213        }
214    }
215
216    #[test]
217    fn primary_language_falls_back_when_empty() {
218        let loc = Locale {
219            country: CountryCode::new("US").unwrap(),
220            accept_language: String::new(),
221            languages: Vec::new(),
222            timezone: String::new(),
223        };
224        assert_eq!(loc.primary_language(), "en-US");
225    }
226
227    #[test]
228    fn languages_from_accept_language_strips_quality() {
229        assert_eq!(
230            languages_from_accept_language("de-DE,de;q=0.9,en;q=0.8"),
231            vec!["de-DE", "de", "en"]
232        );
233        assert_eq!(
234            languages_from_accept_language("en-US,en;q=0.9"),
235            vec!["en-US", "en"]
236        );
237        assert!(languages_from_accept_language("").is_empty());
238    }
239}
stealthscraper_rs/geo.rs

stealthscraper_rs/
geo.rs
