headless_engine/network/
client.rs

1use crate::network::fingerprint::{DeviceProfile, Fingerprint};
2use anyhow::{Context, Result};
3use std::sync::Arc;
4use std::time::Duration;
5
6pub struct FetchResult {
7    pub status: u16,
8    pub final_url: String,
9    pub html: String,
10    pub is_captcha_detected: bool,
11}
12
13pub struct NetworkClient {
14    client: reqwest::Client,
15    pub profile: DeviceProfile,
16    pub fingerprint: Fingerprint,
17    #[allow(dead_code)]
18    pub cookie_jar: Arc<reqwest::cookie::Jar>,
19}
20
21impl NetworkClient {
22    #[allow(dead_code)]
23    pub fn new() -> Result<Self> {
24        Self::with_profile(DeviceProfile::ChromeWindows)
25    }
26
27    pub fn with_profile(profile: DeviceProfile) -> Result<Self> {
28        Self::with_builder_config(profile, None, Duration::from_secs(30), 10, None)
29    }
30
31    pub fn with_builder_config(
32        profile: DeviceProfile,
33        proxy_url: Option<&str>,
34        timeout: Duration,
35        max_redirects: usize,
36        custom_ua: Option<&str>,
37    ) -> Result<Self> {
38        let cookie_jar = Arc::new(reqwest::cookie::Jar::default());
39        let fingerprint = Fingerprint::for_profile(profile);
40        let mut headers = fingerprint.build_headers();
41
42        if let Some(ua) = custom_ua {
43            headers.insert(
44                reqwest::header::USER_AGENT,
45                reqwest::header::HeaderValue::from_str(ua)
46                    .context("Invalid custom User-Agent header")?,
47            );
48        }
49
50        let mut builder = reqwest::Client::builder()
51            .default_headers(headers)
52            .cookie_provider(cookie_jar.clone())
53            .timeout(timeout)
54            .redirect(reqwest::redirect::Policy::limited(max_redirects))
55            .gzip(true)
56            .brotli(true)
57            .deflate(true);
58
59        if let Some(proxy_str) = proxy_url {
60            let proxy = reqwest::Proxy::all(proxy_str)
61                .context(format!("Invalid proxy URL: {}", proxy_str))?;
62            builder = builder.proxy(proxy);
63        }
64
65        // StealthGuard: Pre-warm legitimate session and consent cookies
66        if let Ok(google_url) = "https://www.google.com".parse::<reqwest::Url>() {
67            cookie_jar.add_cookie_str("SOCS=CAESHAgBEhJnd3NfMjAyNDA5MDUtMF9SQzIaAmVuIAEaBgiA_L20Bg; Path=/; Domain=.google.com; Secure", &google_url);
68            cookie_jar.add_cookie_str(
69                "CONSENT=PENDING+987; Path=/; Domain=.google.com; Secure",
70                &google_url,
71            );
72            cookie_jar.add_cookie_str(
73                "AEC=AZ6Zc-Wz2R61_67w88hJ; Path=/; Domain=.google.com; Secure",
74                &google_url,
75            );
76        }
77        if let Ok(ddg_url) = "https://duckduckgo.com".parse::<reqwest::Url>() {
78            cookie_jar.add_cookie_str("5=0; Path=/; Domain=.duckduckgo.com; Secure", &ddg_url);
79            cookie_jar.add_cookie_str("l=en-us; Path=/; Domain=.duckduckgo.com; Secure", &ddg_url);
80        }
81
82        let client = builder.build().context("Failed to build HTTP client")?;
83
84        Ok(Self {
85            client,
86            profile,
87            fingerprint,
88            cookie_jar,
89        })
90    }
91
92    pub fn set_profile(&mut self, profile: DeviceProfile) -> Result<()> {
93        let new_client = Self::with_profile(profile)?;
94        self.client = new_client.client;
95        self.profile = profile;
96        self.fingerprint = new_client.fingerprint;
97        Ok(())
98    }
99
100    pub async fn fetch(&self, url: &str) -> Result<FetchResult> {
101        if url.starts_with("file://") {
102            let file_path = url
103                .trim_start_matches("file:///")
104                .trim_start_matches("file://");
105            let clean_path = file_path.replace('/', "\\");
106            let html = std::fs::read_to_string(&clean_path)
107                .or_else(|_| std::fs::read_to_string(file_path))
108                .context(format!("Failed to read local file: {}", url))?;
109            return Ok(FetchResult {
110                status: 200,
111                final_url: url.to_string(),
112                html,
113                is_captcha_detected: false,
114            });
115        }
116
117        let response = self
118            .client
119            .get(url)
120            .send()
121            .await
122            .context(format!("Failed to send request to {}", url))?;
123
124        let status = response.status().as_u16();
125        let final_url = response.url().to_string();
126        let html = response
127            .text()
128            .await
129            .context("Failed to decode response body")?;
130
131        let is_captcha_detected = html.contains("sorry/index?continue=")
132            || html.contains("Our systems have detected unusual traffic")
133            || html.contains("id=\"captcha-form\"")
134            || (html.contains("challenges.cloudflare.com")
135                && html.contains("cf-turnstile-wrapper"))
136            || html.contains("hcaptcha-box");
137
138        Ok(FetchResult {
139            status,
140            final_url,
141            html,
142            is_captcha_detected,
143        })
144    }
145}
headless_engine/network/client.rs

headless_engine/network/
client.rs
