headless_engine/network/
client.rs

1use crate::network::fingerprint::{DeviceProfile, Fingerprint};
2use anyhow::{Context, Result};
3use std::sync::Arc;
4use std::time::Duration;
5
6pub struct FetchResult {
7    pub status: u16,
8    pub final_url: String,
9    pub html: String,
10    pub is_captcha_detected: bool,
11}
12
13pub struct NetworkClient {
14    client: reqwest::Client,
15    pub profile: DeviceProfile,
16    pub fingerprint: Fingerprint,
17    #[allow(dead_code)]
18    pub cookie_jar: Arc<reqwest::cookie::Jar>,
19}
20
21impl NetworkClient {
22    #[allow(dead_code)]
23    pub fn new() -> Result<Self> {
24        Self::with_profile(DeviceProfile::ChromeWindows)
25    }
26
27    pub fn with_profile(profile: DeviceProfile) -> Result<Self> {
28        Self::with_builder_config(profile, None, Duration::from_secs(30), 10, None)
29    }
30
31    pub fn with_builder_config(
32        profile: DeviceProfile,
33        proxy_url: Option<&str>,
34        timeout: Duration,
35        max_redirects: usize,
36        custom_ua: Option<&str>,
37    ) -> Result<Self> {
38        let cookie_jar = Arc::new(reqwest::cookie::Jar::default());
39        let fingerprint = Fingerprint::for_profile(profile);
40        let mut headers = fingerprint.build_headers();
41
42        if let Some(ua) = custom_ua {
43            headers.insert(
44                reqwest::header::USER_AGENT,
45                reqwest::header::HeaderValue::from_str(ua)
46                    .context("Invalid custom User-Agent header")?,
47            );
48        }
49
50        let mut builder = reqwest::Client::builder()
51            .default_headers(headers)
52            .cookie_provider(cookie_jar.clone())
53            .timeout(timeout)
54            .redirect(reqwest::redirect::Policy::limited(max_redirects))
55            .gzip(true)
56            .brotli(true)
57            .deflate(true);
58
59        let effective_proxy = proxy_url
60            .map(|s| s.to_string())
61            .or_else(|| std::env::var("HTTPS_PROXY").ok())
62            .or_else(|| std::env::var("https_proxy").ok())
63            .or_else(|| std::env::var("HTTP_PROXY").ok())
64            .or_else(|| std::env::var("http_proxy").ok())
65            .or_else(|| std::env::var("ALL_PROXY").ok())
66            .or_else(|| std::env::var("all_proxy").ok());
67
68        if let Some(proxy_str) = &effective_proxy {
69            if !proxy_str.is_empty() {
70                let proxy = reqwest::Proxy::all(proxy_str)
71                    .context(format!("Invalid proxy URL: {}", proxy_str))?;
72                builder = builder.proxy(proxy);
73            }
74        }
75
76        // StealthGuard: Pre-warm legitimate session and consent cookies
77        if let Ok(google_url) = "https://www.google.com".parse::<reqwest::Url>() {
78            cookie_jar.add_cookie_str("SOCS=CAESHAgBEhJnd3NfMjAyNDA5MDUtMF9SQzIaAmVuIAEaBgiA_L20Bg; Path=/; Domain=.google.com; Secure", &google_url);
79            cookie_jar.add_cookie_str(
80                "CONSENT=YES+cb.20230531-04-p0.en+FX+908; Path=/; Domain=.google.com; Secure",
81                &google_url,
82            );
83            cookie_jar.add_cookie_str(
84                "AEC=AZ6Zc-Wz2R61_67w88hJ; Path=/; Domain=.google.com; Secure",
85                &google_url,
86            );
87            cookie_jar.add_cookie_str(
88                "1P_JAR=2024-09-05-12; Path=/; Domain=.google.com; Secure",
89                &google_url,
90            );
91        }
92        if let Ok(ddg_url) = "https://duckduckgo.com".parse::<reqwest::Url>() {
93            cookie_jar.add_cookie_str("5=0; Path=/; Domain=.duckduckgo.com; Secure", &ddg_url);
94            cookie_jar.add_cookie_str("l=en-us; Path=/; Domain=.duckduckgo.com; Secure", &ddg_url);
95        }
96
97        let client = builder.build().context("Failed to build HTTP client")?;
98
99        Ok(Self {
100            client,
101            profile,
102            fingerprint,
103            cookie_jar,
104        })
105    }
106
107    pub fn set_profile(&mut self, profile: DeviceProfile) -> Result<()> {
108        let new_client = Self::with_profile(profile)?;
109        self.client = new_client.client;
110        self.profile = profile;
111        self.fingerprint = new_client.fingerprint;
112        Ok(())
113    }
114
115    pub async fn fetch(&self, url: &str) -> Result<FetchResult> {
116        if url.starts_with("file://") {
117            let file_path = url
118                .trim_start_matches("file:///")
119                .trim_start_matches("file://");
120            let clean_path = file_path.replace('/', "\\");
121            let html = std::fs::read_to_string(&clean_path)
122                .or_else(|_| std::fs::read_to_string(file_path))
123                .context(format!("Failed to read local file: {}", url))?;
124            return Ok(FetchResult {
125                status: 200,
126                final_url: url.to_string(),
127                html,
128                is_captcha_detected: false,
129            });
130        }
131
132        let target_url = if url.contains("google.com/search") && !url.contains("hl=") {
133            if url.contains('?') {
134                format!("{}&hl=en&gl=us", url)
135            } else {
136                format!("{}?hl=en&gl=us", url)
137            }
138        } else {
139            url.to_string()
140        };
141
142        let response = self
143            .client
144            .get(&target_url)
145            .send()
146            .await
147            .context(format!("Failed to send request to {}", url))?;
148
149        let mut status = response.status().as_u16();
150        let mut final_url = response.url().to_string();
151        let bytes = response
152            .bytes()
153            .await
154            .context("Failed to read response body bytes")?;
155        let mut html = String::from_utf8_lossy(&bytes).to_string();
156
157        // If Google serves an interstitial fallback link, follow it directly
158        if html.contains("having trouble accessing Google Search") || html.contains("emsg=SG_REL") {
159            if let Some(start_idx) = html.find("href=\"/search?") {
160                if let Some(end_idx) = html[start_idx + 6..].find('\"') {
161                    let rel_url = &html[start_idx + 6..start_idx + 6 + end_idx];
162                    let clean_rel = rel_url.replace("&amp;", "&");
163                    let redirect_target = format!("https://www.google.com{}", clean_rel);
164                    if let Ok(next_resp) = self.client.get(&redirect_target).send().await {
165                        status = next_resp.status().as_u16();
166                        final_url = next_resp.url().to_string();
167                        if let Ok(next_bytes) = next_resp.bytes().await {
168                            html = String::from_utf8_lossy(&next_bytes).to_string();
169                        }
170                    }
171                }
172            }
173        }
174
175        let mut is_captcha_detected = html.contains("sorry/index?continue=")
176            || html.contains("Our systems have detected unusual traffic")
177            || html.contains("id=\"captcha-form\"")
178            || (html.contains("challenges.cloudflare.com")
179                && html.contains("cf-turnstile-wrapper"))
180            || html.contains("hcaptcha-box");
181
182        // If Google serves an anti-bot fallback, hydrate with live rendered DOM
183        if (url.contains("google.com/search") && (html.contains("having trouble accessing Google Search") || html.contains("emsg=SG_REL"))) || is_captcha_detected {
184            if let Some(rendered_html) = crate::dom::screenshot::RealBrowserScreenshot::dump_rendered_dom(url).await {
185                html = rendered_html;
186                status = 200;
187                final_url = url.to_string();
188                is_captcha_detected = false;
189            }
190        }
191
192        Ok(FetchResult {
193            status,
194            final_url,
195            html,
196            is_captcha_detected,
197        })
198    }
199}
headless_engine/network/client.rs

headless_engine/network/
client.rs
