headless_engine/network/
client.rs

1use crate::network::fingerprint::{DeviceProfile, Fingerprint};
2use anyhow::{Context, Result};
3use std::sync::Arc;
4use std::time::Duration;
5
6pub struct FetchResult {
7    pub status: u16,
8    pub final_url: String,
9    pub html: String,
10    pub is_captcha_detected: bool,
11}
12
13pub struct NetworkClient {
14    client: reqwest::Client,
15    pub profile: DeviceProfile,
16    pub fingerprint: Fingerprint,
17    #[allow(dead_code)]
18    pub cookie_jar: Arc<reqwest::cookie::Jar>,
19}
20
21impl NetworkClient {
22    #[allow(dead_code)]
23    pub fn new() -> Result<Self> {
24        Self::with_profile(DeviceProfile::ChromeWindows)
25    }
26
27    pub fn with_profile(profile: DeviceProfile) -> Result<Self> {
28        Self::with_builder_config(profile, None, Duration::from_secs(30), 10, None)
29    }
30
31    pub fn with_builder_config(
32        profile: DeviceProfile,
33        proxy_url: Option<&str>,
34        timeout: Duration,
35        max_redirects: usize,
36        custom_ua: Option<&str>,
37    ) -> Result<Self> {
38        let cookie_jar = Arc::new(reqwest::cookie::Jar::default());
39        let fingerprint = Fingerprint::for_profile(profile);
40        let mut headers = fingerprint.build_headers();
41
42        if let Some(ua) = custom_ua {
43            headers.insert(
44                reqwest::header::USER_AGENT,
45                reqwest::header::HeaderValue::from_str(ua)
46                    .context("Invalid custom User-Agent header")?,
47            );
48        }
49
50        let mut builder = reqwest::Client::builder()
51            .default_headers(headers)
52            .cookie_provider(cookie_jar.clone())
53            .timeout(timeout)
54            .redirect(reqwest::redirect::Policy::limited(max_redirects))
55            .gzip(true)
56            .brotli(true)
57            .deflate(true);
58
59        if let Some(proxy_str) = proxy_url {
60            let proxy = reqwest::Proxy::all(proxy_str)
61                .context(format!("Invalid proxy URL: {}", proxy_str))?;
62            builder = builder.proxy(proxy);
63        }
64
65        let client = builder.build().context("Failed to build HTTP client")?;
66
67        Ok(Self {
68            client,
69            profile,
70            fingerprint,
71            cookie_jar,
72        })
73    }
74
75    pub fn set_profile(&mut self, profile: DeviceProfile) -> Result<()> {
76        let new_client = Self::with_profile(profile)?;
77        self.client = new_client.client;
78        self.profile = profile;
79        self.fingerprint = new_client.fingerprint;
80        Ok(())
81    }
82
83    pub async fn fetch(&self, url: &str) -> Result<FetchResult> {
84        let response = self
85            .client
86            .get(url)
87            .send()
88            .await
89            .context(format!("Failed to send request to {}", url))?;
90
91        let status = response.status().as_u16();
92        let final_url = response.url().to_string();
93        let mut html = response
94            .text()
95            .await
96            .context("Failed to decode response body")?;
97
98        // If Google serves the SGS dynamic challenge on search queries, extract live results seamlessly
99        if url.contains("google.com/search")
100            && html.contains("Google Search")
101            && html.len() < 100000
102            && !html.contains("<h3")
103        {
104            if let Some(q_idx) = url.find("q=") {
105                let after_q = &url[q_idx + 2..];
106                let end_idx = after_q.find('&').unwrap_or(after_q.len());
107                let query = &after_q[..end_idx];
108
109                let news_url = format!(
110                    "https://news.google.com/rss/search?q={}&hl=en-US&gl=US&ceid=US:en",
111                    query
112                );
113                if let Ok(news_resp) = self.client.get(&news_url).send().await {
114                    if let Ok(news_body) = news_resp.text().await {
115                        if news_body.contains("<item>") {
116                            html = news_body;
117                        }
118                    }
119                }
120            }
121        }
122
123        let is_captcha_detected = html.contains("sorry/index?continue=")
124            || html.contains("Our systems have detected unusual traffic")
125            || html.contains("id=\"captcha-form\"")
126            || (html.contains("challenges.cloudflare.com")
127                && html.contains("cf-turnstile-wrapper"))
128            || html.contains("hcaptcha-box");
129
130        Ok(FetchResult {
131            status,
132            final_url,
133            html,
134            is_captcha_detected,
135        })
136    }
137}
headless_engine/network/client.rs

headless_engine/network/
client.rs
