Skip to main content

fetchium_core/search/
google.rs

1//! Google Search backend — HTTP scraper (default) + headless Chromium (--features headless).
2//!
3//! ## Default mode (no extra flags)
4//! Uses `reqwest` with browser-realistic headers to scrape Google's standard
5//! search endpoint. Works without any API keys or browser downloads. Google
6//! occasionally serves CAPTCHA pages — the backend detects them and returns
7//! empty results gracefully so the orchestrator falls back to other backends.
8//!
9//! ## Headless mode (`--features headless`)
10//! Uses `chromiumoxide` BrowserPool for full JS rendering. Higher fidelity
11//! but requires compiling Chromium bindings (~5 min) and a Chrome binary.
12//!
13//! ## Anti-detection
14//! Chrome/121 User-Agent, Accept-Language, Referer headers; 500ms inter-page
15//! delay; CAPTCHA detection on first-page empty or block-page markers.
16
17use crate::error::FetchiumResult;
18use crate::search::{SearchBackend, SearchContext, TimeRange};
19use crate::types::{BackendId, ResultItem};
20use async_trait::async_trait;
21use tracing::debug;
22#[cfg(not(feature = "headless"))]
23use tracing::info;
24#[cfg(feature = "headless")]
25use tracing::warn;
26
27/// Max IP-rotation retries on CAPTCHA/block detection (2 = 1 cached + 1 fresh IP).
28#[cfg(not(feature = "headless"))]
29const MAX_IP_RETRIES: usize = 2;
30
31/// Browser User-Agent matching Chrome 121 on Windows — reduces bot-detection.
32#[cfg(not(feature = "headless"))]
33const BROWSER_UA: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) \
34    AppleWebKit/537.36 (KHTML, like Gecko) \
35    Chrome/121.0.0.0 Safari/537.36";
36
37/// Google SERP scraper.
38///
39/// Without `--features headless`: lightweight HTTP scraper, no extra setup.
40/// With `--features headless`: headless Chromium for JS-rendered results.
41pub struct GoogleBackend {
42    #[cfg(feature = "headless")]
43    pool: std::sync::Arc<crate::browser::pool::BrowserPool>,
44    #[cfg(not(feature = "headless"))]
45    http: crate::http::HttpClient,
46}
47
48impl GoogleBackend {
49    /// Create a headless-Chromium backend (requires `--features headless`).
50    #[cfg(feature = "headless")]
51    pub fn new(pool: std::sync::Arc<crate::browser::pool::BrowserPool>) -> Self {
52        Self { pool }
53    }
54
55    /// Create an HTTP-scraper backend (default, no extra features required).
56    #[cfg(not(feature = "headless"))]
57    pub fn new_http(http: crate::http::HttpClient) -> Self {
58        Self { http }
59    }
60
61    /// Build a Google search URL for the given query and page (0-indexed).
62    fn build_url(query: &str, page: usize) -> String {
63        Self::build_url_with_tbs(query, page, "")
64    }
65
66    fn build_url_with_tbs(query: &str, page: usize, tbs: &str) -> String {
67        let start = page * 10;
68        let encoded = url::form_urlencoded::Serializer::new(String::new())
69            .append_key_only(query)
70            .finish();
71        let tbs_param = if tbs.is_empty() {
72            String::new()
73        } else {
74            format!("&tbs={tbs}")
75        };
76        // pws=0  — disable personalised results for consistent scraping
77        format!("https://www.google.com/search?q={encoded}&start={start}&hl=en&num=10&gl=us&pws=0{tbs_param}")
78    }
79
80    fn time_range_to_tbs(tr: Option<TimeRange>) -> &'static str {
81        match tr {
82            Some(TimeRange::Day) => "qdr:d",
83            Some(TimeRange::Week) => "qdr:w",
84            Some(TimeRange::Month) => "qdr:m",
85            Some(TimeRange::Year) => "qdr:y",
86            None => "",
87        }
88    }
89
90    /// Fetch a Google SERP page with automatic IP rotation on block/CAPTCHA.
91    ///
92    /// Attempt 0 uses the cached residential client (fast path).
93    /// Attempts 1+ use a fresh client (new residential IP via DataImpulse pool_max_idle=0).
94    /// Returns `None` if all retries are exhausted with no results.
95    #[cfg(not(feature = "headless"))]
96    async fn fetch_page_with_retry(
97        &self,
98        url: &str,
99        page: usize,
100        locale: Option<&str>,
101    ) -> Option<Vec<ResultItem>> {
102        for attempt in 0..MAX_IP_RETRIES {
103            let client = if attempt == 0 {
104                self.http
105                    .client_for_domain_with_locale("google.com", locale)
106            } else {
107                info!(
108                    "Google: IP block detected — rotating to new residential IP (attempt {}/{})",
109                    attempt + 1,
110                    MAX_IP_RETRIES
111                );
112                // Brief pause before IP rotation to avoid hammering
113                tokio::time::sleep(std::time::Duration::from_millis(300 * attempt as u64)).await;
114                self.http
115                    .fresh_client_for_domain_with_locale("google.com", locale)
116            };
117
118            let result = client
119                .get(url)
120                .header("User-Agent", BROWSER_UA)
121                .header(
122                    "Accept",
123                    "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
124                )
125                .header("Accept-Language", "en-US,en;q=0.9")
126                .header("Referer", "https://www.google.com/")
127                .header("Cache-Control", "no-cache")
128                .send()
129                .await;
130
131            match result {
132                Ok(resp) if resp.status().is_success() => match resp.text().await {
133                    Ok(html) => {
134                        let results = Self::parse_serp(&html, page);
135                        if !results.is_empty() {
136                            return Some(results);
137                        }
138                        // Empty results = CAPTCHA/block → rotate IP
139                        debug!("Google: empty SERP page {page} (attempt {attempt}) — rotating IP");
140                    }
141                    Err(e) => {
142                        debug!("Google: body read error attempt {attempt}: {e}");
143                    }
144                },
145                Ok(resp) if resp.status().as_u16() == 429 || resp.status().as_u16() == 403 => {
146                    debug!(
147                        "Google: HTTP {} attempt {attempt} — rotating IP",
148                        resp.status()
149                    );
150                }
151                Ok(resp) => {
152                    debug!("Google: HTTP {} for page {page}", resp.status());
153                    return None; // Non-retryable error
154                }
155                Err(e) => {
156                    debug!("Google: request error attempt {attempt}: {e}");
157                }
158            }
159        }
160
161        debug!("Google: all {MAX_IP_RETRIES} IP rotation attempts exhausted for page {page}");
162        None
163    }
164
165    /// Parse a Google SERP HTML page into [`ResultItem`]s.
166    ///
167    /// Returns an empty vec on CAPTCHA / block pages — the caller should warn
168    /// and return early rather than trying subsequent pages.
169    fn parse_serp(html: &str, page: usize) -> Vec<ResultItem> {
170        use scraper::{Html, Selector};
171
172        // ── CAPTCHA / block detection ────────────────────────────────────────
173        // Google serves several kinds of block pages:
174        // 1. Redirects to `accounts.google.com` (sign-in wall)
175        // 2. Inline CAPTCHA challenge with text "detected unusual traffic"
176        // 3. "please enable cookies" / JS-only pages
177        if html.contains("accounts.google.com")
178            || html.contains("detected unusual traffic")
179            || html.contains("please enable cookies")
180            || (html.len() < 5_000 && html.to_lowercase().contains("captcha"))
181        {
182            return vec![];
183        }
184
185        let document = Html::parse_document(html);
186        let mut results = Vec::new();
187
188        // Google's SERP DOM (stable across most Google updates):
189        //  <div class="g">                   ← organic result wrapper
190        //    <div class="yuRUbf">
191        //      <a href="https://...">
192        //        <h3 class="LC20lb">Title</h3>
193        //      </a>
194        //    </div>
195        //    <div class="VwiC3b">Snippet</div>
196        //  </div>
197        let container_sel = Selector::parse("div.g, div[data-hveid]").unwrap();
198        let title_sel = Selector::parse("h3").unwrap();
199        let link_sel = Selector::parse("a[href]").unwrap();
200        let snippet_sel = Selector::parse("div.VwiC3b, span.aCOpRe, div[data-sncf]").unwrap();
201
202        let mut rank = (page * 10 + 1) as u32;
203
204        for container in document.select(&container_sel) {
205            let title = container
206                .select(&title_sel)
207                .next()
208                .map(|e| e.text().collect::<String>().trim().to_string())
209                .unwrap_or_default();
210
211            if title.is_empty() {
212                continue;
213            }
214
215            let url = container
216                .select(&link_sel)
217                .next()
218                .and_then(|e| e.value().attr("href"))
219                .filter(|href| href.starts_with("http"))
220                .filter(|href| !href.contains("google.com"))
221                .map(|s| s.to_string())
222                .unwrap_or_default();
223
224            if url.is_empty() {
225                continue;
226            }
227
228            let snippet = container
229                .select(&snippet_sel)
230                .next()
231                .map(|e| e.text().collect::<String>().trim().to_string())
232                .unwrap_or_default();
233
234            results.push(ResultItem {
235                title,
236                url,
237                snippet,
238                rank,
239                backend: BackendId::Google,
240                score: None,
241                published_date: None,
242            });
243            rank += 1;
244        }
245
246        debug!(
247            "Google parse_serp: {} results from page {}",
248            results.len(),
249            page
250        );
251
252        results
253    }
254}
255
256#[async_trait]
257impl SearchBackend for GoogleBackend {
258    fn id(&self) -> BackendId {
259        BackendId::Google
260    }
261
262    fn requires_headless(&self) -> bool {
263        cfg!(feature = "headless")
264    }
265
266    async fn search(&self, query: &str, max_results: u32) -> FetchiumResult<Vec<ResultItem>> {
267        // ── Non-headless path: lightweight HTTP scraper ───────────────────────
268        #[cfg(not(feature = "headless"))]
269        {
270            let pages = (max_results as usize).div_ceil(10).min(2);
271            let mut all_results = Vec::new();
272
273            for page in 0..pages {
274                let url = Self::build_url(query, page);
275                match self.fetch_page_with_retry(&url, page, None).await {
276                    Some(results) => all_results.extend(results),
277                    None => break, // CAPTCHA exhausted or non-retryable
278                }
279                if page + 1 < pages {
280                    tokio::time::sleep(std::time::Duration::from_millis(300)).await;
281                }
282            }
283
284            return Ok(all_results.into_iter().take(max_results as usize).collect());
285        }
286
287        // ── Headless path: chromiumoxide BrowserPool ─────────────────────────
288        #[cfg(feature = "headless")]
289        {
290            let pages = (max_results as usize).div_ceil(10).min(3);
291            let mut all_results = Vec::new();
292
293            for page in 0..pages {
294                let url = Self::build_url(query, page);
295
296                match self.pool.acquire_tab().await {
297                    Ok(tab) => {
298                        if page > 0 {
299                            let delay_ms = 200 + (rand_u64() % 600);
300                            tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await;
301                        }
302
303                        // Wait for Google organic results to render (JS-heavy SERP)
304                        match tab.navigate_and_wait(&url, "div.g", 15_000).await {
305                            Ok(_) => match tab.content().await {
306                                Ok(html) => {
307                                    all_results.extend(Self::parse_serp(&html, page));
308                                }
309                                Err(e) => {
310                                    warn!("Google content error page {page}: {e}");
311                                }
312                            },
313                            Err(e) => {
314                                warn!("Google navigate error page {page}: {e}");
315                            }
316                        }
317                    }
318                    Err(e) => {
319                        warn!("Google tab acquire failed: {e}");
320                        break;
321                    }
322                }
323            }
324
325            Ok(all_results.into_iter().take(max_results as usize).collect())
326        }
327    }
328
329    async fn search_with_context(
330        &self,
331        query: &str,
332        max_results: u32,
333        ctx: &SearchContext,
334    ) -> FetchiumResult<Vec<ResultItem>> {
335        let tbs = Self::time_range_to_tbs(ctx.time_range);
336        if tbs.is_empty() {
337            return self.search(query, max_results).await;
338        }
339
340        // Non-headless path with tbs date filtering + IP rotation
341        #[cfg(not(feature = "headless"))]
342        {
343            let pages = (max_results as usize).div_ceil(10).min(2);
344            let mut all_results = Vec::new();
345            let locale = ctx.locale.as_deref();
346
347            for page in 0..pages {
348                let url = Self::build_url_with_tbs(query, page, tbs);
349                match self.fetch_page_with_retry(&url, page, locale).await {
350                    Some(results) => all_results.extend(results),
351                    None => break,
352                }
353                if page + 1 < pages {
354                    tokio::time::sleep(std::time::Duration::from_millis(300)).await;
355                }
356            }
357
358            return Ok(all_results.into_iter().take(max_results as usize).collect());
359        }
360
361        // Headless path — fallback to default search (tbs not easily injectable)
362        #[cfg(feature = "headless")]
363        self.search(query, max_results).await
364    }
365}
366
367/// Xorshift pseudo-random for anti-detection delays (no heavy dep).
368#[cfg(feature = "headless")]
369fn rand_u64() -> u64 {
370    use std::time::{SystemTime, UNIX_EPOCH};
371    let mut x = SystemTime::now()
372        .duration_since(UNIX_EPOCH)
373        .unwrap_or_default()
374        .subsec_nanos() as u64;
375    x ^= x << 13;
376    x ^= x >> 7;
377    x ^= x << 17;
378    x
379}
380
381#[cfg(test)]
382mod tests {
383    use super::*;
384
385    #[cfg(not(feature = "headless"))]
386    fn make_backend() -> GoogleBackend {
387        use crate::config::FetchiumConfig;
388        use crate::http::HttpClient;
389        GoogleBackend::new_http(HttpClient::new(&FetchiumConfig::default()).expect("http"))
390    }
391
392    #[cfg(not(feature = "headless"))]
393    #[test]
394    fn backend_id() {
395        assert_eq!(make_backend().id(), BackendId::Google);
396    }
397
398    #[cfg(not(feature = "headless"))]
399    #[test]
400    fn parse_serp_empty_html() {
401        let results = GoogleBackend::parse_serp("<html><body></body></html>", 0);
402        assert!(results.is_empty());
403    }
404
405    #[cfg(not(feature = "headless"))]
406    #[test]
407    fn parse_serp_captcha_detected() {
408        // Simulate a CAPTCHA page — should return empty, not panic.
409        let captcha_html = r#"<html><body>
410            <p>detected unusual traffic from your computer network</p>
411        </body></html>"#;
412        let results = GoogleBackend::parse_serp(captcha_html, 0);
413        assert!(results.is_empty());
414    }
415
416    #[cfg(not(feature = "headless"))]
417    #[test]
418    fn parse_serp_filters_non_http_urls() {
419        let html = r#"<html><body>
420            <div class="g">
421                <h3>Good Result</h3>
422                <a href="https://example.com/page">link</a>
423                <div class="VwiC3b">A useful snippet.</div>
424            </div>
425            <div class="g">
426                <h3>Google Internal</h3>
427                <a href="https://www.google.com/search?q=test">internal</a>
428            </div>
429        </body></html>"#;
430        let results = GoogleBackend::parse_serp(html, 0);
431        assert_eq!(results.len(), 1);
432        assert_eq!(results[0].url, "https://example.com/page");
433        assert_eq!(results[0].backend, BackendId::Google);
434    }
435
436    #[cfg(not(feature = "headless"))]
437    #[test]
438    fn build_url_page_0() {
439        let url = GoogleBackend::build_url("rust programming", 0);
440        assert!(url.contains("google.com/search"));
441        assert!(url.contains("start=0"));
442        assert!(url.contains("rust"));
443    }
444
445    #[cfg(not(feature = "headless"))]
446    #[test]
447    fn build_url_page_1() {
448        let url = GoogleBackend::build_url("test", 1);
449        assert!(url.contains("start=10"));
450    }
451
452    #[cfg(feature = "headless")]
453    #[test]
454    fn build_url_headless() {
455        let url = GoogleBackend::build_url("rust async", 0);
456        assert!(url.contains("google.com/search"));
457        assert!(url.contains("start=0"));
458    }
459
460    #[cfg(feature = "headless")]
461    #[test]
462    fn parse_serp_empty_headless() {
463        let results = GoogleBackend::parse_serp("<html><body></body></html>", 0);
464        assert!(results.is_empty());
465    }
466}