use crate::error::FetchiumResult;
use crate::search::{SearchBackend, SearchContext, TimeRange};
use crate::types::{BackendId, ResultItem};
use async_trait::async_trait;
use tracing::debug;
#[cfg(not(feature = "headless"))]
use tracing::info;
#[cfg(feature = "headless")]
use tracing::warn;
#[cfg(not(feature = "headless"))]
const MAX_IP_RETRIES: usize = 2;
#[cfg(not(feature = "headless"))]
const BROWSER_UA: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) \
AppleWebKit/537.36 (KHTML, like Gecko) \
Chrome/121.0.0.0 Safari/537.36";
pub struct GoogleBackend {
#[cfg(feature = "headless")]
pool: std::sync::Arc<crate::browser::pool::BrowserPool>,
#[cfg(not(feature = "headless"))]
http: crate::http::HttpClient,
}
impl GoogleBackend {
#[cfg(feature = "headless")]
pub fn new(pool: std::sync::Arc<crate::browser::pool::BrowserPool>) -> Self {
Self { pool }
}
#[cfg(not(feature = "headless"))]
pub fn new_http(http: crate::http::HttpClient) -> Self {
Self { http }
}
fn build_url(query: &str, page: usize) -> String {
Self::build_url_with_tbs(query, page, "")
}
fn build_url_with_tbs(query: &str, page: usize, tbs: &str) -> String {
let start = page * 10;
let encoded = url::form_urlencoded::Serializer::new(String::new())
.append_key_only(query)
.finish();
let tbs_param = if tbs.is_empty() {
String::new()
} else {
format!("&tbs={tbs}")
};
format!("https://www.google.com/search?q={encoded}&start={start}&hl=en&num=10&gl=us&pws=0{tbs_param}")
}
fn time_range_to_tbs(tr: Option<TimeRange>) -> &'static str {
match tr {
Some(TimeRange::Day) => "qdr:d",
Some(TimeRange::Week) => "qdr:w",
Some(TimeRange::Month) => "qdr:m",
Some(TimeRange::Year) => "qdr:y",
None => "",
}
}
#[cfg(not(feature = "headless"))]
async fn fetch_page_with_retry(
&self,
url: &str,
page: usize,
locale: Option<&str>,
) -> Option<Vec<ResultItem>> {
for attempt in 0..MAX_IP_RETRIES {
let client = if attempt == 0 {
self.http
.client_for_domain_with_locale("google.com", locale)
} else {
info!(
"Google: IP block detected — rotating to new residential IP (attempt {}/{})",
attempt + 1,
MAX_IP_RETRIES
);
tokio::time::sleep(std::time::Duration::from_millis(300 * attempt as u64)).await;
self.http
.fresh_client_for_domain_with_locale("google.com", locale)
};
let result = client
.get(url)
.header("User-Agent", BROWSER_UA)
.header(
"Accept",
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
)
.header("Accept-Language", "en-US,en;q=0.9")
.header("Referer", "https://www.google.com/")
.header("Cache-Control", "no-cache")
.send()
.await;
match result {
Ok(resp) if resp.status().is_success() => match resp.text().await {
Ok(html) => {
let results = Self::parse_serp(&html, page);
if !results.is_empty() {
return Some(results);
}
debug!("Google: empty SERP page {page} (attempt {attempt}) — rotating IP");
}
Err(e) => {
debug!("Google: body read error attempt {attempt}: {e}");
}
},
Ok(resp) if resp.status().as_u16() == 429 || resp.status().as_u16() == 403 => {
debug!(
"Google: HTTP {} attempt {attempt} — rotating IP",
resp.status()
);
}
Ok(resp) => {
debug!("Google: HTTP {} for page {page}", resp.status());
return None; }
Err(e) => {
debug!("Google: request error attempt {attempt}: {e}");
}
}
}
debug!("Google: all {MAX_IP_RETRIES} IP rotation attempts exhausted for page {page}");
None
}
fn parse_serp(html: &str, page: usize) -> Vec<ResultItem> {
use scraper::{Html, Selector};
if html.contains("accounts.google.com")
|| html.contains("detected unusual traffic")
|| html.contains("please enable cookies")
|| (html.len() < 5_000 && html.to_lowercase().contains("captcha"))
{
return vec![];
}
let document = Html::parse_document(html);
let mut results = Vec::new();
let container_sel = Selector::parse("div.g, div[data-hveid]").unwrap();
let title_sel = Selector::parse("h3").unwrap();
let link_sel = Selector::parse("a[href]").unwrap();
let snippet_sel = Selector::parse("div.VwiC3b, span.aCOpRe, div[data-sncf]").unwrap();
let mut rank = (page * 10 + 1) as u32;
for container in document.select(&container_sel) {
let title = container
.select(&title_sel)
.next()
.map(|e| e.text().collect::<String>().trim().to_string())
.unwrap_or_default();
if title.is_empty() {
continue;
}
let url = container
.select(&link_sel)
.next()
.and_then(|e| e.value().attr("href"))
.filter(|href| href.starts_with("http"))
.filter(|href| !href.contains("google.com"))
.map(|s| s.to_string())
.unwrap_or_default();
if url.is_empty() {
continue;
}
let snippet = container
.select(&snippet_sel)
.next()
.map(|e| e.text().collect::<String>().trim().to_string())
.unwrap_or_default();
results.push(ResultItem {
title,
url,
snippet,
rank,
backend: BackendId::Google,
score: None,
published_date: None,
});
rank += 1;
}
debug!(
"Google parse_serp: {} results from page {}",
results.len(),
page
);
results
}
}
#[async_trait]
impl SearchBackend for GoogleBackend {
fn id(&self) -> BackendId {
BackendId::Google
}
fn requires_headless(&self) -> bool {
cfg!(feature = "headless")
}
async fn search(&self, query: &str, max_results: u32) -> FetchiumResult<Vec<ResultItem>> {
#[cfg(not(feature = "headless"))]
{
let pages = (max_results as usize).div_ceil(10).min(2);
let mut all_results = Vec::new();
for page in 0..pages {
let url = Self::build_url(query, page);
match self.fetch_page_with_retry(&url, page, None).await {
Some(results) => all_results.extend(results),
None => break, }
if page + 1 < pages {
tokio::time::sleep(std::time::Duration::from_millis(300)).await;
}
}
return Ok(all_results.into_iter().take(max_results as usize).collect());
}
#[cfg(feature = "headless")]
{
let pages = (max_results as usize).div_ceil(10).min(3);
let mut all_results = Vec::new();
for page in 0..pages {
let url = Self::build_url(query, page);
match self.pool.acquire_tab().await {
Ok(tab) => {
if page > 0 {
let delay_ms = 200 + (rand_u64() % 600);
tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await;
}
match tab.navigate_and_wait(&url, "div.g", 15_000).await {
Ok(_) => match tab.content().await {
Ok(html) => {
all_results.extend(Self::parse_serp(&html, page));
}
Err(e) => {
warn!("Google content error page {page}: {e}");
}
},
Err(e) => {
warn!("Google navigate error page {page}: {e}");
}
}
}
Err(e) => {
warn!("Google tab acquire failed: {e}");
break;
}
}
}
Ok(all_results.into_iter().take(max_results as usize).collect())
}
}
async fn search_with_context(
&self,
query: &str,
max_results: u32,
ctx: &SearchContext,
) -> FetchiumResult<Vec<ResultItem>> {
let tbs = Self::time_range_to_tbs(ctx.time_range);
if tbs.is_empty() {
return self.search(query, max_results).await;
}
#[cfg(not(feature = "headless"))]
{
let pages = (max_results as usize).div_ceil(10).min(2);
let mut all_results = Vec::new();
let locale = ctx.locale.as_deref();
for page in 0..pages {
let url = Self::build_url_with_tbs(query, page, tbs);
match self.fetch_page_with_retry(&url, page, locale).await {
Some(results) => all_results.extend(results),
None => break,
}
if page + 1 < pages {
tokio::time::sleep(std::time::Duration::from_millis(300)).await;
}
}
return Ok(all_results.into_iter().take(max_results as usize).collect());
}
#[cfg(feature = "headless")]
self.search(query, max_results).await
}
}
#[cfg(feature = "headless")]
fn rand_u64() -> u64 {
use std::time::{SystemTime, UNIX_EPOCH};
let mut x = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.subsec_nanos() as u64;
x ^= x << 13;
x ^= x >> 7;
x ^= x << 17;
x
}
#[cfg(test)]
mod tests {
use super::*;
#[cfg(not(feature = "headless"))]
fn make_backend() -> GoogleBackend {
use crate::config::FetchiumConfig;
use crate::http::HttpClient;
GoogleBackend::new_http(HttpClient::new(&FetchiumConfig::default()).expect("http"))
}
#[cfg(not(feature = "headless"))]
#[test]
fn backend_id() {
assert_eq!(make_backend().id(), BackendId::Google);
}
#[cfg(not(feature = "headless"))]
#[test]
fn parse_serp_empty_html() {
let results = GoogleBackend::parse_serp("<html><body></body></html>", 0);
assert!(results.is_empty());
}
#[cfg(not(feature = "headless"))]
#[test]
fn parse_serp_captcha_detected() {
let captcha_html = r#"<html><body>
<p>detected unusual traffic from your computer network</p>
</body></html>"#;
let results = GoogleBackend::parse_serp(captcha_html, 0);
assert!(results.is_empty());
}
#[cfg(not(feature = "headless"))]
#[test]
fn parse_serp_filters_non_http_urls() {
let html = r#"<html><body>
<div class="g">
<h3>Good Result</h3>
<a href="https://example.com/page">link</a>
<div class="VwiC3b">A useful snippet.</div>
</div>
<div class="g">
<h3>Google Internal</h3>
<a href="https://www.google.com/search?q=test">internal</a>
</div>
</body></html>"#;
let results = GoogleBackend::parse_serp(html, 0);
assert_eq!(results.len(), 1);
assert_eq!(results[0].url, "https://example.com/page");
assert_eq!(results[0].backend, BackendId::Google);
}
#[cfg(not(feature = "headless"))]
#[test]
fn build_url_page_0() {
let url = GoogleBackend::build_url("rust programming", 0);
assert!(url.contains("google.com/search"));
assert!(url.contains("start=0"));
assert!(url.contains("rust"));
}
#[cfg(not(feature = "headless"))]
#[test]
fn build_url_page_1() {
let url = GoogleBackend::build_url("test", 1);
assert!(url.contains("start=10"));
}
#[cfg(feature = "headless")]
#[test]
fn build_url_headless() {
let url = GoogleBackend::build_url("rust async", 0);
assert!(url.contains("google.com/search"));
assert!(url.contains("start=0"));
}
#[cfg(feature = "headless")]
#[test]
fn parse_serp_empty_headless() {
let results = GoogleBackend::parse_serp("<html><body></body></html>", 0);
assert!(results.is_empty());
}
}