1use crate::error::FetchiumResult;
18use crate::search::{SearchBackend, SearchContext, TimeRange};
19use crate::types::{BackendId, ResultItem};
20use async_trait::async_trait;
21use tracing::debug;
22#[cfg(not(feature = "headless"))]
23use tracing::info;
24#[cfg(feature = "headless")]
25use tracing::warn;
26
27#[cfg(not(feature = "headless"))]
29const MAX_IP_RETRIES: usize = 2;
30
31#[cfg(not(feature = "headless"))]
33const BROWSER_UA: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) \
34 AppleWebKit/537.36 (KHTML, like Gecko) \
35 Chrome/121.0.0.0 Safari/537.36";
36
37pub struct GoogleBackend {
42 #[cfg(feature = "headless")]
43 pool: std::sync::Arc<crate::browser::pool::BrowserPool>,
44 #[cfg(not(feature = "headless"))]
45 http: crate::http::HttpClient,
46}
47
48impl GoogleBackend {
49 #[cfg(feature = "headless")]
51 pub fn new(pool: std::sync::Arc<crate::browser::pool::BrowserPool>) -> Self {
52 Self { pool }
53 }
54
55 #[cfg(not(feature = "headless"))]
57 pub fn new_http(http: crate::http::HttpClient) -> Self {
58 Self { http }
59 }
60
61 fn build_url(query: &str, page: usize) -> String {
63 Self::build_url_with_tbs(query, page, "")
64 }
65
66 fn build_url_with_tbs(query: &str, page: usize, tbs: &str) -> String {
67 let start = page * 10;
68 let encoded = url::form_urlencoded::Serializer::new(String::new())
69 .append_key_only(query)
70 .finish();
71 let tbs_param = if tbs.is_empty() {
72 String::new()
73 } else {
74 format!("&tbs={tbs}")
75 };
76 format!("https://www.google.com/search?q={encoded}&start={start}&hl=en&num=10&gl=us&pws=0{tbs_param}")
78 }
79
80 fn time_range_to_tbs(tr: Option<TimeRange>) -> &'static str {
81 match tr {
82 Some(TimeRange::Day) => "qdr:d",
83 Some(TimeRange::Week) => "qdr:w",
84 Some(TimeRange::Month) => "qdr:m",
85 Some(TimeRange::Year) => "qdr:y",
86 None => "",
87 }
88 }
89
90 #[cfg(not(feature = "headless"))]
96 async fn fetch_page_with_retry(
97 &self,
98 url: &str,
99 page: usize,
100 locale: Option<&str>,
101 ) -> Option<Vec<ResultItem>> {
102 for attempt in 0..MAX_IP_RETRIES {
103 let client = if attempt == 0 {
104 self.http
105 .client_for_domain_with_locale("google.com", locale)
106 } else {
107 info!(
108 "Google: IP block detected — rotating to new residential IP (attempt {}/{})",
109 attempt + 1,
110 MAX_IP_RETRIES
111 );
112 tokio::time::sleep(std::time::Duration::from_millis(300 * attempt as u64)).await;
114 self.http
115 .fresh_client_for_domain_with_locale("google.com", locale)
116 };
117
118 let result = client
119 .get(url)
120 .header("User-Agent", BROWSER_UA)
121 .header(
122 "Accept",
123 "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
124 )
125 .header("Accept-Language", "en-US,en;q=0.9")
126 .header("Referer", "https://www.google.com/")
127 .header("Cache-Control", "no-cache")
128 .send()
129 .await;
130
131 match result {
132 Ok(resp) if resp.status().is_success() => match resp.text().await {
133 Ok(html) => {
134 let results = Self::parse_serp(&html, page);
135 if !results.is_empty() {
136 return Some(results);
137 }
138 debug!("Google: empty SERP page {page} (attempt {attempt}) — rotating IP");
140 }
141 Err(e) => {
142 debug!("Google: body read error attempt {attempt}: {e}");
143 }
144 },
145 Ok(resp) if resp.status().as_u16() == 429 || resp.status().as_u16() == 403 => {
146 debug!(
147 "Google: HTTP {} attempt {attempt} — rotating IP",
148 resp.status()
149 );
150 }
151 Ok(resp) => {
152 debug!("Google: HTTP {} for page {page}", resp.status());
153 return None; }
155 Err(e) => {
156 debug!("Google: request error attempt {attempt}: {e}");
157 }
158 }
159 }
160
161 debug!("Google: all {MAX_IP_RETRIES} IP rotation attempts exhausted for page {page}");
162 None
163 }
164
165 fn parse_serp(html: &str, page: usize) -> Vec<ResultItem> {
170 use scraper::{Html, Selector};
171
172 if html.contains("accounts.google.com")
178 || html.contains("detected unusual traffic")
179 || html.contains("please enable cookies")
180 || (html.len() < 5_000 && html.to_lowercase().contains("captcha"))
181 {
182 return vec![];
183 }
184
185 let document = Html::parse_document(html);
186 let mut results = Vec::new();
187
188 let container_sel = Selector::parse("div.g, div[data-hveid]").unwrap();
198 let title_sel = Selector::parse("h3").unwrap();
199 let link_sel = Selector::parse("a[href]").unwrap();
200 let snippet_sel = Selector::parse("div.VwiC3b, span.aCOpRe, div[data-sncf]").unwrap();
201
202 let mut rank = (page * 10 + 1) as u32;
203
204 for container in document.select(&container_sel) {
205 let title = container
206 .select(&title_sel)
207 .next()
208 .map(|e| e.text().collect::<String>().trim().to_string())
209 .unwrap_or_default();
210
211 if title.is_empty() {
212 continue;
213 }
214
215 let url = container
216 .select(&link_sel)
217 .next()
218 .and_then(|e| e.value().attr("href"))
219 .filter(|href| href.starts_with("http"))
220 .filter(|href| !href.contains("google.com"))
221 .map(|s| s.to_string())
222 .unwrap_or_default();
223
224 if url.is_empty() {
225 continue;
226 }
227
228 let snippet = container
229 .select(&snippet_sel)
230 .next()
231 .map(|e| e.text().collect::<String>().trim().to_string())
232 .unwrap_or_default();
233
234 results.push(ResultItem {
235 title,
236 url,
237 snippet,
238 rank,
239 backend: BackendId::Google,
240 score: None,
241 published_date: None,
242 });
243 rank += 1;
244 }
245
246 debug!(
247 "Google parse_serp: {} results from page {}",
248 results.len(),
249 page
250 );
251
252 results
253 }
254}
255
256#[async_trait]
257impl SearchBackend for GoogleBackend {
258 fn id(&self) -> BackendId {
259 BackendId::Google
260 }
261
262 fn requires_headless(&self) -> bool {
263 cfg!(feature = "headless")
264 }
265
266 async fn search(&self, query: &str, max_results: u32) -> FetchiumResult<Vec<ResultItem>> {
267 #[cfg(not(feature = "headless"))]
269 {
270 let pages = (max_results as usize).div_ceil(10).min(2);
271 let mut all_results = Vec::new();
272
273 for page in 0..pages {
274 let url = Self::build_url(query, page);
275 match self.fetch_page_with_retry(&url, page, None).await {
276 Some(results) => all_results.extend(results),
277 None => break, }
279 if page + 1 < pages {
280 tokio::time::sleep(std::time::Duration::from_millis(300)).await;
281 }
282 }
283
284 return Ok(all_results.into_iter().take(max_results as usize).collect());
285 }
286
287 #[cfg(feature = "headless")]
289 {
290 let pages = (max_results as usize).div_ceil(10).min(3);
291 let mut all_results = Vec::new();
292
293 for page in 0..pages {
294 let url = Self::build_url(query, page);
295
296 match self.pool.acquire_tab().await {
297 Ok(tab) => {
298 if page > 0 {
299 let delay_ms = 200 + (rand_u64() % 600);
300 tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await;
301 }
302
303 match tab.navigate_and_wait(&url, "div.g", 15_000).await {
305 Ok(_) => match tab.content().await {
306 Ok(html) => {
307 all_results.extend(Self::parse_serp(&html, page));
308 }
309 Err(e) => {
310 warn!("Google content error page {page}: {e}");
311 }
312 },
313 Err(e) => {
314 warn!("Google navigate error page {page}: {e}");
315 }
316 }
317 }
318 Err(e) => {
319 warn!("Google tab acquire failed: {e}");
320 break;
321 }
322 }
323 }
324
325 Ok(all_results.into_iter().take(max_results as usize).collect())
326 }
327 }
328
329 async fn search_with_context(
330 &self,
331 query: &str,
332 max_results: u32,
333 ctx: &SearchContext,
334 ) -> FetchiumResult<Vec<ResultItem>> {
335 let tbs = Self::time_range_to_tbs(ctx.time_range);
336 if tbs.is_empty() {
337 return self.search(query, max_results).await;
338 }
339
340 #[cfg(not(feature = "headless"))]
342 {
343 let pages = (max_results as usize).div_ceil(10).min(2);
344 let mut all_results = Vec::new();
345 let locale = ctx.locale.as_deref();
346
347 for page in 0..pages {
348 let url = Self::build_url_with_tbs(query, page, tbs);
349 match self.fetch_page_with_retry(&url, page, locale).await {
350 Some(results) => all_results.extend(results),
351 None => break,
352 }
353 if page + 1 < pages {
354 tokio::time::sleep(std::time::Duration::from_millis(300)).await;
355 }
356 }
357
358 return Ok(all_results.into_iter().take(max_results as usize).collect());
359 }
360
361 #[cfg(feature = "headless")]
363 self.search(query, max_results).await
364 }
365}
366
367#[cfg(feature = "headless")]
369fn rand_u64() -> u64 {
370 use std::time::{SystemTime, UNIX_EPOCH};
371 let mut x = SystemTime::now()
372 .duration_since(UNIX_EPOCH)
373 .unwrap_or_default()
374 .subsec_nanos() as u64;
375 x ^= x << 13;
376 x ^= x >> 7;
377 x ^= x << 17;
378 x
379}
380
381#[cfg(test)]
382mod tests {
383 use super::*;
384
385 #[cfg(not(feature = "headless"))]
386 fn make_backend() -> GoogleBackend {
387 use crate::config::FetchiumConfig;
388 use crate::http::HttpClient;
389 GoogleBackend::new_http(HttpClient::new(&FetchiumConfig::default()).expect("http"))
390 }
391
392 #[cfg(not(feature = "headless"))]
393 #[test]
394 fn backend_id() {
395 assert_eq!(make_backend().id(), BackendId::Google);
396 }
397
398 #[cfg(not(feature = "headless"))]
399 #[test]
400 fn parse_serp_empty_html() {
401 let results = GoogleBackend::parse_serp("<html><body></body></html>", 0);
402 assert!(results.is_empty());
403 }
404
405 #[cfg(not(feature = "headless"))]
406 #[test]
407 fn parse_serp_captcha_detected() {
408 let captcha_html = r#"<html><body>
410 <p>detected unusual traffic from your computer network</p>
411 </body></html>"#;
412 let results = GoogleBackend::parse_serp(captcha_html, 0);
413 assert!(results.is_empty());
414 }
415
416 #[cfg(not(feature = "headless"))]
417 #[test]
418 fn parse_serp_filters_non_http_urls() {
419 let html = r#"<html><body>
420 <div class="g">
421 <h3>Good Result</h3>
422 <a href="https://example.com/page">link</a>
423 <div class="VwiC3b">A useful snippet.</div>
424 </div>
425 <div class="g">
426 <h3>Google Internal</h3>
427 <a href="https://www.google.com/search?q=test">internal</a>
428 </div>
429 </body></html>"#;
430 let results = GoogleBackend::parse_serp(html, 0);
431 assert_eq!(results.len(), 1);
432 assert_eq!(results[0].url, "https://example.com/page");
433 assert_eq!(results[0].backend, BackendId::Google);
434 }
435
436 #[cfg(not(feature = "headless"))]
437 #[test]
438 fn build_url_page_0() {
439 let url = GoogleBackend::build_url("rust programming", 0);
440 assert!(url.contains("google.com/search"));
441 assert!(url.contains("start=0"));
442 assert!(url.contains("rust"));
443 }
444
445 #[cfg(not(feature = "headless"))]
446 #[test]
447 fn build_url_page_1() {
448 let url = GoogleBackend::build_url("test", 1);
449 assert!(url.contains("start=10"));
450 }
451
452 #[cfg(feature = "headless")]
453 #[test]
454 fn build_url_headless() {
455 let url = GoogleBackend::build_url("rust async", 0);
456 assert!(url.contains("google.com/search"));
457 assert!(url.contains("start=0"));
458 }
459
460 #[cfg(feature = "headless")]
461 #[test]
462 fn parse_serp_empty_headless() {
463 let results = GoogleBackend::parse_serp("<html><body></body></html>", 0);
464 assert!(results.is_empty());
465 }
466}