1use crate::config::FetchiumConfig;
7use crate::error::{ErrorKind, FetchiumError, FetchiumResult, StructuredError};
8use crate::proxy::{DataImpulseClient, ProxyPool};
9use dashmap::DashMap;
10use reqwest::{Client, StatusCode};
11use std::sync::Arc;
12use std::time::{Duration, Instant};
13use tokio::time::sleep;
14use tracing::{debug, info};
15use url::Url;
16
17const MAX_RETRIES: u32 = 3;
19const BASE_DELAY_MS: u64 = 500;
21const MAX_BACKOFF_MS: u64 = 30_000;
23const _MAX_BODY_SIZE: u64 = 10 * 1024 * 1024;
25
26#[derive(Debug)]
28struct DomainState {
29 last_request: Instant,
30 min_delay: Duration,
31}
32
33const RESIDENTIAL_REQUIRED_DOMAINS: &[&str] = &["google.com", "www.google.com"];
37
38#[derive(Clone)]
40pub struct HttpClient {
41 inner: Client,
42 config: Arc<FetchiumConfig>,
43 domain_delays: Arc<DashMap<String, DomainState>>,
45 proxy_pool: Option<ProxyPool>,
47 dataimpulse: Option<DataImpulseClient>,
49}
50
51#[derive(Debug)]
53pub struct FetchResult {
54 pub body: String,
55 pub status: u16,
56 pub content_type: String,
57 pub content_length: Option<u64>,
58 pub url: String,
59 pub elapsed_ms: u64,
60 pub retries: u32,
61}
62
63impl HttpClient {
64 pub fn new(config: &FetchiumConfig) -> FetchiumResult<Self> {
66 let client = Client::builder()
67 .user_agent(&config.fetch.user_agent)
68 .timeout(Duration::from_secs(config.fetch.timeout_secs))
69 .connect_timeout(Duration::from_secs(10))
70 .redirect(reqwest::redirect::Policy::limited(
71 config.fetch.max_redirects as usize,
72 ))
73 .gzip(true)
74 .brotli(true)
75 .pool_max_idle_per_host(100)
76 .pool_idle_timeout(Duration::from_secs(90))
77 .build()
78 .map_err(FetchiumError::Network)?;
79
80 let proxy_pool = if config.proxy.enabled {
82 let proxy_file = config
83 .proxy
84 .proxy_file
85 .clone()
86 .unwrap_or_else(|| config.data_dir().join("proxies.txt"));
87 if proxy_file.exists() {
88 match ProxyPool::load_from_file(&proxy_file) {
89 Ok(pool) if !pool.is_empty() => {
90 info!(
91 "Proxy pool loaded: {} proxies from {}",
92 pool.len(),
93 proxy_file.display()
94 );
95 Some(pool)
96 }
97 Ok(_) => {
98 tracing::warn!("Proxy file is empty: {}", proxy_file.display());
99 None
100 }
101 Err(e) => {
102 tracing::warn!("Failed to load proxy file {}: {e}", proxy_file.display());
103 None
104 }
105 }
106 } else {
107 tracing::warn!("Proxy file not found: {}", proxy_file.display());
108 None
109 }
110 } else {
111 None
112 };
113
114 let dataimpulse = if config.dataimpulse.enabled && !config.dataimpulse.username.is_empty() {
116 let di = DataImpulseClient::new(
117 &config.dataimpulse.username,
118 &config.dataimpulse.password,
119 &config.dataimpulse.host,
120 config.dataimpulse.port,
121 &config.fetch.user_agent,
122 Duration::from_secs(config.fetch.timeout_secs),
123 );
124 info!(
125 "DataImpulse residential proxy enabled ({}:{})",
126 config.dataimpulse.host, config.dataimpulse.port
127 );
128 Some(di)
129 } else {
130 None
131 };
132
133 Ok(Self {
134 inner: client,
135 config: Arc::new(config.clone()),
136 domain_delays: Arc::new(DashMap::new()),
137 proxy_pool,
138 dataimpulse,
139 })
140 }
141
142 pub fn client(&self) -> &Client {
144 &self.inner
145 }
146
147 pub fn client_for_domain_with_locale(&self, domain: &str, locale: Option<&str>) -> Client {
156 let needs_residential = self.needs_residential_proxy(domain);
157
158 if needs_residential {
160 if let Some(ref di) = self.dataimpulse {
161 if di.is_configured() {
162 debug!("DataImpulse residential proxy for {domain} locale={locale:?}");
163 return di.client(locale);
164 }
165 }
166 }
167
168 self.inner.clone()
170 }
171
172 pub fn client_for_domain(&self, domain: &str) -> Client {
174 self.client_for_domain_with_locale(domain, None)
175 }
176
177 pub fn client_direct(&self) -> Client {
183 self.inner.clone()
184 }
185
186 pub fn proxy_client_for_domain_with_locale(
191 &self,
192 domain: &str,
193 locale: Option<&str>,
194 ) -> Client {
195 if self.needs_residential_proxy(domain) {
196 if let Some(ref di) = self.dataimpulse {
197 if di.is_configured() {
198 return di.client(locale);
199 }
200 }
201 return self.inner.clone();
202 }
203
204 if let Some(ref pool) = self.proxy_pool {
205 if self.should_proxy_domain(domain) {
206 if let Some(proxy) = pool.proxy_for_domain(domain) {
207 if let Ok(client) = ProxyPool::build_client_with_proxy(
208 &proxy,
209 &self.config.fetch.user_agent,
210 Duration::from_secs(self.config.fetch.timeout_secs),
211 ) {
212 return client;
213 }
214 }
215 }
216 }
217
218 self.inner.clone()
219 }
220
221 pub fn fresh_client_for_domain_with_locale(
227 &self,
228 domain: &str,
229 locale: Option<&str>,
230 ) -> Client {
231 if self.needs_residential_proxy(domain) {
232 if let Some(ref di) = self.dataimpulse {
233 if di.is_configured() {
234 return di.fresh_client(locale);
235 }
236 }
237 }
238 if let Some(ref pool) = self.proxy_pool {
240 if self.should_proxy_domain(domain) {
241 if let Some(proxy) = pool.fresh_proxy_for_domain(domain) {
242 if let Ok(client) = ProxyPool::build_client_with_proxy(
243 &proxy,
244 &self.config.fetch.user_agent,
245 Duration::from_secs(self.config.fetch.timeout_secs),
246 ) {
247 return client;
248 }
249 }
250 }
251 }
252
253 self.client_for_domain_with_locale(domain, locale)
255 }
256
257 fn needs_residential_proxy(&self, domain: &str) -> bool {
259 if !self.config.dataimpulse.proxy_domains.is_empty() {
261 return self
262 .config
263 .dataimpulse
264 .proxy_domains
265 .iter()
266 .any(|d| domain.ends_with(d.as_str()) || domain == d.as_str());
267 }
268 RESIDENTIAL_REQUIRED_DOMAINS
270 .iter()
271 .any(|&d| domain == d || domain.ends_with(&format!(".{d}")))
272 }
273
274 pub fn config(&self) -> &FetchiumConfig {
276 &self.config
277 }
278
279 pub fn extract_domain(url: &str) -> String {
281 Url::parse(url)
282 .map(|u| u.host_str().unwrap_or("unknown").to_string())
283 .unwrap_or_else(|_| "unknown".to_string())
284 }
285
286 async fn enforce_rate_limit(&self, domain: &str) {
288 let min_delay = Duration::from_millis(200);
289 if let Some(state) = self.domain_delays.get(domain) {
290 let elapsed = state.last_request.elapsed();
291 if elapsed < state.min_delay {
292 let wait = state.min_delay - elapsed;
293 debug!("Rate limiting {domain}: waiting {wait:?}");
294 sleep(wait).await;
295 }
296 }
297 self.domain_delays.insert(
298 domain.to_string(),
299 DomainState {
300 last_request: Instant::now(),
301 min_delay,
302 },
303 );
304 }
305
306 fn backoff_delay(attempt: u32) -> Duration {
309 let base = BASE_DELAY_MS.saturating_mul(2u64.saturating_pow(attempt));
310 let jitter = (std::time::SystemTime::now()
311 .duration_since(std::time::UNIX_EPOCH)
312 .unwrap_or_default()
313 .subsec_nanos()
314 % 500) as u64;
315 Duration::from_millis(base.saturating_add(jitter).min(MAX_BACKOFF_MS))
316 }
317
318 fn is_retryable_status(status: StatusCode) -> bool {
320 matches!(
321 status,
322 StatusCode::TOO_MANY_REQUESTS
323 | StatusCode::INTERNAL_SERVER_ERROR
324 | StatusCode::BAD_GATEWAY
325 | StatusCode::SERVICE_UNAVAILABLE
326 | StatusCode::GATEWAY_TIMEOUT
327 )
328 }
329
330 fn status_to_error_kind(status: StatusCode) -> ErrorKind {
332 match status {
333 StatusCode::FORBIDDEN => ErrorKind::Http403,
334 StatusCode::TOO_MANY_REQUESTS => ErrorKind::Http429,
335 s if s.is_server_error() => ErrorKind::Http5xx,
336 _ => ErrorKind::Unknown,
337 }
338 }
339
340 pub async fn fetch(&self, url: &str) -> FetchiumResult<FetchResult> {
343 let domain = Self::extract_domain(url);
344 let start = Instant::now();
345 let max_size = self.config.fetch.max_page_size;
346 let mut last_err: Option<FetchiumError> = None;
347
348 for attempt in 0..=MAX_RETRIES {
349 if attempt > 0 {
350 let delay = Self::backoff_delay(attempt - 1);
351 debug!("Retry {attempt}/{MAX_RETRIES} for {url} after {delay:?}");
352 sleep(delay).await;
353 }
354
355 self.enforce_rate_limit(&domain).await;
356
357 let result = self.inner.get(url).send().await;
358
359 match result {
360 Ok(resp) => {
361 let status = resp.status();
362
363 if !status.is_success() {
364 if Self::is_retryable_status(status) && attempt < MAX_RETRIES {
365 debug!("Retryable status {status} for {url} (attempt {attempt}/{MAX_RETRIES})");
366 last_err = Some(FetchiumError::Structured(StructuredError {
367 kind: Self::status_to_error_kind(status),
368 retryable: true,
369 message: format!("HTTP {status} from {url}"),
370 source_url: Some(url.to_string()),
371 suggested_action: "Retry with backoff".into(),
372 alternatives: vec![],
373 }));
374 continue;
375 }
376
377 return Err(FetchiumError::Structured(StructuredError {
378 kind: Self::status_to_error_kind(status),
379 retryable: false,
380 message: format!("HTTP {status} from {url}"),
381 source_url: Some(url.to_string()),
382 suggested_action: match status {
383 StatusCode::FORBIDDEN => "Site blocks automated access".into(),
384 StatusCode::TOO_MANY_REQUESTS => "Rate limited, try later".into(),
385 _ => "Check URL and try again".into(),
386 },
387 alternatives: vec![],
388 }));
389 }
390
391 let content_length = resp.content_length();
392 if let Some(len) = content_length {
393 if len > max_size {
394 return Err(FetchiumError::Structured(StructuredError {
395 kind: ErrorKind::ExtractionFailed,
396 retryable: false,
397 message: format!(
398 "Response too large: {len} bytes (max {max_size})"
399 ),
400 source_url: Some(url.to_string()),
401 suggested_action: "Increase max_page_size in config".into(),
402 alternatives: vec![],
403 }));
404 }
405 }
406
407 let content_type = resp
408 .headers()
409 .get("content-type")
410 .and_then(|v| v.to_str().ok())
411 .unwrap_or("text/html")
412 .to_string();
413
414 let final_url = resp.url().to_string();
415
416 let body = resp.text().await.map_err(FetchiumError::Network)?;
417
418 if body.len() as u64 > max_size {
419 return Err(FetchiumError::Structured(StructuredError {
420 kind: ErrorKind::ExtractionFailed,
421 retryable: false,
422 message: format!("Body too large: {} bytes", body.len()),
423 source_url: Some(url.to_string()),
424 suggested_action: "Increase max_page_size".into(),
425 alternatives: vec![],
426 }));
427 }
428
429 return Ok(FetchResult {
430 body,
431 status: status.as_u16(),
432 content_type,
433 content_length,
434 url: final_url,
435 elapsed_ms: start.elapsed().as_millis() as u64,
436 retries: attempt,
437 });
438 }
439 Err(e) => {
440 if (e.is_timeout() || e.is_connect()) && attempt < MAX_RETRIES {
441 debug!("Transient error for {url}: {e}");
442 last_err = Some(FetchiumError::Network(e));
443 continue;
444 }
445 return Err(FetchiumError::Network(e));
446 }
447 }
448 }
449
450 Err(last_err.unwrap_or_else(|| {
451 FetchiumError::Structured(StructuredError {
452 kind: ErrorKind::NetworkTimeout,
453 retryable: false,
454 message: format!("All {MAX_RETRIES} retries exhausted for {url}"),
455 source_url: Some(url.to_string()),
456 suggested_action: "Check network connectivity".into(),
457 alternatives: vec![],
458 })
459 }))
460 }
461
462 pub async fn fetch_text(&self, url: &str) -> FetchiumResult<String> {
464 let result = self.fetch(url).await?;
465 Ok(result.body)
466 }
467
468 pub fn proxy_pool(&self) -> Option<&ProxyPool> {
470 self.proxy_pool.as_ref()
471 }
472
473 fn should_proxy_domain(&self, domain: &str) -> bool {
475 if self
477 .config
478 .proxy
479 .bypass_domains
480 .iter()
481 .any(|d| domain.contains(d.as_str()))
482 {
483 return false;
484 }
485 if !self.config.proxy.proxy_domains.is_empty() {
487 return self
488 .config
489 .proxy
490 .proxy_domains
491 .iter()
492 .any(|d| domain.contains(d.as_str()));
493 }
494 true
496 }
497
498 pub async fn fetch_via_proxy(&self, url: &str) -> FetchiumResult<FetchResult> {
500 let domain = Self::extract_domain(url);
501
502 let pool = match &self.proxy_pool {
503 Some(p) if !p.is_empty() && self.should_proxy_domain(&domain) => p,
504 _ => return self.fetch(url).await,
505 };
506
507 let proxy = match pool.proxy_for_domain(&domain) {
508 Some(p) => p,
509 None => {
510 debug!("No available proxy for {domain}, falling back to direct");
511 return self.fetch(url).await;
512 }
513 };
514
515 let start = Instant::now();
516 let client = ProxyPool::build_client_with_proxy(
517 &proxy,
518 &self.config.fetch.user_agent,
519 Duration::from_secs(self.config.fetch.timeout_secs),
520 )
521 .map_err(FetchiumError::Network)?;
522
523 match client.get(url).send().await {
524 Ok(resp) => {
525 let status = resp.status();
526 let latency = start.elapsed().as_millis() as u64;
527
528 if status.is_success() {
529 proxy.record_success(latency);
530 let content_type = resp
531 .headers()
532 .get("content-type")
533 .and_then(|v| v.to_str().ok())
534 .unwrap_or("text/html")
535 .to_string();
536 let content_length = resp.content_length();
537 let final_url = resp.url().to_string();
538 let body = resp.text().await.map_err(FetchiumError::Network)?;
539
540 Ok(FetchResult {
541 body,
542 status: status.as_u16(),
543 content_type,
544 content_length,
545 url: final_url,
546 elapsed_ms: latency,
547 retries: 0,
548 })
549 } else if status == StatusCode::FORBIDDEN || status == StatusCode::TOO_MANY_REQUESTS
550 {
551 proxy.record_failure();
553 debug!(
554 "Proxy {}:{} blocked (HTTP {}) for {domain}, trying next",
555 proxy.host, proxy.port, status
556 );
557 if let Some(next_proxy) = pool.next_proxy() {
559 let client2 = ProxyPool::build_client_with_proxy(
560 &next_proxy,
561 &self.config.fetch.user_agent,
562 Duration::from_secs(self.config.fetch.timeout_secs),
563 )
564 .map_err(FetchiumError::Network)?;
565
566 let start2 = Instant::now();
567 match client2.get(url).send().await {
568 Ok(resp2) if resp2.status().is_success() => {
569 next_proxy.record_success(start2.elapsed().as_millis() as u64);
570 let body = resp2.text().await.map_err(FetchiumError::Network)?;
571 Ok(FetchResult {
572 body,
573 status: 200,
574 content_type: "text/html".into(),
575 content_length: None,
576 url: url.to_string(),
577 elapsed_ms: start.elapsed().as_millis() as u64,
578 retries: 1,
579 })
580 }
581 _ => {
582 next_proxy.record_failure();
583 self.fetch(url).await }
585 }
586 } else {
587 self.fetch(url).await
588 }
589 } else {
590 proxy.record_failure();
591 Err(FetchiumError::Structured(StructuredError {
592 kind: Self::status_to_error_kind(status),
593 retryable: false,
594 message: format!("HTTP {status} via proxy for {url}"),
595 source_url: Some(url.to_string()),
596 suggested_action: "Try different proxy or direct".into(),
597 alternatives: vec![],
598 }))
599 }
600 }
601 Err(e) => {
602 proxy.record_failure();
603 debug!(
604 "Proxy {}:{} connection failed for {domain}: {e}",
605 proxy.host, proxy.port
606 );
607 self.fetch(url).await
609 }
610 }
611 }
612
613 pub async fn fetch_text_via_proxy(&self, url: &str) -> FetchiumResult<String> {
615 let result = self.fetch_via_proxy(url).await?;
616 Ok(result.body)
617 }
618
619 pub async fn fetch_text_once(&self, url: &str) -> FetchiumResult<String> {
625 let resp = self
626 .inner
627 .get(url)
628 .send()
629 .await
630 .map_err(FetchiumError::Network)?;
631
632 if !resp.status().is_success() {
633 return Err(FetchiumError::Structured(crate::error::StructuredError {
634 kind: Self::status_to_error_kind(resp.status()),
635 retryable: false,
636 message: format!("HTTP {} from {url}", resp.status()),
637 source_url: Some(url.to_string()),
638 suggested_action: "Source unavailable".into(),
639 alternatives: vec![],
640 }));
641 }
642
643 resp.text().await.map_err(FetchiumError::Network)
644 }
645
646 pub async fn post_json_once(&self, url: &str, body: String) -> FetchiumResult<String> {
651 let resp = self
652 .inner
653 .post(url)
654 .header("Content-Type", "application/json")
655 .body(body)
656 .send()
657 .await
658 .map_err(FetchiumError::Network)?;
659
660 if !resp.status().is_success() {
661 return Err(FetchiumError::Structured(crate::error::StructuredError {
662 kind: Self::status_to_error_kind(resp.status()),
663 retryable: false,
664 message: format!("HTTP {} from {url}", resp.status()),
665 source_url: Some(url.to_string()),
666 suggested_action: "Source unavailable".into(),
667 alternatives: vec![],
668 }));
669 }
670
671 resp.text().await.map_err(FetchiumError::Network)
672 }
673
674 pub async fn post_json(&self, url: &str, body: &str) -> FetchiumResult<String> {
676 let resp = self
677 .inner
678 .post(url)
679 .header("Content-Type", "application/json")
680 .body(body.to_string())
681 .send()
682 .await
683 .map_err(FetchiumError::Network)?;
684
685 if !resp.status().is_success() {
686 return Err(FetchiumError::Structured(crate::error::StructuredError {
687 kind: Self::status_to_error_kind(resp.status()),
688 retryable: resp.status().is_server_error(),
689 message: format!("HTTP {} from {url}", resp.status()),
690 source_url: Some(url.to_string()),
691 suggested_action: "Check API key and request format".into(),
692 alternatives: vec![],
693 }));
694 }
695
696 resp.text().await.map_err(FetchiumError::Network)
697 }
698
699 pub async fn post_json_with_header(
701 &self,
702 url: &str,
703 body: &str,
704 header_name: &str,
705 header_value: &str,
706 ) -> FetchiumResult<String> {
707 let resp = self
708 .inner
709 .post(url)
710 .header("Content-Type", "application/json")
711 .header(header_name, header_value)
712 .body(body.to_string())
713 .send()
714 .await
715 .map_err(FetchiumError::Network)?;
716
717 if !resp.status().is_success() {
718 return Err(FetchiumError::Structured(crate::error::StructuredError {
719 kind: Self::status_to_error_kind(resp.status()),
720 retryable: resp.status().is_server_error(),
721 message: format!("HTTP {} from {url}", resp.status()),
722 source_url: Some(url.to_string()),
723 suggested_action: "Check API key and request format".into(),
724 alternatives: vec![],
725 }));
726 }
727
728 resp.text().await.map_err(FetchiumError::Network)
729 }
730}
731
732#[cfg(test)]
733mod tests {
734 use super::*;
735
736 #[test]
737 fn client_creation() {
738 let config = FetchiumConfig::default();
739 let client = HttpClient::new(&config);
740 assert!(client.is_ok());
741 }
742
743 #[test]
744 fn domain_extraction() {
745 assert_eq!(
746 HttpClient::extract_domain("https://www.example.com/page"),
747 "www.example.com"
748 );
749 assert_eq!(
750 HttpClient::extract_domain("http://localhost:8080/api"),
751 "localhost"
752 );
753 assert_eq!(HttpClient::extract_domain("not-a-url"), "unknown");
754 }
755
756 #[test]
757 fn backoff_delay_increases() {
758 let d0 = HttpClient::backoff_delay(0);
759 let d1 = HttpClient::backoff_delay(1);
760 let d2 = HttpClient::backoff_delay(2);
761 assert!(d0.as_millis() >= 500 && d0.as_millis() < 1000);
762 assert!(d1.as_millis() >= 1000 && d1.as_millis() < 1500);
763 assert!(d2.as_millis() >= 2000 && d2.as_millis() < 2500);
764 }
765
766 #[test]
767 fn retryable_status_codes() {
768 assert!(HttpClient::is_retryable_status(
769 StatusCode::TOO_MANY_REQUESTS
770 ));
771 assert!(HttpClient::is_retryable_status(StatusCode::BAD_GATEWAY));
772 assert!(HttpClient::is_retryable_status(
773 StatusCode::SERVICE_UNAVAILABLE
774 ));
775 assert!(!HttpClient::is_retryable_status(StatusCode::NOT_FOUND));
776 assert!(!HttpClient::is_retryable_status(StatusCode::FORBIDDEN));
777 }
778
779 #[test]
780 fn residential_proxy_is_google_only_by_default() {
781 let client = HttpClient::new(&FetchiumConfig::default()).unwrap();
782 assert!(client.needs_residential_proxy("google.com"));
783 assert!(client.needs_residential_proxy("www.google.com"));
784 assert!(!client.needs_residential_proxy("duckduckgo.com"));
785 assert!(!client.needs_residential_proxy("html.duckduckgo.com"));
786 assert!(!client.needs_residential_proxy("bing.com"));
787 }
788
789 #[test]
790 fn explicit_residential_domain_override_wins() {
791 let mut config = FetchiumConfig::default();
792 config.dataimpulse.proxy_domains = vec!["example.com".into()];
793 let client = HttpClient::new(&config).unwrap();
794 assert!(client.needs_residential_proxy("www.example.com"));
795 assert!(!client.needs_residential_proxy("google.com"));
796 }
797}