Skip to main content

fetchium_core/http/
client.rs

1//! HTTP client with connection pooling, retries, rate limiting, and size limits.
2//!
3//! PRD SS14: Domain-aware scheduler with per-domain concurrency caps.
4//! PRD SS44: Structured errors with retry info.
5
6use crate::config::FetchiumConfig;
7use crate::error::{ErrorKind, FetchiumError, FetchiumResult, StructuredError};
8use crate::proxy::{DataImpulseClient, ProxyPool};
9use dashmap::DashMap;
10use reqwest::{Client, StatusCode};
11use std::sync::Arc;
12use std::time::{Duration, Instant};
13use tokio::time::sleep;
14use tracing::{debug, info};
15use url::Url;
16
17/// Maximum retry attempts for transient errors.
18const MAX_RETRIES: u32 = 3;
19/// Base delay for exponential backoff (milliseconds).
20const BASE_DELAY_MS: u64 = 500;
21/// Maximum backoff delay cap (30 seconds) to prevent excessive waits.
22const MAX_BACKOFF_MS: u64 = 30_000;
23/// Maximum response body size (10 MB).
24const _MAX_BODY_SIZE: u64 = 10 * 1024 * 1024;
25
26/// Per-domain rate limit state.
27#[derive(Debug)]
28struct DomainState {
29    last_request: Instant,
30    min_delay: Duration,
31}
32
33/// Domains that are blocked hard enough to justify the expensive residential path.
34/// API-based backends (Serper, Exa, Tavily, Gemini) are NOT in this list.
35/// Everything else should use the cheaper fixed proxy pool or direct traffic.
36const RESIDENTIAL_REQUIRED_DOMAINS: &[&str] = &["google.com", "www.google.com"];
37
38/// Shared HTTP client with connection pooling, retries, and rate limiting.
39#[derive(Clone)]
40pub struct HttpClient {
41    inner: Client,
42    config: Arc<FetchiumConfig>,
43    /// Per-domain rate limiting state.
44    domain_delays: Arc<DashMap<String, DomainState>>,
45    /// Optional Webshare proxy pool (datacenter — legacy fallback).
46    proxy_pool: Option<ProxyPool>,
47    /// DataImpulse residential proxy client (country-targeted, pay-per-GB).
48    dataimpulse: Option<DataImpulseClient>,
49}
50
51/// Fetch result with metadata about the request.
52#[derive(Debug)]
53pub struct FetchResult {
54    pub body: String,
55    pub status: u16,
56    pub content_type: String,
57    pub content_length: Option<u64>,
58    pub url: String,
59    pub elapsed_ms: u64,
60    pub retries: u32,
61}
62
63impl HttpClient {
64    /// Create a new HTTP client from config.
65    pub fn new(config: &FetchiumConfig) -> FetchiumResult<Self> {
66        let client = Client::builder()
67            .user_agent(&config.fetch.user_agent)
68            .timeout(Duration::from_secs(config.fetch.timeout_secs))
69            .connect_timeout(Duration::from_secs(10))
70            .redirect(reqwest::redirect::Policy::limited(
71                config.fetch.max_redirects as usize,
72            ))
73            .gzip(true)
74            .brotli(true)
75            .pool_max_idle_per_host(100)
76            .pool_idle_timeout(Duration::from_secs(90))
77            .build()
78            .map_err(FetchiumError::Network)?;
79
80        // Load proxy pool if configured
81        let proxy_pool = if config.proxy.enabled {
82            let proxy_file = config
83                .proxy
84                .proxy_file
85                .clone()
86                .unwrap_or_else(|| config.data_dir().join("proxies.txt"));
87            if proxy_file.exists() {
88                match ProxyPool::load_from_file(&proxy_file) {
89                    Ok(pool) if !pool.is_empty() => {
90                        info!(
91                            "Proxy pool loaded: {} proxies from {}",
92                            pool.len(),
93                            proxy_file.display()
94                        );
95                        Some(pool)
96                    }
97                    Ok(_) => {
98                        tracing::warn!("Proxy file is empty: {}", proxy_file.display());
99                        None
100                    }
101                    Err(e) => {
102                        tracing::warn!("Failed to load proxy file {}: {e}", proxy_file.display());
103                        None
104                    }
105                }
106            } else {
107                tracing::warn!("Proxy file not found: {}", proxy_file.display());
108                None
109            }
110        } else {
111            None
112        };
113
114        // Build DataImpulse residential proxy client if configured
115        let dataimpulse = if config.dataimpulse.enabled && !config.dataimpulse.username.is_empty() {
116            let di = DataImpulseClient::new(
117                &config.dataimpulse.username,
118                &config.dataimpulse.password,
119                &config.dataimpulse.host,
120                config.dataimpulse.port,
121                &config.fetch.user_agent,
122                Duration::from_secs(config.fetch.timeout_secs),
123            );
124            info!(
125                "DataImpulse residential proxy enabled ({}:{})",
126                config.dataimpulse.host, config.dataimpulse.port
127            );
128            Some(di)
129        } else {
130            None
131        };
132
133        Ok(Self {
134            inner: client,
135            config: Arc::new(config.clone()),
136            domain_delays: Arc::new(DashMap::new()),
137            proxy_pool,
138            dataimpulse,
139        })
140    }
141
142    /// Get the inner reqwest client for direct use.
143    pub fn client(&self) -> &Client {
144        &self.inner
145    }
146
147    /// Get a reqwest client for a specific domain.
148    ///
149    /// Priority order:
150    /// 1. DataImpulse residential proxy (country-targeted) — Google only
151    /// 2. Direct connection — default path for everything else
152    ///
153    /// Cheap path by default: non-Google traffic stays direct until a backend
154    /// explicitly asks for proxy escalation.
155    pub fn client_for_domain_with_locale(&self, domain: &str, locale: Option<&str>) -> Client {
156        let needs_residential = self.needs_residential_proxy(domain);
157
158        // Google uses residential immediately.
159        if needs_residential {
160            if let Some(ref di) = self.dataimpulse {
161                if di.is_configured() {
162                    debug!("DataImpulse residential proxy for {domain} locale={locale:?}");
163                    return di.client(locale);
164                }
165            }
166        }
167
168        // Everyone else uses the cheapest path first.
169        self.inner.clone()
170    }
171
172    /// Backwards-compatible wrapper — no locale hint.
173    pub fn client_for_domain(&self, domain: &str) -> Client {
174        self.client_for_domain_with_locale(domain, None)
175    }
176
177    /// Direct client — bypasses ALL proxy routing (residential and datacenter).
178    ///
179    /// Use for first-attempt requests where proxy is not yet needed.
180    /// If the direct attempt fails, escalate to `client_for_domain_with_locale`.
181    /// Saves DataImpulse GB when direct connections succeed (majority of requests).
182    pub fn client_direct(&self) -> Client {
183        self.inner.clone()
184    }
185
186    /// Get a forced proxy client for a domain.
187    ///
188    /// Google uses residential. All other proxy-eligible domains use the fixed
189    /// proxy pool. Falls back to direct when no proxy is available.
190    pub fn proxy_client_for_domain_with_locale(
191        &self,
192        domain: &str,
193        locale: Option<&str>,
194    ) -> Client {
195        if self.needs_residential_proxy(domain) {
196            if let Some(ref di) = self.dataimpulse {
197                if di.is_configured() {
198                    return di.client(locale);
199                }
200            }
201            return self.inner.clone();
202        }
203
204        if let Some(ref pool) = self.proxy_pool {
205            if self.should_proxy_domain(domain) {
206                if let Some(proxy) = pool.proxy_for_domain(domain) {
207                    if let Ok(client) = ProxyPool::build_client_with_proxy(
208                        &proxy,
209                        &self.config.fetch.user_agent,
210                        Duration::from_secs(self.config.fetch.timeout_secs),
211                    ) {
212                        return client;
213                    }
214                }
215            }
216        }
217
218        self.inner.clone()
219    }
220
221    /// Get a **fresh** client that forces a new residential IP on the next request.
222    ///
223    /// Use when a previous request was blocked (CAPTCHA, 403, empty SERP).
224    /// The fresh client has no connection pool, so DataImpulse assigns a new
225    /// residential IP. Falls back to `client_for_domain_with_locale` if not residential.
226    pub fn fresh_client_for_domain_with_locale(
227        &self,
228        domain: &str,
229        locale: Option<&str>,
230    ) -> Client {
231        if self.needs_residential_proxy(domain) {
232            if let Some(ref di) = self.dataimpulse {
233                if di.is_configured() {
234                    return di.fresh_client(locale);
235                }
236            }
237        }
238        // Non-residential: force a new datacenter proxy assignment on retry if possible.
239        if let Some(ref pool) = self.proxy_pool {
240            if self.should_proxy_domain(domain) {
241                if let Some(proxy) = pool.fresh_proxy_for_domain(domain) {
242                    if let Ok(client) = ProxyPool::build_client_with_proxy(
243                        &proxy,
244                        &self.config.fetch.user_agent,
245                        Duration::from_secs(self.config.fetch.timeout_secs),
246                    ) {
247                        return client;
248                    }
249                }
250            }
251        }
252
253        // Fall back to the normal path (direct).
254        self.client_for_domain_with_locale(domain, locale)
255    }
256
257    /// Whether a domain requires residential IPs to avoid blocks.
258    fn needs_residential_proxy(&self, domain: &str) -> bool {
259        // Check user-configured override list first
260        if !self.config.dataimpulse.proxy_domains.is_empty() {
261            return self
262                .config
263                .dataimpulse
264                .proxy_domains
265                .iter()
266                .any(|d| domain.ends_with(d.as_str()) || domain == d.as_str());
267        }
268        // Built-in list of search engines blocked by datacenter IPs
269        RESIDENTIAL_REQUIRED_DOMAINS
270            .iter()
271            .any(|&d| domain == d || domain.ends_with(&format!(".{d}")))
272    }
273
274    /// Get the config reference.
275    pub fn config(&self) -> &FetchiumConfig {
276        &self.config
277    }
278
279    /// Extract domain from a URL for rate limiting.
280    pub fn extract_domain(url: &str) -> String {
281        Url::parse(url)
282            .map(|u| u.host_str().unwrap_or("unknown").to_string())
283            .unwrap_or_else(|_| "unknown".to_string())
284    }
285
286    /// Wait if needed to respect per-domain rate limits.
287    async fn enforce_rate_limit(&self, domain: &str) {
288        let min_delay = Duration::from_millis(200);
289        if let Some(state) = self.domain_delays.get(domain) {
290            let elapsed = state.last_request.elapsed();
291            if elapsed < state.min_delay {
292                let wait = state.min_delay - elapsed;
293                debug!("Rate limiting {domain}: waiting {wait:?}");
294                sleep(wait).await;
295            }
296        }
297        self.domain_delays.insert(
298            domain.to_string(),
299            DomainState {
300                last_request: Instant::now(),
301                min_delay,
302            },
303        );
304    }
305
306    /// Calculate backoff delay with jitter for retry attempt.
307    /// Capped at MAX_BACKOFF_MS to prevent excessive waits on high retry counts.
308    fn backoff_delay(attempt: u32) -> Duration {
309        let base = BASE_DELAY_MS.saturating_mul(2u64.saturating_pow(attempt));
310        let jitter = (std::time::SystemTime::now()
311            .duration_since(std::time::UNIX_EPOCH)
312            .unwrap_or_default()
313            .subsec_nanos()
314            % 500) as u64;
315        Duration::from_millis(base.saturating_add(jitter).min(MAX_BACKOFF_MS))
316    }
317
318    /// Whether a status code is retryable.
319    fn is_retryable_status(status: StatusCode) -> bool {
320        matches!(
321            status,
322            StatusCode::TOO_MANY_REQUESTS
323                | StatusCode::INTERNAL_SERVER_ERROR
324                | StatusCode::BAD_GATEWAY
325                | StatusCode::SERVICE_UNAVAILABLE
326                | StatusCode::GATEWAY_TIMEOUT
327        )
328    }
329
330    /// Map an HTTP status to a structured error kind.
331    fn status_to_error_kind(status: StatusCode) -> ErrorKind {
332        match status {
333            StatusCode::FORBIDDEN => ErrorKind::Http403,
334            StatusCode::TOO_MANY_REQUESTS => ErrorKind::Http429,
335            s if s.is_server_error() => ErrorKind::Http5xx,
336            _ => ErrorKind::Unknown,
337        }
338    }
339
340    /// Fetch a URL with retries, rate limiting, and size enforcement.
341    /// Returns the response body as a string along with metadata.
342    pub async fn fetch(&self, url: &str) -> FetchiumResult<FetchResult> {
343        let domain = Self::extract_domain(url);
344        let start = Instant::now();
345        let max_size = self.config.fetch.max_page_size;
346        let mut last_err: Option<FetchiumError> = None;
347
348        for attempt in 0..=MAX_RETRIES {
349            if attempt > 0 {
350                let delay = Self::backoff_delay(attempt - 1);
351                debug!("Retry {attempt}/{MAX_RETRIES} for {url} after {delay:?}");
352                sleep(delay).await;
353            }
354
355            self.enforce_rate_limit(&domain).await;
356
357            let result = self.inner.get(url).send().await;
358
359            match result {
360                Ok(resp) => {
361                    let status = resp.status();
362
363                    if !status.is_success() {
364                        if Self::is_retryable_status(status) && attempt < MAX_RETRIES {
365                            debug!("Retryable status {status} for {url} (attempt {attempt}/{MAX_RETRIES})");
366                            last_err = Some(FetchiumError::Structured(StructuredError {
367                                kind: Self::status_to_error_kind(status),
368                                retryable: true,
369                                message: format!("HTTP {status} from {url}"),
370                                source_url: Some(url.to_string()),
371                                suggested_action: "Retry with backoff".into(),
372                                alternatives: vec![],
373                            }));
374                            continue;
375                        }
376
377                        return Err(FetchiumError::Structured(StructuredError {
378                            kind: Self::status_to_error_kind(status),
379                            retryable: false,
380                            message: format!("HTTP {status} from {url}"),
381                            source_url: Some(url.to_string()),
382                            suggested_action: match status {
383                                StatusCode::FORBIDDEN => "Site blocks automated access".into(),
384                                StatusCode::TOO_MANY_REQUESTS => "Rate limited, try later".into(),
385                                _ => "Check URL and try again".into(),
386                            },
387                            alternatives: vec![],
388                        }));
389                    }
390
391                    let content_length = resp.content_length();
392                    if let Some(len) = content_length {
393                        if len > max_size {
394                            return Err(FetchiumError::Structured(StructuredError {
395                                kind: ErrorKind::ExtractionFailed,
396                                retryable: false,
397                                message: format!(
398                                    "Response too large: {len} bytes (max {max_size})"
399                                ),
400                                source_url: Some(url.to_string()),
401                                suggested_action: "Increase max_page_size in config".into(),
402                                alternatives: vec![],
403                            }));
404                        }
405                    }
406
407                    let content_type = resp
408                        .headers()
409                        .get("content-type")
410                        .and_then(|v| v.to_str().ok())
411                        .unwrap_or("text/html")
412                        .to_string();
413
414                    let final_url = resp.url().to_string();
415
416                    let body = resp.text().await.map_err(FetchiumError::Network)?;
417
418                    if body.len() as u64 > max_size {
419                        return Err(FetchiumError::Structured(StructuredError {
420                            kind: ErrorKind::ExtractionFailed,
421                            retryable: false,
422                            message: format!("Body too large: {} bytes", body.len()),
423                            source_url: Some(url.to_string()),
424                            suggested_action: "Increase max_page_size".into(),
425                            alternatives: vec![],
426                        }));
427                    }
428
429                    return Ok(FetchResult {
430                        body,
431                        status: status.as_u16(),
432                        content_type,
433                        content_length,
434                        url: final_url,
435                        elapsed_ms: start.elapsed().as_millis() as u64,
436                        retries: attempt,
437                    });
438                }
439                Err(e) => {
440                    if (e.is_timeout() || e.is_connect()) && attempt < MAX_RETRIES {
441                        debug!("Transient error for {url}: {e}");
442                        last_err = Some(FetchiumError::Network(e));
443                        continue;
444                    }
445                    return Err(FetchiumError::Network(e));
446                }
447            }
448        }
449
450        Err(last_err.unwrap_or_else(|| {
451            FetchiumError::Structured(StructuredError {
452                kind: ErrorKind::NetworkTimeout,
453                retryable: false,
454                message: format!("All {MAX_RETRIES} retries exhausted for {url}"),
455                source_url: Some(url.to_string()),
456                suggested_action: "Check network connectivity".into(),
457                alternatives: vec![],
458            })
459        }))
460    }
461
462    /// Convenience: fetch a URL and return just the body text.
463    pub async fn fetch_text(&self, url: &str) -> FetchiumResult<String> {
464        let result = self.fetch(url).await?;
465        Ok(result.body)
466    }
467
468    /// Get the proxy pool (if configured).
469    pub fn proxy_pool(&self) -> Option<&ProxyPool> {
470        self.proxy_pool.as_ref()
471    }
472
473    /// Check if a domain should use proxies.
474    fn should_proxy_domain(&self, domain: &str) -> bool {
475        // Never proxy bypass domains
476        if self
477            .config
478            .proxy
479            .bypass_domains
480            .iter()
481            .any(|d| domain.contains(d.as_str()))
482        {
483            return false;
484        }
485        // If proxy_domains is specified, only proxy those
486        if !self.config.proxy.proxy_domains.is_empty() {
487            return self
488                .config
489                .proxy
490                .proxy_domains
491                .iter()
492                .any(|d| domain.contains(d.as_str()));
493        }
494        // Default: proxy everything
495        true
496    }
497
498    /// Fetch a URL through a rotating proxy. Falls back to direct if no proxy available.
499    pub async fn fetch_via_proxy(&self, url: &str) -> FetchiumResult<FetchResult> {
500        let domain = Self::extract_domain(url);
501
502        let pool = match &self.proxy_pool {
503            Some(p) if !p.is_empty() && self.should_proxy_domain(&domain) => p,
504            _ => return self.fetch(url).await,
505        };
506
507        let proxy = match pool.proxy_for_domain(&domain) {
508            Some(p) => p,
509            None => {
510                debug!("No available proxy for {domain}, falling back to direct");
511                return self.fetch(url).await;
512            }
513        };
514
515        let start = Instant::now();
516        let client = ProxyPool::build_client_with_proxy(
517            &proxy,
518            &self.config.fetch.user_agent,
519            Duration::from_secs(self.config.fetch.timeout_secs),
520        )
521        .map_err(FetchiumError::Network)?;
522
523        match client.get(url).send().await {
524            Ok(resp) => {
525                let status = resp.status();
526                let latency = start.elapsed().as_millis() as u64;
527
528                if status.is_success() {
529                    proxy.record_success(latency);
530                    let content_type = resp
531                        .headers()
532                        .get("content-type")
533                        .and_then(|v| v.to_str().ok())
534                        .unwrap_or("text/html")
535                        .to_string();
536                    let content_length = resp.content_length();
537                    let final_url = resp.url().to_string();
538                    let body = resp.text().await.map_err(FetchiumError::Network)?;
539
540                    Ok(FetchResult {
541                        body,
542                        status: status.as_u16(),
543                        content_type,
544                        content_length,
545                        url: final_url,
546                        elapsed_ms: latency,
547                        retries: 0,
548                    })
549                } else if status == StatusCode::FORBIDDEN || status == StatusCode::TOO_MANY_REQUESTS
550                {
551                    // Proxy got blocked — record failure and retry with next proxy
552                    proxy.record_failure();
553                    debug!(
554                        "Proxy {}:{} blocked (HTTP {}) for {domain}, trying next",
555                        proxy.host, proxy.port, status
556                    );
557                    // One retry with a different proxy
558                    if let Some(next_proxy) = pool.next_proxy() {
559                        let client2 = ProxyPool::build_client_with_proxy(
560                            &next_proxy,
561                            &self.config.fetch.user_agent,
562                            Duration::from_secs(self.config.fetch.timeout_secs),
563                        )
564                        .map_err(FetchiumError::Network)?;
565
566                        let start2 = Instant::now();
567                        match client2.get(url).send().await {
568                            Ok(resp2) if resp2.status().is_success() => {
569                                next_proxy.record_success(start2.elapsed().as_millis() as u64);
570                                let body = resp2.text().await.map_err(FetchiumError::Network)?;
571                                Ok(FetchResult {
572                                    body,
573                                    status: 200,
574                                    content_type: "text/html".into(),
575                                    content_length: None,
576                                    url: url.to_string(),
577                                    elapsed_ms: start.elapsed().as_millis() as u64,
578                                    retries: 1,
579                                })
580                            }
581                            _ => {
582                                next_proxy.record_failure();
583                                self.fetch(url).await // fall back to direct
584                            }
585                        }
586                    } else {
587                        self.fetch(url).await
588                    }
589                } else {
590                    proxy.record_failure();
591                    Err(FetchiumError::Structured(StructuredError {
592                        kind: Self::status_to_error_kind(status),
593                        retryable: false,
594                        message: format!("HTTP {status} via proxy for {url}"),
595                        source_url: Some(url.to_string()),
596                        suggested_action: "Try different proxy or direct".into(),
597                        alternatives: vec![],
598                    }))
599                }
600            }
601            Err(e) => {
602                proxy.record_failure();
603                debug!(
604                    "Proxy {}:{} connection failed for {domain}: {e}",
605                    proxy.host, proxy.port
606                );
607                // Fall back to direct connection
608                self.fetch(url).await
609            }
610        }
611    }
612
613    /// Convenience: fetch text through a proxy.
614    pub async fn fetch_text_via_proxy(&self, url: &str) -> FetchiumResult<String> {
615        let result = self.fetch_via_proxy(url).await?;
616        Ok(result.body)
617    }
618
619    /// Single-shot fetch — no retries, no rate limiting.
620    ///
621    /// Use this for sources where connection errors are *expected* (e.g. third-party
622    /// Piped/Invidious instances that may be down). A single fast attempt is better
623    /// than burning 3.5s on retry sleeps when we're racing many sources in parallel.
624    pub async fn fetch_text_once(&self, url: &str) -> FetchiumResult<String> {
625        let resp = self
626            .inner
627            .get(url)
628            .send()
629            .await
630            .map_err(FetchiumError::Network)?;
631
632        if !resp.status().is_success() {
633            return Err(FetchiumError::Structured(crate::error::StructuredError {
634                kind: Self::status_to_error_kind(resp.status()),
635                retryable: false,
636                message: format!("HTTP {} from {url}", resp.status()),
637                source_url: Some(url.to_string()),
638                suggested_action: "Source unavailable".into(),
639                alternatives: vec![],
640            }));
641        }
642
643        resp.text().await.map_err(FetchiumError::Network)
644    }
645
646    /// Single-shot POST — no retries, no rate limiting.
647    ///
648    /// Like `fetch_text_once` but sends a JSON POST body. Used for YouTube Innertube
649    /// API calls where each call is time-sensitive and retries are not desired.
650    pub async fn post_json_once(&self, url: &str, body: String) -> FetchiumResult<String> {
651        let resp = self
652            .inner
653            .post(url)
654            .header("Content-Type", "application/json")
655            .body(body)
656            .send()
657            .await
658            .map_err(FetchiumError::Network)?;
659
660        if !resp.status().is_success() {
661            return Err(FetchiumError::Structured(crate::error::StructuredError {
662                kind: Self::status_to_error_kind(resp.status()),
663                retryable: false,
664                message: format!("HTTP {} from {url}", resp.status()),
665                source_url: Some(url.to_string()),
666                suggested_action: "Source unavailable".into(),
667                alternatives: vec![],
668            }));
669        }
670
671        resp.text().await.map_err(FetchiumError::Network)
672    }
673
674    /// POST JSON with retry support (uses the standard retry/rate-limit pipeline).
675    pub async fn post_json(&self, url: &str, body: &str) -> FetchiumResult<String> {
676        let resp = self
677            .inner
678            .post(url)
679            .header("Content-Type", "application/json")
680            .body(body.to_string())
681            .send()
682            .await
683            .map_err(FetchiumError::Network)?;
684
685        if !resp.status().is_success() {
686            return Err(FetchiumError::Structured(crate::error::StructuredError {
687                kind: Self::status_to_error_kind(resp.status()),
688                retryable: resp.status().is_server_error(),
689                message: format!("HTTP {} from {url}", resp.status()),
690                source_url: Some(url.to_string()),
691                suggested_action: "Check API key and request format".into(),
692                alternatives: vec![],
693            }));
694        }
695
696        resp.text().await.map_err(FetchiumError::Network)
697    }
698
699    /// POST JSON with a custom header (e.g., API key in non-standard header).
700    pub async fn post_json_with_header(
701        &self,
702        url: &str,
703        body: &str,
704        header_name: &str,
705        header_value: &str,
706    ) -> FetchiumResult<String> {
707        let resp = self
708            .inner
709            .post(url)
710            .header("Content-Type", "application/json")
711            .header(header_name, header_value)
712            .body(body.to_string())
713            .send()
714            .await
715            .map_err(FetchiumError::Network)?;
716
717        if !resp.status().is_success() {
718            return Err(FetchiumError::Structured(crate::error::StructuredError {
719                kind: Self::status_to_error_kind(resp.status()),
720                retryable: resp.status().is_server_error(),
721                message: format!("HTTP {} from {url}", resp.status()),
722                source_url: Some(url.to_string()),
723                suggested_action: "Check API key and request format".into(),
724                alternatives: vec![],
725            }));
726        }
727
728        resp.text().await.map_err(FetchiumError::Network)
729    }
730}
731
732#[cfg(test)]
733mod tests {
734    use super::*;
735
736    #[test]
737    fn client_creation() {
738        let config = FetchiumConfig::default();
739        let client = HttpClient::new(&config);
740        assert!(client.is_ok());
741    }
742
743    #[test]
744    fn domain_extraction() {
745        assert_eq!(
746            HttpClient::extract_domain("https://www.example.com/page"),
747            "www.example.com"
748        );
749        assert_eq!(
750            HttpClient::extract_domain("http://localhost:8080/api"),
751            "localhost"
752        );
753        assert_eq!(HttpClient::extract_domain("not-a-url"), "unknown");
754    }
755
756    #[test]
757    fn backoff_delay_increases() {
758        let d0 = HttpClient::backoff_delay(0);
759        let d1 = HttpClient::backoff_delay(1);
760        let d2 = HttpClient::backoff_delay(2);
761        assert!(d0.as_millis() >= 500 && d0.as_millis() < 1000);
762        assert!(d1.as_millis() >= 1000 && d1.as_millis() < 1500);
763        assert!(d2.as_millis() >= 2000 && d2.as_millis() < 2500);
764    }
765
766    #[test]
767    fn retryable_status_codes() {
768        assert!(HttpClient::is_retryable_status(
769            StatusCode::TOO_MANY_REQUESTS
770        ));
771        assert!(HttpClient::is_retryable_status(StatusCode::BAD_GATEWAY));
772        assert!(HttpClient::is_retryable_status(
773            StatusCode::SERVICE_UNAVAILABLE
774        ));
775        assert!(!HttpClient::is_retryable_status(StatusCode::NOT_FOUND));
776        assert!(!HttpClient::is_retryable_status(StatusCode::FORBIDDEN));
777    }
778
779    #[test]
780    fn residential_proxy_is_google_only_by_default() {
781        let client = HttpClient::new(&FetchiumConfig::default()).unwrap();
782        assert!(client.needs_residential_proxy("google.com"));
783        assert!(client.needs_residential_proxy("www.google.com"));
784        assert!(!client.needs_residential_proxy("duckduckgo.com"));
785        assert!(!client.needs_residential_proxy("html.duckduckgo.com"));
786        assert!(!client.needs_residential_proxy("bing.com"));
787    }
788
789    #[test]
790    fn explicit_residential_domain_override_wins() {
791        let mut config = FetchiumConfig::default();
792        config.dataimpulse.proxy_domains = vec!["example.com".into()];
793        let client = HttpClient::new(&config).unwrap();
794        assert!(client.needs_residential_proxy("www.example.com"));
795        assert!(!client.needs_residential_proxy("google.com"));
796    }
797}