Skip to main content

docsrs_cli/
http.rs

1//! HTTP client: rustls, retry, rate limit, body cap, host allowlist, cancel-aware.
2//!
3//! # Product fetch posture (Rules Rust — web scraping / crawling)
4//!
5//! This is a **one-shot docs client**, not a general crawler:
6//! - **GET-only** product surface against a fixed HTTPS host allowlist.
7//! - **One primary request** per command (no multi-URL frontier, no sitemap/RSS).
8//! - **User-Agent** identifies `docsrs-cli/{version}` plus optional contact.
9//! - **Politeness:** per-host delay floor + additive jitter ([`crate::retry::politeness_delay`]),
10//!   in-process clock and cross-process lock+stamp.
11//! - **Body budget:** stream with `Content-Length` early reject + `try_reserve*`
12//!   (no unbounded `bytes().await`).
13//! - **TLS:** rustls only; never `danger_accept_invalid_certs`.
14//! - **No cookie jar** (`cookie_store` feature not enabled); no shared cookies.
15//! - **Provenance:** HTML payloads expose `source_url` on success envelopes.
16//! - **OOS:** robots.txt REP parser, meta robots / X-Robots-Tag, ETag conditional
17//!   GETs, anti-bot TLS fingerprinting, headless Chrome, proxy rotation, MinHash.
18//!   See `docs/decisions/0003-web-fetch-scope.md`.
19//!
20//! # Workload classification (Rules Rust — parallelism)
21//!
22//! - **Class:** mixed — async I/O stage on multi-thread Tokio + CPU parse stage
23//!   via [`crate::concurrency::ConcurrencyBudget::run_cpu_bound`].
24//! - **Shareable:** methods take `&self`. Per-host in-process clock is a
25//!   `Mutex<HashMap>` with sleep **outside** the lock (no `Mutex` across `.await`
26//!   for the map itself; cross-process flock still owns multi-process throttle).
27//! - **Budget:** each client owns a [`ConcurrencyBudget`] for `spawn_blocking`
28//!   parse work (and future multi-GET fan-out). Bound is explicit (auto or
29//!   `--max-concurrency`).
30//! - **Network:** product commands still issue one primary GET at a time; rate
31//!   limit + body stream cap remain the I/O backpressure.
32//!
33//! # Retry (Rules Rust — retry/backoff)
34//!
35//! Policy lives in [`crate::retry::RetryConfig`] (not inline magic). GET-only
36//! product surface ⇒ idempotent retries for 429/5xx/transport only. Kill switch
37//! `--disable-retry` / `DOCSRS_CLI_DISABLE_RETRY`. See module docs on `retry`.
38
39use std::collections::HashMap;
40use std::fs::{self, File, OpenOptions};
41use std::path::{Path, PathBuf};
42use std::sync::Mutex;
43use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
44
45use bytes::Bytes;
46use fs4::FileExt as Fs4FileExt;
47use fs4::TryLockError as Fs4TryLockError;
48use futures_util::StreamExt;
49use reqwest::header::{ACCEPT, ACCEPT_ENCODING, HeaderMap, HeaderValue, USER_AGENT};
50use reqwest::{Client, Method, StatusCode, Url};
51use tracing::{debug, warn};
52
53use crate::cache::DiskCache;
54use crate::concurrency::ConcurrencyBudget;
55use crate::config::{
56    Config, HARD_MAX_BODY_BYTES, HOST_CRATES_IO, HOST_DOC_RUST_LANG_ORG, HOST_DOCS_RS,
57    HOST_STATIC_DOCS_RS,
58};
59use crate::error::{AppError, AppResult, ErrorKind};
60use crate::retry::{
61    RetryClass, RetryConfig, classify_http_status, parse_retry_after, politeness_delay,
62    wait_for_retry,
63};
64use crate::shutdown::CancelFlag;
65
66/// HTTPS client bound to product allowlist and config.
67///
68/// Shareable across tasks (`&self`): rate-limit clock uses a short-lived
69/// [`Mutex`] around the map only; sleeps run without holding the lock.
70pub struct HttpClient {
71    client: Client,
72    cfg: Config,
73    /// In-process last hit per host. Lock is never held across `.await`.
74    last_host_hit: Mutex<HashMap<String, Instant>>,
75    cancel: CancelFlag,
76    cache: Option<DiskCache>,
77    /// Admission gate for CPU-bound parse (and future multi-GET fan-out).
78    budget: ConcurrencyBudget,
79    /// Explicit retry policy for this client's dependency class.
80    retry: RetryConfig,
81}
82
83/// Successful HTTP response body and metadata.
84#[derive(Debug, Clone)]
85pub struct HttpResponse {
86    /// HTTP status code of the final response.
87    pub status: StatusCode,
88    /// Final URL after redirects.
89    pub final_url: Url,
90    /// Response body bytes (already capped).
91    pub body: Bytes,
92    /// Content-Type header when present.
93    pub content_type: Option<String>,
94}
95
96impl HttpClient {
97    /// Build a client with rustls, default headers, and redirect allowlist.
98    ///
99    /// Owned by one one-shot command. Call sites may `Config::clone` before passing
100    /// `cfg` when they still need the parent config after construction. That clone is
101    /// intentional and cheap relative to TLS/RTT/body I/O. [`ConcurrencyBudget`] is
102    /// built from `cfg.max_concurrency` (0 = auto).
103    ///
104    /// # Errors
105    ///
106    /// Returns [`ErrorKind::Config`] when the User-Agent is not a valid header value.
107    /// Returns [`ErrorKind::Network`] when the reqwest client cannot be built.
108    pub fn new(cfg: Config, cancel: CancelFlag) -> AppResult<Self> {
109        // Defense in depth: clamp even if a caller skipped Config::clamp_resource_limits.
110        let mut cfg = cfg;
111        cfg.clamp_resource_limits();
112        let budget = ConcurrencyBudget::from_configured(cfg.max_concurrency);
113        let retry = RetryConfig::from_config(&cfg);
114
115        let mut headers = HeaderMap::new();
116        headers.insert(
117            USER_AGENT,
118            HeaderValue::from_str(&cfg.user_agent).map_err(|e| {
119                AppError::with_source(ErrorKind::Config, "invalid user-agent header", e)
120            })?,
121        );
122        headers.insert(ACCEPT_ENCODING, HeaderValue::from_static("gzip"));
123
124        let client = Client::builder()
125            .default_headers(headers)
126            .timeout(cfg.timeout())
127            .connect_timeout(cfg.connect_timeout())
128            .redirect(reqwest::redirect::Policy::custom({
129                let max = cfg.max_redirects;
130                move |attempt| {
131                    if attempt.previous().len() as u32 >= max {
132                        return attempt
133                            .error(AppError::new(ErrorKind::Network, "redirect limit exceeded"));
134                    }
135                    // Borrow host_str — no intermediate String on the redirect hot path.
136                    let host = attempt.url().host_str().unwrap_or("");
137                    let scheme_ok = attempt.url().scheme() == "https";
138                    let host_ok = matches!(
139                        host,
140                        HOST_CRATES_IO
141                            | HOST_DOCS_RS
142                            | HOST_STATIC_DOCS_RS
143                            | HOST_DOC_RUST_LANG_ORG
144                    ) || ((host == "127.0.0.1" || host == "localhost")
145                        && (cfg!(test) || std::env::var("DOCSRS_CLI_ALLOW_LOCALHOST").is_ok()));
146                    if !(scheme_ok && host_ok) {
147                        // Materialize the message before moving `attempt` (host is borrowed).
148                        let msg = format!("redirect host not allowlisted: {host}");
149                        return attempt.error(AppError::new(ErrorKind::Network, msg));
150                    }
151                    attempt.follow()
152                }
153            }))
154            .use_rustls_tls()
155            .build()
156            .map_err(|e| {
157                AppError::with_source(ErrorKind::Network, "failed to build HTTP client", e)
158            })?;
159
160        let cache = if cfg.cache_enabled() {
161            cfg.cache_dir
162                .as_ref()
163                .map(|dir| DiskCache::new(dir.clone(), cfg.cache_ttl(), cfg.max_cache_bytes))
164        } else {
165            None
166        };
167
168        Ok(Self {
169            client,
170            cfg,
171            last_host_hit: Mutex::new(HashMap::new()),
172            cancel,
173            cache,
174            budget,
175            retry,
176        })
177    }
178
179    /// Shared concurrency budget for CPU-bound parse work.
180    pub fn budget(&self) -> &ConcurrencyBudget {
181        &self.budget
182    }
183
184    /// Resolved retry policy (for doctor / tests).
185    pub fn retry_policy(&self) -> RetryConfig {
186        self.retry
187    }
188
189    /// GET expecting JSON (`Accept: application/json`).
190    ///
191    /// # Errors
192    ///
193    /// Returns [`ErrorKind::Network`] for allowlist, redirect, TLS, or body-cap failures.
194    /// Returns [`ErrorKind::Timeout`] when the wall-clock budget expires.
195    /// Returns [`ErrorKind::Interrupted`] or [`ErrorKind::Terminated`] on cancel
196    /// (kind preserved from the first `CancelFlag::cancel_with`).
197    /// Maps HTTP statuses via [`AppError::from_http_status`] (e.g. not-found, rate-limit).
198    pub async fn get_json(&self, url: &Url) -> AppResult<HttpResponse> {
199        self.request(Method::GET, url, "application/json", true)
200            .await
201    }
202
203    /// GET expecting HTML (`Accept: text/html`).
204    ///
205    /// # Errors
206    ///
207    /// Same error classes as [`Self::get_json`].
208    pub async fn get_html(&self, url: &Url) -> AppResult<HttpResponse> {
209        self.request(Method::GET, url, "text/html", true).await
210    }
211
212    async fn request(
213        &self,
214        method: Method,
215        url: &Url,
216        accept: &str,
217        retryable: bool,
218    ) -> AppResult<HttpResponse> {
219        if !is_allowed_host(url) {
220            return Err(AppError::new(
221                ErrorKind::Network,
222                format!("host not allowlisted: {}", url.host_str().unwrap_or("")),
223            ));
224        }
225
226        // Disk cache only for GET (product surface is GET-only).
227        if method == Method::GET
228            && let Some(cache) = &self.cache
229            && let Some(hit) = cache.get(url, accept)
230        {
231            return Ok(hit);
232        }
233
234        // Policy is opt-out kill switch + max_retries; `retryable` is for future
235        // non-idempotent methods (product is GET-only today).
236        let policy = if retryable {
237            self.retry
238        } else {
239            RetryConfig {
240                enabled: false,
241                ..self.retry
242            }
243        };
244        // Prefer `url.as_str()` over `url.clone()`: reqwest implements IntoUrl for
245        // `&str` but not for `&Url` (0.12). Method is a small enum — clone is trivial.
246        let url_str = url.as_str();
247        let mut attempt = 0u32;
248        loop {
249            self.cancel.check()?;
250            attempt += 1;
251            self.rate_limit(url).await?;
252            self.cancel.check()?;
253
254            let mut req = self.client.request(method.clone(), url_str);
255            req = req.header(ACCEPT, accept);
256
257            let result = req.send().await;
258            match result {
259                Ok(resp) => {
260                    let status = resp.status();
261                    let final_url = resp.url().clone();
262                    let content_type = resp
263                        .headers()
264                        .get(reqwest::header::CONTENT_TYPE)
265                        .and_then(|v| v.to_str().ok())
266                        .map(|s| s.to_string());
267                    let retry_after = parse_retry_after(resp.headers());
268                    let class = classify_http_status(status, retry_after);
269
270                    match class {
271                        RetryClass::RateLimited { retry_after } if policy.may_retry(attempt) => {
272                            let wait = wait_for_retry(policy, attempt, retry_after);
273                            debug!(
274                                target: "docsrs_cli::retry",
275                                ?wait,
276                                attempt,
277                                max_attempts = policy.max_attempts(),
278                                status = %status,
279                                "retrying after rate-limit / Retry-After"
280                            );
281                            self.sleep_cancelable(wait).await?;
282                            continue;
283                        }
284                        RetryClass::Transient if policy.may_retry(attempt) => {
285                            let wait = wait_for_retry(policy, attempt, None);
286                            debug!(
287                                target: "docsrs_cli::retry",
288                                ?wait,
289                                attempt,
290                                max_attempts = policy.max_attempts(),
291                                status = %status,
292                                "retrying after transient 5xx"
293                            );
294                            self.sleep_cancelable(wait).await?;
295                            continue;
296                        }
297                        RetryClass::RateLimited { retry_after } => {
298                            let secs = retry_after.map(|d| d.as_secs()).unwrap_or(1);
299                            return Err(AppError::from_http_status(
300                                status.as_u16(),
301                                "retries exhausted",
302                            )
303                            .with_retry_after(secs.max(1)));
304                        }
305                        RetryClass::Transient => {
306                            // Exhausted retries on 5xx: map status without body read.
307                            return Err(AppError::from_http_status(
308                                status.as_u16(),
309                                "retries exhausted",
310                            ));
311                        }
312                        RetryClass::Permanent => {
313                            // 2xx and permanent 4xx: read body and return (caller maps status).
314                        }
315                    }
316
317                    let body = read_body_capped(resp, self.cfg.max_body_bytes).await?;
318                    let out = HttpResponse {
319                        status,
320                        final_url,
321                        body,
322                        content_type,
323                    };
324                    if method == Method::GET
325                        && out.status.is_success()
326                        && let Some(cache) = &self.cache
327                        && let Err(e) = cache.put(url, accept, &out)
328                    {
329                        warn!(error = %e, "failed to write HTTP cache entry");
330                    }
331                    return Ok(out);
332                }
333                Err(e) => {
334                    // Classify transport layers without string-matching messages.
335                    let is_timeout = e.is_timeout();
336                    let is_connect = e.is_connect();
337                    let is_request = e.is_request();
338                    let kind = if is_timeout {
339                        ErrorKind::Timeout
340                    } else {
341                        ErrorKind::Network
342                    };
343                    let transport_retryable = is_timeout || is_connect || is_request;
344                    if policy.may_retry(attempt) && transport_retryable {
345                        let wait = wait_for_retry(policy, attempt, None);
346                        warn!(
347                            target: "docsrs_cli::retry",
348                            ?wait,
349                            attempt,
350                            max_attempts = policy.max_attempts(),
351                            error = %e,
352                            is_timeout,
353                            is_connect,
354                            "retrying after transport error"
355                        );
356                        self.sleep_cancelable(wait).await?;
357                        continue;
358                    }
359                    return Err(AppError::with_source(
360                        kind,
361                        format!("HTTP request failed for {url}"),
362                        e,
363                    ));
364                }
365            }
366        }
367    }
368
369    async fn sleep_cancelable(&self, wait: Duration) -> AppResult<()> {
370        let step = Duration::from_millis(50);
371        let mut remaining = wait;
372        while !remaining.is_zero() {
373            self.cancel.check()?;
374            let slice = remaining.min(step);
375            tokio::time::sleep(slice).await;
376            remaining = remaining.saturating_sub(slice);
377        }
378        self.cancel.check()
379    }
380
381    /// Enforce per-host delay: in-process clock + exclusive cross-process lock+stamp.
382    ///
383    /// Floor is `rate_limit_delay_ms`; each wait uses [`politeness_delay`] so the
384    /// effective interval is never fixed (additive jitter up to +20%).
385    ///
386    /// In-process map uses a short [`Mutex`] hold (no sleep under the lock). Cross-process
387    /// flock is released on drop before return (never held across unrelated work).
388    async fn rate_limit(&self, url: &Url) -> AppResult<()> {
389        // Lookup borrows host; allocate only when inserting/updating the clock map.
390        let host = url.host_str().unwrap_or("");
391        let delay = politeness_delay(self.cfg.rate_limit_delay());
392        if delay.is_zero() {
393            return Ok(());
394        }
395
396        // Prefer exclusive cross-process section when cache_dir is available.
397        if let Some(dir) = self.cfg.cache_dir.as_ref() {
398            match self.rate_limit_cross_process(dir, host, delay).await {
399                Ok(()) => {
400                    self.touch_host_clock(host);
401                    return Ok(());
402                }
403                Err(e)
404                    if e.kind() == ErrorKind::Terminated || e.kind() == ErrorKind::Interrupted =>
405                {
406                    return Err(e);
407                }
408                Err(e) => {
409                    // FS without flock or I/O failure: fall back to in-process only.
410                    warn!(error = %e, host = %host, "cross-process rate-limit lock failed; using in-process only");
411                }
412            }
413        }
414
415        let in_proc = {
416            let map = self
417                .last_host_hit
418                .lock()
419                .unwrap_or_else(|e| e.into_inner());
420            map.get(host).and_then(|prev| {
421                let elapsed = prev.elapsed();
422                if elapsed < delay {
423                    Some(delay - elapsed)
424                } else {
425                    None
426                }
427            })
428        };
429        if let Some(d) = in_proc {
430            self.sleep_cancelable(d).await?;
431        }
432        self.touch_host_clock(host);
433        Ok(())
434    }
435
436    fn touch_host_clock(&self, host: &str) {
437        let mut map = self
438            .last_host_hit
439            .lock()
440            .unwrap_or_else(|e| e.into_inner());
441        map.insert(host.to_string(), Instant::now());
442    }
443
444    /// Exclusive lock on `{cache_dir}/rate-limit/{host}.lock`, then stamp throttle.
445    async fn rate_limit_cross_process(
446        &self,
447        cache_dir: &Path,
448        host: &str,
449        delay: Duration,
450    ) -> AppResult<()> {
451        let lock_path = rate_limit_lock_path(cache_dir, host);
452        let stamp_path = rate_limit_stamp_path(cache_dir, host);
453        if let Some(parent) = lock_path.parent() {
454            fs::create_dir_all(parent).map_err(|e| {
455                AppError::with_source(ErrorKind::Network, "rate-limit lock dir create failed", e)
456            })?;
457            crate::platform::restrict_private_dir(parent);
458        }
459
460        let file = OpenOptions::new()
461            .create(true)
462            .read(true)
463            .write(true)
464            .truncate(false)
465            .open(&lock_path)
466            .map_err(|e| {
467                AppError::with_source(ErrorKind::Network, "rate-limit lock open failed", e)
468            })?;
469        crate::platform::restrict_private_file(&lock_path);
470
471        // Cancel-aware exclusive acquisition (fs4 try_lock + short sleep).
472        // Fully-qualified calls avoid collision with std::fs::File::try_lock (Rust 1.89+).
473        loop {
474            self.cancel.check()?;
475            match Fs4FileExt::try_lock(&file) {
476                Ok(()) => break,
477                Err(Fs4TryLockError::WouldBlock) => {
478                    self.sleep_cancelable(Duration::from_millis(50)).await?;
479                }
480                Err(Fs4TryLockError::Error(e)) => {
481                    return Err(AppError::with_source(
482                        ErrorKind::Network,
483                        "rate-limit exclusive lock failed",
484                        e,
485                    ));
486                }
487            }
488        }
489
490        // Guard unlock on all exit paths.
491        struct UnlockOnDrop<'a>(&'a File);
492        impl Drop for UnlockOnDrop<'_> {
493            fn drop(&mut self) {
494                let _ = Fs4FileExt::unlock(self.0);
495            }
496        }
497        let _guard = UnlockOnDrop(&file);
498
499        let remaining = stamp_remaining(&stamp_path, delay);
500        if let Some(d) = remaining {
501            self.sleep_cancelable(d).await?;
502        }
503        write_stamp(&stamp_path)?;
504        Ok(())
505    }
506}
507
508fn safe_host_name(host: &str) -> String {
509    let safe: String = host
510        .chars()
511        .map(|c| {
512            if c.is_ascii_alphanumeric() || c == '.' || c == '-' {
513                c
514            } else {
515                '_'
516            }
517        })
518        .collect();
519    if safe.is_empty() {
520        "unknown".to_string()
521    } else {
522        safe
523    }
524}
525
526/// Path for host rate-limit stamp: `{cache_dir}/rate-limit/{safe_host}.stamp`.
527fn rate_limit_stamp_path(cache_dir: &Path, host: &str) -> PathBuf {
528    cache_dir
529        .join("rate-limit")
530        .join(format!("{}.stamp", safe_host_name(host)))
531}
532
533/// Path for host rate-limit exclusive lock: `{cache_dir}/rate-limit/{safe_host}.lock`.
534fn rate_limit_lock_path(cache_dir: &Path, host: &str) -> PathBuf {
535    cache_dir
536        .join("rate-limit")
537        .join(format!("{}.lock", safe_host_name(host)))
538}
539
540fn now_unix_ms() -> Option<u128> {
541    SystemTime::now()
542        .duration_since(UNIX_EPOCH)
543        .ok()
544        .map(|d| d.as_millis())
545}
546
547/// Remaining sleep from stamp file, if any.
548fn stamp_remaining(path: &Path, delay: Duration) -> Option<Duration> {
549    let text = fs::read_to_string(path).ok()?;
550    let last_ms: u128 = text.trim().parse().ok()?;
551    let now = now_unix_ms()?;
552    if now < last_ms {
553        return Some(delay);
554    }
555    let elapsed = Duration::from_millis((now - last_ms) as u64);
556    if elapsed < delay {
557        Some(delay - elapsed)
558    } else {
559        None
560    }
561}
562
563/// Write stamp under exclusive lock (temp + rename).
564fn write_stamp(path: &Path) -> AppResult<()> {
565    if let Some(parent) = path.parent() {
566        fs::create_dir_all(parent).map_err(|e| {
567            AppError::with_source(ErrorKind::Network, "rate-limit stamp dir create failed", e)
568        })?;
569    }
570    let now = now_unix_ms()
571        .ok_or_else(|| AppError::new(ErrorKind::Internal, "system clock before UNIX epoch"))?;
572    let tmp = path.with_extension("stamp.tmp");
573    fs::write(&tmp, now.to_string()).map_err(|e| {
574        AppError::with_source(ErrorKind::Network, "rate-limit stamp write failed", e)
575    })?;
576    fs::rename(&tmp, path).map_err(|e| {
577        let _ = fs::remove_file(&tmp);
578        AppError::with_source(ErrorKind::Network, "rate-limit stamp rename failed", e)
579    })?;
580    crate::platform::restrict_private_file(path);
581    Ok(())
582}
583
584fn is_allowed_host(url: &Url) -> bool {
585    match url.host_str() {
586        Some(HOST_CRATES_IO)
587        | Some(HOST_DOCS_RS)
588        | Some(HOST_STATIC_DOCS_RS)
589        | Some(HOST_DOC_RUST_LANG_ORG) => url.scheme() == "https",
590        Some("127.0.0.1") | Some("localhost") => {
591            cfg!(test) || std::env::var("DOCSRS_CLI_ALLOW_LOCALHOST").is_ok()
592        }
593        _ => false,
594    }
595}
596
597/// Read a response body with a hard byte budget and fallible allocation.
598///
599/// Primary defense is [`HARD_MAX_BODY_BYTES`] (operators may only lower the cap).
600/// `try_reserve` / `try_reserve_exact` map allocation failure to
601/// [`ErrorKind::Network`] instead of aborting via `with_capacity` on hostile sizes.
602/// On Linux overcommit, the allocator may still report success and the OOM killer
603/// can fire later — the hard ceiling remains the main bound.
604async fn read_body_capped(resp: reqwest::Response, max_bytes: u64) -> AppResult<Bytes> {
605    // Never honor a budget above the product hard ceiling (defense in depth).
606    let max_bytes = max_bytes.min(HARD_MAX_BODY_BYTES);
607    // When Content-Length is known and already over budget, fail without buffering.
608    // (With gzip, length may be compressed size or absent; still a useful early guard.)
609    let content_length = resp.content_length();
610    if let Some(n) = content_length
611        && n > max_bytes
612    {
613        return Err(AppError::new(
614            ErrorKind::Network,
615            format!("response body exceeds max_body_bytes ({max_bytes})"),
616        ));
617    }
618    // Pre-size when Content-Length is present to avoid realloc churn on large docs pages.
619    // Prefer try_reserve_exact when length is known; never with_capacity on external size.
620    let capacity = content_length
621        .map(|n| (n as usize).min(max_bytes as usize))
622        .unwrap_or(0);
623    let mut stream = resp.bytes_stream();
624    let mut buf: Vec<u8> = Vec::new();
625    if capacity > 0 {
626        buf.try_reserve_exact(capacity).map_err(|e| {
627            AppError::with_source(
628                ErrorKind::Network,
629                format!("failed to reserve {capacity} bytes for response body"),
630                e,
631            )
632        })?;
633    }
634    while let Some(chunk) = stream.next().await {
635        let chunk = chunk.map_err(|e| {
636            AppError::with_source(ErrorKind::Network, "failed reading response body", e)
637        })?;
638        if (buf.len() as u64).saturating_add(chunk.len() as u64) > max_bytes {
639            // Body cap is a transport/resource limit, not user input validation.
640            return Err(AppError::new(
641                ErrorKind::Network,
642                format!("response body exceeds max_body_bytes ({max_bytes})"),
643            ));
644        }
645        // Grow with try_reserve when stream chunks exceed Content-Length estimate.
646        let need = chunk.len();
647        if buf.capacity().saturating_sub(buf.len()) < need {
648            buf.try_reserve(need).map_err(|e| {
649                AppError::with_source(
650                    ErrorKind::Network,
651                    format!("failed to reserve {need} more body bytes"),
652                    e,
653                )
654            })?;
655        }
656        buf.extend_from_slice(&chunk);
657    }
658    Ok(Bytes::from(buf))
659}
660
661/// Decode body as UTF-8, stripping a leading UTF-8 BOM.
662///
663/// # Errors
664///
665/// Returns [`ErrorKind::Parse`] when the body is not valid UTF-8.
666pub fn decode_utf8(body: &Bytes) -> AppResult<String> {
667    let mut bytes = body.as_ref();
668    if bytes.starts_with(&[0xEF, 0xBB, 0xBF]) {
669        bytes = &bytes[3..];
670    }
671    std::str::from_utf8(bytes)
672        .map(|s| s.to_string())
673        .map_err(|e| AppError::with_source(ErrorKind::Parse, "response body is not valid UTF-8", e))
674}
675
676/// ASCII case-insensitive substring check without heap allocation.
677///
678/// Used on Content-Type sniffing (hot path after every successful GET) so we
679/// never build a lowercased temporary `String` for a few-byte header.
680fn ascii_contains_ignore_case(haystack: &str, needle: &str) -> bool {
681    if needle.is_empty() {
682        return true;
683    }
684    let n = needle.as_bytes();
685    haystack
686        .as_bytes()
687        .windows(n.len())
688        .any(|w| w.eq_ignore_ascii_case(n))
689}
690
691/// True when Content-Type looks like JSON.
692pub fn content_type_looks_json(ct: Option<&str>) -> bool {
693    ct.is_some_and(|c| ascii_contains_ignore_case(c, "json"))
694}
695
696/// True when Content-Type looks like HTML/XHTML.
697pub fn content_type_looks_html(ct: Option<&str>) -> bool {
698    ct.is_some_and(|c| {
699        ascii_contains_ignore_case(c, "html") || ascii_contains_ignore_case(c, "xhtml")
700    })
701}
702
703#[cfg(test)]
704mod tests {
705    use super::*;
706
707    #[test]
708    fn allowlist_https_only() {
709        let ok = Url::parse("https://docs.rs/serde").unwrap();
710        assert!(is_allowed_host(&ok));
711        let stdlib = Url::parse("https://doc.rust-lang.org/stable/std/").unwrap();
712        assert!(is_allowed_host(&stdlib));
713        let bad = Url::parse("http://docs.rs/serde").unwrap();
714        assert!(!is_allowed_host(&bad));
715        let evil = Url::parse("https://evil.example/").unwrap();
716        assert!(!is_allowed_host(&evil));
717    }
718
719    #[test]
720    fn utf8_bom_strip() {
721        let mut v = vec![0xEF, 0xBB, 0xBF];
722        v.extend_from_slice(b"hello");
723        let s = decode_utf8(&Bytes::from(v)).unwrap();
724        assert_eq!(s, "hello");
725    }
726
727    #[test]
728    fn content_types() {
729        assert!(content_type_looks_json(Some(
730            "application/json; charset=utf-8"
731        )));
732        assert!(content_type_looks_json(Some("Application/JSON")));
733        assert!(content_type_looks_html(Some("text/html")));
734        assert!(content_type_looks_html(Some("Text/HTML; charset=utf-8")));
735        assert!(content_type_looks_html(Some("application/xhtml+xml")));
736        assert!(!content_type_looks_json(Some("text/html")));
737        assert!(!content_type_looks_json(None));
738        assert!(!content_type_looks_html(None));
739    }
740
741    #[test]
742    fn ascii_contains_ignore_case_edges() {
743        assert!(ascii_contains_ignore_case("abJSON", "json"));
744        assert!(ascii_contains_ignore_case("json", "json"));
745        assert!(!ascii_contains_ignore_case("js", "json"));
746        assert!(ascii_contains_ignore_case("x", ""));
747    }
748
749    #[test]
750    fn retry_policy_kill_switch() {
751        let cfg = Config {
752            disable_retry: true,
753            max_retries: 5,
754            ..Config::default()
755        };
756        let http = HttpClient::new(cfg, CancelFlag::new()).unwrap();
757        assert!(!http.retry_policy().enabled);
758        assert_eq!(http.retry_policy().max_attempts(), 1);
759    }
760
761    #[test]
762    fn invalid_user_agent_rejected() {
763        let cfg = Config {
764            user_agent: "bad\nua".into(),
765            ..Config::default()
766        };
767        match HttpClient::new(cfg, CancelFlag::new()) {
768            Ok(_) => panic!("expected invalid user-agent to fail"),
769            Err(err) => assert_eq!(err.kind(), ErrorKind::Config),
770        }
771    }
772
773    #[test]
774    fn localhost_allowed_in_unit_tests() {
775        let ok = Url::parse("http://127.0.0.1:9/x").unwrap();
776        assert!(is_allowed_host(&ok));
777        let local = Url::parse("http://localhost:9/x").unwrap();
778        assert!(is_allowed_host(&local));
779    }
780
781    #[test]
782    fn decode_utf8_rejects_invalid() {
783        let err = decode_utf8(&Bytes::from(vec![0xff, 0xfe])).unwrap_err();
784        assert_eq!(err.kind(), ErrorKind::Parse);
785    }
786}