Skip to main content

docsrs_cli/
http.rs

1//! HTTP client: rustls, retry, rate limit, body cap, host allowlist, cancel-aware.
2//!
3//! # Product fetch posture (Rules Rust — web scraping / crawling)
4//!
5//! This is a **one-shot docs client**, not a general crawler:
6//! - **GET-only** product surface against a fixed HTTPS host allowlist.
7//! - **One primary request** per command (no multi-URL frontier, no sitemap/RSS).
8//! - **User-Agent** identifies `docsrs-cli/{version}` plus optional contact.
9//! - **Politeness:** per-host delay floor + additive jitter ([`crate::retry::politeness_delay`]),
10//!   in-process clock and cross-process lock+stamp.
11//! - **Body budget:** stream with `Content-Length` early reject + `try_reserve*`
12//!   (no unbounded `bytes().await`).
13//! - **TLS:** rustls only; never `danger_accept_invalid_certs`.
14//! - **No cookie jar** (`cookie_store` feature not enabled); no shared cookies.
15//! - **Provenance:** HTML payloads expose `source_url` on success envelopes.
16//! - **OOS:** robots.txt REP parser, meta robots / X-Robots-Tag, ETag conditional
17//!   GETs, anti-bot TLS fingerprinting, headless Chrome, proxy rotation, MinHash.
18//!   See `docs/decisions/0003-web-fetch-scope.md`.
19//!
20//! # Workload classification (Rules Rust — parallelism)
21//!
22//! - **Class:** mixed — async I/O stage on multi-thread Tokio + CPU parse stage
23//!   via [`crate::concurrency::ConcurrencyBudget::run_cpu_bound`].
24//! - **Shareable:** methods take `&self`. Per-host in-process clock is a
25//!   `Mutex<HashMap>` with sleep **outside** the lock (no `Mutex` across `.await`
26//!   for the map itself; cross-process flock still owns multi-process throttle).
27//! - **Budget:** each client owns a [`ConcurrencyBudget`] for `spawn_blocking`
28//!   parse work (and future multi-GET fan-out). Bound is explicit (auto or
29//!   `--max-concurrency`).
30//! - **Network:** product commands still issue one primary GET at a time; rate
31//!   limit + body stream cap remain the I/O backpressure.
32//!
33//! # Retry (Rules Rust — retry/backoff)
34//!
35//! Policy lives in [`crate::retry::RetryConfig`] (not inline magic). GET-only
36//! product surface ⇒ idempotent retries for 429/5xx/transport only. Kill switch
37//! `--disable-retry` / TOML `disable_retry` / `max_retries=0`. See module docs on `retry`.
38
39use std::collections::HashMap;
40use std::fs::{self, File, OpenOptions};
41use std::path::{Path, PathBuf};
42use std::sync::Mutex;
43use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
44
45use bytes::Bytes;
46use fs4::FileExt as Fs4FileExt;
47use fs4::TryLockError as Fs4TryLockError;
48use futures_util::StreamExt;
49use reqwest::header::{ACCEPT, ACCEPT_ENCODING, HeaderMap, HeaderValue, USER_AGENT};
50use reqwest::{Client, Method, StatusCode, Url};
51use tracing::{debug, warn};
52
53use crate::cache::DiskCache;
54use crate::concurrency::ConcurrencyBudget;
55use crate::config::{
56    Config, HARD_MAX_BODY_BYTES, HOST_CRATES_IO, HOST_DOC_RUST_LANG_ORG, HOST_DOCS_RS,
57    HOST_STATIC_DOCS_RS,
58};
59use crate::error::{AppError, AppResult, ErrorKind};
60use crate::retry::{
61    RetryClass, RetryConfig, classify_http_status, parse_retry_after, politeness_delay,
62    wait_for_retry,
63};
64use crate::shutdown::CancelFlag;
65
66/// HTTPS client bound to product allowlist and config.
67///
68/// Shareable across tasks (`&self`): rate-limit clock uses a short-lived
69/// [`Mutex`] around the map only; sleeps run without holding the lock.
70pub struct HttpClient {
71    client: Client,
72    cfg: Config,
73    /// In-process last hit per host. Lock is never held across `.await`.
74    last_host_hit: Mutex<HashMap<String, Instant>>,
75    cancel: CancelFlag,
76    cache: Option<DiskCache>,
77    /// Admission gate for CPU-bound parse (and future multi-GET fan-out).
78    budget: ConcurrencyBudget,
79    /// Explicit retry policy for this client's dependency class.
80    retry: RetryConfig,
81}
82
83/// Successful HTTP response body and metadata.
84#[derive(Debug, Clone)]
85pub struct HttpResponse {
86    /// HTTP status code of the final response.
87    pub status: StatusCode,
88    /// Final URL after redirects.
89    pub final_url: Url,
90    /// Response body bytes (already capped).
91    pub body: Bytes,
92    /// Content-Type header when present.
93    pub content_type: Option<String>,
94    /// True when this response was served from the local disk cache (GAP-020).
95    pub cache_hit: bool,
96}
97
98impl HttpClient {
99    /// Build a client with rustls, default headers, and redirect allowlist.
100    ///
101    /// Owned by one one-shot command. Call sites may `Config::clone` before passing
102    /// `cfg` when they still need the parent config after construction. That clone is
103    /// intentional and cheap relative to TLS/RTT/body I/O. [`ConcurrencyBudget`] is
104    /// built from `cfg.max_concurrency` (0 = auto).
105    ///
106    /// # Errors
107    ///
108    /// Returns [`ErrorKind::Config`] when the User-Agent is not a valid header value.
109    /// Returns [`ErrorKind::Network`] when the reqwest client cannot be built.
110    pub fn new(cfg: Config, cancel: CancelFlag) -> AppResult<Self> {
111        // Defense in depth: clamp even if a caller skipped Config::clamp_resource_limits.
112        let mut cfg = cfg;
113        cfg.clamp_resource_limits();
114        let budget = ConcurrencyBudget::from_configured(cfg.max_concurrency);
115        let retry = RetryConfig::from_config(&cfg);
116
117        let mut headers = HeaderMap::new();
118        headers.insert(
119            USER_AGENT,
120            HeaderValue::from_str(&cfg.user_agent).map_err(|e| {
121                AppError::with_source(ErrorKind::Config, "invalid user-agent header", e)
122            })?,
123        );
124        headers.insert(ACCEPT_ENCODING, HeaderValue::from_static("gzip"));
125
126        let client = Client::builder()
127            .default_headers(headers)
128            .timeout(cfg.timeout())
129            .connect_timeout(cfg.connect_timeout())
130            .redirect(reqwest::redirect::Policy::custom({
131                let max = cfg.max_redirects;
132                move |attempt| {
133                    if attempt.previous().len() as u32 >= max {
134                        return attempt
135                            .error(AppError::new(ErrorKind::Network, "redirect limit exceeded"));
136                    }
137                    // Borrow host_str — no intermediate String on the redirect hot path.
138                    let host = attempt.url().host_str().unwrap_or("");
139                    let scheme_ok = attempt.url().scheme() == "https";
140                    let host_ok = matches!(
141                        host,
142                        HOST_CRATES_IO
143                            | HOST_DOCS_RS
144                            | HOST_STATIC_DOCS_RS
145                            | HOST_DOC_RUST_LANG_ORG
146                    ) || ((host == "127.0.0.1" || host == "localhost")
147                        && (cfg!(test) || std::env::var("DOCSRS_CLI_ALLOW_LOCALHOST").is_ok()));
148                    if !(scheme_ok && host_ok) {
149                        // Materialize the message before moving `attempt` (host is borrowed).
150                        let msg = format!("redirect host not allowlisted: {host}");
151                        return attempt.error(AppError::new(ErrorKind::Network, msg));
152                    }
153                    attempt.follow()
154                }
155            }))
156            .use_rustls_tls()
157            .build()
158            .map_err(|e| {
159                AppError::with_source(ErrorKind::Network, "failed to build HTTP client", e)
160            })?;
161
162        let cache = if cfg.cache_enabled() {
163            cfg.cache_dir
164                .as_ref()
165                .map(|dir| DiskCache::new(dir.clone(), cfg.cache_ttl(), cfg.max_cache_bytes))
166        } else {
167            None
168        };
169
170        Ok(Self {
171            client,
172            cfg,
173            last_host_hit: Mutex::new(HashMap::new()),
174            cancel,
175            cache,
176            budget,
177            retry,
178        })
179    }
180
181    /// Shared concurrency budget for CPU-bound parse work.
182    pub fn budget(&self) -> &ConcurrencyBudget {
183        &self.budget
184    }
185
186    /// Resolved retry policy (for doctor / tests).
187    pub fn retry_policy(&self) -> RetryConfig {
188        self.retry
189    }
190
191    /// GET expecting JSON (`Accept: application/json`).
192    ///
193    /// # Errors
194    ///
195    /// Returns [`ErrorKind::Network`] for allowlist, redirect, TLS, or body-cap failures.
196    /// Returns [`ErrorKind::Timeout`] when the wall-clock budget expires.
197    /// Returns [`ErrorKind::Interrupted`] or [`ErrorKind::Terminated`] on cancel
198    /// (kind preserved from the first `CancelFlag::cancel_with`).
199    /// Maps HTTP statuses via [`AppError::from_http_status`] (e.g. not-found, rate-limit).
200    pub async fn get_json(&self, url: &Url) -> AppResult<HttpResponse> {
201        self.request(Method::GET, url, "application/json", true)
202            .await
203    }
204
205    /// GET expecting HTML (`Accept: text/html`).
206    ///
207    /// # Errors
208    ///
209    /// Same error classes as [`Self::get_json`].
210    pub async fn get_html(&self, url: &Url) -> AppResult<HttpResponse> {
211        self.request(Method::GET, url, "text/html", true).await
212    }
213
214    async fn request(
215        &self,
216        method: Method,
217        url: &Url,
218        accept: &str,
219        retryable: bool,
220    ) -> AppResult<HttpResponse> {
221        if !is_allowed_host(url) {
222            return Err(AppError::new(
223                ErrorKind::Network,
224                format!("host not allowlisted: {}", url.host_str().unwrap_or("")),
225            ));
226        }
227
228        // Disk cache only for GET (product surface is GET-only).
229        if method == Method::GET
230            && let Some(cache) = &self.cache
231            && let Some(hit) = cache.get(url, accept)
232        {
233            return Ok(hit);
234        }
235
236        // Policy is opt-out kill switch + max_retries; `retryable` is for future
237        // non-idempotent methods (product is GET-only today).
238        let policy = if retryable {
239            self.retry
240        } else {
241            RetryConfig {
242                enabled: false,
243                ..self.retry
244            }
245        };
246        // Prefer `url.as_str()` over `url.clone()`: reqwest implements IntoUrl for
247        // `&str` but not for `&Url` (0.12). Method is a small enum — clone is trivial.
248        let url_str = url.as_str();
249        let mut attempt = 0u32;
250        loop {
251            self.cancel.check()?;
252            attempt += 1;
253            self.rate_limit(url).await?;
254            self.cancel.check()?;
255
256            let mut req = self.client.request(method.clone(), url_str);
257            req = req.header(ACCEPT, accept);
258
259            let result = req.send().await;
260            match result {
261                Ok(resp) => {
262                    let status = resp.status();
263                    let final_url = resp.url().clone();
264                    let content_type = resp
265                        .headers()
266                        .get(reqwest::header::CONTENT_TYPE)
267                        .and_then(|v| v.to_str().ok())
268                        .map(|s| s.to_string());
269                    let retry_after = parse_retry_after(resp.headers());
270                    let class = classify_http_status(status, retry_after);
271
272                    match class {
273                        RetryClass::RateLimited { retry_after } if policy.may_retry(attempt) => {
274                            let wait = wait_for_retry(policy, attempt, retry_after);
275                            debug!(
276                                target: "docsrs_cli::retry",
277                                ?wait,
278                                attempt,
279                                max_attempts = policy.max_attempts(),
280                                status = %status,
281                                "retrying after rate-limit / Retry-After"
282                            );
283                            self.sleep_cancelable(wait).await?;
284                            continue;
285                        }
286                        RetryClass::Transient if policy.may_retry(attempt) => {
287                            let wait = wait_for_retry(policy, attempt, None);
288                            debug!(
289                                target: "docsrs_cli::retry",
290                                ?wait,
291                                attempt,
292                                max_attempts = policy.max_attempts(),
293                                status = %status,
294                                "retrying after transient 5xx"
295                            );
296                            self.sleep_cancelable(wait).await?;
297                            continue;
298                        }
299                        RetryClass::RateLimited { retry_after } => {
300                            let secs = retry_after.map(|d| d.as_secs()).unwrap_or(1);
301                            return Err(AppError::from_http_status(
302                                status.as_u16(),
303                                "retries exhausted",
304                            )
305                            .with_retry_after(secs.max(1)));
306                        }
307                        RetryClass::Transient => {
308                            // Exhausted retries on 5xx: map status without body read.
309                            return Err(AppError::from_http_status(
310                                status.as_u16(),
311                                "retries exhausted",
312                            ));
313                        }
314                        RetryClass::Permanent => {
315                            // 2xx and permanent 4xx: read body and return (caller maps status).
316                        }
317                    }
318
319                    let body = read_body_capped(resp, self.cfg.max_body_bytes).await?;
320                    let out = HttpResponse {
321                        status,
322                        final_url,
323                        body,
324                        content_type,
325                        cache_hit: false,
326                    };
327                    if method == Method::GET
328                        && out.status.is_success()
329                        && let Some(cache) = &self.cache
330                        && let Err(e) = cache.put(url, accept, &out)
331                    {
332                        warn!(error = %e, "failed to write HTTP cache entry");
333                    }
334                    return Ok(out);
335                }
336                Err(e) => {
337                    // Classify transport layers without string-matching messages.
338                    let is_timeout = e.is_timeout();
339                    let is_connect = e.is_connect();
340                    let is_request = e.is_request();
341                    let kind = if is_timeout {
342                        ErrorKind::Timeout
343                    } else {
344                        ErrorKind::Network
345                    };
346                    let transport_retryable = is_timeout || is_connect || is_request;
347                    if policy.may_retry(attempt) && transport_retryable {
348                        let wait = wait_for_retry(policy, attempt, None);
349                        warn!(
350                            target: "docsrs_cli::retry",
351                            ?wait,
352                            attempt,
353                            max_attempts = policy.max_attempts(),
354                            error = %e,
355                            is_timeout,
356                            is_connect,
357                            "retrying after transport error"
358                        );
359                        self.sleep_cancelable(wait).await?;
360                        continue;
361                    }
362                    return Err(AppError::with_source(
363                        kind,
364                        format!("HTTP request failed for {url}"),
365                        e,
366                    ));
367                }
368            }
369        }
370    }
371
372    async fn sleep_cancelable(&self, wait: Duration) -> AppResult<()> {
373        let step = Duration::from_millis(50);
374        let mut remaining = wait;
375        while !remaining.is_zero() {
376            self.cancel.check()?;
377            let slice = remaining.min(step);
378            tokio::time::sleep(slice).await;
379            remaining = remaining.saturating_sub(slice);
380        }
381        self.cancel.check()
382    }
383
384    /// Enforce per-host delay: in-process clock + exclusive cross-process lock+stamp.
385    ///
386    /// Floor is `rate_limit_delay_ms`; each wait uses [`politeness_delay`] so the
387    /// effective interval is never fixed (additive jitter up to +20%).
388    ///
389    /// In-process map uses a short [`Mutex`] hold (no sleep under the lock). Cross-process
390    /// flock is released on drop before return (never held across unrelated work).
391    async fn rate_limit(&self, url: &Url) -> AppResult<()> {
392        // Lookup borrows host; allocate only when inserting/updating the clock map.
393        let host = url.host_str().unwrap_or("");
394        let delay = politeness_delay(self.cfg.rate_limit_delay());
395        if delay.is_zero() {
396            return Ok(());
397        }
398
399        // Prefer exclusive cross-process section when cache_dir is available.
400        if let Some(dir) = self.cfg.cache_dir.as_ref() {
401            match self.rate_limit_cross_process(dir, host, delay).await {
402                Ok(()) => {
403                    self.touch_host_clock(host);
404                    return Ok(());
405                }
406                Err(e)
407                    if e.kind() == ErrorKind::Terminated || e.kind() == ErrorKind::Interrupted =>
408                {
409                    return Err(e);
410                }
411                Err(e) => {
412                    // FS without flock or I/O failure: fall back to in-process only.
413                    warn!(error = %e, host = %host, "cross-process rate-limit lock failed; using in-process only");
414                }
415            }
416        }
417
418        let in_proc = {
419            let map = self.last_host_hit.lock().unwrap_or_else(|e| e.into_inner());
420            map.get(host).and_then(|prev| {
421                let elapsed = prev.elapsed();
422                if elapsed < delay {
423                    Some(delay - elapsed)
424                } else {
425                    None
426                }
427            })
428        };
429        if let Some(d) = in_proc {
430            self.sleep_cancelable(d).await?;
431        }
432        self.touch_host_clock(host);
433        Ok(())
434    }
435
436    fn touch_host_clock(&self, host: &str) {
437        let mut map = self.last_host_hit.lock().unwrap_or_else(|e| e.into_inner());
438        map.insert(host.to_string(), Instant::now());
439    }
440
441    /// Exclusive lock on `{cache_dir}/rate-limit/{host}.lock`, then stamp throttle.
442    async fn rate_limit_cross_process(
443        &self,
444        cache_dir: &Path,
445        host: &str,
446        delay: Duration,
447    ) -> AppResult<()> {
448        let lock_path = rate_limit_lock_path(cache_dir, host);
449        let stamp_path = rate_limit_stamp_path(cache_dir, host);
450        if let Some(parent) = lock_path.parent() {
451            fs::create_dir_all(parent).map_err(|e| {
452                AppError::with_source(ErrorKind::Network, "rate-limit lock dir create failed", e)
453            })?;
454            crate::platform::restrict_private_dir(parent);
455        }
456
457        let file = OpenOptions::new()
458            .create(true)
459            .read(true)
460            .write(true)
461            .truncate(false)
462            .open(&lock_path)
463            .map_err(|e| {
464                AppError::with_source(ErrorKind::Network, "rate-limit lock open failed", e)
465            })?;
466        crate::platform::restrict_private_file(&lock_path);
467
468        // Cancel-aware exclusive acquisition (fs4 try_lock + short sleep).
469        // Fully-qualified calls avoid collision with std::fs::File::try_lock (Rust 1.89+).
470        loop {
471            self.cancel.check()?;
472            match Fs4FileExt::try_lock(&file) {
473                Ok(()) => break,
474                Err(Fs4TryLockError::WouldBlock) => {
475                    self.sleep_cancelable(Duration::from_millis(50)).await?;
476                }
477                Err(Fs4TryLockError::Error(e)) => {
478                    return Err(AppError::with_source(
479                        ErrorKind::Network,
480                        "rate-limit exclusive lock failed",
481                        e,
482                    ));
483                }
484            }
485        }
486
487        // Guard unlock on all exit paths.
488        struct UnlockOnDrop<'a>(&'a File);
489        impl Drop for UnlockOnDrop<'_> {
490            fn drop(&mut self) {
491                let _ = Fs4FileExt::unlock(self.0);
492            }
493        }
494        let _guard = UnlockOnDrop(&file);
495
496        let remaining = stamp_remaining(&stamp_path, delay);
497        if let Some(d) = remaining {
498            self.sleep_cancelable(d).await?;
499        }
500        write_stamp(&stamp_path)?;
501        Ok(())
502    }
503}
504
505fn safe_host_name(host: &str) -> String {
506    let safe: String = host
507        .chars()
508        .map(|c| {
509            if c.is_ascii_alphanumeric() || c == '.' || c == '-' {
510                c
511            } else {
512                '_'
513            }
514        })
515        .collect();
516    if safe.is_empty() {
517        "unknown".to_string()
518    } else {
519        safe
520    }
521}
522
523/// Path for host rate-limit stamp: `{cache_dir}/rate-limit/{safe_host}.stamp`.
524fn rate_limit_stamp_path(cache_dir: &Path, host: &str) -> PathBuf {
525    cache_dir
526        .join("rate-limit")
527        .join(format!("{}.stamp", safe_host_name(host)))
528}
529
530/// Path for host rate-limit exclusive lock: `{cache_dir}/rate-limit/{safe_host}.lock`.
531fn rate_limit_lock_path(cache_dir: &Path, host: &str) -> PathBuf {
532    cache_dir
533        .join("rate-limit")
534        .join(format!("{}.lock", safe_host_name(host)))
535}
536
537fn now_unix_ms() -> Option<u128> {
538    SystemTime::now()
539        .duration_since(UNIX_EPOCH)
540        .ok()
541        .map(|d| d.as_millis())
542}
543
544/// Remaining sleep from stamp file, if any.
545fn stamp_remaining(path: &Path, delay: Duration) -> Option<Duration> {
546    let text = fs::read_to_string(path).ok()?;
547    let last_ms: u128 = text.trim().parse().ok()?;
548    let now = now_unix_ms()?;
549    if now < last_ms {
550        return Some(delay);
551    }
552    let elapsed = Duration::from_millis((now - last_ms) as u64);
553    if elapsed < delay {
554        Some(delay - elapsed)
555    } else {
556        None
557    }
558}
559
560/// Write stamp under exclusive lock (temp + rename).
561fn write_stamp(path: &Path) -> AppResult<()> {
562    if let Some(parent) = path.parent() {
563        fs::create_dir_all(parent).map_err(|e| {
564            AppError::with_source(ErrorKind::Network, "rate-limit stamp dir create failed", e)
565        })?;
566    }
567    let now = now_unix_ms()
568        .ok_or_else(|| AppError::new(ErrorKind::Internal, "system clock before UNIX epoch"))?;
569    let tmp = path.with_extension("stamp.tmp");
570    fs::write(&tmp, now.to_string()).map_err(|e| {
571        AppError::with_source(ErrorKind::Network, "rate-limit stamp write failed", e)
572    })?;
573    fs::rename(&tmp, path).map_err(|e| {
574        let _ = fs::remove_file(&tmp);
575        AppError::with_source(ErrorKind::Network, "rate-limit stamp rename failed", e)
576    })?;
577    crate::platform::restrict_private_file(path);
578    Ok(())
579}
580
581fn is_allowed_host(url: &Url) -> bool {
582    match url.host_str() {
583        Some(HOST_CRATES_IO)
584        | Some(HOST_DOCS_RS)
585        | Some(HOST_STATIC_DOCS_RS)
586        | Some(HOST_DOC_RUST_LANG_ORG) => url.scheme() == "https",
587        Some("127.0.0.1") | Some("localhost") => {
588            cfg!(test) || std::env::var("DOCSRS_CLI_ALLOW_LOCALHOST").is_ok()
589        }
590        _ => false,
591    }
592}
593
594/// Read a response body with a hard byte budget and fallible allocation.
595///
596/// Primary defense is [`HARD_MAX_BODY_BYTES`] (operators may only lower the cap).
597/// `try_reserve` / `try_reserve_exact` map allocation failure to
598/// [`ErrorKind::Network`] instead of aborting via `with_capacity` on hostile sizes.
599/// On Linux overcommit, the allocator may still report success and the OOM killer
600/// can fire later — the hard ceiling remains the main bound.
601async fn read_body_capped(resp: reqwest::Response, max_bytes: u64) -> AppResult<Bytes> {
602    // Never honor a budget above the product hard ceiling (defense in depth).
603    let max_bytes = max_bytes.min(HARD_MAX_BODY_BYTES);
604    // When Content-Length is known and already over budget, fail without buffering.
605    // (With gzip, length may be compressed size or absent; still a useful early guard.)
606    let content_length = resp.content_length();
607    if let Some(n) = content_length
608        && n > max_bytes
609    {
610        // Permanent local budget — not a transport failure; do not auto-retry.
611        return Err(AppError::new(
612            ErrorKind::Budget,
613            format!("response body exceeds max_body_bytes ({max_bytes})"),
614        ));
615    }
616    // Pre-size when Content-Length is present to avoid realloc churn on large docs pages.
617    // Prefer try_reserve_exact when length is known; never with_capacity on external size.
618    let capacity = content_length
619        .map(|n| (n as usize).min(max_bytes as usize))
620        .unwrap_or(0);
621    let mut stream = resp.bytes_stream();
622    let mut buf: Vec<u8> = Vec::new();
623    if capacity > 0 {
624        buf.try_reserve_exact(capacity).map_err(|e| {
625            AppError::with_source(
626                ErrorKind::Network,
627                format!("failed to reserve {capacity} bytes for response body"),
628                e,
629            )
630        })?;
631    }
632    while let Some(chunk) = stream.next().await {
633        let chunk = chunk.map_err(|e| {
634            AppError::with_source(ErrorKind::Network, "failed reading response body", e)
635        })?;
636        if (buf.len() as u64).saturating_add(chunk.len() as u64) > max_bytes {
637            // Permanent local budget — not a transport failure; do not auto-retry.
638            return Err(AppError::new(
639                ErrorKind::Budget,
640                format!("response body exceeds max_body_bytes ({max_bytes})"),
641            ));
642        }
643        // Grow with try_reserve when stream chunks exceed Content-Length estimate.
644        let need = chunk.len();
645        if buf.capacity().saturating_sub(buf.len()) < need {
646            buf.try_reserve(need).map_err(|e| {
647                AppError::with_source(
648                    ErrorKind::Network,
649                    format!("failed to reserve {need} more body bytes"),
650                    e,
651                )
652            })?;
653        }
654        buf.extend_from_slice(&chunk);
655    }
656    Ok(Bytes::from(buf))
657}
658
659/// Decode body as UTF-8, stripping a leading UTF-8 BOM.
660///
661/// # Errors
662///
663/// Returns [`ErrorKind::Parse`] when the body is not valid UTF-8.
664pub fn decode_utf8(body: &Bytes) -> AppResult<String> {
665    let mut bytes = body.as_ref();
666    if bytes.starts_with(&[0xEF, 0xBB, 0xBF]) {
667        bytes = &bytes[3..];
668    }
669    std::str::from_utf8(bytes)
670        .map(|s| s.to_string())
671        .map_err(|e| AppError::with_source(ErrorKind::Parse, "response body is not valid UTF-8", e))
672}
673
674/// ASCII case-insensitive substring check without heap allocation.
675///
676/// Used on Content-Type sniffing (hot path after every successful GET) so we
677/// never build a lowercased temporary `String` for a few-byte header.
678fn ascii_contains_ignore_case(haystack: &str, needle: &str) -> bool {
679    if needle.is_empty() {
680        return true;
681    }
682    let n = needle.as_bytes();
683    haystack
684        .as_bytes()
685        .windows(n.len())
686        .any(|w| w.eq_ignore_ascii_case(n))
687}
688
689/// True when Content-Type looks like JSON.
690pub fn content_type_looks_json(ct: Option<&str>) -> bool {
691    ct.is_some_and(|c| ascii_contains_ignore_case(c, "json"))
692}
693
694/// True when Content-Type looks like HTML/XHTML.
695pub fn content_type_looks_html(ct: Option<&str>) -> bool {
696    ct.is_some_and(|c| {
697        ascii_contains_ignore_case(c, "html") || ascii_contains_ignore_case(c, "xhtml")
698    })
699}
700
701#[cfg(test)]
702mod tests {
703    use super::*;
704
705    #[test]
706    fn allowlist_https_only() {
707        let ok = Url::parse("https://docs.rs/serde").unwrap();
708        assert!(is_allowed_host(&ok));
709        let stdlib = Url::parse("https://doc.rust-lang.org/stable/std/").unwrap();
710        assert!(is_allowed_host(&stdlib));
711        let bad = Url::parse("http://docs.rs/serde").unwrap();
712        assert!(!is_allowed_host(&bad));
713        let evil = Url::parse("https://evil.example/").unwrap();
714        assert!(!is_allowed_host(&evil));
715    }
716
717    #[test]
718    fn utf8_bom_strip() {
719        let mut v = vec![0xEF, 0xBB, 0xBF];
720        v.extend_from_slice(b"hello");
721        let s = decode_utf8(&Bytes::from(v)).unwrap();
722        assert_eq!(s, "hello");
723    }
724
725    #[test]
726    fn content_types() {
727        assert!(content_type_looks_json(Some(
728            "application/json; charset=utf-8"
729        )));
730        assert!(content_type_looks_json(Some("Application/JSON")));
731        assert!(content_type_looks_html(Some("text/html")));
732        assert!(content_type_looks_html(Some("Text/HTML; charset=utf-8")));
733        assert!(content_type_looks_html(Some("application/xhtml+xml")));
734        assert!(!content_type_looks_json(Some("text/html")));
735        assert!(!content_type_looks_json(None));
736        assert!(!content_type_looks_html(None));
737    }
738
739    #[test]
740    fn ascii_contains_ignore_case_edges() {
741        assert!(ascii_contains_ignore_case("abJSON", "json"));
742        assert!(ascii_contains_ignore_case("json", "json"));
743        assert!(!ascii_contains_ignore_case("js", "json"));
744        assert!(ascii_contains_ignore_case("x", ""));
745    }
746
747    #[test]
748    fn retry_policy_kill_switch() {
749        let cfg = Config {
750            disable_retry: true,
751            max_retries: 5,
752            ..Config::default()
753        };
754        let http = HttpClient::new(cfg, CancelFlag::new()).unwrap();
755        assert!(!http.retry_policy().enabled);
756        assert_eq!(http.retry_policy().max_attempts(), 1);
757    }
758
759    #[test]
760    fn invalid_user_agent_rejected() {
761        let cfg = Config {
762            user_agent: "bad\nua".into(),
763            ..Config::default()
764        };
765        match HttpClient::new(cfg, CancelFlag::new()) {
766            Ok(_) => panic!("expected invalid user-agent to fail"),
767            Err(err) => assert_eq!(err.kind(), ErrorKind::Config),
768        }
769    }
770
771    #[test]
772    fn localhost_allowed_in_unit_tests() {
773        let ok = Url::parse("http://127.0.0.1:9/x").unwrap();
774        assert!(is_allowed_host(&ok));
775        let local = Url::parse("http://localhost:9/x").unwrap();
776        assert!(is_allowed_host(&local));
777    }
778
779    #[test]
780    fn decode_utf8_rejects_invalid() {
781        let err = decode_utf8(&Bytes::from(vec![0xff, 0xfe])).unwrap_err();
782        assert_eq!(err.kind(), ErrorKind::Parse);
783    }
784}