Skip to main content

docsrs_cli/http/
client.rs

1//! HTTPS client: rustls, retry, rate limit, body cap, host allowlist, cancel-aware.
2
3use std::collections::HashMap;
4use std::sync::Mutex;
5use std::time::{Duration, Instant};
6
7use bytes::Bytes;
8use reqwest::header::{ACCEPT, ACCEPT_ENCODING, HeaderMap, HeaderValue, USER_AGENT};
9use reqwest::{Client, Method, StatusCode, Url};
10use tracing::{Instrument, debug, info_span, warn};
11
12use crate::cache::DiskCache;
13use crate::concurrency::ConcurrencyBudget;
14use crate::config::{Config, HARD_MAX_BODY_BYTES};
15use crate::error::{AllowlistStage, AppError, AppResult, ErrorDetail};
16use crate::retry::{
17    ErrorLayer, RetryClass, RetryConfig, classify_http_status, classify_reqwest_error,
18    parse_retry_after, wait_for_retry,
19};
20use crate::shutdown::CancelFlag;
21
22use super::allowlist::is_allowed_host;
23use super::body::read_body_capped;
24use super::constants::{
25    POOL_IDLE_TIMEOUT_SECS, POOL_MAX_IDLE_PER_HOST, TCP_KEEPALIVE_IDLE_SECS,
26    TCP_KEEPALIVE_INTERVAL_SECS, TCP_KEEPALIVE_RETRIES,
27};
28
29/// Install the ring CryptoProvider once when none is configured (ADR 0007).
30///
31/// Safe to call repeatedly. Does not overwrite an existing process default
32/// (binary `main` already installed it, or a prior call succeeded).
33///
34/// `ring` compiles C, which the product rule forbids in principle. Both
35/// pure-Rust replacements were measured and rejected in 2026-08 — see ADR 0007
36/// and GAP-TOOLCHAIN-001 — so the exception is recorded rather than silent.
37fn ensure_crypto_provider() {
38    use std::sync::Once;
39    static ONCE: Once = Once::new();
40    ONCE.call_once(|| {
41        if rustls::crypto::CryptoProvider::get_default().is_none() {
42            let _ = rustls::crypto::ring::default_provider().install_default();
43        }
44    });
45}
46
47/// HTTPS client bound to product allowlist and config.
48///
49/// Shareable across tasks (`&self`): rate-limit clock uses a short-lived
50/// [`Mutex`] around the map only; sleeps run without holding the lock.
51pub struct HttpClient {
52    client: Client,
53    // `pub(super)` for the fields the sibling `rate_limit` module drives; the
54    // throttle policy lives next to its stamp/lock I/O, not in the request loop.
55    pub(super) cfg: Config,
56    /// In-process last hit per host. Lock is never held across `.await`.
57    pub(super) last_host_hit: Mutex<HashMap<String, Instant>>,
58    pub(super) cancel: CancelFlag,
59    cache: Option<DiskCache>,
60    budget: ConcurrencyBudget,
61    retry: RetryConfig,
62}
63
64/// Successful HTTP response body and metadata.
65#[derive(Debug, Clone)]
66pub struct HttpResponse {
67    /// Final HTTP status after redirects.
68    pub status: StatusCode,
69    /// Final URL after redirects.
70    pub final_url: Url,
71    /// Response body bytes (capped).
72    pub body: Bytes,
73    /// Content-Type header when present.
74    pub content_type: Option<String>,
75    /// True when served from the local disk cache.
76    pub cache_hit: bool,
77}
78
79impl HttpClient {
80    /// Build a client with rustls, default headers, and redirect allowlist.
81    ///
82    /// Owned by one one-shot command. Call sites may `Config::clone` before passing
83    /// `cfg` when they still need the parent config after construction. That clone is
84    /// intentional and cheap relative to TLS/RTT/body I/O. [`crate::concurrency::ConcurrencyBudget`] is
85    /// built from `cfg.max_concurrency` (0 = auto).
86    ///
87    /// # Errors
88    ///
89    /// Returns [`crate::error::ErrorKind::Config`] when the User-Agent is not a valid header value.
90    /// Returns [`crate::error::ErrorKind::Network`] when the reqwest client cannot be built.
91    pub fn new(cfg: Config, cancel: CancelFlag) -> AppResult<Self> {
92        // reqwest 0.13 `rustls-no-provider`: a process CryptoProvider is required.
93        // Binary installs ring first in `main` (fail-closed dual-init). Library
94        // callers and tests reach this path without `main` — install once if missing.
95        ensure_crypto_provider();
96
97        // Defense in depth: clamp even if a caller skipped Config::clamp_resource_limits.
98        let mut cfg = cfg;
99        cfg.clamp_resource_limits();
100        let budget = ConcurrencyBudget::from_configured(cfg.max_concurrency);
101        let retry = RetryConfig::from_config(&cfg);
102
103        let mut headers = HeaderMap::new();
104        headers.insert(
105            USER_AGENT,
106            HeaderValue::from_str(&cfg.user_agent)
107                .map_err(|e| AppError::of_with_source(ErrorDetail::UserAgentHeaderInvalid, e))?,
108        );
109        // Explicit encodings: feature `brotli` alone does not add `br` when this
110        // header is set manually (SCRAPE-S-004).
111        headers.insert(ACCEPT_ENCODING, HeaderValue::from_static("gzip, br"));
112
113        let client = Client::builder()
114            .default_headers(headers)
115            .timeout(cfg.timeout())
116            .connect_timeout(cfg.connect_timeout())
117            .redirect(reqwest::redirect::Policy::custom({
118                let max = cfg.max_redirects;
119                let allow_loopback = cfg.allow_loopback;
120                move |attempt| {
121                    if attempt.previous().len() as u32 >= max {
122                        return attempt.error(AppError::of(ErrorDetail::RedirectLimitExceeded));
123                    }
124                    // Borrow host_str — no intermediate String on the redirect hot path.
125                    // Same allowlist as request gate + config origins (SSRF defense in depth).
126                    let host = attempt.url().host_str().unwrap_or("");
127                    let scheme = attempt.url().scheme();
128                    if !crate::config::is_allowed_origin_scheme_host(scheme, host, allow_loopback) {
129                        // Materialize the detail before moving `attempt` (host is borrowed).
130                        let detail = ErrorDetail::HostNotAllowlisted {
131                            host: host.to_string(),
132                            stage: AllowlistStage::Redirect,
133                        };
134                        return attempt.error(AppError::of(detail));
135                    }
136                    attempt.follow()
137                }
138            }))
139            // Preconfigured rather than `use_rustls_tls()`: the roots are ours
140            // (Mozilla set, compiled in) instead of whatever reqwest would wire
141            // up, which since 0.13 is the OS store. See `super::tls`.
142            //
143            // The TLS 1.2 floor moves into that config too — `min_tls_version`
144            // is ignored once a config is supplied, so leaving it here would be
145            // a setting that reads as applied and enforces nothing.
146            //
147            // Not `https_only(true)`: offline wiremock uses `http://127.0.0.1` when
148            // `allow_loopback` is set (CLI/XDG); production hosts still require HTTPS.
149            .use_preconfigured_tls(super::tls::client_config()?)
150            // Low-latency request/response docs GETs (disable Nagle).
151            .tcp_nodelay(true)
152            .tcp_keepalive(Duration::from_secs(TCP_KEEPALIVE_IDLE_SECS))
153            .tcp_keepalive_interval(Duration::from_secs(TCP_KEEPALIVE_INTERVAL_SECS))
154            .tcp_keepalive_retries(TCP_KEEPALIVE_RETRIES)
155            // One-shot pool: tiny idle set, short idle TTL (process dies soon).
156            .pool_max_idle_per_host(POOL_MAX_IDLE_PER_HOST)
157            .pool_idle_timeout(Duration::from_secs(POOL_IDLE_TIMEOUT_SECS))
158            // HTTP/2 adaptive flow control when ALPN selects h2.
159            .http2_adaptive_window(true)
160            // system-proxy feature: honor HTTP(S)_PROXY / ALL_PROXY / NO_PROXY.
161            .build()
162            .map_err(|e| AppError::of_with_source(ErrorDetail::HttpClientBuild, e))?;
163
164        let cache = if cfg.cache_enabled() {
165            cfg.cache_dir.as_ref().map(|dir| {
166                DiskCache::new(
167                    dir.clone(),
168                    cfg.cache_ttl(),
169                    cfg.max_cache_bytes,
170                    cfg.allow_loopback,
171                )
172            })
173        } else {
174            None
175        };
176
177        Ok(Self {
178            client,
179            cfg,
180            last_host_hit: Mutex::new(HashMap::new()),
181            cancel,
182            cache,
183            budget,
184            retry,
185        })
186    }
187
188    /// Shared cancel flag for cooperative shutdown (HTTP + CPU scrape).
189    pub fn cancel_flag(&self) -> &crate::shutdown::CancelFlag {
190        &self.cancel
191    }
192
193    /// Shared concurrency budget for CPU-bound parse work.
194    pub fn budget(&self) -> &ConcurrencyBudget {
195        &self.budget
196    }
197
198    /// Resolved retry policy (for doctor / tests).
199    pub fn retry_policy(&self) -> RetryConfig {
200        self.retry
201    }
202
203    /// GET expecting JSON (`Accept: application/json`).
204    ///
205    /// # Errors
206    ///
207    /// Returns [`crate::error::ErrorKind::Network`] for allowlist, redirect, TLS, or body-cap failures.
208    /// Returns [`crate::error::ErrorKind::Timeout`] when the wall-clock budget expires.
209    /// Returns [`crate::error::ErrorKind::Interrupted`] or [`crate::error::ErrorKind::Terminated`] on cancel
210    /// (kind preserved from the first `CancelFlag::cancel_with`).
211    /// Maps HTTP statuses via [`AppError::from_http_status`] (e.g. not-found, rate-limit).
212    pub async fn get_json(&self, url: &Url) -> AppResult<HttpResponse> {
213        self.request(Method::GET, url, "application/json", true)
214            .await
215    }
216
217    /// GET expecting HTML (`Accept: text/html`).
218    ///
219    /// # Errors
220    ///
221    /// Same error classes as [`Self::get_json`].
222    pub async fn get_html(&self, url: &Url) -> AppResult<HttpResponse> {
223        self.request(Method::GET, url, "text/html", true).await
224    }
225
226    async fn request(
227        &self,
228        method: Method,
229        url: &Url,
230        accept: &str,
231        retryable: bool,
232    ) -> AppResult<HttpResponse> {
233        if !is_allowed_host(url, self.cfg.allow_loopback) {
234            return Err(AppError::of(ErrorDetail::HostNotAllowlisted {
235                host: url.host_str().unwrap_or("").to_string(),
236                stage: AllowlistStage::Request,
237            }));
238        }
239
240        // Disk cache only for GET (product surface is GET-only).
241        // Invariant: every HttpResponse respects max_body_bytes (network *and* cache).
242        if method == Method::GET
243            && let Some(cache) = &self.cache
244            && let Some(hit) = cache.get(url, accept)
245        {
246            let cap = self.cfg.max_body_bytes.min(HARD_MAX_BODY_BYTES);
247            if (hit.body.len() as u64) > cap {
248                return Err(AppError::of(ErrorDetail::CachedBodyOverBudget {
249                    max_bytes: cap,
250                }));
251            }
252            return Ok(hit);
253        }
254
255        // Policy is opt-out kill switch + max_retries; `retryable` is for future
256        // non-idempotent methods (product is GET-only today).
257        let policy = if retryable {
258            self.retry
259        } else {
260            RetryConfig {
261                enabled: false,
262                ..self.retry
263            }
264        };
265        // Prefer `url.as_str()` over `url.clone()`: reqwest implements IntoUrl for
266        // `&str` but not for `&Url` (0.12). Method is a small enum — clone is trivial.
267        let url_str = url.as_str();
268        // Monotonic start for max_elapsed_ms budget (Rules: Instant, not SystemTime).
269        let started = Instant::now();
270        let mut attempt = 0u32;
271        loop {
272            self.cancel.check()?;
273            attempt += 1;
274            self.rate_limit(url).await?;
275            self.cancel.check()?;
276
277            let mut req = self.client.request(method.clone(), url_str);
278            req = req.header(ACCEPT, accept);
279
280            let result = req.send().await;
281            match result {
282                Ok(resp) => {
283                    let status = resp.status();
284                    let final_url = resp.url().clone();
285                    let content_type = resp
286                        .headers()
287                        .get(reqwest::header::CONTENT_TYPE)
288                        .and_then(|v| v.to_str().ok())
289                        .map(|s| s.to_string());
290                    let retry_after = parse_retry_after(resp.headers());
291                    let class = classify_http_status(status, retry_after);
292
293                    match class {
294                        RetryClass::RateLimited { retry_after } => {
295                            let wait = wait_for_retry(policy, attempt, retry_after);
296                            if policy.may_retry_within_budget(attempt, started.elapsed(), wait) {
297                                let span = info_span!(
298                                    target: "docsrs_cli::retry",
299                                    "retry_attempt",
300                                    attempt,
301                                    max_attempts = policy.max_attempts(),
302                                    status = %status,
303                                    layer = ErrorLayer::HttpStatus.as_str(),
304                                    wait_ms = wait.as_millis() as u64,
305                                    reason = "rate_limited",
306                                );
307                                async {
308                                    debug!(
309                                        target: "docsrs_cli::retry",
310                                        ?wait,
311                                        attempt,
312                                        max_attempts = policy.max_attempts(),
313                                        status = %status,
314                                        "retrying after rate-limit / Retry-After"
315                                    );
316                                    self.sleep_cancelable(wait).await
317                                }
318                                .instrument(span)
319                                .await?;
320                                continue;
321                            }
322                            let secs = retry_after.map(|d| d.as_secs()).unwrap_or(1);
323                            return Err(AppError::from_http_status(
324                                status.as_u16(),
325                                "retries exhausted",
326                            )
327                            .with_retry_after(secs.max(1)));
328                        }
329                        RetryClass::Transient => {
330                            let wait = wait_for_retry(policy, attempt, None);
331                            if policy.may_retry_within_budget(attempt, started.elapsed(), wait) {
332                                let span = info_span!(
333                                    target: "docsrs_cli::retry",
334                                    "retry_attempt",
335                                    attempt,
336                                    max_attempts = policy.max_attempts(),
337                                    status = %status,
338                                    layer = ErrorLayer::HttpStatus.as_str(),
339                                    wait_ms = wait.as_millis() as u64,
340                                    reason = "transient_status",
341                                );
342                                async {
343                                    debug!(
344                                        target: "docsrs_cli::retry",
345                                        ?wait,
346                                        attempt,
347                                        max_attempts = policy.max_attempts(),
348                                        status = %status,
349                                        "retrying after transient status"
350                                    );
351                                    self.sleep_cancelable(wait).await
352                                }
353                                .instrument(span)
354                                .await?;
355                                continue;
356                            }
357                            // Exhausted attempts or elapsed budget on 5xx/408.
358                            return Err(AppError::from_http_status(
359                                status.as_u16(),
360                                "retries exhausted",
361                            ));
362                        }
363                        RetryClass::Permanent => {
364                            // 2xx and permanent 4xx: read body and return (caller maps status).
365                        }
366                    }
367
368                    // Defense in depth: redirect policy already gates hops; re-check
369                    // final URL so a policy regression cannot leak off-allowlist bodies.
370                    if !is_allowed_host(&final_url, self.cfg.allow_loopback) {
371                        return Err(AppError::of(ErrorDetail::HostNotAllowlisted {
372                            host: final_url.host_str().unwrap_or("").to_string(),
373                            stage: AllowlistStage::FinalUrl,
374                        }));
375                    }
376                    let body = read_body_capped(resp, self.cfg.max_body_bytes).await?;
377                    let out = HttpResponse {
378                        status,
379                        final_url,
380                        body,
381                        content_type,
382                        cache_hit: false,
383                    };
384                    if method == Method::GET
385                        && out.status.is_success()
386                        && let Some(cache) = &self.cache
387                        && let Err(e) = cache.put(url, accept, &out)
388                    {
389                        warn!(error = %e, "failed to write HTTP cache entry");
390                    }
391                    return Ok(out);
392                }
393                Err(e) => {
394                    // Classify transport layers without string-matching messages.
395                    let (kind, layer, transport_retryable) = classify_reqwest_error(&e);
396                    if transport_retryable {
397                        let wait = wait_for_retry(policy, attempt, None);
398                        if policy.may_retry_within_budget(attempt, started.elapsed(), wait) {
399                            let span = info_span!(
400                                target: "docsrs_cli::retry",
401                                "retry_attempt",
402                                attempt,
403                                max_attempts = policy.max_attempts(),
404                                layer = layer.as_str(),
405                                wait_ms = wait.as_millis() as u64,
406                                reason = "transport",
407                            );
408                            async {
409                                warn!(
410                                    target: "docsrs_cli::retry",
411                                    ?wait,
412                                    attempt,
413                                    max_attempts = policy.max_attempts(),
414                                    error = %e,
415                                    layer = layer.as_str(),
416                                    "retrying after transport error"
417                                );
418                                self.sleep_cancelable(wait).await
419                            }
420                            .instrument(span)
421                            .await?;
422                            continue;
423                        }
424                    }
425                    return Err(AppError::of_with_source(
426                        ErrorDetail::HttpRequestFailed {
427                            url: url.to_string(),
428                            kind,
429                        },
430                        e,
431                    ));
432                }
433            }
434        }
435    }
436}