Skip to main content

docsrs_cli/http/
client.rs

1//! HTTPS client: rustls, retry, rate limit, body cap, host allowlist, cancel-aware.
2
3use std::collections::HashMap;
4use std::fs::{self, File, OpenOptions};
5use std::path::Path;
6use std::sync::Mutex;
7use std::time::{Duration, Instant};
8
9use bytes::Bytes;
10use fs4::FileExt as Fs4FileExt;
11use fs4::TryLockError as Fs4TryLockError;
12use reqwest::header::{ACCEPT, ACCEPT_ENCODING, HeaderMap, HeaderValue, USER_AGENT};
13use reqwest::{Client, Method, StatusCode, Url};
14use tracing::{debug, info_span, warn, Instrument};
15
16use crate::cache::DiskCache;
17use crate::concurrency::ConcurrencyBudget;
18use crate::config::{CANCEL_POLL_INTERVAL_MS, Config, HARD_MAX_BODY_BYTES};
19use crate::error::{AppError, AppResult, ErrorKind};
20use crate::retry::{
21    ErrorLayer, RetryClass, RetryConfig, classify_http_status, classify_reqwest_error,
22    parse_retry_after, politeness_delay, wait_for_retry,
23};
24use crate::shutdown::CancelFlag;
25
26use super::allowlist::is_allowed_host;
27use super::body::read_body_capped;
28use super::constants::{
29    POOL_IDLE_TIMEOUT_SECS, POOL_MAX_IDLE_PER_HOST, TCP_KEEPALIVE_IDLE_SECS,
30    TCP_KEEPALIVE_INTERVAL_SECS, TCP_KEEPALIVE_RETRIES,
31};
32use super::rate_limit::{
33    rate_limit_lock_path, rate_limit_stamp_path, stamp_remaining, write_stamp,
34};
35
36/// Install ring CryptoProvider once when none is configured (ADR 0007).
37///
38/// Safe to call repeatedly. Does not overwrite an existing process default
39/// (binary `main` already installed ring, or a prior call succeeded).
40fn ensure_ring_crypto_provider() {
41    use std::sync::Once;
42    static ONCE: Once = Once::new();
43    ONCE.call_once(|| {
44        if rustls::crypto::CryptoProvider::get_default().is_none() {
45            let _ = rustls::crypto::ring::default_provider().install_default();
46        }
47    });
48}
49
50/// HTTPS client bound to product allowlist and config.
51///
52/// Shareable across tasks (`&self`): rate-limit clock uses a short-lived
53/// [`Mutex`] around the map only; sleeps run without holding the lock.
54pub struct HttpClient {
55    client: Client,
56    cfg: Config,
57    /// In-process last hit per host. Lock is never held across `.await`.
58    last_host_hit: Mutex<HashMap<String, Instant>>,
59    cancel: CancelFlag,
60    cache: Option<DiskCache>,
61    budget: ConcurrencyBudget,
62    retry: RetryConfig,
63}
64
65/// Successful HTTP response body and metadata.
66#[derive(Debug, Clone)]
67pub struct HttpResponse {
68    /// Final HTTP status after redirects.
69    pub status: StatusCode,
70    /// Final URL after redirects.
71    pub final_url: Url,
72    /// Response body bytes (capped).
73    pub body: Bytes,
74    /// Content-Type header when present.
75    pub content_type: Option<String>,
76    /// True when served from the local disk cache.
77    pub cache_hit: bool,
78}
79
80impl HttpClient {
81    /// Build a client with rustls, default headers, and redirect allowlist.
82    ///
83    /// Owned by one one-shot command. Call sites may `Config::clone` before passing
84    /// `cfg` when they still need the parent config after construction. That clone is
85    /// intentional and cheap relative to TLS/RTT/body I/O. [`ConcurrencyBudget`] is
86    /// built from `cfg.max_concurrency` (0 = auto).
87    ///
88    /// # Errors
89    ///
90    /// Returns [`ErrorKind::Config`] when the User-Agent is not a valid header value.
91    /// Returns [`ErrorKind::Network`] when the reqwest client cannot be built.
92    pub fn new(cfg: Config, cancel: CancelFlag) -> AppResult<Self> {
93        // reqwest 0.13 `rustls-no-provider`: a process CryptoProvider is required.
94        // Binary installs ring first in `main` (fail-closed dual-init). Library
95        // callers and tests reach this path without `main` — install once if missing.
96        ensure_ring_crypto_provider();
97
98        // Defense in depth: clamp even if a caller skipped Config::clamp_resource_limits.
99        let mut cfg = cfg;
100        cfg.clamp_resource_limits();
101        let budget = ConcurrencyBudget::from_configured(cfg.max_concurrency);
102        let retry = RetryConfig::from_config(&cfg);
103
104        let mut headers = HeaderMap::new();
105        headers.insert(
106            USER_AGENT,
107            HeaderValue::from_str(&cfg.user_agent).map_err(|e| {
108                AppError::with_source(ErrorKind::Config, "invalid user-agent header", e)
109            })?,
110        );
111        // Explicit encodings: feature `brotli` alone does not add `br` when this
112        // header is set manually (SCRAPE-S-004).
113        headers.insert(ACCEPT_ENCODING, HeaderValue::from_static("gzip, br"));
114
115        let client = Client::builder()
116            .default_headers(headers)
117            .timeout(cfg.timeout())
118            .connect_timeout(cfg.connect_timeout())
119            .redirect(reqwest::redirect::Policy::custom({
120                let max = cfg.max_redirects;
121                let allow_loopback = cfg.allow_loopback;
122                move |attempt| {
123                    if attempt.previous().len() as u32 >= max {
124                        return attempt
125                            .error(AppError::new(ErrorKind::Network, "redirect limit exceeded"));
126                    }
127                    // Borrow host_str — no intermediate String on the redirect hot path.
128                    // Same allowlist as request gate + config origins (SSRF defense in depth).
129                    let host = attempt.url().host_str().unwrap_or("");
130                    let scheme = attempt.url().scheme();
131                    if !crate::config::is_allowed_origin_scheme_host(scheme, host, allow_loopback)
132                    {
133                        // Materialize the message before moving `attempt` (host is borrowed).
134                        let msg = format!("redirect host not allowlisted: {host}");
135                        return attempt.error(AppError::new(ErrorKind::Network, msg));
136                    }
137                    attempt.follow()
138                }
139            }))
140            .use_rustls_tls()
141            // TLS 1.2 floor (Rules Rust — TLS). Origins negotiate 1.3 when available.
142            // Not `https_only(true)`: offline wiremock uses `http://127.0.0.1` when
143            // `allow_loopback` is set (CLI/XDG); production hosts still require HTTPS.
144            .min_tls_version(reqwest::tls::Version::TLS_1_2)
145            // Low-latency request/response docs GETs (disable Nagle).
146            .tcp_nodelay(true)
147            .tcp_keepalive(Duration::from_secs(TCP_KEEPALIVE_IDLE_SECS))
148            .tcp_keepalive_interval(Duration::from_secs(TCP_KEEPALIVE_INTERVAL_SECS))
149            .tcp_keepalive_retries(TCP_KEEPALIVE_RETRIES)
150            // One-shot pool: tiny idle set, short idle TTL (process dies soon).
151            .pool_max_idle_per_host(POOL_MAX_IDLE_PER_HOST)
152            .pool_idle_timeout(Duration::from_secs(POOL_IDLE_TIMEOUT_SECS))
153            // HTTP/2 adaptive flow control when ALPN selects h2.
154            .http2_adaptive_window(true)
155            // system-proxy feature: honor HTTP(S)_PROXY / ALL_PROXY / NO_PROXY.
156            .build()
157            .map_err(|e| {
158                AppError::with_source(ErrorKind::Network, "failed to build HTTP client", e)
159            })?;
160
161        let cache = if cfg.cache_enabled() {
162            cfg.cache_dir.as_ref().map(|dir| {
163                DiskCache::new(
164                    dir.clone(),
165                    cfg.cache_ttl(),
166                    cfg.max_cache_bytes,
167                    cfg.allow_loopback,
168                )
169            })
170        } else {
171            None
172        };
173
174        Ok(Self {
175            client,
176            cfg,
177            last_host_hit: Mutex::new(HashMap::new()),
178            cancel,
179            cache,
180            budget,
181            retry,
182        })
183    }
184
185    /// Shared cancel flag for cooperative shutdown (HTTP + CPU scrape).
186    pub fn cancel_flag(&self) -> &crate::shutdown::CancelFlag {
187        &self.cancel
188    }
189
190    /// Shared concurrency budget for CPU-bound parse work.
191    pub fn budget(&self) -> &ConcurrencyBudget {
192        &self.budget
193    }
194
195    /// Resolved retry policy (for doctor / tests).
196    pub fn retry_policy(&self) -> RetryConfig {
197        self.retry
198    }
199
200    /// GET expecting JSON (`Accept: application/json`).
201    ///
202    /// # Errors
203    ///
204    /// Returns [`ErrorKind::Network`] for allowlist, redirect, TLS, or body-cap failures.
205    /// Returns [`ErrorKind::Timeout`] when the wall-clock budget expires.
206    /// Returns [`ErrorKind::Interrupted`] or [`ErrorKind::Terminated`] on cancel
207    /// (kind preserved from the first `CancelFlag::cancel_with`).
208    /// Maps HTTP statuses via [`AppError::from_http_status`] (e.g. not-found, rate-limit).
209    pub async fn get_json(&self, url: &Url) -> AppResult<HttpResponse> {
210        self.request(Method::GET, url, "application/json", true)
211            .await
212    }
213
214    /// GET expecting HTML (`Accept: text/html`).
215    ///
216    /// # Errors
217    ///
218    /// Same error classes as [`Self::get_json`].
219    pub async fn get_html(&self, url: &Url) -> AppResult<HttpResponse> {
220        self.request(Method::GET, url, "text/html", true).await
221    }
222
223    async fn request(
224        &self,
225        method: Method,
226        url: &Url,
227        accept: &str,
228        retryable: bool,
229    ) -> AppResult<HttpResponse> {
230        if !is_allowed_host(url, self.cfg.allow_loopback) {
231            return Err(AppError::new(
232                ErrorKind::Network,
233                format!("host not allowlisted: {}", url.host_str().unwrap_or("")),
234            ));
235        }
236
237        // Disk cache only for GET (product surface is GET-only).
238        // Invariant: every HttpResponse respects max_body_bytes (network *and* cache).
239        if method == Method::GET
240            && let Some(cache) = &self.cache
241            && let Some(hit) = cache.get(url, accept)
242        {
243            let cap = self.cfg.max_body_bytes.min(HARD_MAX_BODY_BYTES);
244            if (hit.body.len() as u64) > cap {
245                return Err(AppError::new(
246                    ErrorKind::Budget,
247                    format!("cached response body exceeds max_body_bytes ({cap})"),
248                ));
249            }
250            return Ok(hit);
251        }
252
253        // Policy is opt-out kill switch + max_retries; `retryable` is for future
254        // non-idempotent methods (product is GET-only today).
255        let policy = if retryable {
256            self.retry
257        } else {
258            RetryConfig {
259                enabled: false,
260                ..self.retry
261            }
262        };
263        // Prefer `url.as_str()` over `url.clone()`: reqwest implements IntoUrl for
264        // `&str` but not for `&Url` (0.12). Method is a small enum — clone is trivial.
265        let url_str = url.as_str();
266        // Monotonic start for max_elapsed_ms budget (Rules: Instant, not SystemTime).
267        let started = Instant::now();
268        let mut attempt = 0u32;
269        loop {
270            self.cancel.check()?;
271            attempt += 1;
272            self.rate_limit(url).await?;
273            self.cancel.check()?;
274
275            let mut req = self.client.request(method.clone(), url_str);
276            req = req.header(ACCEPT, accept);
277
278            let result = req.send().await;
279            match result {
280                Ok(resp) => {
281                    let status = resp.status();
282                    let final_url = resp.url().clone();
283                    let content_type = resp
284                        .headers()
285                        .get(reqwest::header::CONTENT_TYPE)
286                        .and_then(|v| v.to_str().ok())
287                        .map(|s| s.to_string());
288                    let retry_after = parse_retry_after(resp.headers());
289                    let class = classify_http_status(status, retry_after);
290
291                    match class {
292                        RetryClass::RateLimited { retry_after } => {
293                            let wait = wait_for_retry(policy, attempt, retry_after);
294                            if policy.may_retry_within_budget(attempt, started.elapsed(), wait) {
295                                let span = info_span!(
296                                    target: "docsrs_cli::retry",
297                                    "retry_attempt",
298                                    attempt,
299                                    max_attempts = policy.max_attempts(),
300                                    status = %status,
301                                    layer = ErrorLayer::HttpStatus.as_str(),
302                                    wait_ms = wait.as_millis() as u64,
303                                    reason = "rate_limited",
304                                );
305                                async {
306                                    debug!(
307                                        target: "docsrs_cli::retry",
308                                        ?wait,
309                                        attempt,
310                                        max_attempts = policy.max_attempts(),
311                                        status = %status,
312                                        "retrying after rate-limit / Retry-After"
313                                    );
314                                    self.sleep_cancelable(wait).await
315                                }
316                                .instrument(span)
317                                .await?;
318                                continue;
319                            }
320                            let secs = retry_after.map(|d| d.as_secs()).unwrap_or(1);
321                            return Err(AppError::from_http_status(
322                                status.as_u16(),
323                                "retries exhausted",
324                            )
325                            .with_retry_after(secs.max(1)));
326                        }
327                        RetryClass::Transient => {
328                            let wait = wait_for_retry(policy, attempt, None);
329                            if policy.may_retry_within_budget(attempt, started.elapsed(), wait) {
330                                let span = info_span!(
331                                    target: "docsrs_cli::retry",
332                                    "retry_attempt",
333                                    attempt,
334                                    max_attempts = policy.max_attempts(),
335                                    status = %status,
336                                    layer = ErrorLayer::HttpStatus.as_str(),
337                                    wait_ms = wait.as_millis() as u64,
338                                    reason = "transient_status",
339                                );
340                                async {
341                                    debug!(
342                                        target: "docsrs_cli::retry",
343                                        ?wait,
344                                        attempt,
345                                        max_attempts = policy.max_attempts(),
346                                        status = %status,
347                                        "retrying after transient status"
348                                    );
349                                    self.sleep_cancelable(wait).await
350                                }
351                                .instrument(span)
352                                .await?;
353                                continue;
354                            }
355                            // Exhausted attempts or elapsed budget on 5xx/408.
356                            return Err(AppError::from_http_status(
357                                status.as_u16(),
358                                "retries exhausted",
359                            ));
360                        }
361                        RetryClass::Permanent => {
362                            // 2xx and permanent 4xx: read body and return (caller maps status).
363                        }
364                    }
365
366                    // Defense in depth: redirect policy already gates hops; re-check
367                    // final URL so a policy regression cannot leak off-allowlist bodies.
368                    if !is_allowed_host(&final_url, self.cfg.allow_loopback) {
369                        return Err(AppError::new(
370                            ErrorKind::Network,
371                            format!(
372                                "final URL host not allowlisted: {}",
373                                final_url.host_str().unwrap_or("")
374                            ),
375                        ));
376                    }
377                    let body = read_body_capped(resp, self.cfg.max_body_bytes).await?;
378                    let out = HttpResponse {
379                        status,
380                        final_url,
381                        body,
382                        content_type,
383                        cache_hit: false,
384                    };
385                    if method == Method::GET
386                        && out.status.is_success()
387                        && let Some(cache) = &self.cache
388                        && let Err(e) = cache.put(url, accept, &out)
389                    {
390                        warn!(error = %e, "failed to write HTTP cache entry");
391                    }
392                    return Ok(out);
393                }
394                Err(e) => {
395                    // Classify transport layers without string-matching messages.
396                    let (kind, layer, transport_retryable) = classify_reqwest_error(&e);
397                    if transport_retryable {
398                        let wait = wait_for_retry(policy, attempt, None);
399                        if policy.may_retry_within_budget(attempt, started.elapsed(), wait) {
400                            let span = info_span!(
401                                target: "docsrs_cli::retry",
402                                "retry_attempt",
403                                attempt,
404                                max_attempts = policy.max_attempts(),
405                                layer = layer.as_str(),
406                                wait_ms = wait.as_millis() as u64,
407                                reason = "transport",
408                            );
409                            async {
410                                warn!(
411                                    target: "docsrs_cli::retry",
412                                    ?wait,
413                                    attempt,
414                                    max_attempts = policy.max_attempts(),
415                                    error = %e,
416                                    layer = layer.as_str(),
417                                    "retrying after transport error"
418                                );
419                                self.sleep_cancelable(wait).await
420                            }
421                            .instrument(span)
422                            .await?;
423                            continue;
424                        }
425                    }
426                    return Err(AppError::with_source(
427                        kind,
428                        format!("http request failed for {url}"),
429                        e,
430                    ));
431                }
432            }
433        }
434    }
435
436    async fn sleep_cancelable(&self, wait: Duration) -> AppResult<()> {
437        let step = Duration::from_millis(CANCEL_POLL_INTERVAL_MS);
438        let mut remaining = wait;
439        while !remaining.is_zero() {
440            self.cancel.check()?;
441            let slice = remaining.min(step);
442            tokio::time::sleep(slice).await;
443            remaining = remaining.saturating_sub(slice);
444        }
445        self.cancel.check()
446    }
447
448    /// Enforce per-host delay: in-process clock + exclusive cross-process lock+stamp.
449    ///
450    /// Floor is `rate_limit_delay_ms`; each wait uses [`politeness_delay`] so the
451    /// effective interval is never fixed (additive jitter up to +20%).
452    ///
453    /// In-process map uses a short [`Mutex`] hold (no sleep under the lock). Cross-process
454    /// flock is released on drop before return (never held across unrelated work).
455    async fn rate_limit(&self, url: &Url) -> AppResult<()> {
456        // Lookup borrows host; allocate only when inserting/updating the clock map.
457        let host = url.host_str().unwrap_or("");
458        let delay = politeness_delay(self.cfg.rate_limit_delay());
459        if delay.is_zero() {
460            return Ok(());
461        }
462
463        // Prefer exclusive cross-process section when cache_dir is available.
464        if let Some(dir) = self.cfg.cache_dir.as_ref() {
465            match self.rate_limit_cross_process(dir, host, delay).await {
466                Ok(()) => {
467                    self.touch_host_clock(host);
468                    return Ok(());
469                }
470                Err(e)
471                    if e.kind() == ErrorKind::Terminated || e.kind() == ErrorKind::Interrupted =>
472                {
473                    return Err(e);
474                }
475                Err(e) => {
476                    // FS without flock or I/O failure: fall back to in-process only.
477                    warn!(error = %e, host = %host, "cross-process rate-limit lock failed; using in-process only");
478                }
479            }
480        }
481
482        let in_proc = {
483            let map = self.last_host_hit.lock().unwrap_or_else(|e| e.into_inner());
484            map.get(host).and_then(|prev| {
485                let elapsed = prev.elapsed();
486                if elapsed < delay {
487                    Some(delay - elapsed)
488                } else {
489                    None
490                }
491            })
492        };
493        if let Some(d) = in_proc {
494            self.sleep_cancelable(d).await?;
495        }
496        self.touch_host_clock(host);
497        Ok(())
498    }
499
500    fn touch_host_clock(&self, host: &str) {
501        let mut map = self.last_host_hit.lock().unwrap_or_else(|e| e.into_inner());
502        map.insert(host.to_string(), Instant::now());
503    }
504
505    /// Exclusive lock on `{cache_dir}/rate-limit/{host}.lock`, then stamp throttle.
506    async fn rate_limit_cross_process(
507        &self,
508        cache_dir: &Path,
509        host: &str,
510        delay: Duration,
511    ) -> AppResult<()> {
512        let lock_path = rate_limit_lock_path(cache_dir, host);
513        let stamp_path = rate_limit_stamp_path(cache_dir, host);
514        if let Some(parent) = lock_path.parent() {
515            fs::create_dir_all(parent).map_err(|e| {
516                AppError::with_source(ErrorKind::Network, "rate-limit lock dir create failed", e)
517            })?;
518            crate::platform::restrict_private_dir(parent);
519        }
520
521        let file = OpenOptions::new()
522            .create(true)
523            .read(true)
524            .write(true)
525            .truncate(false)
526            .open(&lock_path)
527            .map_err(|e| {
528                AppError::with_source(ErrorKind::Network, "rate-limit lock open failed", e)
529            })?;
530        crate::platform::restrict_private_file(&lock_path);
531
532        // Cancel-aware exclusive acquisition (fs4 try_lock + short sleep).
533        // Fully-qualified calls avoid collision with std::fs::File::try_lock (Rust 1.89+).
534        loop {
535            self.cancel.check()?;
536            match Fs4FileExt::try_lock(&file) {
537                Ok(()) => break,
538                Err(Fs4TryLockError::WouldBlock) => {
539                    self.sleep_cancelable(Duration::from_millis(CANCEL_POLL_INTERVAL_MS))
540                        .await?;
541                }
542                Err(Fs4TryLockError::Error(e)) => {
543                    return Err(AppError::with_source(
544                        ErrorKind::Network,
545                        "rate-limit exclusive lock failed",
546                        e,
547                    ));
548                }
549            }
550        }
551
552        // Guard unlock on all exit paths.
553        struct UnlockOnDrop<'a>(&'a File);
554        impl Drop for UnlockOnDrop<'_> {
555            fn drop(&mut self) {
556                let _ = Fs4FileExt::unlock(self.0);
557            }
558        }
559        let _guard = UnlockOnDrop(&file);
560
561        let remaining = stamp_remaining(&stamp_path, delay);
562        if let Some(d) = remaining {
563            self.sleep_cancelable(d).await?;
564        }
565        write_stamp(&stamp_path)?;
566        Ok(())
567    }
568}
569