docsrs_cli/http/client.rs
1//! HTTPS client: rustls, retry, rate limit, body cap, host allowlist, cancel-aware.
2
3use std::collections::HashMap;
4use std::sync::Mutex;
5use std::time::{Duration, Instant};
6
7use bytes::Bytes;
8use reqwest::header::{ACCEPT, ACCEPT_ENCODING, HeaderMap, HeaderValue, USER_AGENT};
9use reqwest::{Client, Method, StatusCode, Url};
10use tracing::{Instrument, debug, info_span, warn};
11
12use crate::cache::DiskCache;
13use crate::concurrency::ConcurrencyBudget;
14use crate::config::{Config, HARD_MAX_BODY_BYTES};
15use crate::error::{AllowlistStage, AppError, AppResult, ErrorDetail};
16use crate::retry::{
17 ErrorLayer, RetryClass, RetryConfig, classify_http_status, classify_reqwest_error,
18 parse_retry_after, wait_for_retry,
19};
20use crate::shutdown::CancelFlag;
21
22use super::allowlist::is_allowed_host;
23use super::body::read_body_capped;
24use super::constants::{
25 POOL_IDLE_TIMEOUT_SECS, POOL_MAX_IDLE_PER_HOST, TCP_KEEPALIVE_IDLE_SECS,
26 TCP_KEEPALIVE_INTERVAL_SECS, TCP_KEEPALIVE_RETRIES,
27};
28
29/// Install the ring CryptoProvider once when none is configured (ADR 0007).
30///
31/// Safe to call repeatedly. Does not overwrite an existing process default
32/// (binary `main` already installed it, or a prior call succeeded).
33///
34/// `ring` compiles C, which the product rule forbids in principle. Both
35/// pure-Rust replacements were measured and rejected in 2026-08 — see ADR 0007
36/// and GAP-TOOLCHAIN-001 — so the exception is recorded rather than silent.
37fn ensure_crypto_provider() {
38 use std::sync::Once;
39 static ONCE: Once = Once::new();
40 ONCE.call_once(|| {
41 if rustls::crypto::CryptoProvider::get_default().is_none() {
42 let _ = rustls::crypto::ring::default_provider().install_default();
43 }
44 });
45}
46
47/// HTTPS client bound to product allowlist and config.
48///
49/// Shareable across tasks (`&self`): rate-limit clock uses a short-lived
50/// [`Mutex`] around the map only; sleeps run without holding the lock.
51pub struct HttpClient {
52 client: Client,
53 // `pub(super)` for the fields the sibling `rate_limit` module drives; the
54 // throttle policy lives next to its stamp/lock I/O, not in the request loop.
55 pub(super) cfg: Config,
56 /// In-process last hit per host. Lock is never held across `.await`.
57 pub(super) last_host_hit: Mutex<HashMap<String, Instant>>,
58 pub(super) cancel: CancelFlag,
59 cache: Option<DiskCache>,
60 budget: ConcurrencyBudget,
61 retry: RetryConfig,
62}
63
64/// Successful HTTP response body and metadata.
65#[derive(Debug, Clone)]
66pub struct HttpResponse {
67 /// Final HTTP status after redirects.
68 pub status: StatusCode,
69 /// Final URL after redirects.
70 pub final_url: Url,
71 /// Response body bytes (capped).
72 pub body: Bytes,
73 /// Content-Type header when present.
74 pub content_type: Option<String>,
75 /// True when served from the local disk cache.
76 pub cache_hit: bool,
77}
78
79impl HttpClient {
80 /// Build a client with rustls, default headers, and redirect allowlist.
81 ///
82 /// Owned by one one-shot command. Call sites may `Config::clone` before passing
83 /// `cfg` when they still need the parent config after construction. That clone is
84 /// intentional and cheap relative to TLS/RTT/body I/O. [`crate::concurrency::ConcurrencyBudget`] is
85 /// built from `cfg.max_concurrency` (0 = auto).
86 ///
87 /// # Errors
88 ///
89 /// Returns [`crate::error::ErrorKind::Config`] when the User-Agent is not a valid header value.
90 /// Returns [`crate::error::ErrorKind::Network`] when the reqwest client cannot be built.
91 pub fn new(cfg: Config, cancel: CancelFlag) -> AppResult<Self> {
92 // reqwest 0.13 `rustls-no-provider`: a process CryptoProvider is required.
93 // Binary installs ring first in `main` (fail-closed dual-init). Library
94 // callers and tests reach this path without `main` — install once if missing.
95 ensure_crypto_provider();
96
97 // Defense in depth: clamp even if a caller skipped Config::clamp_resource_limits.
98 let mut cfg = cfg;
99 cfg.clamp_resource_limits();
100 let budget = ConcurrencyBudget::from_configured(cfg.max_concurrency);
101 let retry = RetryConfig::from_config(&cfg);
102
103 let mut headers = HeaderMap::new();
104 headers.insert(
105 USER_AGENT,
106 HeaderValue::from_str(&cfg.user_agent)
107 .map_err(|e| AppError::of_with_source(ErrorDetail::UserAgentHeaderInvalid, e))?,
108 );
109 // Explicit encodings: feature `brotli` alone does not add `br` when this
110 // header is set manually (SCRAPE-S-004).
111 headers.insert(ACCEPT_ENCODING, HeaderValue::from_static("gzip, br"));
112
113 let client = Client::builder()
114 .default_headers(headers)
115 .timeout(cfg.timeout())
116 .connect_timeout(cfg.connect_timeout())
117 .redirect(reqwest::redirect::Policy::custom({
118 let max = cfg.max_redirects;
119 let allow_loopback = cfg.allow_loopback;
120 move |attempt| {
121 if attempt.previous().len() as u32 >= max {
122 return attempt.error(AppError::of(ErrorDetail::RedirectLimitExceeded));
123 }
124 // Borrow host_str — no intermediate String on the redirect hot path.
125 // Same allowlist as request gate + config origins (SSRF defense in depth).
126 let host = attempt.url().host_str().unwrap_or("");
127 let scheme = attempt.url().scheme();
128 if !crate::config::is_allowed_origin_scheme_host(scheme, host, allow_loopback) {
129 // Materialize the detail before moving `attempt` (host is borrowed).
130 let detail = ErrorDetail::HostNotAllowlisted {
131 host: host.to_string(),
132 stage: AllowlistStage::Redirect,
133 };
134 return attempt.error(AppError::of(detail));
135 }
136 attempt.follow()
137 }
138 }))
139 // Preconfigured rather than `use_rustls_tls()`: the roots are ours
140 // (Mozilla set, compiled in) instead of whatever reqwest would wire
141 // up, which since 0.13 is the OS store. See `super::tls`.
142 //
143 // The TLS 1.2 floor moves into that config too — `min_tls_version`
144 // is ignored once a config is supplied, so leaving it here would be
145 // a setting that reads as applied and enforces nothing.
146 //
147 // Not `https_only(true)`: offline wiremock uses `http://127.0.0.1` when
148 // `allow_loopback` is set (CLI/XDG); production hosts still require HTTPS.
149 .use_preconfigured_tls(super::tls::client_config()?)
150 // Low-latency request/response docs GETs (disable Nagle).
151 .tcp_nodelay(true)
152 .tcp_keepalive(Duration::from_secs(TCP_KEEPALIVE_IDLE_SECS))
153 .tcp_keepalive_interval(Duration::from_secs(TCP_KEEPALIVE_INTERVAL_SECS))
154 .tcp_keepalive_retries(TCP_KEEPALIVE_RETRIES)
155 // One-shot pool: tiny idle set, short idle TTL (process dies soon).
156 .pool_max_idle_per_host(POOL_MAX_IDLE_PER_HOST)
157 .pool_idle_timeout(Duration::from_secs(POOL_IDLE_TIMEOUT_SECS))
158 // HTTP/2 adaptive flow control when ALPN selects h2.
159 .http2_adaptive_window(true)
160 // system-proxy feature: honor HTTP(S)_PROXY / ALL_PROXY / NO_PROXY.
161 .build()
162 .map_err(|e| AppError::of_with_source(ErrorDetail::HttpClientBuild, e))?;
163
164 let cache = if cfg.cache_enabled() {
165 cfg.cache_dir.as_ref().map(|dir| {
166 DiskCache::new(
167 dir.clone(),
168 cfg.cache_ttl(),
169 cfg.max_cache_bytes,
170 cfg.allow_loopback,
171 )
172 })
173 } else {
174 None
175 };
176
177 Ok(Self {
178 client,
179 cfg,
180 last_host_hit: Mutex::new(HashMap::new()),
181 cancel,
182 cache,
183 budget,
184 retry,
185 })
186 }
187
188 /// Shared cancel flag for cooperative shutdown (HTTP + CPU scrape).
189 pub fn cancel_flag(&self) -> &crate::shutdown::CancelFlag {
190 &self.cancel
191 }
192
193 /// Shared concurrency budget for CPU-bound parse work.
194 pub fn budget(&self) -> &ConcurrencyBudget {
195 &self.budget
196 }
197
198 /// Resolved retry policy (for doctor / tests).
199 pub fn retry_policy(&self) -> RetryConfig {
200 self.retry
201 }
202
203 /// GET expecting JSON (`Accept: application/json`).
204 ///
205 /// # Errors
206 ///
207 /// Returns [`crate::error::ErrorKind::Network`] for allowlist, redirect, TLS, or body-cap failures.
208 /// Returns [`crate::error::ErrorKind::Timeout`] when the wall-clock budget expires.
209 /// Returns [`crate::error::ErrorKind::Interrupted`] or [`crate::error::ErrorKind::Terminated`] on cancel
210 /// (kind preserved from the first `CancelFlag::cancel_with`).
211 /// Maps HTTP statuses via [`AppError::from_http_status`] (e.g. not-found, rate-limit).
212 pub async fn get_json(&self, url: &Url) -> AppResult<HttpResponse> {
213 self.request(Method::GET, url, "application/json", true)
214 .await
215 }
216
217 /// GET expecting HTML (`Accept: text/html`).
218 ///
219 /// # Errors
220 ///
221 /// Same error classes as [`Self::get_json`].
222 pub async fn get_html(&self, url: &Url) -> AppResult<HttpResponse> {
223 self.request(Method::GET, url, "text/html", true).await
224 }
225
226 async fn request(
227 &self,
228 method: Method,
229 url: &Url,
230 accept: &str,
231 retryable: bool,
232 ) -> AppResult<HttpResponse> {
233 if !is_allowed_host(url, self.cfg.allow_loopback) {
234 return Err(AppError::of(ErrorDetail::HostNotAllowlisted {
235 host: url.host_str().unwrap_or("").to_string(),
236 stage: AllowlistStage::Request,
237 }));
238 }
239
240 // Disk cache only for GET (product surface is GET-only).
241 // Invariant: every HttpResponse respects max_body_bytes (network *and* cache).
242 if method == Method::GET
243 && let Some(cache) = &self.cache
244 && let Some(hit) = cache.get(url, accept)
245 {
246 let cap = self.cfg.max_body_bytes.min(HARD_MAX_BODY_BYTES);
247 if (hit.body.len() as u64) > cap {
248 return Err(AppError::of(ErrorDetail::CachedBodyOverBudget {
249 max_bytes: cap,
250 }));
251 }
252 return Ok(hit);
253 }
254
255 // Policy is opt-out kill switch + max_retries; `retryable` is for future
256 // non-idempotent methods (product is GET-only today).
257 let policy = if retryable {
258 self.retry
259 } else {
260 RetryConfig {
261 enabled: false,
262 ..self.retry
263 }
264 };
265 // Prefer `url.as_str()` over `url.clone()`: reqwest implements IntoUrl for
266 // `&str` but not for `&Url` (0.12). Method is a small enum — clone is trivial.
267 let url_str = url.as_str();
268 // Monotonic start for max_elapsed_ms budget (Rules: Instant, not SystemTime).
269 let started = Instant::now();
270 let mut attempt = 0u32;
271 loop {
272 self.cancel.check()?;
273 attempt += 1;
274 self.rate_limit(url).await?;
275 self.cancel.check()?;
276
277 let mut req = self.client.request(method.clone(), url_str);
278 req = req.header(ACCEPT, accept);
279
280 let result = req.send().await;
281 match result {
282 Ok(resp) => {
283 let status = resp.status();
284 let final_url = resp.url().clone();
285 let content_type = resp
286 .headers()
287 .get(reqwest::header::CONTENT_TYPE)
288 .and_then(|v| v.to_str().ok())
289 .map(|s| s.to_string());
290 let retry_after = parse_retry_after(resp.headers());
291 let class = classify_http_status(status, retry_after);
292
293 match class {
294 RetryClass::RateLimited { retry_after } => {
295 let wait = wait_for_retry(policy, attempt, retry_after);
296 if policy.may_retry_within_budget(attempt, started.elapsed(), wait) {
297 let span = info_span!(
298 target: "docsrs_cli::retry",
299 "retry_attempt",
300 attempt,
301 max_attempts = policy.max_attempts(),
302 status = %status,
303 layer = ErrorLayer::HttpStatus.as_str(),
304 wait_ms = wait.as_millis() as u64,
305 reason = "rate_limited",
306 );
307 async {
308 debug!(
309 target: "docsrs_cli::retry",
310 ?wait,
311 attempt,
312 max_attempts = policy.max_attempts(),
313 status = %status,
314 "retrying after rate-limit / Retry-After"
315 );
316 self.sleep_cancelable(wait).await
317 }
318 .instrument(span)
319 .await?;
320 continue;
321 }
322 let secs = retry_after.map(|d| d.as_secs()).unwrap_or(1);
323 return Err(AppError::from_http_status(
324 status.as_u16(),
325 "retries exhausted",
326 )
327 .with_retry_after(secs.max(1)));
328 }
329 RetryClass::Transient => {
330 let wait = wait_for_retry(policy, attempt, None);
331 if policy.may_retry_within_budget(attempt, started.elapsed(), wait) {
332 let span = info_span!(
333 target: "docsrs_cli::retry",
334 "retry_attempt",
335 attempt,
336 max_attempts = policy.max_attempts(),
337 status = %status,
338 layer = ErrorLayer::HttpStatus.as_str(),
339 wait_ms = wait.as_millis() as u64,
340 reason = "transient_status",
341 );
342 async {
343 debug!(
344 target: "docsrs_cli::retry",
345 ?wait,
346 attempt,
347 max_attempts = policy.max_attempts(),
348 status = %status,
349 "retrying after transient status"
350 );
351 self.sleep_cancelable(wait).await
352 }
353 .instrument(span)
354 .await?;
355 continue;
356 }
357 // Exhausted attempts or elapsed budget on 5xx/408.
358 return Err(AppError::from_http_status(
359 status.as_u16(),
360 "retries exhausted",
361 ));
362 }
363 RetryClass::Permanent => {
364 // 2xx and permanent 4xx: read body and return (caller maps status).
365 }
366 }
367
368 // Defense in depth: redirect policy already gates hops; re-check
369 // final URL so a policy regression cannot leak off-allowlist bodies.
370 if !is_allowed_host(&final_url, self.cfg.allow_loopback) {
371 return Err(AppError::of(ErrorDetail::HostNotAllowlisted {
372 host: final_url.host_str().unwrap_or("").to_string(),
373 stage: AllowlistStage::FinalUrl,
374 }));
375 }
376 let body = read_body_capped(resp, self.cfg.max_body_bytes).await?;
377 let out = HttpResponse {
378 status,
379 final_url,
380 body,
381 content_type,
382 cache_hit: false,
383 };
384 if method == Method::GET
385 && out.status.is_success()
386 && let Some(cache) = &self.cache
387 && let Err(e) = cache.put(url, accept, &out)
388 {
389 warn!(error = %e, "failed to write HTTP cache entry");
390 }
391 return Ok(out);
392 }
393 Err(e) => {
394 // Classify transport layers without string-matching messages.
395 let (kind, layer, transport_retryable) = classify_reqwest_error(&e);
396 if transport_retryable {
397 let wait = wait_for_retry(policy, attempt, None);
398 if policy.may_retry_within_budget(attempt, started.elapsed(), wait) {
399 let span = info_span!(
400 target: "docsrs_cli::retry",
401 "retry_attempt",
402 attempt,
403 max_attempts = policy.max_attempts(),
404 layer = layer.as_str(),
405 wait_ms = wait.as_millis() as u64,
406 reason = "transport",
407 );
408 async {
409 warn!(
410 target: "docsrs_cli::retry",
411 ?wait,
412 attempt,
413 max_attempts = policy.max_attempts(),
414 error = %e,
415 layer = layer.as_str(),
416 "retrying after transport error"
417 );
418 self.sleep_cancelable(wait).await
419 }
420 .instrument(span)
421 .await?;
422 continue;
423 }
424 }
425 return Err(AppError::of_with_source(
426 ErrorDetail::HttpRequestFailed {
427 url: url.to_string(),
428 kind,
429 },
430 e,
431 ));
432 }
433 }
434 }
435 }
436}