1use std::collections::HashMap;
4use std::fs::{self, File, OpenOptions};
5use std::path::Path;
6use std::sync::Mutex;
7use std::time::{Duration, Instant};
8
9use bytes::Bytes;
10use fs4::FileExt as Fs4FileExt;
11use fs4::TryLockError as Fs4TryLockError;
12use reqwest::header::{ACCEPT, ACCEPT_ENCODING, HeaderMap, HeaderValue, USER_AGENT};
13use reqwest::{Client, Method, StatusCode, Url};
14use tracing::{debug, info_span, warn, Instrument};
15
16use crate::cache::DiskCache;
17use crate::concurrency::ConcurrencyBudget;
18use crate::config::{CANCEL_POLL_INTERVAL_MS, Config, HARD_MAX_BODY_BYTES};
19use crate::error::{AppError, AppResult, ErrorKind};
20use crate::retry::{
21 ErrorLayer, RetryClass, RetryConfig, classify_http_status, classify_reqwest_error,
22 parse_retry_after, politeness_delay, wait_for_retry,
23};
24use crate::shutdown::CancelFlag;
25
26use super::allowlist::is_allowed_host;
27use super::body::read_body_capped;
28use super::constants::{
29 POOL_IDLE_TIMEOUT_SECS, POOL_MAX_IDLE_PER_HOST, TCP_KEEPALIVE_IDLE_SECS,
30 TCP_KEEPALIVE_INTERVAL_SECS, TCP_KEEPALIVE_RETRIES,
31};
32use super::rate_limit::{
33 rate_limit_lock_path, rate_limit_stamp_path, stamp_remaining, write_stamp,
34};
35
36fn ensure_ring_crypto_provider() {
41 use std::sync::Once;
42 static ONCE: Once = Once::new();
43 ONCE.call_once(|| {
44 if rustls::crypto::CryptoProvider::get_default().is_none() {
45 let _ = rustls::crypto::ring::default_provider().install_default();
46 }
47 });
48}
49
50pub struct HttpClient {
55 client: Client,
56 cfg: Config,
57 last_host_hit: Mutex<HashMap<String, Instant>>,
59 cancel: CancelFlag,
60 cache: Option<DiskCache>,
61 budget: ConcurrencyBudget,
62 retry: RetryConfig,
63}
64
65#[derive(Debug, Clone)]
67pub struct HttpResponse {
68 pub status: StatusCode,
70 pub final_url: Url,
72 pub body: Bytes,
74 pub content_type: Option<String>,
76 pub cache_hit: bool,
78}
79
80impl HttpClient {
81 pub fn new(cfg: Config, cancel: CancelFlag) -> AppResult<Self> {
93 ensure_ring_crypto_provider();
97
98 let mut cfg = cfg;
100 cfg.clamp_resource_limits();
101 let budget = ConcurrencyBudget::from_configured(cfg.max_concurrency);
102 let retry = RetryConfig::from_config(&cfg);
103
104 let mut headers = HeaderMap::new();
105 headers.insert(
106 USER_AGENT,
107 HeaderValue::from_str(&cfg.user_agent).map_err(|e| {
108 AppError::with_source(ErrorKind::Config, "invalid user-agent header", e)
109 })?,
110 );
111 headers.insert(ACCEPT_ENCODING, HeaderValue::from_static("gzip, br"));
114
115 let client = Client::builder()
116 .default_headers(headers)
117 .timeout(cfg.timeout())
118 .connect_timeout(cfg.connect_timeout())
119 .redirect(reqwest::redirect::Policy::custom({
120 let max = cfg.max_redirects;
121 let allow_loopback = cfg.allow_loopback;
122 move |attempt| {
123 if attempt.previous().len() as u32 >= max {
124 return attempt
125 .error(AppError::new(ErrorKind::Network, "redirect limit exceeded"));
126 }
127 let host = attempt.url().host_str().unwrap_or("");
130 let scheme = attempt.url().scheme();
131 if !crate::config::is_allowed_origin_scheme_host(scheme, host, allow_loopback)
132 {
133 let msg = format!("redirect host not allowlisted: {host}");
135 return attempt.error(AppError::new(ErrorKind::Network, msg));
136 }
137 attempt.follow()
138 }
139 }))
140 .use_rustls_tls()
141 .min_tls_version(reqwest::tls::Version::TLS_1_2)
145 .tcp_nodelay(true)
147 .tcp_keepalive(Duration::from_secs(TCP_KEEPALIVE_IDLE_SECS))
148 .tcp_keepalive_interval(Duration::from_secs(TCP_KEEPALIVE_INTERVAL_SECS))
149 .tcp_keepalive_retries(TCP_KEEPALIVE_RETRIES)
150 .pool_max_idle_per_host(POOL_MAX_IDLE_PER_HOST)
152 .pool_idle_timeout(Duration::from_secs(POOL_IDLE_TIMEOUT_SECS))
153 .http2_adaptive_window(true)
155 .build()
157 .map_err(|e| {
158 AppError::with_source(ErrorKind::Network, "failed to build HTTP client", e)
159 })?;
160
161 let cache = if cfg.cache_enabled() {
162 cfg.cache_dir.as_ref().map(|dir| {
163 DiskCache::new(
164 dir.clone(),
165 cfg.cache_ttl(),
166 cfg.max_cache_bytes,
167 cfg.allow_loopback,
168 )
169 })
170 } else {
171 None
172 };
173
174 Ok(Self {
175 client,
176 cfg,
177 last_host_hit: Mutex::new(HashMap::new()),
178 cancel,
179 cache,
180 budget,
181 retry,
182 })
183 }
184
185 pub fn cancel_flag(&self) -> &crate::shutdown::CancelFlag {
187 &self.cancel
188 }
189
190 pub fn budget(&self) -> &ConcurrencyBudget {
192 &self.budget
193 }
194
195 pub fn retry_policy(&self) -> RetryConfig {
197 self.retry
198 }
199
200 pub async fn get_json(&self, url: &Url) -> AppResult<HttpResponse> {
210 self.request(Method::GET, url, "application/json", true)
211 .await
212 }
213
214 pub async fn get_html(&self, url: &Url) -> AppResult<HttpResponse> {
220 self.request(Method::GET, url, "text/html", true).await
221 }
222
223 async fn request(
224 &self,
225 method: Method,
226 url: &Url,
227 accept: &str,
228 retryable: bool,
229 ) -> AppResult<HttpResponse> {
230 if !is_allowed_host(url, self.cfg.allow_loopback) {
231 return Err(AppError::new(
232 ErrorKind::Network,
233 format!("host not allowlisted: {}", url.host_str().unwrap_or("")),
234 ));
235 }
236
237 if method == Method::GET
240 && let Some(cache) = &self.cache
241 && let Some(hit) = cache.get(url, accept)
242 {
243 let cap = self.cfg.max_body_bytes.min(HARD_MAX_BODY_BYTES);
244 if (hit.body.len() as u64) > cap {
245 return Err(AppError::new(
246 ErrorKind::Budget,
247 format!("cached response body exceeds max_body_bytes ({cap})"),
248 ));
249 }
250 return Ok(hit);
251 }
252
253 let policy = if retryable {
256 self.retry
257 } else {
258 RetryConfig {
259 enabled: false,
260 ..self.retry
261 }
262 };
263 let url_str = url.as_str();
266 let started = Instant::now();
268 let mut attempt = 0u32;
269 loop {
270 self.cancel.check()?;
271 attempt += 1;
272 self.rate_limit(url).await?;
273 self.cancel.check()?;
274
275 let mut req = self.client.request(method.clone(), url_str);
276 req = req.header(ACCEPT, accept);
277
278 let result = req.send().await;
279 match result {
280 Ok(resp) => {
281 let status = resp.status();
282 let final_url = resp.url().clone();
283 let content_type = resp
284 .headers()
285 .get(reqwest::header::CONTENT_TYPE)
286 .and_then(|v| v.to_str().ok())
287 .map(|s| s.to_string());
288 let retry_after = parse_retry_after(resp.headers());
289 let class = classify_http_status(status, retry_after);
290
291 match class {
292 RetryClass::RateLimited { retry_after } => {
293 let wait = wait_for_retry(policy, attempt, retry_after);
294 if policy.may_retry_within_budget(attempt, started.elapsed(), wait) {
295 let span = info_span!(
296 target: "docsrs_cli::retry",
297 "retry_attempt",
298 attempt,
299 max_attempts = policy.max_attempts(),
300 status = %status,
301 layer = ErrorLayer::HttpStatus.as_str(),
302 wait_ms = wait.as_millis() as u64,
303 reason = "rate_limited",
304 );
305 async {
306 debug!(
307 target: "docsrs_cli::retry",
308 ?wait,
309 attempt,
310 max_attempts = policy.max_attempts(),
311 status = %status,
312 "retrying after rate-limit / Retry-After"
313 );
314 self.sleep_cancelable(wait).await
315 }
316 .instrument(span)
317 .await?;
318 continue;
319 }
320 let secs = retry_after.map(|d| d.as_secs()).unwrap_or(1);
321 return Err(AppError::from_http_status(
322 status.as_u16(),
323 "retries exhausted",
324 )
325 .with_retry_after(secs.max(1)));
326 }
327 RetryClass::Transient => {
328 let wait = wait_for_retry(policy, attempt, None);
329 if policy.may_retry_within_budget(attempt, started.elapsed(), wait) {
330 let span = info_span!(
331 target: "docsrs_cli::retry",
332 "retry_attempt",
333 attempt,
334 max_attempts = policy.max_attempts(),
335 status = %status,
336 layer = ErrorLayer::HttpStatus.as_str(),
337 wait_ms = wait.as_millis() as u64,
338 reason = "transient_status",
339 );
340 async {
341 debug!(
342 target: "docsrs_cli::retry",
343 ?wait,
344 attempt,
345 max_attempts = policy.max_attempts(),
346 status = %status,
347 "retrying after transient status"
348 );
349 self.sleep_cancelable(wait).await
350 }
351 .instrument(span)
352 .await?;
353 continue;
354 }
355 return Err(AppError::from_http_status(
357 status.as_u16(),
358 "retries exhausted",
359 ));
360 }
361 RetryClass::Permanent => {
362 }
364 }
365
366 if !is_allowed_host(&final_url, self.cfg.allow_loopback) {
369 return Err(AppError::new(
370 ErrorKind::Network,
371 format!(
372 "final URL host not allowlisted: {}",
373 final_url.host_str().unwrap_or("")
374 ),
375 ));
376 }
377 let body = read_body_capped(resp, self.cfg.max_body_bytes).await?;
378 let out = HttpResponse {
379 status,
380 final_url,
381 body,
382 content_type,
383 cache_hit: false,
384 };
385 if method == Method::GET
386 && out.status.is_success()
387 && let Some(cache) = &self.cache
388 && let Err(e) = cache.put(url, accept, &out)
389 {
390 warn!(error = %e, "failed to write HTTP cache entry");
391 }
392 return Ok(out);
393 }
394 Err(e) => {
395 let (kind, layer, transport_retryable) = classify_reqwest_error(&e);
397 if transport_retryable {
398 let wait = wait_for_retry(policy, attempt, None);
399 if policy.may_retry_within_budget(attempt, started.elapsed(), wait) {
400 let span = info_span!(
401 target: "docsrs_cli::retry",
402 "retry_attempt",
403 attempt,
404 max_attempts = policy.max_attempts(),
405 layer = layer.as_str(),
406 wait_ms = wait.as_millis() as u64,
407 reason = "transport",
408 );
409 async {
410 warn!(
411 target: "docsrs_cli::retry",
412 ?wait,
413 attempt,
414 max_attempts = policy.max_attempts(),
415 error = %e,
416 layer = layer.as_str(),
417 "retrying after transport error"
418 );
419 self.sleep_cancelable(wait).await
420 }
421 .instrument(span)
422 .await?;
423 continue;
424 }
425 }
426 return Err(AppError::with_source(
427 kind,
428 format!("http request failed for {url}"),
429 e,
430 ));
431 }
432 }
433 }
434 }
435
436 async fn sleep_cancelable(&self, wait: Duration) -> AppResult<()> {
437 let step = Duration::from_millis(CANCEL_POLL_INTERVAL_MS);
438 let mut remaining = wait;
439 while !remaining.is_zero() {
440 self.cancel.check()?;
441 let slice = remaining.min(step);
442 tokio::time::sleep(slice).await;
443 remaining = remaining.saturating_sub(slice);
444 }
445 self.cancel.check()
446 }
447
448 async fn rate_limit(&self, url: &Url) -> AppResult<()> {
456 let host = url.host_str().unwrap_or("");
458 let delay = politeness_delay(self.cfg.rate_limit_delay());
459 if delay.is_zero() {
460 return Ok(());
461 }
462
463 if let Some(dir) = self.cfg.cache_dir.as_ref() {
465 match self.rate_limit_cross_process(dir, host, delay).await {
466 Ok(()) => {
467 self.touch_host_clock(host);
468 return Ok(());
469 }
470 Err(e)
471 if e.kind() == ErrorKind::Terminated || e.kind() == ErrorKind::Interrupted =>
472 {
473 return Err(e);
474 }
475 Err(e) => {
476 warn!(error = %e, host = %host, "cross-process rate-limit lock failed; using in-process only");
478 }
479 }
480 }
481
482 let in_proc = {
483 let map = self.last_host_hit.lock().unwrap_or_else(|e| e.into_inner());
484 map.get(host).and_then(|prev| {
485 let elapsed = prev.elapsed();
486 if elapsed < delay {
487 Some(delay - elapsed)
488 } else {
489 None
490 }
491 })
492 };
493 if let Some(d) = in_proc {
494 self.sleep_cancelable(d).await?;
495 }
496 self.touch_host_clock(host);
497 Ok(())
498 }
499
500 fn touch_host_clock(&self, host: &str) {
501 let mut map = self.last_host_hit.lock().unwrap_or_else(|e| e.into_inner());
502 map.insert(host.to_string(), Instant::now());
503 }
504
505 async fn rate_limit_cross_process(
507 &self,
508 cache_dir: &Path,
509 host: &str,
510 delay: Duration,
511 ) -> AppResult<()> {
512 let lock_path = rate_limit_lock_path(cache_dir, host);
513 let stamp_path = rate_limit_stamp_path(cache_dir, host);
514 if let Some(parent) = lock_path.parent() {
515 fs::create_dir_all(parent).map_err(|e| {
516 AppError::with_source(ErrorKind::Network, "rate-limit lock dir create failed", e)
517 })?;
518 crate::platform::restrict_private_dir(parent);
519 }
520
521 let file = OpenOptions::new()
522 .create(true)
523 .read(true)
524 .write(true)
525 .truncate(false)
526 .open(&lock_path)
527 .map_err(|e| {
528 AppError::with_source(ErrorKind::Network, "rate-limit lock open failed", e)
529 })?;
530 crate::platform::restrict_private_file(&lock_path);
531
532 loop {
535 self.cancel.check()?;
536 match Fs4FileExt::try_lock(&file) {
537 Ok(()) => break,
538 Err(Fs4TryLockError::WouldBlock) => {
539 self.sleep_cancelable(Duration::from_millis(CANCEL_POLL_INTERVAL_MS))
540 .await?;
541 }
542 Err(Fs4TryLockError::Error(e)) => {
543 return Err(AppError::with_source(
544 ErrorKind::Network,
545 "rate-limit exclusive lock failed",
546 e,
547 ));
548 }
549 }
550 }
551
552 struct UnlockOnDrop<'a>(&'a File);
554 impl Drop for UnlockOnDrop<'_> {
555 fn drop(&mut self) {
556 let _ = Fs4FileExt::unlock(self.0);
557 }
558 }
559 let _guard = UnlockOnDrop(&file);
560
561 let remaining = stamp_remaining(&stamp_path, delay);
562 if let Some(d) = remaining {
563 self.sleep_cancelable(d).await?;
564 }
565 write_stamp(&stamp_path)?;
566 Ok(())
567 }
568}
569