1use std::collections::HashMap;
40use std::fs::{self, File, OpenOptions};
41use std::path::{Path, PathBuf};
42use std::sync::Mutex;
43use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
44
45use bytes::Bytes;
46use fs4::FileExt as Fs4FileExt;
47use fs4::TryLockError as Fs4TryLockError;
48use futures_util::StreamExt;
49use reqwest::header::{ACCEPT, ACCEPT_ENCODING, HeaderMap, HeaderValue, USER_AGENT};
50use reqwest::{Client, Method, StatusCode, Url};
51use tracing::{debug, warn};
52
53use crate::cache::DiskCache;
54use crate::concurrency::ConcurrencyBudget;
55use crate::config::{
56 Config, HARD_MAX_BODY_BYTES, HOST_CRATES_IO, HOST_DOC_RUST_LANG_ORG, HOST_DOCS_RS,
57 HOST_STATIC_DOCS_RS,
58};
59use crate::error::{AppError, AppResult, ErrorKind};
60use crate::retry::{
61 RetryClass, RetryConfig, classify_http_status, parse_retry_after, politeness_delay,
62 wait_for_retry,
63};
64use crate::shutdown::CancelFlag;
65
66pub struct HttpClient {
71 client: Client,
72 cfg: Config,
73 last_host_hit: Mutex<HashMap<String, Instant>>,
75 cancel: CancelFlag,
76 cache: Option<DiskCache>,
77 budget: ConcurrencyBudget,
79 retry: RetryConfig,
81}
82
83#[derive(Debug, Clone)]
85pub struct HttpResponse {
86 pub status: StatusCode,
88 pub final_url: Url,
90 pub body: Bytes,
92 pub content_type: Option<String>,
94}
95
96impl HttpClient {
97 pub fn new(cfg: Config, cancel: CancelFlag) -> AppResult<Self> {
109 let mut cfg = cfg;
111 cfg.clamp_resource_limits();
112 let budget = ConcurrencyBudget::from_configured(cfg.max_concurrency);
113 let retry = RetryConfig::from_config(&cfg);
114
115 let mut headers = HeaderMap::new();
116 headers.insert(
117 USER_AGENT,
118 HeaderValue::from_str(&cfg.user_agent).map_err(|e| {
119 AppError::with_source(ErrorKind::Config, "invalid user-agent header", e)
120 })?,
121 );
122 headers.insert(ACCEPT_ENCODING, HeaderValue::from_static("gzip"));
123
124 let client = Client::builder()
125 .default_headers(headers)
126 .timeout(cfg.timeout())
127 .connect_timeout(cfg.connect_timeout())
128 .redirect(reqwest::redirect::Policy::custom({
129 let max = cfg.max_redirects;
130 move |attempt| {
131 if attempt.previous().len() as u32 >= max {
132 return attempt
133 .error(AppError::new(ErrorKind::Network, "redirect limit exceeded"));
134 }
135 let host = attempt.url().host_str().unwrap_or("");
137 let scheme_ok = attempt.url().scheme() == "https";
138 let host_ok = matches!(
139 host,
140 HOST_CRATES_IO
141 | HOST_DOCS_RS
142 | HOST_STATIC_DOCS_RS
143 | HOST_DOC_RUST_LANG_ORG
144 ) || ((host == "127.0.0.1" || host == "localhost")
145 && (cfg!(test) || std::env::var("DOCSRS_CLI_ALLOW_LOCALHOST").is_ok()));
146 if !(scheme_ok && host_ok) {
147 let msg = format!("redirect host not allowlisted: {host}");
149 return attempt.error(AppError::new(ErrorKind::Network, msg));
150 }
151 attempt.follow()
152 }
153 }))
154 .use_rustls_tls()
155 .build()
156 .map_err(|e| {
157 AppError::with_source(ErrorKind::Network, "failed to build HTTP client", e)
158 })?;
159
160 let cache = if cfg.cache_enabled() {
161 cfg.cache_dir
162 .as_ref()
163 .map(|dir| DiskCache::new(dir.clone(), cfg.cache_ttl(), cfg.max_cache_bytes))
164 } else {
165 None
166 };
167
168 Ok(Self {
169 client,
170 cfg,
171 last_host_hit: Mutex::new(HashMap::new()),
172 cancel,
173 cache,
174 budget,
175 retry,
176 })
177 }
178
179 pub fn budget(&self) -> &ConcurrencyBudget {
181 &self.budget
182 }
183
184 pub fn retry_policy(&self) -> RetryConfig {
186 self.retry
187 }
188
189 pub async fn get_json(&self, url: &Url) -> AppResult<HttpResponse> {
199 self.request(Method::GET, url, "application/json", true)
200 .await
201 }
202
203 pub async fn get_html(&self, url: &Url) -> AppResult<HttpResponse> {
209 self.request(Method::GET, url, "text/html", true).await
210 }
211
212 async fn request(
213 &self,
214 method: Method,
215 url: &Url,
216 accept: &str,
217 retryable: bool,
218 ) -> AppResult<HttpResponse> {
219 if !is_allowed_host(url) {
220 return Err(AppError::new(
221 ErrorKind::Network,
222 format!("host not allowlisted: {}", url.host_str().unwrap_or("")),
223 ));
224 }
225
226 if method == Method::GET
228 && let Some(cache) = &self.cache
229 && let Some(hit) = cache.get(url, accept)
230 {
231 return Ok(hit);
232 }
233
234 let policy = if retryable {
237 self.retry
238 } else {
239 RetryConfig {
240 enabled: false,
241 ..self.retry
242 }
243 };
244 let url_str = url.as_str();
247 let mut attempt = 0u32;
248 loop {
249 self.cancel.check()?;
250 attempt += 1;
251 self.rate_limit(url).await?;
252 self.cancel.check()?;
253
254 let mut req = self.client.request(method.clone(), url_str);
255 req = req.header(ACCEPT, accept);
256
257 let result = req.send().await;
258 match result {
259 Ok(resp) => {
260 let status = resp.status();
261 let final_url = resp.url().clone();
262 let content_type = resp
263 .headers()
264 .get(reqwest::header::CONTENT_TYPE)
265 .and_then(|v| v.to_str().ok())
266 .map(|s| s.to_string());
267 let retry_after = parse_retry_after(resp.headers());
268 let class = classify_http_status(status, retry_after);
269
270 match class {
271 RetryClass::RateLimited { retry_after } if policy.may_retry(attempt) => {
272 let wait = wait_for_retry(policy, attempt, retry_after);
273 debug!(
274 target: "docsrs_cli::retry",
275 ?wait,
276 attempt,
277 max_attempts = policy.max_attempts(),
278 status = %status,
279 "retrying after rate-limit / Retry-After"
280 );
281 self.sleep_cancelable(wait).await?;
282 continue;
283 }
284 RetryClass::Transient if policy.may_retry(attempt) => {
285 let wait = wait_for_retry(policy, attempt, None);
286 debug!(
287 target: "docsrs_cli::retry",
288 ?wait,
289 attempt,
290 max_attempts = policy.max_attempts(),
291 status = %status,
292 "retrying after transient 5xx"
293 );
294 self.sleep_cancelable(wait).await?;
295 continue;
296 }
297 RetryClass::RateLimited { retry_after } => {
298 let secs = retry_after.map(|d| d.as_secs()).unwrap_or(1);
299 return Err(AppError::from_http_status(
300 status.as_u16(),
301 "retries exhausted",
302 )
303 .with_retry_after(secs.max(1)));
304 }
305 RetryClass::Transient => {
306 return Err(AppError::from_http_status(
308 status.as_u16(),
309 "retries exhausted",
310 ));
311 }
312 RetryClass::Permanent => {
313 }
315 }
316
317 let body = read_body_capped(resp, self.cfg.max_body_bytes).await?;
318 let out = HttpResponse {
319 status,
320 final_url,
321 body,
322 content_type,
323 };
324 if method == Method::GET
325 && out.status.is_success()
326 && let Some(cache) = &self.cache
327 && let Err(e) = cache.put(url, accept, &out)
328 {
329 warn!(error = %e, "failed to write HTTP cache entry");
330 }
331 return Ok(out);
332 }
333 Err(e) => {
334 let is_timeout = e.is_timeout();
336 let is_connect = e.is_connect();
337 let is_request = e.is_request();
338 let kind = if is_timeout {
339 ErrorKind::Timeout
340 } else {
341 ErrorKind::Network
342 };
343 let transport_retryable = is_timeout || is_connect || is_request;
344 if policy.may_retry(attempt) && transport_retryable {
345 let wait = wait_for_retry(policy, attempt, None);
346 warn!(
347 target: "docsrs_cli::retry",
348 ?wait,
349 attempt,
350 max_attempts = policy.max_attempts(),
351 error = %e,
352 is_timeout,
353 is_connect,
354 "retrying after transport error"
355 );
356 self.sleep_cancelable(wait).await?;
357 continue;
358 }
359 return Err(AppError::with_source(
360 kind,
361 format!("HTTP request failed for {url}"),
362 e,
363 ));
364 }
365 }
366 }
367 }
368
369 async fn sleep_cancelable(&self, wait: Duration) -> AppResult<()> {
370 let step = Duration::from_millis(50);
371 let mut remaining = wait;
372 while !remaining.is_zero() {
373 self.cancel.check()?;
374 let slice = remaining.min(step);
375 tokio::time::sleep(slice).await;
376 remaining = remaining.saturating_sub(slice);
377 }
378 self.cancel.check()
379 }
380
381 async fn rate_limit(&self, url: &Url) -> AppResult<()> {
389 let host = url.host_str().unwrap_or("");
391 let delay = politeness_delay(self.cfg.rate_limit_delay());
392 if delay.is_zero() {
393 return Ok(());
394 }
395
396 if let Some(dir) = self.cfg.cache_dir.as_ref() {
398 match self.rate_limit_cross_process(dir, host, delay).await {
399 Ok(()) => {
400 self.touch_host_clock(host);
401 return Ok(());
402 }
403 Err(e)
404 if e.kind() == ErrorKind::Terminated || e.kind() == ErrorKind::Interrupted =>
405 {
406 return Err(e);
407 }
408 Err(e) => {
409 warn!(error = %e, host = %host, "cross-process rate-limit lock failed; using in-process only");
411 }
412 }
413 }
414
415 let in_proc = {
416 let map = self
417 .last_host_hit
418 .lock()
419 .unwrap_or_else(|e| e.into_inner());
420 map.get(host).and_then(|prev| {
421 let elapsed = prev.elapsed();
422 if elapsed < delay {
423 Some(delay - elapsed)
424 } else {
425 None
426 }
427 })
428 };
429 if let Some(d) = in_proc {
430 self.sleep_cancelable(d).await?;
431 }
432 self.touch_host_clock(host);
433 Ok(())
434 }
435
436 fn touch_host_clock(&self, host: &str) {
437 let mut map = self
438 .last_host_hit
439 .lock()
440 .unwrap_or_else(|e| e.into_inner());
441 map.insert(host.to_string(), Instant::now());
442 }
443
444 async fn rate_limit_cross_process(
446 &self,
447 cache_dir: &Path,
448 host: &str,
449 delay: Duration,
450 ) -> AppResult<()> {
451 let lock_path = rate_limit_lock_path(cache_dir, host);
452 let stamp_path = rate_limit_stamp_path(cache_dir, host);
453 if let Some(parent) = lock_path.parent() {
454 fs::create_dir_all(parent).map_err(|e| {
455 AppError::with_source(ErrorKind::Network, "rate-limit lock dir create failed", e)
456 })?;
457 crate::platform::restrict_private_dir(parent);
458 }
459
460 let file = OpenOptions::new()
461 .create(true)
462 .read(true)
463 .write(true)
464 .truncate(false)
465 .open(&lock_path)
466 .map_err(|e| {
467 AppError::with_source(ErrorKind::Network, "rate-limit lock open failed", e)
468 })?;
469 crate::platform::restrict_private_file(&lock_path);
470
471 loop {
474 self.cancel.check()?;
475 match Fs4FileExt::try_lock(&file) {
476 Ok(()) => break,
477 Err(Fs4TryLockError::WouldBlock) => {
478 self.sleep_cancelable(Duration::from_millis(50)).await?;
479 }
480 Err(Fs4TryLockError::Error(e)) => {
481 return Err(AppError::with_source(
482 ErrorKind::Network,
483 "rate-limit exclusive lock failed",
484 e,
485 ));
486 }
487 }
488 }
489
490 struct UnlockOnDrop<'a>(&'a File);
492 impl Drop for UnlockOnDrop<'_> {
493 fn drop(&mut self) {
494 let _ = Fs4FileExt::unlock(self.0);
495 }
496 }
497 let _guard = UnlockOnDrop(&file);
498
499 let remaining = stamp_remaining(&stamp_path, delay);
500 if let Some(d) = remaining {
501 self.sleep_cancelable(d).await?;
502 }
503 write_stamp(&stamp_path)?;
504 Ok(())
505 }
506}
507
508fn safe_host_name(host: &str) -> String {
509 let safe: String = host
510 .chars()
511 .map(|c| {
512 if c.is_ascii_alphanumeric() || c == '.' || c == '-' {
513 c
514 } else {
515 '_'
516 }
517 })
518 .collect();
519 if safe.is_empty() {
520 "unknown".to_string()
521 } else {
522 safe
523 }
524}
525
526fn rate_limit_stamp_path(cache_dir: &Path, host: &str) -> PathBuf {
528 cache_dir
529 .join("rate-limit")
530 .join(format!("{}.stamp", safe_host_name(host)))
531}
532
533fn rate_limit_lock_path(cache_dir: &Path, host: &str) -> PathBuf {
535 cache_dir
536 .join("rate-limit")
537 .join(format!("{}.lock", safe_host_name(host)))
538}
539
540fn now_unix_ms() -> Option<u128> {
541 SystemTime::now()
542 .duration_since(UNIX_EPOCH)
543 .ok()
544 .map(|d| d.as_millis())
545}
546
547fn stamp_remaining(path: &Path, delay: Duration) -> Option<Duration> {
549 let text = fs::read_to_string(path).ok()?;
550 let last_ms: u128 = text.trim().parse().ok()?;
551 let now = now_unix_ms()?;
552 if now < last_ms {
553 return Some(delay);
554 }
555 let elapsed = Duration::from_millis((now - last_ms) as u64);
556 if elapsed < delay {
557 Some(delay - elapsed)
558 } else {
559 None
560 }
561}
562
563fn write_stamp(path: &Path) -> AppResult<()> {
565 if let Some(parent) = path.parent() {
566 fs::create_dir_all(parent).map_err(|e| {
567 AppError::with_source(ErrorKind::Network, "rate-limit stamp dir create failed", e)
568 })?;
569 }
570 let now = now_unix_ms()
571 .ok_or_else(|| AppError::new(ErrorKind::Internal, "system clock before UNIX epoch"))?;
572 let tmp = path.with_extension("stamp.tmp");
573 fs::write(&tmp, now.to_string()).map_err(|e| {
574 AppError::with_source(ErrorKind::Network, "rate-limit stamp write failed", e)
575 })?;
576 fs::rename(&tmp, path).map_err(|e| {
577 let _ = fs::remove_file(&tmp);
578 AppError::with_source(ErrorKind::Network, "rate-limit stamp rename failed", e)
579 })?;
580 crate::platform::restrict_private_file(path);
581 Ok(())
582}
583
584fn is_allowed_host(url: &Url) -> bool {
585 match url.host_str() {
586 Some(HOST_CRATES_IO)
587 | Some(HOST_DOCS_RS)
588 | Some(HOST_STATIC_DOCS_RS)
589 | Some(HOST_DOC_RUST_LANG_ORG) => url.scheme() == "https",
590 Some("127.0.0.1") | Some("localhost") => {
591 cfg!(test) || std::env::var("DOCSRS_CLI_ALLOW_LOCALHOST").is_ok()
592 }
593 _ => false,
594 }
595}
596
597async fn read_body_capped(resp: reqwest::Response, max_bytes: u64) -> AppResult<Bytes> {
605 let max_bytes = max_bytes.min(HARD_MAX_BODY_BYTES);
607 let content_length = resp.content_length();
610 if let Some(n) = content_length
611 && n > max_bytes
612 {
613 return Err(AppError::new(
614 ErrorKind::Network,
615 format!("response body exceeds max_body_bytes ({max_bytes})"),
616 ));
617 }
618 let capacity = content_length
621 .map(|n| (n as usize).min(max_bytes as usize))
622 .unwrap_or(0);
623 let mut stream = resp.bytes_stream();
624 let mut buf: Vec<u8> = Vec::new();
625 if capacity > 0 {
626 buf.try_reserve_exact(capacity).map_err(|e| {
627 AppError::with_source(
628 ErrorKind::Network,
629 format!("failed to reserve {capacity} bytes for response body"),
630 e,
631 )
632 })?;
633 }
634 while let Some(chunk) = stream.next().await {
635 let chunk = chunk.map_err(|e| {
636 AppError::with_source(ErrorKind::Network, "failed reading response body", e)
637 })?;
638 if (buf.len() as u64).saturating_add(chunk.len() as u64) > max_bytes {
639 return Err(AppError::new(
641 ErrorKind::Network,
642 format!("response body exceeds max_body_bytes ({max_bytes})"),
643 ));
644 }
645 let need = chunk.len();
647 if buf.capacity().saturating_sub(buf.len()) < need {
648 buf.try_reserve(need).map_err(|e| {
649 AppError::with_source(
650 ErrorKind::Network,
651 format!("failed to reserve {need} more body bytes"),
652 e,
653 )
654 })?;
655 }
656 buf.extend_from_slice(&chunk);
657 }
658 Ok(Bytes::from(buf))
659}
660
661pub fn decode_utf8(body: &Bytes) -> AppResult<String> {
667 let mut bytes = body.as_ref();
668 if bytes.starts_with(&[0xEF, 0xBB, 0xBF]) {
669 bytes = &bytes[3..];
670 }
671 std::str::from_utf8(bytes)
672 .map(|s| s.to_string())
673 .map_err(|e| AppError::with_source(ErrorKind::Parse, "response body is not valid UTF-8", e))
674}
675
676fn ascii_contains_ignore_case(haystack: &str, needle: &str) -> bool {
681 if needle.is_empty() {
682 return true;
683 }
684 let n = needle.as_bytes();
685 haystack
686 .as_bytes()
687 .windows(n.len())
688 .any(|w| w.eq_ignore_ascii_case(n))
689}
690
691pub fn content_type_looks_json(ct: Option<&str>) -> bool {
693 ct.is_some_and(|c| ascii_contains_ignore_case(c, "json"))
694}
695
696pub fn content_type_looks_html(ct: Option<&str>) -> bool {
698 ct.is_some_and(|c| {
699 ascii_contains_ignore_case(c, "html") || ascii_contains_ignore_case(c, "xhtml")
700 })
701}
702
703#[cfg(test)]
704mod tests {
705 use super::*;
706
707 #[test]
708 fn allowlist_https_only() {
709 let ok = Url::parse("https://docs.rs/serde").unwrap();
710 assert!(is_allowed_host(&ok));
711 let stdlib = Url::parse("https://doc.rust-lang.org/stable/std/").unwrap();
712 assert!(is_allowed_host(&stdlib));
713 let bad = Url::parse("http://docs.rs/serde").unwrap();
714 assert!(!is_allowed_host(&bad));
715 let evil = Url::parse("https://evil.example/").unwrap();
716 assert!(!is_allowed_host(&evil));
717 }
718
719 #[test]
720 fn utf8_bom_strip() {
721 let mut v = vec![0xEF, 0xBB, 0xBF];
722 v.extend_from_slice(b"hello");
723 let s = decode_utf8(&Bytes::from(v)).unwrap();
724 assert_eq!(s, "hello");
725 }
726
727 #[test]
728 fn content_types() {
729 assert!(content_type_looks_json(Some(
730 "application/json; charset=utf-8"
731 )));
732 assert!(content_type_looks_json(Some("Application/JSON")));
733 assert!(content_type_looks_html(Some("text/html")));
734 assert!(content_type_looks_html(Some("Text/HTML; charset=utf-8")));
735 assert!(content_type_looks_html(Some("application/xhtml+xml")));
736 assert!(!content_type_looks_json(Some("text/html")));
737 assert!(!content_type_looks_json(None));
738 assert!(!content_type_looks_html(None));
739 }
740
741 #[test]
742 fn ascii_contains_ignore_case_edges() {
743 assert!(ascii_contains_ignore_case("abJSON", "json"));
744 assert!(ascii_contains_ignore_case("json", "json"));
745 assert!(!ascii_contains_ignore_case("js", "json"));
746 assert!(ascii_contains_ignore_case("x", ""));
747 }
748
749 #[test]
750 fn retry_policy_kill_switch() {
751 let cfg = Config {
752 disable_retry: true,
753 max_retries: 5,
754 ..Config::default()
755 };
756 let http = HttpClient::new(cfg, CancelFlag::new()).unwrap();
757 assert!(!http.retry_policy().enabled);
758 assert_eq!(http.retry_policy().max_attempts(), 1);
759 }
760
761 #[test]
762 fn invalid_user_agent_rejected() {
763 let cfg = Config {
764 user_agent: "bad\nua".into(),
765 ..Config::default()
766 };
767 match HttpClient::new(cfg, CancelFlag::new()) {
768 Ok(_) => panic!("expected invalid user-agent to fail"),
769 Err(err) => assert_eq!(err.kind(), ErrorKind::Config),
770 }
771 }
772
773 #[test]
774 fn localhost_allowed_in_unit_tests() {
775 let ok = Url::parse("http://127.0.0.1:9/x").unwrap();
776 assert!(is_allowed_host(&ok));
777 let local = Url::parse("http://localhost:9/x").unwrap();
778 assert!(is_allowed_host(&local));
779 }
780
781 #[test]
782 fn decode_utf8_rejects_invalid() {
783 let err = decode_utf8(&Bytes::from(vec![0xff, 0xfe])).unwrap_err();
784 assert_eq!(err.kind(), ErrorKind::Parse);
785 }
786}