1use std::collections::HashMap;
40use std::fs::{self, File, OpenOptions};
41use std::path::{Path, PathBuf};
42use std::sync::Mutex;
43use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
44
45use bytes::Bytes;
46use fs4::FileExt as Fs4FileExt;
47use fs4::TryLockError as Fs4TryLockError;
48use futures_util::StreamExt;
49use reqwest::header::{ACCEPT, ACCEPT_ENCODING, HeaderMap, HeaderValue, USER_AGENT};
50use reqwest::{Client, Method, StatusCode, Url};
51use tracing::{debug, warn};
52
53use crate::cache::DiskCache;
54use crate::concurrency::ConcurrencyBudget;
55use crate::config::{
56 Config, HARD_MAX_BODY_BYTES, HOST_CRATES_IO, HOST_DOC_RUST_LANG_ORG, HOST_DOCS_RS,
57 HOST_STATIC_DOCS_RS,
58};
59use crate::error::{AppError, AppResult, ErrorKind};
60use crate::retry::{
61 RetryClass, RetryConfig, classify_http_status, parse_retry_after, politeness_delay,
62 wait_for_retry,
63};
64use crate::shutdown::CancelFlag;
65
66pub struct HttpClient {
71 client: Client,
72 cfg: Config,
73 last_host_hit: Mutex<HashMap<String, Instant>>,
75 cancel: CancelFlag,
76 cache: Option<DiskCache>,
77 budget: ConcurrencyBudget,
79 retry: RetryConfig,
81}
82
83#[derive(Debug, Clone)]
85pub struct HttpResponse {
86 pub status: StatusCode,
88 pub final_url: Url,
90 pub body: Bytes,
92 pub content_type: Option<String>,
94 pub cache_hit: bool,
96}
97
98impl HttpClient {
99 pub fn new(cfg: Config, cancel: CancelFlag) -> AppResult<Self> {
111 let mut cfg = cfg;
113 cfg.clamp_resource_limits();
114 let budget = ConcurrencyBudget::from_configured(cfg.max_concurrency);
115 let retry = RetryConfig::from_config(&cfg);
116
117 let mut headers = HeaderMap::new();
118 headers.insert(
119 USER_AGENT,
120 HeaderValue::from_str(&cfg.user_agent).map_err(|e| {
121 AppError::with_source(ErrorKind::Config, "invalid user-agent header", e)
122 })?,
123 );
124 headers.insert(ACCEPT_ENCODING, HeaderValue::from_static("gzip"));
125
126 let client = Client::builder()
127 .default_headers(headers)
128 .timeout(cfg.timeout())
129 .connect_timeout(cfg.connect_timeout())
130 .redirect(reqwest::redirect::Policy::custom({
131 let max = cfg.max_redirects;
132 move |attempt| {
133 if attempt.previous().len() as u32 >= max {
134 return attempt
135 .error(AppError::new(ErrorKind::Network, "redirect limit exceeded"));
136 }
137 let host = attempt.url().host_str().unwrap_or("");
139 let scheme_ok = attempt.url().scheme() == "https";
140 let host_ok = matches!(
141 host,
142 HOST_CRATES_IO
143 | HOST_DOCS_RS
144 | HOST_STATIC_DOCS_RS
145 | HOST_DOC_RUST_LANG_ORG
146 ) || ((host == "127.0.0.1" || host == "localhost")
147 && (cfg!(test) || std::env::var("DOCSRS_CLI_ALLOW_LOCALHOST").is_ok()));
148 if !(scheme_ok && host_ok) {
149 let msg = format!("redirect host not allowlisted: {host}");
151 return attempt.error(AppError::new(ErrorKind::Network, msg));
152 }
153 attempt.follow()
154 }
155 }))
156 .use_rustls_tls()
157 .build()
158 .map_err(|e| {
159 AppError::with_source(ErrorKind::Network, "failed to build HTTP client", e)
160 })?;
161
162 let cache = if cfg.cache_enabled() {
163 cfg.cache_dir
164 .as_ref()
165 .map(|dir| DiskCache::new(dir.clone(), cfg.cache_ttl(), cfg.max_cache_bytes))
166 } else {
167 None
168 };
169
170 Ok(Self {
171 client,
172 cfg,
173 last_host_hit: Mutex::new(HashMap::new()),
174 cancel,
175 cache,
176 budget,
177 retry,
178 })
179 }
180
181 pub fn budget(&self) -> &ConcurrencyBudget {
183 &self.budget
184 }
185
186 pub fn retry_policy(&self) -> RetryConfig {
188 self.retry
189 }
190
191 pub async fn get_json(&self, url: &Url) -> AppResult<HttpResponse> {
201 self.request(Method::GET, url, "application/json", true)
202 .await
203 }
204
205 pub async fn get_html(&self, url: &Url) -> AppResult<HttpResponse> {
211 self.request(Method::GET, url, "text/html", true).await
212 }
213
214 async fn request(
215 &self,
216 method: Method,
217 url: &Url,
218 accept: &str,
219 retryable: bool,
220 ) -> AppResult<HttpResponse> {
221 if !is_allowed_host(url) {
222 return Err(AppError::new(
223 ErrorKind::Network,
224 format!("host not allowlisted: {}", url.host_str().unwrap_or("")),
225 ));
226 }
227
228 if method == Method::GET
230 && let Some(cache) = &self.cache
231 && let Some(hit) = cache.get(url, accept)
232 {
233 return Ok(hit);
234 }
235
236 let policy = if retryable {
239 self.retry
240 } else {
241 RetryConfig {
242 enabled: false,
243 ..self.retry
244 }
245 };
246 let url_str = url.as_str();
249 let mut attempt = 0u32;
250 loop {
251 self.cancel.check()?;
252 attempt += 1;
253 self.rate_limit(url).await?;
254 self.cancel.check()?;
255
256 let mut req = self.client.request(method.clone(), url_str);
257 req = req.header(ACCEPT, accept);
258
259 let result = req.send().await;
260 match result {
261 Ok(resp) => {
262 let status = resp.status();
263 let final_url = resp.url().clone();
264 let content_type = resp
265 .headers()
266 .get(reqwest::header::CONTENT_TYPE)
267 .and_then(|v| v.to_str().ok())
268 .map(|s| s.to_string());
269 let retry_after = parse_retry_after(resp.headers());
270 let class = classify_http_status(status, retry_after);
271
272 match class {
273 RetryClass::RateLimited { retry_after } if policy.may_retry(attempt) => {
274 let wait = wait_for_retry(policy, attempt, retry_after);
275 debug!(
276 target: "docsrs_cli::retry",
277 ?wait,
278 attempt,
279 max_attempts = policy.max_attempts(),
280 status = %status,
281 "retrying after rate-limit / Retry-After"
282 );
283 self.sleep_cancelable(wait).await?;
284 continue;
285 }
286 RetryClass::Transient if policy.may_retry(attempt) => {
287 let wait = wait_for_retry(policy, attempt, None);
288 debug!(
289 target: "docsrs_cli::retry",
290 ?wait,
291 attempt,
292 max_attempts = policy.max_attempts(),
293 status = %status,
294 "retrying after transient 5xx"
295 );
296 self.sleep_cancelable(wait).await?;
297 continue;
298 }
299 RetryClass::RateLimited { retry_after } => {
300 let secs = retry_after.map(|d| d.as_secs()).unwrap_or(1);
301 return Err(AppError::from_http_status(
302 status.as_u16(),
303 "retries exhausted",
304 )
305 .with_retry_after(secs.max(1)));
306 }
307 RetryClass::Transient => {
308 return Err(AppError::from_http_status(
310 status.as_u16(),
311 "retries exhausted",
312 ));
313 }
314 RetryClass::Permanent => {
315 }
317 }
318
319 let body = read_body_capped(resp, self.cfg.max_body_bytes).await?;
320 let out = HttpResponse {
321 status,
322 final_url,
323 body,
324 content_type,
325 cache_hit: false,
326 };
327 if method == Method::GET
328 && out.status.is_success()
329 && let Some(cache) = &self.cache
330 && let Err(e) = cache.put(url, accept, &out)
331 {
332 warn!(error = %e, "failed to write HTTP cache entry");
333 }
334 return Ok(out);
335 }
336 Err(e) => {
337 let is_timeout = e.is_timeout();
339 let is_connect = e.is_connect();
340 let is_request = e.is_request();
341 let kind = if is_timeout {
342 ErrorKind::Timeout
343 } else {
344 ErrorKind::Network
345 };
346 let transport_retryable = is_timeout || is_connect || is_request;
347 if policy.may_retry(attempt) && transport_retryable {
348 let wait = wait_for_retry(policy, attempt, None);
349 warn!(
350 target: "docsrs_cli::retry",
351 ?wait,
352 attempt,
353 max_attempts = policy.max_attempts(),
354 error = %e,
355 is_timeout,
356 is_connect,
357 "retrying after transport error"
358 );
359 self.sleep_cancelable(wait).await?;
360 continue;
361 }
362 return Err(AppError::with_source(
363 kind,
364 format!("HTTP request failed for {url}"),
365 e,
366 ));
367 }
368 }
369 }
370 }
371
372 async fn sleep_cancelable(&self, wait: Duration) -> AppResult<()> {
373 let step = Duration::from_millis(50);
374 let mut remaining = wait;
375 while !remaining.is_zero() {
376 self.cancel.check()?;
377 let slice = remaining.min(step);
378 tokio::time::sleep(slice).await;
379 remaining = remaining.saturating_sub(slice);
380 }
381 self.cancel.check()
382 }
383
384 async fn rate_limit(&self, url: &Url) -> AppResult<()> {
392 let host = url.host_str().unwrap_or("");
394 let delay = politeness_delay(self.cfg.rate_limit_delay());
395 if delay.is_zero() {
396 return Ok(());
397 }
398
399 if let Some(dir) = self.cfg.cache_dir.as_ref() {
401 match self.rate_limit_cross_process(dir, host, delay).await {
402 Ok(()) => {
403 self.touch_host_clock(host);
404 return Ok(());
405 }
406 Err(e)
407 if e.kind() == ErrorKind::Terminated || e.kind() == ErrorKind::Interrupted =>
408 {
409 return Err(e);
410 }
411 Err(e) => {
412 warn!(error = %e, host = %host, "cross-process rate-limit lock failed; using in-process only");
414 }
415 }
416 }
417
418 let in_proc = {
419 let map = self
420 .last_host_hit
421 .lock()
422 .unwrap_or_else(|e| e.into_inner());
423 map.get(host).and_then(|prev| {
424 let elapsed = prev.elapsed();
425 if elapsed < delay {
426 Some(delay - elapsed)
427 } else {
428 None
429 }
430 })
431 };
432 if let Some(d) = in_proc {
433 self.sleep_cancelable(d).await?;
434 }
435 self.touch_host_clock(host);
436 Ok(())
437 }
438
439 fn touch_host_clock(&self, host: &str) {
440 let mut map = self
441 .last_host_hit
442 .lock()
443 .unwrap_or_else(|e| e.into_inner());
444 map.insert(host.to_string(), Instant::now());
445 }
446
447 async fn rate_limit_cross_process(
449 &self,
450 cache_dir: &Path,
451 host: &str,
452 delay: Duration,
453 ) -> AppResult<()> {
454 let lock_path = rate_limit_lock_path(cache_dir, host);
455 let stamp_path = rate_limit_stamp_path(cache_dir, host);
456 if let Some(parent) = lock_path.parent() {
457 fs::create_dir_all(parent).map_err(|e| {
458 AppError::with_source(ErrorKind::Network, "rate-limit lock dir create failed", e)
459 })?;
460 crate::platform::restrict_private_dir(parent);
461 }
462
463 let file = OpenOptions::new()
464 .create(true)
465 .read(true)
466 .write(true)
467 .truncate(false)
468 .open(&lock_path)
469 .map_err(|e| {
470 AppError::with_source(ErrorKind::Network, "rate-limit lock open failed", e)
471 })?;
472 crate::platform::restrict_private_file(&lock_path);
473
474 loop {
477 self.cancel.check()?;
478 match Fs4FileExt::try_lock(&file) {
479 Ok(()) => break,
480 Err(Fs4TryLockError::WouldBlock) => {
481 self.sleep_cancelable(Duration::from_millis(50)).await?;
482 }
483 Err(Fs4TryLockError::Error(e)) => {
484 return Err(AppError::with_source(
485 ErrorKind::Network,
486 "rate-limit exclusive lock failed",
487 e,
488 ));
489 }
490 }
491 }
492
493 struct UnlockOnDrop<'a>(&'a File);
495 impl Drop for UnlockOnDrop<'_> {
496 fn drop(&mut self) {
497 let _ = Fs4FileExt::unlock(self.0);
498 }
499 }
500 let _guard = UnlockOnDrop(&file);
501
502 let remaining = stamp_remaining(&stamp_path, delay);
503 if let Some(d) = remaining {
504 self.sleep_cancelable(d).await?;
505 }
506 write_stamp(&stamp_path)?;
507 Ok(())
508 }
509}
510
511fn safe_host_name(host: &str) -> String {
512 let safe: String = host
513 .chars()
514 .map(|c| {
515 if c.is_ascii_alphanumeric() || c == '.' || c == '-' {
516 c
517 } else {
518 '_'
519 }
520 })
521 .collect();
522 if safe.is_empty() {
523 "unknown".to_string()
524 } else {
525 safe
526 }
527}
528
529fn rate_limit_stamp_path(cache_dir: &Path, host: &str) -> PathBuf {
531 cache_dir
532 .join("rate-limit")
533 .join(format!("{}.stamp", safe_host_name(host)))
534}
535
536fn rate_limit_lock_path(cache_dir: &Path, host: &str) -> PathBuf {
538 cache_dir
539 .join("rate-limit")
540 .join(format!("{}.lock", safe_host_name(host)))
541}
542
543fn now_unix_ms() -> Option<u128> {
544 SystemTime::now()
545 .duration_since(UNIX_EPOCH)
546 .ok()
547 .map(|d| d.as_millis())
548}
549
550fn stamp_remaining(path: &Path, delay: Duration) -> Option<Duration> {
552 let text = fs::read_to_string(path).ok()?;
553 let last_ms: u128 = text.trim().parse().ok()?;
554 let now = now_unix_ms()?;
555 if now < last_ms {
556 return Some(delay);
557 }
558 let elapsed = Duration::from_millis((now - last_ms) as u64);
559 if elapsed < delay {
560 Some(delay - elapsed)
561 } else {
562 None
563 }
564}
565
566fn write_stamp(path: &Path) -> AppResult<()> {
568 if let Some(parent) = path.parent() {
569 fs::create_dir_all(parent).map_err(|e| {
570 AppError::with_source(ErrorKind::Network, "rate-limit stamp dir create failed", e)
571 })?;
572 }
573 let now = now_unix_ms()
574 .ok_or_else(|| AppError::new(ErrorKind::Internal, "system clock before UNIX epoch"))?;
575 let tmp = path.with_extension("stamp.tmp");
576 fs::write(&tmp, now.to_string()).map_err(|e| {
577 AppError::with_source(ErrorKind::Network, "rate-limit stamp write failed", e)
578 })?;
579 fs::rename(&tmp, path).map_err(|e| {
580 let _ = fs::remove_file(&tmp);
581 AppError::with_source(ErrorKind::Network, "rate-limit stamp rename failed", e)
582 })?;
583 crate::platform::restrict_private_file(path);
584 Ok(())
585}
586
587fn is_allowed_host(url: &Url) -> bool {
588 match url.host_str() {
589 Some(HOST_CRATES_IO)
590 | Some(HOST_DOCS_RS)
591 | Some(HOST_STATIC_DOCS_RS)
592 | Some(HOST_DOC_RUST_LANG_ORG) => url.scheme() == "https",
593 Some("127.0.0.1") | Some("localhost") => {
594 cfg!(test) || std::env::var("DOCSRS_CLI_ALLOW_LOCALHOST").is_ok()
595 }
596 _ => false,
597 }
598}
599
600async fn read_body_capped(resp: reqwest::Response, max_bytes: u64) -> AppResult<Bytes> {
608 let max_bytes = max_bytes.min(HARD_MAX_BODY_BYTES);
610 let content_length = resp.content_length();
613 if let Some(n) = content_length
614 && n > max_bytes
615 {
616 return Err(AppError::new(
617 ErrorKind::Network,
618 format!("response body exceeds max_body_bytes ({max_bytes})"),
619 ));
620 }
621 let capacity = content_length
624 .map(|n| (n as usize).min(max_bytes as usize))
625 .unwrap_or(0);
626 let mut stream = resp.bytes_stream();
627 let mut buf: Vec<u8> = Vec::new();
628 if capacity > 0 {
629 buf.try_reserve_exact(capacity).map_err(|e| {
630 AppError::with_source(
631 ErrorKind::Network,
632 format!("failed to reserve {capacity} bytes for response body"),
633 e,
634 )
635 })?;
636 }
637 while let Some(chunk) = stream.next().await {
638 let chunk = chunk.map_err(|e| {
639 AppError::with_source(ErrorKind::Network, "failed reading response body", e)
640 })?;
641 if (buf.len() as u64).saturating_add(chunk.len() as u64) > max_bytes {
642 return Err(AppError::new(
644 ErrorKind::Network,
645 format!("response body exceeds max_body_bytes ({max_bytes})"),
646 ));
647 }
648 let need = chunk.len();
650 if buf.capacity().saturating_sub(buf.len()) < need {
651 buf.try_reserve(need).map_err(|e| {
652 AppError::with_source(
653 ErrorKind::Network,
654 format!("failed to reserve {need} more body bytes"),
655 e,
656 )
657 })?;
658 }
659 buf.extend_from_slice(&chunk);
660 }
661 Ok(Bytes::from(buf))
662}
663
664pub fn decode_utf8(body: &Bytes) -> AppResult<String> {
670 let mut bytes = body.as_ref();
671 if bytes.starts_with(&[0xEF, 0xBB, 0xBF]) {
672 bytes = &bytes[3..];
673 }
674 std::str::from_utf8(bytes)
675 .map(|s| s.to_string())
676 .map_err(|e| AppError::with_source(ErrorKind::Parse, "response body is not valid UTF-8", e))
677}
678
679fn ascii_contains_ignore_case(haystack: &str, needle: &str) -> bool {
684 if needle.is_empty() {
685 return true;
686 }
687 let n = needle.as_bytes();
688 haystack
689 .as_bytes()
690 .windows(n.len())
691 .any(|w| w.eq_ignore_ascii_case(n))
692}
693
694pub fn content_type_looks_json(ct: Option<&str>) -> bool {
696 ct.is_some_and(|c| ascii_contains_ignore_case(c, "json"))
697}
698
699pub fn content_type_looks_html(ct: Option<&str>) -> bool {
701 ct.is_some_and(|c| {
702 ascii_contains_ignore_case(c, "html") || ascii_contains_ignore_case(c, "xhtml")
703 })
704}
705
706#[cfg(test)]
707mod tests {
708 use super::*;
709
710 #[test]
711 fn allowlist_https_only() {
712 let ok = Url::parse("https://docs.rs/serde").unwrap();
713 assert!(is_allowed_host(&ok));
714 let stdlib = Url::parse("https://doc.rust-lang.org/stable/std/").unwrap();
715 assert!(is_allowed_host(&stdlib));
716 let bad = Url::parse("http://docs.rs/serde").unwrap();
717 assert!(!is_allowed_host(&bad));
718 let evil = Url::parse("https://evil.example/").unwrap();
719 assert!(!is_allowed_host(&evil));
720 }
721
722 #[test]
723 fn utf8_bom_strip() {
724 let mut v = vec![0xEF, 0xBB, 0xBF];
725 v.extend_from_slice(b"hello");
726 let s = decode_utf8(&Bytes::from(v)).unwrap();
727 assert_eq!(s, "hello");
728 }
729
730 #[test]
731 fn content_types() {
732 assert!(content_type_looks_json(Some(
733 "application/json; charset=utf-8"
734 )));
735 assert!(content_type_looks_json(Some("Application/JSON")));
736 assert!(content_type_looks_html(Some("text/html")));
737 assert!(content_type_looks_html(Some("Text/HTML; charset=utf-8")));
738 assert!(content_type_looks_html(Some("application/xhtml+xml")));
739 assert!(!content_type_looks_json(Some("text/html")));
740 assert!(!content_type_looks_json(None));
741 assert!(!content_type_looks_html(None));
742 }
743
744 #[test]
745 fn ascii_contains_ignore_case_edges() {
746 assert!(ascii_contains_ignore_case("abJSON", "json"));
747 assert!(ascii_contains_ignore_case("json", "json"));
748 assert!(!ascii_contains_ignore_case("js", "json"));
749 assert!(ascii_contains_ignore_case("x", ""));
750 }
751
752 #[test]
753 fn retry_policy_kill_switch() {
754 let cfg = Config {
755 disable_retry: true,
756 max_retries: 5,
757 ..Config::default()
758 };
759 let http = HttpClient::new(cfg, CancelFlag::new()).unwrap();
760 assert!(!http.retry_policy().enabled);
761 assert_eq!(http.retry_policy().max_attempts(), 1);
762 }
763
764 #[test]
765 fn invalid_user_agent_rejected() {
766 let cfg = Config {
767 user_agent: "bad\nua".into(),
768 ..Config::default()
769 };
770 match HttpClient::new(cfg, CancelFlag::new()) {
771 Ok(_) => panic!("expected invalid user-agent to fail"),
772 Err(err) => assert_eq!(err.kind(), ErrorKind::Config),
773 }
774 }
775
776 #[test]
777 fn localhost_allowed_in_unit_tests() {
778 let ok = Url::parse("http://127.0.0.1:9/x").unwrap();
779 assert!(is_allowed_host(&ok));
780 let local = Url::parse("http://localhost:9/x").unwrap();
781 assert!(is_allowed_host(&local));
782 }
783
784 #[test]
785 fn decode_utf8_rejects_invalid() {
786 let err = decode_utf8(&Bytes::from(vec![0xff, 0xfe])).unwrap_err();
787 assert_eq!(err.kind(), ErrorKind::Parse);
788 }
789}