1use std::collections::HashMap;
40use std::fs::{self, File, OpenOptions};
41use std::path::{Path, PathBuf};
42use std::sync::Mutex;
43use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
44
45use bytes::Bytes;
46use fs4::FileExt as Fs4FileExt;
47use fs4::TryLockError as Fs4TryLockError;
48use futures_util::StreamExt;
49use reqwest::header::{ACCEPT, ACCEPT_ENCODING, HeaderMap, HeaderValue, USER_AGENT};
50use reqwest::{Client, Method, StatusCode, Url};
51use tracing::{debug, warn};
52
53use crate::cache::DiskCache;
54use crate::concurrency::ConcurrencyBudget;
55use crate::config::{
56 Config, HARD_MAX_BODY_BYTES, HOST_CRATES_IO, HOST_DOC_RUST_LANG_ORG, HOST_DOCS_RS,
57 HOST_STATIC_DOCS_RS,
58};
59use crate::error::{AppError, AppResult, ErrorKind};
60use crate::retry::{
61 RetryClass, RetryConfig, classify_http_status, parse_retry_after, politeness_delay,
62 wait_for_retry,
63};
64use crate::shutdown::CancelFlag;
65
66pub struct HttpClient {
71 client: Client,
72 cfg: Config,
73 last_host_hit: Mutex<HashMap<String, Instant>>,
75 cancel: CancelFlag,
76 cache: Option<DiskCache>,
77 budget: ConcurrencyBudget,
79 retry: RetryConfig,
81}
82
83#[derive(Debug, Clone)]
85pub struct HttpResponse {
86 pub status: StatusCode,
88 pub final_url: Url,
90 pub body: Bytes,
92 pub content_type: Option<String>,
94 pub cache_hit: bool,
96}
97
98impl HttpClient {
99 pub fn new(cfg: Config, cancel: CancelFlag) -> AppResult<Self> {
111 let mut cfg = cfg;
113 cfg.clamp_resource_limits();
114 let budget = ConcurrencyBudget::from_configured(cfg.max_concurrency);
115 let retry = RetryConfig::from_config(&cfg);
116
117 let mut headers = HeaderMap::new();
118 headers.insert(
119 USER_AGENT,
120 HeaderValue::from_str(&cfg.user_agent).map_err(|e| {
121 AppError::with_source(ErrorKind::Config, "invalid user-agent header", e)
122 })?,
123 );
124 headers.insert(ACCEPT_ENCODING, HeaderValue::from_static("gzip"));
125
126 let client = Client::builder()
127 .default_headers(headers)
128 .timeout(cfg.timeout())
129 .connect_timeout(cfg.connect_timeout())
130 .redirect(reqwest::redirect::Policy::custom({
131 let max = cfg.max_redirects;
132 move |attempt| {
133 if attempt.previous().len() as u32 >= max {
134 return attempt
135 .error(AppError::new(ErrorKind::Network, "redirect limit exceeded"));
136 }
137 let host = attempt.url().host_str().unwrap_or("");
139 let scheme_ok = attempt.url().scheme() == "https";
140 let host_ok = matches!(
141 host,
142 HOST_CRATES_IO
143 | HOST_DOCS_RS
144 | HOST_STATIC_DOCS_RS
145 | HOST_DOC_RUST_LANG_ORG
146 ) || ((host == "127.0.0.1" || host == "localhost")
147 && (cfg!(test) || std::env::var("DOCSRS_CLI_ALLOW_LOCALHOST").is_ok()));
148 if !(scheme_ok && host_ok) {
149 let msg = format!("redirect host not allowlisted: {host}");
151 return attempt.error(AppError::new(ErrorKind::Network, msg));
152 }
153 attempt.follow()
154 }
155 }))
156 .use_rustls_tls()
157 .build()
158 .map_err(|e| {
159 AppError::with_source(ErrorKind::Network, "failed to build HTTP client", e)
160 })?;
161
162 let cache = if cfg.cache_enabled() {
163 cfg.cache_dir
164 .as_ref()
165 .map(|dir| DiskCache::new(dir.clone(), cfg.cache_ttl(), cfg.max_cache_bytes))
166 } else {
167 None
168 };
169
170 Ok(Self {
171 client,
172 cfg,
173 last_host_hit: Mutex::new(HashMap::new()),
174 cancel,
175 cache,
176 budget,
177 retry,
178 })
179 }
180
181 pub fn budget(&self) -> &ConcurrencyBudget {
183 &self.budget
184 }
185
186 pub fn retry_policy(&self) -> RetryConfig {
188 self.retry
189 }
190
191 pub async fn get_json(&self, url: &Url) -> AppResult<HttpResponse> {
201 self.request(Method::GET, url, "application/json", true)
202 .await
203 }
204
205 pub async fn get_html(&self, url: &Url) -> AppResult<HttpResponse> {
211 self.request(Method::GET, url, "text/html", true).await
212 }
213
214 async fn request(
215 &self,
216 method: Method,
217 url: &Url,
218 accept: &str,
219 retryable: bool,
220 ) -> AppResult<HttpResponse> {
221 if !is_allowed_host(url) {
222 return Err(AppError::new(
223 ErrorKind::Network,
224 format!("host not allowlisted: {}", url.host_str().unwrap_or("")),
225 ));
226 }
227
228 if method == Method::GET
230 && let Some(cache) = &self.cache
231 && let Some(hit) = cache.get(url, accept)
232 {
233 return Ok(hit);
234 }
235
236 let policy = if retryable {
239 self.retry
240 } else {
241 RetryConfig {
242 enabled: false,
243 ..self.retry
244 }
245 };
246 let url_str = url.as_str();
249 let mut attempt = 0u32;
250 loop {
251 self.cancel.check()?;
252 attempt += 1;
253 self.rate_limit(url).await?;
254 self.cancel.check()?;
255
256 let mut req = self.client.request(method.clone(), url_str);
257 req = req.header(ACCEPT, accept);
258
259 let result = req.send().await;
260 match result {
261 Ok(resp) => {
262 let status = resp.status();
263 let final_url = resp.url().clone();
264 let content_type = resp
265 .headers()
266 .get(reqwest::header::CONTENT_TYPE)
267 .and_then(|v| v.to_str().ok())
268 .map(|s| s.to_string());
269 let retry_after = parse_retry_after(resp.headers());
270 let class = classify_http_status(status, retry_after);
271
272 match class {
273 RetryClass::RateLimited { retry_after } if policy.may_retry(attempt) => {
274 let wait = wait_for_retry(policy, attempt, retry_after);
275 debug!(
276 target: "docsrs_cli::retry",
277 ?wait,
278 attempt,
279 max_attempts = policy.max_attempts(),
280 status = %status,
281 "retrying after rate-limit / Retry-After"
282 );
283 self.sleep_cancelable(wait).await?;
284 continue;
285 }
286 RetryClass::Transient if policy.may_retry(attempt) => {
287 let wait = wait_for_retry(policy, attempt, None);
288 debug!(
289 target: "docsrs_cli::retry",
290 ?wait,
291 attempt,
292 max_attempts = policy.max_attempts(),
293 status = %status,
294 "retrying after transient 5xx"
295 );
296 self.sleep_cancelable(wait).await?;
297 continue;
298 }
299 RetryClass::RateLimited { retry_after } => {
300 let secs = retry_after.map(|d| d.as_secs()).unwrap_or(1);
301 return Err(AppError::from_http_status(
302 status.as_u16(),
303 "retries exhausted",
304 )
305 .with_retry_after(secs.max(1)));
306 }
307 RetryClass::Transient => {
308 return Err(AppError::from_http_status(
310 status.as_u16(),
311 "retries exhausted",
312 ));
313 }
314 RetryClass::Permanent => {
315 }
317 }
318
319 let body = read_body_capped(resp, self.cfg.max_body_bytes).await?;
320 let out = HttpResponse {
321 status,
322 final_url,
323 body,
324 content_type,
325 cache_hit: false,
326 };
327 if method == Method::GET
328 && out.status.is_success()
329 && let Some(cache) = &self.cache
330 && let Err(e) = cache.put(url, accept, &out)
331 {
332 warn!(error = %e, "failed to write HTTP cache entry");
333 }
334 return Ok(out);
335 }
336 Err(e) => {
337 let is_timeout = e.is_timeout();
339 let is_connect = e.is_connect();
340 let is_request = e.is_request();
341 let kind = if is_timeout {
342 ErrorKind::Timeout
343 } else {
344 ErrorKind::Network
345 };
346 let transport_retryable = is_timeout || is_connect || is_request;
347 if policy.may_retry(attempt) && transport_retryable {
348 let wait = wait_for_retry(policy, attempt, None);
349 warn!(
350 target: "docsrs_cli::retry",
351 ?wait,
352 attempt,
353 max_attempts = policy.max_attempts(),
354 error = %e,
355 is_timeout,
356 is_connect,
357 "retrying after transport error"
358 );
359 self.sleep_cancelable(wait).await?;
360 continue;
361 }
362 return Err(AppError::with_source(
363 kind,
364 format!("HTTP request failed for {url}"),
365 e,
366 ));
367 }
368 }
369 }
370 }
371
372 async fn sleep_cancelable(&self, wait: Duration) -> AppResult<()> {
373 let step = Duration::from_millis(50);
374 let mut remaining = wait;
375 while !remaining.is_zero() {
376 self.cancel.check()?;
377 let slice = remaining.min(step);
378 tokio::time::sleep(slice).await;
379 remaining = remaining.saturating_sub(slice);
380 }
381 self.cancel.check()
382 }
383
384 async fn rate_limit(&self, url: &Url) -> AppResult<()> {
392 let host = url.host_str().unwrap_or("");
394 let delay = politeness_delay(self.cfg.rate_limit_delay());
395 if delay.is_zero() {
396 return Ok(());
397 }
398
399 if let Some(dir) = self.cfg.cache_dir.as_ref() {
401 match self.rate_limit_cross_process(dir, host, delay).await {
402 Ok(()) => {
403 self.touch_host_clock(host);
404 return Ok(());
405 }
406 Err(e)
407 if e.kind() == ErrorKind::Terminated || e.kind() == ErrorKind::Interrupted =>
408 {
409 return Err(e);
410 }
411 Err(e) => {
412 warn!(error = %e, host = %host, "cross-process rate-limit lock failed; using in-process only");
414 }
415 }
416 }
417
418 let in_proc = {
419 let map = self.last_host_hit.lock().unwrap_or_else(|e| e.into_inner());
420 map.get(host).and_then(|prev| {
421 let elapsed = prev.elapsed();
422 if elapsed < delay {
423 Some(delay - elapsed)
424 } else {
425 None
426 }
427 })
428 };
429 if let Some(d) = in_proc {
430 self.sleep_cancelable(d).await?;
431 }
432 self.touch_host_clock(host);
433 Ok(())
434 }
435
436 fn touch_host_clock(&self, host: &str) {
437 let mut map = self.last_host_hit.lock().unwrap_or_else(|e| e.into_inner());
438 map.insert(host.to_string(), Instant::now());
439 }
440
441 async fn rate_limit_cross_process(
443 &self,
444 cache_dir: &Path,
445 host: &str,
446 delay: Duration,
447 ) -> AppResult<()> {
448 let lock_path = rate_limit_lock_path(cache_dir, host);
449 let stamp_path = rate_limit_stamp_path(cache_dir, host);
450 if let Some(parent) = lock_path.parent() {
451 fs::create_dir_all(parent).map_err(|e| {
452 AppError::with_source(ErrorKind::Network, "rate-limit lock dir create failed", e)
453 })?;
454 crate::platform::restrict_private_dir(parent);
455 }
456
457 let file = OpenOptions::new()
458 .create(true)
459 .read(true)
460 .write(true)
461 .truncate(false)
462 .open(&lock_path)
463 .map_err(|e| {
464 AppError::with_source(ErrorKind::Network, "rate-limit lock open failed", e)
465 })?;
466 crate::platform::restrict_private_file(&lock_path);
467
468 loop {
471 self.cancel.check()?;
472 match Fs4FileExt::try_lock(&file) {
473 Ok(()) => break,
474 Err(Fs4TryLockError::WouldBlock) => {
475 self.sleep_cancelable(Duration::from_millis(50)).await?;
476 }
477 Err(Fs4TryLockError::Error(e)) => {
478 return Err(AppError::with_source(
479 ErrorKind::Network,
480 "rate-limit exclusive lock failed",
481 e,
482 ));
483 }
484 }
485 }
486
487 struct UnlockOnDrop<'a>(&'a File);
489 impl Drop for UnlockOnDrop<'_> {
490 fn drop(&mut self) {
491 let _ = Fs4FileExt::unlock(self.0);
492 }
493 }
494 let _guard = UnlockOnDrop(&file);
495
496 let remaining = stamp_remaining(&stamp_path, delay);
497 if let Some(d) = remaining {
498 self.sleep_cancelable(d).await?;
499 }
500 write_stamp(&stamp_path)?;
501 Ok(())
502 }
503}
504
505fn safe_host_name(host: &str) -> String {
506 let safe: String = host
507 .chars()
508 .map(|c| {
509 if c.is_ascii_alphanumeric() || c == '.' || c == '-' {
510 c
511 } else {
512 '_'
513 }
514 })
515 .collect();
516 if safe.is_empty() {
517 "unknown".to_string()
518 } else {
519 safe
520 }
521}
522
523fn rate_limit_stamp_path(cache_dir: &Path, host: &str) -> PathBuf {
525 cache_dir
526 .join("rate-limit")
527 .join(format!("{}.stamp", safe_host_name(host)))
528}
529
530fn rate_limit_lock_path(cache_dir: &Path, host: &str) -> PathBuf {
532 cache_dir
533 .join("rate-limit")
534 .join(format!("{}.lock", safe_host_name(host)))
535}
536
537fn now_unix_ms() -> Option<u128> {
538 SystemTime::now()
539 .duration_since(UNIX_EPOCH)
540 .ok()
541 .map(|d| d.as_millis())
542}
543
544fn stamp_remaining(path: &Path, delay: Duration) -> Option<Duration> {
546 let text = fs::read_to_string(path).ok()?;
547 let last_ms: u128 = text.trim().parse().ok()?;
548 let now = now_unix_ms()?;
549 if now < last_ms {
550 return Some(delay);
551 }
552 let elapsed = Duration::from_millis((now - last_ms) as u64);
553 if elapsed < delay {
554 Some(delay - elapsed)
555 } else {
556 None
557 }
558}
559
560fn write_stamp(path: &Path) -> AppResult<()> {
562 if let Some(parent) = path.parent() {
563 fs::create_dir_all(parent).map_err(|e| {
564 AppError::with_source(ErrorKind::Network, "rate-limit stamp dir create failed", e)
565 })?;
566 }
567 let now = now_unix_ms()
568 .ok_or_else(|| AppError::new(ErrorKind::Internal, "system clock before UNIX epoch"))?;
569 let tmp = path.with_extension("stamp.tmp");
570 fs::write(&tmp, now.to_string()).map_err(|e| {
571 AppError::with_source(ErrorKind::Network, "rate-limit stamp write failed", e)
572 })?;
573 fs::rename(&tmp, path).map_err(|e| {
574 let _ = fs::remove_file(&tmp);
575 AppError::with_source(ErrorKind::Network, "rate-limit stamp rename failed", e)
576 })?;
577 crate::platform::restrict_private_file(path);
578 Ok(())
579}
580
581fn is_allowed_host(url: &Url) -> bool {
582 match url.host_str() {
583 Some(HOST_CRATES_IO)
584 | Some(HOST_DOCS_RS)
585 | Some(HOST_STATIC_DOCS_RS)
586 | Some(HOST_DOC_RUST_LANG_ORG) => url.scheme() == "https",
587 Some("127.0.0.1") | Some("localhost") => {
588 cfg!(test) || std::env::var("DOCSRS_CLI_ALLOW_LOCALHOST").is_ok()
589 }
590 _ => false,
591 }
592}
593
594async fn read_body_capped(resp: reqwest::Response, max_bytes: u64) -> AppResult<Bytes> {
602 let max_bytes = max_bytes.min(HARD_MAX_BODY_BYTES);
604 let content_length = resp.content_length();
607 if let Some(n) = content_length
608 && n > max_bytes
609 {
610 return Err(AppError::new(
612 ErrorKind::Budget,
613 format!("response body exceeds max_body_bytes ({max_bytes})"),
614 ));
615 }
616 let capacity = content_length
619 .map(|n| (n as usize).min(max_bytes as usize))
620 .unwrap_or(0);
621 let mut stream = resp.bytes_stream();
622 let mut buf: Vec<u8> = Vec::new();
623 if capacity > 0 {
624 buf.try_reserve_exact(capacity).map_err(|e| {
625 AppError::with_source(
626 ErrorKind::Network,
627 format!("failed to reserve {capacity} bytes for response body"),
628 e,
629 )
630 })?;
631 }
632 while let Some(chunk) = stream.next().await {
633 let chunk = chunk.map_err(|e| {
634 AppError::with_source(ErrorKind::Network, "failed reading response body", e)
635 })?;
636 if (buf.len() as u64).saturating_add(chunk.len() as u64) > max_bytes {
637 return Err(AppError::new(
639 ErrorKind::Budget,
640 format!("response body exceeds max_body_bytes ({max_bytes})"),
641 ));
642 }
643 let need = chunk.len();
645 if buf.capacity().saturating_sub(buf.len()) < need {
646 buf.try_reserve(need).map_err(|e| {
647 AppError::with_source(
648 ErrorKind::Network,
649 format!("failed to reserve {need} more body bytes"),
650 e,
651 )
652 })?;
653 }
654 buf.extend_from_slice(&chunk);
655 }
656 Ok(Bytes::from(buf))
657}
658
659pub fn decode_utf8(body: &Bytes) -> AppResult<String> {
665 let mut bytes = body.as_ref();
666 if bytes.starts_with(&[0xEF, 0xBB, 0xBF]) {
667 bytes = &bytes[3..];
668 }
669 std::str::from_utf8(bytes)
670 .map(|s| s.to_string())
671 .map_err(|e| AppError::with_source(ErrorKind::Parse, "response body is not valid UTF-8", e))
672}
673
674fn ascii_contains_ignore_case(haystack: &str, needle: &str) -> bool {
679 if needle.is_empty() {
680 return true;
681 }
682 let n = needle.as_bytes();
683 haystack
684 .as_bytes()
685 .windows(n.len())
686 .any(|w| w.eq_ignore_ascii_case(n))
687}
688
689pub fn content_type_looks_json(ct: Option<&str>) -> bool {
691 ct.is_some_and(|c| ascii_contains_ignore_case(c, "json"))
692}
693
694pub fn content_type_looks_html(ct: Option<&str>) -> bool {
696 ct.is_some_and(|c| {
697 ascii_contains_ignore_case(c, "html") || ascii_contains_ignore_case(c, "xhtml")
698 })
699}
700
701#[cfg(test)]
702mod tests {
703 use super::*;
704
705 #[test]
706 fn allowlist_https_only() {
707 let ok = Url::parse("https://docs.rs/serde").unwrap();
708 assert!(is_allowed_host(&ok));
709 let stdlib = Url::parse("https://doc.rust-lang.org/stable/std/").unwrap();
710 assert!(is_allowed_host(&stdlib));
711 let bad = Url::parse("http://docs.rs/serde").unwrap();
712 assert!(!is_allowed_host(&bad));
713 let evil = Url::parse("https://evil.example/").unwrap();
714 assert!(!is_allowed_host(&evil));
715 }
716
717 #[test]
718 fn utf8_bom_strip() {
719 let mut v = vec![0xEF, 0xBB, 0xBF];
720 v.extend_from_slice(b"hello");
721 let s = decode_utf8(&Bytes::from(v)).unwrap();
722 assert_eq!(s, "hello");
723 }
724
725 #[test]
726 fn content_types() {
727 assert!(content_type_looks_json(Some(
728 "application/json; charset=utf-8"
729 )));
730 assert!(content_type_looks_json(Some("Application/JSON")));
731 assert!(content_type_looks_html(Some("text/html")));
732 assert!(content_type_looks_html(Some("Text/HTML; charset=utf-8")));
733 assert!(content_type_looks_html(Some("application/xhtml+xml")));
734 assert!(!content_type_looks_json(Some("text/html")));
735 assert!(!content_type_looks_json(None));
736 assert!(!content_type_looks_html(None));
737 }
738
739 #[test]
740 fn ascii_contains_ignore_case_edges() {
741 assert!(ascii_contains_ignore_case("abJSON", "json"));
742 assert!(ascii_contains_ignore_case("json", "json"));
743 assert!(!ascii_contains_ignore_case("js", "json"));
744 assert!(ascii_contains_ignore_case("x", ""));
745 }
746
747 #[test]
748 fn retry_policy_kill_switch() {
749 let cfg = Config {
750 disable_retry: true,
751 max_retries: 5,
752 ..Config::default()
753 };
754 let http = HttpClient::new(cfg, CancelFlag::new()).unwrap();
755 assert!(!http.retry_policy().enabled);
756 assert_eq!(http.retry_policy().max_attempts(), 1);
757 }
758
759 #[test]
760 fn invalid_user_agent_rejected() {
761 let cfg = Config {
762 user_agent: "bad\nua".into(),
763 ..Config::default()
764 };
765 match HttpClient::new(cfg, CancelFlag::new()) {
766 Ok(_) => panic!("expected invalid user-agent to fail"),
767 Err(err) => assert_eq!(err.kind(), ErrorKind::Config),
768 }
769 }
770
771 #[test]
772 fn localhost_allowed_in_unit_tests() {
773 let ok = Url::parse("http://127.0.0.1:9/x").unwrap();
774 assert!(is_allowed_host(&ok));
775 let local = Url::parse("http://localhost:9/x").unwrap();
776 assert!(is_allowed_host(&local));
777 }
778
779 #[test]
780 fn decode_utf8_rejects_invalid() {
781 let err = decode_utf8(&Bytes::from(vec![0xff, 0xfe])).unwrap_err();
782 assert_eq!(err.kind(), ErrorKind::Parse);
783 }
784}