docsrs_cli/http/
rate_limit.rs1use std::fs::{self, File, OpenOptions};
7use std::path::{Path, PathBuf};
8use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
9
10use fs4::FileExt as Fs4FileExt;
11use fs4::TryLockError as Fs4TryLockError;
12use reqwest::Url;
13use tracing::warn;
14
15use crate::config::CANCEL_POLL_INTERVAL_MS;
16use crate::error::{AppError, AppResult, ErrorDetail, ErrorKind, InternalOp, IoOp, io_at};
17use crate::retry::politeness_delay;
18
19use super::client::HttpClient;
20
21impl HttpClient {
22 pub(super) async fn sleep_cancelable(&self, wait: Duration) -> AppResult<()> {
28 let step = Duration::from_millis(CANCEL_POLL_INTERVAL_MS);
29 let mut remaining = wait;
30 while !remaining.is_zero() {
31 self.cancel.check()?;
32 let slice = remaining.min(step);
33 tokio::time::sleep(slice).await;
34 remaining = remaining.saturating_sub(slice);
35 }
36 self.cancel.check()
37 }
38
39 pub(super) async fn rate_limit(&self, url: &Url) -> AppResult<()> {
53 let host = url.host_str().unwrap_or("");
55 let delay = politeness_delay(self.cfg.rate_limit_delay());
56 if delay.is_zero() {
57 return Ok(());
58 }
59
60 if let Some(dir) = self.cfg.cache_dir.as_ref() {
62 match self.rate_limit_cross_process(dir, host, delay).await {
63 Ok(()) => {
64 self.touch_host_clock(host);
65 return Ok(());
66 }
67 Err(e)
68 if e.kind() == ErrorKind::Terminated || e.kind() == ErrorKind::Interrupted =>
69 {
70 return Err(e);
71 }
72 Err(e) => {
73 warn!(error = %e, host = %host, "cross-process rate-limit lock failed; using in-process only");
75 }
76 }
77 }
78
79 let in_proc = {
80 let map = self.last_host_hit.lock().unwrap_or_else(|e| e.into_inner());
81 map.get(host).and_then(|prev| {
82 let elapsed = prev.elapsed();
83 if elapsed < delay {
84 Some(delay - elapsed)
85 } else {
86 None
87 }
88 })
89 };
90 if let Some(d) = in_proc {
91 self.sleep_cancelable(d).await?;
92 }
93 self.touch_host_clock(host);
94 Ok(())
95 }
96
97 fn touch_host_clock(&self, host: &str) {
98 let mut map = self.last_host_hit.lock().unwrap_or_else(|e| e.into_inner());
99 map.insert(host.to_string(), Instant::now());
100 }
101
102 async fn rate_limit_cross_process(
104 &self,
105 cache_dir: &Path,
106 host: &str,
107 delay: Duration,
108 ) -> AppResult<()> {
109 let lock_path = rate_limit_lock_path(cache_dir, host);
110 let stamp_path = rate_limit_stamp_path(cache_dir, host);
111 if let Some(parent) = lock_path.parent() {
112 fs::create_dir_all(parent).map_err(io_at(IoOp::CreateDir, parent))?;
113 crate::platform::restrict_private_dir(parent);
114 }
115
116 let file = OpenOptions::new()
117 .create(true)
118 .read(true)
119 .write(true)
120 .truncate(false)
121 .open(&lock_path)
122 .map_err(io_at(IoOp::OpenLock, &lock_path))?;
123 crate::platform::restrict_private_file(&lock_path);
124
125 loop {
128 self.cancel.check()?;
129 match Fs4FileExt::try_lock(&file) {
130 Ok(()) => break,
131 Err(Fs4TryLockError::WouldBlock) => {
132 self.sleep_cancelable(Duration::from_millis(CANCEL_POLL_INTERVAL_MS))
133 .await?;
134 }
135 Err(Fs4TryLockError::Error(e)) => {
136 return Err(io_at(IoOp::Lock, &lock_path)(e));
137 }
138 }
139 }
140
141 struct UnlockOnDrop<'a>(&'a File);
143 impl Drop for UnlockOnDrop<'_> {
144 fn drop(&mut self) {
145 let _ = Fs4FileExt::unlock(self.0);
146 }
147 }
148 let _guard = UnlockOnDrop(&file);
149
150 let remaining = stamp_remaining(&stamp_path, delay);
151 if let Some(d) = remaining {
152 self.sleep_cancelable(d).await?;
153 }
154 write_stamp(&stamp_path)?;
155 Ok(())
156 }
157}
158
159pub(super) fn safe_host_name(host: &str) -> String {
160 let safe: String = host
161 .chars()
162 .map(|c| {
163 if c.is_ascii_alphanumeric() || c == '.' || c == '-' {
164 c
165 } else {
166 '_'
167 }
168 })
169 .collect();
170 if safe.is_empty() {
171 "unknown".to_string()
172 } else {
173 safe
174 }
175}
176
177pub const RATE_LIMIT_DIR_NAME: &str = "rate-limit";
182
183pub(super) fn rate_limit_stamp_path(cache_dir: &Path, host: &str) -> PathBuf {
185 cache_dir
186 .join(RATE_LIMIT_DIR_NAME)
187 .join(format!("{}.stamp", safe_host_name(host)))
188}
189
190pub(super) fn rate_limit_lock_path(cache_dir: &Path, host: &str) -> PathBuf {
192 cache_dir
193 .join(RATE_LIMIT_DIR_NAME)
194 .join(format!("{}.lock", safe_host_name(host)))
195}
196
197pub(super) fn now_unix_ms() -> Option<u128> {
198 SystemTime::now()
199 .duration_since(UNIX_EPOCH)
200 .ok()
201 .map(|d| d.as_millis())
202}
203
204pub(super) fn stamp_remaining(path: &Path, delay: Duration) -> Option<Duration> {
206 let text = fs::read_to_string(path).ok()?;
207 let last_ms: u128 = text.trim().parse().ok()?;
208 let now = now_unix_ms()?;
209 if now < last_ms {
210 return Some(delay);
211 }
212 let elapsed = Duration::from_millis((now - last_ms) as u64);
213 if elapsed < delay {
214 Some(delay - elapsed)
215 } else {
216 None
217 }
218}
219
220pub(super) fn write_stamp(path: &Path) -> AppResult<()> {
222 if let Some(parent) = path.parent() {
223 fs::create_dir_all(parent).map_err(io_at(IoOp::CreateDir, parent))?;
224 }
225 let now = now_unix_ms().ok_or_else(|| {
226 AppError::of(ErrorDetail::Internal {
227 op: InternalOp::ClockBeforeEpoch,
228 })
229 })?;
230 let tmp = path.with_extension("stamp.tmp");
231 fs::write(&tmp, now.to_string()).map_err(io_at(IoOp::Write, &tmp))?;
232 fs::rename(&tmp, path).map_err(|e| {
233 let _ = fs::remove_file(&tmp);
234 io_at(IoOp::Rename, path)(e)
235 })?;
236 crate::platform::restrict_private_file(path);
237 Ok(())
238}