Skip to main content

docsrs_cli/http/
rate_limit.rs

1//! Per-host politeness throttle: in-process clock plus cross-process lock+stamp.
2//!
3//! Policy and its stamp/lock I/O live together so the request loop in `client`
4//! only calls [`HttpClient::rate_limit`] and never open-codes the throttle.
5
6use std::fs::{self, File, OpenOptions};
7use std::path::{Path, PathBuf};
8use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
9
10use fs4::FileExt as Fs4FileExt;
11use fs4::TryLockError as Fs4TryLockError;
12use reqwest::Url;
13use tracing::warn;
14
15use crate::config::CANCEL_POLL_INTERVAL_MS;
16use crate::error::{AppError, AppResult, ErrorDetail, ErrorKind, InternalOp, IoOp, io_at};
17use crate::retry::politeness_delay;
18
19use super::client::HttpClient;
20
21impl HttpClient {
22    /// Sleep in cancel-poll slices so SIGINT/SIGTERM never waits out a full delay.
23    ///
24    /// # Errors
25    ///
26    /// Returns the cancel error when the process was interrupted or terminated.
27    pub(super) async fn sleep_cancelable(&self, wait: Duration) -> AppResult<()> {
28        let step = Duration::from_millis(CANCEL_POLL_INTERVAL_MS);
29        let mut remaining = wait;
30        while !remaining.is_zero() {
31            self.cancel.check()?;
32            let slice = remaining.min(step);
33            tokio::time::sleep(slice).await;
34            remaining = remaining.saturating_sub(slice);
35        }
36        self.cancel.check()
37    }
38
39    /// Enforce per-host delay: in-process clock + exclusive cross-process lock+stamp.
40    ///
41    /// Floor is `rate_limit_delay_ms`; each wait uses [`politeness_delay`] so the
42    /// effective interval is never fixed (additive jitter up to +20%).
43    ///
44    /// In-process map uses a short [`std::sync::Mutex`] hold (no sleep under the
45    /// lock). Cross-process flock is released on drop before return (never held
46    /// across unrelated work).
47    ///
48    /// # Errors
49    ///
50    /// Returns the cancel error when interrupted; lock/stamp I/O failures degrade
51    /// to the in-process clock instead of failing the request.
52    pub(super) async fn rate_limit(&self, url: &Url) -> AppResult<()> {
53        // Lookup borrows host; allocate only when inserting/updating the clock map.
54        let host = url.host_str().unwrap_or("");
55        let delay = politeness_delay(self.cfg.rate_limit_delay());
56        if delay.is_zero() {
57            return Ok(());
58        }
59
60        // Prefer exclusive cross-process section when cache_dir is available.
61        if let Some(dir) = self.cfg.cache_dir.as_ref() {
62            match self.rate_limit_cross_process(dir, host, delay).await {
63                Ok(()) => {
64                    self.touch_host_clock(host);
65                    return Ok(());
66                }
67                Err(e)
68                    if e.kind() == ErrorKind::Terminated || e.kind() == ErrorKind::Interrupted =>
69                {
70                    return Err(e);
71                }
72                Err(e) => {
73                    // FS without flock or I/O failure: fall back to in-process only.
74                    warn!(error = %e, host = %host, "cross-process rate-limit lock failed; using in-process only");
75                }
76            }
77        }
78
79        let in_proc = {
80            let map = self.last_host_hit.lock().unwrap_or_else(|e| e.into_inner());
81            map.get(host).and_then(|prev| {
82                let elapsed = prev.elapsed();
83                if elapsed < delay {
84                    Some(delay - elapsed)
85                } else {
86                    None
87                }
88            })
89        };
90        if let Some(d) = in_proc {
91            self.sleep_cancelable(d).await?;
92        }
93        self.touch_host_clock(host);
94        Ok(())
95    }
96
97    fn touch_host_clock(&self, host: &str) {
98        let mut map = self.last_host_hit.lock().unwrap_or_else(|e| e.into_inner());
99        map.insert(host.to_string(), Instant::now());
100    }
101
102    /// Exclusive lock on `{cache_dir}/rate-limit/{host}.lock`, then stamp throttle.
103    async fn rate_limit_cross_process(
104        &self,
105        cache_dir: &Path,
106        host: &str,
107        delay: Duration,
108    ) -> AppResult<()> {
109        let lock_path = rate_limit_lock_path(cache_dir, host);
110        let stamp_path = rate_limit_stamp_path(cache_dir, host);
111        if let Some(parent) = lock_path.parent() {
112            fs::create_dir_all(parent).map_err(io_at(IoOp::CreateDir, parent))?;
113            crate::platform::restrict_private_dir(parent);
114        }
115
116        let file = OpenOptions::new()
117            .create(true)
118            .read(true)
119            .write(true)
120            .truncate(false)
121            .open(&lock_path)
122            .map_err(io_at(IoOp::OpenLock, &lock_path))?;
123        crate::platform::restrict_private_file(&lock_path);
124
125        // Cancel-aware exclusive acquisition (fs4 try_lock + short sleep).
126        // Fully-qualified calls avoid collision with std::fs::File::try_lock (Rust 1.89+).
127        loop {
128            self.cancel.check()?;
129            match Fs4FileExt::try_lock(&file) {
130                Ok(()) => break,
131                Err(Fs4TryLockError::WouldBlock) => {
132                    self.sleep_cancelable(Duration::from_millis(CANCEL_POLL_INTERVAL_MS))
133                        .await?;
134                }
135                Err(Fs4TryLockError::Error(e)) => {
136                    return Err(io_at(IoOp::Lock, &lock_path)(e));
137                }
138            }
139        }
140
141        // Guard unlock on all exit paths.
142        struct UnlockOnDrop<'a>(&'a File);
143        impl Drop for UnlockOnDrop<'_> {
144            fn drop(&mut self) {
145                let _ = Fs4FileExt::unlock(self.0);
146            }
147        }
148        let _guard = UnlockOnDrop(&file);
149
150        let remaining = stamp_remaining(&stamp_path, delay);
151        if let Some(d) = remaining {
152            self.sleep_cancelable(d).await?;
153        }
154        write_stamp(&stamp_path)?;
155        Ok(())
156    }
157}
158
159pub(super) fn safe_host_name(host: &str) -> String {
160    let safe: String = host
161        .chars()
162        .map(|c| {
163            if c.is_ascii_alphanumeric() || c == '.' || c == '-' {
164                c
165            } else {
166                '_'
167            }
168        })
169        .collect();
170    if safe.is_empty() {
171        "unknown".to_string()
172    } else {
173        safe
174    }
175}
176
177/// Cache subdirectory holding per-host rate-limit stamps and locks.
178///
179/// `doctor` reports on this directory and the limiter creates it; a literal in
180/// each would let the health check inspect a path the limiter no longer uses.
181pub const RATE_LIMIT_DIR_NAME: &str = "rate-limit";
182
183/// Path for host rate-limit stamp: `{cache_dir}/rate-limit/{safe_host}.stamp`.
184pub(super) fn rate_limit_stamp_path(cache_dir: &Path, host: &str) -> PathBuf {
185    cache_dir
186        .join(RATE_LIMIT_DIR_NAME)
187        .join(format!("{}.stamp", safe_host_name(host)))
188}
189
190/// Path for host rate-limit exclusive lock: `{cache_dir}/rate-limit/{safe_host}.lock`.
191pub(super) fn rate_limit_lock_path(cache_dir: &Path, host: &str) -> PathBuf {
192    cache_dir
193        .join(RATE_LIMIT_DIR_NAME)
194        .join(format!("{}.lock", safe_host_name(host)))
195}
196
197pub(super) fn now_unix_ms() -> Option<u128> {
198    SystemTime::now()
199        .duration_since(UNIX_EPOCH)
200        .ok()
201        .map(|d| d.as_millis())
202}
203
204/// Remaining sleep from stamp file, if any.
205pub(super) fn stamp_remaining(path: &Path, delay: Duration) -> Option<Duration> {
206    let text = fs::read_to_string(path).ok()?;
207    let last_ms: u128 = text.trim().parse().ok()?;
208    let now = now_unix_ms()?;
209    if now < last_ms {
210        return Some(delay);
211    }
212    let elapsed = Duration::from_millis((now - last_ms) as u64);
213    if elapsed < delay {
214        Some(delay - elapsed)
215    } else {
216        None
217    }
218}
219
220/// Write stamp under exclusive lock (temp + rename).
221pub(super) fn write_stamp(path: &Path) -> AppResult<()> {
222    if let Some(parent) = path.parent() {
223        fs::create_dir_all(parent).map_err(io_at(IoOp::CreateDir, parent))?;
224    }
225    let now = now_unix_ms().ok_or_else(|| {
226        AppError::of(ErrorDetail::Internal {
227            op: InternalOp::ClockBeforeEpoch,
228        })
229    })?;
230    let tmp = path.with_extension("stamp.tmp");
231    fs::write(&tmp, now.to_string()).map_err(io_at(IoOp::Write, &tmp))?;
232    fs::rename(&tmp, path).map_err(|e| {
233        let _ = fs::remove_file(&tmp);
234        io_at(IoOp::Rename, path)(e)
235    })?;
236    crate::platform::restrict_private_file(path);
237    Ok(())
238}