Skip to main content

jev_client/
throttle.rs

1//! Pacing shared by many calls, so that one call being rate limited slows all of them.
2
3use std::sync::Mutex;
4use std::time::{Duration, Instant};
5
6use crate::clock::Clock;
7
8/// The gap between call starts after the first slow-down.
9const FIRST_SPACING: Duration = Duration::from_millis(100);
10
11/// The widest gap between call starts.
12const MAX_SPACING: Duration = Duration::from_secs(5);
13
14/// Below this gap, pacing stops altogether.
15const MIN_SPACING: Duration = Duration::from_millis(10);
16
17/// Pacing shared by every call through the transports that hold it.
18///
19/// Retries alone slow down only the call that was refused: the others keep sending at full speed
20/// and are refused in turn. A `Throttle` shared by concurrent calls makes a 429 or 529 on any of
21/// them slow all of them, the way a well-behaved client pool should:
22///
23/// - **No call starts before the server's delay is over.** The delay is the `Retry-After` the
24///   server sent, or else the retry policy's backoff.
25/// - **Calls then start one at a time, spaced out.** The gap doubles with each refusal, from
26///   100 ms up to 5 s, and shrinks by an eighth with each success until pacing stops: a pool
27///   that keeps being refused slows down fast, and one that stops being refused speeds up again
28///   gradually.
29///
30/// Without refusals it does nothing. Attach one with
31/// [`HttpTransport::with_throttle`](crate::HttpTransport::with_throttle); clones of a transport
32/// share it.
33#[derive(Debug, Default)]
34pub struct Throttle {
35    state: Mutex<State>,
36}
37
38#[derive(Debug, Default)]
39struct State {
40    /// The gap between call starts; zero when calls are not paced.
41    spacing: Duration,
42    /// No call starts before this.
43    next_start: Option<Instant>,
44}
45
46impl Throttle {
47    /// A throttle that does not slow anything down until a call is refused.
48    #[must_use]
49    pub fn new() -> Self {
50        Self::default()
51    }
52
53    /// The current gap between call starts; zero when calls are not paced.
54    #[must_use]
55    pub fn spacing(&self) -> Duration {
56        self.lock().spacing
57    }
58
59    /// Waits until this call may start, and books the next start after it.
60    pub(crate) async fn wait(&self, clock: &dyn Clock) {
61        let now = clock.now();
62        let start = {
63            let mut state = self.lock();
64            let start = state.next_start.map_or(now, |next| next.max(now));
65            state.next_start = if state.spacing.is_zero() && start == now {
66                None
67            } else {
68                Some(start + state.spacing)
69            };
70            start
71        };
72        if start > now {
73            clock.sleep(start.duration_since(now)).await;
74        }
75    }
76
77    /// A call was refused with 429 or 529: nothing starts before `delay` from `now`, and calls
78    /// are spaced further apart.
79    pub(crate) fn slow_down(&self, now: Instant, delay: Duration) {
80        let mut state = self.lock();
81        let wider = state.spacing * 2;
82        state.spacing = wider.clamp(FIRST_SPACING, MAX_SPACING);
83        let resume = now + delay;
84        state.next_start = Some(state.next_start.map_or(resume, |next| next.max(resume)));
85    }
86
87    /// A call succeeded: calls may be spaced a little closer together.
88    pub(crate) fn speed_up(&self) {
89        let mut state = self.lock();
90        let narrower = state.spacing.saturating_sub(state.spacing / 8);
91        state.spacing = if narrower < MIN_SPACING {
92            Duration::ZERO
93        } else {
94            narrower
95        };
96    }
97
98    fn lock(&self) -> std::sync::MutexGuard<'_, State> {
99        // The state is two plain values, valid after any panic, so a poisoned lock is still usable.
100        self.state
101            .lock()
102            .unwrap_or_else(std::sync::PoisonError::into_inner)
103    }
104}
105
106#[cfg(test)]
107mod tests {
108    use std::future::Future;
109    use std::pin::Pin;
110    use std::sync::Mutex;
111    use std::time::{Duration, Instant, SystemTime};
112
113    use super::Throttle;
114    use crate::clock::Clock;
115
116    /// A clock that stands still, even when slept on, and remembers every sleep.
117    struct Frozen {
118        origin: Instant,
119        sleeps: Mutex<Vec<Duration>>,
120    }
121
122    impl Frozen {
123        fn new() -> Self {
124            Self {
125                origin: Instant::now(),
126                sleeps: Mutex::new(Vec::new()),
127            }
128        }
129
130        fn take_sleeps(&self) -> Vec<Duration> {
131            std::mem::take(&mut *self.sleeps.lock().unwrap())
132        }
133    }
134
135    impl Clock for Frozen {
136        fn now(&self) -> Instant {
137            self.origin
138        }
139
140        fn wall_time(&self) -> SystemTime {
141            SystemTime::UNIX_EPOCH
142        }
143
144        fn sleep(&self, duration: Duration) -> Pin<Box<dyn Future<Output = ()> + Send + '_>> {
145            self.sleeps.lock().unwrap().push(duration);
146            Box::pin(async {})
147        }
148    }
149
150    fn block_on(future: impl Future<Output = ()>) {
151        tokio::runtime::Builder::new_current_thread()
152            .build()
153            .unwrap()
154            .block_on(future);
155    }
156
157    #[test]
158    fn without_a_refusal_no_call_waits() {
159        let (throttle, clock) = (Throttle::new(), Frozen::new());
160
161        block_on(async {
162            for _ in 0..5 {
163                throttle.wait(&clock).await;
164            }
165        });
166
167        assert!(clock.take_sleeps().is_empty());
168        assert_eq!(throttle.spacing(), Duration::ZERO);
169    }
170
171    #[test]
172    fn a_refusal_holds_every_call_until_the_delay_is_over_then_spaces_them_out() {
173        let (throttle, clock) = (Throttle::new(), Frozen::new());
174
175        throttle.slow_down(clock.now(), Duration::from_secs(2));
176        block_on(async {
177            for _ in 0..3 {
178                throttle.wait(&clock).await;
179            }
180        });
181
182        let ms = Duration::from_millis;
183        assert_eq!(clock.take_sleeps(), [ms(2000), ms(2100), ms(2200)]);
184    }
185
186    #[test]
187    fn refusals_widen_the_gap_up_to_a_limit_and_successes_narrow_it_to_nothing() {
188        let (throttle, clock) = (Throttle::new(), Frozen::new());
189
190        let mut widths = Vec::new();
191        for _ in 0..8 {
192            throttle.slow_down(clock.now(), Duration::ZERO);
193            widths.push(throttle.spacing().as_millis());
194        }
195        let mut successes = 0;
196        while !throttle.spacing().is_zero() {
197            throttle.speed_up();
198            successes += 1;
199        }
200
201        assert_eq!(widths, [100, 200, 400, 800, 1600, 3200, 5000, 5000]);
202        assert_eq!(successes, 47, "about fifty successes undo the widest gap");
203    }
204
205    #[test]
206    fn a_later_refusal_never_shortens_a_pause_already_asked_for() {
207        let (throttle, clock) = (Throttle::new(), Frozen::new());
208
209        throttle.slow_down(clock.now(), Duration::from_secs(10));
210        throttle.slow_down(clock.now(), Duration::from_secs(1));
211        block_on(throttle.wait(&clock));
212
213        assert_eq!(clock.take_sleeps(), [Duration::from_secs(10)]);
214    }
215}