spire_core/crawler/
delay.rs

1//! The gap between the requests to one host.
2
3use std::ops::RangeInclusive;
4use std::time::Duration;
5
6/// The gap between starting two requests to the same host, see
7/// [`Crawler::with_host_delay`](crate::crawler::Crawler::with_host_delay): at least
8/// a minimum, growing towards a maximum while the host is throttled.
9///
10/// A host is throttled when it answers `429 Too Many Requests` or `503
11/// Service Unavailable`, or a request to it times out.
12/// Once its concurrency cannot go lower, at a fixed limit or at one with
13/// [`HostConcurrency::adaptive`](crate::crawler::HostConcurrency::adaptive),
14/// each round of throttling doubles its gap, from the minimum, or one
15/// second without one, up to the maximum. Each success halves it back to
16/// the minimum. A `Retry-After` from the host pauses it for exactly as long
17/// as it asks, beyond the maximum if need be, without widening its gap.
18/// The next request waits for the gap after the last one started, as the
19/// gap is when it is due, so a widened gap applies at once.
20///
21/// A [`Duration`] is the minimum, with the [default maximum](Self::DEFAULT_MAX). A
22/// range sets both ends; the same value at both ends keeps the gap fixed.
23///
24/// ```
25/// use std::time::Duration;
26///
27/// use spire_core::crawler::Crawler;
28/// use spire_core::backend::Noop;
29/// use spire_core::worker::from_fn;
30///
31/// let worker = from_fn(|_: spire_core::Context<Noop>| async {});
32/// let crawler = Crawler::new(Noop, worker)
33///     .with_host_delay(Duration::from_millis(200)..=Duration::from_secs(300));
34/// ```
35#[derive(Debug, Clone, Copy, PartialEq, Eq)]
36pub struct HostDelay {
37    min: Duration,
38    max: Duration,
39}
40
41impl HostDelay {
42    /// The maximum gap when only a minimum is given: one request a minute.
43    pub const DEFAULT_MAX: Duration = Duration::from_mins(1);
44
45    /// A gap of at least `min`, growing up to `max`, or `min` if `max` is
46    /// smaller.
47    #[must_use]
48    pub fn new(min: Duration, max: Duration) -> Self {
49        Self {
50            min,
51            max: max.max(min),
52        }
53    }
54
55    /// Returns the smallest gap.
56    #[must_use]
57    pub const fn min(self) -> Duration {
58        self.min
59    }
60
61    /// Returns the largest gap.
62    #[must_use]
63    pub const fn max(self) -> Duration {
64        self.max
65    }
66
67    /// Raises the smallest gap to at least `min`, e.g. for a robots.txt
68    /// `Crawl-delay`.
69    #[cfg(feature = "robots")]
70    pub(crate) fn raise(&mut self, min: Duration) {
71        *self = Self::new(self.min.max(min), self.max);
72    }
73
74    /// The gap after `gap` when the host is throttled: doubled, from the
75    /// minimum, or one second if the minimum is zero, up to the maximum.
76    pub(crate) fn grow(self, gap: Duration) -> Duration {
77        (gap * 2).max(self.base()).min(self.max)
78    }
79
80    /// The gap after `gap` when the host recovers: halved, back to the
81    /// minimum.
82    pub(crate) fn shrink(self, gap: Duration) -> Duration {
83        let half = gap / 2;
84        if half < self.base() { self.min } else { half }
85    }
86
87    /// The smallest gap a throttled host gets.
88    fn base(self) -> Duration {
89        if self.min.is_zero() {
90            Duration::from_secs(1)
91        } else {
92            self.min
93        }
94    }
95}
96
97impl Default for HostDelay {
98    /// No gap, growing up to [`DEFAULT_MAX`](Self::DEFAULT_MAX) while a host is throttled.
99    fn default() -> Self {
100        Self::from(Duration::ZERO)
101    }
102}
103
104impl From<Duration> for HostDelay {
105    fn from(min: Duration) -> Self {
106        Self::new(min, Self::DEFAULT_MAX)
107    }
108}
109
110impl From<RangeInclusive<Duration>> for HostDelay {
111    fn from(range: RangeInclusive<Duration>) -> Self {
112        Self::new(*range.start(), *range.end())
113    }
114}
115
116#[cfg(test)]
117mod tests {
118    use super::*;
119
120    const fn ms(ms: u64) -> Duration {
121        Duration::from_millis(ms)
122    }
123
124    #[test]
125    fn converts_from_a_minimum_or_a_range() {
126        assert_eq!(
127            HostDelay::from(ms(200)),
128            HostDelay::new(ms(200), HostDelay::DEFAULT_MAX)
129        );
130        assert_eq!(HostDelay::from(ms(200)..=ms(300)).max(), ms(300));
131        assert_eq!(HostDelay::from(ms(500)..=ms(300)).max(), ms(500), "fixed");
132        assert_eq!(HostDelay::default(), HostDelay::from(Duration::ZERO));
133    }
134
135    #[test]
136    fn grows_from_the_minimum_and_shrinks_back() {
137        let delay = HostDelay::from(ms(200)..=ms(1000));
138        let mut gap = delay.min();
139        let mut gaps = Vec::new();
140        for _ in 0..4 {
141            gap = delay.grow(gap);
142            gaps.push(gap);
143        }
144        assert_eq!(gaps, [ms(400), ms(800), ms(1000), ms(1000)]);
145        gaps.clear();
146        for _ in 0..4 {
147            gap = delay.shrink(gap);
148            gaps.push(gap);
149        }
150        assert_eq!(gaps, [ms(500), ms(250), ms(200), ms(200)]);
151    }
152
153    #[test]
154    fn grows_from_a_second_without_a_minimum() {
155        let delay = HostDelay::default();
156        assert_eq!(delay.grow(Duration::ZERO), Duration::from_secs(1));
157        assert_eq!(delay.grow(Duration::from_secs(1)), Duration::from_secs(2));
158        assert_eq!(delay.grow(Duration::from_secs(50)), HostDelay::DEFAULT_MAX);
159        assert_eq!(delay.shrink(Duration::from_secs(1)), Duration::ZERO);
160        // A fixed gap never grows.
161        let fixed = HostDelay::from(ms(200)..=ms(200));
162        assert_eq!(fixed.grow(ms(200)), ms(200));
163    }
164}
spire_core/crawler/delay.rs

spire_core/crawler/
delay.rs
