spire_core/worker/
outcome.rs

1use std::convert::Infallible;
2use std::time::Duration;
3
4use crate::crawler::Tag;
5use crate::{BoxError, Error};
6
7/// What the crawler should do after a worker handled a request.
8#[must_use]
9#[derive(Debug, Default)]
10pub enum Outcome {
11    /// The request is done.
12    #[default]
13    Continue,
14    /// Try the request again, after the given delay or the frontier's
15    /// default delay.
16    Retry {
17        /// Delay before the next attempt; `None` defers to the frontier.
18        after: Option<Duration>,
19    },
20    /// Pause every request in `scope` for `duration`, then try this request
21    /// again. Typically returned on `429 Too Many Requests`.
22    Backoff {
23        /// The requests to pause.
24        scope: Scope,
25        /// How long to pause them.
26        duration: Duration,
27    },
28    /// Stop crawling `scope`: its queued requests are skipped from now on.
29    /// Returned on the first request of a tag, this drops the whole tag.
30    Abort(Scope),
31    /// Stop the crawl: requests in flight finish, nothing new starts.
32    Stop,
33    /// The request failed. Retryable errors are tried again, others are
34    /// recorded as failures.
35    Fail(Error),
36    /// Fail the crawl, returning the error after in-flight requests drain.
37    /// No new work starts, and this failure is not retried.
38    Fatal(Error),
39}
40
41impl Outcome {
42    /// Retries the request after the frontier's default delay.
43    pub const fn retry() -> Self {
44        Self::Retry { after: None }
45    }
46
47    /// Retries the request after `delay`.
48    pub const fn retry_after(delay: Duration) -> Self {
49        Self::Retry { after: Some(delay) }
50    }
51
52    /// Pauses requests with the same tag as this one for `duration`.
53    pub const fn backoff(duration: Duration) -> Self {
54        Self::Backoff {
55            scope: Scope::SameTag,
56            duration,
57        }
58    }
59
60    /// Skips every further request with the same tag as this one.
61    pub const fn abort() -> Self {
62        Self::Abort(Scope::SameTag)
63    }
64
65    /// Skips every further request with `tag`.
66    pub fn abort_tag(tag: impl Into<Tag>) -> Self {
67        Self::Abort(Scope::Tag(tag.into()))
68    }
69
70    /// Pauses requests with `tag` for `duration`.
71    pub fn backoff_tag(tag: impl Into<Tag>, duration: Duration) -> Self {
72        Self::Backoff {
73            scope: Scope::Tag(tag.into()),
74            duration,
75        }
76    }
77}
78
79/// The requests an [`Outcome::Backoff`] or [`Outcome::Abort`] applies to.
80#[derive(Debug, Clone, Default, PartialEq, Eq)]
81pub enum Scope {
82    /// Requests sharing the tag of the request that was handled, including
83    /// untagged requests when it had none.
84    #[default]
85    SameTag,
86    /// Requests with the given tag.
87    Tag(Tag),
88}
89
90/// Conversion into an [`Outcome`], implemented for handler return types.
91pub trait IntoOutcome {
92    /// Converts `self` into an [`Outcome`].
93    fn into_outcome(self) -> Outcome;
94}
95
96impl IntoOutcome for Outcome {
97    fn into_outcome(self) -> Outcome {
98        self
99    }
100}
101
102impl IntoOutcome for () {
103    fn into_outcome(self) -> Outcome {
104        Outcome::Continue
105    }
106}
107
108impl IntoOutcome for Infallible {
109    fn into_outcome(self) -> Outcome {
110        match self {}
111    }
112}
113
114impl IntoOutcome for Error {
115    fn into_outcome(self) -> Outcome {
116        Outcome::Fail(self)
117    }
118}
119
120impl IntoOutcome for BoxError {
121    /// Fails with the error, keeping its kind if it is an [`Error`] and
122    /// classifying it as a [`Worker`](crate::ErrorKind::Worker) error
123    /// otherwise.
124    fn into_outcome(self) -> Outcome {
125        Outcome::Fail(Error::from_boxed(crate::ErrorKind::Worker, self))
126    }
127}
128
129impl<T> IntoOutcome for Option<T>
130where
131    T: IntoOutcome,
132{
133    fn into_outcome(self) -> Outcome {
134        self.map_or(Outcome::Continue, IntoOutcome::into_outcome)
135    }
136}
137
138impl<T, E> IntoOutcome for Result<T, E>
139where
140    T: IntoOutcome,
141    E: IntoOutcome,
142{
143    fn into_outcome(self) -> Outcome {
144        match self {
145            Ok(value) => value.into_outcome(),
146            Err(error) => error.into_outcome(),
147        }
148    }
149}
150
151#[cfg(test)]
152mod tests {
153    use super::*;
154    use crate::{ErrorKind, Result};
155
156    #[test]
157    fn conversions() {
158        assert!(matches!(().into_outcome(), Outcome::Continue));
159        assert!(matches!(None::<Outcome>.into_outcome(), Outcome::Continue));
160
161        let ok: Result<()> = Ok(());
162        assert!(matches!(ok.into_outcome(), Outcome::Continue));
163
164        let err: Result<()> = Err(Error::new(ErrorKind::Http));
165        assert!(matches!(err.into_outcome(), Outcome::Fail(_)));
166
167        let retry: Result<(), Outcome> = Err(Outcome::retry());
168        assert!(matches!(
169            retry.into_outcome(),
170            Outcome::Retry { after: None }
171        ));
172    }
173}
spire_core/worker/outcome.rs

spire_core/worker/
outcome.rs
