millipede_core/
errors.rs

1//! Crawl errors and their retry semantics.
2
3use crate::request::{Method, RequestBuildError};
4use std::time::Duration;
5
6/// An error produced while processing a crawl request.
7#[derive(Debug, thiserror::Error)]
8#[non_exhaustive]
9pub enum CrawlError {
10    /// A retryable error that counts against the request retry limit.
11    #[error("retryable: {0}")]
12    Retry(#[source] anyhow::Error),
13    /// A retryable error that rotates the current session.
14    #[error("session: {0}")]
15    Session(#[source] anyhow::Error),
16    /// A retryable error that ignores the request retry limit.
17    #[error("force-retry: {0}")]
18    ForceRetry(#[source] anyhow::Error),
19    /// A permanent request error that invokes the failure handler.
20    #[error("non-retryable: {0}")]
21    NonRetryable(#[source] anyhow::Error),
22    /// A critical error that aborts the crawler.
23    #[error("critical: {0}")]
24    Critical(#[source] anyhow::Error),
25    /// No registered route matched a request's label and method.
26    #[error("missing route for label {label:?} ({method})")]
27    MissingRoute {
28        /// The request routing label.
29        label: Option<String>,
30        /// The request HTTP method.
31        method: Method,
32    },
33    /// A known anti-bot or web application firewall response was detected.
34    #[error("anti-bot detected: {tech:?}")]
35    AntiBotDetected {
36        /// The detected anti-bot technology.
37        tech: AntiBotTech,
38        /// The underlying detection error.
39        #[source]
40        source: anyhow::Error,
41    },
42}
43
44/// A recognized anti-bot or web application firewall technology.
45#[derive(Debug, Clone, PartialEq, Eq)]
46#[non_exhaustive]
47pub enum AntiBotTech {
48    /// Cloudflare protection.
49    Cloudflare,
50    /// DataDome protection.
51    DataDome,
52    /// PerimeterX protection.
53    PerimeterX,
54    /// Kasada protection.
55    Kasada,
56    /// Imperva protection.
57    Imperva,
58    /// Akamai protection.
59    Akamai,
60    /// A user-defined technology name.
61    Custom(String),
62    /// An unidentified anti-bot technology.
63    Unknown,
64}
65
66impl CrawlError {
67    /// Returns an HTTP status carried anywhere in the underlying error chain.
68    pub fn http_status(&self) -> Option<http::StatusCode> {
69        let source = match self {
70            Self::Retry(error)
71            | Self::Session(error)
72            | Self::ForceRetry(error)
73            | Self::NonRetryable(error)
74            | Self::Critical(error) => error,
75            Self::AntiBotDetected { source, .. } => source,
76            Self::MissingRoute { .. } => return None,
77        };
78        source.chain().find_map(|error| {
79            error
80                .downcast_ref::<crate::http_client::HttpStatusError>()
81                .map(|status| status.status)
82        })
83    }
84
85    /// Returns a Retry-After duration carried anywhere in the underlying error chain.
86    pub fn retry_after(&self) -> Option<Duration> {
87        let source = match self {
88            Self::Retry(error)
89            | Self::Session(error)
90            | Self::ForceRetry(error)
91            | Self::NonRetryable(error)
92            | Self::Critical(error) => error,
93            Self::AntiBotDetected { source, .. } => source,
94            Self::MissingRoute { .. } => return None,
95        };
96        source.chain().find_map(|error| {
97            error
98                .downcast_ref::<crate::http_client::HttpStatusError>()
99                .and_then(|status| status.retry_after)
100        })
101    }
102
103    /// Creates a retryable error that counts against the retry limit.
104    pub fn retry<E: Into<anyhow::Error>>(error: E) -> Self {
105        Self::Retry(error.into())
106    }
107
108    /// Creates a retryable error that rotates the current session.
109    pub fn session<E: Into<anyhow::Error>>(error: E) -> Self {
110        Self::Session(error.into())
111    }
112
113    /// Creates a retryable error that ignores the retry limit.
114    pub fn force_retry<E: Into<anyhow::Error>>(error: E) -> Self {
115        Self::ForceRetry(error.into())
116    }
117
118    /// Creates a permanent, non-retryable request error.
119    pub fn non_retryable<E: Into<anyhow::Error>>(error: E) -> Self {
120        Self::NonRetryable(error.into())
121    }
122
123    /// Creates a critical error that aborts the crawler.
124    pub fn critical<E: Into<anyhow::Error>>(error: E) -> Self {
125        Self::Critical(error.into())
126    }
127
128    /// Creates a permanent error that is never retried and runs the failure handler.
129    ///
130    /// This is an alias for [`Self::non_retryable`].
131    pub fn fatal<E: Into<anyhow::Error>>(error: E) -> Self {
132        Self::non_retryable(error)
133    }
134
135    /// Returns whether the request should be attempted again.
136    pub fn is_retryable(&self) -> bool {
137        matches!(
138            self,
139            Self::Retry(_) | Self::Session(_) | Self::ForceRetry(_) | Self::AntiBotDetected { .. }
140        )
141    }
142
143    /// Returns whether retrying should rotate the current session.
144    pub fn rotates_session(&self) -> bool {
145        matches!(self, Self::Session(_) | Self::AntiBotDetected { .. })
146    }
147
148    /// Returns whether the error ignores the request retry limit.
149    pub fn ignores_max_retries(&self) -> bool {
150        matches!(self, Self::ForceRetry(_))
151    }
152
153    /// Returns whether the error should abort the entire crawler.
154    pub fn is_critical(&self) -> bool {
155        matches!(self, Self::Critical(_))
156    }
157
158    /// Returns whether retrying counts against the request retry limit.
159    pub fn counts_against_retries(&self) -> bool {
160        matches!(self, Self::Retry(_))
161    }
162}
163
164impl From<std::io::Error> for CrawlError {
165    fn from(error: std::io::Error) -> Self {
166        Self::retry(error)
167    }
168}
169
170impl From<serde_json::Error> for CrawlError {
171    fn from(error: serde_json::Error) -> Self {
172        Self::non_retryable(error)
173    }
174}
175
176impl From<RequestBuildError> for CrawlError {
177    fn from(error: RequestBuildError) -> Self {
178        Self::non_retryable(error)
179    }
180}
181
182#[cfg(test)]
183mod tests {
184    use super::*;
185    use http::StatusCode;
186
187    #[test]
188    fn constructors_produce_expected_variants() {
189        assert!(matches!(
190            CrawlError::retry(anyhow::anyhow!("x")),
191            CrawlError::Retry(_)
192        ));
193        assert!(matches!(
194            CrawlError::session(anyhow::anyhow!("x")),
195            CrawlError::Session(_)
196        ));
197        assert!(matches!(
198            CrawlError::force_retry(anyhow::anyhow!("x")),
199            CrawlError::ForceRetry(_)
200        ));
201        assert!(matches!(
202            CrawlError::non_retryable(anyhow::anyhow!("x")),
203            CrawlError::NonRetryable(_)
204        ));
205        assert!(matches!(
206            CrawlError::fatal(anyhow::anyhow!("x")),
207            CrawlError::NonRetryable(_)
208        ));
209        assert!(matches!(
210            CrawlError::critical(anyhow::anyhow!("x")),
211            CrawlError::Critical(_)
212        ));
213    }
214
215    #[test]
216    fn extracts_http_status_from_error_chain() {
217        let direct = CrawlError::retry(crate::http_client::HttpStatusError::new(
218            StatusCode::TOO_MANY_REQUESTS,
219        ));
220        assert_eq!(direct.http_status(), Some(StatusCode::TOO_MANY_REQUESTS));
221
222        let wrapped = anyhow::Error::new(crate::http_client::HttpStatusError::new(
223            StatusCode::BAD_GATEWAY,
224        ))
225        .context("fetch failed");
226        assert_eq!(
227            CrawlError::retry(wrapped).http_status(),
228            Some(StatusCode::BAD_GATEWAY)
229        );
230    }
231
232    #[test]
233    fn retry_after_extracted_through_chain() {
234        let retry_after = Duration::from_secs(2);
235        let direct = CrawlError::retry(
236            crate::http_client::HttpStatusError::new(StatusCode::TOO_MANY_REQUESTS)
237                .with_retry_after(retry_after),
238        );
239        assert_eq!(direct.retry_after(), Some(retry_after));
240
241        let plain = CrawlError::retry(crate::http_client::HttpStatusError::new(
242            StatusCode::TOO_MANY_REQUESTS,
243        ));
244        assert_eq!(plain.retry_after(), None);
245
246        let wrapped = anyhow::Error::new(
247            crate::http_client::HttpStatusError::new(StatusCode::TOO_MANY_REQUESTS)
248                .with_retry_after(retry_after),
249        )
250        .context("fetch failed");
251        assert_eq!(CrawlError::retry(wrapped).retry_after(), Some(retry_after));
252    }
253
254    #[test]
255    fn retry_after_is_none_for_missing_route_and_anti_bot() {
256        let missing_route = CrawlError::MissingRoute {
257            label: None,
258            method: Method::GET,
259        };
260        let anti_bot = CrawlError::AntiBotDetected {
261            tech: AntiBotTech::Cloudflare,
262            source: anyhow::anyhow!("challenge"),
263        };
264
265        assert_eq!(missing_route.retry_after(), None);
266        assert_eq!(anti_bot.retry_after(), None);
267    }
268
269    #[test]
270    fn helpers_cover_the_full_classification_matrix() {
271        let cases = [
272            (
273                CrawlError::retry(anyhow::anyhow!("x")),
274                [true, false, false, false, true],
275            ),
276            (
277                CrawlError::session(anyhow::anyhow!("x")),
278                [true, true, false, false, false],
279            ),
280            (
281                CrawlError::force_retry(anyhow::anyhow!("x")),
282                [true, false, true, false, false],
283            ),
284            (CrawlError::non_retryable(anyhow::anyhow!("x")), [false; 5]),
285            (
286                CrawlError::critical(anyhow::anyhow!("x")),
287                [false, false, false, true, false],
288            ),
289            (
290                CrawlError::MissingRoute {
291                    label: Some("detail".into()),
292                    method: Method::GET,
293                },
294                [false; 5],
295            ),
296            (
297                CrawlError::AntiBotDetected {
298                    tech: AntiBotTech::Cloudflare,
299                    source: anyhow::anyhow!("x"),
300                },
301                [true, true, false, false, false],
302            ),
303        ];
304
305        for (error, expected) in cases {
306            assert_eq!(error.is_retryable(), expected[0]);
307            assert_eq!(error.rotates_session(), expected[1]);
308            assert_eq!(error.ignores_max_retries(), expected[2]);
309            assert_eq!(error.is_critical(), expected[3]);
310            assert_eq!(error.counts_against_retries(), expected[4]);
311        }
312    }
313
314    #[test]
315    fn standard_errors_have_default_classifications() {
316        let io_error: CrawlError = std::io::Error::other("io").into();
317        assert!(matches!(io_error, CrawlError::Retry(_)));
318
319        let json_error: CrawlError = serde_json::from_str::<serde_json::Value>("{")
320            .expect_err("invalid JSON")
321            .into();
322        assert!(matches!(json_error, CrawlError::NonRetryable(_)));
323
324        let request_error: CrawlError = RequestBuildError::MissingUrl.into();
325        assert!(matches!(request_error, CrawlError::NonRetryable(_)));
326    }
327
328    #[test]
329    fn display_strings_include_classification_prefixes() {
330        assert!(
331            CrawlError::retry(anyhow::anyhow!("x"))
332                .to_string()
333                .contains("retryable:")
334        );
335        assert!(
336            CrawlError::session(anyhow::anyhow!("x"))
337                .to_string()
338                .contains("session:")
339        );
340        assert!(
341            CrawlError::force_retry(anyhow::anyhow!("x"))
342                .to_string()
343                .contains("force-retry:")
344        );
345        assert!(
346            CrawlError::non_retryable(anyhow::anyhow!("x"))
347                .to_string()
348                .contains("non-retryable:")
349        );
350        assert!(
351            CrawlError::critical(anyhow::anyhow!("x"))
352                .to_string()
353                .contains("critical:")
354        );
355        assert!(
356            CrawlError::AntiBotDetected {
357                tech: AntiBotTech::Cloudflare,
358                source: anyhow::anyhow!("x"),
359            }
360            .to_string()
361            .contains("anti-bot detected:")
362        );
363    }
364
365    #[test]
366    fn missing_route_display_contains_label_and_method() {
367        let error = CrawlError::MissingRoute {
368            label: Some("detail".into()),
369            method: Method::POST,
370        };
371        let display = error.to_string();
372        assert!(display.contains("detail"));
373        assert!(display.contains("POST"));
374    }
375}
millipede_core/errors.rs

millipede_core/
errors.rs
