spire_core/backend/
status.rs

1use std::time::{Duration, SystemTime};
2
3use http::header::RETRY_AFTER;
4use http::{HeaderMap, StatusCode};
5
6use crate::request::{Response, ResponseExt};
7use crate::{Error, ErrorKind, Result};
8
9/// How a backend treats error statuses, e.g. `HttpClient` from
10/// `spire-reqwest` or `BrowserBackend` from `spire-thirtyfour`.
11#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
12#[non_exhaustive]
13pub enum StatusPolicy {
14    /// `4xx` and `5xx` responses become errors, so the crawler retries or
15    /// fails them without a handler having to check:
16    ///
17    /// - `408`, `429` and `5xx` are retryable [`Http`] errors; `429` and
18    ///   `503` honour `Retry-After` by pausing the request's tag.
19    /// - Other `4xx` are [`Rejected`] errors, which are not retried.
20    ///
21    /// [`Http`]: ErrorKind::Http
22    /// [`Rejected`]: ErrorKind::Rejected
23    #[default]
24    Classify,
25    /// Every response is returned, whatever its status.
26    Accept,
27}
28
29impl StatusPolicy {
30    /// Applies the policy to a response, returning it unless its status is
31    /// an error under the policy.
32    ///
33    /// # Errors
34    ///
35    /// Returns the error the status maps to, with its final URI and any
36    /// `Retry-After` delay.
37    pub fn check<B>(self, response: Response<B>) -> Result<Response<B>> {
38        let status = response.status();
39        if self == Self::Accept || !(status.is_client_error() || status.is_server_error()) {
40            return Ok(response);
41        }
42
43        let retryable = status.is_server_error()
44            || matches!(
45                status,
46                StatusCode::REQUEST_TIMEOUT | StatusCode::TOO_MANY_REQUESTS
47            );
48        let kind = if retryable {
49            ErrorKind::Http
50        } else {
51            ErrorKind::Rejected
52        };
53
54        let message = match response.final_uri() {
55            Some(uri) => format!("{uri} responded with {status}"),
56            None => format!("responded with {status}"),
57        };
58        let mut error = Error::new(kind).with_message(message).with_status(status);
59        if let Some(uri) = response.final_uri() {
60            error = error.with_uri(uri.clone());
61        }
62
63        let honours_retry_after = matches!(
64            status,
65            StatusCode::TOO_MANY_REQUESTS | StatusCode::SERVICE_UNAVAILABLE
66        );
67        if honours_retry_after && let Some(delay) = retry_after(response.headers()) {
68            error = error.with_retry_after(delay);
69        }
70        Err(error)
71    }
72}
73
74/// Parses a `Retry-After` header given in seconds or as an HTTP date.
75fn retry_after(headers: &HeaderMap) -> Option<Duration> {
76    let value = headers.get(RETRY_AFTER)?.to_str().ok()?.trim();
77    if let Ok(seconds) = value.parse::<u64>() {
78        return Some(Duration::from_secs(seconds));
79    }
80
81    let at = httpdate::parse_http_date(value).ok()?;
82    Some(at.duration_since(SystemTime::now()).unwrap_or_default())
83}
84
85#[cfg(test)]
86mod tests {
87    use http::{HeaderValue, Uri};
88
89    use super::*;
90
91    fn response(status: u16, retry_after: Option<&'static str>) -> Response {
92        let mut response = Response::default();
93        *response.status_mut() = StatusCode::from_u16(status).unwrap();
94        response.set_final_uri(Uri::from_static("https://example.com/"));
95        if let Some(value) = retry_after {
96            let value = HeaderValue::from_static(value);
97            response.headers_mut().insert(RETRY_AFTER, value);
98        }
99        response
100    }
101
102    fn classify(status: u16, retry_after: Option<&'static str>) -> Error {
103        StatusPolicy::Classify
104            .check(response(status, retry_after))
105            .unwrap_err()
106    }
107
108    #[test]
109    fn success_and_redirects_pass() {
110        for status in [200, 204, 301, 304] {
111            assert!(StatusPolicy::Classify.check(response(status, None)).is_ok());
112        }
113    }
114
115    #[test]
116    fn classifies_error_statuses() {
117        assert_eq!(classify(404, None).kind(), ErrorKind::Rejected);
118        assert_eq!(classify(403, None).kind(), ErrorKind::Rejected);
119        assert_eq!(classify(408, None).kind(), ErrorKind::Http);
120        assert_eq!(classify(500, None).kind(), ErrorKind::Http);
121
122        let error = classify(502, None);
123        assert_eq!(error.status(), Some(StatusCode::BAD_GATEWAY));
124        assert_eq!(error.uri().unwrap(), "https://example.com/");
125        assert_eq!(
126            error.to_string(),
127            "https://example.com/ responded with 502 Bad Gateway"
128        );
129    }
130
131    #[test]
132    fn honours_retry_after() {
133        let error = classify(429, Some("120"));
134        assert!(error.is_retryable());
135        assert_eq!(error.retry_after(), Some(Duration::from_mins(2)));
136
137        let past = classify(503, Some("Wed, 21 Oct 2015 07:28:00 GMT"));
138        assert_eq!(past.retry_after(), Some(Duration::ZERO));
139
140        // Only rate limiting and unavailability carry a pause.
141        assert_eq!(classify(500, Some("120")).retry_after(), None);
142        assert_eq!(classify(429, Some("soon")).retry_after(), None);
143    }
144
145    #[test]
146    fn accept_returns_everything() {
147        assert!(StatusPolicy::Accept.check(response(500, None)).is_ok());
148    }
149}
spire_core/backend/status.rs

spire_core/backend/
status.rs
