spire_core/backend/
status.rs

1use std::time::{Duration, SystemTime};
2
3use ::http::header::RETRY_AFTER;
4use ::http::{HeaderMap, StatusCode};
5
6use crate::http::{Response, ResponseExt};
7use crate::{Error, ErrorKind, Result};
8
9/// How a backend treats error statuses.
10#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
11#[non_exhaustive]
12pub enum StatusPolicy {
13    /// `4xx` and `5xx` responses become errors, so the crawler retries or
14    /// fails them without a handler having to check:
15    ///
16    /// - `408`, `429` and `5xx` are retryable [`Http`] errors; `429` and
17    ///   `503` honour `Retry-After` by pausing the request's host.
18    /// - Other `4xx` are [`Rejected`] errors, which are not retried.
19    ///
20    /// [`Http`]: ErrorKind::Http
21    /// [`Rejected`]: ErrorKind::Rejected
22    #[default]
23    Classify,
24    /// Every response is returned, whatever its status.
25    Accept,
26}
27
28impl StatusPolicy {
29    /// Applies the policy to a response, returning it unless its status is
30    /// an error under the policy.
31    ///
32    /// # Errors
33    ///
34    /// Returns the error the status maps to, with its final URI and any
35    /// `Retry-After` delay.
36    pub fn check<B>(self, response: Response<B>) -> Result<Response<B>> {
37        let status = response.status();
38        if self == Self::Accept || !(status.is_client_error() || status.is_server_error()) {
39            return Ok(response);
40        }
41
42        let retryable = status.is_server_error()
43            || matches!(
44                status,
45                StatusCode::REQUEST_TIMEOUT | StatusCode::TOO_MANY_REQUESTS
46            );
47        let kind = if retryable {
48            ErrorKind::Http
49        } else {
50            ErrorKind::Rejected
51        };
52
53        let message = match response.final_uri() {
54            Some(uri) => format!("{uri} responded with {status}"),
55            None => format!("responded with {status}"),
56        };
57        let mut error = Error::new(kind).with_message(message).with_status(status);
58        if let Some(uri) = response.final_uri() {
59            error = error.with_uri(uri.clone());
60        }
61
62        let honours_retry_after = matches!(
63            status,
64            StatusCode::TOO_MANY_REQUESTS | StatusCode::SERVICE_UNAVAILABLE
65        );
66        if honours_retry_after && let Some(delay) = retry_after(response.headers()) {
67            error = error.with_retry_after(delay);
68        }
69        Err(error)
70    }
71}
72
73/// Parses a `Retry-After` header given in seconds or as an HTTP date.
74fn retry_after(headers: &HeaderMap) -> Option<Duration> {
75    let value = headers.get(RETRY_AFTER)?.to_str().ok()?.trim();
76    if let Ok(seconds) = value.parse::<u64>() {
77        return Some(Duration::from_secs(seconds));
78    }
79
80    let at = httpdate::parse_http_date(value).ok()?;
81    Some(at.duration_since(SystemTime::now()).unwrap_or_default())
82}
83
84#[cfg(test)]
85mod tests {
86    use ::http::{HeaderValue, Uri};
87
88    use super::*;
89
90    fn response(status: u16, retry_after: Option<&'static str>) -> Response {
91        let mut response = Response::default();
92        *response.status_mut() = StatusCode::from_u16(status).unwrap();
93        response.set_final_uri(Uri::from_static("https://example.com/"));
94        if let Some(value) = retry_after {
95            let value = HeaderValue::from_static(value);
96            response.headers_mut().insert(RETRY_AFTER, value);
97        }
98        response
99    }
100
101    fn classify(status: u16, retry_after: Option<&'static str>) -> Error {
102        StatusPolicy::Classify
103            .check(response(status, retry_after))
104            .unwrap_err()
105    }
106
107    #[test]
108    fn success_and_redirects_pass() {
109        for status in [200, 204, 301, 304] {
110            assert!(StatusPolicy::Classify.check(response(status, None)).is_ok());
111        }
112    }
113
114    #[test]
115    fn classifies_error_statuses() {
116        assert_eq!(classify(404, None).kind(), ErrorKind::Rejected);
117        assert_eq!(classify(403, None).kind(), ErrorKind::Rejected);
118        assert_eq!(classify(408, None).kind(), ErrorKind::Http);
119        assert_eq!(classify(500, None).kind(), ErrorKind::Http);
120
121        let error = classify(502, None);
122        assert_eq!(error.status(), Some(StatusCode::BAD_GATEWAY));
123        assert_eq!(error.uri().unwrap(), "https://example.com/");
124        assert_eq!(
125            error.to_string(),
126            "https://example.com/ responded with 502 Bad Gateway"
127        );
128    }
129
130    #[test]
131    fn honours_retry_after() {
132        let error = classify(429, Some("120"));
133        assert!(error.is_retryable());
134        assert_eq!(error.retry_after(), Some(Duration::from_mins(2)));
135
136        let past = classify(503, Some("Wed, 21 Oct 2015 07:28:00 GMT"));
137        assert_eq!(past.retry_after(), Some(Duration::ZERO));
138
139        // Only rate limiting and unavailability carry a pause.
140        assert_eq!(classify(500, Some("120")).retry_after(), None);
141        assert_eq!(classify(429, Some("soon")).retry_after(), None);
142    }
143
144    #[test]
145    fn accept_returns_everything() {
146        assert!(StatusPolicy::Accept.check(response(500, None)).is_ok());
147    }
148}
spire_core/backend/status.rs

spire_core/backend/
status.rs
