Skip to main content

spider_cloud_agent/
error.rs

1//! What can go wrong, and how to tell the cases apart.
2
3use crate::credits::Credits;
4use crate::policy::StopReason;
5use crate::response::{Attempt, FailedPage};
6use crate::status::ApiStatus;
7use std::time::Duration;
8
9/// An error from a call to Spider Cloud.
10///
11/// Running out of credits is its own variant rather than a status code, because
12/// it is the one failure that must never be retried: every attempt after the
13/// balance hits zero fails the same way, and some of them still cost money.
14///
15/// A rate limit is not its own variant either. A 429 arrives as [`Error::Api`]
16/// with its status and its `retry_after`, and [`Error::recovery`] is what an
17/// agent reads to learn that the answer is to wait.
18#[derive(Debug, thiserror::Error)]
19#[non_exhaustive]
20pub enum Error {
21    /// A call error together with its operation trail and optional run totals.
22    #[error("{source}")]
23    Accounted {
24        /// Original error, preserving its variant and recovery advice.
25        source: Box<Error>,
26        /// Every attempted call in this operation.
27        attempts: Vec<Attempt>,
28        /// Totals across the shared run, when configured.
29        run: Option<crate::client::RunSpend>,
30    },
31    /// The service rejected the call itself. This is never a target site's
32    /// status, only the status of your request to Spider Cloud.
33    #[error("{status}{}", .message.as_deref().map(|m| format!(": {m}")).unwrap_or_default())]
34    Api {
35        /// The status the service returned.
36        status: ApiStatus,
37        /// What the service said, when it said anything.
38        message: Option<String>,
39        /// How long to wait before trying again, when the service said.
40        retry_after: Option<Duration>,
41    },
42
43    /// The account has no credits left. Do not retry this.
44    #[error("no credits left on the account")]
45    InsufficientCredits,
46
47    /// Pages came back, none of them usable, and the walk then stopped.
48    ///
49    /// A walk that never got a page back keeps the call error in [`Error::Accounted`],
50    /// [`Error::Api`], [`Error::Auth`] or [`Error::Transport`], because what
51    /// stopped it is a fact about the call rather than about a page.
52    #[error(
53        "no usable page after {} attempts: {reason}{}",
54        .attempts.len(),
55        .source.as_deref().map(|s| format!(", {s}")).unwrap_or_default()
56    )]
57    Exhausted {
58        /// What was tried, in order.
59        attempts: Vec<Attempt>,
60        /// The last failure, when there was one to keep.
61        last: Option<Box<FailedPage>>,
62        /// Why the policy stopped the walk.
63        reason: StopReason,
64        /// The call error that stopped the walk, when a call rather than a
65        /// page was what stopped it.
66        source: Option<Box<Error>>,
67    },
68
69    /// The call would have cost more than the budget allowed.
70    #[error("budget exceeded: {kind}")]
71    BudgetExceeded {
72        /// Which cap stopped it.
73        kind: BudgetKind,
74        /// What was tried before the cap stopped it, in order. Empty when the
75        /// cap was reached before anything was sent.
76        attempts: Vec<Attempt>,
77    },
78
79    /// No usable API key, the key was rejected, or signing in did not produce
80    /// one.
81    #[error("authentication: {message}")]
82    Auth {
83        /// Which kind of trouble it was, so a caller can tell a key the service
84        /// refused from a key that was never there.
85        cause: AuthCause,
86        /// What went wrong, naming places and never values.
87        message: String,
88    },
89
90    /// Sending the request or reading its response failed at the HTTP layer.
91    #[error("transport: {0}")]
92    Transport(#[from] reqwest::Error),
93
94    /// The response did not have the shape this version of the crate expects.
95    #[error("could not read the response: {0}")]
96    Decode(#[source] serde_json::Error),
97
98    /// The answer ran past the size the client was built to read.
99    ///
100    /// Only a client built with `max_response_bytes` can see this. Inside a
101    /// page operation it arrives as the source of an [`Error::Exhausted`]
102    /// whose attempts include the cut-off call, so nothing spent before it is
103    /// lost.
104    #[error("the answer ran past {limit} bytes")]
105    ResponseTooLarge {
106        /// The most the client was built to read.
107        limit: usize,
108        /// The status the answer arrived with. The body behind it was not read.
109        status: ApiStatus,
110    },
111
112    /// The client was built with settings that cannot work.
113    #[error("configuration: {0}")]
114    Config(String),
115}
116
117/// Why authentication failed.
118///
119/// The split that matters is whether signing in again could help. It can when
120/// the service refused a key or the sign in itself failed. It cannot create a
121/// key that was never configured, or fix a path on this machine.
122#[derive(Debug, Clone, Copy, PartialEq, Eq)]
123#[non_exhaustive]
124pub enum AuthCause {
125    /// Nothing is configured: no environment variable, no keychain entry and
126    /// no credentials file.
127    NoKey,
128    /// A key was supplied and it was blank.
129    EmptyKey,
130    /// The service answered 401.
131    Refused,
132    /// The sign in flow itself failed: the browser step, the token exchange or
133    /// a bad state.
134    SignInFailed,
135    /// A problem on this machine: no home directory, a file that could not be
136    /// written or narrowed, or a configured address that is not one.
137    Local,
138    /// The call went out with no key and the service asked for an account.
139    /// Signing in is the way forward, and the same call then works.
140    SignInRequired,
141}
142
143/// What a caller can do about an error.
144#[derive(Debug, Clone, Copy, PartialEq, Eq)]
145#[non_exhaustive]
146pub enum Recovery {
147    /// Send the same call again after this wait, or after a wait of the
148    /// caller's choosing when the service named none.
149    Wait(Option<Duration>),
150    /// Sign in again, or supply a different key.
151    Reauthenticate,
152    /// Nothing about sending again will change the answer.
153    Permanent,
154    /// The error does not say.
155    Unknown,
156}
157
158/// Which budget ran out.
159#[derive(Debug, Clone, Copy, PartialEq, Eq)]
160#[non_exhaustive]
161pub enum BudgetKind {
162    /// The credit cap.
163    Credits,
164    /// The wall clock cap.
165    Time,
166    /// The attempt count.
167    Attempts,
168}
169
170impl std::fmt::Display for BudgetKind {
171    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
172        match self {
173            BudgetKind::Credits => write!(f, "credits"),
174            BudgetKind::Time => write!(f, "time"),
175            BudgetKind::Attempts => write!(f, "attempts"),
176        }
177    }
178}
179
180impl Error {
181    /// Original error without the accounting wrapper.
182    pub fn cause(&self) -> &Error {
183        match self {
184            Self::Accounted { source, .. } => source.cause(),
185            _ => self,
186        }
187    }
188
189    /// Consume the accounting wrapper after recording its trail.
190    pub fn into_cause(self) -> Error {
191        match self {
192            Self::Accounted { source, .. } => source.into_cause(),
193            _ => self,
194        }
195    }
196
197    /// Run totals on an error from a client with shared credit admission.
198    pub fn run_spend(&self) -> Option<crate::RunSpend> {
199        match self {
200            Self::Accounted { run, .. } => *run,
201            _ => None,
202        }
203    }
204
205    pub(crate) fn with_run(mut self, snapshot: crate::RunSpend) -> Self {
206        if let Self::Accounted { run, .. } = &mut self {
207            *run = Some(snapshot);
208            self
209        } else {
210            let attempts = self.attempts().to_vec();
211            self.accounted(attempts, Some(snapshot))
212        }
213    }
214
215    /// Calls made before this error, including calls with unknown charges.
216    pub fn attempts(&self) -> &[Attempt] {
217        match self {
218            Self::Accounted { attempts, .. }
219            | Self::Exhausted { attempts, .. }
220            | Self::BudgetExceeded { attempts, .. } => attempts,
221            _ => &[],
222        }
223    }
224
225    pub(crate) fn accounted(
226        self,
227        attempts: Vec<Attempt>,
228        run: Option<crate::client::RunSpend>,
229    ) -> Self {
230        Self::Accounted {
231            source: Box::new(self),
232            attempts,
233            run,
234        }
235    }
236
237    /// Whether trying the same call again could plausibly work.
238    ///
239    /// False for anything caused by the request itself, and false for running
240    /// out of credits, which only gets worse with repetition. On the call plane it
241    /// is a rate limit or a status [`ApiStatus::is_transient`] names, the same set
242    /// the send loop retries.
243    pub fn is_retryable(&self) -> bool {
244        match self {
245            Error::Accounted { source, .. } => source.is_retryable(),
246            Error::Api { status, .. } => status.code() == 429 || status.is_transient(),
247            Error::Transport(e) => e.is_timeout() || e.is_connect(),
248            _ => false,
249        }
250    }
251
252    /// What to do about this error.
253    ///
254    /// A rate limit and a transient service failure say to wait, with the
255    /// service's own figure when it sent one. A refused key or a failed sign in
256    /// say to sign in again. A key that was never there, a local path problem,
257    /// an empty balance, a cap and a malformed request or response are
258    /// permanent, because sending again does not create a key, fix a path or
259    /// refill an account. An exhausted walk answers from the call error that
260    /// stopped it, when there was one, and from the policy's reason otherwise.
261    pub fn recovery(&self) -> Recovery {
262        match self {
263            Error::Accounted { source, .. } => source.recovery(),
264            Error::Api {
265                status,
266                retry_after,
267                ..
268            } => {
269                if status.code() == 429 || status.is_transient() {
270                    Recovery::Wait(*retry_after)
271                } else {
272                    Recovery::Permanent
273                }
274            }
275            Error::Transport(e) if e.is_timeout() || e.is_connect() => Recovery::Wait(None),
276            Error::Transport(_) => Recovery::Permanent,
277            Error::Auth { cause, .. } => match cause {
278                AuthCause::Refused | AuthCause::SignInFailed | AuthCause::SignInRequired => {
279                    Recovery::Reauthenticate
280                }
281                AuthCause::NoKey | AuthCause::EmptyKey | AuthCause::Local => Recovery::Permanent,
282            },
283            Error::InsufficientCredits | Error::BudgetExceeded { .. } => Recovery::Permanent,
284            Error::Exhausted { reason, source, .. } => match source.as_deref() {
285                Some(Error::Api {
286                    status,
287                    retry_after,
288                    ..
289                }) if status.code() == 429 || status.is_transient() => Recovery::Wait(*retry_after),
290                _ if matches!(reason, StopReason::Rejected { .. }) => Recovery::Permanent,
291                _ => Recovery::Unknown,
292            },
293            Error::Decode(_) | Error::ResponseTooLarge { .. } | Error::Config(_) => {
294                Recovery::Permanent
295            }
296        }
297    }
298
299    /// The spend that has already happened, when the error knows about any.
300    pub fn spent(&self) -> Credits {
301        match self {
302            Error::Accounted { attempts, .. }
303            | Error::Exhausted { attempts, .. }
304            | Error::BudgetExceeded { attempts, .. } => attempts.iter().map(|a| a.cost).sum(),
305            _ => Credits::ZERO,
306        }
307    }
308}
309
310#[cfg(test)]
311mod tests {
312    // A test may unwrap and may panic: a test that cannot set itself up should
313    // fail loudly rather than quietly measure nothing.
314    #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
315    use super::*;
316    use crate::response::Hint;
317
318    fn api(code: u16, retry_after: Option<Duration>) -> Error {
319        Error::Api {
320            status: ApiStatus::new(code),
321            message: None,
322            retry_after,
323        }
324    }
325
326    fn auth(cause: AuthCause) -> Error {
327        Error::Auth {
328            cause,
329            message: "a message naming no value".to_string(),
330        }
331    }
332
333    fn exhausted(reason: StopReason, source: Option<Error>) -> Error {
334        Error::Exhausted {
335            attempts: Vec::new(),
336            last: None,
337            reason,
338            source: source.map(Box::new),
339        }
340    }
341
342    /// Every stop reason there is, so a test over them cannot quietly miss one
343    /// that a later change adds.
344    const REASONS: [StopReason; 6] = [
345        StopReason::Rejected {
346            hint: Hint::Permanent,
347        },
348        StopReason::OutOfCredits,
349        StopReason::Budget(BudgetKind::Credits),
350        StopReason::RetriesExhausted,
351        StopReason::LadderExhausted,
352        StopReason::Unhandled,
353    ];
354
355    /// A real timeout: a listener that accepts and never answers, against a
356    /// client that gives up first.
357    async fn timed_out() -> reqwest::Error {
358        let listener = std::net::TcpListener::bind((std::net::Ipv4Addr::new(127, 0, 0, 1), 0))
359            .expect("a port");
360        let address = listener.local_addr().expect("an address");
361        let client = reqwest::Client::builder()
362            .timeout(Duration::from_millis(50))
363            .build()
364            .expect("a client");
365        let error = client
366            .get(format!("http://{address}/"))
367            .send()
368            .await
369            .expect_err("a timeout");
370        drop(listener);
371        error
372    }
373
374    /// A real refused connection: a port that was open a moment ago and is not
375    /// any more.
376    async fn refused_connection() -> reqwest::Error {
377        let listener = std::net::TcpListener::bind((std::net::Ipv4Addr::new(127, 0, 0, 1), 0))
378            .expect("a port");
379        let address = listener.local_addr().expect("an address");
380        drop(listener);
381        reqwest::get(format!("http://{address}/"))
382            .await
383            .expect_err("a refused connection")
384    }
385
386    #[test]
387    fn a_rate_limit_says_to_wait_for_what_the_service_asked() {
388        let wait = Some(Duration::from_secs(7));
389        assert_eq!(api(429, wait).recovery(), Recovery::Wait(wait));
390        assert_eq!(api(429, None).recovery(), Recovery::Wait(None));
391    }
392
393    #[test]
394    fn a_transient_service_failure_says_to_wait() {
395        for code in [408, 500, 502, 503, 504] {
396            let wait = Some(Duration::from_secs(2));
397            assert_eq!(api(code, wait).recovery(), Recovery::Wait(wait), "{code}");
398        }
399    }
400
401    #[test]
402    fn a_call_the_service_refused_for_good_is_permanent() {
403        for code in [400, 403, 404, 413, 422] {
404            assert_eq!(api(code, None).recovery(), Recovery::Permanent, "{code}");
405        }
406    }
407
408    #[tokio::test]
409    async fn a_timeout_on_the_wire_says_to_wait_with_no_figure() {
410        let error = Error::Transport(timed_out().await);
411        assert!(error.is_retryable(), "{error}");
412        assert_eq!(error.recovery(), Recovery::Wait(None));
413    }
414
415    #[tokio::test]
416    async fn a_refused_connection_says_to_wait_with_no_figure() {
417        let error = Error::Transport(refused_connection().await);
418        assert!(error.is_retryable(), "{error}");
419        assert_eq!(error.recovery(), Recovery::Wait(None));
420    }
421
422    #[test]
423    fn any_other_transport_failure_is_permanent() {
424        let error = reqwest::Client::new()
425            .get("not a url")
426            .build()
427            .expect_err("a request that cannot be built");
428        assert!(!error.is_timeout() && !error.is_connect());
429        assert_eq!(Error::Transport(error).recovery(), Recovery::Permanent);
430    }
431
432    #[test]
433    fn a_refused_key_or_a_failed_sign_in_says_to_sign_in_again() {
434        assert_eq!(
435            auth(AuthCause::Refused).recovery(),
436            Recovery::Reauthenticate
437        );
438        assert_eq!(
439            auth(AuthCause::SignInFailed).recovery(),
440            Recovery::Reauthenticate
441        );
442    }
443
444    #[test]
445    fn a_key_that_was_never_there_or_a_local_problem_is_permanent() {
446        for cause in [AuthCause::NoKey, AuthCause::EmptyKey, AuthCause::Local] {
447            assert_eq!(auth(cause).recovery(), Recovery::Permanent, "{cause:?}");
448        }
449    }
450
451    #[test]
452    fn an_auth_error_still_reads_as_one() {
453        let shown = auth(AuthCause::Refused).to_string();
454        assert!(shown.starts_with("authentication: "), "{shown}");
455    }
456
457    #[test]
458    fn an_empty_balance_is_permanent() {
459        assert_eq!(Error::InsufficientCredits.recovery(), Recovery::Permanent);
460    }
461
462    #[test]
463    fn a_cap_is_permanent() {
464        for kind in [BudgetKind::Credits, BudgetKind::Time, BudgetKind::Attempts] {
465            let error = Error::BudgetExceeded {
466                kind,
467                attempts: Vec::new(),
468            };
469            assert_eq!(error.recovery(), Recovery::Permanent, "{kind}");
470        }
471    }
472
473    #[test]
474    fn a_walk_a_rate_limit_stopped_says_to_wait_for_the_figure_it_carried() {
475        let wait = Some(Duration::from_secs(9));
476        for reason in REASONS {
477            let error = exhausted(reason, Some(api(429, wait)));
478            assert_eq!(error.recovery(), Recovery::Wait(wait), "{reason:?}");
479        }
480    }
481
482    #[test]
483    fn a_walk_a_transient_failure_stopped_says_to_wait() {
484        for code in [408, 500, 502, 503, 504] {
485            let error = exhausted(StopReason::RetriesExhausted, Some(api(code, None)));
486            assert_eq!(error.recovery(), Recovery::Wait(None), "{code}");
487        }
488    }
489
490    #[test]
491    fn a_walk_the_site_settled_is_permanent_whatever_stopped_the_last_call() {
492        let rejected = StopReason::Rejected {
493            hint: Hint::TryBrowser,
494        };
495        assert_eq!(exhausted(rejected, None).recovery(), Recovery::Permanent);
496        let error = exhausted(rejected, Some(api(403, None)));
497        assert_eq!(error.recovery(), Recovery::Permanent);
498    }
499
500    #[test]
501    fn every_other_exhausted_walk_does_not_say() {
502        for reason in REASONS {
503            if matches!(reason, StopReason::Rejected { .. }) {
504                continue;
505            }
506            assert_eq!(
507                exhausted(reason, None).recovery(),
508                Recovery::Unknown,
509                "{reason:?}"
510            );
511            let error = exhausted(reason, Some(api(403, None)));
512            assert_eq!(error.recovery(), Recovery::Unknown, "{reason:?}");
513        }
514    }
515
516    #[tokio::test]
517    async fn a_walk_a_timeout_stopped_does_not_say() {
518        let error = exhausted(
519            StopReason::RetriesExhausted,
520            Some(Error::Transport(timed_out().await)),
521        );
522        assert_eq!(error.recovery(), Recovery::Unknown);
523    }
524
525    #[test]
526    fn a_body_that_cannot_be_read_is_permanent() {
527        let error =
528            serde_json::from_str::<serde_json::Value>("not json").expect_err("a decode error");
529        assert_eq!(Error::Decode(error).recovery(), Recovery::Permanent);
530    }
531
532    #[test]
533    fn a_configuration_that_cannot_work_is_permanent() {
534        let error = Error::Config("an address that is not one".to_string());
535        assert_eq!(error.recovery(), Recovery::Permanent);
536    }
537
538    #[test]
539    fn an_exhausted_walk_says_why_and_what_stopped_it() {
540        let alone = exhausted(StopReason::LadderExhausted, None).to_string();
541        assert_eq!(
542            alone,
543            "no usable page after 0 attempts: out of escalation steps"
544        );
545
546        let stopped = exhausted(
547            StopReason::RetriesExhausted,
548            Some(Error::Api {
549                status: ApiStatus::new(429),
550                message: Some("slow down".to_string()),
551                retry_after: None,
552            }),
553        );
554        assert_eq!(
555            stopped.to_string(),
556            "no usable page after 0 attempts: out of retries, api status 429: slow down"
557        );
558        let source = std::error::Error::source(&stopped).expect("the call error");
559        assert_eq!(source.to_string(), "api status 429: slow down");
560    }
561}