microvms-domain 1.2.0

The MicroVMs client's rules and values: sizing, cost, regions, names, errors, with no I/O
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
// SPDX-License-Identifier: Apache-2.0
//! One error type, two granularities.
//!
//! # Why two
//!
//! The daemon reports distinct statuses such as `Conflict`, `NotFound`, and
//! `ProtocolError`. The CLI groups them into a coarse process exit category while
//! retaining the finer wire kind in its JSON envelope.
//!
//! [`Error::kind`] answers the exit-category question; [`Error::wire_kind`] answers
//! which daemon condition occurred. The conformance suite checks both contracts.
//!
//! # What a caller matches on
//!
//! [`ErrorKind`], nearly always. [`WireKind`] exists for the two consumers that
//! need the finer view: the conformance driver, and a caller retrying a specific
//! daemon condition. Nobody should parse a message.

use std::fmt;

/// A failure, classified once at the point it is raised.
///
/// Carries the coarse [`ErrorKind`] every consumer branches on, the optional
/// [`WireKind`] naming the daemon status that produced it, a message that names
/// the `docs/PLATFORM.md` finding when a trap closure raised it, and the
/// underlying error when one exists.
///
/// Deliberately a struct with a private body rather than an enum: the variants a
/// caller wants to match on are the *kinds*, and an enum over every raise site
/// would make every new failure a breaking change for a binding that matched
/// exhaustively.
#[derive(Debug, thiserror::Error)]
#[error("{message}")]
pub struct Error {
    kind: ErrorKind,
    wire_kind: Option<WireKind>,
    message: String,
    throttled: bool,
    attempts: u32,
    backoff: std::time::Duration,
    #[source]
    source: Option<Box<dyn std::error::Error + Send + Sync + 'static>>,
}

impl Error {
    /// A failure of `kind`, with a message that should name its finding.
    pub fn new(kind: ErrorKind, message: impl Into<String>) -> Self {
        Self {
            kind,
            wire_kind: None,
            message: message.into(),
            throttled: false,
            attempts: 0,
            backoff: std::time::Duration::ZERO,
            source: None,
        }
    }

    /// A local reject: the request was refused before any AWS call.
    ///
    /// The shorthand exists because every trap closure in this crate ends in one,
    /// and `ERR_INVALID_ARG` is the code that tells a caller the fix is theirs and
    /// costs nothing.
    pub fn invalid_arg(message: impl Into<String>) -> Self {
        Self::new(ErrorKind::InvalidArg, message)
    }

    /// A failure the daemon reported by choosing a status.
    ///
    /// The [`ErrorKind`] is derived from the [`WireKind`] rather than passed in,
    /// because the two must not be able to disagree: `Unauthorized` is
    /// `ERR_CREDENTIALS` and never `ERR_PROTOCOL`, and a call site free to pick
    /// both is a call site that can get that wrong once.
    pub fn wire(wire_kind: WireKind, message: impl Into<String>) -> Self {
        Self {
            kind: wire_kind.error_kind(),
            wire_kind: Some(wire_kind),
            message: message.into(),
            throttled: false,
            attempts: 0,
            backoff: std::time::Duration::ZERO,
            source: None,
        }
    }

    /// Attaches the underlying error, reachable through [`std::error::Error::source`].
    #[must_use]
    pub fn with_source(mut self, source: impl std::error::Error + Send + Sync + 'static) -> Self {
        // Wrapping a refusal (for example, a resume of an ended VM) keeps the
        // work already done by the underlying control-plane request visible.
        if self.attempts == 0
            && let Some(error) =
                (&source as &(dyn std::error::Error + 'static)).downcast_ref::<Self>()
        {
            self.attempts = error.attempts;
            self.backoff = error.backoff;
            self.throttled = error.throttled;
        }
        self.source = Some(Box::new(source));
        self
    }

    /// Marks a control-plane throttle without parsing its explanation.
    #[must_use]
    pub fn with_throttled(mut self, throttled: bool) -> Self {
        self.throttled = throttled;
        self
    }

    /// Attaches the attempts and completed retry sleeps for this control-plane call.
    #[must_use]
    pub fn with_retry_details(mut self, attempts: u32, backoff: std::time::Duration) -> Self {
        self.attempts = attempts;
        self.backoff = backoff;
        self
    }

    /// Whether the service refused this call because of throttling.
    pub fn throttled(&self) -> bool {
        self.throttled
    }

    /// Attempts sent for the failed call, or zero when no request telemetry is available.
    pub fn attempt_count(&self) -> u32 {
        self.attempts
    }

    /// Total completed retry sleeps for this call.
    pub fn backoff(&self) -> std::time::Duration {
        self.backoff
    }

    /// The coarse class, one per row of the exit-code table.
    pub fn kind(&self) -> ErrorKind {
        self.kind
    }

    /// The daemon status this came from, when it came from one.
    ///
    /// `None` for every local reject and every control-plane failure — those never
    /// reached the in-VM daemon, so there is no status to report.
    pub fn wire_kind(&self) -> Option<WireKind> {
        self.wire_kind
    }

    /// The machine-readable `ERR_*` code, as `cli.py`'s `Code` spells it.
    pub fn code(&self) -> &'static str {
        self.kind.code()
    }

    /// Whether retrying the identical request could plausibly succeed.
    ///
    /// Derived from the kind rather than stored, so it cannot drift from it: the
    /// five retryable daemon conditions all map to [`ErrorKind::Retryable`], which
    /// is what makes this one comparison instead of a second table.
    pub fn retryable(&self) -> bool {
        self.kind.retryable()
    }
}

/// The coarse failure classes, one per non-zero row of the CLI's exit table.
///
/// These categories are mapped to stable process exit codes by the CLI.
/// The integer exit code lives in the CLI, not here: a library that owned process
/// exit codes would be a library with an opinion about being a process.
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
pub enum ErrorKind {
    /// No handler claimed this — a bug in this crate, not the platform.
    Unexpected,
    /// Refused locally, before any AWS call. Every trap closure lands here.
    InvalidArg,
    /// Transient. Run the identical request again.
    Retryable,
    /// An identity is wrong or absent; waiting will not fix it.
    Credentials,
    /// The daemon rejected the request on its merits.
    Protocol,
    /// The image build was never scheduled — the `clientToken` replay signature.
    BuildWedged,
    /// The MicroVM reached a terminal state before RUNNING; read `stateReason`.
    LaunchDied,
    /// The MicroVM ended before the resume (the suspended window, the maximum duration, or a
    /// terminate), and it can't be resumed; launch a new one. The service refused the resume,
    /// and `GetMicrovm` reports the VM TERMINATING or TERMINATED.
    WindowClosed,
    /// A control-plane failure with no more specific class.
    Platform,
    /// A client-side deadline elapsed. The VM and the exec are untouched.
    Timeout,
    /// Interrupted after launch; teardown ran and any leak is named in the payload.
    Interrupted,
    /// A prerequisite is missing.
    Precondition,
    /// The sandbox worked and the command in it exited non-zero.
    ///
    /// Its own class because it is the one failure that means nothing is wrong with
    /// the platform, the credentials, or this client — a CI caller needs to tell
    /// "your tests failed" from "we never got a VM", and one shared class cannot say
    /// both.
    ExecFailed,
}

impl ErrorKind {
    /// Every kind, in exit-code order.
    ///
    /// Public so a binding can enumerate the catalog and a test can assert the
    /// mapping is total rather than sampling it.
    pub const ALL: [ErrorKind; 13] = [
        ErrorKind::Unexpected,
        ErrorKind::InvalidArg,
        ErrorKind::Retryable,
        ErrorKind::Credentials,
        ErrorKind::Protocol,
        ErrorKind::BuildWedged,
        ErrorKind::LaunchDied,
        ErrorKind::WindowClosed,
        ErrorKind::Platform,
        ErrorKind::Timeout,
        ErrorKind::Interrupted,
        ErrorKind::Precondition,
        ErrorKind::ExecFailed,
    ];

    /// Whether a failure of this kind could plausibly succeed on an identical retry.
    ///
    /// The one answer [`Error::retryable`] reads, public so a binding that carries only
    /// the code across its boundary (the TypeScript one) asks this rather than keeping a
    /// list of retryable codes of its own.
    pub fn retryable(self) -> bool {
        matches!(self, ErrorKind::Retryable)
    }

    /// The `ERR_*` string, byte-identical to `cli.py`'s `Code` member.
    ///
    /// A string beside the eventual integer because the two are read by different
    /// consumers: a shell branches on `$?`, an agent parsing `--json` branches on
    /// `code` and should never have to keep an integer table.
    pub fn code(self) -> &'static str {
        match self {
            ErrorKind::Unexpected => "ERR_UNEXPECTED",
            ErrorKind::InvalidArg => "ERR_INVALID_ARG",
            ErrorKind::Retryable => "ERR_RETRYABLE",
            ErrorKind::Credentials => "ERR_CREDENTIALS",
            ErrorKind::Protocol => "ERR_PROTOCOL",
            ErrorKind::BuildWedged => "ERR_BUILD_WEDGED",
            ErrorKind::LaunchDied => "ERR_LAUNCH_DIED",
            ErrorKind::WindowClosed => "ERR_WINDOW_CLOSED",
            ErrorKind::Platform => "ERR_PLATFORM",
            ErrorKind::Timeout => "ERR_TIMEOUT",
            ErrorKind::Interrupted => "ERR_INTERRUPTED",
            ErrorKind::Precondition => "ERR_PRECONDITION",
            ErrorKind::ExecFailed => "ERR_EXEC_FAILED",
        }
    }
}

impl fmt::Display for ErrorKind {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        f.write_str(self.code())
    }
}

/// The fine-grained taxonomy: one variant per condition the daemon or the
/// transport distinguishes.
///
/// These are the `errors.py` exception *classes*, which is what the conformance
/// suite asserts on. Several collapse onto one [`ErrorKind`] and that is the
/// point — the collapse happens at the exit code, not at the raise site.
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
pub enum WireKind {
    /// 401: the presented bearer token is not the installed one. Fatal.
    Unauthorized,
    /// 400: malformed body, missing query key, bad mode, refused tar member.
    ///
    /// Never 404. A 404 here reads as a missing file, which is exactly how one
    /// defect hid for a review round.
    ProtocolError,
    /// 404: a genuinely absent exec id, file, or directory.
    NotFound,
    /// 409: well-formed, but the target is in the wrong state.
    Conflict,
    /// 410: stdin already saw EOF, or the child stopped reading.
    ///
    /// Distinct from [`WireKind::Conflict`], which is "you did not ask for stdin"
    /// and is fixed at start time. This one is a lifecycle fact.
    StdinClosed,
    /// 413: over a configured cap (body, tar members or bytes, stdin write).
    TooLarge,
    /// 408: the child did not drain its stdin pipe within the write timeout.
    ///
    /// Retryable, and the daemon deliberately keeps its stdin handle open so a
    /// retry can succeed. Some bytes may already have landed; reconciling that is
    /// the caller's problem, which is why it is a distinct variant.
    RequestTimeout,
    /// 503: the run hook has not landed, so the control API is closed.
    ///
    /// Not 404 and not a dropped connection. Retry: the platform is about to
    /// deliver the token.
    NotBootstrapped,
    /// 5xx other than 503 and 507: spawn failure, io failure, a panicking task.
    ServerError,
    /// 507: a write would take the target filesystem under the daemon's disk reserve.
    ///
    /// Not retryable, and that's why the daemon answers 507 rather than 500
    /// (`crates/agentd/src/fs.rs`): an identical retry against a full disk fails the
    /// same way, and a retry loop keyed on a 5xx would repeat it. The body names the
    /// free and reserved bytes. The remedy is freeing space, then retrying.
    InsufficientStorage,
    /// The request never produced a status: connection refused, reset, timeout.
    ///
    /// Retryable because it says nothing about the daemon's state. A VM that has
    /// just reached RUNNING commonly refuses a connection or two before the proxy
    /// path is wired up.
    Transport,
    /// `CreateMicrovmAuthToken` failed.
    ///
    /// Retryable, and load-bearing rather than optimistic: proxy tokens expire at
    /// 60 minutes, so a long run *will* mint mid-flight, and a throttle at that
    /// moment must not kill a trial that is otherwise healthy.
    AuthTokenMint,
    /// A client-side wait or stream deadline elapsed. The exec is untouched.
    ExecTimeout,
    /// Output bytes are gone for good — the replay ring evicted them, or this
    /// subscriber lagged the live channel.
    OutputGap,
}

impl WireKind {
    /// Every variant, so a test can assert the mapping is total.
    pub const ALL: [WireKind; 14] = [
        WireKind::Unauthorized,
        WireKind::ProtocolError,
        WireKind::NotFound,
        WireKind::Conflict,
        WireKind::StdinClosed,
        WireKind::TooLarge,
        WireKind::RequestTimeout,
        WireKind::NotBootstrapped,
        WireKind::ServerError,
        WireKind::InsufficientStorage,
        WireKind::Transport,
        WireKind::AuthTokenMint,
        WireKind::ExecTimeout,
        WireKind::OutputGap,
    ];

    /// The stable name the CLI puts in `data.kind` and the bindings use for the
    /// corresponding exception. Conformance tests compare these names.
    pub fn as_str(self) -> &'static str {
        match self {
            WireKind::Unauthorized => "Unauthorized",
            WireKind::ProtocolError => "ProtocolError",
            WireKind::NotFound => "NotFound",
            WireKind::Conflict => "Conflict",
            WireKind::StdinClosed => "StdinClosed",
            WireKind::TooLarge => "TooLarge",
            WireKind::RequestTimeout => "RequestTimeout",
            WireKind::NotBootstrapped => "NotBootstrapped",
            WireKind::ServerError => "ServerError",
            WireKind::InsufficientStorage => "InsufficientStorage",
            WireKind::Transport => "Transport",
            WireKind::AuthTokenMint => "AuthTokenMint",
            WireKind::ExecTimeout => "ExecTimeout",
            WireKind::OutputGap => "OutputGap",
        }
    }

    /// The status that produces this variant, for the ones that come from one.
    ///
    /// `None` for the four that have no status: two never got a response
    /// ([`WireKind::Transport`], [`WireKind::AuthTokenMint`]) and two are client-side
    /// facts ([`WireKind::ExecTimeout`], [`WireKind::OutputGap`]).
    pub fn status(self) -> Option<u16> {
        match self {
            WireKind::ProtocolError => Some(400),
            WireKind::Unauthorized => Some(401),
            WireKind::NotFound => Some(404),
            WireKind::RequestTimeout => Some(408),
            WireKind::Conflict => Some(409),
            WireKind::StdinClosed => Some(410),
            WireKind::TooLarge => Some(413),
            WireKind::ServerError => Some(500),
            WireKind::NotBootstrapped => Some(503),
            WireKind::InsufficientStorage => Some(507),
            WireKind::Transport
            | WireKind::AuthTokenMint
            | WireKind::ExecTimeout
            | WireKind::OutputGap => None,
        }
    }

    /// The variant a response status means, or `None` for a status the daemon does
    /// not use to mean anything.
    ///
    /// The table is explicit and there is deliberately **no generic 4xx fallback**.
    /// The daemon's whole point is that 400 and 404 mean different things, and a
    /// fallback that mapped "some 4xx" to one variant would reintroduce the
    /// phantom-missing-file defect that hid in the Python client for a review
    /// round. 5xx does fall back — to [`WireKind::ServerError`], which is what
    /// `errors.py` does — because every 5xx means the same thing to a caller:
    /// the daemon broke, try again. The two the daemon chooses on purpose keep their
    /// own variants: 503 is "come back in a moment", and 507 is "free space first".
    pub fn from_status(status: u16) -> Option<WireKind> {
        match status {
            400 => Some(WireKind::ProtocolError),
            401 => Some(WireKind::Unauthorized),
            404 => Some(WireKind::NotFound),
            408 => Some(WireKind::RequestTimeout),
            409 => Some(WireKind::Conflict),
            410 => Some(WireKind::StdinClosed),
            413 => Some(WireKind::TooLarge),
            503 => Some(WireKind::NotBootstrapped),
            // Before the 5xx fallback, which would make a full disk retryable.
            507 => Some(WireKind::InsufficientStorage),
            s if s >= 500 => Some(WireKind::ServerError),
            _ => None,
        }
    }

    /// The exit-code class this collapses onto.
    ///
    /// The ordering `cli.py`'s `classify` documents as "the contract" is a `match`
    /// here rather than a sequence of `isinstance` tests, which removes the way it
    /// could be got wrong: [`WireKind::Unauthorized`] is an HTTP error whose remedy
    /// is a credential rather than a wait, and in Python it had to be *checked
    /// before* the generic retryable test to avoid being swallowed by it. A match on
    /// a closed enum has no order to get wrong.
    pub fn error_kind(self) -> ErrorKind {
        match self {
            // A credential, not a wait. Retrying a 401 forever is one of the two
            // mistakes the Python ordering existed to prevent.
            WireKind::Unauthorized => ErrorKind::Credentials,
            // The retryable daemon conditions. Failing a launch that
            // was 200 ms from ready is the other mistake.
            WireKind::Transport
            | WireKind::AuthTokenMint
            | WireKind::NotBootstrapped
            | WireKind::RequestTimeout
            | WireKind::ServerError => ErrorKind::Retryable,
            // The daemon rejected the request on its merits. Four classes, one exit
            // code: a shell cannot act differently on them, and a caller that can
            // reads `data.kind`.
            WireKind::ProtocolError
            | WireKind::NotFound
            | WireKind::Conflict
            | WireKind::StdinClosed
            | WireKind::TooLarge => ErrorKind::Protocol,
            // A client-side deadline. `cli.py` reaches ERR_PLATFORM for this by
            // accident of `isinstance` ordering — `ExecTimeout` is an `AgentdError`
            // and is tested before the `TimeoutError` branch — but ERR_TIMEOUT's own
            // row reads "a client-side deadline elapsed; the VM and the exec are
            // untouched", which is `ExecTimeout`'s docstring. Corrected here.
            WireKind::ExecTimeout => ErrorKind::Timeout,
            // No better row exists: a gap is not the caller's argument, not the
            // daemon refusing the request, and not retryable — the bytes are gone.
            // `cli.py` reaches ERR_PLATFORM for it too, and this one is right.
            WireKind::OutputGap => ErrorKind::Platform,
            // A condition of the VM rather than of the request, and not retryable
            // until space is freed: the row the CLI's `sync` already reported it on.
            WireKind::InsufficientStorage => ErrorKind::Platform,
        }
    }

    /// Whether retrying the identical request could plausibly succeed.
    ///
    /// The `errors.py` contract, restated as an independent table so the test below
    /// can compare it against [`Error::retryable`] rather than trusting the two
    /// agree. Test-only and deliberately not public: a second public answer to
    /// "is this retryable" is a second thing to keep in step, and the production
    /// answer must stay derived from [`WireKind::error_kind`] so a new variant
    /// cannot be retryable in one place and fatal in the other.
    #[cfg(test)]
    fn retryable(self) -> bool {
        matches!(
            self,
            WireKind::Transport
                | WireKind::AuthTokenMint
                | WireKind::NotBootstrapped
                | WireKind::RequestTimeout
                | WireKind::ServerError
        )
    }
}

impl fmt::Display for WireKind {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        f.write_str(self.as_str())
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn retry_details_distinguish_unmeasured_and_observed_failures() {
        for error in [
            Error::new(ErrorKind::Platform, "local failure"),
            Error::wire(WireKind::Transport, "daemon failure"),
        ] {
            assert!(!error.throttled());
            assert_eq!(error.attempt_count(), 0);
            assert_eq!(error.backoff(), std::time::Duration::ZERO);
        }

        let delay = std::time::Duration::from_millis(750);
        let error = Error::new(ErrorKind::Retryable, "service throttle")
            .with_throttled(true)
            .with_retry_details(4, delay);
        assert!(error.throttled());
        assert_eq!(error.attempt_count(), 4);
        assert_eq!(error.backoff(), delay);
    }

    #[test]
    fn wrapping_retry_details_inherits_only_when_outer_has_no_attempts() {
        let source = || {
            Error::new(ErrorKind::Retryable, "service throttle")
                .with_throttled(true)
                .with_retry_details(4, std::time::Duration::from_millis(750))
        };
        let inherited = Error::new(ErrorKind::WindowClosed, "wrapped").with_source(source());
        assert_eq!(inherited.kind(), ErrorKind::WindowClosed);
        assert!(inherited.throttled());
        assert_eq!(inherited.attempt_count(), 4);
        assert_eq!(inherited.backoff(), std::time::Duration::from_millis(750));

        let own_delay = std::time::Duration::from_millis(200);
        let own = Error::new(ErrorKind::Platform, "outer request")
            .with_retry_details(2, own_delay)
            .with_source(source());
        assert!(!own.throttled());
        assert_eq!(own.attempt_count(), 2);
        assert_eq!(own.backoff(), own_delay);
        assert!(
            std::error::Error::source(&own)
                .unwrap()
                .downcast_ref::<Error>()
                .unwrap()
                .throttled()
        );

        let unrelated =
            Error::new(ErrorKind::Platform, "formatting failure").with_source(std::fmt::Error);
        assert_eq!(unrelated.attempt_count(), 0);
        assert!(!unrelated.throttled());
        assert_eq!(unrelated.backoff(), std::time::Duration::ZERO);
    }

    /// The thirteen codes, spelled as `cli.py` spells them. Written out rather
    /// than generated, because a generated list would agree with a typo.
    #[test]
    fn every_kind_carries_its_python_err_code() {
        let codes: Vec<&str> = ErrorKind::ALL.iter().map(|k| k.code()).collect();
        assert_eq!(
            codes,
            [
                "ERR_UNEXPECTED",
                "ERR_INVALID_ARG",
                "ERR_RETRYABLE",
                "ERR_CREDENTIALS",
                "ERR_PROTOCOL",
                "ERR_BUILD_WEDGED",
                "ERR_LAUNCH_DIED",
                "ERR_WINDOW_CLOSED",
                "ERR_PLATFORM",
                "ERR_TIMEOUT",
                "ERR_INTERRUPTED",
                "ERR_PRECONDITION",
                "ERR_EXEC_FAILED",
            ]
        );
    }

    /// Thirteen distinct codes for thirteen kinds. Two kinds sharing a code would
    /// make the CLI's failure envelope ambiguous, which is the one thing the string
    /// beside the integer exists to prevent.
    #[test]
    fn no_two_kinds_share_a_code() {
        let mut seen: Vec<&str> = ErrorKind::ALL.iter().map(|k| k.code()).collect();
        seen.sort_unstable();
        let before = seen.len();
        seen.dedup();
        assert_eq!(seen.len(), before, "duplicate ERR_* code");
    }

    /// `retryable()` reads the kind, and the kind is derived from the wire kind, so
    /// the only way the two can disagree is a wrong row in `error_kind`. Compared
    /// against the `errors.py` table restated independently in `WireKind::retryable`.
    #[test]
    fn retryable_agrees_with_the_python_exception_contract() {
        for wire in WireKind::ALL {
            let err = Error::wire(wire, "detail");
            assert_eq!(
                err.retryable(),
                wire.retryable(),
                "{wire} disagrees with errors.py on retryability"
            );
        }
    }

    /// Exactly the five `errors.py` marks retryable, named so a sixth added by
    /// mistake fails here rather than in a retry loop that never terminates.
    #[test]
    fn exactly_five_wire_kinds_are_retryable() {
        let retryable: Vec<&str> = WireKind::ALL
            .iter()
            .filter(|w| w.retryable())
            .map(|w| w.as_str())
            .collect();
        assert_eq!(
            retryable,
            [
                "RequestTimeout",
                "NotBootstrapped",
                "ServerError",
                "Transport",
                "AuthTokenMint"
            ]
        );
    }

    /// A 401 is a credential and never a protocol error, whatever order the
    /// classification is written in. This is the `isinstance`-ordering trap from
    /// `cli.py` made unreachable.
    #[test]
    fn unauthorized_is_a_credential_failure_not_a_protocol_one() {
        let err = Error::wire(WireKind::Unauthorized, "401");
        assert_eq!(err.kind(), ErrorKind::Credentials);
        assert_eq!(err.code(), "ERR_CREDENTIALS");
        assert!(!err.retryable());
    }

    /// The load-bearing distinction: 400 and 404 are different variants, and no
    /// generic 4xx fallback can produce either. 402 is a 4xx the daemon never
    /// chooses, and it must map to nothing at all rather than to the nearest
    /// neighbour.
    #[test]
    fn no_generic_four_hundred_fallback_can_produce_a_protocol_error() {
        assert_eq!(WireKind::from_status(400), Some(WireKind::ProtocolError));
        assert_eq!(WireKind::from_status(404), Some(WireKind::NotFound));
        for unmapped in [402, 403, 405, 418, 429, 451] {
            assert_eq!(
                WireKind::from_status(unmapped),
                None,
                "{unmapped} must not resolve to a daemon-chosen variant"
            );
        }
    }

    /// 5xx *does* fall back, because every 5xx means the same thing to a caller.
    /// 503 is the exception and keeps its own variant: "come back in a moment" is
    /// not "the daemon broke". 507 is the other one, below.
    #[test]
    fn five_hundreds_fall_back_to_server_error_except_the_bootstrap_one() {
        assert_eq!(WireKind::from_status(500), Some(WireKind::ServerError));
        assert_eq!(WireKind::from_status(502), Some(WireKind::ServerError));
        assert_eq!(WireKind::from_status(506), Some(WireKind::ServerError));
        assert_eq!(WireKind::from_status(508), Some(WireKind::ServerError));
        assert_eq!(WireKind::from_status(599), Some(WireKind::ServerError));
        assert_eq!(WireKind::from_status(503), Some(WireKind::NotBootstrapped));
    }

    /// A 507 is `InsufficientStorage`, `ERR_PLATFORM`, and not retryable (#256).
    ///
    /// The daemon answers 507 when a write would take the filesystem under its disk
    /// reserve, and it chose 507 over 500 because a client retries a 500: right for a
    /// defect, harmful for a full disk. The 5xx fallback would have made it
    /// `ServerError`, one of the retryable five, on every surface at once.
    ///
    /// **Falsification**: delete the 507 arm from `from_status` and this is red, since
    /// 507 then resolves to `ServerError` and its error is `ERR_RETRYABLE`.
    #[test]
    fn a_disk_pressure_507_is_insufficient_storage_and_not_retryable() {
        let wire = WireKind::from_status(507);
        assert_eq!(
            wire,
            Some(WireKind::InsufficientStorage),
            "507 resolves to {wire:?}"
        );
        let err = Error::wire(
            wire.expect("507 is mapped"),
            "PUT /v1/fs/file?path=%2Fw -> 507: 4096 bytes available, below the reserve",
        );
        assert_eq!(err.kind(), ErrorKind::Platform);
        assert_eq!(err.code(), "ERR_PLATFORM");
        assert!(!err.retryable(), "a full disk is not retryable unchanged");
        assert_eq!(WireKind::InsufficientStorage.status(), Some(507));
        assert_eq!(
            WireKind::InsufficientStorage.as_str(),
            "InsufficientStorage"
        );
    }

    /// `Error::retryable` is its kind's answer, for every kind, so a binding that asks
    /// the kind (it carries only the code) gets the answer a Rust caller gets.
    #[test]
    fn an_errors_retryability_is_its_kinds() {
        for kind in ErrorKind::ALL {
            assert_eq!(
                Error::new(kind, "detail").retryable(),
                kind.retryable(),
                "{kind}"
            );
        }
        let retryable: Vec<ErrorKind> = ErrorKind::ALL
            .into_iter()
            .filter(|kind| kind.retryable())
            .collect();
        assert_eq!(retryable, [ErrorKind::Retryable]);
    }

    /// Every status in the table round-trips: the status a variant reports is the
    /// status that produces it. Two tables that must agree are one table that will
    /// not, and this is the cheap way to keep them one.
    #[test]
    fn status_and_from_status_are_inverses_where_both_are_defined() {
        for wire in WireKind::ALL {
            if let Some(status) = wire.status() {
                assert_eq!(
                    WireKind::from_status(status),
                    Some(wire),
                    "{wire} reports {status} but {status} resolves elsewhere"
                );
            }
        }
    }

    /// A local reject has no wire kind: nothing reached the daemon, so there is no
    /// status to report and the CLI's `data.kind` must be absent rather than
    /// invented.
    #[test]
    fn a_local_reject_carries_no_wire_kind() {
        let err = Error::invalid_arg("refused before any AWS call");
        assert_eq!(err.kind(), ErrorKind::InvalidArg);
        assert_eq!(err.wire_kind(), None);
        assert_eq!(err.to_string(), "refused before any AWS call");
    }

    /// The underlying error stays reachable through `source()`, so a caller can
    /// print a chain rather than a message that already threw the cause away.
    #[test]
    fn an_attached_source_survives_into_the_error_trait() {
        let io = std::io::Error::other("connection reset");
        let err = Error::new(ErrorKind::Retryable, "endpoint request failed").with_source(io);
        let source = std::error::Error::source(&err).expect("source is reachable");
        assert_eq!(source.to_string(), "connection reset");
    }
}