aws-s3-transfer 1.0.0

Transfer Manager for Amazon S3 - a high-performance data plane client for uploading and downloading objects and directories.
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
/*
 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 * SPDX-License-Identifier: Apache-2.0
 */

//! Retry loop for transfer operations and its classifiers.
//!
//! [`retry`](crate::retry::retry) re-issues an operation up to `MAX_ATTEMPTS` times,
//! deciding per failure via a caller-supplied classifier and backing off per
//! [`Backoff`](crate::retry::Backoff).
//! The loop is deadline-agnostic: any latency deadline is composed by the caller
//! inside the built future (see
//! [`LatencyTracker::guarded`](crate::metrics::latency::LatencyTracker::guarded)),
//! surfacing a timeout as [`crate::retry::GuardError::DeadlineExceeded`].

use std::future::Future;
use std::time::Duration;

use aws_sdk_s3::config::retry::{RetryPartition, TokenBucket};

use crate::error::{Error, ErrorKind};
pub(crate) use crate::metrics::latency::GuardError;
use crate::metrics::RequestMetrics;

/// Maximum attempts per operation before the last error is returned.
const MAX_ATTEMPTS: u32 = 3;

/// Base of the full-jitter backoff: the first retry waits a random duration in
/// `[0, INITIAL_BACKOFF]`, doubling the ceiling each attempt.
///
/// Larger than the SDK's own transient base because this is an *outer* retry
/// that fires only after the SDK's inner retry exhausted; a re-issue should span
/// the window in which in-flight work completes and refills the shared retry
/// quota, rather than collide with the still-drained bucket.
const INITIAL_BACKOFF: Duration = Duration::from_millis(100);

/// Base of the full-jitter backoff for a throttle retry.
///
/// The SEP backs off a throttling error with a 1s base — 10× the transient base
/// — so a service shedding load is given room to recover before a re-issue. A
/// transient transport error (a body-part IO failure) is safe to re-issue fast;
/// a throttle is not, so the two retry reasons carry distinct bases while
/// sharing [`MAX_ATTEMPTS`].
const THROTTLE_INITIAL_BACKOFF: Duration = Duration::from_secs(1);

/// Ceiling for a single backoff delay, before jitter. Capped below the SDK SEP's
/// 20s because a multi-second stall on one part of a larger transfer is
/// pathological.
const MAX_BACKOFF: Duration = Duration::from_secs(5);

/// Full-jitter truncated-exponential backoff schedule: `delay = b · min(initial
/// · 2^i, max)`, `b` uniform in `[0, 1)`.
///
/// Full jitter (the whole capped value scaled, not a band around the base)
/// de-correlates a burst of simultaneous failures by spreading re-issues across
/// the entire `[0, cap]` window.
///
/// Owns no RNG: the caller supplies the `[0, 1)` draw, so [`delay`](Self::delay)
/// is a pure function of `(retry_index, rand_unit)`.
#[derive(Clone, Copy)]
pub(crate) struct Backoff {
    initial: Duration,
    max: Duration,
}

impl Backoff {
    /// Backoff tuned for transient-transport retries (upload part-send and
    /// download body-read paths).
    pub(crate) const fn transient() -> Self {
        Self {
            initial: INITIAL_BACKOFF,
            max: MAX_BACKOFF,
        }
    }

    /// Backoff tuned for throttle retries.
    ///
    /// Uses the SEP's 1s throttling base (10× the transient base), giving a
    /// service that is shedding load room to recover before a re-issue. The SDK
    /// applies this base on its own inner retries, but when its retry quota is
    /// drained it returns the throttle without retrying (and without backoff), so
    /// the outer loop applies the hard base itself.
    pub(crate) const fn throttle() -> Self {
        Self {
            initial: THROTTLE_INITIAL_BACKOFF,
            max: MAX_BACKOFF,
        }
    }

    /// Un-jittered ceiling for a 0-based retry index: `min(initial · 2^i, max)`.
    fn ceiling(&self, retry_index: u32) -> Duration {
        let factor = 2u32.saturating_pow(retry_index);
        self.initial.saturating_mul(factor).min(self.max)
    }

    /// Full-jittered delay for a 0-based retry index (`0` = first retry).
    /// `rand_unit` must be in `[0, 1)`; the result lands in `[0, ceiling(i)]`.
    pub(crate) fn delay(&self, retry_index: u32, rand_unit: f64) -> Duration {
        self.ceiling(retry_index).mul_f64(rand_unit)
    }
}

/// A classifier's verdict for a failed attempt: whether to retry, never how
/// long to wait. [`retry`] owns the schedule, so classifiers are pure functions
/// of the error.
#[derive(Debug, PartialEq, Eq)]
pub(crate) enum RetryDecision {
    /// Re-issue after a fast full-jittered backoff (transient transport or a
    /// latency-deadline straggler).
    Retry,
    /// Re-issue after a hard full-jittered backoff appropriate for a service
    /// shedding load (throttling).
    RetryThrottle,
    /// Give up and return the last error.
    NoRetry,
}

/// Retry an operation up to [`MAX_ATTEMPTS`] times, deciding each failure via
/// `classify` and backing off per `backoff`.
///
/// `metrics` records transfer-manager reissues, retry exhaustion, and selected
/// backoff. The caller remains responsible for recording the logical request's
/// final elapsed time and publishing the completed measurement.
///
/// Each attempt calls `build(allow_hedge)` for a fresh future; `build` must
/// produce an identical request each call so a retry re-sends/re-reads the same
/// data. Any latency deadline is composed inside `build` (via
/// [`LatencyTracker::guarded`](crate::metrics::latency::LatencyTracker::guarded),
/// which arms only when `allow_hedge` is set) and surfaces as
/// [`GuardError::DeadlineExceeded`]; other errors surface as
/// [`GuardError::Inner`]. `allow_hedge` is `false` after the first deadline
/// exceedance (hedge-once), so a build with no deadline can ignore the flag.
///
/// `classify` decides per failure; the loop owns the
/// schedule, selecting the [`Backoff`] by decision — [`RetryDecision::Retry`]
/// uses the fast transient base, [`RetryDecision::RetryThrottle`] the hard
/// throttle base — and sleeps a full-jittered delay keyed on the 0-based retry
/// index. Both retry decisions share [`MAX_ATTEMPTS`]; on [`RetryDecision::NoRetry`]
/// or after the last attempt it returns the error via [`into_error`].
///
/// The one-time hedge (the first [`GuardError::DeadlineExceeded`]) does not
/// consume an attempt: a genuine failure gets the full [`MAX_ATTEMPTS`] whether
/// or not the operation also hedged, so a speculative cancel never steals a
/// transport retry. The loop still terminates — at most one exceedance is free
/// (hedge-once), bounding it to [`MAX_ATTEMPTS`] + 1 iterations.
pub(crate) async fn retry<T, F, Fut>(
    metrics: &mut RequestMetrics,
    classify: impl Fn(&GuardError<Error>) -> RetryDecision,
    mut build: F,
) -> Result<T, Error>
where
    F: FnMut(bool) -> Fut,
    Fut: Future<Output = Result<T, GuardError<Error>>>,
{
    let transient = Backoff::transient();
    let throttle = Backoff::throttle();
    let mut attempt = 1u32;
    // Hedge at most once per operation: the first deadline exceedance cancels and
    // re-issues (the hedge), but a second would mean speculatively cancelling a
    // path already shown to be slow — so once hedged, later attempts run untimed
    // (via `allow_hedge = false`) and ride to completion. This bounds a single
    // operation's deadline-driven cancels to one, so no operation can exhaust its
    // attempt budget on the deadline; the budget bounds aggregate speculation.
    let mut hedged = false;
    loop {
        match build(!hedged).await {
            Ok(val) => return Ok(val),
            Err(ge) => {
                let is_hedge = matches!(ge, GuardError::DeadlineExceeded(_));
                // The one-time hedge does not consume a retry attempt: a genuine
                // failure still gets the full MAX_ATTEMPTS whether or not the
                // operation also hedged. "Free" applies only to the first deadline
                // exceedance (hedge-once); `guarded` runs untimed thereafter, so a
                // further exceedance should not occur, but if one does it consumes
                // an attempt so the loop is still bounded (at most MAX_ATTEMPTS + 1
                // iterations).
                let free_hedge = is_hedge && !hedged;
                // Select the backoff by retry reason; both share MAX_ATTEMPTS.
                let decision = classify(&ge);
                let backoff = match decision {
                    RetryDecision::NoRetry => return Err(into_error(ge)),
                    _ if !free_hedge && attempt >= MAX_ATTEMPTS => {
                        metrics.record_retry_exhaustion();
                        tracing::debug!(
                            target: crate::telemetry::TARGET_TRANSFER,
                            attempts = attempt,
                            max_attempts = MAX_ATTEMPTS,
                            cause = %retry_cause(&ge),
                            "retry budget exhausted, returning last error"
                        );
                        return Err(into_error(ge));
                    }
                    RetryDecision::Retry => &transient,
                    RetryDecision::RetryThrottle => &throttle,
                };
                // `attempt - 1` is the 0-based retry index: 0 on the first
                // failure, so the first backoff uses 2^0. A free hedge reissues at
                // the current index without advancing it, so genuine retries keep
                // their normal backoff progression.
                let delay = backoff.delay(attempt - 1, fastrand::f64());
                if free_hedge {
                    metrics.record_hedge_reissue(delay);
                    tracing::debug!(
                        target: crate::telemetry::TARGET_TRANSFER,
                        backoff_ms = delay.as_millis() as u64,
                        "hedging request after latency deadline (free reissue)"
                    );
                } else {
                    match decision {
                        RetryDecision::Retry => metrics.record_retry_reissue(delay),
                        RetryDecision::RetryThrottle => metrics.record_throttle_reissue(delay),
                        RetryDecision::NoRetry => unreachable!("terminal decisions return above"),
                    }
                    tracing::debug!(
                        target: crate::telemetry::TARGET_TRANSFER,
                        attempt,
                        max_attempts = MAX_ATTEMPTS,
                        throttled = matches!(decision, RetryDecision::RetryThrottle),
                        backoff_ms = delay.as_millis() as u64,
                        cause = %retry_cause(&ge),
                        "retrying operation"
                    );
                }
                // The backoff sleep binds to tokio directly rather than the
                // runtime's `AsyncSleep`, so the loop requires a tokio reactor.
                tokio::time::sleep(delay).await;
                if is_hedge {
                    hedged = true;
                }
                if !free_hedge {
                    attempt += 1;
                }
            }
        }
    }
}

/// Convert a terminal [`GuardError`] into the error returned to the caller.
///
/// [`GuardError::DeadlineExceeded`] carries no inner error, so it becomes an
/// [`ErrorKind::IOError`] naming the deadline
/// the final attempt exceeded. An inner error is returned verbatim.
fn into_error(ge: GuardError<Error>) -> Error {
    match ge {
        GuardError::DeadlineExceeded(deadline) => Error::new(
            ErrorKind::IOError,
            format!(
                "request exceeded the {}ms latency deadline",
                deadline.as_millis()
            ),
        ),
        GuardError::Inner(e) => e,
    }
}

/// Render the cause of a failed attempt for retry log events. `Error`'s
/// `Display` prints only its own context (kind, operation, service error code,
/// request ids), not object keys, buckets, urls, or credentials.
fn retry_cause(ge: &GuardError<Error>) -> String {
    match ge {
        GuardError::DeadlineExceeded(dl) => format!("deadline exceeded ({}ms)", dl.as_millis()),
        GuardError::Inner(e) => e.to_string(),
    }
}

/// Classifier for the upload part-send path.
///
/// Retries a **transient transport** inner error (connection IO error or
/// client-side timeout). The SDK normally retries these over the rewindable
/// in-memory part body, but its retry can be defeated by token-bucket exhaustion
/// when many parts fail concurrently (an ENOBUFS-style burst); the outer
/// re-issue (with the loop's jittered backoff) lands after in-flight parts
/// complete and refill the quota, instead of re-colliding.
///
/// A throttle ([`is_throttle`]) is retried with the hard throttle backoff. Under
/// high fan-out the SDK's shared retry token bucket drains, surfacing throttles
/// un-retried; the tuned per-bucket partition ([`bucket_retry_partition`]) refills
/// that budget over time while the outer re-issue — bounded by [`MAX_ATTEMPTS`]
/// and paced by the 1s throttle base — gives the service room to recover.
///
/// Everything else is terminal: a non-throttle modeled service error and any
/// non-transport inner error. That includes a `412` from a user-set `If-Match` /
/// `If-None-Match` — retrying cannot change the object state that caused it.
///
/// [`GuardError::DeadlineExceeded`]: uploads carry no latency deadline (no
/// `guarded` composition), so this arm is unreachable in practice; retained for
/// exhaustiveness over the shared [`GuardError`].
pub(crate) fn classify_upload_part_retry(ge: &GuardError<Error>) -> RetryDecision {
    match ge {
        GuardError::DeadlineExceeded(_) => RetryDecision::NoRetry,
        GuardError::Inner(e) if e.is_transient_transport() => RetryDecision::Retry,
        GuardError::Inner(e) if is_throttle(e) => RetryDecision::RetryThrottle,
        GuardError::Inner(_) => RetryDecision::NoRetry,
    }
}

/// Whether a transfer error is an S3 throttle (a service error whose code denotes
/// throttling).
///
/// Matches on service error code. The set mirrors the scheduler's throttle
/// classifier (`scheduler::concurrency`). [`classify_discovery_retry`] also
/// recognizes bodyless HEAD errors by HTTP status.
fn is_throttle(e: &Error) -> bool {
    matches!(
        e.code(),
        Some(
            "SlowDown"
                | "Throttling"
                | "ThrottlingException"
                | "RequestLimitExceeded"
                | "BandwidthLimitExceeded"
        )
    )
}

/// Classifier for the download discovery send path.
///
/// Discovery is a single send (GetObject, HeadObject, or GetObject with
/// partNumber) with no body read and no latency deadline. Retries a throttle
/// with the hard throttle backoff and a transient transport error with the fast
/// backoff. A HEAD response with HTTP 429 or 503 also uses throttle backoff:
/// HEAD carries no error body from which the SDK could parse a throttle code.
/// Everything else (including `InvalidRange`, `NotFound`, and precondition
/// failures) is terminal.
pub(crate) fn classify_discovery_retry(ge: &GuardError<Error>) -> RetryDecision {
    match ge {
        GuardError::DeadlineExceeded(_) => RetryDecision::NoRetry,
        GuardError::Inner(e) if e.is_transient_transport() => RetryDecision::Retry,
        GuardError::Inner(e)
            if is_throttle(e)
                || (e.operation_name() == Some("HeadObject")
                    && matches!(e.http_status(), Some(429 | 503))) =>
        {
            RetryDecision::RetryThrottle
        }
        GuardError::Inner(_) => RetryDecision::NoRetry,
    }
}

/// Classifier for download body reads.
///
/// A chunk's body is consumed after the SDK's GetObject orchestration completes,
/// so the SDK's own retry never covers a mid-stream failure. This loop recovers
/// exactly that, re-issuing the ranged GET, and it is the SOLE retrier for the
/// body path. Retries (all with the loop's full-jittered backoff):
///   - a **latency-deadline timeout** (a slow-TTFB straggler) → re-issue on a
///     fresh connection.
///   - a **transient transport** failure on the re-issue's send (connection IO
///     error or client-side timeout) → these correlate under load (e.g. macOS
///     mbuf/ENOBUFS fails many connections at once), so the jittered backoff
///     spreads the retries instead of re-colliding on the same empty pool.
///   - a **body-stream** failure ([`ErrorKind::IOError`]: reset, truncated, short
///     body, SSP stall) → also clusters under a saturated link, same backoff.
///   - a **throttle** ([`is_throttle`]) on the send → re-issue with the hard
///     throttle backoff (see [`classify_upload_part_retry`] for the rationale).
///
/// A checksum mismatch ([`ErrorKind::IntegrityError`], classified at the body-read
/// boundary) is terminal: a corrupt body must never be re-fetched and masked.
/// Everything else is terminal.
pub(crate) fn classify_body_retry(ge: &GuardError<Error>) -> RetryDecision {
    let err = match ge {
        GuardError::DeadlineExceeded(_) => return RetryDecision::Retry,
        GuardError::Inner(e) => e,
    };

    match err.kind() {
        // A checksum mismatch must never be re-fetched and masked.
        ErrorKind::IntegrityError(_) => RetryDecision::NoRetry,
        // Throttle on the send: re-issue with the hard throttle backoff.
        ErrorKind::ServiceError if is_throttle(err) => RetryDecision::RetryThrottle,
        // Transient transport on the send (surfaced as ServiceError with the
        // transient-transport flag).
        ErrorKind::ServiceError if err.is_transient_transport() => RetryDecision::Retry,
        // Mid-stream body-read failure (reset, truncation, SSP stall).
        ErrorKind::IOError => RetryDecision::Retry,
        _ => RetryDecision::NoRetry,
    }
}

/// Per-bucket retry token bucket refill rate, tokens/second.
///
/// The SDK's default bucket has no time-based refill (`refill_rate = 0`): retry
/// budget returns only as requests succeed. Under a high fan-out against one
/// prefix a throttle storm drains it before any success lands, leaving no
/// recovery path — the transfer aborts. A low refill restores a bounded retry
/// budget over time so re-issues proceed once the throttle relents.
///
/// Cached per bucket on the Handle so the budget and its refill are shared
/// across all operations and retries to that bucket; under a total outage the
/// sustained retry rate to S3 is `REFILL_RATE / throttling_retry_cost` (the
/// SDK's throttle cost is 5) ≈ 2 retries/sec per bucket, independent of
/// concurrency. Kept low to trickle-probe recovery rather than re-flood a
/// service that is shedding load; per-operation attempts remain bounded by
/// [`MAX_ATTEMPTS`].
const RETRY_BUCKET_REFILL_RATE: f32 = 10.0;

/// Per-bucket retry token bucket capacity. Matches the SDK default; a larger
/// pool would only absorb a larger initial retry burst (more amplification)
/// without addressing recovery, which [`RETRY_BUCKET_REFILL_RATE`] owns.
const RETRY_BUCKET_CAPACITY: usize = 500;

/// Retry partition for an S3 bucket, carrying a TM-tuned token bucket.
///
/// The SDK shares one retry token bucket per [`RetryPartition`]; the default is
/// region-wide (`s3-{region}`), so a throttle storm on one bucket drains the
/// retry budget for every other bucket on the client. Keying the partition by
/// bucket isolates them: each bucket gets its own token bucket, matching CRT,
/// which partitions per S3 host. The bucket is given a time-based refill
/// ([`RETRY_BUCKET_REFILL_RATE`]) the SDK default lacks, so a drained bucket
/// recovers a bounded retry budget over time.
///
/// Callers MUST NOT call this per-operation: the returned partition carries a
/// freshly-allocated token bucket. Cache the partition on the Handle (via
/// [`Handle::bucket_partition_override`](crate::client::Handle::bucket_partition_override))
/// so all operations to a bucket share one live bucket instance.
pub(crate) fn bucket_retry_partition(bucket: &str) -> RetryPartition {
    RetryPartition::custom(format!("s3-tm-{bucket}"))
        .token_bucket(
            TokenBucket::builder()
                .capacity(RETRY_BUCKET_CAPACITY)
                .refill_rate(RETRY_BUCKET_REFILL_RATE)
                .build(),
        )
        .build()
}

/// Stalled-stream grace period applied to transfer bodies.
///
/// Stalled-stream protection aborts a body whose *throughput* stays at zero for
/// this long, re-issuing the operation on a fresh connection. It fires only on
/// zero byte-progress, so a slow-but-progressing stream is never affected;
/// tightening below the SDK's 5s default only shortens how long a genuinely dead
/// gap wedges the transfer. Bounds a mid-download-body dead connection and a
/// mid-upload-body stall (the transferring peer stops making progress); it does
/// not bound a response that never arrives after the request body is fully sent.
const STALL_GRACE: Duration = Duration::from_secs(2);

/// Stalled-stream protection enabled with the tightened [`STALL_GRACE`].
pub(crate) fn tightened_ssp() -> aws_sdk_s3::config::StalledStreamProtectionConfig {
    aws_sdk_s3::config::StalledStreamProtectionConfig::enabled()
        .grace_period(STALL_GRACE)
        .build()
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::sync::atomic::{AtomicUsize, Ordering};

    /// Classifier for the loop tests: retry a `DeadlineExceeded` or an `IOError`
    /// inner; everything else terminal. Exercises both `GuardError` arms.
    fn classify_test(ge: &GuardError<Error>) -> RetryDecision {
        match ge {
            GuardError::DeadlineExceeded(_) => RetryDecision::Retry,
            GuardError::Inner(e) if *e.kind() == ErrorKind::IOError => RetryDecision::Retry,
            GuardError::Inner(_) => RetryDecision::NoRetry,
        }
    }

    // --- retry loop (pure; no deadline — timing is composed by callers) -----

    #[cfg_attr(miri, ignore)]
    #[tokio::test(start_paused = true)]
    async fn retries_then_succeeds() {
        // First attempt fails retryably (IOError), second succeeds.
        let attempts = AtomicUsize::new(0);
        let mut metrics = RequestMetrics::default();
        let result = retry(&mut metrics, classify_test, |_| {
            let n = attempts.fetch_add(1, Ordering::Relaxed);
            async move {
                if n == 0 {
                    return Err(GuardError::Inner(Error::new(ErrorKind::IOError, "reset")));
                }
                Ok::<_, GuardError<Error>>(42)
            }
        })
        .await;

        assert_eq!(result.unwrap(), 42);
        assert_eq!(attempts.load(Ordering::Relaxed), 2);
        assert_eq!(metrics.retry_reissues, 1);
        assert_eq!(metrics.throttle_reissues, 0);
        assert_eq!(metrics.hedge_reissues, 0);
        assert_eq!(metrics.retry_exhaustions, 0);
    }

    #[cfg_attr(miri, ignore)]
    #[tokio::test(start_paused = true)]
    async fn deadline_exceeded_is_retried() {
        // A `DeadlineExceeded` (what a composed `guarded` produces on timeout) is
        // a retryable arm; the terminal one renders to an IOError via `into_error`.
        // The first exceedance is a free hedge (no attempt consumed), so a build
        // that keeps tripping the deadline runs MAX_ATTEMPTS + 1 times before the
        // last error surfaces. (A real `guarded` cannot exceed the deadline once
        // `allow_hedge` is cleared; this build ignores the flag to drive the loop.)
        let attempts = AtomicUsize::new(0);
        let mut metrics = RequestMetrics::default();
        let result: Result<(), _> = retry(&mut metrics, classify_test, |_| {
            attempts.fetch_add(1, Ordering::Relaxed);
            async { Err::<(), _>(GuardError::DeadlineExceeded(Duration::from_millis(200))) }
        })
        .await;

        let err = result.unwrap_err();
        assert_eq!(err.kind(), &ErrorKind::IOError);
        assert_eq!(attempts.load(Ordering::Relaxed), MAX_ATTEMPTS as usize + 1);
        assert_eq!(metrics.hedge_reissues, 1);
        assert_eq!(metrics.retry_reissues, 2);
        assert_eq!(metrics.retry_exhaustions, 1);
    }

    #[cfg_attr(miri, ignore)]
    #[tokio::test(start_paused = true)]
    async fn hedge_allowed_only_until_first_deadline_exceedance() {
        // hedge-once: the loop passes `allow_hedge = true` until a
        // `DeadlineExceeded` fires, then `false` for every later attempt — so a
        // chunk is speculatively cancelled at most once and later attempts run
        // untimed. Record the flag each attempt.
        let flags = std::sync::Mutex::new(Vec::new());
        let mut metrics = RequestMetrics::default();
        let result: Result<(), _> = retry(&mut metrics, classify_test, |allow_hedge| {
            flags.lock().unwrap().push(allow_hedge);
            // Every call trips the deadline, so after the first (free) hedge the
            // loop must clear the flag. The free hedge does not consume an attempt,
            // so the loop runs MAX_ATTEMPTS + 1 times: one hedge + MAX_ATTEMPTS
            // untimed attempts.
            async { Err::<(), _>(GuardError::DeadlineExceeded(Duration::from_millis(200))) }
        })
        .await;

        assert!(result.is_err());
        assert_eq!(
            *flags.lock().unwrap(),
            vec![true, false, false, false],
            "hedge allowed on the first call only, then cleared; the free hedge \
             adds one iteration on top of MAX_ATTEMPTS"
        );
        assert_eq!(metrics.hedge_reissues, 1);
        assert_eq!(metrics.retry_reissues, 2);
        assert_eq!(metrics.retry_exhaustions, 1);
    }

    #[cfg_attr(miri, ignore)]
    #[tokio::test(start_paused = true)]
    async fn non_deadline_failures_do_not_clear_the_hedge_flag() {
        // Only a `DeadlineExceeded` is a hedge; a transient inner error must not
        // consume the once-per-operation hedge allowance.
        let flags = std::sync::Mutex::new(Vec::new());
        let attempts = AtomicUsize::new(0);
        let mut metrics = RequestMetrics::default();
        let result: Result<(), _> = retry(&mut metrics, classify_test, |allow_hedge| {
            flags.lock().unwrap().push(allow_hedge);
            let n = attempts.fetch_add(1, Ordering::Relaxed);
            async move {
                // Two inner IO errors (not hedges), then succeed.
                if n < 2 {
                    return Err(GuardError::Inner(Error::new(ErrorKind::IOError, "reset")));
                }
                Ok::<(), GuardError<Error>>(())
            }
        })
        .await;

        assert!(result.is_ok());
        assert_eq!(
            *flags.lock().unwrap(),
            vec![true, true, true],
            "inner errors are not hedges; the hedge allowance stays open"
        );
        assert_eq!(metrics.retry_reissues, 2);
        assert_eq!(metrics.hedge_reissues, 0);
        assert_eq!(metrics.retry_exhaustions, 0);
    }

    #[cfg_attr(miri, ignore)]
    #[tokio::test(start_paused = true)]
    async fn hedge_does_not_consume_a_transport_retry() {
        // The seq-700 case: a chunk hedges once (deadline exceedance) and then
        // hits genuine transient-transport failures. The hedge must not steal a
        // transport attempt — the operation still gets the full MAX_ATTEMPTS
        // genuine retries after the free hedge, succeeding on the last one.
        let attempts = AtomicUsize::new(0);
        let mut metrics = RequestMetrics::default();
        let result = retry(&mut metrics, classify_test, |allow_hedge| {
            let n = attempts.fetch_add(1, Ordering::Relaxed);
            async move {
                // Call 0: the one-time hedge (deadline exceedance), free.
                if n == 0 {
                    assert!(allow_hedge, "hedge must be allowed on the first call");
                    return Err(GuardError::DeadlineExceeded(Duration::from_millis(200)));
                }
                // Calls 1..MAX_ATTEMPTS: genuine transport failures (untimed).
                assert!(!allow_hedge, "hedge cleared after the first exceedance");
                if (n as u32) < MAX_ATTEMPTS {
                    return Err(GuardError::Inner(Error::new(ErrorKind::IOError, "connect")));
                }
                // The MAX_ATTEMPTS-th genuine attempt succeeds.
                Ok::<_, GuardError<Error>>(7)
            }
        })
        .await;

        assert_eq!(
            result.unwrap(),
            7,
            "the full transport-retry budget survives a hedge"
        );
        // 1 free hedge + MAX_ATTEMPTS genuine attempts (last succeeds).
        assert_eq!(
            attempts.load(Ordering::Relaxed),
            MAX_ATTEMPTS as usize + 1,
            "hedge is free; genuine failures keep the full MAX_ATTEMPTS"
        );
        assert_eq!(metrics.hedge_reissues, 1);
        assert_eq!(metrics.retry_reissues, 2);
        assert_eq!(metrics.retry_exhaustions, 0);
    }

    #[cfg_attr(miri, ignore)]
    #[tokio::test(start_paused = true)]
    async fn exhausts_attempts_on_persistent_retryable_error() {
        let attempts = AtomicUsize::new(0);
        let mut metrics = RequestMetrics::default();
        let result: Result<(), _> = retry(&mut metrics, classify_test, |_| {
            attempts.fetch_add(1, Ordering::Relaxed);
            async { Err::<(), _>(GuardError::Inner(Error::new(ErrorKind::IOError, "reset"))) }
        })
        .await;

        assert!(result.is_err());
        assert_eq!(attempts.load(Ordering::Relaxed), MAX_ATTEMPTS as usize);
        assert_eq!(metrics.retry_reissues, 2);
        assert_eq!(metrics.retry_exhaustions, 1);
    }

    #[cfg_attr(miri, ignore)]
    #[tokio::test(start_paused = true)]
    async fn does_not_retry_terminal_inner_error() {
        let attempts = AtomicUsize::new(0);
        let mut metrics = RequestMetrics::default();
        let result: Result<(), _> = retry(&mut metrics, classify_test, |_| {
            attempts.fetch_add(1, Ordering::Relaxed);
            async {
                Err::<(), _>(GuardError::Inner(Error::new(
                    ErrorKind::RuntimeError,
                    "simulated SDK error",
                )))
            }
        })
        .await;

        assert!(result.is_err());
        assert_eq!(attempts.load(Ordering::Relaxed), 1);
        assert_eq!(metrics, RequestMetrics::default());
    }

    #[cfg_attr(miri, ignore)]
    #[tokio::test(start_paused = true)]
    async fn throttle_decision_retries_and_uses_the_throttle_backoff() {
        // A `RetryThrottle` decision re-issues up to MAX_ATTEMPTS AND the loop
        // paces it with the throttle base (1s), not the transient base (100ms).
        // This times the REAL loop, so it fails if the loop selects the wrong
        // backoff for a throttle decision.
        //
        // Full jitter makes an unseeded lower bound flaky, so seed `fastrand`:
        // with seed 2 the two throttle draws (ceilings 1s, 2s) sum to ~2.03s of
        // paused-time sleep. The transient base (ceilings 100ms, 200ms) can sleep
        // at most 300ms total for ANY draw, so a `>= 1s` lower bound is reachable
        // only by the throttle schedule and cannot false-pass under transient.
        fn classify_throttle(ge: &GuardError<Error>) -> RetryDecision {
            match ge {
                GuardError::Inner(e) if is_throttle(e) => RetryDecision::RetryThrottle,
                _ => RetryDecision::NoRetry,
            }
        }
        fastrand::seed(2);

        let attempts = AtomicUsize::new(0);
        let start = tokio::time::Instant::now();
        let mut metrics = RequestMetrics::default();
        let result: Result<(), _> = retry(&mut metrics, classify_throttle, |_| {
            attempts.fetch_add(1, Ordering::Relaxed);
            async { Err::<(), _>(GuardError::Inner(Error::test_service_error("SlowDown"))) }
        })
        .await;

        assert!(result.is_err());
        assert_eq!(attempts.load(Ordering::Relaxed), MAX_ATTEMPTS as usize);
        let elapsed = start.elapsed();
        // Lower bound only the throttle base can reach (transient tops out at 300ms
        // across both retries): proves the loop selected the throttle backoff.
        assert!(
            elapsed >= Duration::from_secs(1),
            "elapsed {elapsed:?} is below the throttle base; the loop must use the \
             throttle backoff (1s), not the transient backoff (max 300ms total)"
        );
        // Upper bound: the two throttle ceilings (1s + 2s) cap total sleep at 3s.
        assert!(
            elapsed <= Duration::from_secs(3),
            "elapsed {elapsed:?} exceeds the throttle backoff ceiling sum"
        );
        assert_eq!(metrics.throttle_reissues, 2);
        assert_eq!(metrics.retry_reissues, 0);
        assert_eq!(metrics.retry_exhaustions, 1);
        assert!(
            metrics.backoff_duration.abs_diff(elapsed) <= Duration::from_millis(1),
            "selected backoff {:?} must account for paused-clock elapsed {elapsed:?}",
            metrics.backoff_duration
        );
    }

    // --- Backoff schedule (pure; rand supplied) -----------------------------

    #[test]
    fn backoff_ceiling_progression_doubles_then_caps() {
        // rand_unit = 1.0 yields the full ceiling, so the geometric progression
        // for INITIAL_BACKOFF=100ms, factor 2, MAX_BACKOFF=5s is directly checked.
        let b = Backoff::transient();
        assert_eq!(b.delay(0, 1.0), Duration::from_millis(100));
        assert_eq!(b.delay(1, 1.0), Duration::from_millis(200));
        assert_eq!(b.delay(2, 1.0), Duration::from_millis(400));
        assert_eq!(b.delay(3, 1.0), Duration::from_millis(800));
        assert_eq!(b.delay(4, 1.0), Duration::from_millis(1600));
        assert_eq!(b.delay(5, 1.0), Duration::from_millis(3200));
        // 100ms * 2^6 = 6400ms, capped at MAX_BACKOFF = 5s.
        assert_eq!(b.delay(6, 1.0), Duration::from_secs(5));
        assert_eq!(b.delay(7, 1.0), Duration::from_secs(5));
    }

    #[test]
    fn backoff_large_index_saturates_without_overflow() {
        // 2^u32::MAX must not panic; saturating_pow/mul keep it at the cap.
        let b = Backoff::transient();
        assert_eq!(b.delay(u32::MAX, 1.0), Duration::from_secs(5));
    }

    #[test]
    fn backoff_full_jitter_scales_linearly_within_ceiling() {
        let b = Backoff::transient();
        // index 2 -> 400ms ceiling. Full jitter spans [0, ceiling].
        assert_eq!(b.delay(2, 0.0), Duration::ZERO);
        assert_eq!(b.delay(2, 0.5), Duration::from_millis(200));
        assert_eq!(b.delay(2, 0.25), Duration::from_millis(100));
    }

    #[test]
    fn backoff_real_draws_stay_within_ceiling() {
        let b = Backoff::transient();
        for _ in 0..10_000 {
            let d = b.delay(2, fastrand::f64());
            assert!(
                d <= Duration::from_millis(400),
                "delay {d:?} exceeded ceiling"
            );
        }
    }

    #[test]
    fn throttle_backoff_uses_the_1s_base_and_shares_the_cap() {
        // The throttle base is 1s (10× the transient base); the same 5s cap.
        let b = Backoff::throttle();
        assert_eq!(b.delay(0, 1.0), Duration::from_secs(1));
        assert_eq!(b.delay(1, 1.0), Duration::from_secs(2));
        assert_eq!(b.delay(2, 1.0), Duration::from_secs(4));
        // 1s * 2^3 = 8s, capped at MAX_BACKOFF = 5s.
        assert_eq!(b.delay(3, 1.0), Duration::from_secs(5));
        // Full jitter still spans [0, ceiling].
        assert_eq!(b.delay(0, 0.0), Duration::ZERO);
    }

    // --- classify_upload_part_retry -----------------------------------------

    #[test]
    fn upload_transient_transport_retries() {
        // A transient-transport ServiceError (the ENOBUFS shape) is retried.
        let d = classify_upload_part_retry(&GuardError::Inner(Error::test_transient_transport()));
        assert_eq!(d, RetryDecision::Retry);
    }

    #[test]
    fn upload_non_throttle_service_error_is_terminal() {
        // A modeled service error with no throttle code is terminal: not transient
        // transport, not a throttle. Covers `AccessDenied` and equally a `412` from
        // a caller's `If-Match` / `If-None-Match`, which retrying cannot resolve.
        let err = Error::test_service_error("AccessDenied");
        assert_eq!(
            classify_upload_part_retry(&GuardError::Inner(err)),
            RetryDecision::NoRetry
        );
    }

    #[test]
    fn upload_throttle_retries_with_throttle_decision() {
        // A throttle (SlowDown) is retried with the hard throttle backoff.
        let err = Error::test_service_error("SlowDown");
        assert_eq!(
            classify_upload_part_retry(&GuardError::Inner(err)),
            RetryDecision::RetryThrottle
        );
    }

    #[test]
    fn upload_other_inner_error_is_terminal() {
        let err = Error::new(ErrorKind::RuntimeError, "not a transport error");
        assert_eq!(
            classify_upload_part_retry(&GuardError::Inner(err)),
            RetryDecision::NoRetry
        );
    }

    // --- classify_body_retry ------------------------------------------------

    #[test]
    fn body_retry_deadline_exceeded_retries() {
        let d = classify_body_retry(&GuardError::DeadlineExceeded(Duration::from_secs(1)));
        assert_eq!(d, RetryDecision::Retry);
    }

    #[test]
    fn body_retry_cancellation_does_not_retry() {
        let err = Error::new(ErrorKind::OperationCancelled, "cancelled mid body");
        assert_eq!(
            classify_body_retry(&GuardError::Inner(err)),
            RetryDecision::NoRetry
        );
    }

    #[test]
    fn body_retry_io_error_retries() {
        // A body-stream failure (reset, truncation, SSP stall) is retried.
        let err = Error::new(ErrorKind::IOError, "connection reset by peer");
        assert_eq!(
            classify_body_retry(&GuardError::Inner(err)),
            RetryDecision::Retry
        );
    }

    #[test]
    fn body_retry_transient_transport_retries() {
        // A transient-transport send failure (ENOBUFS/connect/timeout class).
        let d = classify_body_retry(&GuardError::Inner(Error::test_transient_transport()));
        assert_eq!(d, RetryDecision::Retry);
    }

    #[test]
    fn body_retry_throttle_retries_with_throttle_decision() {
        let err = Error::test_service_error("SlowDown");
        assert_eq!(
            classify_body_retry(&GuardError::Inner(err)),
            RetryDecision::RetryThrottle
        );
    }

    #[test]
    fn body_retry_non_throttle_service_error_is_terminal() {
        let err = Error::test_service_error("AccessDenied");
        assert_eq!(
            classify_body_retry(&GuardError::Inner(err)),
            RetryDecision::NoRetry
        );
    }

    #[test]
    fn discovery_head_retry_uses_status_without_an_error_body() {
        use aws_sdk_s3::operation::head_object::HeadObjectError;
        use aws_smithy_runtime_api::{
            client::orchestrator::HttpResponse, client::result::SdkError, http::StatusCode,
        };
        use aws_smithy_types::{body::SdkBody, error::ErrorMetadata};

        for status in [429, 503, 400, 403, 404, 412, 500] {
            let sdk = SdkError::service_error(
                HeadObjectError::generic(ErrorMetadata::builder().build()),
                HttpResponse::new(StatusCode::try_from(status).unwrap(), SdkBody::empty()),
            );
            let error = crate::error::service_error(sdk);
            assert_eq!(error.code(), None);
            assert_eq!(error.http_status(), Some(status));
            assert_eq!(
                classify_discovery_retry(&GuardError::Inner(error)),
                if matches!(status, 429 | 503) {
                    RetryDecision::RetryThrottle
                } else {
                    RetryDecision::NoRetry
                },
                "HEAD status {status}",
            );
        }
    }

    #[test]
    fn discovery_get_retry_does_not_treat_a_codeless_status_as_a_head_error() {
        use aws_sdk_s3::operation::get_object::GetObjectError;
        use aws_smithy_runtime_api::{
            client::orchestrator::HttpResponse, client::result::SdkError, http::StatusCode,
        };
        use aws_smithy_types::{body::SdkBody, error::ErrorMetadata};

        let sdk = SdkError::service_error(
            GetObjectError::generic(ErrorMetadata::builder().build()),
            HttpResponse::new(StatusCode::try_from(503).unwrap(), SdkBody::empty()),
        );
        assert_eq!(
            classify_discovery_retry(&GuardError::Inner(crate::error::service_error(sdk))),
            RetryDecision::NoRetry
        );
    }

    // --- throttle detection -------------------------------------------------

    #[test]
    fn is_throttle_matches_the_known_codes() {
        for code in [
            "SlowDown",
            "Throttling",
            "ThrottlingException",
            "RequestLimitExceeded",
            "BandwidthLimitExceeded",
        ] {
            assert!(
                is_throttle(&Error::test_service_error(code)),
                "{code} must be detected as a throttle"
            );
        }
    }

    #[test]
    fn is_throttle_rejects_non_throttle_and_codeless() {
        assert!(!is_throttle(&Error::test_service_error("AccessDenied")));
        // No service code at all (e.g. a non-service error) is not a throttle.
        assert!(!is_throttle(&Error::new(ErrorKind::IOError, "reset")));
    }

    // The integrity-error-is-terminal contract is covered end-to-end by the
    // `corrupt_body_once_is_not_retried` integration test (a real checksum
    // mismatch through the full stack), which is stronger than a unit assertion
    // over a hand-built error.
}