Skip to main content

autumn_web/
http_client.rs

1//! Traced outbound HTTP client with retries and test mocks.
2//!
3//! Exposes [`Client`](crate::http_client::Client) as `autumn_web::http::Client` — a thin `reqwest`-backed
4//! outbound HTTP client that propagates the active span's `traceparent` /
5//! `tracestate` headers and the current request's `x-request-id`, opens one
6//! CLIENT span per attempt, retries transient failures, and is mockable in
7//! tests via [`TestApp::http_mock`](crate::test::TestApp::http_mock).
8//!
9//! # Quick start
10//!
11//! ```rust,no_run
12//! use autumn_web::prelude::*;
13//! use autumn_web::http::Client;
14//!
15//! #[post("/pay")]
16//! async fn pay(client: Client) -> AutumnResult<Json<serde_json::Value>> {
17//!     let resp = client
18//!         .post("https://api.stripe.com/v1/charges")
19//!         .header("authorization", "Bearer sk_test_xxx")
20//!         .json(&serde_json::json!({"amount": 1000, "currency": "usd"}))
21//!         .send()
22//!         .await?;
23//!     Ok(Json(resp.json()?))
24//! }
25//! ```
26//!
27//! # Test mocks
28//!
29//! ```rust,no_run
30//! use autumn_web::test::TestApp;
31//! use autumn_web::prelude::*;
32//! use serde_json::json;
33//!
34//! // (handler shown above)
35//!
36//! #[tokio::test]
37//! async fn pay_calls_stripe() {
38//!     let mut app = TestApp::new().routes(routes![pay]);
39//!     let mock = app.http_mock("stripe")
40//!         .post("/v1/charges")
41//!         .respond_with(200, json!({"id": "ch_123", "amount": 1000}));
42//!
43//!     let client = app.build();
44//!     client.post("/pay").send().await.assert_status(200);
45//!     mock.expect_called(1);
46//! }
47//! ```
48
49// autumn-determinism-gate: production code in this module must read time and
50// mint identifiers through the framework's injected seams (ClockSource /
51// Entropy), never `Instant::now()` / `Utc::now()` / `SystemTime::now()` /
52// `Uuid::new_v4()` directly. See CONTRIBUTING.md "Determinism seam gate"
53// (issue #1797). Justify exceptions with
54// #[allow(clippy::disallowed_methods, reason = "…")] at the narrowest scope.
55#![cfg_attr(not(test), deny(clippy::disallowed_methods))]
56
57use std::collections::HashMap;
58use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr};
59use std::sync::atomic::{AtomicBool, AtomicU32, AtomicUsize, Ordering};
60use std::sync::{Arc, Mutex};
61use std::time::{Duration, Instant};
62
63use bytes::Bytes;
64use reqwest::Method;
65use reqwest::header::{HeaderMap, HeaderName, HeaderValue};
66use serde::Serialize;
67use serde::de::DeserializeOwned;
68
69use crate::config::RetryBudgetConfig;
70use crate::deadline::{DEADLINE_HEADER, Deadline};
71use crate::retry_budget::{RetryBudget, RetryBudgets, RetryKind};
72
73// ── Error ────────────────────────────────────────────────────────────────────
74
75/// Errors produced by [`Client`] and [`RequestBuilder`].
76///
77/// `#[non_exhaustive]`: match it with a `_` arm, so a new variant is not a
78/// breaking change.
79#[non_exhaustive]
80#[derive(Debug, thiserror::Error)]
81pub enum ClientError {
82    /// An underlying `reqwest` transport error.
83    #[error("outbound HTTP request failed: {0}")]
84    Request(#[from] reqwest::Error),
85    /// JSON (de)serialisation failed.
86    #[error("JSON error: {0}")]
87    Json(#[from] serde_json::Error),
88    /// No mock entry matched the outgoing request.
89    #[error("no mock registered for {0} {1}")]
90    NoMock(String, String),
91    /// The outbound circuit breaker is open.
92    #[error("outbound circuit breaker is open")]
93    CircuitBreakerOpen,
94    /// A staging fault (`[fault_injection]`, target `http`) failed the call
95    /// before it was sent.
96    #[error("{0}")]
97    FaultInjected(String),
98    /// The client-side adaptive throttle rejected the attempt locally,
99    /// because the host rejected too many recent attempts (issue #3068).
100    /// See `[http.client.adaptive_throttle]`.
101    #[error("outbound request to {host} throttled locally: the host rejects too many requests")]
102    ThrottledLocally {
103        /// The host (`host` or `host:port`).
104        host: String,
105    },
106    /// The capsule recorded a transport failure for this call, and replay
107    /// reproduced it (#1634).
108    ///
109    /// A recorded transport error cannot be rebuilt as a `reqwest::Error` — it
110    /// has no public constructor — but it must not be downgraded to a status
111    /// either: a handler whose bug is "we do not handle a connection reset"
112    /// has to meet the reset again.
113    ///
114    /// Its `Display` is the recorded text *exactly*, with no replay marker of
115    /// its own. A handler that propagates this error puts the text into the
116    /// capsule's outcome, and the replay verdict compares outcome messages
117    /// verbatim — so a marker here would report an unchanged failure as a
118    /// mismatch, which is the one thing the comparison exists to get right.
119    #[error("{0}")]
120    ReplayedRequestFailure(String),
121    /// Outbound HTTP is blocked because this process is replaying a failure
122    /// capsule (see the crate-internal `block_outbound_for_replay`).
123    #[error(
124        "outbound HTTP is not recorded in a failure capsule and is blocked during replay: {0} {1}"
125    )]
126    BlockedDuringReplay(String, String),
127    /// A resolved connection target (or redirect target) is a blocked
128    /// (private / link-local / loopback / reserved) IP address per the built-in
129    /// SSRF policy. See [`is_blocked_ip`].
130    #[error("SSRF policy blocked address: {0}")]
131    SsrfBlocked(String),
132    /// A redirect chain exceeded the configured maximum number of hops.
133    #[error("too many redirects (max {0})")]
134    TooManyRedirects(usize),
135    /// A redirect's absolute `Location` target was rejected by the caller's
136    /// validator (or by the built-in scheme-downgrade guard).
137    #[error("redirect rejected: {0}")]
138    RedirectRejected(String),
139    /// A URL could not be parsed, was missing a host, or DNS resolution failed
140    /// while composing a custom (redirect / pin / SSRF-safe) request.
141    #[error("invalid or unresolvable URL: {0}")]
142    InvalidUrl(String),
143    /// [`pin_to`](RequestBuilder::pin_to) was combined with
144    /// [`follow_redirects`](RequestBuilder::follow_redirects) — an incompatible
145    /// pair. A single pinned `SocketAddr` only covers the first hop; later
146    /// redirect hops re-resolve via normal DNS, so following a cross-host `3xx`
147    /// would silently escape the pin and defeat its purpose. Use
148    /// [`Client::get_ssrf_safe`] for pinned, per-hop-revalidated redirect
149    /// following; `pin_to` alone (returns the `3xx` unfollowed); or
150    /// `follow_redirects` without `pin_to`.
151    #[error("{0}")]
152    IncompatiblePinRedirect(&'static str),
153    /// [`pin_to`](RequestBuilder::pin_to) was used on a request whose URL host is
154    /// an IP literal. reqwest/hyper treat an IP-literal host as already-resolved
155    /// and never consult the DNS resolver, so the `resolve_to_addrs` override
156    /// that installs the pin is skipped and the socket connects to the literal in
157    /// the URL — not the pinned address — silently bypassing the pin. `pin_to`
158    /// therefore requires a domain (hostname) host; put the desired IP directly
159    /// in the URL, or use a domain host. (`get_ssrf_safe` never sets a pin: it
160    /// validates the literal and connects to that same IP, so it is unaffected.)
161    #[error("{0}")]
162    PinRequiresDomainHost(&'static str),
163    /// [`pin_to`](RequestBuilder::pin_to) was combined with
164    /// [`Client::get_ssrf_safe`] — an incompatible pair. The SSRF-safe path runs
165    /// its own per-hop resolve→validate→pin and never reads the caller's
166    /// `pin_to` address, so an explicit pin would be silently ignored. Use
167    /// `pin_to` alone for a caller-chosen fixed address, or `get_ssrf_safe`
168    /// alone for guarded automatic per-hop pinning — not both.
169    #[error("{0}")]
170    PinNotAllowedWithSsrfSafe(&'static str),
171    /// The simulated network ([`crate::sim::SimNet`], issue #2967) failed the
172    /// call: a drop, a partition, a timeout, a host it does not know, or a
173    /// request or response it could not build.
174    #[error("simulated network: {0}")]
175    SimNetwork(String),
176
177    /// The request deadline passed before an attempt could start (issue
178    /// #3058). See [`crate::deadline`].
179    #[error("request deadline exceeded before the outbound request could start")]
180    DeadlineExceeded,
181}
182
183// ── Response ─────────────────────────────────────────────────────────────────
184
185/// Completed outbound HTTP response with eagerly-collected body bytes.
186///
187/// Body is consumed once — call exactly one of [`json`](Self::json),
188/// [`text`](Self::text), or [`bytes`](Self::bytes).
189#[derive(Debug)]
190pub struct Response {
191    status: reqwest::StatusCode,
192    headers: HeaderMap,
193    body: Bytes,
194    url: Option<reqwest::Url>,
195}
196
197impl Response {
198    /// HTTP status code.
199    pub const fn status(&self) -> reqwest::StatusCode {
200        self.status
201    }
202
203    /// Response headers (sensitive values are **not** redacted here).
204    pub const fn headers(&self) -> &HeaderMap {
205        &self.headers
206    }
207
208    /// `true` when the status code is in the 2xx range.
209    pub fn is_success(&self) -> bool {
210        self.status.is_success()
211    }
212
213    /// URL that was ultimately requested (after redirects, if any).
214    pub const fn url(&self) -> Option<&reqwest::Url> {
215        self.url.as_ref()
216    }
217
218    /// Deserialise the body as JSON.
219    ///
220    /// # Errors
221    /// Returns [`ClientError::Json`] if the body is not valid JSON for `T`.
222    pub fn json<T: DeserializeOwned>(self) -> Result<T, ClientError> {
223        serde_json::from_slice(&self.body).map_err(ClientError::Json)
224    }
225
226    /// Return the body as a UTF-8 string (lossy).
227    pub fn text(self) -> String {
228        String::from_utf8_lossy(&self.body).into_owned()
229    }
230
231    /// Return the raw body bytes.
232    pub fn bytes(self) -> Bytes {
233        self.body
234    }
235}
236
237// ── SSRF address policy ──────────────────────────────────────────────────────
238
239/// Return `true` when `ip` must **not** be connected to because it belongs to a
240/// private, loopback, link-local, CGNAT, benchmarking, documentation, multicast
241/// or otherwise reserved range.
242///
243/// This is the built-in Server-Side Request Forgery (SSRF) deny-list used by
244/// [`Client::get_ssrf_safe`]. Ranges are checked explicitly (rather than via the
245/// unstable `IpAddr::is_global` family) so the code compiles on stable Rust.
246///
247/// IPv6 addresses that embed an IPv4 via a transition mechanism — IPv4-mapped
248/// (`::ffff:a.b.c.d`), the deprecated IPv4-compatible (`::a.b.c.d`), NAT64
249/// (`64:ff9b::/96`), 6to4 (`2002::/16`), and the SIIT IPv4-translated prefix
250/// (`::ffff:0:0:0/96`) — are unwrapped and re-checked as IPv4, so encodings
251/// such as `::ffff:169.254.169.254`, `64:ff9b::a9fe:a9fe`, `2002:a9fe:a9fe::`,
252/// and `::ffff:0:169.254.169.254` are all correctly blocked.
253#[must_use]
254pub fn is_blocked_ip(ip: IpAddr) -> bool {
255    match ip {
256        IpAddr::V4(v4) => is_blocked_ipv4(v4),
257        IpAddr::V6(v6) => is_blocked_ipv6(v6),
258    }
259}
260
261/// Return `true` when `ip` is safe to connect to — the exact negation of
262/// [`is_blocked_ip`].
263#[must_use]
264pub fn is_public_ip(ip: IpAddr) -> bool {
265    !is_blocked_ip(ip)
266}
267
268fn is_blocked_ipv4(ip: Ipv4Addr) -> bool {
269    let [a, b, c, _d] = ip.octets();
270    // 0.0.0.0/8 (incl. unspecified)
271    if a == 0 {
272        return true;
273    }
274    // 10.0.0.0/8
275    if a == 10 {
276        return true;
277    }
278    // 100.64.0.0/10 (CGNAT)
279    if a == 100 && (64..=127).contains(&b) {
280        return true;
281    }
282    // 127.0.0.0/8 (loopback)
283    if a == 127 {
284        return true;
285    }
286    // 169.254.0.0/16 (link-local, incl. 169.254.169.254 cloud metadata)
287    if a == 169 && b == 254 {
288        return true;
289    }
290    // 172.16.0.0/12
291    if a == 172 && (16..=31).contains(&b) {
292        return true;
293    }
294    // 192.0.0.0/24 (IETF protocol assignments)
295    if a == 192 && b == 0 && c == 0 {
296        return true;
297    }
298    // 192.0.2.0/24 (TEST-NET-1)
299    if a == 192 && b == 0 && c == 2 {
300        return true;
301    }
302    // 192.88.99.0/24 (6to4 anycast relay, RFC 3068 / RFC 7526)
303    if a == 192 && b == 88 && c == 99 {
304        return true;
305    }
306    // 192.168.0.0/16
307    if a == 192 && b == 168 {
308        return true;
309    }
310    // 198.18.0.0/15 (benchmarking)
311    if a == 198 && (18..=19).contains(&b) {
312        return true;
313    }
314    // 198.51.100.0/24 (TEST-NET-2)
315    if a == 198 && b == 51 && c == 100 {
316        return true;
317    }
318    // 203.0.113.0/24 (TEST-NET-3)
319    if a == 203 && b == 0 && c == 113 {
320        return true;
321    }
322    // 224.0.0.0/4 (multicast) and 240.0.0.0/4 (reserved, incl. 255.255.255.255)
323    if a >= 224 {
324        return true;
325    }
326    false
327}
328
329/// Extract any IPv4 address embedded in an IPv6 address via a transition
330/// mechanism, so the IPv4 SSRF policy can be re-applied to it:
331///
332/// - IPv4-mapped `::ffff:a.b.c.d`
333/// - deprecated IPv4-compatible `::a.b.c.d`
334/// - NAT64 well-known prefix `64:ff9b::/96` (RFC 6052) — e.g.
335///   `64:ff9b::a9fe:a9fe` decodes to `169.254.169.254`
336/// - 6to4 `2002::/16` (RFC 3056) — e.g. `2002:a9fe:a9fe::` embeds
337///   `169.254.169.254`
338/// - SIIT "IPv4-translated" prefix `::ffff:0:0:0/96` (RFC 6052) — e.g.
339///   `::ffff:0:169.254.169.254` decodes to `169.254.169.254`. Note this is a
340///   DIFFERENT segment layout from IPv4-mapped `::ffff:0:0/96` (here
341///   `segments()[4] == 0xffff && segments()[5] == 0`, whereas IPv4-mapped has
342///   `segments()[5] == 0xffff`), so it is NOT caught by `to_ipv4_mapped()`.
343///
344/// Returns `None` for a genuinely-native IPv6 address (no embedded v4). Using
345/// `octets()` avoids any lossy `u16 -> u8` casts.
346fn embedded_ipv4(ip: Ipv6Addr) -> Option<Ipv4Addr> {
347    if let Some(v4) = ip.to_ipv4_mapped() {
348        return Some(v4);
349    }
350    let segs = ip.segments();
351    // SIIT IPv4-translated `::ffff:0:0:0/96` (RFC 6052): 64 zero bits, then
352    // 0xffff, then 16 zero bits, then the IPv4 in the last 32 bits. Distinct
353    // from IPv4-mapped `::ffff:0:0/96` (segment[5] == 0xffff) — here
354    // segment[4] == 0xffff and segment[5] == 0 — so `to_ipv4_mapped()` above
355    // does NOT catch it. Decode it so the embedded IPv4 is re-checked by the
356    // IPv4 policy (e.g. `::ffff:0:169.254.169.254` must be blocked).
357    if segs[0] == 0
358        && segs[1] == 0
359        && segs[2] == 0
360        && segs[3] == 0
361        && segs[4] == 0xffff
362        && segs[5] == 0
363    {
364        let o = ip.octets();
365        return Some(Ipv4Addr::new(o[12], o[13], o[14], o[15]));
366    }
367    let o = ip.octets();
368    // NAT64 64:ff9b::/96 — embedded IPv4 in the last 32 bits (octets 12..16),
369    // with octets 4..12 all zero.
370    if o[0] == 0x00
371        && o[1] == 0x64
372        && o[2] == 0xff
373        && o[3] == 0x9b
374        && o[4..12].iter().all(|&b| b == 0)
375    {
376        return Some(Ipv4Addr::new(o[12], o[13], o[14], o[15]));
377    }
378    // 6to4 2002::/16 — embedded IPv4 in bits 16..48 (octets 2..6).
379    if o[0] == 0x20 && o[1] == 0x02 {
380        return Some(Ipv4Addr::new(o[2], o[3], o[4], o[5]));
381    }
382    // Deprecated IPv4-compatible ::a.b.c.d (upper 96 bits zero).
383    ip.to_ipv4()
384}
385
386fn is_blocked_ipv6(ip: Ipv6Addr) -> bool {
387    // Block any private / loopback / link-local / metadata IPv4 that is
388    // tunnelled inside this v6 address (IPv4-mapped, IPv4-compatible, NAT64,
389    // 6to4, or SIIT IPv4-translated `::ffff:0:0:0/96`) by re-running the IPv4
390    // policy on the embedded address. A public
391    // embedded IPv4 (e.g. 6to4 `2002:0808:0808::` == 8.8.8.8) is NOT blocked
392    // here — it falls through to the native v6 range checks below.
393    if let Some(v4) = embedded_ipv4(ip)
394        && is_blocked_ipv4(v4)
395    {
396        return true;
397    }
398
399    let segs = ip.segments();
400    // RFC 8215 local-use NAT64 prefix `64:ff9b:1::/48`: deny the whole prefix
401    // outright. Unlike the well-known `64:ff9b::/96` (where a public embedded
402    // IPv4 like 8.8.8.8 is legitimately allowed), this is a private/site-local
403    // NAT64 allocation with no legitimate public destination, and the RFC 6052
404    // embedding position varies with prefix length — a blanket deny is both
405    // simpler and strictly safer (e.g. `64:ff9b:1::a9fe:a9fe` == 169.254.169.254).
406    if segs[0] == 0x0064 && segs[1] == 0xff9b && segs[2] == 0x0001 {
407        return true;
408    }
409    // :: (unspecified)
410    if ip == Ipv6Addr::UNSPECIFIED {
411        return true;
412    }
413    // ::1 (loopback)
414    if ip == Ipv6Addr::LOCALHOST {
415        return true;
416    }
417    // fc00::/7 (unique local address)
418    if segs[0] & 0xfe00 == 0xfc00 {
419        return true;
420    }
421    // fe80::/10 (link-local)
422    if segs[0] & 0xffc0 == 0xfe80 {
423        return true;
424    }
425    // fec0::/10 (deprecated site-local — defence-in-depth)
426    if segs[0] & 0xffc0 == 0xfec0 {
427        return true;
428    }
429    // ff00::/8 (multicast)
430    if segs[0] & 0xff00 == 0xff00 {
431        return true;
432    }
433    // 2001:db8::/32 (documentation)
434    if segs[0] == 0x2001 && segs[1] == 0x0db8 {
435        return true;
436    }
437
438    // Remaining IANA IPv6 special-purpose prefixes with Globally Reachable =
439    // False. These are reserved / benchmarking / documentation / discard ranges
440    // that must never be dialled, matching the deny-list's reserved policy.
441    // Each check uses explicit masks so it cannot catch an adjacent *public*
442    // address (e.g. benchmarking 2001:2::/48 must not swallow Teredo 2001:0::/32
443    // where s[1] == 0, and documentation 2001:db8::/32 stays its own check).
444    //
445    //   Prefix               RFC        Purpose
446    //   100::/64             RFC 6666   Discard-Only address block
447    //   2001:2::/48          RFC 5180   Benchmarking
448    //   2001:10::/28         RFC 4843   ORCHID (deprecated)
449    //   2001:20::/28         RFC 7343   ORCHIDv2
450    //   3fff::/20            RFC 9637   Documentation
451    //   5f00::/16            RFC 9602   Segment Routing (SRv6) SIDs
452    //   2620:4f:8000::/48    RFC 7534   Direct Delegation AS112 service
453    let s = segs;
454    // 100::/64 (Discard-Only, RFC 6666)
455    if s[0] == 0x0100 && s[1] == 0 && s[2] == 0 && s[3] == 0 {
456        return true;
457    }
458    // 2001:2::/48 (Benchmarking, RFC 5180)
459    if s[0] == 0x2001 && s[1] == 0x0002 && s[2] == 0x0000 {
460        return true;
461    }
462    // 2001:10::/28 (ORCHID, deprecated RFC 4843)
463    if s[0] == 0x2001 && (s[1] & 0xFFF0) == 0x0010 {
464        return true;
465    }
466    // 2001:20::/28 (ORCHIDv2, RFC 7343)
467    if s[0] == 0x2001 && (s[1] & 0xFFF0) == 0x0020 {
468        return true;
469    }
470    // 3fff::/20 (Documentation, RFC 9637)
471    if s[0] == 0x3fff && (s[1] & 0xF000) == 0x0000 {
472        return true;
473    }
474    // 5f00::/16 (SRv6 SIDs, RFC 9602)
475    if s[0] == 0x5f00 {
476        return true;
477    }
478    // 2620:4f:8000::/48 (Direct Delegation AS112, RFC 7534)
479    if s[0] == 0x2620 && s[1] == 0x004f && s[2] == 0x8000 {
480        return true;
481    }
482
483    false
484}
485
486// ── RetryPolicy ──────────────────────────────────────────────────────────────
487
488/// Retry configuration for a [`RequestBuilder`].
489///
490/// The wait before retry `n` (0 = first retry) is full jitter:
491/// `random(0, min(max_backoff, 100 ms * 2^n))`. See [`crate::backoff`].
492#[derive(Clone, Debug)]
493pub struct RetryPolicy {
494    /// Maximum number of additional attempts after the first failure.  Zero
495    /// means no retries (one attempt total).
496    pub max_retries: u32,
497    /// When `true` (the default), only GET / HEAD / PUT / DELETE / OPTIONS /
498    /// TRACE are retried; POST and PATCH are not. Clear it with
499    /// [`RequestBuilder::retry_non_idempotent`].
500    pub retry_idempotent_only: bool,
501    /// Cap on a `Retry-After` hint. The wait is also at most the backoff
502    /// plus 5 s, so a value above 5 s has no effect.
503    pub max_retry_after: Duration,
504    /// Per-request timeout.
505    pub request_timeout: Option<Duration>,
506    /// Cap on the jittered backoff between attempts. Default: 20 s.
507    pub max_backoff: Duration,
508}
509
510impl Default for RetryPolicy {
511    fn default() -> Self {
512        Self {
513            max_retries: 3,
514            retry_idempotent_only: true,
515            max_retry_after: Duration::from_secs(10),
516            request_timeout: Some(Duration::from_secs(30)),
517            max_backoff: Duration::from_millis(crate::backoff::DEFAULT_HTTP_MAX_BACKOFF_MS),
518        }
519    }
520}
521
522/// Base of the exponential retry backoff.
523const BASE_BACKOFF: Duration = Duration::from_millis(100);
524
525/// The header that [`RequestBuilder::retry_non_idempotent`] adds.
526const IDEMPOTENCY_KEY: &str = "idempotency-key";
527
528impl RetryPolicy {
529    /// The wait before the retry that follows failed attempt `attempt`
530    /// (0-indexed).
531    ///
532    /// Without a hint, this is the jittered backoff. With a `Retry-After`
533    /// hint, the hint is capped by `max_retry_after` and `request_timeout`.
534    /// The wait is then `backoff + min(hint, 5 s)`. See
535    /// [`crate::backoff::retry_after_wait`].
536    fn retry_delay(
537        &self,
538        entropy: &dyn crate::entropy::Entropy,
539        attempt: u32,
540        hint: Option<Duration>,
541    ) -> Duration {
542        let backoff = crate::backoff::full_jitter(entropy, BASE_BACKOFF, self.max_backoff, attempt);
543        let Some(mut hint) = hint else {
544            return backoff;
545        };
546        hint = hint.min(self.max_retry_after);
547        if let Some(timeout) = self.request_timeout {
548            hint = hint.min(timeout);
549        }
550        crate::backoff::retry_after_wait(hint, backoff)
551    }
552}
553
554/// `true` for a status the client retries: `429` and `502`-`504`.
555const fn is_retryable_response(status: u16) -> bool {
556    status == 429 || is_retryable_status(status)
557}
558
559/// The server's wait hint for a retryable status: `Retry-After` on `429`
560/// (1 s when absent) and on `503`.
561fn retry_hint(status: u16, headers: &HeaderMap) -> Option<Duration> {
562    match status {
563        429 => Some(parse_retry_after(headers).unwrap_or(Duration::from_secs(1))),
564        503 => parse_retry_after(headers),
565        _ => None,
566    }
567}
568
569// ── MockRegistry ─────────────────────────────────────────────────────────────
570
571/// Internal mock entry stored by [`MockRegistry`].
572pub(crate) struct MockEntry {
573    pub(crate) method: Option<Method>,
574    /// URL path to match against the path component of the outbound URL.
575    pub(crate) path: String,
576    /// Optional alias that must match the `Client`'s alias.
577    pub(crate) alias: Option<String>,
578    pub(crate) status: u16,
579    pub(crate) body: Option<serde_json::Value>,
580    pub(crate) call_count: Arc<AtomicUsize>,
581}
582
583/// Canned response returned by a [`MockRegistry`] match.
584pub(crate) struct MockResponse {
585    pub(crate) status: u16,
586    pub(crate) body: Option<serde_json::Value>,
587}
588
589/// In-process mock registry used by [`TestApp::http_mock`](crate::test::TestApp::http_mock).
590///
591/// Stored in [`AppState`](crate::AppState) extensions during test builds so
592/// that any [`Client`] extracted from state will intercept matching requests
593/// and return canned responses without hitting the network.
594pub struct MockRegistry {
595    entries: Mutex<Vec<MockEntry>>,
596}
597
598impl MockRegistry {
599    /// Create an empty registry.
600    #[must_use]
601    pub const fn new() -> Self {
602        Self {
603            entries: Mutex::new(Vec::new()),
604        }
605    }
606
607    /// Register a new mock entry.
608    pub(crate) fn register(&self, entry: MockEntry) {
609        self.entries
610            .lock()
611            .expect("mock registry lock poisoned")
612            .push(entry);
613    }
614
615    /// Find the first entry matching `(method, url, alias)` and increment its
616    /// call counter.  Returns `None` when no entry matches.
617    pub(crate) fn find_match(
618        &self,
619        method: &Method,
620        url: &str,
621        alias: Option<&str>,
622    ) -> Option<MockResponse> {
623        // Extract the URL path component for matching, stripping query and fragment.
624        // For full URLs (https://…) reqwest::Url::parse gives us the clean path.
625        // For relative paths we strip manually so "?query" doesn't break matching.
626        // Extract the path without query/fragment. For full URLs reqwest parses
627        // cleanly; for relative strings we strip manually.
628        let url_path_owned: String = reqwest::Url::parse(url).map_or_else(
629            |_| {
630                let s = url.split_once('?').map_or(url, |(p, _)| p);
631                s.split_once('#').map_or(s, |(p, _)| p).to_owned()
632            },
633            |parsed| parsed.path().to_owned(),
634        );
635        let url_path = url_path_owned.as_str();
636
637        // Hold the lock only for the search; release before fetching metadata.
638        let found = {
639            let entries = self.entries.lock().expect("mock registry lock poisoned");
640            entries.iter().find_map(|entry| {
641                let method_ok = entry.method.as_ref().is_none_or(|m| m == method);
642                // Path match: exact equality OR suffix at a segment boundary.
643                // When the mock path starts with '/' the leading slash IS the
644                // segment separator, so a non-empty prefix is also valid
645                // (e.g. mock "/charges" matches URL path "/v1/charges").
646                let path_ok = url_path == entry.path.as_str()
647                    || url_path
648                        .strip_suffix(entry.path.as_str())
649                        .is_some_and(|prefix| {
650                            prefix.is_empty()
651                                || prefix.ends_with('/')
652                                || entry.path.starts_with('/')
653                        });
654                let alias_ok = entry
655                    .alias
656                    .as_deref()
657                    .is_none_or(|a| alias.is_some_and(|b| a == b));
658                if method_ok && path_ok && alias_ok {
659                    Some((entry.call_count.clone(), entry.status, entry.body.clone()))
660                } else {
661                    None
662                }
663            })
664        };
665
666        found.map(|(call_count, status, body)| {
667            call_count.fetch_add(1, Ordering::SeqCst);
668            MockResponse { status, body }
669        })
670    }
671}
672
673impl Default for MockRegistry {
674    fn default() -> Self {
675        Self::new()
676    }
677}
678
679/// Set while this process is replaying a failure capsule, blocking every
680/// outbound request (see [`block_outbound_for_replay`]).
681static OUTBOUND_BLOCKED: AtomicBool = AtomicBool::new(false);
682
683/// Fail every outbound HTTP request from now on.
684///
685/// Called by the capsule replay one-shot
686/// ([`AppBuilder::run`](crate::app::AppBuilder::run) under
687/// `AUTUMN_REPLAY_CAPSULE`), which rebuilds the real application — including
688/// its real `reqwest` client — and must not let it reach anything the capsule
689/// did not record. There is deliberately no way to unset it: a process that has
690/// started replaying a capsule never goes back to serving traffic.
691#[cfg_attr(not(feature = "reporting"), allow(dead_code))]
692pub(crate) fn block_outbound_for_replay() {
693    OUTBOUND_BLOCKED.store(true, Ordering::SeqCst);
694}
695
696/// Whether outbound HTTP is blocked for capsule replay.
697#[must_use]
698pub(crate) fn outbound_blocked_for_replay() -> bool {
699    OUTBOUND_BLOCKED.load(Ordering::SeqCst)
700}
701
702// ── Failure-capsule seam (#1634) ─────────────────────────────────────────────
703//
704// The whole seam is behind the `reporting` feature, which is what gates the
705// `capsule` module. `OutboundRecorder` has a no-op twin below so `send` reads
706// the same on either build.
707
708/// The prefix of a recorded header value that is escaped bytes.
709#[cfg(feature = "reporting")]
710const OPAQUE_HEADER_PREFIX: &str = "[bytes] ";
711
712/// The headers the *caller* set, in the order they were set.
713///
714/// One definition for both halves of the seam: the recorder stores this, and
715/// replay compares against it. Headers the client adds later — the injected
716/// W3C trace context, and any default the underlying `reqwest::Client` was
717/// built with — are deliberately outside it, so a client-side default can
718/// never be read as the handler changing its request. So is a Web Bot Auth
719/// `Signature` and `Signature-Input`: the client makes them new for each
720/// send, so they differ on every replay.
721#[cfg(feature = "reporting")]
722fn caller_headers(builder: &RequestBuilder) -> Vec<(String, String)> {
723    builder
724        .extra_headers
725        .iter()
726        .filter(|(name, _)| {
727            builder.web_bot_auth.is_none()
728                || !matches!(name.as_str(), "signature" | "signature-input")
729        })
730        .map(|(name, value)| {
731            (
732                name.as_str().to_owned(),
733                // An opaque value is kept as escaped bytes, so two different
734                // values never record the same text. A text value with the
735                // prefix is escaped too, so it cannot look like bytes.
736                match value.to_str() {
737                    Ok(text) if !text.starts_with(OPAQUE_HEADER_PREFIX) => text.to_owned(),
738                    _ => format!("{OPAQUE_HEADER_PREFIX}{}", value.as_bytes().escape_ascii()),
739                },
740            )
741        })
742        .collect()
743}
744
745/// Serve one outbound call from a capsule's effect tape.
746///
747/// Which [`HttpErrorKind`] a failure is, so replay can rebuild the variant and
748/// not merely quote the text.
749#[cfg(feature = "reporting")]
750const fn http_error_kind(error: &ClientError) -> crate::capsule::schema::HttpErrorKind {
751    use crate::capsule::schema::HttpErrorKind as Kind;
752    match error {
753        ClientError::Json(_) => Kind::Json,
754        ClientError::NoMock(..) => Kind::NoMock,
755        ClientError::CircuitBreakerOpen => Kind::CircuitBreakerOpen,
756        ClientError::ThrottledLocally { .. } => Kind::ThrottledLocally,
757        ClientError::SsrfBlocked(_) => Kind::SsrfBlocked,
758        ClientError::TooManyRedirects(_) => Kind::TooManyRedirects,
759        ClientError::RedirectRejected(_) => Kind::RedirectRejected,
760        ClientError::InvalidUrl(_) => Kind::InvalidUrl,
761        ClientError::FaultInjected(_) => Kind::FaultInjected,
762        // Everything else — a `reqwest` transport error, and the replay-only
763        // variants a recorded run cannot have produced — keeps its text alone.
764        _ => Kind::Transport,
765    }
766}
767
768/// Rebuild the error a recorded call produced, as the handler observed it.
769///
770/// Variants whose payload survives in the recording are rebuilt exactly, so
771/// code that branches on them takes the branch it took in production. The rest
772/// come back as [`ClientError::ReplayedRequestFailure`], whose `Display` is the
773/// recorded text verbatim — the closest an unreconstructible foreign error can
774/// be brought.
775#[cfg(feature = "reporting")]
776fn rebuild_client_error(
777    kind: Option<crate::capsule::schema::HttpErrorKind>,
778    text: String,
779) -> ClientError {
780    use crate::capsule::schema::HttpErrorKind as Kind;
781    match kind {
782        Some(Kind::CircuitBreakerOpen) => ClientError::CircuitBreakerOpen,
783        Some(Kind::ThrottledLocally) => text
784            .strip_prefix("outbound request to ")
785            .and_then(|rest| rest.split_once(" throttled locally"))
786            .map_or_else(
787                || ClientError::ReplayedRequestFailure(text.clone()),
788                |(host, _)| ClientError::ThrottledLocally {
789                    host: host.to_owned(),
790                },
791            ),
792        Some(Kind::SsrfBlocked) => {
793            ClientError::SsrfBlocked(strip_prefix_payload(&text, "SSRF policy blocked address: "))
794        }
795        Some(Kind::RedirectRejected) => {
796            ClientError::RedirectRejected(strip_prefix_payload(&text, "redirect rejected: "))
797        }
798        Some(Kind::InvalidUrl) => {
799            ClientError::InvalidUrl(strip_prefix_payload(&text, "invalid or unresolvable URL: "))
800        }
801        Some(Kind::FaultInjected) => ClientError::FaultInjected(text),
802        Some(Kind::TooManyRedirects) => text
803            .trim_end_matches(')')
804            .rsplit_once("(max ")
805            .and_then(|(_, hops)| hops.parse().ok())
806            .map_or_else(
807                || ClientError::ReplayedRequestFailure(text.clone()),
808                ClientError::TooManyRedirects,
809            ),
810        // `NoMock` carries the method and URL, which the effect already holds;
811        // it is a test-harness error a recorded production run cannot produce,
812        // so it is not worth rebuilding structurally.
813        Some(Kind::Json | Kind::NoMock | Kind::Transport) | None => {
814            ClientError::ReplayedRequestFailure(text)
815        }
816    }
817}
818
819/// The payload of a single-field error, recovered from its recorded `Display`.
820///
821/// The prefixes are this module's own `#[error(...)]` strings, so they cannot
822/// drift without this file changing; a recording that does not carry one is
823/// handed back whole rather than silently truncated.
824#[cfg(feature = "reporting")]
825fn strip_prefix_payload(text: &str, prefix: &str) -> String {
826    text.strip_prefix(prefix).unwrap_or(text).to_owned()
827}
828
829/// A recorded transport failure is reproduced as one, not as a status: a
830/// handler whose bug is "we do not handle a connection reset" must meet the
831/// reset again.
832#[cfg(feature = "reporting")]
833fn replayed_response(
834    tape: &crate::capsule::effects::ReplayEffects,
835    request: &crate::capsule::effects::OutboundRequest<'_>,
836) -> Result<Response, ClientError> {
837    let Some(recorded) = tape.next_http(request) else {
838        // `next_http` already logged the divergence; the call fails closed.
839        return Err(ClientError::BlockedDuringReplay(
840            request.method.to_owned(),
841            crate::capsule::redact::hide_url_userinfo(request.url),
842        ));
843    };
844    if let Some(error) = recorded.error {
845        return Err(rebuild_client_error(recorded.error_kind, error));
846    }
847    let mut headers = HeaderMap::new();
848    for (name, value) in &recorded.response_headers {
849        if let (Ok(name), Ok(value)) = (
850            HeaderName::from_bytes(name.as_bytes()),
851            HeaderValue::from_str(value),
852        ) {
853            headers.append(name, value);
854        }
855    }
856    Ok(Response {
857        status: reqwest::StatusCode::from_u16(recorded.status)
858            .unwrap_or(reqwest::StatusCode::INTERNAL_SERVER_ERROR),
859        headers,
860        body: Bytes::from(crate::capsule::effects::body_bytes(&recorded.response_body)),
861        // The recorded final location when redirects moved it, so a handler
862        // inspecting `Response::url()` sees what it saw live.
863        url: recorded
864            .final_url
865            .as_deref()
866            .or(Some(request.url))
867            .and_then(|url| url::Url::parse(url).ok()),
868    })
869}
870
871/// Tees one outbound exchange into the in-flight request's capsule.
872///
873/// Armed before the send so the request half is captured even when the call
874/// never produces a response, and disarmed to a no-op whenever no capture
875/// scope is active — which is every request in an application that has not
876/// turned `[failure_capture]` on, so the cost on the ordinary path is one
877/// task-local probe.
878#[cfg(feature = "reporting")]
879struct OutboundRecorder {
880    scope: Option<Arc<crate::capsule::CaptureScope>>,
881    /// Tape position, taken when the call *starts*. Two calls a handler
882    /// `join!`s therefore land in initiation order rather than completion
883    /// order — which is the order replay consumes them in.
884    slot: Option<usize>,
885    /// Cached from the scope's settings so `finish` need not re-read them.
886    max_body_bytes: usize,
887    method: String,
888    url: String,
889    request_headers: Vec<(String, String)>,
890    request_body: crate::capsule::CapsuleBody,
891}
892
893#[cfg(feature = "reporting")]
894impl OutboundRecorder {
895    /// Snapshot the request half, if a capsule is being recorded.
896    ///
897    /// Records the headers the *caller* set. Headers the client adds later —
898    /// the injected W3C trace context, and any default the underlying
899    /// `reqwest::Client` was built with — are not on the tape. They change from
900    /// run to run, and replay compares the method, the URL, the caller-set
901    /// headers and the body.
902    fn arm(builder: &RequestBuilder) -> Self {
903        let scope = crate::capsule::current_scope();
904        let Some(scope) = scope else {
905            return Self {
906                scope: None,
907                slot: None,
908                max_body_bytes: 0,
909                method: String::new(),
910                url: String::new(),
911                request_headers: Vec::new(),
912                request_body: crate::capsule::CapsuleBody::Absent,
913            };
914        };
915        let max_body_bytes = scope.settings().max_body_bytes;
916        let slot = scope.reserve_http();
917        let request_headers = caller_headers(builder);
918        Self {
919            scope: Some(scope),
920            slot,
921            max_body_bytes,
922            method: builder.method.to_string(),
923            url: builder.url.clone(),
924            request_headers,
925            request_body: builder
926                .body
927                .as_ref()
928                .map_or(crate::capsule::CapsuleBody::Absent, |body| {
929                    encode_body(body, max_body_bytes)
930                }),
931        }
932    }
933
934    /// Complete the reserved slot with the finished exchange.
935    fn finish(self, result: &Result<Response, ClientError>) {
936        let (Some(scope), Some(slot)) = (self.scope, self.slot) else {
937            return;
938        };
939        let (status, response_headers, response_body, error, final_url) = match result {
940            Ok(response) => (
941                response.status.as_u16(),
942                response
943                    .headers
944                    .iter()
945                    .map(|(name, value)| {
946                        // A header value is bytes, not text. Replacing an
947                        // opaque non-UTF-8 value with an empty string would
948                        // hand replay a header the peer never sent, and code
949                        // reading it through `as_bytes()` would branch on
950                        // nothing; the lossy form at least preserves its shape
951                        // and length rather than deleting it.
952                        (
953                            name.as_str().to_owned(),
954                            value.to_str().map_or_else(
955                                |_| String::from_utf8_lossy(value.as_bytes()).into_owned(),
956                                ToOwned::to_owned,
957                            ),
958                        )
959                    })
960                    .collect(),
961                encode_body(&response.body, self.max_body_bytes),
962                None,
963                response
964                    .url()
965                    .map(reqwest::Url::as_str)
966                    .filter(|final_url| *final_url != self.url)
967                    .map(ToOwned::to_owned),
968            ),
969            Err(error) => (
970                0,
971                Vec::new(),
972                crate::capsule::CapsuleBody::Absent,
973                Some((error.to_string(), http_error_kind(error))),
974                None,
975            ),
976        };
977        let (error, error_kind) = match error {
978            Some((text, kind)) => (Some(text), Some(kind)),
979            None => (None, None),
980        };
981        scope.fill_http(
982            slot,
983            crate::capsule::HttpEffect {
984                method: self.method,
985                url: self.url,
986                request_headers: self.request_headers,
987                request_body: self.request_body,
988                status,
989                response_headers,
990                final_url,
991                response_body,
992                error,
993                error_kind,
994            },
995        );
996    }
997}
998
999/// Record a body as capsule text when it is UTF-8, base64 otherwise.
1000///
1001/// Text is what makes a capsule diffable and what lets redaction parse a JSON
1002/// payload by key; base64 is the honest fallback for a protobuf or an image.
1003#[cfg(feature = "reporting")]
1004fn encode_body(bytes: &Bytes, max_body_bytes: usize) -> crate::capsule::CapsuleBody {
1005    if bytes.is_empty() {
1006        return crate::capsule::CapsuleBody::Absent;
1007    }
1008    // Checked *before* the copy, not after: a 50 MB download must not be
1009    // duplicated into a `String` (or grown by a third into base64) only to be
1010    // thrown away by the capsule's body cap a moment later.
1011    if bytes.len() > max_body_bytes {
1012        return crate::capsule::CapsuleBody::Skipped {
1013            declared_len: Some(bytes.len()),
1014        };
1015    }
1016    std::str::from_utf8(bytes).map_or_else(
1017        |_| {
1018            use base64::Engine as _;
1019            crate::capsule::CapsuleBody::Base64(
1020                base64::engine::general_purpose::STANDARD.encode(bytes),
1021            )
1022        },
1023        |text| crate::capsule::CapsuleBody::Text(text.to_owned()),
1024    )
1025}
1026
1027/// No capsule support compiled in: the recorder is an empty value the
1028/// optimizer removes.
1029#[cfg(not(feature = "reporting"))]
1030struct OutboundRecorder;
1031
1032#[cfg(not(feature = "reporting"))]
1033impl OutboundRecorder {
1034    const fn arm(_builder: &RequestBuilder) -> Self {
1035        Self
1036    }
1037
1038    const fn finish(self, _result: &Result<Response, ClientError>) {}
1039}
1040
1041/// Newtype stored in [`AppState`](crate::AppState) extensions so the
1042/// `MockRegistry` `Arc` survives a `build()` without double-wrapping.
1043pub struct HttpMockRegistryExt(pub Arc<MockRegistry>);
1044
1045/// Shared, process-wide `reqwest::Client` registered in [`AppState`] at server
1046/// boot. Cloning is O(1) because `reqwest::Client` is internally `Arc`-backed,
1047/// and the connection pool is preserved across the clone.
1048///
1049/// `timeout_secs` records the per-request timeout the inner client was built
1050/// with.  [`Client::from_state`] compares this against the currently installed
1051/// config so that a `state_initializer` that replaces `AutumnConfig` with a
1052/// different timeout causes the stale inner to be discarded and a fresh client
1053/// to be built from the new config instead.
1054#[derive(Clone)]
1055pub(crate) struct SharedReqwestClient {
1056    pub(crate) client: reqwest::Client,
1057    pub(crate) timeout_secs: u64,
1058}
1059
1060/// Handle returned by
1061/// [`MockSetupBuilder::respond_with`] that lets tests assert call counts.
1062pub struct MockHandle {
1063    alias: String,
1064    method: String,
1065    path: String,
1066    call_count: Arc<AtomicUsize>,
1067}
1068
1069impl MockHandle {
1070    /// Assert that the mocked endpoint was called exactly `expected` times.
1071    ///
1072    /// # Panics
1073    ///
1074    /// Panics with a diagnostic message when the actual call count differs.
1075    pub fn expect_called(&self, expected: usize) {
1076        let actual = self.call_count.load(Ordering::SeqCst);
1077        assert_eq!(
1078            actual, expected,
1079            "http mock for {} {} {} expected {} call(s) but got {}",
1080            self.alias, self.method, self.path, expected, actual,
1081        );
1082    }
1083
1084    /// Return the raw call count without asserting.
1085    #[must_use]
1086    pub fn call_count(&self) -> usize {
1087        self.call_count.load(Ordering::SeqCst)
1088    }
1089}
1090
1091/// Builder returned by [`TestApp::http_mock`](crate::test::TestApp::http_mock).
1092///
1093/// Chain a method call (`get`, `post`, …) and a path, then call
1094/// [`respond_with`](Self::respond_with) to register the entry and obtain a
1095/// [`MockHandle`] for later assertions.
1096pub struct MockSetupBuilder {
1097    pub(crate) registry: Arc<MockRegistry>,
1098    pub(crate) alias: String,
1099    pub(crate) method: Option<Method>,
1100    pub(crate) path: Option<String>,
1101}
1102
1103impl MockSetupBuilder {
1104    /// Match `GET <path>`.
1105    #[must_use]
1106    pub fn get(mut self, path: &str) -> Self {
1107        self.method = Some(Method::GET);
1108        self.path = Some(path.to_owned());
1109        self
1110    }
1111    /// Match `POST <path>`.
1112    #[must_use]
1113    pub fn post(mut self, path: &str) -> Self {
1114        self.method = Some(Method::POST);
1115        self.path = Some(path.to_owned());
1116        self
1117    }
1118    /// Match `PUT <path>`.
1119    #[must_use]
1120    pub fn put(mut self, path: &str) -> Self {
1121        self.method = Some(Method::PUT);
1122        self.path = Some(path.to_owned());
1123        self
1124    }
1125    /// Match `PATCH <path>`.
1126    #[must_use]
1127    pub fn patch(mut self, path: &str) -> Self {
1128        self.method = Some(Method::PATCH);
1129        self.path = Some(path.to_owned());
1130        self
1131    }
1132    /// Match `DELETE <path>`.
1133    #[must_use]
1134    pub fn delete(mut self, path: &str) -> Self {
1135        self.method = Some(Method::DELETE);
1136        self.path = Some(path.to_owned());
1137        self
1138    }
1139
1140    /// Match `HEAD <path>`.
1141    #[must_use]
1142    pub fn head(mut self, path: &str) -> Self {
1143        self.method = Some(Method::HEAD);
1144        self.path = Some(path.to_owned());
1145        self
1146    }
1147
1148    /// Register the mock entry and return a [`MockHandle`] for assertions.
1149    ///
1150    /// `status` is the HTTP status code to return.
1151    /// `body` is serialised as JSON and returned as the response body.
1152    #[must_use]
1153    pub fn respond_with(self, status: u16, body: serde_json::Value) -> MockHandle {
1154        let path = self.path.clone().unwrap_or_default();
1155        let method_str = self
1156            .method
1157            .as_ref()
1158            .map_or_else(|| "*".to_owned(), ToString::to_string);
1159        let call_count = Arc::new(AtomicUsize::new(0));
1160
1161        self.registry.register(MockEntry {
1162            method: self.method,
1163            path: path.clone(),
1164            alias: Some(self.alias.clone()),
1165            status,
1166            body: Some(body),
1167            call_count: call_count.clone(),
1168        });
1169
1170        MockHandle {
1171            alias: self.alias,
1172            method: method_str,
1173            path,
1174            call_count,
1175        }
1176    }
1177
1178    /// Convenience variant that returns the given status with an empty body.
1179    ///
1180    /// Unlike [`respond_with`](Self::respond_with), this stores `body: None` so
1181    /// the mock response truly has zero body bytes (not the JSON literal `null`).
1182    #[must_use]
1183    pub fn respond_with_status(self, status: u16) -> MockHandle {
1184        let path = self.path.clone().unwrap_or_default();
1185        let method_str = self
1186            .method
1187            .as_ref()
1188            .map_or_else(|| "*".to_owned(), ToString::to_string);
1189        let call_count = Arc::new(AtomicUsize::new(0));
1190
1191        self.registry.register(MockEntry {
1192            method: self.method,
1193            path: path.clone(),
1194            alias: Some(self.alias.clone()),
1195            status,
1196            body: None,
1197            call_count: call_count.clone(),
1198        });
1199
1200        MockHandle {
1201            alias: self.alias,
1202            method: method_str,
1203            path,
1204            call_count,
1205        }
1206    }
1207}
1208
1209// ── Client ───────────────────────────────────────────────────────────────────
1210
1211/// Traced outbound HTTP client with automatic retries and test-mock support.
1212///
1213/// Extracted from `AppState` via Axum's extractor machinery — declare it as a
1214/// handler parameter to get a pre-configured instance that respects
1215/// `[http.client]` config and, in test builds, intercepts requests against any
1216/// registered mocks.
1217///
1218/// ```rust,no_run
1219/// use autumn_web::prelude::*;
1220/// use autumn_web::http::Client;
1221///
1222/// #[get("/ping-upstream")]
1223/// async fn ping(client: Client) -> AutumnResult<&'static str> {
1224///     client.get("https://api.example.com/health").send().await?;
1225///     Ok("ok")
1226/// }
1227/// ```
1228///
1229/// You can also construct a standalone client outside of a handler:
1230///
1231/// ```rust
1232/// use autumn_web::http::Client;
1233///
1234/// let client = Client::new();
1235/// ```
1236#[derive(Clone)]
1237pub struct Client {
1238    inner: reqwest::Client,
1239    /// Named alias — used to look up base URLs from config and to match mocks.
1240    alias: Option<String>,
1241    /// Base URL prepended to relative paths.
1242    base_url: Option<String>,
1243    /// Alias → base URL map loaded from `[http.client.base_urls]` config.
1244    base_urls: HashMap<String, String>,
1245    retry_policy: RetryPolicy,
1246    /// When present (test builds), matching requests bypass the network.
1247    mock: Option<Arc<MockRegistry>>,
1248    /// Resilience configuration for circuit breakers.
1249    resilience_config: Option<Arc<crate::config::ResilienceConfig>>,
1250    /// When present (a sim with a `SimNet`), calls go through the simulated
1251    /// network instead of the real one.
1252    sim_net: Option<Arc<crate::sim::SimNet>>,
1253    /// Retry budgets and deadline header settings (issue #3058).
1254    retry: RetrySettings,
1255    /// Source of retry jitter. `from_state` uses the app's entropy, so a sim
1256    /// seed replays the same delays.
1257    entropy: Arc<dyn crate::entropy::Entropy>,
1258    /// Client-side adaptive throttle (issue #3068). `None` when off.
1259    throttle: Option<Arc<crate::admission::AdaptiveThrottle>>,
1260}
1261
1262/// The app-wide client-side throttle, so that every `Client::from_state`
1263/// shares one set of per-host counts (issue #3068). `config` is the setting
1264/// it was built from.
1265#[derive(Clone)]
1266pub(crate) struct SharedThrottle {
1267    throttle: Arc<crate::admission::AdaptiveThrottle>,
1268    config: crate::config::AdaptiveThrottleConfig,
1269}
1270
1271/// Put the app-wide throttle in `state` when `[http.client.adaptive_throttle]`
1272/// is on.
1273pub(crate) fn install_shared_throttle(
1274    state: &crate::AppState,
1275    config: &crate::config::HttpClientConfig,
1276) {
1277    let _ = shared_throttle(state, config);
1278}
1279
1280/// Serializes the replacement of the shared throttle.
1281static THROTTLE_REPLACE: std::sync::Mutex<()> = std::sync::Mutex::new(());
1282
1283/// The app-wide throttle for the effective `config`, or `None` when it is
1284/// off. A `state_initializer` can replace the config after boot. If the
1285/// shared throttle was built from other settings, or there is none, a new
1286/// one replaces it, so that all clients share counts for the settings in
1287/// force.
1288fn shared_throttle(
1289    state: &crate::AppState,
1290    config: &crate::config::HttpClientConfig,
1291) -> Option<Arc<crate::admission::AdaptiveThrottle>> {
1292    let settings = config.adaptive_throttle;
1293    if !settings.enabled {
1294        return None;
1295    }
1296    let current = || {
1297        state
1298            .extension::<SharedThrottle>()
1299            .filter(|shared| shared.config == settings)
1300            .map(|shared| Arc::clone(&shared.throttle))
1301    };
1302    if let Some(throttle) = current() {
1303        return Some(throttle);
1304    }
1305    // Replace under a lock and check again, so that concurrent first calls
1306    // after a config change share one throttle. Only this slow path locks.
1307    let _guard = THROTTLE_REPLACE
1308        .lock()
1309        .unwrap_or_else(std::sync::PoisonError::into_inner);
1310    if let Some(throttle) = current() {
1311        return Some(throttle);
1312    }
1313    let throttle = throttle_from_config(config)?;
1314    state.insert_extension(SharedThrottle {
1315        throttle: Arc::clone(&throttle),
1316        config: settings,
1317    });
1318    Some(throttle)
1319}
1320
1321/// A new throttle for `config`, or `None` when it is off.
1322fn throttle_from_config(
1323    config: &crate::config::HttpClientConfig,
1324) -> Option<Arc<crate::admission::AdaptiveThrottle>> {
1325    let t = config.adaptive_throttle;
1326    t.enabled.then(|| {
1327        Arc::new(crate::admission::AdaptiveThrottle::new(
1328            t.k,
1329            Duration::from_secs(t.window_secs),
1330        ))
1331    })
1332}
1333
1334/// The entropy a client without app state uses.
1335fn os_entropy() -> Arc<dyn crate::entropy::Entropy> {
1336    Arc::new(crate::entropy::OsEntropy)
1337}
1338
1339impl Client {
1340    /// Create a new client with default settings (30 s timeout, 3 retries on
1341    /// idempotent methods).
1342    #[must_use]
1343    pub fn new() -> Self {
1344        Self::with_timeout(Duration::from_secs(30))
1345    }
1346
1347    /// Create a client with a custom per-request timeout.
1348    ///
1349    /// # Panics
1350    ///
1351    /// Panics if the underlying TLS backend cannot be initialised (should not
1352    /// happen with the default `rustls-tls` feature).
1353    #[must_use]
1354    pub fn with_timeout(timeout: Duration) -> Self {
1355        let inner = reqwest::ClientBuilder::new()
1356            .timeout(timeout)
1357            .redirect(pooled_redirect_policy())
1358            .build()
1359            .expect("failed to build reqwest client");
1360        Self {
1361            inner,
1362            alias: None,
1363            base_url: None,
1364            base_urls: HashMap::new(),
1365            retry_policy: RetryPolicy {
1366                request_timeout: Some(timeout),
1367                ..RetryPolicy::default()
1368            },
1369            mock: None,
1370            resilience_config: None,
1371            sim_net: None,
1372            retry: RetrySettings::standalone(),
1373            entropy: os_entropy(),
1374            throttle: None,
1375        }
1376    }
1377
1378    /// Build a bare `reqwest::Client` from `[http.client]` config.
1379    ///
1380    /// Used by `build_state` to create the single shared instance registered
1381    /// in `AppState` at server boot, and as the fallback when no shared client
1382    /// is available.
1383    ///
1384    /// # Panics
1385    ///
1386    /// Panics if the underlying TLS backend cannot be initialised.
1387    pub(crate) fn build_inner(config: &crate::config::HttpClientConfig) -> reqwest::Client {
1388        reqwest::ClientBuilder::new()
1389            .timeout(Duration::from_secs(config.timeout_secs))
1390            .redirect(pooled_redirect_policy())
1391            .build()
1392            .expect("failed to build reqwest client")
1393    }
1394
1395    /// Assemble a `Client` around an already-built `reqwest::Client` using the
1396    /// policy fields from `config`.  The caller supplies the inner client so
1397    /// the connection pool can be shared across requests.
1398    fn from_config_with_inner(
1399        inner: reqwest::Client,
1400        config: &crate::config::HttpClientConfig,
1401    ) -> Self {
1402        let timeout = Duration::from_secs(config.timeout_secs);
1403        Self {
1404            inner,
1405            alias: None,
1406            base_url: None,
1407            base_urls: config.base_urls.clone(),
1408            retry_policy: RetryPolicy {
1409                max_retries: config.max_retries,
1410                retry_idempotent_only: true,
1411                max_retry_after: Duration::from_secs(config.max_retry_after_secs),
1412                request_timeout: Some(timeout),
1413                max_backoff: Duration::from_millis(config.max_backoff_ms),
1414            },
1415            mock: None,
1416            resilience_config: None,
1417            sim_net: None,
1418            // `from_config` and `from_state` set the budgets.
1419            retry: RetrySettings {
1420                budgets: None,
1421                send_deadline_header: config.send_deadline_header,
1422            },
1423            entropy: os_entropy(),
1424            throttle: throttle_from_config(config),
1425        }
1426    }
1427
1428    /// Assemble a `Client` with default policy around an already-built
1429    /// `reqwest::Client`.  Used when a shared inner client is available but
1430    /// no explicit `[http.client]` config is registered.
1431    fn with_inner(inner: reqwest::Client) -> Self {
1432        Self {
1433            inner,
1434            alias: None,
1435            base_url: None,
1436            base_urls: HashMap::new(),
1437            retry_policy: RetryPolicy::default(),
1438            mock: None,
1439            resilience_config: None,
1440            sim_net: None,
1441            // `from_state` sets the budgets.
1442            retry: RetrySettings {
1443                budgets: None,
1444                send_deadline_header: true,
1445            },
1446            entropy: os_entropy(),
1447            throttle: None,
1448        }
1449    }
1450
1451    /// Create a client from `[http.client]` framework configuration.
1452    ///
1453    /// # Panics
1454    ///
1455    /// Panics if the underlying TLS backend cannot be initialised (should not
1456    /// happen with the default `rustls-tls` feature).
1457    #[must_use]
1458    pub fn from_config(config: &crate::config::HttpClientConfig) -> Self {
1459        let mut client = Self::from_config_with_inner(Self::build_inner(config), config);
1460        client.retry.budgets = config
1461            .retry_budget
1462            .enabled
1463            .then(|| Arc::new(RetryBudgets::new(&config.retry_budget)));
1464        client
1465    }
1466
1467    /// Attach a mock registry (used by the test harness).
1468    pub(crate) fn with_mock(mut self, registry: Arc<MockRegistry>) -> Self {
1469        self.mock = Some(registry);
1470        self
1471    }
1472
1473    /// Build a client from runtime application state.
1474    ///
1475    /// When the server was started via `AppBuilder`, a single `reqwest::Client`
1476    /// is registered in `AppState` at boot as a `SharedReqwestClient`.  This
1477    /// method clones that shared instance (O(1), preserves the connection pool)
1478    /// instead of constructing a new one, eliminating per-request TCP/TLS
1479    /// handshakes and DNS-resolver-spawn overhead.
1480    ///
1481    /// Falls back to `Self::new()` for detached or test state that does not
1482    /// carry a shared client.
1483    #[must_use]
1484    pub fn from_state(state: &crate::AppState) -> Self {
1485        let autumn_config = state.extension::<crate::config::AutumnConfig>();
1486        let config = state
1487            .extension::<crate::config::HttpConfig>()
1488            .or_else(|| autumn_config.as_ref().map(|c| Arc::new(c.http.clone())));
1489
1490        // Only reuse the shared inner client when its baked-in timeout still
1491        // matches the effective config timeout.  A state_initializer that
1492        // replaces AutumnConfig/HttpConfig with a different timeout_secs runs
1493        // after build_state, so without this check the stale inner would
1494        // silently override the new config's per-request timeout.
1495        let effective_timeout_secs = config.as_ref().map_or_else(
1496            || crate::config::HttpClientConfig::default().timeout_secs,
1497            |c| c.client.timeout_secs,
1498        );
1499        let shared = state.extension::<SharedReqwestClient>().and_then(|s| {
1500            if s.timeout_secs == effective_timeout_secs {
1501                Some(s.client.clone())
1502            } else {
1503                None
1504            }
1505        });
1506
1507        let budget_config = config
1508            .as_ref()
1509            .map_or_else(RetryBudgetConfig::default, |cfg| {
1510                cfg.client.retry_budget.clone()
1511            });
1512        // One throttle for the whole app, so per-host counts add up.
1513        let throttle = config
1514            .as_ref()
1515            .and_then(|cfg| shared_throttle(state, &cfg.client));
1516        let mut client = match (config, shared) {
1517            (Some(cfg), Some(inner)) => Self::from_config_with_inner(inner, &cfg.client),
1518            (Some(cfg), None) => Self::from_config(&cfg.client),
1519            (None, Some(inner)) => Self::with_inner(inner),
1520            (None, None) => Self::new(),
1521        };
1522
1523        // The extractor builds a new client for each request, so the budgets
1524        // live in the app state. Thus all requests of one app share them.
1525        client.retry.budgets = shared_retry_budgets(state, &budget_config);
1526
1527        client.resilience_config = autumn_config.map(|c| Arc::new(c.resilience.clone()));
1528
1529        if let Some(ext) = state.extension::<HttpMockRegistryExt>() {
1530            client = client.with_mock(ext.0.clone());
1531        }
1532        client.sim_net = state.extension::<crate::sim::SimNet>();
1533        client.throttle = throttle;
1534        // Retry jitter and the automatic key are not made again on capsule
1535        // replay, so they must not go on the capsule's random tape.
1536        let entropy = state.entropy_arc();
1537        client.entropy = entropy.unrecorded().unwrap_or(entropy);
1538
1539        client
1540    }
1541
1542    /// Return a clone of this client scoped to the named alias.
1543    ///
1544    /// When a `[http.client.base_urls]` entry exists for the alias the client
1545    /// will prepend that URL to all relative paths. Mocks registered for the
1546    /// alias via [`TestApp::http_mock`](crate::test::TestApp::http_mock) will
1547    /// match requests made through this named client.
1548    #[must_use]
1549    pub fn named(&self, alias: &str) -> Self {
1550        let base_url = self
1551            .base_urls
1552            .get(alias)
1553            .cloned()
1554            .or_else(|| self.base_url.clone());
1555        Self {
1556            inner: self.inner.clone(),
1557            alias: Some(alias.to_owned()),
1558            base_url,
1559            base_urls: self.base_urls.clone(),
1560            retry_policy: self.retry_policy.clone(),
1561            mock: self.mock.clone(),
1562            resilience_config: self.resilience_config.clone(),
1563            sim_net: self.sim_net.clone(),
1564            retry: self.retry.clone(),
1565            entropy: self.entropy.clone(),
1566            throttle: self.throttle.clone(),
1567        }
1568    }
1569
1570    /// Set (or override) the base URL prepended to relative request paths.
1571    #[must_use]
1572    pub fn with_base_url(&self, base_url: impl Into<String>) -> Self {
1573        Self {
1574            inner: self.inner.clone(),
1575            alias: self.alias.clone(),
1576            base_url: Some(base_url.into()),
1577            base_urls: self.base_urls.clone(),
1578            retry_policy: self.retry_policy.clone(),
1579            mock: self.mock.clone(),
1580            resilience_config: self.resilience_config.clone(),
1581            sim_net: self.sim_net.clone(),
1582            retry: self.retry.clone(),
1583            entropy: self.entropy.clone(),
1584            throttle: self.throttle.clone(),
1585        }
1586    }
1587
1588    fn build_request(&self, method: Method, url: impl AsRef<str>) -> RequestBuilder {
1589        let url_str = url.as_ref();
1590        let full_url = if url_str.starts_with("http://") || url_str.starts_with("https://") {
1591            url_str.to_owned()
1592        } else if let Some(base) = &self.base_url {
1593            format!(
1594                "{}/{}",
1595                base.trim_end_matches('/'),
1596                url_str.trim_start_matches('/')
1597            )
1598        } else {
1599            url_str.to_owned()
1600        };
1601
1602        RequestBuilder {
1603            client: self.inner.clone(),
1604            method,
1605            url: full_url,
1606            extra_headers: HeaderMap::new(),
1607            body: None,
1608            retry_policy: self.retry_policy.clone(),
1609            mock: self.mock.clone(),
1610            alias: self.alias.clone(),
1611            pending_error: None,
1612            resilience_config: self.resilience_config.clone(),
1613            redirect_mode: RedirectMode::Default,
1614            pin_addr: None,
1615            ssrf_safe: false,
1616            discard_response_body: false,
1617            breaker_scoped: false,
1618            sim_net: self.sim_net.clone(),
1619            forward_request_id: true,
1620            retry: self.retry.clone(),
1621            entropy: self.entropy.clone(),
1622            throttle: self.throttle.clone(),
1623            criticality: None,
1624            web_bot_auth: None,
1625        }
1626    }
1627
1628    /// Build a `GET` request.
1629    #[must_use]
1630    pub fn get(&self, url: impl AsRef<str>) -> RequestBuilder {
1631        self.build_request(Method::GET, url)
1632    }
1633    /// Build a `POST` request.
1634    #[must_use]
1635    pub fn post(&self, url: impl AsRef<str>) -> RequestBuilder {
1636        self.build_request(Method::POST, url)
1637    }
1638    /// Build a `PUT` request.
1639    #[must_use]
1640    pub fn put(&self, url: impl AsRef<str>) -> RequestBuilder {
1641        self.build_request(Method::PUT, url)
1642    }
1643    /// Build a `PATCH` request.
1644    #[must_use]
1645    pub fn patch(&self, url: impl AsRef<str>) -> RequestBuilder {
1646        self.build_request(Method::PATCH, url)
1647    }
1648    /// Build a `DELETE` request.
1649    #[must_use]
1650    pub fn delete(&self, url: impl AsRef<str>) -> RequestBuilder {
1651        self.build_request(Method::DELETE, url)
1652    }
1653
1654    /// Build a `HEAD` request.
1655    #[must_use]
1656    pub fn head(&self, url: impl AsRef<str>) -> RequestBuilder {
1657        self.build_request(Method::HEAD, url)
1658    }
1659
1660    /// Build an SSRF-safe `GET` request.
1661    ///
1662    /// This is the composed safe path for fetching **untrusted** URLs. Before
1663    /// connecting it resolves the host **once**, validates every resolved IP
1664    /// against the built-in SSRF deny-list ([`is_blocked_ip`]) and rejects the
1665    /// request if *any* address is blocked. The connection is then pinned to the
1666    /// full set of validated addresses so reqwest cannot re-resolve the host
1667    /// (closing the DNS-rebinding / TOCTOU window) yet can still fall back across
1668    /// them in order if the first is unreachable. Redirects are followed manually up to
1669    /// `SSRF_SAFE_MAX_REDIRECTS` hops, re-running resolve→validate→pin on each
1670    /// hop; a hop that downgrades the scheme from `https` to `http` is rejected
1671    /// as defence-in-depth.
1672    ///
1673    /// The redirect count and per-hop validation honour a chained builder
1674    /// override (the built-in resolve→validate→pin and scheme-downgrade guards
1675    /// always apply):
1676    ///
1677    /// - by default, up to `SSRF_SAFE_MAX_REDIRECTS` hops are followed;
1678    /// - a chained [`no_redirect`](RequestBuilder::no_redirect) returns the
1679    ///   initial `3xx` verbatim — the initial URL is still resolved, validated
1680    ///   and pinned, but no redirect is followed;
1681    /// - a chained [`follow_redirects(max, validator)`](RequestBuilder::follow_redirects)
1682    ///   caps following at `max` hops (so `max == 0` turns the first `3xx` into
1683    ///   [`ClientError::TooManyRedirects`]) and additionally runs the caller's
1684    ///   `validator(&next)` on every hop, on top of the built-in guards.
1685    ///
1686    /// Like the test-mock path, this custom send path bypasses the process-wide
1687    /// circuit-breaker registry to avoid entangling per-URL SSRF fetches with
1688    /// the shared per-host breaker state.
1689    ///
1690    /// **Env proxies are bypassed.** Each pinned per-hop client is built with
1691    /// `.no_proxy()`, so `HTTP_PROXY` / `HTTPS_PROXY` / `ALL_PROXY` are ignored
1692    /// and the socket connects directly to the validated/pinned address. This is
1693    /// required for the pin to hold: reqwest checks proxy interception before the
1694    /// connector where the `resolve()` override applies, so a configured proxy
1695    /// would otherwise receive the request and re-resolve the host — reopening
1696    /// the DNS-rebinding / SSRF window this API closes.
1697    ///
1698    /// **Cannot be combined with [`pin_to`](RequestBuilder::pin_to).** This path
1699    /// performs its own per-hop resolve→validate→pin and never reads the
1700    /// `pin_to` address, so an explicit pin would be silently ignored. Chaining
1701    /// the two is therefore rejected at send time with
1702    /// [`ClientError::PinNotAllowedWithSsrfSafe`]. Use `pin_to` alone for a
1703    /// caller-chosen fixed address, or `get_ssrf_safe` alone for guarded
1704    /// automatic per-hop pinning.
1705    #[must_use]
1706    pub fn get_ssrf_safe(&self, url: impl Into<String>) -> RequestBuilder {
1707        self.build_request(Method::GET, url.into()).ssrf_safe()
1708    }
1709}
1710
1711impl Default for Client {
1712    fn default() -> Self {
1713        Self::new()
1714    }
1715}
1716
1717impl axum::extract::FromRequestParts<crate::AppState> for Client {
1718    type Rejection = std::convert::Infallible;
1719
1720    async fn from_request_parts(
1721        _parts: &mut http::request::Parts,
1722        state: &crate::AppState,
1723    ) -> Result<Self, std::convert::Infallible> {
1724        Ok(Self::from_state(state))
1725    }
1726}
1727
1728// ── RequestBuilder ───────────────────────────────────────────────────────────
1729
1730/// Type alias for a redirect-`Location` validator.
1731type RedirectValidator = Arc<dyn Fn(&str) -> bool + Send + Sync>;
1732
1733tokio::task_local! {
1734    /// The redirect targets the pooled client followed during one send, in
1735    /// order. An error does not say which host failed after a redirect, and
1736    /// every host on the way gets its refill, so the plain path reads them
1737    /// here.
1738    static FOLLOWED: std::cell::RefCell<Vec<String>>;
1739    /// `true` while the plain path follows redirects itself, so the pooled
1740    /// client returns each redirect (see `pooled_redirect_policy`).
1741    static MANUAL_REDIRECTS: bool;
1742}
1743
1744/// Send `req` on the pooled client. Also return the redirect targets the
1745/// HTTP stack followed, in order.
1746async fn send_tracking_redirects(
1747    req: reqwest::RequestBuilder,
1748) -> (Result<reqwest::Response, reqwest::Error>, Vec<String>) {
1749    FOLLOWED
1750        .scope(std::cell::RefCell::new(Vec::new()), async {
1751            let sent = req.send().await;
1752            (sent, FOLLOWED.with(std::cell::RefCell::take))
1753        })
1754        .await
1755}
1756
1757/// Set `Referer` for a redirect from `previous` to `next`, as reqwest does:
1758/// the previous URL without credentials or fragment, and none on an
1759/// `https` to `http` hop.
1760fn set_referer(headers: &mut HeaderMap, next: &str, previous: &str) {
1761    let (Ok(next), Ok(mut referer)) = (url::Url::parse(next), url::Url::parse(previous)) else {
1762        return;
1763    };
1764    if next.scheme() == "http" && referer.scheme() == "https" {
1765        headers.remove(reqwest::header::REFERER);
1766        return;
1767    }
1768    let _ = referer.set_username("");
1769    let _ = referer.set_password(None);
1770    referer.set_fragment(None);
1771    if let Ok(value) = HeaderValue::from_str(referer.as_str()) {
1772        headers.insert(reqwest::header::REFERER, value);
1773    }
1774}
1775
1776/// The hop limit of the plain send path, as in reqwest's default policy.
1777const PLAIN_MAX_REDIRECTS: usize = 10;
1778
1779/// The redirect policy of the pooled client: reqwest's default of
1780/// [`PLAIN_MAX_REDIRECTS`] hops, but no automatic hop while a request
1781/// deadline is set. The plain send path then follows the redirect itself, so
1782/// each hop sends the time left at that hop in [`DEADLINE_HEADER`] (issue
1783/// #3058).
1784fn pooled_redirect_policy() -> reqwest::redirect::Policy {
1785    reqwest::redirect::Policy::custom(|attempt| {
1786        if Deadline::current().is_some() || MANUAL_REDIRECTS.try_with(|m| *m).unwrap_or(false) {
1787            attempt.stop()
1788        } else if attempt.previous().len() > PLAIN_MAX_REDIRECTS {
1789            // `previous` holds the first URL too, as in reqwest's own limit.
1790            attempt.error("too many redirects")
1791        } else {
1792            let _ = FOLLOWED.try_with(|hops| hops.borrow_mut().push(attempt.url().to_string()));
1793            attempt.follow()
1794        }
1795    })
1796}
1797
1798/// The client of one hop of [`RequestBuilder::follow_loop`].
1799#[derive(Clone, Copy)]
1800enum HopClient<'a> {
1801    /// The pooled client of the plain path.
1802    Pooled(&'a reqwest::Client),
1803    /// A new client per hop, with this timeout.
1804    OneShot(Duration),
1805}
1806
1807/// Per-request redirect handling.
1808///
1809/// `Default` preserves the historical behaviour exactly (the shared-client fast
1810/// path with reqwest's built-in auto-follow). `None` and `Follow` route the
1811/// request through the custom one-shot-client send path.
1812enum RedirectMode {
1813    /// Historical behaviour: shared client, reqwest auto-follows up to 10 hops.
1814    Default,
1815    /// Never follow: a 3xx is returned to the caller verbatim.
1816    None,
1817    /// Follow up to `max` hops, calling `validator` on each absolute target
1818    /// before following it.
1819    Follow {
1820        max: usize,
1821        validator: RedirectValidator,
1822    },
1823}
1824
1825/// Default hop cap for the composed [`Client::get_ssrf_safe`] safe path.
1826const SSRF_SAFE_MAX_REDIRECTS: usize = 5;
1827
1828/// Fluent outbound request builder produced by [`Client`] methods.
1829#[allow(
1830    clippy::struct_excessive_bools,
1831    reason = "independent per-request switches; each has its own builder method"
1832)]
1833pub struct RequestBuilder {
1834    client: reqwest::Client,
1835    method: Method,
1836    url: String,
1837    extra_headers: HeaderMap,
1838    /// Request body. `Bytes` gives O(1) clones across retry attempts.
1839    body: Option<Bytes>,
1840    retry_policy: RetryPolicy,
1841    mock: Option<Arc<MockRegistry>>,
1842    alias: Option<String>,
1843    /// Captures errors from `json()` or invalid headers to surface in `send()`.
1844    pending_error: Option<ClientError>,
1845    /// Resilience configuration for circuit breakers.
1846    resilience_config: Option<Arc<crate::config::ResilienceConfig>>,
1847    /// Per-request redirect handling (see [`RedirectMode`]).
1848    redirect_mode: RedirectMode,
1849    /// When set, connect directly to this socket, skipping DNS resolution while
1850    /// preserving the original `Host` header + SNI. See [`RequestBuilder::pin_to`].
1851    pin_addr: Option<Vec<SocketAddr>>,
1852    /// When `true`, use the composed SSRF-safe send path (resolve→validate→pin
1853    /// with per-hop redirect validation). Set by [`Client::get_ssrf_safe`].
1854    ssrf_safe: bool,
1855    /// When `true`, the response body is dropped unread. See
1856    /// [`RequestBuilder::discard_response_body`].
1857    discard_response_body: bool,
1858    /// When `true`, `send_recorded` keeps circuit-breaker accounting even on
1859    /// the custom send path (`needs_custom_path()`). See
1860    /// [`RequestBuilder::breaker_scoped`].
1861    breaker_scoped: bool,
1862    /// The simulated network, when the client came from a sim app state.
1863    sim_net: Option<Arc<crate::sim::SimNet>>,
1864    /// When `true` (the default), `send` adds the current request's id as
1865    /// `x-request-id`. See [`RequestBuilder::without_request_id`].
1866    forward_request_id: bool,
1867    /// Retry budgets and deadline header settings (issue #3058).
1868    retry: RetrySettings,
1869    /// Source of retry jitter and of the automatic `Idempotency-Key`.
1870    entropy: Arc<dyn crate::entropy::Entropy>,
1871    /// Client-side adaptive throttle (issue #3068).
1872    throttle: Option<Arc<crate::admission::AdaptiveThrottle>>,
1873    /// The criticality to send. `None` sends the inbound request's.
1874    criticality: Option<crate::admission::Criticality>,
1875    /// Web Bot Auth signer: each attempt sends a fresh signature.
1876    web_bot_auth: Option<Arc<crate::aeo::web_bot_auth::WebBotAuthSigner>>,
1877}
1878
1879impl RequestBuilder {
1880    /// Do not send the current request's id as `x-request-id`.
1881    ///
1882    /// By default, a request sent during an inbound request carries that
1883    /// request's id (issue #3064). Use this for a host that must not see it.
1884    #[must_use]
1885    pub const fn without_request_id(mut self) -> Self {
1886        self.forward_request_id = false;
1887        self
1888    }
1889
1890    /// Send this criticality in the `X-Autumn-Criticality` header (issue
1891    /// #3068). Without this call, the client sends the criticality of the
1892    /// inbound request that it serves, if any.
1893    #[must_use]
1894    pub const fn criticality(mut self, criticality: crate::admission::Criticality) -> Self {
1895        self.criticality = Some(criticality);
1896        self
1897    }
1898
1899    /// Append a request header.
1900    ///
1901    /// Headers named `authorization`, `cookie`, or `set-cookie` are accepted
1902    /// normally but are **redacted** in tracing events and log output.
1903    /// Invalid header names or values emit a `tracing::warn!` and are skipped.
1904    #[must_use]
1905    pub fn header(mut self, name: impl AsRef<str>, value: impl AsRef<str>) -> Self {
1906        let name_str = name.as_ref();
1907        let value_str = value.as_ref();
1908        match (
1909            HeaderName::from_bytes(name_str.as_bytes()),
1910            HeaderValue::from_str(value_str),
1911        ) {
1912            (Ok(n), Ok(v)) => {
1913                self.extra_headers.insert(n, v);
1914            }
1915            (Err(e), _) => {
1916                tracing::warn!(header.name = name_str, error = %e, "invalid header name — header skipped");
1917            }
1918            (_, Err(e)) => {
1919                tracing::warn!(header.name = name_str, error = %e, "invalid header value — header skipped");
1920            }
1921        }
1922        self
1923    }
1924
1925    /// Sign this request with Web Bot Auth (RFC 9421 HTTP Message
1926    /// Signatures), so the receiving site can verify that it comes from this
1927    /// app. Adds `Signature-Agent`, `Signature-Input`, and `Signature`.
1928    ///
1929    /// The URL must be absolute. Each attempt (a retry, a same-origin
1930    /// redirect) sends a new signature, so a held builder or a late retry
1931    /// never sends an expired one. See [`crate::aeo::web_bot_auth`].
1932    #[must_use]
1933    pub fn sign_web_bot_auth(
1934        mut self,
1935        signer: &crate::aeo::web_bot_auth::WebBotAuthSigner,
1936    ) -> Self {
1937        if let Some(h) = signer.sign_url(&self.url) {
1938            self.web_bot_auth = Some(Arc::new(signer.clone()));
1939            return self
1940                .header("signature-agent", h.signature_agent)
1941                .header("signature-input", h.signature_input)
1942                .header("signature", h.signature);
1943        }
1944        self.pending_error = Some(ClientError::InvalidUrl(format!(
1945            "Web Bot Auth needs an absolute URL, got {:?}",
1946            self.url
1947        )));
1948        self
1949    }
1950
1951    /// Serialise `body` as JSON and set `Content-Type: application/json`.
1952    ///
1953    /// Serialisation errors are captured and returned when [`send`](Self::send)
1954    /// is called rather than being silently discarded.
1955    #[must_use]
1956    pub fn json<T: Serialize>(mut self, body: &T) -> Self {
1957        match serde_json::to_vec(body) {
1958            Ok(bytes) => {
1959                self.body = Some(Bytes::from(bytes));
1960                self = self.header("content-type", "application/json");
1961            }
1962            Err(e) => {
1963                self.pending_error = Some(ClientError::Json(e));
1964            }
1965        }
1966        self
1967    }
1968
1969    /// Set a plain-text body.
1970    #[must_use]
1971    pub fn text_body(mut self, body: impl Into<String>) -> Self {
1972        self.body = Some(Bytes::from(body.into().into_bytes()));
1973        self
1974    }
1975
1976    /// Set a raw byte body, without assuming any content type.
1977    ///
1978    /// Use this for binary payloads — [`text_body`](Self::text_body) takes a
1979    /// `String` and so cannot carry bytes that are not valid UTF-8. Set the
1980    /// content type yourself with [`header`](Self::header). The Web Push
1981    /// transport uses this for RFC 8291-encrypted bodies.
1982    #[must_use]
1983    pub fn bytes_body(mut self, body: impl Into<Bytes>) -> Self {
1984        self.body = Some(body.into());
1985        self
1986    }
1987
1988    /// Override the maximum retry count for this request.
1989    ///
1990    /// This does not enable retries for `POST` and `PATCH`. Use
1991    /// [`retry_non_idempotent`](Self::retry_non_idempotent) for that.
1992    #[must_use]
1993    pub const fn retries(mut self, max: u32) -> Self {
1994        self.retry_policy.max_retries = max;
1995        self
1996    }
1997
1998    /// Retry non-idempotent methods (`POST`, `PATCH`) too.
1999    ///
2000    /// A retried `POST` can run two times on the server. So for a
2001    /// non-idempotent method, the client sends an `Idempotency-Key` header
2002    /// with a random value, the same on every attempt. A key you set with
2003    /// [`header`](Self::header) is kept.
2004    #[must_use]
2005    pub const fn retry_non_idempotent(mut self) -> Self {
2006        self.retry_policy.retry_idempotent_only = false;
2007        self
2008    }
2009
2010    /// Override the cap on the jittered backoff between attempts.
2011    #[must_use]
2012    pub const fn max_backoff(mut self, max: Duration) -> Self {
2013        self.retry_policy.max_backoff = max;
2014        self
2015    }
2016
2017    /// Override the maximum `Retry-After` sleep duration for this request.
2018    #[must_use]
2019    pub const fn max_retry_after(mut self, max: Duration) -> Self {
2020        self.retry_policy.max_retry_after = max;
2021        self
2022    }
2023
2024    /// Disable retries for this request.
2025    #[must_use]
2026    pub const fn no_retry(mut self) -> Self {
2027        self.retry_policy.max_retries = 0;
2028        self
2029    }
2030
2031    /// Disable redirect following for this request.
2032    ///
2033    /// A `3xx` response is returned to the caller verbatim (status, headers and
2034    /// body) rather than being followed. Routes the request through the custom
2035    /// one-shot-client send path, which bypasses the process-wide circuit
2036    /// breaker.
2037    #[must_use]
2038    pub fn no_redirect(mut self) -> Self {
2039        self.redirect_mode = RedirectMode::None;
2040        self
2041    }
2042
2043    /// Return the status and headers without reading the response body.
2044    ///
2045    /// Every other path collects the whole body into memory with no ceiling,
2046    /// which is the right default for an API call whose payload the caller
2047    /// wants. It is the wrong default when the *remote host* is untrusted and
2048    /// the caller needs only the status: a server that streams indefinitely
2049    /// then costs one unbounded allocation per request until the timeout
2050    /// fires.
2051    ///
2052    /// Web Push is exactly that shape — the endpoint URL is chosen by the
2053    /// client, and the transport only ever reads the status code — so it sets
2054    /// this. [`Response::bytes`] then returns empty; dropping the underlying
2055    /// response closes the connection without draining it.
2056    #[must_use]
2057    pub const fn discard_response_body(mut self) -> Self {
2058        self.discard_response_body = true;
2059        self
2060    }
2061
2062    /// Follow up to `max` redirects, validating each hop before following it.
2063    ///
2064    /// Before following a `3xx`, the `Location` header is resolved to an
2065    /// absolute URL (relative locations are joined against the current URL) and
2066    /// `validator(&absolute_location)` is called. If it returns `false` the
2067    /// request fails with [`ClientError::RedirectRejected`]. If the chain would
2068    /// exceed `max` hops the request fails with
2069    /// [`ClientError::TooManyRedirects`] (so `max == 0` turns the first `3xx`
2070    /// into an error). Routes the request through the custom one-shot-client
2071    /// send path, which bypasses the process-wide circuit breaker.
2072    ///
2073    /// **TOCTOU / rebinding limitation.** The `validator` receives the redirect
2074    /// target as a *string* (not a resolved IP), and the subsequent connection
2075    /// re-resolves that host via normal DNS. So a validator that inspects IP
2076    /// literals sees only literal hosts, has a connect-time TOCTOU window
2077    /// against a hostname that re-resolves between check and connect, and
2078    /// provides no address pinning. When you need pinned, rebind-safe following
2079    /// that validates every resolved IP and pins the connection per hop, use
2080    /// [`Client::get_ssrf_safe`] instead.
2081    #[must_use]
2082    pub fn follow_redirects<F>(mut self, max: usize, validator: F) -> Self
2083    where
2084        F: Fn(&str) -> bool + Send + Sync + 'static,
2085    {
2086        self.redirect_mode = RedirectMode::Follow {
2087            max,
2088            validator: Arc::new(validator),
2089        };
2090        self
2091    }
2092
2093    /// Pin the connection to `addr`, skipping DNS resolution.
2094    ///
2095    /// The original `Host` header and TLS SNI are preserved; only the
2096    /// address the socket connects to is overridden. This protects against
2097    /// DNS-rebinding / TOCTOU attacks where a hostname re-resolves to a
2098    /// different (private) address between validation and connection.
2099    ///
2100    /// **Requires a domain (hostname) URL host.** The pin is enforced via a DNS
2101    /// `resolve` override, but reqwest/hyper treat an IP-literal URL host as
2102    /// already-resolved and never consult the resolver — so the override is
2103    /// skipped and the socket would connect to the literal in the URL, not the
2104    /// pinned address. A `pin_to` request whose URL host is an IP literal
2105    /// (IPv4 or IPv6) is therefore **rejected at send time** with
2106    /// [`ClientError::PinRequiresDomainHost`]. Put the desired IP directly in the
2107    /// URL (no pin needed), or use a domain host.
2108    ///
2109    /// **Pinning applies to the initial connection only.** Redirects are **not**
2110    /// followed under `pin_to`: a `3xx` is returned to the caller verbatim (as if
2111    /// [`no_redirect`](Self::no_redirect) were set) rather than being
2112    /// auto-followed to a possibly-different host that would be re-resolved via
2113    /// normal DNS, silently escaping the pin. If you need pinned, rebind-safe
2114    /// per-hop following, use [`Client::get_ssrf_safe`] instead.
2115    ///
2116    /// **Cannot be combined with [`follow_redirects`](Self::follow_redirects).**
2117    /// Because the pin only covers the first hop while later hops would re-resolve
2118    /// via DNS, chaining `pin_to` with `follow_redirects` (in either order) is
2119    /// rejected at send time with [`ClientError::IncompatiblePinRedirect`] rather
2120    /// than silently following a redirect off the pinned address. Use
2121    /// [`Client::get_ssrf_safe`] for pinned, per-hop-revalidated redirect
2122    /// following, `pin_to` alone (which returns the `3xx` unfollowed), or
2123    /// `follow_redirects` without `pin_to`.
2124    ///
2125    /// Implemented via a one-shot `reqwest::ClientBuilder::resolve(host, addr)`
2126    /// scoped to this request. Note that reqwest ignores the **port** in the
2127    /// resolve override and connects to the port from the request URL, so
2128    /// `addr.port()` is only honoured when it matches the URL's port (which it
2129    /// does for addresses obtained by resolving that same URL). Routes the
2130    /// request through the custom one-shot-client send path, which bypasses the
2131    /// process-wide circuit breaker.
2132    ///
2133    /// **Env proxies are bypassed.** The pinned client is built with
2134    /// `.no_proxy()`, so `HTTP_PROXY` / `HTTPS_PROXY` / `ALL_PROXY` are ignored
2135    /// and the socket connects directly to `addr`. This is required for the pin
2136    /// to hold: reqwest checks proxy interception before the connector where the
2137    /// `resolve()` override applies, so a configured proxy would otherwise
2138    /// receive the request and re-resolve the host — defeating the pin.
2139    #[must_use]
2140    pub fn pin_to(self, addr: SocketAddr) -> Self {
2141        self.pin_to_addrs([addr])
2142    }
2143
2144    /// Like [`pin_to`](Self::pin_to), but accepts several addresses for one
2145    /// host.
2146    ///
2147    /// All addresses go to one request. reqwest tries the next address only
2148    /// if it cannot connect. A loop over `pin_to` could send a `POST` body
2149    /// twice.
2150    ///
2151    /// An empty set makes `send` fail. It never drops the pin. Every other
2152    /// rule of `pin_to` applies.
2153    #[must_use]
2154    pub fn pin_to_addrs(mut self, addrs: impl IntoIterator<Item = SocketAddr>) -> Self {
2155        self.pin_addr = Some(addrs.into_iter().collect());
2156        self
2157    }
2158
2159    /// Route this request through the SSRF-safe resolve→validate→pin send path
2160    /// documented on [`Client::get_ssrf_safe`], for any HTTP method.
2161    ///
2162    /// [`Client::get_ssrf_safe`] only builds `GET` requests, but the guarantee
2163    /// it describes — reject a resolved address on the built-in SSRF deny-list,
2164    /// pin the connection to the validated set, re-validate on every redirect
2165    /// hop — is implemented by [`Self::send`] purely from this flag and is not
2166    /// GET-specific. Any outbound call whose destination is not a value the
2167    /// app itself chose — a webhook subscriber's `target_url`, a user-supplied
2168    /// callback, an OAuth discovery endpoint — needs this on **every** verb it
2169    /// uses, not only reads. Chain it after [`Client::post`], [`Client::put`],
2170    /// etc.:
2171    ///
2172    /// ```rust,ignore
2173    /// client.post(target_url).ssrf_safe().json(&payload).send().await?;
2174    /// ```
2175    ///
2176    /// Same incompatibility with [`pin_to`](Self::pin_to) as `get_ssrf_safe`:
2177    /// this path performs its own per-hop resolve→validate→pin, so chaining an
2178    /// explicit pin is rejected at send time with
2179    /// [`ClientError::PinNotAllowedWithSsrfSafe`].
2180    #[must_use]
2181    pub const fn ssrf_safe(mut self) -> Self {
2182        self.ssrf_safe = true;
2183        self
2184    }
2185
2186    /// Keep circuit-breaker accounting even when this request also needs the
2187    /// custom send path (`ssrf_safe()`, `pin_to()`, `no_redirect()`,
2188    /// `follow_redirects()`).
2189    ///
2190    /// That custom path otherwise bypasses `send_recorded`'s breaker block
2191    /// entirely — right for its usual case, a one-off fetch of an arbitrary
2192    /// caller-chosen URL, where per-host breaker state is mostly noise. A
2193    /// caller instead making *repeated* calls to a small, durable set of
2194    /// hosts — outbound webhook delivery is the motivating case (#2480 code
2195    /// review) — still wants the breaker: without it, a down receiver never
2196    /// fails fast, and every queued delivery pays a full
2197    /// DNS-resolve-and-connect timeout instead of the open-breaker
2198    /// short-circuit every other outbound call gets.
2199    ///
2200    /// This flag is read from *inside* `send_recorded`, not layered on
2201    /// externally, which is what makes it free to get right on every other
2202    /// axis `send` already handles correctly: a mocked client's `self.mock.is_some()`
2203    /// check runs first regardless (mock behavior is unaffected); a capsule
2204    /// replay's `current_tape()` check happens even earlier, in `send` itself,
2205    /// before `send_recorded` is ever reached (an open-breaker attempt is
2206    /// therefore never gated on live state during replay, and — because it
2207    /// stays behind `send`'s own capture tee — an open-breaker attempt made
2208    /// *while capturing* is recorded into the capsule exactly like any other
2209    /// outcome, so a later replay of that exact run reproduces
2210    /// `CircuitBreakerOpen` instead of diverging).
2211    ///
2212    /// Breaker keying always uses this request's *resolved* URL (`self.url`
2213    /// — already expanded past any `[http.client.base_urls]` alias by
2214    /// `Client::build_request`), so an alias-targeted destination keys the
2215    /// same breaker bucket a literal-URL request to the same host would.
2216    #[must_use]
2217    pub(crate) const fn breaker_scoped(mut self) -> Self {
2218        self.breaker_scoped = true;
2219        self
2220    }
2221
2222    /// Send the request, applying retries and returning a [`Response`].
2223    ///
2224    /// # Errors
2225    ///
2226    /// Returns [`ClientError::Json`] if a prior `.json()` call failed to
2227    /// serialise the body.  Returns [`ClientError::Request`] for transport
2228    /// errors that exhaust all retry attempts.  Returns [`ClientError::NoMock`]
2229    /// if the request is made in a test context without a matching mock entry.
2230    ///
2231    /// # Panics
2232    ///
2233    /// Contains an internal `unreachable!()` that guards against a logic error
2234    /// in the retry loop; it cannot be reached in practice.
2235    pub async fn send(self) -> Result<Response, ClientError> {
2236        // Surface any error captured during builder construction.
2237        if let Some(err) = self.pending_error {
2238            return Err(err);
2239        }
2240
2241        // Replay serves outbound calls from the capsule's effect tape (#1634) and never
2242        // dials the peer. A call the tape cannot answer is a divergence, already logged by
2243        // `next_http`, and fails closed here. The unconditional block below stays as the
2244        // backstop for a replay process whose current task carries no tape — app boot, a
2245        // state initializer, work the handler spawned — because a capsule records no
2246        // response for any of those either. The check is here rather than on the client
2247        // because a handler can build one any way it likes (`Client::new()`, `from_state`,
2248        // a stored one) and every path must be closed.
2249        #[cfg(feature = "reporting")]
2250        if let Some(tape) = crate::capsule::effects::current_tape() {
2251            let method = self.method.to_string();
2252            // Derived exactly as `OutboundRecorder::arm` derives the recorded
2253            // half, so the comparison is like with like. No cap on the body:
2254            // the capsule's own cap already applied at record time, and a
2255            // recording that hit it is refused before it ever reaches replay.
2256            let headers = caller_headers(&self);
2257            let body = self
2258                .body
2259                .as_ref()
2260                .map_or(crate::capsule::CapsuleBody::Absent, |body| {
2261                    encode_body(body, usize::MAX)
2262                });
2263            return replayed_response(
2264                &tape,
2265                &crate::capsule::effects::OutboundRequest {
2266                    method: &method,
2267                    url: &self.url,
2268                    headers: &headers,
2269                    body: &body,
2270                },
2271            );
2272        }
2273        // Only capsule replay (`reporting`) sets the block.
2274        #[cfg(feature = "reporting")]
2275        if outbound_blocked_for_replay() {
2276            return Err(ClientError::BlockedDuringReplay(
2277                self.method.to_string(),
2278                crate::capsule::redact::hide_url_userinfo(&self.url),
2279            ));
2280        }
2281
2282        // Capture: the exchange is recorded into the in-flight request's
2283        // capsule scope, when there is one. Recording wraps every send path
2284        // below (mock, custom, breaker-guarded) so a capsule cannot miss an
2285        // exchange because the caller happened to use `no_redirect` or a
2286        // pinned address.
2287        let recorder = OutboundRecorder::arm(&self);
2288        let result = self.send_recorded().await;
2289        recorder.finish(&result);
2290        result
2291    }
2292
2293    /// [`send`](Self::send), minus the replay gate and the capture tee.
2294    async fn send_recorded(mut self) -> Result<Response, ClientError> {
2295        // Staging fault injection (#3071). Inert outside a fault scope.
2296        crate::fault_injection::inject(crate::fault_injection::FaultTarget::Http)
2297            .await
2298            .map_err(|fault| ClientError::FaultInjected(fault.to_string()))?;
2299        // After the capture tee armed, so a capsule records only the caller's
2300        // own headers and replay compares like with like.
2301        if self.forward_request_id {
2302            add_current_request_id(&mut self.extra_headers);
2303        }
2304        // After the capture tee, so a capsule records the caller's headers
2305        // only and replays without a random key.
2306        self.ensure_idempotency_key();
2307        self.ensure_criticality_header();
2308
2309        // A sim network serves every send path, so nothing reaches the real
2310        // network. Like mocks, it bypasses the process-global breaker.
2311        if let Some(net) = self.sim_net.clone() {
2312            return self.send_sim(&net).await;
2313        }
2314
2315        // Bypassing circuit breaker if a mock registry is present.
2316        if self.mock.is_some() {
2317            return self.send_inner(false).await;
2318        }
2319
2320        // Custom send path: any of no_redirect / follow_redirects / pin_to /
2321        // get_ssrf_safe builds one-shot reqwest client(s) with Policy::none()
2322        // (+ optional .resolve()) and does manual redirect handling. Like the
2323        // mock path it deliberately BYPASSES the process-global circuit breaker
2324        // to avoid entangling these one-off, per-URL requests with the shared
2325        // per-host breaker registry — unless the caller opted in via
2326        // `breaker_scoped()` (repeated calls to a small, durable host set; see
2327        // its doc comment).
2328        if self.needs_custom_path() {
2329            if self.breaker_scoped {
2330                return self.send_custom_breaker_guarded().await;
2331            }
2332            return self.send_custom(false).await;
2333        }
2334
2335        // ── Resilience / Circuit Breaker ──────────────────────────────────
2336        let breaker = breaker_for_url(self.resilience_config.as_ref(), &self.url);
2337
2338        // Check if circuit breaker is open
2339        let Ok(guard) = breaker.admit() else {
2340            return Err(ClientError::CircuitBreakerOpen);
2341        };
2342
2343        let is_half_open = breaker.state() == crate::circuit_breaker::CircuitState::HalfOpen;
2344        let res = self.send_inner(is_half_open).await;
2345        match &res {
2346            Ok(resp) => {
2347                let success = resp.status().as_u16() < 500;
2348                if success {
2349                    guard.success();
2350                } else {
2351                    guard.failure();
2352                }
2353            }
2354            // Neither says anything about the host. The caller ran out of
2355            // time (issue #3058): a cancelled call, which counts only past the
2356            // slow-call threshold. A local throttle reject (#3068): dropping
2357            // the guard records nothing for a fast call and frees a half-open
2358            // slot.
2359            Err(ClientError::DeadlineExceeded | ClientError::ThrottledLocally { .. }) => {
2360                drop(guard);
2361            }
2362            Err(_) => {
2363                guard.failure();
2364            }
2365        }
2366        res
2367    }
2368
2369    /// The custom send path (`send_custom`), with breaker accounting exactly
2370    /// like the plain-path breaker block above: `admit` gate,
2371    /// `CircuitBreakerGuard` covering the call (its `Drop` releases a
2372    /// half-open slot if this future is cancelled or panics before finishing),
2373    /// `< 500` success threshold. Only reached when `breaker_scoped()` was
2374    /// set — see its doc comment for why this has to live here rather than in
2375    /// an external wrapper around `send()`.
2376    async fn send_custom_breaker_guarded(self) -> Result<Response, ClientError> {
2377        let breaker = breaker_for_url(self.resilience_config.as_ref(), &self.url);
2378        let Ok(guard) = breaker.admit() else {
2379            return Err(ClientError::CircuitBreakerOpen);
2380        };
2381        // Like the breaker, the throttle keeps per-host state, so on the
2382        // custom path it covers only `breaker_scoped` calls: a small, durable
2383        // host set. Unscoped custom calls go to user-supplied URLs, and one
2384        // entry per host would grow without bound. The ticket is taken after
2385        // the breaker admits, so a fail-fast `CircuitBreakerOpen` call never
2386        // reaches the throttle. `send_one` retries inside, so the throttle
2387        // counts the call once. A local reject leaves the breaker neutral.
2388        let ticket = match self.throttle_attempt(None) {
2389            Ok(ticket) => ticket,
2390            Err(error) => {
2391                drop(guard);
2392                return Err(error);
2393            }
2394        };
2395        // Mirrors the
2396        // plain-path breaker block's own `is_half_open` (passed to
2397        // `send_inner` to force a single attempt): a half-open probe is a
2398        // budgeted, limited trial (`half_open_trial_count`), and letting
2399        // `send_one`'s own retry loop turn one trial into several real
2400        // network attempts spends that budget on one logical delivery
2401        // instead of testing recovery with independent probes (#2480
2402        // review, round 10).
2403        let is_half_open = breaker.state() == crate::circuit_breaker::CircuitState::HalfOpen;
2404        let res = self.send_custom(is_half_open).await;
2405        record_call(ticket, &res);
2406        match &res {
2407            Ok(resp) if resp.status().as_u16() < 500 => guard.success(),
2408            // The caller ran out of time, not the upstream (issue #3058): a
2409            // cancelled call, which counts only past the slow-call threshold.
2410            Err(ClientError::DeadlineExceeded) => drop(guard),
2411            _ => guard.failure(),
2412        }
2413        res
2414    }
2415
2416    async fn send_inner(self, suppress_retries: bool) -> Result<Response, ClientError> {
2417        // ── Mock short-circuit ──────────────────────────────────────────────
2418        if let Some(ref mock) = self.mock {
2419            return self.mock_response(mock);
2420        }
2421
2422        // ── Real network request with retries ───────────────────────────────
2423        let start = crate::time::ambient_instant();
2424        let max_attempts = self.max_attempts(suppress_retries);
2425        let mut gate = self.retry_gate(url_host(&self.url).as_deref());
2426        if gate.deadline.is_some() || self.follows_redirects_itself() {
2427            return self.follow_pooled(suppress_retries, &gate).await;
2428        }
2429        let mut last_retry = None;
2430        let mut delay = Duration::ZERO;
2431        // Hosts refilled for this request: the first one already was.
2432        let mut refilled: std::collections::HashSet<String> =
2433            url_host(&self.url).into_iter().collect();
2434
2435        for attempt in 0..max_attempts {
2436            if attempt > 0 {
2437                tokio::time::sleep(delay).await;
2438            }
2439            // A throttled attempt ends the call, retry or not.
2440            let ticket = self.begin_attempt(&gate, None)?;
2441            let last = attempt + 1 == max_attempts;
2442            let (span, sent, followed) =
2443                self.send_plain_attempt(&gate, attempt, &mut refilled).await;
2444            match sent {
2445                Ok(resp) => {
2446                    let status = resp.status();
2447                    let headers = resp.headers().clone();
2448                    let url_used = resp.url().clone();
2449
2450                    // 429 and 502-504: retry while attempts, time and budget
2451                    // remain. The HTTP stack may have followed a redirect, so
2452                    // charge the host that answered.
2453                    if is_retryable_response(status.as_u16()) && !last {
2454                        gate.rekey(url_used.as_str());
2455                        let wait = self.retry_policy.retry_delay(
2456                            &*self.entropy,
2457                            attempt,
2458                            retry_hint(status.as_u16(), &headers),
2459                        );
2460                        let kind = retry_kind(status.as_u16());
2461                        if gate.allow(kind, wait) {
2462                            if let Some(ticket) = ticket {
2463                                ticket.record(throttle_accepts(status.as_u16()));
2464                            }
2465                            delay = wait;
2466                            last_retry = Some(kind);
2467                            continue;
2468                        }
2469                    }
2470
2471                    let body = if self.discard_response_body {
2472                        // Dropped unread — see `discard_response_body`.
2473                        Ok(Bytes::new())
2474                    } else {
2475                        read_body_in_span(resp, &span, &gate).await
2476                    };
2477                    // Count the attempt only now: a body that fails to arrive
2478                    // is a transport error, not an accept.
2479                    if let Some(ticket) = ticket {
2480                        ticket.record(body.is_ok() && throttle_accepts(status.as_u16()));
2481                    }
2482                    let body = body?;
2483                    // Refund only after the body arrived.
2484                    gate.finish(last_retry, status.as_u16());
2485                    let elapsed = crate::time::ambient_instant().saturating_duration_since(start);
2486                    log_request(
2487                        self.method.as_str(),
2488                        &url_used,
2489                        status.as_u16(),
2490                        elapsed,
2491                        &self.extra_headers,
2492                    );
2493
2494                    return Ok(Response {
2495                        status,
2496                        headers,
2497                        body,
2498                        url: Some(url_used),
2499                    });
2500                }
2501                // The request deadline stopped the attempt.
2502                Err(e) if e.is_timeout() && gate.expired() => {
2503                    return Err(ClientError::DeadlineExceeded);
2504                }
2505                // Only retry transient connect/timeout errors; non-transient errors
2506                // (e.g. malformed URL) fail immediately.
2507                Err(e) if (e.is_connect() || e.is_timeout()) && !last => {
2508                    if let Some(ticket) = ticket {
2509                        ticket.record(false);
2510                    }
2511                    // The HTTP stack may have followed a redirect: charge the
2512                    // host that failed.
2513                    // The host that failed: the last redirect target, or the
2514                    // request's own host.
2515                    gate.rekey(followed.last().map_or(self.url.as_str(), String::as_str));
2516                    let wait = self.retry_policy.retry_delay(&*self.entropy, attempt, None);
2517                    if !gate.allow(RetryKind::Transient, wait) {
2518                        return Err(ClientError::Request(e.without_url()));
2519                    }
2520                    delay = wait;
2521                    last_retry = Some(RetryKind::Transient);
2522                }
2523                Err(e) => {
2524                    if let Some(ticket) = ticket {
2525                        ticket.record(false);
2526                    }
2527                    return Err(ClientError::Request(e.without_url()));
2528                }
2529            }
2530        }
2531
2532        // The retry loop always returns inside the last attempt; this is unreachable.
2533        unreachable!("retry loop exited without returning a result — this is a bug")
2534    }
2535
2536    /// The deadline and retry budget for one send to `host`.
2537    fn retry_gate(&self, host: Option<&str>) -> RetryGate {
2538        let mut gate = RetryGate::start(
2539            self.retry.budgets.clone(),
2540            host,
2541            self.retry.send_deadline_header,
2542        );
2543        gate.signer.clone_from(&self.web_bot_auth);
2544        gate
2545    }
2546
2547    /// `true` when a header needs the client to follow redirects itself:
2548    ///
2549    /// - A caller's own deadline header needs a new value per attempt and
2550    ///   per hop.
2551    /// - An HTTP message signature is a credential that the pooled client's
2552    ///   redirects would carry to the next host; the manual follow strips it
2553    ///   on a cross-origin hop.
2554    fn follows_redirects_itself(&self) -> bool {
2555        self.extra_headers
2556            .get(DEADLINE_HEADER)
2557            .and_then(crate::deadline::parse_header)
2558            .is_some()
2559            || self.extra_headers.contains_key("signature")
2560    }
2561
2562    /// The plain path under a request deadline. The pooled client does not
2563    /// follow a redirect then (see `pooled_redirect_policy`), so this follows
2564    /// it, and each hop sends the time left at that hop.
2565    async fn follow_pooled(
2566        self,
2567        suppress_retries: bool,
2568        gate: &RetryGate,
2569    ) -> Result<Response, ClientError> {
2570        // `send_one` asks the throttle (issue #3068) for each attempt of
2571        // each hop, as the plain path does for each attempt.
2572        let client = self.client.clone();
2573        MANUAL_REDIRECTS
2574            .scope(
2575                true,
2576                self.follow_loop(
2577                    PLAIN_MAX_REDIRECTS,
2578                    Arc::new(|_: &str| true),
2579                    HopClient::Pooled(&client),
2580                    suppress_retries,
2581                    gate,
2582                ),
2583            )
2584            .await
2585    }
2586
2587    /// Add an `Idempotency-Key` header when this request can retry a
2588    /// non-idempotent method and the caller set no key. Every attempt sends
2589    /// the same key, so the server can drop a duplicate.
2590    fn ensure_idempotency_key(&mut self) {
2591        let retries_unsafe_method = !is_idempotent_method(&self.method)
2592            && !self.retry_policy.retry_idempotent_only
2593            && self.retry_policy.max_retries > 0;
2594        if !retries_unsafe_method || self.extra_headers.contains_key(IDEMPOTENCY_KEY) {
2595            return;
2596        }
2597        let key = self.entropy.uuid_v4().to_string();
2598        if let Ok(value) = HeaderValue::from_str(&key) {
2599            self.extra_headers
2600                .insert(HeaderName::from_static(IDEMPOTENCY_KEY), value);
2601        }
2602    }
2603
2604    /// Add the `X-Autumn-Criticality` header (issue #3068): the value set by
2605    /// [`Self::criticality`], else the inbound request's when it is not
2606    /// `default` (a missing header means `default`, so the client does not
2607    /// send it to every host). A header that the caller set wins.
2608    fn ensure_criticality_header(&mut self) {
2609        if self
2610            .extra_headers
2611            .contains_key(crate::admission::CRITICALITY_HEADER)
2612        {
2613            return;
2614        }
2615        let inbound = crate::admission::current_criticality()
2616            .filter(|c| *c != crate::admission::Criticality::Default);
2617        if let Some(c) = self.criticality.or(inbound) {
2618            self.extra_headers.insert(
2619                HeaderName::from_static(crate::admission::CRITICALITY_HEADER),
2620                HeaderValue::from_static(c.as_str()),
2621            );
2622        }
2623    }
2624
2625    /// Start one attempt: stop at the deadline, ask the throttle, then let
2626    /// the gate record the attempt. The throttle is asked before
2627    /// [`RetryGate::check`], so a throttled attempt refills no budget.
2628    fn begin_attempt(
2629        &self,
2630        gate: &RetryGate,
2631        host: Option<&str>,
2632    ) -> Result<Option<ThrottleTicket>, ClientError> {
2633        if gate.expired() {
2634            return Err(ClientError::DeadlineExceeded);
2635        }
2636        let ticket = self.throttle_attempt(host)?;
2637        gate.check()?;
2638        Ok(ticket)
2639    }
2640
2641    /// Send one plain-path attempt in its own CLIENT span (issue #3064). The
2642    /// trace context is injected inside the span, so the next service's
2643    /// parent is this attempt. Returns the span, so the caller reads the body
2644    /// in it, the send result and the redirect targets followed. Each host
2645    /// that served a redirect gets its refill, even when the send or the body
2646    /// fails later.
2647    async fn send_plain_attempt(
2648        &self,
2649        gate: &RetryGate,
2650        attempt: u32,
2651        refilled: &mut std::collections::HashSet<String>,
2652    ) -> (
2653        tracing::Span,
2654        Result<reqwest::Response, reqwest::Error>,
2655        Vec<String>,
2656    ) {
2657        let span = client_attempt_span(&self.method, &self.url, attempt);
2658        let req = self.plain_attempt(gate, &span);
2659        let (sent, followed) =
2660            tracing::Instrument::instrument(send_tracking_redirects(req), span.clone()).await;
2661        record_send_outcome(&span, &sent);
2662        gate.record_destinations(&followed, refilled);
2663        (span, sent, followed)
2664    }
2665
2666    /// The request of one plain-path attempt: the attempt timeout under a
2667    /// deadline, the trace context, then the caller's headers, which may
2668    /// override or extend the trace headers.
2669    fn plain_attempt(&self, gate: &RetryGate, span: &tracing::Span) -> reqwest::RequestBuilder {
2670        let timeout = gate.attempt_timeout(self.retry_policy.request_timeout);
2671        let mut req = self.client.request(self.method.clone(), &self.url);
2672        if gate.deadline.is_some()
2673            && let Some(timeout) = timeout
2674        {
2675            req = req.timeout(timeout);
2676        }
2677        req = span.in_scope(|| inject_trace_context(req, &self.extra_headers));
2678        req = with_caller_headers(req, gate, timeout, &self.extra_headers, &self.url);
2679        if let Some(body) = &self.body {
2680            req = req.body(body.clone());
2681        }
2682        req
2683    }
2684
2685    /// Ask the throttle for one attempt (issue #3068). `host` is the throttle
2686    /// key; `None` takes it from the URL. Returns `Ok(None)` without a
2687    /// throttle, and [`ClientError::ThrottledLocally`] when it rejects.
2688    fn throttle_attempt(&self, host: Option<&str>) -> Result<Option<ThrottleTicket>, ClientError> {
2689        let Some(throttle) = &self.throttle else {
2690            return Ok(None);
2691        };
2692        let Some(host) = host.map(str::to_owned).or_else(|| throttle_host(&self.url)) else {
2693            return Ok(None);
2694        };
2695        admit_throttle(throttle, host, &*self.entropy).map(Some)
2696    }
2697
2698    /// How many attempts the retry policy allows for this request.
2699    const fn max_attempts(&self, suppress_retries: bool) -> u32 {
2700        if suppress_retries {
2701            1
2702        } else if is_idempotent_method(&self.method) || !self.retry_policy.retry_idempotent_only {
2703            self.retry_policy.max_retries.saturating_add(1)
2704        } else {
2705            1
2706        }
2707    }
2708
2709    /// The canned response for this request from `mock`, or
2710    /// [`ClientError::NoMock`].
2711    fn mock_response(&self, mock: &MockRegistry) -> Result<Response, ClientError> {
2712        let Some(mock_resp) = mock.find_match(&self.method, &self.url, self.alias.as_deref())
2713        else {
2714            // A mock registry is present but nothing matched — treat as a test
2715            // failure rather than falling through to the network.
2716            return Err(ClientError::NoMock(
2717                self.method.to_string(),
2718                self.url.clone(),
2719            ));
2720        };
2721        let status =
2722            reqwest::StatusCode::from_u16(mock_resp.status).unwrap_or(reqwest::StatusCode::OK);
2723        let body_bytes = mock_resp
2724            .body
2725            .as_ref()
2726            .map(|v| serde_json::to_vec(v).unwrap_or_default())
2727            .unwrap_or_default();
2728
2729        tracing::info!(
2730            http.method = %self.method,
2731            http.url = %self.url,
2732            http.status = mock_resp.status,
2733            "[mock] outbound request intercepted"
2734        );
2735
2736        Ok(Response {
2737            status,
2738            headers: HeaderMap::new(),
2739            body: Bytes::from(body_bytes),
2740            url: None,
2741        })
2742    }
2743
2744    /// Send through the simulated network (issue #2967), with the attempts,
2745    /// jittered backoff, `Retry-After` on 429 and 503, and per-attempt
2746    /// timeout of the real retry loop.
2747    async fn send_sim(self, net: &crate::sim::SimNet) -> Result<Response, ClientError> {
2748        use tracing::Instrument as _;
2749
2750        let url = self.sim_url()?;
2751        let host = url
2752            .host_str()
2753            .ok_or_else(|| ClientError::InvalidUrl(format!("{}: no host", self.url)))?
2754            .to_owned();
2755        let max_attempts = self.max_attempts(false);
2756        let gate = self.retry_gate(url_host(url.as_str()).as_deref());
2757        let mut last_retry = None;
2758        let mut delay = Duration::ZERO;
2759        for attempt in 0..max_attempts {
2760            if attempt > 0 {
2761                tokio::time::sleep(delay).await;
2762            }
2763            let ticket = self.begin_attempt(&gate, Some(&host))?;
2764            let last = attempt + 1 == max_attempts;
2765            // One CLIENT span per attempt, as on the real send paths. The host
2766            // router sees this span in its `traceparent`.
2767            let span = client_attempt_span(&self.method, url.as_str(), attempt);
2768            let timeout = gate.attempt_timeout(self.retry_policy.request_timeout);
2769            let deadline_header = gate.header(timeout, &self.extra_headers);
2770            let exchange = self
2771                .sim_attempt(net, &host, &url, timeout, deadline_header)
2772                .instrument(span.clone());
2773            // Under a deadline, check it before each poll of the attempt, as
2774            // the real paths' request timeout does: `tokio::time::timeout`
2775            // polls the attempt first, so an answer ready at the deadline
2776            // would still be used.
2777            let exchange = async {
2778                match gate.deadline {
2779                    Some(deadline) => crate::deadline::Bounded::until(deadline, exchange)
2780                        .await
2781                        .unwrap_or_else(|crate::deadline::DeadlineExceeded| {
2782                            Err(SimAttemptError::Transient(format!(
2783                                "request to {host} reached the request deadline"
2784                            )))
2785                        }),
2786                    None => exchange.await,
2787                }
2788            };
2789            let outcome = match timeout {
2790                Some(limit) => {
2791                    tokio::time::timeout(limit, exchange)
2792                        .await
2793                        .unwrap_or_else(|_elapsed| {
2794                            Err(SimAttemptError::Transient(format!(
2795                                "request to {host} timed out after {limit:?}"
2796                            )))
2797                        })
2798                }
2799                None => exchange.await,
2800            };
2801            match &outcome {
2802                Ok(response) => record_attempt_status(&span, response.status.as_u16()),
2803                Err(SimAttemptError::Transient(_)) => record_attempt_error(&span, "network"),
2804                Err(SimAttemptError::Fatal(_)) => record_attempt_error(&span, "request"),
2805            }
2806            drop(span);
2807            // A transient failure at the caller's deadline says nothing about
2808            // the host: its ticket is dropped, which counts as an accept.
2809            let deadline_stop =
2810                matches!(outcome, Err(SimAttemptError::Transient(_))) && gate.expired();
2811            if let Some(ticket) = ticket
2812                && !deadline_stop
2813            {
2814                ticket.record(matches!(&outcome, Ok(r) if throttle_accepts(r.status.as_u16())));
2815            }
2816            let response = match outcome {
2817                Ok(response) => response,
2818                // A drop or a timeout is transient, like a real connect or
2819                // timeout error, so it is retried.
2820                Err(SimAttemptError::Transient(_)) if gate.expired() => {
2821                    return Err(ClientError::DeadlineExceeded);
2822                }
2823                Err(SimAttemptError::Transient(message)) if !last => {
2824                    let wait = self.retry_policy.retry_delay(&*self.entropy, attempt, None);
2825                    if !gate.allow(RetryKind::Transient, wait) {
2826                        return Err(ClientError::SimNetwork(message));
2827                    }
2828                    delay = wait;
2829                    last_retry = Some(RetryKind::Transient);
2830                    continue;
2831                }
2832                Err(SimAttemptError::Transient(message)) => {
2833                    return Err(ClientError::SimNetwork(message));
2834                }
2835                Err(SimAttemptError::Fatal(error)) => return Err(error),
2836            };
2837            let status = response.status.as_u16();
2838            if is_retryable_response(status) && !last {
2839                let wait = self.retry_policy.retry_delay(
2840                    &*self.entropy,
2841                    attempt,
2842                    retry_hint(status, &response.headers),
2843                );
2844                let kind = retry_kind(status);
2845                if gate.allow(kind, wait) {
2846                    delay = wait;
2847                    last_retry = Some(kind);
2848                    continue;
2849                }
2850            }
2851            gate.finish(last_retry, status);
2852            return Ok(response);
2853        }
2854        unreachable!("the sim retry loop returns on its last attempt")
2855    }
2856
2857    /// The absolute URL a sim call goes to. A relative URL on a named client
2858    /// (`client.named("payments").get("/charge")`) goes to the host named by
2859    /// the alias, as a named http mock would match it.
2860    fn sim_url(&self) -> Result<reqwest::Url, ClientError> {
2861        match (reqwest::Url::parse(&self.url), self.alias.as_deref()) {
2862            (Ok(url), _) => Ok(url),
2863            (Err(url::ParseError::RelativeUrlWithoutBase), Some(alias)) => {
2864                let path = self.url.trim_start_matches('/');
2865                reqwest::Url::parse(&format!("http://{alias}/{path}"))
2866                    .map_err(|error| ClientError::InvalidUrl(format!("{}: {error}", self.url)))
2867            }
2868            (Err(error), _) => Err(ClientError::InvalidUrl(format!("{}: {error}", self.url))),
2869        }
2870    }
2871
2872    /// One attempt through the simulated network: the network, then the host
2873    /// router or the http mocks.
2874    async fn sim_attempt(
2875        &self,
2876        net: &crate::sim::SimNet,
2877        host: &str,
2878        url: &reqwest::Url,
2879        timeout: Option<Duration>,
2880        deadline_header: Option<HeaderValue>,
2881    ) -> Result<Response, SimAttemptError> {
2882        net.transmit(host, timeout)
2883            .await
2884            .map_err(|fault| SimAttemptError::Transient(format!("request to {host} {fault}")))?;
2885        match (net.service(host), self.mock.as_ref()) {
2886            (Some(router), _) => serve_sim_host(router, self, url.clone(), deadline_header)
2887                .await
2888                .map_err(SimAttemptError::Fatal),
2889            (None, Some(mock)) => self.mock_response(mock).map_err(SimAttemptError::Fatal),
2890            (None, None) => Err(SimAttemptError::Fatal(ClientError::SimNetwork(format!(
2891                "no sim host named {host}"
2892            )))),
2893        }
2894    }
2895
2896    /// `true` when any security-hardening option requires the custom send path.
2897    const fn needs_custom_path(&self) -> bool {
2898        self.ssrf_safe
2899            || self.pin_addr.is_some()
2900            || !matches!(self.redirect_mode, RedirectMode::Default)
2901    }
2902
2903    /// Dispatch to the appropriate custom send path. Consumes `self`.
2904    ///
2905    /// `is_half_open`: `true` when this call is a circuit-breaker half-open
2906    /// probe (only ever passed by [`Self::send_custom_breaker_guarded`]);
2907    /// forces every `send_one` this dispatch reaches down to a single
2908    /// attempt, exactly like the plain-path breaker block's own
2909    /// `send_inner(is_half_open)` already does — see
2910    /// `send_custom_breaker_guarded`'s doc comment.
2911    async fn send_custom(self, is_half_open: bool) -> Result<Response, ClientError> {
2912        // Reject the incompatible `get_ssrf_safe` + `pin_to` combination up
2913        // front — deterministically, before any network I/O. `get_ssrf_safe`
2914        // routes through `send_ssrf_safe`, which runs its OWN per-hop
2915        // resolve→validate→pin and never reads `self.pin_addr`; a caller's
2916        // explicit `pin_to(addr)` would therefore be silently ignored. Fail
2917        // loudly instead so the mismatch is caught rather than masked. Use
2918        // `pin_to` alone for a caller-chosen fixed address, or `get_ssrf_safe`
2919        // alone for guarded automatic per-hop pinning.
2920        if self.ssrf_safe && self.pin_addr.is_some() {
2921            return Err(ClientError::PinNotAllowedWithSsrfSafe(
2922                "get_ssrf_safe cannot be combined with pin_to: the SSRF-safe path \
2923                 performs its own per-hop resolve/validate/pin and never reads the \
2924                 pin_to address, so an explicit pin would be silently ignored. Use \
2925                 pin_to alone for a caller-chosen address, or get_ssrf_safe alone \
2926                 for guarded automatic per-hop pinning.",
2927            ));
2928        }
2929
2930        // Reject the incompatible `pin_to` + `follow_redirects` combination up
2931        // front — deterministically, before issuing any request. A single pinned
2932        // `SocketAddr` only applies to hop 0 of `follow_loop`; later hops resolve
2933        // via normal DNS, so following a cross-host `3xx` would silently escape
2934        // the pin and defeat its purpose. `get_ssrf_safe` never sets `pin_addr`,
2935        // so its per-hop resolve→validate→pin path is unaffected by this guard.
2936        if self.pin_addr.is_some() && matches!(self.redirect_mode, RedirectMode::Follow { .. }) {
2937            return Err(ClientError::IncompatiblePinRedirect(
2938                "pin_to cannot be combined with follow_redirects: the pin only \
2939                 covers the first hop and later redirect hops re-resolve via DNS, \
2940                 escaping the pin. Use get_ssrf_safe for pinned, per-hop-revalidated \
2941                 redirect following; pin_to alone (which returns the 3xx unfollowed); \
2942                 or follow_redirects without pin_to.",
2943            ));
2944        }
2945
2946        // An empty pin set would send the request unpinned. Fail closed.
2947        if self.pin_addr.as_ref().is_some_and(Vec::is_empty) {
2948            return Err(ClientError::InvalidUrl(
2949                "pin_to_addrs needs at least one address".to_owned(),
2950            ));
2951        }
2952
2953        // Reject `pin_to` on an IP-literal URL host up front, deterministically, before
2954        // any network I/O. reqwest and hyper treat an IP-literal host as already resolved
2955        // and do not consult the DNS resolver, so the `resolve_to_addrs` override
2956        // `build_oneshot_client` installs — which is what enforces the pin — is skipped
2957        // and the socket connects to the literal in the URL rather than the pinned
2958        // address, silently violating `pin_to`'s documented guarantee. `get_ssrf_safe`
2959        // never sets `pin_addr`, and validates and connects to the same literal, so it
2960        // stays safe; this guard targets only the explicit `pin_to` primitive.
2961        if self.pin_addr.is_some() && url_host_is_ip_literal(&self.url)? {
2962            return Err(ClientError::PinRequiresDomainHost(
2963                "pin_to cannot be honored for an IP-literal URL host because the \
2964                 HTTP stack connects to the literal directly and skips the pinned \
2965                 address; put the desired IP directly in the URL, or use a domain host.",
2966            ));
2967        }
2968
2969        // One gate for the whole send, across redirect hops. The request
2970        // deadline also makes the one-shot client timeout shorter.
2971        let gate = self.retry_gate(url_host(&self.url).as_deref());
2972        // Only the deadline here: the first-attempt refill waits for
2973        // `send_one`, after DNS and address checks that may stop the call.
2974        if gate.expired() {
2975            return Err(ClientError::DeadlineExceeded);
2976        }
2977        let timeout = gate
2978            .attempt_timeout(self.retry_policy.request_timeout)
2979            .unwrap_or_else(|| Duration::from_secs(30));
2980
2981        if self.ssrf_safe {
2982            return self.send_ssrf_safe(timeout, is_half_open, &gate).await;
2983        }
2984
2985        // Extract the follow parameters (ending the borrow) before moving `self`.
2986        let follow = match &self.redirect_mode {
2987            RedirectMode::Follow { max, validator } => Some((*max, validator.clone())),
2988            RedirectMode::None | RedirectMode::Default => None,
2989        };
2990        if let Some((max, validator)) = follow {
2991            return self
2992                .follow_loop(
2993                    max,
2994                    validator,
2995                    HopClient::OneShot(timeout),
2996                    is_half_open,
2997                    &gate,
2998                )
2999                .await;
3000        }
3001
3002        // Only `RedirectMode::None` (explicit `no_redirect`) and the pin-only
3003        // `RedirectMode::Default` reach here (`Follow` and `ssrf_safe` returned
3004        // above). Both use `Policy::none()`: a 3xx is returned to the caller
3005        // verbatim rather than being auto-followed. Critically, for the
3006        // pin-only path this stops reqwest from silently following a cross-host
3007        // redirect and re-resolving the new host via normal DNS — which would
3008        // defeat the pin. Callers who want to follow redirects while staying
3009        // pinned/rebind-safe use `get_ssrf_safe` (or `follow_redirects`).
3010        let policy = reqwest::redirect::Policy::none();
3011        let resolve = self.pin_resolve()?;
3012        let client = build_oneshot_client(resolve, policy, timeout)?;
3013        send_one(
3014            &client,
3015            &self.method,
3016            &self.url,
3017            &self.extra_headers,
3018            self.body.as_ref(),
3019            &self.retry_policy,
3020            &*self.entropy,
3021            self.discard_response_body,
3022            None,
3023            is_half_open,
3024            &gate,
3025            false,
3026            None,
3027            None,
3028        )
3029        .await
3030    }
3031
3032    /// Compute the `(host, addr)` resolve override for a pinned request, if any.
3033    fn pin_resolve(&self) -> Result<Option<(String, Vec<SocketAddr>)>, ClientError> {
3034        match &self.pin_addr {
3035            Some(addrs) => Ok(Some((host_of(&self.url)?, addrs.clone()))),
3036            None => Ok(None),
3037        }
3038    }
3039
3040    /// Manual redirect-following loop with per-hop validation (Feature #1238).
3041    async fn follow_loop(
3042        self,
3043        max: usize,
3044        validator: RedirectValidator,
3045        hop_client: HopClient<'_>,
3046        is_half_open: bool,
3047        gate: &RetryGate,
3048    ) -> Result<Response, ClientError> {
3049        let original =
3050            url::Url::parse(&self.url).map_err(|e| ClientError::InvalidUrl(e.to_string()))?;
3051        let mut current = self.url.clone();
3052        // Threaded across hops so cross-origin header stripping (Fix A) and
3053        // RFC method/body rewriting (Fix B) accumulate correctly.
3054        let mut method = self.method.clone();
3055        let mut headers = self.extra_headers.clone();
3056        let mut body = self.body.clone();
3057        // On the pooled path the whole chain shares one retry count, as when
3058        // reqwest follows the redirects. It comes from the original method:
3059        // a `POST` that a 303 turns into a `GET` gets no retries the `POST`
3060        // did not have.
3061        let chain_retries = matches!(hop_client, HopClient::Pooled(_))
3062            .then(|| ChainRetries::new(self.max_attempts(is_half_open).saturating_sub(1)));
3063        // Hosts refilled for this chain: the first one already was.
3064        let mut refilled: std::collections::HashSet<String> =
3065            url_host(&self.url).into_iter().collect();
3066        for hop in 0.. {
3067            // Pin only applies to the first hop's original target.
3068            let resolve = if hop == 0 {
3069                match &self.pin_addr {
3070                    Some(addrs) => Some((host_of(&current)?, addrs.clone())),
3071                    None => None,
3072                }
3073            } else {
3074                None
3075            };
3076            // On any post-origin hop, drop credential-bearing headers if the
3077            // current target is cross-origin (stays stripped once stripped).
3078            if hop > 0 {
3079                strip_sensitive_headers_if_cross_origin(&mut headers, &original, &current)?;
3080            }
3081            let client = match hop_client {
3082                HopClient::Pooled(client) => client.clone(),
3083                HopClient::OneShot(timeout) => {
3084                    build_oneshot_client(resolve, reqwest::redirect::Policy::none(), timeout)?
3085                }
3086            };
3087            // Each hop uses the retry budget of its own host.
3088            let hop_gate;
3089            let hop_gate = if hop == 0 {
3090                gate
3091            } else {
3092                hop_gate = gate.for_hop(&current, &mut refilled);
3093                &hop_gate
3094            };
3095            let resp = send_one(
3096                &client,
3097                &method,
3098                &current,
3099                &headers,
3100                body.as_ref(),
3101                &self.retry_policy,
3102                &*self.entropy,
3103                self.discard_response_body,
3104                None,
3105                is_half_open,
3106                hop_gate,
3107                true,
3108                chain_retries.as_ref(),
3109                // The custom path counts a whole call instead (see
3110                // `send_custom_breaker_guarded`).
3111                matches!(hop_client, HopClient::Pooled(_))
3112                    .then_some(self.throttle.as_ref())
3113                    .flatten(),
3114            )
3115            .await?;
3116
3117            let next = match redirect_target(&resp, &current) {
3118                Ok(Some(next)) => next,
3119                Ok(None) => return Ok(resp),
3120                // reqwest returns a redirect with a bad `Location` as is.
3121                Err(ClientError::InvalidUrl(_)) if matches!(hop_client, HopClient::Pooled(_)) => {
3122                    return Ok(resp);
3123                }
3124                Err(error) => return Err(error),
3125            };
3126            if hop >= max {
3127                return Err(ClientError::TooManyRedirects(max));
3128            }
3129            if !validator(&next) {
3130                return Err(ClientError::RedirectRejected(next));
3131            }
3132            // RFC 7231/7538 method+body rewriting before the next hop.
3133            rewrite_after_redirect(resp.status(), &mut method, &mut body, &mut headers);
3134            // The pooled client stands in for reqwest's own redirect
3135            // handling, which sets `Referer`.
3136            if matches!(hop_client, HopClient::Pooled(_)) {
3137                set_referer(&mut headers, &next, &current);
3138            }
3139            current = next;
3140        }
3141        unreachable!("redirect loop is bounded by `max` and always returns")
3142    }
3143
3144    /// Derive the SSRF-safe redirect plan `(follow, max)` from the builder's
3145    /// [`RedirectMode`], so a chained `no_redirect()` / `follow_redirects(..)`
3146    /// overrides the default hop cap on the SSRF-safe path:
3147    ///
3148    /// - [`RedirectMode::Default`] → `(true, SSRF_SAFE_MAX_REDIRECTS)`.
3149    /// - [`RedirectMode::None`] (`no_redirect()`) → `(false, 0)` — the initial
3150    ///   `3xx` is returned verbatim (the `max` is unused).
3151    /// - [`RedirectMode::Follow { max, .. }`] (`follow_redirects(max, ..)`) →
3152    ///   `(true, max)`. The caller's per-hop validator is pulled from
3153    ///   `self.redirect_mode` separately inside the send loop.
3154    const fn ssrf_redirect_plan(&self) -> (bool, usize) {
3155        match &self.redirect_mode {
3156            RedirectMode::Default => (true, SSRF_SAFE_MAX_REDIRECTS),
3157            RedirectMode::None => (false, 0),
3158            RedirectMode::Follow { max, .. } => (true, *max),
3159        }
3160    }
3161
3162    /// Composed SSRF-safe send path (Features #1238 + #1239). Resolves and
3163    /// validates every hop, pins the connection, and rejects scheme downgrades.
3164    ///
3165    /// The follow/hop-cap behaviour comes from [`ssrf_redirect_plan`](Self::ssrf_redirect_plan),
3166    /// so a chained `no_redirect()` / `follow_redirects(max, ..)` overrides the
3167    /// default cap while every per-hop safety step (resolve→validate→pin,
3168    /// https→http downgrade block, sensitive-header stripping, method/body
3169    /// rewrite) still applies.
3170    /// Runs the whole resolve→validate→pin→redirect operation against one
3171    /// deadline `timeout` from now, rather than handing every phase of every
3172    /// hop a fresh `timeout`-length budget.
3173    ///
3174    /// Without this, `timeout` only bounded each hop's own connect/response
3175    /// wait — the DNS lookup in [`resolve_and_validate`] had no timeout of
3176    /// its own, and every redirect hop got a full fresh `timeout` regardless
3177    /// of how long earlier hops already took. A subscriber-controlled
3178    /// destination (the motivating case: outbound webhook delivery, #2480
3179    /// code review) with stalled DNS or a slow multi-hop redirect chain
3180    /// could therefore occupy a job worker for `timeout × (hops + 1)` plus
3181    /// unbounded DNS wait, rather than the single `timeout` every other
3182    /// outbound call is bounded by.
3183    ///
3184    /// The budget shrinks across two seams per hop — the DNS lookup, then
3185    /// the connect/response reqwest performs — rather than being enforced by
3186    /// one coarse outer `tokio::time::timeout` wrapping the whole call: a
3187    /// coarse wrap would race the *same* duration against both this
3188    /// operation's start and each hop's own reqwest-level timeout, and since
3189    /// the outer clock always starts first it would almost always fire
3190    /// first, silently reclassifying an ordinary single-hop stall from
3191    /// `ClientError::Request` (a `reqwest::Error` callers can query with
3192    /// `.is_timeout()`) into `ClientError::InvalidUrl` (#2480 review, round
3193    /// 7). Shrinking the *reqwest* timeout instead means a connect/response
3194    /// stall in the common (no-redirect) case still times out inside
3195    /// reqwest itself and keeps that exact, already-documented error shape;
3196    /// only a stall in the DNS lookup — which had no error shape of its own
3197    /// to preserve, because it had no timeout at all before this fix —
3198    /// surfaces as `InvalidUrl`, consistent with the DNS-failure case
3199    /// immediately below reusing the same variant.
3200    async fn send_ssrf_safe(
3201        self,
3202        timeout: Duration,
3203        is_half_open: bool,
3204        gate: &RetryGate,
3205    ) -> Result<Response, ClientError> {
3206        let deadline = crate::time::ambient_instant() + timeout;
3207        let (follow, max) = self.ssrf_redirect_plan();
3208        let original =
3209            url::Url::parse(&self.url).map_err(|e| ClientError::InvalidUrl(e.to_string()))?;
3210        let mut current = self.url.clone();
3211        // Threaded across hops so cross-origin header stripping (Fix A) and
3212        // RFC method/body rewriting (Fix B) accumulate correctly.
3213        let mut method = self.method.clone();
3214        let mut headers = self.extra_headers.clone();
3215        let mut body = self.body.clone();
3216        // Hosts refilled for this chain: the first one already was.
3217        let mut refilled: std::collections::HashSet<String> =
3218            url_host(&self.url).into_iter().collect();
3219        for hop in 0.. {
3220            let remaining_for_lookup = deadline_remaining_or_timeout(deadline, &current)
3221                .map_err(|error| gate.classify(error))?;
3222            // Resolve host → ALL validated addresses (rejects if ANY resolved IP
3223            // is blocked), then pin the full set so reqwest cannot re-resolve but
3224            // can still fall back across the validated addresses in order.
3225            // Wrapped in the *remaining* budget, not the full per-request
3226            // `timeout`: resolve_and_validate's DNS lookup previously had no
3227            // timeout of its own at all.
3228            let addrs = tokio::time::timeout(remaining_for_lookup, resolve_and_validate(&current))
3229                .await
3230                .map_err(|_| gate.classify(ssrf_safe_deadline_error(&current)))??;
3231            let host = host_of(&current)?;
3232            // Re-measured after the lookup, so a slow DNS response shrinks
3233            // what's left for the connect/response phase below rather than
3234            // that phase getting a fresh full `timeout` regardless.
3235            let remaining_for_connect = deadline_remaining_or_timeout(deadline, &current)
3236                .map_err(|error| gate.classify(error))?;
3237            let client = build_oneshot_client(
3238                Some((host, addrs)),
3239                reqwest::redirect::Policy::none(),
3240                remaining_for_connect,
3241            )?;
3242            // On any post-origin hop, drop credential-bearing headers if the
3243            // current target is cross-origin (stays stripped once stripped).
3244            if hop > 0 {
3245                strip_sensitive_headers_if_cross_origin(&mut headers, &original, &current)?;
3246            }
3247            // Each hop uses the retry budget of its own host.
3248            let hop_gate;
3249            let hop_gate = if hop == 0 {
3250                gate
3251            } else {
3252                hop_gate = gate.for_hop(&current, &mut refilled);
3253                &hop_gate
3254            };
3255            let resp = send_one(
3256                &client,
3257                &method,
3258                &current,
3259                &headers,
3260                body.as_ref(),
3261                &self.retry_policy,
3262                &*self.entropy,
3263                self.discard_response_body,
3264                Some(deadline),
3265                is_half_open,
3266                hop_gate,
3267                false,
3268                None,
3269                None,
3270            )
3271            .await?;
3272
3273            // Honour a chained `no_redirect()`: return the response verbatim
3274            // BEFORE parsing the `Location` header. The initial URL was still
3275            // resolved / validated / pinned above. Parsing `Location` here (via
3276            // `redirect_target`) would let an untrusted server force an
3277            // `InvalidUrl` error out of a `no_redirect()` fetch by returning a
3278            // followable 3xx with a malformed `Location`, violating the
3279            // documented "return the initial 3xx verbatim" contract.
3280            if !follow {
3281                return Ok(resp);
3282            }
3283            let Some(next) = redirect_target(&resp, &current)? else {
3284                return Ok(resp);
3285            };
3286            if hop >= max {
3287                return Err(ClientError::TooManyRedirects(max));
3288            }
3289            // Defence-in-depth: reject an https→http downgrade on redirect.
3290            if scheme_is_https(&current)? && !scheme_is_https(&next)? {
3291                return Err(ClientError::RedirectRejected(format!(
3292                    "https→http scheme downgrade on redirect to {next}"
3293                )));
3294            }
3295            // Caller-supplied per-hop validator, present only in the
3296            // `follow_redirects` override. It runs in ADDITION to the built-in
3297            // resolve/validate/pin already applied at the top of the loop.
3298            if let RedirectMode::Follow { validator, .. } = &self.redirect_mode
3299                && !validator(&next)
3300            {
3301                return Err(ClientError::RedirectRejected(next));
3302            }
3303            // RFC 7231/7538 method+body rewriting before the next hop.
3304            rewrite_after_redirect(resp.status(), &mut method, &mut body, &mut headers);
3305            current = next;
3306            // The next loop iteration re-resolves + re-validates `current`
3307            // before connecting, so a redirect to a blocked address is rejected
3308            // there with `SsrfBlocked`.
3309        }
3310        unreachable!("redirect loop is bounded by the SSRF-safe redirect plan")
3311    }
3312}
3313
3314// ── Internal helpers ─────────────────────────────────────────────────────────
3315
3316const fn is_idempotent_method(method: &Method) -> bool {
3317    matches!(
3318        *method,
3319        Method::GET | Method::HEAD | Method::PUT | Method::DELETE | Method::OPTIONS | Method::TRACE
3320    )
3321}
3322
3323const fn is_retryable_status(status: u16) -> bool {
3324    matches!(status, 502..=504)
3325}
3326
3327/// Time left until `deadline`, or the deadline error if it has already
3328/// passed before this hop's next phase (DNS lookup or connect) even began.
3329/// Used by [`RequestBuilder::send_ssrf_safe`] to shrink the budget handed to
3330/// each successive phase rather than resetting it every hop.
3331fn deadline_remaining_or_timeout(
3332    deadline: std::time::Instant,
3333    current: &str,
3334) -> Result<Duration, ClientError> {
3335    let now = crate::time::ambient_instant();
3336    if now >= deadline {
3337        return Err(ssrf_safe_deadline_error(current));
3338    }
3339    Ok(deadline - now)
3340}
3341
3342/// The error [`RequestBuilder::send_ssrf_safe`] returns when its overall
3343/// deadline is exhausted — reusing [`ClientError::InvalidUrl`] rather than a
3344/// new variant (see that call site's doc comment for why), consistent with
3345/// the adjacent DNS-lookup-failure case in [`resolve_and_validate`] already
3346/// using the same variant for the same phase.
3347fn ssrf_safe_deadline_error(current: &str) -> ClientError {
3348    ClientError::InvalidUrl(format!(
3349        "SSRF-safe resolve/redirect operation exceeded its deadline resolving {current}"
3350    ))
3351}
3352
3353/// Resolve (or create) the circuit breaker for `url`'s host, honouring a
3354/// per-client `resilience_config` override exactly as [`RequestBuilder::send_recorded`]'s
3355/// own breaker path does. Shared by both the plain-path breaker block and
3356/// [`RequestBuilder::send_custom_breaker_guarded`] so the two cannot drift on
3357/// how a host name is derived from the URL.
3358fn breaker_for_url(
3359    resilience_config: Option<&Arc<crate::config::ResilienceConfig>>,
3360    url: &str,
3361) -> crate::circuit_breaker::CircuitBreaker {
3362    let host = url::Url::parse(url).ok().map_or_else(
3363        || "unknown".to_owned(),
3364        |u| {
3365            let h = u.host_str().unwrap_or("unknown");
3366            u.port()
3367                .map_or_else(|| h.to_owned(), |port| format!("{h}:{port}"))
3368        },
3369    );
3370
3371    resilience_config.map_or_else(
3372        || {
3373            crate::circuit_breaker::global_registry().get_or_create(
3374                &host,
3375                crate::circuit_breaker::CircuitBreakerPolicy::default(),
3376            )
3377        },
3378        |rc| {
3379            let policy = crate::circuit_breaker::CircuitBreakerPolicy::from_config(rc, &host);
3380            crate::circuit_breaker::global_registry().get_or_create_with_config(&host, policy)
3381        },
3382    )
3383}
3384
3385// ── Deadline and retry budget (issue #3058) ──────────────────────────────────
3386
3387/// Why a retry of a response with `status` is necessary.
3388const fn retry_kind(status: u16) -> RetryKind {
3389    if status == 429 {
3390        RetryKind::Throttling
3391    } else {
3392        RetryKind::Transient
3393    }
3394}
3395
3396/// The retry budget key of `url`: its host and its effective port. `None` for
3397/// a relative URL.
3398fn url_host(url: &str) -> Option<String> {
3399    let parsed = reqwest::Url::parse(url).ok()?;
3400    let host = parsed.host_str()?;
3401    Some(
3402        parsed
3403            .port_or_known_default()
3404            .map_or_else(|| host.to_owned(), |port| format!("{host}:{port}")),
3405    )
3406}
3407
3408/// Retry budgets and deadline header settings of a [`Client`].
3409#[derive(Clone)]
3410struct RetrySettings {
3411    /// Retry budgets, one for each host. `None` when the budget is off.
3412    budgets: Option<Arc<RetryBudgets>>,
3413    /// Send [`DEADLINE_HEADER`] when a request deadline is set.
3414    send_deadline_header: bool,
3415}
3416
3417impl RetrySettings {
3418    /// Settings for a client made outside an app: its own default budgets.
3419    fn standalone() -> Self {
3420        Self {
3421            budgets: Some(Arc::new(RetryBudgets::new(&RetryBudgetConfig::default()))),
3422            send_deadline_header: true,
3423        }
3424    }
3425}
3426
3427/// The shortest time left for which a retry starts. A shorter attempt cannot
3428/// get an answer, and tokio timers round up to whole milliseconds.
3429const MIN_ATTEMPT: Duration = Duration::from_millis(10);
3430
3431/// The app's retry budgets, stored in `AppState`. `config` is the setting
3432/// they were built from.
3433#[derive(Clone)]
3434pub(crate) struct SharedRetryBudgets {
3435    budgets: Arc<RetryBudgets>,
3436    config: RetryBudgetConfig,
3437}
3438
3439/// Serializes the replacement of the shared retry budgets.
3440static BUDGETS_REPLACE: std::sync::Mutex<()> = std::sync::Mutex::new(());
3441
3442/// The app-wide retry budgets for the effective `config`, or `None` when
3443/// they are off. A `state_initializer` may replace the config after boot, so
3444/// budgets built from other settings are replaced, as the shared throttle is.
3445fn shared_retry_budgets(
3446    state: &crate::AppState,
3447    config: &RetryBudgetConfig,
3448) -> Option<Arc<RetryBudgets>> {
3449    if !config.enabled {
3450        return None;
3451    }
3452    let current = || {
3453        state
3454            .extension::<SharedRetryBudgets>()
3455            .filter(|shared| shared.config == *config)
3456            .map(|shared| Arc::clone(&shared.budgets))
3457    };
3458    if let Some(budgets) = current() {
3459        return Some(budgets);
3460    }
3461    // Replace under a lock and check again, so that concurrent first calls
3462    // after a config change share one set. Only this slow path locks.
3463    let _guard = BUDGETS_REPLACE
3464        .lock()
3465        .unwrap_or_else(std::sync::PoisonError::into_inner);
3466    if let Some(budgets) = current() {
3467        return Some(budgets);
3468    }
3469    let budgets = Arc::new(RetryBudgets::new(config));
3470    state.insert_extension(SharedRetryBudgets {
3471        budgets: Arc::clone(&budgets),
3472        config: config.clone(),
3473    });
3474    Some(budgets)
3475}
3476
3477/// The request deadline and the retry budget for one send.
3478///
3479/// All three retry loops (plain, sim, custom) ask it the same questions.
3480struct RetryGate {
3481    deadline: Option<Deadline>,
3482    /// All budgets of the client, to pick the budget of a redirect hop.
3483    budgets: Option<Arc<RetryBudgets>>,
3484    /// The budget of the current host.
3485    budget: Option<Arc<RetryBudget>>,
3486    /// The key of the current host's budget.
3487    host: Option<String>,
3488    send_header: bool,
3489    /// The first-attempt refill of `budget`, done by the first
3490    /// [`check`](Self::check) that finds time left, so a request that never
3491    /// starts an attempt refills nothing.
3492    refill_pending: AtomicBool,
3493    /// The tokens [`allow`](Self::allow) took for a retry that has not
3494    /// started. The next [`check`](Self::check) that lets the retry run keeps
3495    /// them; a send cancelled in the backoff gives them back on drop.
3496    pending_retry: std::sync::Mutex<Option<(Arc<RetryBudget>, RetryKind)>>,
3497    /// The caller's own [`DEADLINE_HEADER`] value as a deadline, fixed at the
3498    /// first attempt. The value is relative, so a retry or a later hop sends
3499    /// what is left of it, not the whole value again.
3500    caller_deadline: std::sync::OnceLock<Option<Deadline>>,
3501    /// Signs each attempt again (Web Bot Auth), see [`with_caller_headers`].
3502    signer: Option<Arc<crate::aeo::web_bot_auth::WebBotAuthSigner>>,
3503}
3504
3505impl Drop for RetryGate {
3506    fn drop(&mut self) {
3507        let pending = self
3508            .pending_retry
3509            .get_mut()
3510            .unwrap_or_else(std::sync::PoisonError::into_inner)
3511            .take();
3512        if let Some((budget, kind)) = pending {
3513            budget.release(kind);
3514        }
3515    }
3516}
3517
3518impl RetryGate {
3519    /// Read the current deadline. The first attempt is recorded in the
3520    /// budget of `host` when it starts.
3521    fn start(budgets: Option<Arc<RetryBudgets>>, host: Option<&str>, send_header: bool) -> Self {
3522        Self::with_deadline(Deadline::current(), budgets, host, send_header)
3523    }
3524
3525    fn with_deadline(
3526        deadline: Option<Deadline>,
3527        budgets: Option<Arc<RetryBudgets>>,
3528        host: Option<&str>,
3529        send_header: bool,
3530    ) -> Self {
3531        let budget = budgets
3532            .as_deref()
3533            .zip(host)
3534            .map(|(budgets, host)| budgets.for_host(host));
3535        Self {
3536            deadline,
3537            budgets,
3538            budget,
3539            host: host.map(str::to_owned),
3540            send_header,
3541            refill_pending: AtomicBool::new(true),
3542            pending_retry: std::sync::Mutex::new(None),
3543            caller_deadline: std::sync::OnceLock::new(),
3544            signer: None,
3545        }
3546    }
3547
3548    /// Move to the budget of `url`'s host when it is not the current host,
3549    /// for example after the HTTP stack followed a redirect.
3550    ///
3551    /// The plain path refills a redirect host with
3552    /// [`record_destinations`](Self::record_destinations), so this does not.
3553    fn rekey(&mut self, url: &str) {
3554        let host = url_host(url);
3555        if host.is_some() && host != self.host {
3556            self.budget = self
3557                .budgets
3558                .as_deref()
3559                .zip(host.as_deref())
3560                .map(|(budgets, host)| budgets.for_host(host));
3561            self.host = host;
3562        }
3563    }
3564
3565    /// Record a first attempt in the budget of `url`'s host when it is not
3566    /// the current host: the HTTP stack followed a redirect and the answer is
3567    /// final. The gate keeps its budget, so a refund still goes to the budget
3568    /// that paid for the retry.
3569    fn record_destination(&self, url: &str) {
3570        let host = url_host(url);
3571        // No deadline check: the HTTP stack only lists a host it sent to.
3572        if host.is_none() || host == self.host {
3573            return;
3574        }
3575        if let Some((budgets, host)) = self.budgets.as_deref().zip(host) {
3576            budgets.for_host(&host).record_request();
3577        }
3578    }
3579
3580    /// [`record_destination`](Self::record_destination) for each host of a
3581    /// redirect chain that is not in `seen`, then add it to `seen`.
3582    fn record_destinations(&self, urls: &[String], seen: &mut std::collections::HashSet<String>) {
3583        for url in urls {
3584            if url_host(url).is_some_and(|host| seen.insert(host)) {
3585                self.record_destination(url);
3586            }
3587        }
3588    }
3589
3590    /// The gate of a redirect hop to `url`: the same deadline, and the
3591    /// budget of the hop's host.
3592    ///
3593    /// The hop's host gets its first-attempt refill only when it is not in
3594    /// `refilled`, so a host that a chain visits again is credited once.
3595    fn for_hop(&self, url: &str, refilled: &mut std::collections::HashSet<String>) -> Self {
3596        let mut hop = Self {
3597            deadline: self.deadline,
3598            budgets: self.budgets.clone(),
3599            budget: self.budget.clone(),
3600            host: self.host.clone(),
3601            send_header: self.send_header,
3602            refill_pending: AtomicBool::new(false),
3603            pending_retry: std::sync::Mutex::new(None),
3604            caller_deadline: self.caller_deadline.clone(),
3605            signer: self.signer.clone(),
3606        };
3607        hop.rekey(url);
3608        if url_host(url).is_some_and(|host| refilled.insert(host)) {
3609            hop.refill_pending = AtomicBool::new(true);
3610        }
3611        hop
3612    }
3613
3614    /// The error of a failed body read: [`ClientError::DeadlineExceeded`] for
3615    /// a timeout the request deadline caused, else [`ClientError::Request`].
3616    fn body_error(&self, error: reqwest::Error) -> ClientError {
3617        if error.is_timeout() && self.expired() {
3618            ClientError::DeadlineExceeded
3619        } else {
3620            ClientError::Request(error.without_url())
3621        }
3622    }
3623
3624    /// `error`, or [`ClientError::DeadlineExceeded`] when the request deadline
3625    /// has passed and so is the cause.
3626    fn classify(&self, error: ClientError) -> ClientError {
3627        if self.expired() {
3628            ClientError::DeadlineExceeded
3629        } else {
3630            error
3631        }
3632    }
3633
3634    /// An error when no time is left to start an attempt.
3635    ///
3636    /// The first check with time left does the first-attempt refill.
3637    fn check(&self) -> Result<(), ClientError> {
3638        if self.expired() {
3639            return Err(ClientError::DeadlineExceeded);
3640        }
3641        // The retry starts, so it keeps its tokens.
3642        self.pending_retry
3643            .lock()
3644            .unwrap_or_else(std::sync::PoisonError::into_inner)
3645            .take();
3646        if self.refill_pending.swap(false, Ordering::Relaxed)
3647            && let Some(budget) = &self.budget
3648        {
3649            budget.record_request();
3650        }
3651        Ok(())
3652    }
3653
3654    /// The timeout of the next attempt: `per_try`, or the time left if that is
3655    /// shorter.
3656    fn attempt_timeout(&self, per_try: Option<Duration>) -> Option<Duration> {
3657        match (per_try, self.deadline) {
3658            (Some(limit), Some(deadline)) => Some(deadline.clamp(limit)),
3659            (None, Some(deadline)) => Some(deadline.remaining()),
3660            (limit, None) => limit,
3661        }
3662    }
3663
3664    /// `true` when the request deadline has passed.
3665    fn expired(&self) -> bool {
3666        self.deadline.is_some_and(Deadline::is_expired)
3667    }
3668
3669    /// `true` when a retry of `kind` can start after `wait`. It must have at
3670    /// least [`MIN_ATTEMPT`] left after the wait, and it takes tokens from the
3671    /// budget. The tokens come back if the retry never starts.
3672    fn allow(&self, kind: RetryKind, wait: Duration) -> bool {
3673        if self
3674            .deadline
3675            .is_some_and(|deadline| deadline.remaining() <= wait + MIN_ATTEMPT)
3676        {
3677            return false;
3678        }
3679        let Some(budget) = &self.budget else {
3680            return true;
3681        };
3682        if !budget.try_acquire(kind) {
3683            return false;
3684        }
3685        *self
3686            .pending_retry
3687            .lock()
3688            .unwrap_or_else(std::sync::PoisonError::into_inner) = Some((Arc::clone(budget), kind));
3689        true
3690    }
3691
3692    /// Give back the tokens of the last retry when the final `status` is a
3693    /// success (below 400).
3694    fn finish(&self, last_retry: Option<RetryKind>, status: u16) {
3695        if let (Some(budget), Some(kind)) = (&self.budget, last_retry)
3696            && status < 400
3697        {
3698            budget.release(kind);
3699        }
3700    }
3701
3702    /// The [`DEADLINE_HEADER`] value for an attempt with `timeout`. `None`
3703    /// with no deadline and no caller value the client can read.
3704    ///
3705    /// A value the caller set is kept when it is shorter, so the header never
3706    /// says more than the time left. When this returns a value, the caller's
3707    /// own copy of the header is not sent.
3708    fn header(&self, timeout: Option<Duration>, caller: &HeaderMap) -> Option<HeaderValue> {
3709        let millis = |d: Duration| u64::try_from(d.as_millis()).unwrap_or(u64::MAX);
3710        // The time left of the request deadline, when one is set.
3711        let left = self.deadline.and(timeout).map(millis);
3712        let Some(value) = caller.get(DEADLINE_HEADER) else {
3713            return left.filter(|_| self.send_header).map(HeaderValue::from);
3714        };
3715        // The first attempt sends the caller's value as set; later ones send
3716        // what is left of it, with or without a request deadline.
3717        let mut first = None;
3718        let caller_deadline = self.caller_deadline.get_or_init(|| {
3719            first = crate::deadline::parse_header(value);
3720            first.map(Deadline::after)
3721        });
3722        let theirs = first.map(millis).or_else(|| {
3723            caller_deadline
3724                .as_ref()
3725                .map(|deadline| millis(deadline.remaining()))
3726        });
3727        match (left, theirs) {
3728            (Some(left), Some(theirs)) => Some(HeaderValue::from(left.min(theirs))),
3729            (Some(value), None) | (None, Some(value)) => Some(HeaderValue::from(value)),
3730            // No deadline, and a value the client cannot read: sent as set.
3731            (None, None) => None,
3732        }
3733    }
3734}
3735
3736/// Add the [`DEADLINE_HEADER`] for an attempt with `timeout`, then the
3737/// caller's headers. The caller's own deadline header is left out when the
3738/// gate sends one, which is never longer than the caller's.
3739fn with_caller_headers(
3740    mut req: reqwest::RequestBuilder,
3741    gate: &RetryGate,
3742    timeout: Option<Duration>,
3743    caller: &HeaderMap,
3744    url: &str,
3745) -> reqwest::RequestBuilder {
3746    let deadline_value = gate.header(timeout, caller);
3747    let sends_deadline = deadline_value.is_some();
3748    if let Some(value) = deadline_value {
3749        req = req.header(DEADLINE_HEADER, value);
3750    }
3751    // A Web Bot Auth signature is made new for each attempt. A cross-origin
3752    // hop already dropped the old one, and gets none.
3753    let fresh = gate
3754        .signer
3755        .as_ref()
3756        .filter(|_| caller.contains_key("signature"))
3757        .and_then(|signer| signer.sign_url(url));
3758    for (name, value) in caller {
3759        if sends_deadline && name == DEADLINE_HEADER {
3760            continue;
3761        }
3762        if fresh.is_some()
3763            && matches!(
3764                name.as_str(),
3765                "signature" | "signature-input" | "signature-agent"
3766            )
3767        {
3768            continue;
3769        }
3770        req = req.header(name.clone(), value.clone());
3771    }
3772    if let Some(h) = fresh {
3773        req = req
3774            .header("signature-agent", h.signature_agent)
3775            .header("signature-input", h.signature_input)
3776            .header("signature", h.signature);
3777    }
3778    req
3779}
3780
3781// ── Custom send-path helpers (redirect / pin / SSRF-safe) ─────────────────────
3782
3783/// Why one simulated attempt failed.
3784enum SimAttemptError {
3785    /// A drop, a partition or a timeout: retried while attempts remain.
3786    Transient(String),
3787    /// Not retried.
3788    Fatal(ClientError),
3789}
3790
3791/// Serve one request from a simulated host's router (issue #2967).
3792async fn serve_sim_host(
3793    router: axum::Router,
3794    request: &RequestBuilder,
3795    url: reqwest::Url,
3796    deadline_header: Option<HeaderValue>,
3797) -> Result<Response, ClientError> {
3798    let target = url.query().map_or_else(
3799        || url.path().to_owned(),
3800        |query| format!("{}?{query}", url.path()),
3801    );
3802    let mut builder = axum::http::Request::builder()
3803        .method(request.method.clone())
3804        .uri(target);
3805    if !request.extra_headers.contains_key(reqwest::header::HOST) {
3806        // Host and port as the URL writes them, so an IPv6 host keeps its
3807        // brackets (`[::1]:8080`).
3808        let authority = &url[url::Position::BeforeHost..url::Position::AfterPort];
3809        builder = builder.header(reqwest::header::HOST, authority);
3810    }
3811    // Trace context first, as on the real send path. A caller header wins.
3812    for (name, value) in trace_headers_to_send(trace_context_headers(), &request.extra_headers) {
3813        builder = builder.header(name, value);
3814    }
3815    let sends_deadline = deadline_header.is_some();
3816    if let Some(value) = deadline_header {
3817        builder = builder.header(DEADLINE_HEADER, value);
3818    }
3819    // The real client sends a `Content-Length` for a known-size body. A
3820    // caller header of the same name wins.
3821    if let Some(body) = &request.body
3822        && !request
3823            .extra_headers
3824            .contains_key(reqwest::header::CONTENT_LENGTH)
3825    {
3826        builder = builder.header(reqwest::header::CONTENT_LENGTH, body.len());
3827    }
3828    // A Web Bot Auth signature is made new for each attempt, as on the real
3829    // send path (`with_caller_headers`).
3830    let fresh = request
3831        .web_bot_auth
3832        .as_ref()
3833        .filter(|_| request.extra_headers.contains_key("signature"))
3834        .and_then(|signer| signer.sign_url(url.as_str()));
3835    for (name, value) in &request.extra_headers {
3836        if sends_deadline && name == DEADLINE_HEADER {
3837            continue;
3838        }
3839        if fresh.is_some()
3840            && matches!(
3841                name.as_str(),
3842                "signature" | "signature-input" | "signature-agent"
3843            )
3844        {
3845            continue;
3846        }
3847        builder = builder.header(name, value);
3848    }
3849    if let Some(h) = fresh {
3850        builder = builder
3851            .header("signature-agent", h.signature_agent)
3852            .header("signature-input", h.signature_input)
3853            .header("signature", h.signature);
3854    }
3855    let body = request.body.clone().unwrap_or_default();
3856    let http_request = builder
3857        .body(axum::body::Body::from(body))
3858        .map_err(|error| ClientError::SimNetwork(error.to_string()))?;
3859    // The host is another process: it must not see the caller's task-local
3860    // deadline. Only the deadline header goes to it.
3861    let response = crate::deadline::unscoped(tower::ServiceExt::oneshot(router, http_request))
3862        .await
3863        .unwrap_or_else(|never| match never {});
3864    let status = response.status();
3865    let headers = response.headers().clone();
3866    let body = if request.discard_response_body {
3867        Bytes::new()
3868    } else {
3869        axum::body::to_bytes(response.into_body(), usize::MAX)
3870            .await
3871            .map_err(|error| ClientError::SimNetwork(error.to_string()))?
3872    };
3873    Ok(Response {
3874        status,
3875        headers,
3876        body,
3877        url: Some(url),
3878    })
3879}
3880
3881/// Build a one-shot `reqwest::Client` for the custom send path, with the given
3882/// redirect policy, per-request timeout, and optional DNS `resolve` override.
3883///
3884/// **Proxy bypass on pinned clients.** When a `resolve` override is present the
3885/// client is built with `.no_proxy()` so the request connects DIRECTLY to the
3886/// validated/pinned address. reqwest evaluates proxy interception (from
3887/// `HTTP_PROXY` / `HTTPS_PROXY` / `ALL_PROXY`) BEFORE the connector where the
3888/// `resolve()` override applies, so a configured env proxy would otherwise send
3889/// the request to the proxy — which re-resolves the target host, reopening the
3890/// exact DNS-rebinding / SSRF window that pinning closes. Non-pinned callers
3891/// (`resolve == None`) keep reqwest's default proxy behaviour untouched.
3892fn build_oneshot_client(
3893    resolve: Option<(String, Vec<SocketAddr>)>,
3894    policy: reqwest::redirect::Policy,
3895    timeout: Duration,
3896) -> Result<reqwest::Client, ClientError> {
3897    let mut builder = reqwest::ClientBuilder::new()
3898        .timeout(timeout)
3899        .redirect(policy);
3900    if let Some((host, addrs)) = resolve
3901        && !addrs.is_empty()
3902    {
3903        // Pin to the FULL validated set: reqwest tries the addresses in order and
3904        // falls back on connection failure, so an unreachable first address no
3905        // longer dooms the request. Every pinned address was already validated,
3906        // so the TOCTOU/SSRF guarantee is preserved. A pinned request must never
3907        // route through a re-resolving proxy.
3908        builder = builder.no_proxy().resolve_to_addrs(&host, &addrs);
3909    }
3910    builder.build().map_err(ClientError::Request)
3911}
3912
3913/// The retries a redirect chain shares (see `follow_pooled`).
3914struct ChainRetries {
3915    /// Retries the chain may still make.
3916    left: AtomicU32,
3917    /// Retries already made, so a later hop's backoff goes on from them.
3918    used: AtomicU32,
3919    /// When the attempt that answered with the redirect being followed runs
3920    /// out of its `request_timeout`. As with reqwest's own redirects, one
3921    /// timeout covers all hops of an attempt.
3922    attempt_end: Mutex<Option<Instant>>,
3923}
3924
3925impl ChainRetries {
3926    const fn new(retries: u32) -> Self {
3927        Self {
3928            left: AtomicU32::new(retries),
3929            used: AtomicU32::new(0),
3930            attempt_end: Mutex::new(None),
3931        }
3932    }
3933
3934    /// Take the end of the attempt that led to this hop.
3935    fn take_attempt_end(&self) -> Option<Instant> {
3936        self.attempt_end
3937            .lock()
3938            .unwrap_or_else(std::sync::PoisonError::into_inner)
3939            .take()
3940    }
3941
3942    /// Carry `end` to the next hop's first attempt.
3943    fn set_attempt_end(&self, end: Option<Instant>) {
3944        *self
3945            .attempt_end
3946            .lock()
3947            .unwrap_or_else(std::sync::PoisonError::into_inner) = end;
3948    }
3949}
3950
3951/// One attempt that the throttle let through. [`Self::record`] counts the
3952/// outcome. A ticket dropped without an outcome (the caller cancelled the
3953/// call) counts as an accept: the host did not reject it.
3954struct ThrottleTicket {
3955    throttle: Arc<crate::admission::AdaptiveThrottle>,
3956    host: String,
3957    recorded: bool,
3958}
3959
3960impl ThrottleTicket {
3961    fn record(mut self, accepted: bool) {
3962        self.recorded = true;
3963        self.throttle
3964            .record(&self.host, crate::time::ambient_instant(), accepted);
3965    }
3966}
3967
3968impl Drop for ThrottleTicket {
3969    fn drop(&mut self) {
3970        if !self.recorded {
3971            self.throttle
3972                .record(&self.host, crate::time::ambient_instant(), true);
3973        }
3974    }
3975}
3976
3977/// Ask `throttle` for one attempt to `host` (issue #3068).
3978/// [`ClientError::ThrottledLocally`] when it rejects.
3979fn admit_throttle(
3980    throttle: &Arc<crate::admission::AdaptiveThrottle>,
3981    host: String,
3982    entropy: &dyn crate::entropy::Entropy,
3983) -> Result<ThrottleTicket, ClientError> {
3984    let now = crate::time::ambient_instant();
3985    if throttle.admit(&host, now, || entropy.next_u64()) {
3986        Ok(ThrottleTicket {
3987            throttle: Arc::clone(throttle),
3988            host,
3989            recorded: false,
3990        })
3991    } else {
3992        tracing::debug!(host = %host, "outbound request throttled locally");
3993        Err(ClientError::ThrottledLocally { host })
3994    }
3995}
3996
3997/// Count a whole call that took one throttle `ticket`. A call that the
3998/// caller's deadline stopped says nothing about the host (issue #3058): its
3999/// ticket is dropped, which counts as an accept.
4000fn record_call(ticket: Option<ThrottleTicket>, res: &Result<Response, ClientError>) {
4001    if let Some(ticket) = ticket
4002        && !matches!(res, Err(ClientError::DeadlineExceeded))
4003    {
4004        ticket.record(matches!(res, Ok(r) if throttle_accepts(r.status.as_u16())));
4005    }
4006}
4007
4008/// `false` for the statuses a host uses to reject for overload.
4009const fn throttle_accepts(status: u16) -> bool {
4010    !matches!(status, 429 | 503)
4011}
4012
4013/// The throttle key of `url`: `host`, or `host:port` with an explicit port.
4014fn throttle_host(url: &str) -> Option<String> {
4015    let parsed = url::Url::parse(url).ok()?;
4016    let host = parsed.host_str()?;
4017    Some(
4018        parsed
4019            .port()
4020            .map_or_else(|| host.to_owned(), |port| format!("{host}:{port}")),
4021    )
4022}
4023
4024/// Send a single request through `client` (no manual redirect following — the
4025/// client's redirect policy governs that) with the same transient-error and
4026/// 429/5xx retry behaviour as the shared path, and collect the [`Response`].
4027///
4028/// `deadline`, when set, bounds the *retry loop* — checked before each
4029/// backoff sleep, so a retry that would start after the deadline is skipped
4030/// in favour of failing the call immediately with a deadline error, rather
4031/// than a stale response/error from an attempt that already ran. This exists
4032/// for
4033/// [`RequestBuilder::send_ssrf_safe`] (#2480 review, round 8): passing a
4034/// shrunk per-hop `timeout` into `client`'s own reqwest-level timeout, as
4035/// every other caller here already does, bounds one attempt's connect/
4036/// response wait, but `client` is reused across every retry `send_one`
4037/// itself performs — each gets that same per-attempt timeout again, and the
4038/// backoff/`Retry-After` sleeps between attempts are outside it entirely, so
4039/// a retried, timing-out SSRF-safe hop could still run well past the overall
4040/// deadline the caller computed. Checked only before sleeping (never wrapped
4041/// around an in-flight attempt), so it cannot race an attempt's own
4042/// reqwest-level timeout the way an outer `tokio::time::timeout` would (see
4043/// round 7's `ClientError::Request`/`.is_timeout()` regression) — the first
4044/// attempt's error shape is always preserved unchanged. Other callers pass
4045/// `None`, so their behavior is exactly as before this parameter existed.
4046///
4047/// Two more `deadline` seams round 9 found `send_ssrf_safe`'s own deadline
4048/// fix had missed: `client`'s own reqwest-level timeout is fixed at
4049/// build-hop-start, so a retry within the same hop still got the *original*
4050/// per-attempt budget rather than what's actually left; and the
4051/// `Retry-After` wait was not checked against the deadline. Both are fixed
4052/// here. Every attempt sends with an explicit per-request `.timeout()`
4053/// recomputed from `deadline`. This overrides `client`'s own timeout only
4054/// when `deadline` is set, so callers that pass `None` see no change. If a
4055/// `Retry-After` wait (on a 429 or 503, issue #3054) reaches the deadline,
4056/// the loop does not sleep. It returns that response as final, as it does
4057/// when `max_attempts` runs out.
4058///
4059/// `suppress_retries`, when `true`, forces a single attempt regardless of
4060/// `retry_policy` — the same thing [`RequestBuilder::send_inner`]'s own
4061/// `suppress_retries` parameter does for the plain (non-custom) breaker path
4062/// during a circuit-breaker half-open probe (#2480 review, round 10): a
4063/// probe is a budgeted, limited trial, and letting this retry loop turn one
4064/// trial into several real network attempts spends that budget on one
4065/// logical delivery rather than testing recovery with independent probes.
4066#[allow(
4067    clippy::too_many_arguments,
4068    reason = "one seam shared by three call sites (plain custom path, follow_loop, send_ssrf_safe); \
4069              splitting the request-shape fields into their own struct would still leave the \
4070              retry/discard/deadline/half-open knobs alongside it, for no reduction in what a \
4071              caller reasons about"
4072)]
4073#[allow(
4074    clippy::too_many_lines,
4075    reason = "one retry loop; the deadline and budget checks (#3058) sit at each decision"
4076)]
4077async fn send_one(
4078    client: &reqwest::Client,
4079    method: &Method,
4080    url: &str,
4081    extra_headers: &HeaderMap,
4082    body: Option<&Bytes>,
4083    retry_policy: &RetryPolicy,
4084    entropy: &dyn crate::entropy::Entropy,
4085    discard_response_body: bool,
4086    deadline: Option<Instant>,
4087    suppress_retries: bool,
4088    gate: &RetryGate,
4089    skip_redirect_body: bool,
4090    chain: Option<&ChainRetries>,
4091    throttle: Option<&Arc<crate::admission::AdaptiveThrottle>>,
4092) -> Result<Response, ClientError> {
4093    let start = crate::time::ambient_instant();
4094    let mut last_retry = None;
4095    let mut max_attempts = if suppress_retries {
4096        1
4097    } else if is_idempotent_method(method) || !retry_policy.retry_idempotent_only {
4098        retry_policy.max_retries.saturating_add(1)
4099    } else {
4100        1
4101    };
4102    // A redirect chain shares one retry count (see `follow_pooled`), and
4103    // its backoff goes on from the retries earlier hops made.
4104    let retries_before = chain.map_or(0, |chain| chain.used.load(Ordering::Relaxed));
4105    if let Some(chain) = chain {
4106        max_attempts = max_attempts.min(chain.left.load(Ordering::Relaxed).saturating_add(1));
4107    }
4108    let mut last_transient_err: Option<reqwest::Error> = None;
4109
4110    // A prior attempt's own connect/timeout error may be why the deadline is
4111    // already gone — surface that error (preserving
4112    // `ClientError::Request(e).is_timeout()` for callers) instead of masking
4113    // it as an unrelated `InvalidUrl`.
4114    let deadline_exceeded_err = |last_transient_err: &mut Option<reqwest::Error>| {
4115        last_transient_err.take().map_or_else(
4116            || ssrf_safe_deadline_error(url),
4117            |e| ClientError::Request(e.without_url()),
4118        )
4119    };
4120
4121    // `true` when a wait of `wait` leaves less than `MIN_ATTEMPT` before the
4122    // hop deadline, so no retry can run after it. `RetryGate::allow` keeps
4123    // the same margin before the request deadline.
4124    let reaches_hop_deadline = |wait: Duration| {
4125        deadline.is_some_and(|d| {
4126            crate::time::ambient_instant()
4127                .checked_add(wait.saturating_add(MIN_ATTEMPT))
4128                .is_none_or(|resume| resume >= d)
4129        })
4130    };
4131
4132    let mut delay = Duration::ZERO;
4133    for attempt in 0..max_attempts {
4134        let last = attempt + 1 == max_attempts;
4135        if attempt > 0 {
4136            if let Some(chain) = chain {
4137                chain.left.fetch_sub(1, Ordering::Relaxed);
4138                chain.used.fetch_add(1, Ordering::Relaxed);
4139            }
4140            if deadline.is_some_and(|d| crate::time::ambient_instant() >= d) {
4141                return Err(gate.classify(deadline_exceeded_err(&mut last_transient_err)));
4142            }
4143            let mut sleep_for = delay;
4144            if let Some(d) = deadline {
4145                sleep_for =
4146                    sleep_for.min(d.saturating_duration_since(crate::time::ambient_instant()));
4147            }
4148            tokio::time::sleep(sleep_for).await;
4149            if deadline.is_some_and(|d| crate::time::ambient_instant() >= d) {
4150                return Err(gate.classify(deadline_exceeded_err(&mut last_transient_err)));
4151            }
4152        }
4153
4154        // With `throttle`, each attempt asks it (issue #3068), as on the plain
4155        // path: a throttled retry ends the call. It asks before
4156        // `RetryGate::check`, so a throttled attempt refills no budget.
4157        if gate.expired() {
4158            return Err(ClientError::DeadlineExceeded);
4159        }
4160        let ticket = match throttle.zip(throttle_host(url)) {
4161            Some((throttle, host)) => Some(admit_throttle(throttle, host, entropy)?),
4162            None => None,
4163        };
4164        gate.check()?;
4165        let mut req = client.request(method.clone(), url);
4166        // Recomputed fresh every attempt (not just retries) rather than
4167        // relying solely on `client`'s own timeout, which was fixed when the
4168        // caller built it at hop-start: without this override, a retry deep
4169        // into a hop's budget would still get the full original per-attempt
4170        // timeout rather than what's actually left before `deadline`.
4171        let attempt_start = crate::time::ambient_instant();
4172        let hop_timeout = deadline.map(|d| d.saturating_duration_since(attempt_start));
4173        // On a chain the client follows itself, the first attempt of a later
4174        // hop goes on with the attempt that answered with the redirect, and
4175        // keeps what is left of its `request_timeout`. A retry is a new
4176        // attempt, with a new timeout.
4177        let carried_end = if attempt == 0 {
4178            chain.and_then(ChainRetries::take_attempt_end)
4179        } else {
4180            None
4181        };
4182        let per_try = match (retry_policy.request_timeout, carried_end) {
4183            (Some(_), Some(end)) => Some(end.saturating_duration_since(attempt_start)),
4184            (limit, _) => limit,
4185        };
4186        let attempt_end = carried_end.or_else(|| {
4187            retry_policy
4188                .request_timeout
4189                .and_then(|limit| attempt_start.checked_add(limit))
4190        });
4191        // The request deadline (issue #3058) can make it shorter again.
4192        let attempt_timeout = if gate.deadline.is_some() {
4193            gate.attempt_timeout(hop_timeout.or(per_try))
4194        } else if carried_end.is_some() {
4195            // A later hop of a chain without a deadline.
4196            hop_timeout.or(per_try)
4197        } else {
4198            hop_timeout
4199        };
4200        if let Some(timeout) = attempt_timeout {
4201            req = req.timeout(timeout);
4202        }
4203        let span = client_attempt_span(method, url, attempt);
4204        req = span.in_scope(|| inject_trace_context(req, extra_headers));
4205        req = with_caller_headers(req, gate, attempt_timeout, extra_headers, url);
4206        if let Some(body) = body {
4207            req = req.body(body.clone());
4208        }
4209
4210        match send_in_span(req, &span).await {
4211            Ok(resp) => {
4212                let status = resp.status();
4213                let headers = resp.headers().clone();
4214                let url_used = resp.url().clone();
4215
4216                if is_retryable_response(status.as_u16())
4217                    && !last
4218                    && deadline.is_none_or(|d| crate::time::ambient_instant() < d)
4219                {
4220                    let hint = retry_hint(status.as_u16(), &headers);
4221                    let next = retry_policy.retry_delay(
4222                        entropy,
4223                        retries_before.saturating_add(attempt),
4224                        hint,
4225                    );
4226                    // A wait (hinted or not) that reaches the deadline cannot
4227                    // retry in time, so this response is the final outcome.
4228                    let kind = retry_kind(status.as_u16());
4229                    if !reaches_hop_deadline(next) && gate.allow(kind, next) {
4230                        if let Some(ticket) = ticket {
4231                            ticket.record(throttle_accepts(status.as_u16()));
4232                        }
4233                        delay = next;
4234                        last_retry = Some(kind);
4235                        continue;
4236                    }
4237                }
4238                // A redirect the caller follows: its body is not needed, and
4239                // may be large or never end.
4240                let followed = skip_redirect_body
4241                    && matches!(redirect_location(status, &headers, url), Ok(Some(_)));
4242                if followed && let Some(chain) = chain {
4243                    chain.set_attempt_end(attempt_end);
4244                }
4245                let body = if discard_response_body || followed {
4246                    // Dropped unread — see `RequestBuilder::discard_response_body`.
4247                    Ok(Bytes::new())
4248                } else {
4249                    read_body_in_span(resp, &span, gate).await
4250                };
4251                // Count the attempt only now: a body that fails to arrive is
4252                // a transport error, not an accept. A body the caller's
4253                // deadline stopped drops the ticket, which counts as an accept.
4254                if let Some(ticket) = ticket
4255                    && !matches!(body, Err(ClientError::DeadlineExceeded))
4256                {
4257                    ticket.record(body.is_ok() && throttle_accepts(status.as_u16()));
4258                }
4259                let body = body?;
4260                // Refund only after the body arrived.
4261                gate.finish(last_retry, status.as_u16());
4262                log_request(
4263                    method.as_str(),
4264                    &url_used,
4265                    status.as_u16(),
4266                    crate::time::ambient_instant().saturating_duration_since(start),
4267                    extra_headers,
4268                );
4269                return Ok(Response {
4270                    status,
4271                    headers,
4272                    body,
4273                    url: Some(url_used),
4274                });
4275            }
4276            // The request deadline stopped the attempt.
4277            Err(e) if e.is_timeout() && gate.expired() => {
4278                return Err(ClientError::DeadlineExceeded);
4279            }
4280            Err(e) if (e.is_connect() || e.is_timeout()) && !last => {
4281                if let Some(ticket) = ticket {
4282                    ticket.record(false);
4283                }
4284                let wait =
4285                    retry_policy.retry_delay(entropy, retries_before.saturating_add(attempt), None);
4286                // No retry fits in the hop, so none is charged.
4287                if reaches_hop_deadline(wait) || !gate.allow(RetryKind::Transient, wait) {
4288                    return Err(ClientError::Request(e.without_url()));
4289                }
4290                delay = wait;
4291                last_retry = Some(RetryKind::Transient);
4292                last_transient_err = Some(e);
4293            }
4294            Err(e) => {
4295                if let Some(ticket) = ticket {
4296                    ticket.record(false);
4297                }
4298                return Err(ClientError::Request(e.without_url()));
4299            }
4300        }
4301    }
4302
4303    unreachable!("retry loop exited without returning a result — this is a bug")
4304}
4305
4306/// Extract the host portion of a URL as an owned `String`.
4307fn host_of(url: &str) -> Result<String, ClientError> {
4308    let parsed = url::Url::parse(url).map_err(|e| ClientError::InvalidUrl(e.to_string()))?;
4309    parsed
4310        .host_str()
4311        .map(str::to_owned)
4312        .ok_or_else(|| ClientError::InvalidUrl(format!("URL has no host: {url}")))
4313}
4314
4315/// `true` when the URL's host is an IP literal (IPv4 or IPv6) rather than a
4316/// domain name. Uses the `url` crate's parsed [`url::Host`] so bracketed IPv6
4317/// literals and decimal/octal/hex IPv4 encodings are classified correctly.
4318fn url_host_is_ip_literal(url: &str) -> Result<bool, ClientError> {
4319    let parsed = url::Url::parse(url).map_err(|e| ClientError::InvalidUrl(e.to_string()))?;
4320    match parsed.host() {
4321        Some(url::Host::Ipv4(_) | url::Host::Ipv6(_)) => Ok(true),
4322        Some(url::Host::Domain(_)) => Ok(false),
4323        None => Err(ClientError::InvalidUrl(format!("URL has no host: {url}"))),
4324    }
4325}
4326
4327/// `true` when the URL's scheme is `https` (case-insensitive).
4328fn scheme_is_https(url: &str) -> Result<bool, ClientError> {
4329    let parsed = url::Url::parse(url).map_err(|e| ClientError::InvalidUrl(e.to_string()))?;
4330    Ok(parsed.scheme().eq_ignore_ascii_case("https"))
4331}
4332
4333/// If `resp` is a followable redirect (status `301`, `302`, `303`, `307`, or
4334/// `308`) carrying a `Location` header, resolve it to an absolute URL (joining
4335/// relative locations against `base`). Returns `Ok(None)` when the response is
4336/// not a followable redirect: any status outside that set (including non-3xx and
4337/// the non-followable 3xx `300`/`304`/`305`/`306`), or a followable status
4338/// missing its `Location` header.
4339fn redirect_target(resp: &Response, base: &str) -> Result<Option<String>, ClientError> {
4340    redirect_location(resp.status(), resp.headers(), base)
4341}
4342
4343/// [`redirect_target`] from a response's status and headers, before its body
4344/// is read.
4345fn redirect_location(
4346    status: reqwest::StatusCode,
4347    headers: &HeaderMap,
4348    base: &str,
4349) -> Result<Option<String>, ClientError> {
4350    // Only the statuses reqwest itself follows are treated as redirects. A
4351    // response like `304 Not Modified` (or 300/305/306) can legitimately carry
4352    // a `Location` header without being a followable redirect, so matching the
4353    // entire 300–399 range via `is_redirection()` would wrongly issue an extra
4354    // request instead of returning the response to the caller.
4355    match status {
4356        reqwest::StatusCode::MOVED_PERMANENTLY
4357        | reqwest::StatusCode::FOUND
4358        | reqwest::StatusCode::SEE_OTHER
4359        | reqwest::StatusCode::TEMPORARY_REDIRECT
4360        | reqwest::StatusCode::PERMANENT_REDIRECT => {}
4361        _ => return Ok(None),
4362    }
4363    let Some(location) = headers
4364        .get(reqwest::header::LOCATION)
4365        .and_then(|v| v.to_str().ok())
4366    else {
4367        return Ok(None);
4368    };
4369    let base_url = url::Url::parse(base).map_err(|e| ClientError::InvalidUrl(e.to_string()))?;
4370    let joined = base_url
4371        .join(location)
4372        .map_err(|e| ClientError::InvalidUrl(e.to_string()))?;
4373    Ok(Some(joined.to_string()))
4374}
4375
4376/// Strip credential-bearing headers when a redirect hop crosses origins.
4377///
4378/// If `current`'s origin (scheme + host + port, per [`url::Url::origin`])
4379/// differs from the `original` request URL's origin, the `Authorization`,
4380/// `Cookie`, `Cookie2`, `Proxy-Authorization` and `WWW-Authenticate` headers
4381/// are removed from `headers` so they are never forwarded to a cross-origin
4382/// target (credential leak).
4383/// Because the caller threads a single mutable `headers` map across hops, once
4384/// these headers are stripped on any hop they stay stripped for the remainder
4385/// of the chain — the safe, conservative behaviour.
4386fn strip_sensitive_headers_if_cross_origin(
4387    headers: &mut HeaderMap,
4388    original: &url::Url,
4389    current: &str,
4390) -> Result<(), ClientError> {
4391    let current_url =
4392        url::Url::parse(current).map_err(|e| ClientError::InvalidUrl(e.to_string()))?;
4393    if current_url.origin() != original.origin() {
4394        // The set reqwest's own redirect handling removes.
4395        headers.remove(reqwest::header::AUTHORIZATION);
4396        headers.remove(reqwest::header::COOKIE);
4397        headers.remove("cookie2");
4398        headers.remove(reqwest::header::PROXY_AUTHORIZATION);
4399        headers.remove(reqwest::header::WWW_AUTHENTICATE);
4400        // RFC 9421 signatures (Web Bot Auth) bind to the first authority.
4401        headers.remove("signature");
4402        headers.remove("signature-input");
4403        headers.remove("signature-agent");
4404    }
4405    Ok(())
4406}
4407
4408/// Apply RFC 7231 §6.4 / RFC 7538 method-and-body rewriting after receiving a
4409/// redirect `status`, before issuing the next hop. Mutates `method` and `body`
4410/// in place.
4411///
4412/// - **303 See Other**: switch to `GET` (a `HEAD` stays `HEAD`) and drop the body.
4413/// - **301 Moved Permanently / 302 Found**: a `POST` becomes a bodyless `GET`;
4414///   every other method (and its body) is preserved — matching prevailing
4415///   browser behaviour.
4416/// - **307 Temporary Redirect / 308 Permanent Redirect**: preserve the method
4417///   **and** the body verbatim (the RFC-correct behaviour — the body must NOT
4418///   be dropped).
4419/// - Any other redirect status: leave method and body untouched.
4420///
4421/// Whenever the body is dropped (the POST→GET / 303→GET rewrites), the payload
4422/// (entity) headers threaded across hops are also removed from `headers` so the
4423/// bodyless follow-up hop does not carry a misleading `Content-Type` /
4424/// `Content-Length` / `Transfer-Encoding` / `Content-Encoding` /
4425/// `Content-Language` — matching reqwest's redirect layer. On 307/308 the body
4426/// is preserved, so those headers are left intact.
4427fn rewrite_after_redirect(
4428    status: reqwest::StatusCode,
4429    method: &mut Method,
4430    body: &mut Option<Bytes>,
4431    headers: &mut HeaderMap,
4432) {
4433    match status.as_u16() {
4434        303 => {
4435            if *method != Method::HEAD {
4436                *method = Method::GET;
4437            }
4438            *body = None;
4439            strip_payload_headers(headers);
4440        }
4441        301 | 302 if *method == Method::POST => {
4442            *method = Method::GET;
4443            *body = None;
4444            strip_payload_headers(headers);
4445        }
4446        _ => {}
4447    }
4448}
4449
4450/// Remove payload (entity) headers from the threaded per-hop header map. Called
4451/// when a redirect rewrite drops the request body so a bodyless GET does not
4452/// keep carrying the original payload's `Content-Type` etc.
4453fn strip_payload_headers(headers: &mut HeaderMap) {
4454    use reqwest::header::{
4455        CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_LENGTH, CONTENT_TYPE, TRANSFER_ENCODING,
4456    };
4457    headers.remove(CONTENT_TYPE);
4458    headers.remove(CONTENT_LENGTH);
4459    headers.remove(TRANSFER_ENCODING);
4460    headers.remove(CONTENT_ENCODING);
4461    headers.remove(CONTENT_LANGUAGE);
4462}
4463
4464/// Validate a set of resolved socket addresses against the built-in SSRF
4465/// deny-list, failing closed.
4466///
4467/// Returns `Err(SsrfBlocked)` (naming the first blocked address) if **any**
4468/// address's IP is blocked ([`is_blocked_ip`]); otherwise returns the whole set
4469/// unchanged, preserving order. Factored out of [`resolve_and_validate`] as a
4470/// pure, synchronous helper so the validation policy is unit-testable without
4471/// real multi-record DNS.
4472fn validate_resolved_addrs(addrs: Vec<SocketAddr>) -> Result<Vec<SocketAddr>, ClientError> {
4473    for addr in &addrs {
4474        if is_blocked_ip(addr.ip()) {
4475            return Err(ClientError::SsrfBlocked(addr.ip().to_string()));
4476        }
4477    }
4478    Ok(addrs)
4479}
4480
4481/// Resolve `url`'s host to **all** validated [`SocketAddr`]s, rejecting with
4482/// [`ClientError::SsrfBlocked`] if the host is (or resolves to) any blocked IP.
4483///
4484/// IP-literal hosts (including decimal/octal/hex encodings, which the `url`
4485/// crate normalises to an `Ipv4Addr` at parse time) are validated directly with
4486/// no DNS lookup. Domain hosts are resolved **once** via `tokio::net::lookup_host`
4487/// (keeping the TOCTOU window closed) and rejected if **any** resolved address
4488/// is blocked. Since the whole DNS response is rejected when any IP is blocked,
4489/// it is safe to return the full validated set (order preserved) so a caller can
4490/// pin all of them and let reqwest try them in order — an unreachable first
4491/// address no longer dooms the request.
4492async fn resolve_and_validate(url: &str) -> Result<Vec<SocketAddr>, ClientError> {
4493    let parsed = url::Url::parse(url).map_err(|e| ClientError::InvalidUrl(e.to_string()))?;
4494    // Explicit scheme allowlist (defence-in-depth): only http/https may be
4495    // resolved and connected on the safe path. Reject ftp://, gopher://,
4496    // file://, etc. here — before any DNS lookup or connection — rather than
4497    // relying on reqwest to reject them after the fact.
4498    let scheme = parsed.scheme();
4499    if !scheme.eq_ignore_ascii_case("http") && !scheme.eq_ignore_ascii_case("https") {
4500        return Err(ClientError::InvalidUrl(format!(
4501            "unsupported URL scheme `{scheme}` (only http/https are allowed): {url}"
4502        )));
4503    }
4504    let port = parsed.port_or_known_default().ok_or_else(|| {
4505        ClientError::InvalidUrl(format!("URL has no port and unknown scheme: {url}"))
4506    })?;
4507    let host = parsed
4508        .host()
4509        .ok_or_else(|| ClientError::InvalidUrl(format!("URL has no host: {url}")))?;
4510
4511    match host {
4512        url::Host::Ipv4(v4) => validate_resolved_addrs(vec![SocketAddr::new(IpAddr::V4(v4), port)]),
4513        url::Host::Ipv6(v6) => validate_resolved_addrs(vec![SocketAddr::new(IpAddr::V6(v6), port)]),
4514        url::Host::Domain(name) => {
4515            let addrs: Vec<SocketAddr> = tokio::net::lookup_host((name, port))
4516                .await
4517                .map_err(|e| ClientError::InvalidUrl(format!("DNS lookup failed for {name}: {e}")))?
4518                .collect();
4519            if addrs.is_empty() {
4520                return Err(ClientError::InvalidUrl(format!(
4521                    "DNS lookup for {name} returned no addresses"
4522                )));
4523            }
4524            // Reject if ANY resolved address is blocked (fail closed); otherwise
4525            // return the whole validated set (order preserved from the lookup).
4526            validate_resolved_addrs(addrs)
4527        }
4528    }
4529}
4530
4531fn parse_retry_after(headers: &HeaderMap) -> Option<Duration> {
4532    let value = headers.get("retry-after")?.to_str().ok()?;
4533    // Integer seconds (most common form).
4534    if let Ok(secs) = value.parse::<u64>() {
4535        return Some(Duration::from_secs(secs));
4536    }
4537    // HTTP-date format per RFC 9110 (e.g. "Tue, 01 Jan 2030 00:00:00 GMT").
4538    let dt = chrono::DateTime::parse_from_rfc2822(value).ok()?;
4539    let now = crate::time::ambient_now();
4540    let future = dt.with_timezone(&chrono::Utc);
4541    let secs = u64::try_from((future - now).num_seconds().max(0)).unwrap_or(0);
4542    Some(Duration::from_secs(secs))
4543}
4544
4545const REDACTED_HEADERS: &[&str] = &["authorization", "cookie", "set-cookie"];
4546
4547fn is_sensitive_header(name: &str) -> bool {
4548    REDACTED_HEADERS
4549        .iter()
4550        .any(|h| h.eq_ignore_ascii_case(name))
4551}
4552
4553fn log_request(
4554    method: &str,
4555    url: &reqwest::Url,
4556    status: u16,
4557    elapsed: Duration,
4558    headers: &HeaderMap,
4559) {
4560    let host = url.host_str().unwrap_or("unknown");
4561    let path = url.path();
4562
4563    // Collect non-sensitive header names for the span (values are omitted).
4564    let sent_headers: Vec<&str> = headers
4565        .keys()
4566        .map(HeaderName::as_str)
4567        .filter(|k| !is_sensitive_header(k))
4568        .collect();
4569
4570    tracing::info!(
4571        http.method = method,
4572        http.host = host,
4573        http.path = path,
4574        http.status = status,
4575        http.elapsed_ms = u64::try_from(elapsed.as_millis()).unwrap_or(u64::MAX),
4576        http.sent_headers = ?sent_headers,
4577        "outbound request"
4578    );
4579}
4580
4581/// Open the CLIENT span for one outbound attempt (issue #3064).
4582///
4583/// Field names follow the OpenTelemetry HTTP client conventions. The span
4584/// holds the URL path only, not the query: a query can hold secrets.
4585fn client_attempt_span(method: &Method, url: &str, attempt: u32) -> tracing::Span {
4586    let span = tracing::info_span!(
4587        "http.client.request",
4588        otel.name = %method,
4589        otel.kind = "client",
4590        http.request.method = %method,
4591        server.address = tracing::field::Empty,
4592        url.path = tracing::field::Empty,
4593        http.request.resend_count = tracing::field::Empty,
4594        http.response.status_code = tracing::field::Empty,
4595        error.type = tracing::field::Empty,
4596        otel.status_code = tracing::field::Empty,
4597    );
4598    // The conventions leave `resend_count` unset on the first attempt.
4599    if attempt > 0 {
4600        span.record("http.request.resend_count", attempt);
4601    }
4602    // Parse the URL only when a subscriber listens.
4603    if !span.is_disabled()
4604        && let Ok(parsed) = reqwest::Url::parse(url)
4605    {
4606        span.record("server.address", parsed.host_str().unwrap_or_default());
4607        span.record("url.path", parsed.path());
4608    }
4609    span
4610}
4611
4612/// Record the response status on an attempt span. A 4xx or 5xx status sets
4613/// the span status to `ERROR`, as the HTTP client conventions require.
4614fn record_attempt_status(span: &tracing::Span, status: u16) {
4615    span.record("http.response.status_code", status);
4616    if status >= 400 {
4617        span.record("otel.status_code", "ERROR");
4618    }
4619}
4620
4621/// Record a failed attempt: the error class and the `ERROR` span status.
4622fn record_attempt_error(span: &tracing::Span, kind: &'static str) {
4623    span.record("error.type", kind);
4624    span.record("otel.status_code", "ERROR");
4625}
4626
4627/// Read the response body inside the attempt `span`, so the span covers the
4628/// body transfer. A read failure sets `error.type = "body"`, and `gate`
4629/// classifies it (see [`RetryGate::body_error`]).
4630async fn read_body_in_span(
4631    response: reqwest::Response,
4632    span: &tracing::Span,
4633    gate: &RetryGate,
4634) -> Result<Bytes, ClientError> {
4635    use tracing::Instrument as _;
4636
4637    let body = response.bytes().instrument(span.clone()).await;
4638    if body.is_err() {
4639        record_attempt_error(span, "body");
4640    }
4641    body.map_err(|e| gate.body_error(e))
4642}
4643
4644/// Send `request` inside `span`, then record the status code or the error
4645/// class on the span.
4646///
4647/// The caller keeps `span` open through the body read, and ends it before a
4648/// `Retry-After` sleep: the sleep is not part of the attempt.
4649async fn send_in_span(
4650    request: reqwest::RequestBuilder,
4651    span: &tracing::Span,
4652) -> Result<reqwest::Response, reqwest::Error> {
4653    use tracing::Instrument as _;
4654
4655    let sent = request.send().instrument(span.clone()).await;
4656    record_send_outcome(span, &sent);
4657    sent
4658}
4659
4660/// Record the status code or the error class of one send on its attempt
4661/// `span`.
4662fn record_send_outcome(span: &tracing::Span, sent: &Result<reqwest::Response, reqwest::Error>) {
4663    match sent {
4664        Ok(response) => record_attempt_status(span, response.status().as_u16()),
4665        Err(error) => {
4666            let kind = if error.is_timeout() {
4667                "timeout"
4668            } else if error.is_connect() {
4669                "connect"
4670            } else {
4671                "request"
4672            };
4673            record_attempt_error(span, kind);
4674        }
4675    }
4676}
4677
4678/// Inject the W3C trace-context headers of the active span, except the ones
4679/// that [`trace_headers_to_send`] drops for `caller_headers`.
4680fn inject_trace_context(
4681    builder: reqwest::RequestBuilder,
4682    caller_headers: &HeaderMap,
4683) -> reqwest::RequestBuilder {
4684    let mut builder = builder;
4685    for (name, value) in trace_headers_to_send(trace_context_headers(), caller_headers) {
4686        builder = builder.header(name, value);
4687    }
4688    builder
4689}
4690
4691/// The injected trace headers to send. Skip a name that `caller_headers`
4692/// holds: `RequestBuilder::header` appends, so if not, the request sends both
4693/// values.
4694///
4695/// `traceparent` and `tracestate` are one W3C context. When the caller sets
4696/// either one, skip both, so the request never mixes two contexts.
4697fn trace_headers_to_send(
4698    injected: Vec<(String, HeaderValue)>,
4699    caller_headers: &HeaderMap,
4700) -> Vec<(String, HeaderValue)> {
4701    const W3C_PAIR: [&str; 2] = ["traceparent", "tracestate"];
4702    let caller_has_context = W3C_PAIR
4703        .iter()
4704        .any(|name| caller_headers.contains_key(*name));
4705    injected
4706        .into_iter()
4707        .filter(|(name, _)| {
4708            let paired = caller_has_context && W3C_PAIR.contains(&name.as_str());
4709            !paired && !caller_headers.contains_key(name.as_str())
4710        })
4711        .collect()
4712}
4713
4714/// Add the current request's id as `x-request-id` (issue #3064). Do nothing
4715/// when the caller set the header, or when there is no current request.
4716fn add_current_request_id(headers: &mut HeaderMap) {
4717    static X_REQUEST_ID: HeaderName = HeaderName::from_static("x-request-id");
4718    if headers.contains_key(&X_REQUEST_ID) {
4719        return;
4720    }
4721    if let Some(value) =
4722        crate::log::context::current_request_id().and_then(|id| HeaderValue::from_str(&id).ok())
4723    {
4724        headers.insert(X_REQUEST_ID.clone(), value);
4725    }
4726}
4727
4728/// The W3C trace-context headers for the active span. Empty when the
4729/// `telemetry-otlp` feature is disabled or no span has a valid context.
4730#[allow(clippy::missing_const_for_fn)]
4731fn trace_context_headers() -> Vec<(String, HeaderValue)> {
4732    #[cfg(not(feature = "telemetry-otlp"))]
4733    {
4734        Vec::new()
4735    }
4736    #[cfg(feature = "telemetry-otlp")]
4737    {
4738        use std::collections::HashMap;
4739        use tracing_opentelemetry::OpenTelemetrySpanExt as _;
4740        let cx = tracing::Span::current().context();
4741        let mut map = HashMap::<String, String>::new();
4742        opentelemetry::global::get_text_map_propagator(|propagator| {
4743            propagator.inject_context(&cx, &mut TraceHeaderInjector(&mut map));
4744        });
4745        map.into_iter()
4746            .filter_map(|(name, value)| Some((name, HeaderValue::from_str(&value).ok()?)))
4747            .collect()
4748    }
4749}
4750
4751#[cfg(feature = "telemetry-otlp")]
4752struct TraceHeaderInjector<'a>(&'a mut std::collections::HashMap<String, String>);
4753
4754#[cfg(feature = "telemetry-otlp")]
4755impl opentelemetry::propagation::Injector for TraceHeaderInjector<'_> {
4756    fn set(&mut self, key: &str, value: String) {
4757        self.0.insert(key.to_owned(), value);
4758    }
4759}
4760
4761// ── Tests ────────────────────────────────────────────────────────────────────
4762
4763#[cfg(test)]
4764mod tests {
4765    use super::*;
4766
4767    /// Issue #3068: a capsule replays a local throttle reject as the same
4768    /// variant, so a handler that matches it takes the same branch.
4769    #[cfg(feature = "reporting")]
4770    #[test]
4771    fn throttled_locally_round_trips_through_a_capsule() {
4772        let err = ClientError::ThrottledLocally {
4773            host: "api.example.com:8443".to_owned(),
4774        };
4775        let kind = http_error_kind(&err);
4776        assert_eq!(
4777            kind,
4778            crate::capsule::schema::HttpErrorKind::ThrottledLocally
4779        );
4780        match rebuild_client_error(Some(kind), err.to_string()) {
4781            ClientError::ThrottledLocally { host } => assert_eq!(host, "api.example.com:8443"),
4782            other => panic!("rebuilt as {other:?}"),
4783        }
4784    }
4785
4786    /// Codex review on #3222: two different opaque header values record
4787    /// different text, and a text value cannot look like bytes.
4788    #[cfg(feature = "reporting")]
4789    #[test]
4790    fn opaque_header_values_record_without_collision() {
4791        let recorded = |value: HeaderValue| {
4792            let mut builder = Client::new().get("http://example.test/");
4793            builder.extra_headers.insert("x-sig", value);
4794            caller_headers(&builder).remove(0).1
4795        };
4796        let first = recorded(HeaderValue::from_bytes(b"a\xfe").expect("value"));
4797        let second = recorded(HeaderValue::from_bytes(b"a\xff").expect("value"));
4798        assert_ne!(first, second);
4799        assert_eq!(recorded(HeaderValue::from_static("plain")), "plain");
4800        let look_alike = recorded(HeaderValue::from_static("[bytes] a\\xfe"));
4801        assert_ne!(look_alike, first);
4802    }
4803
4804    /// Issue #3071: a capsule replays an injected fault as the same variant.
4805    #[cfg(feature = "reporting")]
4806    #[test]
4807    fn fault_injected_round_trips_through_a_capsule() {
4808        let err = ClientError::FaultInjected("fault injection: injected http error".to_owned());
4809        let kind = http_error_kind(&err);
4810        assert_eq!(kind, crate::capsule::schema::HttpErrorKind::FaultInjected);
4811        match rebuild_client_error(Some(kind), err.to_string()) {
4812            ClientError::FaultInjected(text) => {
4813                assert_eq!(text, "fault injection: injected http error");
4814            }
4815            other => panic!("rebuilt as {other:?}"),
4816        }
4817    }
4818
4819    /// Regression (#3068 review): a local throttle reject says nothing about
4820    /// the host, so it must not count as a circuit-breaker failure.
4821    #[tokio::test]
4822    #[allow(clippy::await_holding_lock)]
4823    async fn throttled_calls_do_not_open_the_circuit_breaker() {
4824        let _lock = crate::circuit_breaker::TEST_LOCK
4825            .lock()
4826            .unwrap_or_else(std::sync::PoisonError::into_inner);
4827        crate::circuit_breaker::global_registry().clear();
4828
4829        // 429 is a reject for the throttle, but a success for the breaker.
4830        let app = axum::Router::new().route(
4831            "/x",
4832            axum::routing::get(|| async { axum::http::StatusCode::TOO_MANY_REQUESTS }),
4833        );
4834        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
4835        let addr = listener.local_addr().unwrap();
4836        tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
4837
4838        let mut config = HttpClientConfig::default();
4839        config.adaptive_throttle.enabled = true;
4840        let client = Client::from_config(&config);
4841        let url = format!("http://{addr}/x");
4842        let mut throttled = 0;
4843        for _ in 0..200 {
4844            match client.get(&url).no_retry().send().await {
4845                Err(ClientError::ThrottledLocally { .. }) => throttled += 1,
4846                Ok(r) => assert_eq!(r.status().as_u16(), 429),
4847                Err(e) => panic!("unexpected error: {e}"),
4848            }
4849        }
4850        assert!(
4851            throttled > 50,
4852            "the throttle must reject locally: {throttled}"
4853        );
4854        let breaker = crate::circuit_breaker::global_registry().get_or_create(
4855            &addr.to_string(),
4856            crate::circuit_breaker::CircuitBreakerPolicy::default(),
4857        );
4858        assert_eq!(
4859            breaker.state(),
4860            crate::circuit_breaker::CircuitState::Closed,
4861            "local rejects must not open the breaker"
4862        );
4863        crate::circuit_breaker::global_registry().clear();
4864    }
4865
4866    /// Regression (#3183 review): a `state_initializer` that replaces the
4867    /// config after boot gets one shared throttle for the new settings.
4868    #[test]
4869    fn from_state_shares_a_throttle_built_from_the_effective_config() {
4870        let state = crate::AppState::for_test();
4871        let boot = HttpClientConfig::default();
4872        install_shared_throttle(&state, &boot);
4873        assert!(state.extension::<SharedThrottle>().is_none(), "off at boot");
4874        assert!(Client::from_state(&state).throttle.is_none());
4875
4876        let mut replaced = crate::config::HttpConfig::default();
4877        replaced.client.adaptive_throttle.enabled = true;
4878        state.insert_extension(replaced.clone());
4879        let a = Client::from_state(&state)
4880            .throttle
4881            .expect("on after replace");
4882        let b = Client::from_state(&state)
4883            .throttle
4884            .expect("on after replace");
4885        assert!(Arc::ptr_eq(&a, &b), "clients share one throttle");
4886
4887        replaced.client.adaptive_throttle.k = 3.0;
4888        state.insert_extension(replaced.clone());
4889        let c = Client::from_state(&state).throttle.expect("still on");
4890        assert!(!Arc::ptr_eq(&a, &c), "new settings build a new throttle");
4891
4892        replaced.client.adaptive_throttle.enabled = false;
4893        state.insert_extension(replaced);
4894        assert!(Client::from_state(&state).throttle.is_none(), "off again");
4895    }
4896
4897    /// A `state_initializer` that replaces the config after boot gets retry
4898    /// budgets built from the new settings, shared by all clients.
4899    #[test]
4900    fn from_state_rebuilds_the_retry_budgets_when_their_config_changes() {
4901        let state = crate::AppState::for_test();
4902        let mut config = crate::config::HttpConfig::default();
4903        state.insert_extension(config.clone());
4904        let a = Client::from_state(&state).retry.budgets.expect("on");
4905        let b = Client::from_state(&state).retry.budgets.expect("on");
4906        assert!(Arc::ptr_eq(&a, &b), "clients share one set of budgets");
4907
4908        config.client.retry_budget.capacity = 50;
4909        state.insert_extension(config.clone());
4910        let c = Client::from_state(&state).retry.budgets.expect("still on");
4911        assert!(!Arc::ptr_eq(&a, &c), "new settings build new budgets");
4912        assert!((c.for_host("h:80").available() - 50.0).abs() < f64::EPSILON);
4913
4914        config.client.retry_budget.enabled = false;
4915        state.insert_extension(config);
4916        assert!(Client::from_state(&state).retry.budgets.is_none(), "off");
4917    }
4918
4919    /// Regression (#3183 review): concurrent first calls after a config
4920    /// change share one throttle.
4921    #[test]
4922    fn concurrent_first_calls_share_one_throttle() {
4923        let state = crate::AppState::for_test();
4924        let mut config = crate::config::HttpConfig::default();
4925        config.client.adaptive_throttle.enabled = true;
4926        state.insert_extension(config);
4927        let throttles: Vec<_> = std::thread::scope(|scope| {
4928            // Spawn all threads before the first join, so the calls overlap.
4929            let mut handles = Vec::with_capacity(8);
4930            for _ in 0..8 {
4931                handles.push(scope.spawn(|| Client::from_state(&state).throttle.expect("on")));
4932            }
4933            handles
4934                .into_iter()
4935                .map(|h| h.join().expect("thread"))
4936                .collect()
4937        });
4938        assert!(
4939            throttles.iter().all(|t| Arc::ptr_eq(t, &throttles[0])),
4940            "every client must share the one throttle"
4941        );
4942    }
4943
4944    /// Regression (#3183 review): a response whose body fails to arrive is
4945    /// a transport error for the throttle, not an accept.
4946    #[tokio::test]
4947    #[allow(clippy::await_holding_lock)]
4948    async fn a_truncated_body_is_not_an_accept() {
4949        use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _};
4950        let _lock = crate::circuit_breaker::TEST_LOCK
4951            .lock()
4952            .unwrap_or_else(std::sync::PoisonError::into_inner);
4953        crate::circuit_breaker::global_registry().clear();
4954
4955        // A server that sends a 200 head with a 100-byte length, 3 bytes of
4956        // body, then closes.
4957        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
4958        let addr = listener.local_addr().unwrap();
4959        tokio::spawn(async move {
4960            loop {
4961                let Ok((mut socket, _)) = listener.accept().await else {
4962                    return;
4963                };
4964                let mut buf = [0_u8; 1024];
4965                let _ = socket.read(&mut buf).await;
4966                let _ = socket
4967                    .write_all(b"HTTP/1.1 200 OK\r\ncontent-length: 100\r\n\r\nabc")
4968                    .await;
4969                drop(socket);
4970            }
4971        });
4972
4973        let mut config = HttpClientConfig::default();
4974        config.adaptive_throttle.enabled = true;
4975        let client = Client::from_config(&config);
4976        let url = format!("http://{addr}/x");
4977        for _ in 0..20 {
4978            let res = client.get(&url).no_retry().send().await;
4979            assert!(
4980                matches!(
4981                    res,
4982                    Err(ClientError::Request(_) | ClientError::ThrottledLocally { .. })
4983                ),
4984                "a truncated body is an error: {res:?}"
4985            );
4986        }
4987        let throttle = client.throttle.as_ref().expect("on");
4988        assert!(
4989            throttle.reject_probability(&addr.to_string(), crate::time::ambient_instant()) > 0.5,
4990            "truncated bodies must count as rejects"
4991        );
4992        crate::circuit_breaker::global_registry().clear();
4993    }
4994
4995    #[test]
4996    fn throttle_host_keeps_an_explicit_port() {
4997        assert_eq!(
4998            throttle_host("https://a.example/x").as_deref(),
4999            Some("a.example")
5000        );
5001        assert_eq!(
5002            throttle_host("http://a.example:8080/x").as_deref(),
5003            Some("a.example:8080")
5004        );
5005        assert_eq!(throttle_host("not a url"), None);
5006        assert!(throttle_accepts(200) && throttle_accepts(500));
5007        assert!(!throttle_accepts(429) && !throttle_accepts(503));
5008    }
5009    use crate::config::HttpClientConfig;
5010
5011    // RED-PHASE TEST 1: Client can be constructed with defaults.
5012    #[test]
5013    fn client_constructs_with_defaults() {
5014        let client = Client::new();
5015        assert!(client.alias.is_none());
5016        assert!(client.base_url.is_none());
5017        assert_eq!(client.retry_policy.max_retries, 3);
5018    }
5019
5020    // RED-PHASE TEST 2: Fluent RequestBuilder API compiles.
5021    #[test]
5022    fn request_builder_fluent_api_compiles() {
5023        let client = Client::new();
5024        let _builder = client
5025            .post("https://example.com/api")
5026            .header("x-api-key", "secret")
5027            .json(&serde_json::json!({"key": "value"}))
5028            .retries(2);
5029    }
5030
5031    // RED-PHASE TEST 3: Response accessors work.
5032    #[test]
5033    fn response_accessors_work() {
5034        let payload = serde_json::json!({"id": 42, "name": "Alice"});
5035        let body = serde_json::to_vec(&payload).unwrap();
5036        let resp = Response {
5037            status: reqwest::StatusCode::OK,
5038            headers: HeaderMap::new(),
5039            body: Bytes::from(body),
5040            url: None,
5041        };
5042        assert_eq!(resp.status().as_u16(), 200);
5043        assert!(resp.is_success());
5044    }
5045
5046    // RED-PHASE TEST 4: Response::json() deserialises correctly.
5047    #[test]
5048    fn response_json_deserialises() {
5049        #[derive(serde::Deserialize, PartialEq, Debug)]
5050        struct User {
5051            id: i32,
5052            name: String,
5053        }
5054        let payload = serde_json::json!({"id": 1, "name": "Bob"});
5055        let resp = Response {
5056            status: reqwest::StatusCode::OK,
5057            headers: HeaderMap::new(),
5058            body: Bytes::from(serde_json::to_vec(&payload).unwrap()),
5059            url: None,
5060        };
5061        let user: User = resp.json().unwrap();
5062        assert_eq!(user.id, 1);
5063        assert_eq!(user.name, "Bob");
5064    }
5065
5066    // RED-PHASE TEST 5: Response::text() returns UTF-8 string.
5067    #[test]
5068    fn response_text_returns_string() {
5069        let resp = Response {
5070            status: reqwest::StatusCode::OK,
5071            headers: HeaderMap::new(),
5072            body: Bytes::from_static(b"hello world"),
5073            url: None,
5074        };
5075        assert_eq!(resp.text(), "hello world");
5076    }
5077
5078    // RED-PHASE TEST 6: Response::bytes() returns raw bytes.
5079    #[test]
5080    fn response_bytes_returns_raw() {
5081        let resp = Response {
5082            status: reqwest::StatusCode::CREATED,
5083            headers: HeaderMap::new(),
5084            body: Bytes::from_static(b"\x00\x01\x02"),
5085            url: None,
5086        };
5087        assert_eq!(resp.bytes(), Bytes::from_static(b"\x00\x01\x02"));
5088    }
5089
5090    // RED-PHASE TEST 7: HttpClientConfig deserialises from [http.client] TOML.
5091    #[test]
5092    fn config_deserialises_from_toml() {
5093        // Simulate the [http.client] section as it appears in autumn.toml.
5094        let toml = r#"
5095            [client]
5096            timeout_secs = 60
5097            max_retries = 5
5098            [client.base_urls]
5099            stripe = "https://api.stripe.com"
5100            sendgrid = "https://api.sendgrid.com"
5101        "#;
5102        let http_cfg: crate::config::HttpConfig = toml::from_str(toml).unwrap();
5103        let config = &http_cfg.client;
5104        assert_eq!(config.timeout_secs, 60);
5105        assert_eq!(config.max_retries, 5);
5106        assert_eq!(
5107            config.base_urls.get("stripe").map(String::as_str),
5108            Some("https://api.stripe.com")
5109        );
5110        assert_eq!(
5111            config.base_urls.get("sendgrid").map(String::as_str),
5112            Some("https://api.sendgrid.com")
5113        );
5114    }
5115
5116    // RED-PHASE TEST 8: HttpClientConfig has correct defaults.
5117    #[test]
5118    fn config_has_correct_defaults() {
5119        let config = HttpClientConfig::default();
5120        assert_eq!(config.timeout_secs, 30);
5121        assert_eq!(config.max_retries, 3);
5122        assert!(config.base_urls.is_empty());
5123    }
5124
5125    // RED-PHASE TEST 9: is_idempotent_method returns correct values.
5126    #[test]
5127    fn idempotent_method_classification() {
5128        assert!(is_idempotent_method(&Method::GET));
5129        assert!(is_idempotent_method(&Method::HEAD));
5130        assert!(is_idempotent_method(&Method::PUT));
5131        assert!(is_idempotent_method(&Method::DELETE));
5132        assert!(is_idempotent_method(&Method::OPTIONS));
5133        assert!(is_idempotent_method(&Method::TRACE));
5134        assert!(!is_idempotent_method(&Method::POST));
5135        assert!(!is_idempotent_method(&Method::PATCH));
5136    }
5137
5138    // RED-PHASE TEST 10: is_retryable_status returns correct values.
5139    #[test]
5140    fn retryable_status_classification() {
5141        assert!(is_retryable_status(502));
5142        assert!(is_retryable_status(503));
5143        assert!(is_retryable_status(504));
5144        assert!(!is_retryable_status(200));
5145        assert!(!is_retryable_status(400));
5146        assert!(!is_retryable_status(404));
5147        assert!(!is_retryable_status(500));
5148        assert!(!is_retryable_status(429));
5149    }
5150
5151    // RED-PHASE TEST 11: parse_retry_after parses seconds correctly.
5152    #[test]
5153    fn retry_after_header_parsing() {
5154        let mut headers = HeaderMap::new();
5155        headers.insert(
5156            reqwest::header::HeaderName::from_static("retry-after"),
5157            HeaderValue::from_static("5"),
5158        );
5159        assert_eq!(parse_retry_after(&headers), Some(Duration::from_secs(5)));
5160
5161        let empty = HeaderMap::new();
5162        assert_eq!(parse_retry_after(&empty), None);
5163    }
5164
5165    // RED-PHASE TEST 12: Sensitive header detection.
5166    #[test]
5167    fn sensitive_header_detection() {
5168        assert!(is_sensitive_header("authorization"));
5169        assert!(is_sensitive_header("Authorization"));
5170        assert!(is_sensitive_header("AUTHORIZATION"));
5171        assert!(is_sensitive_header("cookie"));
5172        assert!(is_sensitive_header("set-cookie"));
5173        assert!(!is_sensitive_header("content-type"));
5174        assert!(!is_sensitive_header("x-api-key"));
5175    }
5176
5177    // RED-PHASE TEST 13: MockRegistry captures and matches calls.
5178    #[tokio::test]
5179    async fn mock_registry_captures_calls() {
5180        let registry = Arc::new(MockRegistry::new());
5181        let call_count = Arc::new(AtomicUsize::new(0));
5182
5183        registry.register(MockEntry {
5184            method: Some(Method::POST),
5185            path: "/charges".to_owned(),
5186            alias: Some("stripe".to_owned()),
5187            status: 200,
5188            body: Some(serde_json::json!({"id": "ch_123"})),
5189            call_count: call_count.clone(),
5190        });
5191
5192        let client = Client::new().with_mock(registry).named("stripe");
5193
5194        let resp = client
5195            .post("https://api.stripe.com/charges")
5196            .json(&serde_json::json!({"amount": 1000}))
5197            .send()
5198            .await
5199            .unwrap();
5200
5201        assert_eq!(resp.status().as_u16(), 200);
5202        let body: serde_json::Value = resp.json().unwrap();
5203        assert_eq!(body["id"], "ch_123");
5204        assert_eq!(call_count.load(Ordering::SeqCst), 1);
5205    }
5206
5207    // RED-PHASE TEST 14: MockHandle::expect_called passes when count matches.
5208    #[tokio::test]
5209    async fn mock_handle_expect_called_passes() {
5210        let registry = Arc::new(MockRegistry::new());
5211        let call_count = Arc::new(AtomicUsize::new(0));
5212
5213        registry.register(MockEntry {
5214            method: Some(Method::GET),
5215            path: "/users/1".to_owned(),
5216            alias: None,
5217            status: 200,
5218            body: Some(serde_json::json!({"name": "Alice"})),
5219            call_count: call_count.clone(),
5220        });
5221
5222        let handle = MockHandle {
5223            alias: "test".to_owned(),
5224            method: "GET".to_owned(),
5225            path: "/users/1".to_owned(),
5226            call_count: call_count.clone(),
5227        };
5228
5229        let client = Client::new().with_mock(registry);
5230        client
5231            .get("https://api.example.com/users/1")
5232            .send()
5233            .await
5234            .unwrap();
5235
5236        handle.expect_called(1);
5237        assert_eq!(handle.call_count(), 1);
5238    }
5239
5240    // RED-PHASE TEST 15: MockRegistry matches by URL path suffix.
5241    #[tokio::test]
5242    async fn mock_matches_by_path_suffix() {
5243        let registry = Arc::new(MockRegistry::new());
5244        let call_count = Arc::new(AtomicUsize::new(0));
5245
5246        registry.register(MockEntry {
5247            method: Some(Method::POST),
5248            path: "/v1/charges".to_owned(),
5249            alias: None,
5250            status: 201,
5251            body: Some(serde_json::json!({"created": true})),
5252            call_count: call_count.clone(),
5253        });
5254
5255        let client = Client::new().with_mock(registry);
5256        let resp = client
5257            .post("https://api.stripe.com/v1/charges")
5258            .send()
5259            .await
5260            .unwrap();
5261
5262        assert_eq!(resp.status().as_u16(), 201);
5263        assert_eq!(call_count.load(Ordering::SeqCst), 1);
5264    }
5265
5266    // RED-PHASE TEST 16: NoMock error when mock registry has no match.
5267    #[tokio::test]
5268    async fn no_mock_error_when_unmatched() {
5269        let registry = Arc::new(MockRegistry::new());
5270        let client = Client::new().with_mock(registry);
5271        let result = client.post("https://api.example.com/unknown").send().await;
5272        assert!(matches!(result, Err(ClientError::NoMock(_, _))));
5273    }
5274
5275    // RED-PHASE TEST 17: MockSetupBuilder registers and returns MockHandle.
5276    #[tokio::test]
5277    async fn mock_setup_builder_registers_entry() {
5278        let registry = Arc::new(MockRegistry::new());
5279        let builder = MockSetupBuilder {
5280            registry: registry.clone(),
5281            alias: "myservice".to_owned(),
5282            method: None,
5283            path: None,
5284        };
5285
5286        let handle = builder
5287            .post("/api/resource")
5288            .respond_with(201, serde_json::json!({"ok": true}));
5289
5290        let client = Client::new().with_mock(registry).named("myservice");
5291        client
5292            .post("https://myservice.example.com/api/resource")
5293            .send()
5294            .await
5295            .unwrap();
5296
5297        handle.expect_called(1);
5298    }
5299
5300    // RED-PHASE TEST 18: Client::from_config respects timeout and retries.
5301    #[test]
5302    fn client_from_config() {
5303        let config = HttpClientConfig {
5304            timeout_secs: 10,
5305            max_retries: 1,
5306            max_retry_after_secs: 10,
5307            max_backoff_ms: 20_000,
5308            base_urls: std::collections::HashMap::new(),
5309            ..HttpClientConfig::default()
5310        };
5311        let client = Client::from_config(&config);
5312        assert_eq!(client.retry_policy.max_retries, 1);
5313    }
5314
5315    // RED-PHASE TEST 19: Client.named() preserves mock registry.
5316    #[test]
5317    fn named_client_preserves_mock_registry() {
5318        let registry = Arc::new(MockRegistry::new());
5319        let client = Client::new().with_mock(registry);
5320        let named = client.named("stripe");
5321        assert!(named.mock.is_some());
5322        assert_eq!(named.alias.as_deref(), Some("stripe"));
5323    }
5324
5325    // RED-PHASE TEST 20: base_url is prepended to relative paths.
5326    #[test]
5327    fn base_url_prepended_to_relative_path() {
5328        let client = Client::new();
5329        let client = client.with_base_url("https://api.stripe.com");
5330        let builder = client.post("/v1/charges");
5331        assert_eq!(builder.url, "https://api.stripe.com/v1/charges");
5332    }
5333
5334    // RED-PHASE TEST 21: Absolute URLs bypass base_url.
5335    #[test]
5336    fn absolute_url_bypasses_base_url() {
5337        let client = Client::new().with_base_url("https://ignored.example.com");
5338        let builder = client.get("https://actual.example.com/path");
5339        assert_eq!(builder.url, "https://actual.example.com/path");
5340    }
5341
5342    // RED-PHASE TEST 22: RetryPolicy can be overridden per-request.
5343    #[test]
5344    fn retry_override_per_request() {
5345        let client = Client::new(); // default: 3 retries
5346        let builder = client.get("https://example.com").retries(0);
5347        assert_eq!(builder.retry_policy.max_retries, 0);
5348
5349        let no_retry = client.get("https://example.com").no_retry();
5350        assert_eq!(no_retry.retry_policy.max_retries, 0);
5351    }
5352
5353    // RED-PHASE TEST 23: Client extracts from AppState.
5354    #[tokio::test]
5355    async fn client_extracts_from_state() {
5356        use axum::extract::FromRequestParts;
5357        let state = crate::AppState::for_test();
5358        let mut parts = axum::http::Request::new(axum::body::Body::empty())
5359            .into_parts()
5360            .0;
5361        let client = Client::from_request_parts(&mut parts, &state)
5362            .await
5363            .unwrap();
5364        // Default client: no mock, no alias
5365        assert!(client.mock.is_none());
5366        assert!(client.alias.is_none());
5367    }
5368
5369    // RED-PHASE TEST 24: MockRegistryExt round-trips through AppState extensions.
5370    #[test]
5371    fn mock_registry_ext_round_trips_through_state() {
5372        let registry = Arc::new(MockRegistry::new());
5373        let ext = HttpMockRegistryExt(registry);
5374        let state = crate::AppState::for_test();
5375        state.insert_extension(ext);
5376        let retrieved = state.extension::<HttpMockRegistryExt>();
5377        assert!(retrieved.is_some());
5378    }
5379
5380    // TEST 25: named() resolves base URL from base_urls map in config.
5381    #[test]
5382    fn named_client_resolves_base_url_from_config() {
5383        let mut base_urls = std::collections::HashMap::new();
5384        base_urls.insert("stripe".to_owned(), "https://api.stripe.com".to_owned());
5385        let config = HttpClientConfig {
5386            timeout_secs: 30,
5387            max_retries: 3,
5388            max_retry_after_secs: 10,
5389            max_backoff_ms: 20_000,
5390            base_urls,
5391            ..HttpClientConfig::default()
5392        };
5393        let client = Client::from_config(&config);
5394        let stripe = client.named("stripe");
5395        assert_eq!(stripe.base_url.as_deref(), Some("https://api.stripe.com"));
5396        assert_eq!(stripe.alias.as_deref(), Some("stripe"));
5397
5398        // Unknown alias falls back to client-level base_url (None in this case).
5399        let other = client.named("sendgrid");
5400        assert!(other.base_url.is_none());
5401    }
5402
5403    // PR #2480 review, round 4 (redesign): `breaker_scoped()` moved breaker accounting for
5404    // the custom send path from an external wrapper — `BreakerGuardedCall`, now removed —
5405    // to a flag `send_recorded` itself reads. That is what makes the mock bypass, the
5406    // capsule-replay bypass, and correct capsule recording of an open-breaker attempt all
5407    // free: they were already correctly ordered around `send_recorded` before this flag
5408    // existed, for every other caller.
5409    //
5410    // This test locks in the one thing the external wrapper got wrong (P1, round 1):
5411    // keying the breaker by the alias a caller passed to `post`/`get` rather than the
5412    // expanded destination. `self.url` is set once, by `Client::build_request`, before any
5413    // `breaker_scoped()` or `ssrf_safe()` flag is read, so there is no separate "pass the
5414    // right URL" step left to get wrong.
5415    #[test]
5416    fn breaker_scoped_keys_by_resolved_url_not_alias() {
5417        // breaker_for_url really does get_or_create against the shared global
5418        // registry, so this needs the same isolation as every other test that
5419        // touches it directly — otherwise a concurrently-running test's "no
5420        // breaker entry for this host" assertion can observe the entry this
5421        // test creates (and never cleans up otherwise).
5422        let _lock = crate::circuit_breaker::TEST_LOCK
5423            .lock()
5424            .unwrap_or_else(std::sync::PoisonError::into_inner);
5425
5426        let mut base_urls = std::collections::HashMap::new();
5427        base_urls.insert(
5428            "hook-service".to_owned(),
5429            "http://mock-receiver/base".to_owned(),
5430        );
5431        let config = HttpClientConfig {
5432            timeout_secs: 30,
5433            max_retries: 3,
5434            max_retry_after_secs: 10,
5435            max_backoff_ms: 20_000,
5436            base_urls,
5437            ..HttpClientConfig::default()
5438        };
5439        let client = Client::from_config(&config);
5440
5441        let req = client
5442            .named("hook-service")
5443            .post("hook-service")
5444            .ssrf_safe()
5445            .breaker_scoped();
5446        assert_eq!(req.url, "http://mock-receiver/base/hook-service");
5447        assert!(req.breaker_scoped);
5448
5449        // The bare alias would neither parse as a URL nor name the real
5450        // destination host — exactly the bug the P1 finding caught.
5451        assert!(url::Url::parse("hook-service").is_err());
5452        let breaker = super::breaker_for_url(None, &req.url);
5453        assert_eq!(breaker.name(), "mock-receiver");
5454
5455        crate::circuit_breaker::global_registry().clear();
5456    }
5457
5458    // PR #2480 review, round 4: `breaker_scoped()` on the custom send path
5459    // must trip and fail fast exactly like the plain-path breaker already
5460    // does (`test_http_client_circuit_breaker_integration`), and must key on
5461    // the same host a non-custom-path request to the same URL would.
5462    //
5463    // No listener needed: `send_ssrf_safe` refuses a loopback destination
5464    // (`SsrfBlocked`) before ever dialing, and that refusal is exactly the
5465    // kind of failure the breaker must count — a subscriber pointing
5466    // `target_url` at a blocked destination repeatedly must trip the breaker
5467    // on those refusals just as it would on real 5xxs, not bypass accounting
5468    // entirely.
5469    #[tokio::test]
5470    #[allow(clippy::await_holding_lock)]
5471    async fn breaker_scoped_custom_path_trips_and_fails_fast() {
5472        let _lock = crate::circuit_breaker::TEST_LOCK
5473            .lock()
5474            .unwrap_or_else(std::sync::PoisonError::into_inner);
5475        crate::circuit_breaker::global_registry().clear();
5476
5477        let mut rc = crate::config::ResilienceConfig::default();
5478        rc.circuit_breaker.defaults.failure_ratio_threshold = Some(0.5);
5479        rc.circuit_breaker.defaults.minimum_sample_count = Some(3);
5480        rc.circuit_breaker.defaults.open_duration_secs = Some(10);
5481        let client = Client {
5482            resilience_config: Some(Arc::new(rc)),
5483            ..Client::new()
5484        };
5485
5486        let url = "http://127.0.0.1:1/blocked";
5487
5488        for _ in 0..3 {
5489            let res = client.post(url).ssrf_safe().breaker_scoped().send().await;
5490            assert!(
5491                matches!(res, Err(ClientError::SsrfBlocked(_))),
5492                "expected SsrfBlocked, got {res:?}"
5493            );
5494        }
5495
5496        // The breaker for 127.0.0.1 should now be OPEN — the next attempt
5497        // must fail fast with CircuitBreakerOpen rather than SsrfBlocked,
5498        // proving it never re-entered send_custom (no re-resolution, no
5499        // reqwest client built).
5500        let res = client.post(url).ssrf_safe().breaker_scoped().send().await;
5501        assert!(matches!(res, Err(ClientError::CircuitBreakerOpen)));
5502
5503        crate::circuit_breaker::global_registry().clear();
5504    }
5505
5506    /// Regression (#3183 review): a fail-fast `CircuitBreakerOpen` call on
5507    /// the breaker-scoped custom path never reaches the host, so the
5508    /// throttle must not count it.
5509    #[tokio::test]
5510    #[allow(clippy::await_holding_lock)]
5511    async fn open_breaker_calls_are_not_charged_to_the_throttle() {
5512        let _lock = crate::circuit_breaker::TEST_LOCK
5513            .lock()
5514            .unwrap_or_else(std::sync::PoisonError::into_inner);
5515        crate::circuit_breaker::global_registry().clear();
5516
5517        let mut rc = crate::config::ResilienceConfig::default();
5518        rc.circuit_breaker.defaults.failure_ratio_threshold = Some(0.5);
5519        rc.circuit_breaker.defaults.minimum_sample_count = Some(3);
5520        rc.circuit_breaker.defaults.open_duration_secs = Some(10);
5521        let rc = Arc::new(rc);
5522        let url = "http://127.0.0.1:1/blocked";
5523
5524        // Trip the breaker with a client that has no throttle.
5525        let plain = Client {
5526            resilience_config: Some(Arc::clone(&rc)),
5527            ..Client::new()
5528        };
5529        for _ in 0..3 {
5530            let _ = plain.post(url).ssrf_safe().breaker_scoped().send().await;
5531        }
5532
5533        let throttle = Arc::new(crate::admission::AdaptiveThrottle::new(
5534            2.0,
5535            Duration::from_secs(120),
5536        ));
5537        let throttled = Client {
5538            resilience_config: Some(rc),
5539            throttle: Some(Arc::clone(&throttle)),
5540            ..Client::new()
5541        };
5542        for _ in 0..50 {
5543            let res = throttled
5544                .post(url)
5545                .ssrf_safe()
5546                .breaker_scoped()
5547                .send()
5548                .await;
5549            assert!(
5550                matches!(res, Err(ClientError::CircuitBreakerOpen)),
5551                "an open breaker fails fast, not through the throttle: {res:?}"
5552            );
5553        }
5554        assert!(
5555            throttle.reject_probability("127.0.0.1:1", crate::time::ambient_instant())
5556                < f64::EPSILON,
5557            "fail-fast calls must not be counted"
5558        );
5559        crate::circuit_breaker::global_registry().clear();
5560    }
5561
5562    // PR #2480 review, round 4: a client with a mock registry attached must
5563    // never touch the real breaker even with `breaker_scoped()` set —
5564    // `send_recorded` checks `self.mock.is_some()` before it ever reads
5565    // `breaker_scoped`, so a deliberately-mocked failure in one test cannot
5566    // open the shared global breaker for a host name reused by an unrelated
5567    // later test.
5568    #[tokio::test]
5569    #[allow(clippy::await_holding_lock)]
5570    async fn breaker_scoped_bypassed_for_mocked_client() {
5571        let _lock = crate::circuit_breaker::TEST_LOCK
5572            .lock()
5573            .unwrap_or_else(std::sync::PoisonError::into_inner);
5574        crate::circuit_breaker::global_registry().clear();
5575
5576        let registry = Arc::new(MockRegistry::new());
5577        let mock = MockSetupBuilder {
5578            registry: registry.clone(),
5579            alias: "http://mock-receiver/hook".to_owned(),
5580            method: None,
5581            path: None,
5582        }
5583        .post("/hook")
5584        .respond_with(500, serde_json::json!({ "error": "down" }));
5585        let client = Client::new().with_mock(registry);
5586
5587        // More attempts than minimum_sample_count would need to trip a real
5588        // breaker at default thresholds — every one must still reach the
5589        // mock rather than short-circuiting on CircuitBreakerOpen.
5590        for _ in 0..12 {
5591            let res = client
5592                .named("http://mock-receiver/hook")
5593                .post("http://mock-receiver/hook")
5594                .ssrf_safe()
5595                .breaker_scoped()
5596                .send()
5597                .await;
5598            let res = res.expect("a mocked client must never see CircuitBreakerOpen");
5599            assert_eq!(res.status().as_u16(), 500);
5600        }
5601        mock.expect_called(12);
5602
5603        // No breaker entry should exist for the mocked host at all.
5604        assert!(
5605            crate::circuit_breaker::global_registry()
5606                .all_breakers()
5607                .iter()
5608                .all(|b| b.name() != "mock-receiver"),
5609            "a mocked send must never create a real breaker entry"
5610        );
5611
5612        crate::circuit_breaker::global_registry().clear();
5613    }
5614
5615    // TEST 26: from_request_parts uses AutumnConfig.http when no HttpConfig extension.
5616    #[tokio::test]
5617    async fn client_extracts_from_autumn_config_in_state() {
5618        use axum::extract::FromRequestParts;
5619        let mut cfg = crate::config::AutumnConfig::default();
5620        cfg.http.client.max_retries = 7;
5621        let state = crate::AppState::for_test();
5622        state.insert_extension(cfg);
5623
5624        let mut parts = axum::http::Request::new(axum::body::Body::empty())
5625            .into_parts()
5626            .0;
5627        let client = Client::from_request_parts(&mut parts, &state)
5628            .await
5629            .unwrap();
5630        assert_eq!(client.retry_policy.max_retries, 7);
5631    }
5632
5633    // TEST 27: respond_with_status produces a truly empty body (not JSON null).
5634    #[tokio::test]
5635    async fn respond_with_status_produces_empty_body() {
5636        let registry = Arc::new(MockRegistry::new());
5637        let builder = MockSetupBuilder {
5638            registry: registry.clone(),
5639            alias: "svc".to_owned(),
5640            method: None,
5641            path: None,
5642        };
5643        let _handle = builder.delete("/items/1").respond_with_status(204);
5644
5645        let client = Client::new().with_mock(registry).named("svc");
5646        let resp = client
5647            .delete("https://svc.example.com/items/1")
5648            .send()
5649            .await
5650            .unwrap();
5651
5652        assert_eq!(resp.status().as_u16(), 204);
5653        assert_eq!(
5654            resp.bytes(),
5655            bytes::Bytes::new(),
5656            "body must be empty, not \"null\""
5657        );
5658    }
5659
5660    // TEST 28: parse_retry_after handles HTTP-date format.
5661    #[test]
5662    fn retry_after_http_date_parsing() {
5663        let mut headers = HeaderMap::new();
5664        // A date far in the future to ensure the computed seconds > 0.
5665        headers.insert(
5666            reqwest::header::HeaderName::from_static("retry-after"),
5667            HeaderValue::from_static("Tue, 01 Jan 2030 00:00:00 GMT"),
5668        );
5669        let duration = parse_retry_after(&headers);
5670        assert!(duration.is_some(), "should parse HTTP-date Retry-After");
5671        assert!(
5672            duration.unwrap().as_secs() > 0,
5673            "future date should yield positive delay"
5674        );
5675    }
5676
5677    // TEST 29: non-idempotent POST with retries disabled makes only one attempt.
5678    #[tokio::test]
5679    async fn non_idempotent_post_no_retry() {
5680        let registry = Arc::new(MockRegistry::new());
5681        let call_count = Arc::new(AtomicUsize::new(0));
5682        registry.register(MockEntry {
5683            method: Some(Method::POST),
5684            path: "/endpoint".to_owned(),
5685            alias: None,
5686            status: 503,
5687            body: None,
5688            call_count: call_count.clone(),
5689        });
5690
5691        // With retry_idempotent_only=true (default), POST should NOT retry.
5692        let client = Client::new().with_mock(registry);
5693        let resp = client
5694            .post("https://example.com/endpoint")
5695            .send()
5696            .await
5697            .unwrap();
5698
5699        assert_eq!(resp.status().as_u16(), 503);
5700        // Mock was called exactly once — no retry for non-idempotent method.
5701        assert_eq!(call_count.load(Ordering::SeqCst), 1);
5702    }
5703
5704    // TEST 30: find_match strips query string from relative URLs before comparing.
5705    #[tokio::test]
5706    async fn mock_strips_query_from_url_before_matching() {
5707        let registry = Arc::new(MockRegistry::new());
5708        let call_count = Arc::new(AtomicUsize::new(0));
5709        registry.register(MockEntry {
5710            method: Some(Method::GET),
5711            path: "/v1/charges".to_owned(),
5712            alias: None,
5713            status: 200,
5714            body: Some(serde_json::json!({"ok": true})),
5715            call_count: call_count.clone(),
5716        });
5717
5718        // The URL has a query string; the mock is registered without one.
5719        let client = Client::new().with_mock(registry);
5720        let resp = client
5721            .get("https://api.stripe.com/v1/charges?expand[]=balance_transaction")
5722            .send()
5723            .await
5724            .unwrap();
5725
5726        assert_eq!(resp.status().as_u16(), 200);
5727        assert_eq!(call_count.load(Ordering::SeqCst), 1);
5728    }
5729
5730    // TEST 31: suffix match works when mock path starts with '/' and URL has a prefix.
5731    #[tokio::test]
5732    async fn mock_suffix_match_with_leading_slash_path() {
5733        let registry = Arc::new(MockRegistry::new());
5734        let call_count = Arc::new(AtomicUsize::new(0));
5735        // Register only the leaf segment (with leading slash).
5736        registry.register(MockEntry {
5737            method: Some(Method::POST),
5738            path: "/charges".to_owned(),
5739            alias: None,
5740            status: 201,
5741            body: Some(serde_json::json!({"matched": true})),
5742            call_count: call_count.clone(),
5743        });
5744
5745        let client = Client::new().with_mock(registry);
5746        // Full URL path is /v1/charges; mock path is /charges.
5747        let resp = client
5748            .post("https://api.stripe.com/v1/charges")
5749            .send()
5750            .await
5751            .unwrap();
5752
5753        assert_eq!(resp.status().as_u16(), 201);
5754        assert_eq!(call_count.load(Ordering::SeqCst), 1);
5755    }
5756
5757    // TEST 32 (issue #3054): retries() sets only the count. POST retries need
5758    // retry_non_idempotent().
5759    #[test]
5760    fn retries_does_not_clear_idempotent_only_flag() {
5761        let client = Client::new();
5762        let builder = client.post("https://example.com").retries(2);
5763        assert_eq!(builder.retry_policy.max_retries, 2);
5764        assert!(builder.retry_policy.retry_idempotent_only);
5765        assert_eq!(builder.max_attempts(false), 1, "POST must not retry");
5766
5767        let builder = builder.retry_non_idempotent();
5768        assert!(!builder.retry_policy.retry_idempotent_only);
5769        assert_eq!(builder.max_attempts(false), 3);
5770    }
5771
5772    #[test]
5773    fn idempotency_key_is_added_only_for_opted_in_unsafe_retries() {
5774        let client = Client::new();
5775        let key_of = |builder: &RequestBuilder| {
5776            builder
5777                .extra_headers
5778                .get(IDEMPOTENCY_KEY)
5779                .map(|value| value.to_str().unwrap().to_owned())
5780        };
5781
5782        let mut post = client.post("https://example.com").retry_non_idempotent();
5783        post.ensure_idempotency_key();
5784        let first = key_of(&post).expect("an opted-in POST gets a key");
5785        post.ensure_idempotency_key();
5786        assert_eq!(key_of(&post), Some(first), "the key does not change");
5787
5788        let mut caller = client
5789            .post("https://example.com")
5790            .header("Idempotency-Key", "mine")
5791            .retry_non_idempotent();
5792        caller.ensure_idempotency_key();
5793        assert_eq!(key_of(&caller).as_deref(), Some("mine"));
5794
5795        for mut builder in [
5796            client.post("https://example.com").retries(3),
5797            client
5798                .post("https://example.com")
5799                .retry_non_idempotent()
5800                .no_retry(),
5801            client.get("https://example.com").retry_non_idempotent(),
5802        ] {
5803            builder.ensure_idempotency_key();
5804            assert_eq!(key_of(&builder), None);
5805        }
5806    }
5807
5808    #[test]
5809    fn retry_delay_is_jittered_and_capped() {
5810        let entropy = crate::entropy::SeededEntropy::new(9);
5811        let policy = RetryPolicy {
5812            max_backoff: Duration::from_secs(1),
5813            ..RetryPolicy::default()
5814        };
5815        for attempt in 0..40 {
5816            let delay = policy.retry_delay(&entropy, attempt, None);
5817            let ceiling = Duration::from_millis(crate::backoff::ceiling_ms(100, 1_000, attempt));
5818            assert!(delay <= ceiling, "attempt {attempt}: {delay:?}");
5819        }
5820    }
5821
5822    #[test]
5823    fn retry_delay_clamps_a_hint_to_the_backoff_window() {
5824        let entropy = crate::entropy::SeededEntropy::new(10);
5825        let policy = RetryPolicy::default();
5826        // First retry: the backoff is in [0, 100 ms], added to the hint.
5827        let wait = policy.retry_delay(&entropy, 0, Some(Duration::from_secs(2)));
5828        assert!(wait >= Duration::from_secs(2) && wait <= Duration::from_millis(2_100));
5829        // The 5 s slack bounds a long hint.
5830        let wait = policy.retry_delay(&entropy, 0, Some(Duration::from_secs(3_600)));
5831        assert!(wait >= Duration::from_secs(5) && wait <= Duration::from_millis(5_100));
5832    }
5833
5834    #[test]
5835    fn retry_hint_reads_retry_after_on_429_and_503_only() {
5836        let mut headers = HeaderMap::new();
5837        headers.insert("retry-after", HeaderValue::from_static("2"));
5838        assert_eq!(retry_hint(429, &headers), Some(Duration::from_secs(2)));
5839        assert_eq!(retry_hint(503, &headers), Some(Duration::from_secs(2)));
5840        assert_eq!(retry_hint(502, &headers), None);
5841        let empty = HeaderMap::new();
5842        assert_eq!(retry_hint(429, &empty), Some(Duration::from_secs(1)));
5843        assert_eq!(retry_hint(503, &empty), None);
5844    }
5845
5846    // TEST 33: log_request covers the sensitive-header redaction path.
5847    #[test]
5848    fn log_request_completes_with_sensitive_headers() {
5849        let url = reqwest::Url::parse("https://api.example.com/v1/resource?q=1").unwrap();
5850        let mut headers = HeaderMap::new();
5851        headers.insert(
5852            HeaderName::from_static("content-type"),
5853            HeaderValue::from_static("application/json"),
5854        );
5855        headers.insert(
5856            HeaderName::from_static("authorization"),
5857            HeaderValue::from_static("Bearer sk_test_xxx"),
5858        );
5859        // Should complete without panicking; authorization is redacted from span.
5860        log_request("POST", &url, 201, Duration::from_millis(12), &headers);
5861    }
5862
5863    /// The sim `Host` header keeps IPv6 brackets and the port (issue #2967).
5864    #[test]
5865    fn sim_host_header_keeps_ipv6_brackets() {
5866        let runtime = tokio::runtime::Builder::new_current_thread()
5867            .enable_all()
5868            .build()
5869            .expect("runtime");
5870        let echo = axum::Router::new().route(
5871            "/status",
5872            axum::routing::get(|headers: HeaderMap| async move {
5873                headers
5874                    .get(reqwest::header::HOST)
5875                    .and_then(|value| value.to_str().ok())
5876                    .unwrap_or_default()
5877                    .to_owned()
5878            }),
5879        );
5880        runtime.block_on(async {
5881            for (url, host) in [
5882                ("http://[::1]:8080/status", "[::1]:8080"),
5883                ("http://[::1]/status", "[::1]"),
5884                ("http://payments:8443/status", "payments:8443"),
5885            ] {
5886                let request = Client::new().get(url);
5887                let parsed = reqwest::Url::parse(url).unwrap();
5888                let seen = serve_sim_host(echo.clone(), &request, parsed, None)
5889                    .await
5890                    .unwrap()
5891                    .text();
5892                assert_eq!(seen, host, "{url}");
5893            }
5894        });
5895    }
5896
5897    /// A sim attempt signs Web Bot Auth again, as the real send path does, so
5898    /// a held builder never sends a stale signature to a simulated host.
5899    #[test]
5900    fn sim_host_gets_a_fresh_web_bot_auth_signature() {
5901        let runtime = tokio::runtime::Builder::new_current_thread()
5902            .enable_all()
5903            .build()
5904            .expect("runtime");
5905        let echo = axum::Router::new().fallback(|headers: HeaderMap| async move {
5906            let count = headers.get_all("signature").iter().count();
5907            let input = headers
5908                .get("signature-input")
5909                .and_then(|v| v.to_str().ok())
5910                .unwrap_or_default()
5911                .to_owned();
5912            format!("{count} {input}")
5913        });
5914        let key = crate::aeo::web_bot_auth::WebBotAuthKey::from_seed_b64(
5915            "nWGxne_9WmC6hEr0kuwsxERJxWl7MmkZcDusAxyuf2A",
5916        )
5917        .unwrap();
5918        let signer = crate::aeo::web_bot_auth::WebBotAuthSigner::new(key, "https://bot.example");
5919        let created = |input: &str| -> u64 {
5920            input
5921                .split(";created=")
5922                .nth(1)
5923                .and_then(|r| r.split(';').next())
5924                .and_then(|v| v.parse().ok())
5925                .unwrap()
5926        };
5927        let url = "https://payments/charge";
5928        let first = created(&signer.sign_url(url).unwrap().signature_input);
5929        let request = Client::new().get(url).sign_web_bot_auth(&signer);
5930        std::thread::sleep(Duration::from_millis(1100));
5931        let parsed = reqwest::Url::parse(url).unwrap();
5932        let seen = runtime.block_on(async {
5933            serve_sim_host(echo, &request, parsed, None)
5934                .await
5935                .unwrap()
5936                .text()
5937        });
5938        let (count, input) = seen.split_once(' ').unwrap();
5939        assert_eq!(count, "1", "{seen}");
5940        assert!(created(input) > first, "{seen}");
5941    }
5942
5943    /// A sim request with a body carries its `Content-Length`, as the real
5944    /// client's does, and a caller value wins (issue #2967).
5945    #[test]
5946    fn sim_host_receives_content_length() {
5947        let runtime = tokio::runtime::Builder::new_current_thread()
5948            .enable_all()
5949            .build()
5950            .expect("runtime");
5951        let echo = axum::Router::new().fallback(|headers: HeaderMap| async move {
5952            headers
5953                .get(reqwest::header::CONTENT_LENGTH)
5954                .and_then(|value| value.to_str().ok())
5955                .unwrap_or("none")
5956                .to_owned()
5957        });
5958        let url = "http://payments/charge";
5959        let parsed = reqwest::Url::parse(url).unwrap();
5960        runtime.block_on(async {
5961            for (request, expected) in [
5962                (Client::new().post(url).text_body("hello"), "5"),
5963                (
5964                    Client::new().post(url).json(&serde_json::json!({"a": 1})),
5965                    "7",
5966                ),
5967                (Client::new().post(url).bytes_body(Bytes::new()), "0"),
5968                (Client::new().get(url), "none"),
5969            ] {
5970                let seen = serve_sim_host(echo.clone(), &request, parsed.clone(), None)
5971                    .await
5972                    .unwrap()
5973                    .text();
5974                assert_eq!(seen, expected);
5975            }
5976        });
5977    }
5978
5979    /// A sim host sees the active span's `traceparent`, as a real upstream
5980    /// does, and a caller header of the same name wins (issue #2967).
5981    #[cfg(feature = "telemetry-otlp")]
5982    #[test]
5983    fn sim_host_receives_trace_context() {
5984        use opentelemetry::trace::TracerProvider as _;
5985        use opentelemetry_sdk::propagation::TraceContextPropagator;
5986        use opentelemetry_sdk::trace::SdkTracerProvider;
5987        use tracing_subscriber::prelude::*;
5988
5989        opentelemetry::global::set_text_map_propagator(TraceContextPropagator::new());
5990        let provider = SdkTracerProvider::builder().build();
5991        let subscriber = tracing_subscriber::registry()
5992            .with(tracing_opentelemetry::layer().with_tracer(provider.tracer("test")));
5993        let runtime = tokio::runtime::Builder::new_current_thread()
5994            .enable_all()
5995            .build()
5996            .expect("runtime");
5997        let echo = || {
5998            axum::Router::new().route(
5999                "/echo",
6000                axum::routing::get(|headers: HeaderMap| async move {
6001                    headers
6002                        .get("traceparent")
6003                        .and_then(|value| value.to_str().ok())
6004                        .unwrap_or_default()
6005                        .to_owned()
6006                }),
6007            )
6008        };
6009        let url = reqwest::Url::parse("http://payments/echo").unwrap();
6010
6011        tracing::subscriber::with_default(subscriber, || {
6012            let span = tracing::info_span!("sim_trace_test");
6013            let _guard = span.enter();
6014            runtime.block_on(async {
6015                let request = Client::new().get("http://payments/echo");
6016                let traceparent = serve_sim_host(echo(), &request, url.clone(), None)
6017                    .await
6018                    .unwrap()
6019                    .text();
6020                assert!(traceparent.starts_with("00-"), "{traceparent}");
6021
6022                let request = Client::new()
6023                    .get("http://payments/echo")
6024                    .header("traceparent", "caller-value");
6025                let response = serve_sim_host(echo(), &request, url.clone(), None)
6026                    .await
6027                    .unwrap();
6028                assert_eq!(response.text(), "caller-value");
6029            });
6030        });
6031    }
6032
6033    // TEST 34: the request id is added only inside a request.
6034    #[test]
6035    fn no_request_id_outside_a_request() {
6036        let mut headers = HeaderMap::new();
6037        add_current_request_id(&mut headers);
6038        assert!(headers.is_empty());
6039    }
6040
6041    #[tokio::test]
6042    async fn current_request_id_is_added() {
6043        let ctx = crate::log::context::LogContext::new(Some("rid-1".to_owned()));
6044        let mut headers = HeaderMap::new();
6045        crate::log::context::scope(ctx, async { add_current_request_id(&mut headers) }).await;
6046        assert_eq!(headers.get("x-request-id").unwrap(), "rid-1");
6047    }
6048
6049    #[tokio::test]
6050    async fn caller_request_id_is_kept() {
6051        let ctx = crate::log::context::LogContext::new(Some("rid-1".to_owned()));
6052        let mut headers = HeaderMap::new();
6053        headers.insert("x-request-id", HeaderValue::from_static("mine"));
6054        crate::log::context::scope(ctx, async { add_current_request_id(&mut headers) }).await;
6055        assert_eq!(headers.get_all("x-request-id").iter().count(), 1);
6056        assert_eq!(headers.get("x-request-id").unwrap(), "mine");
6057    }
6058
6059    /// `traceparent` and `tracestate` are one W3C context. A caller value for
6060    /// either one drops both injected headers, so the request never mixes two
6061    /// contexts.
6062    #[test]
6063    fn caller_trace_header_drops_the_whole_injected_pair() {
6064        let injected = || {
6065            vec![
6066                ("traceparent".to_owned(), HeaderValue::from_static("ours-p")),
6067                ("tracestate".to_owned(), HeaderValue::from_static("ours-s")),
6068                ("baggage".to_owned(), HeaderValue::from_static("ours-b")),
6069            ]
6070        };
6071        let names = |caller: &[&'static str]| {
6072            let mut headers = HeaderMap::new();
6073            for name in caller {
6074                headers.insert(*name, HeaderValue::from_static("mine"));
6075            }
6076            let mut sent: Vec<String> = trace_headers_to_send(injected(), &headers)
6077                .into_iter()
6078                .map(|(name, _)| name)
6079                .collect();
6080            sent.sort();
6081            sent
6082        };
6083        assert_eq!(names(&[]), ["baggage", "traceparent", "tracestate"]);
6084        assert_eq!(names(&["traceparent"]), ["baggage"]);
6085        assert_eq!(names(&["tracestate"]), ["baggage"]);
6086        assert_eq!(names(&["baggage"]), ["traceparent", "tracestate"]);
6087    }
6088
6089    #[test]
6090    fn caller_trace_header_suppresses_the_injected_one() {
6091        let mut caller = HeaderMap::new();
6092        caller.insert("traceparent", HeaderValue::from_static("mine"));
6093        let request =
6094            inject_trace_context(reqwest::Client::new().get("https://example.com"), &caller)
6095                .build()
6096                .unwrap();
6097        assert!(request.headers().get("traceparent").is_none());
6098    }
6099
6100    // A request built with `without_request_id` sends no id.
6101    #[tokio::test]
6102    #[allow(clippy::await_holding_lock)]
6103    async fn without_request_id_sends_no_id() {
6104        use axum::{Router, http::HeaderMap as AxumHeaders, routing::get};
6105
6106        let _lock = crate::circuit_breaker::TEST_LOCK
6107            .lock()
6108            .unwrap_or_else(std::sync::PoisonError::into_inner);
6109        crate::circuit_breaker::global_registry().clear();
6110
6111        let app = Router::new().route(
6112            "/echo",
6113            get(|headers: AxumHeaders| async move {
6114                headers
6115                    .get("x-request-id")
6116                    .map_or_else(|| "none".to_owned(), |v| v.to_str().unwrap().to_owned())
6117            }),
6118        );
6119        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6120        let addr = listener.local_addr().unwrap();
6121        tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
6122
6123        let ctx = crate::log::context::LogContext::new(Some("rid-1".to_owned()));
6124        let url = format!("http://{addr}/echo");
6125        let (forwarded, opted_out) = crate::log::context::scope(ctx, async {
6126            let forwarded = Client::new().get(&url).send().await.unwrap().text();
6127            let opted_out = Client::new()
6128                .get(&url)
6129                .without_request_id()
6130                .send()
6131                .await
6132                .unwrap()
6133                .text();
6134            (forwarded, opted_out)
6135        })
6136        .await;
6137        assert_eq!(forwarded, "rid-1");
6138        assert_eq!(opted_out, "none");
6139
6140        crate::circuit_breaker::global_registry().clear();
6141    }
6142
6143    // TEST 35: Real GET request exercises inject_trace_context, log_request, and
6144    // the success branch of the retry loop.
6145    #[tokio::test]
6146    #[allow(clippy::await_holding_lock)]
6147    async fn real_get_request_covers_network_path() {
6148        use axum::{Router, routing::get};
6149
6150        let _lock = crate::circuit_breaker::TEST_LOCK
6151            .lock()
6152            .unwrap_or_else(std::sync::PoisonError::into_inner);
6153        crate::circuit_breaker::global_registry().clear();
6154
6155        let app = Router::new().route("/ping", get(|| async { "pong" }));
6156        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6157        let addr = listener.local_addr().unwrap();
6158        tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
6159
6160        let client = Client::new();
6161        let resp = client
6162            .get(format!("http://127.0.0.1:{}/ping", addr.port()))
6163            .header("x-request-id", "test-35")
6164            .send()
6165            .await
6166            .unwrap();
6167
6168        assert_eq!(resp.status().as_u16(), 200);
6169        assert!(resp.url().is_some());
6170        assert_eq!(resp.text(), "pong");
6171
6172        crate::circuit_breaker::global_registry().clear();
6173    }
6174
6175    /// Records each `http.client.request` span and the fields set on it.
6176    #[derive(Clone, Default)]
6177    struct ClientSpanCapture {
6178        spans: std::sync::Arc<
6179            std::sync::Mutex<
6180                std::collections::HashMap<u64, std::collections::BTreeMap<String, String>>,
6181            >,
6182        >,
6183    }
6184
6185    struct SpanFieldVisitor<'a>(&'a mut std::collections::BTreeMap<String, String>);
6186
6187    impl tracing::field::Visit for SpanFieldVisitor<'_> {
6188        fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) {
6189            self.0.insert(
6190                field.name().to_owned(),
6191                format!("{value:?}").trim_matches('"').to_owned(),
6192            );
6193        }
6194    }
6195
6196    impl<S> tracing_subscriber::Layer<S> for ClientSpanCapture
6197    where
6198        S: tracing::Subscriber + for<'a> tracing_subscriber::registry::LookupSpan<'a>,
6199    {
6200        fn on_new_span(
6201            &self,
6202            attrs: &tracing::span::Attributes<'_>,
6203            id: &tracing::span::Id,
6204            _ctx: tracing_subscriber::layer::Context<'_, S>,
6205        ) {
6206            if attrs.metadata().name() != "http.client.request" {
6207                return;
6208            }
6209            let mut fields = std::collections::BTreeMap::new();
6210            attrs.record(&mut SpanFieldVisitor(&mut fields));
6211            self.spans.lock().unwrap().insert(id.into_u64(), fields);
6212        }
6213
6214        fn on_record(
6215            &self,
6216            id: &tracing::span::Id,
6217            values: &tracing::span::Record<'_>,
6218            _ctx: tracing_subscriber::layer::Context<'_, S>,
6219        ) {
6220            if let Some(fields) = self.spans.lock().unwrap().get_mut(&id.into_u64()) {
6221                values.record(&mut SpanFieldVisitor(fields));
6222            }
6223        }
6224    }
6225
6226    /// Records how long each `http.client.request` span stays open.
6227    #[derive(Clone, Default)]
6228    struct ClientSpanDurations {
6229        opened: std::sync::Arc<std::sync::Mutex<std::collections::HashMap<u64, Instant>>>,
6230        closed: std::sync::Arc<std::sync::Mutex<Vec<Duration>>>,
6231    }
6232
6233    impl<S> tracing_subscriber::Layer<S> for ClientSpanDurations
6234    where
6235        S: tracing::Subscriber + for<'a> tracing_subscriber::registry::LookupSpan<'a>,
6236    {
6237        fn on_new_span(
6238            &self,
6239            attrs: &tracing::span::Attributes<'_>,
6240            id: &tracing::span::Id,
6241            _ctx: tracing_subscriber::layer::Context<'_, S>,
6242        ) {
6243            if attrs.metadata().name() == "http.client.request" {
6244                self.opened
6245                    .lock()
6246                    .unwrap()
6247                    .insert(id.into_u64(), Instant::now());
6248            }
6249        }
6250
6251        fn on_close(&self, id: tracing::span::Id, _ctx: tracing_subscriber::layer::Context<'_, S>) {
6252            let start = self.opened.lock().unwrap().remove(&id.into_u64());
6253            if let Some(start) = start {
6254                self.closed.lock().unwrap().push(start.elapsed());
6255            }
6256        }
6257    }
6258
6259    /// Install both span capture layers on this thread, warm up the callsite,
6260    /// and return them.
6261    fn capture_client_spans() -> (
6262        ClientSpanCapture,
6263        ClientSpanDurations,
6264        tracing::subscriber::DefaultGuard,
6265    ) {
6266        use tracing_subscriber::layer::SubscriberExt as _;
6267
6268        let fields = ClientSpanCapture::default();
6269        let durations = ClientSpanDurations::default();
6270        let guard = tracing::subscriber::set_default(
6271            tracing_subscriber::registry()
6272                .with(fields.clone())
6273                .with(durations.clone()),
6274        );
6275        drop(client_attempt_span(&Method::GET, "http://warm.up/", 0));
6276        tracing::callsite::rebuild_interest_cache();
6277        fields.spans.lock().unwrap().clear();
6278        durations.closed.lock().unwrap().clear();
6279        (fields, durations, guard)
6280    }
6281
6282    // The attempt span stays open while the body streams, and records a body
6283    // read failure.
6284    #[tokio::test]
6285    #[allow(clippy::await_holding_lock)]
6286    async fn attempt_span_covers_the_response_body() {
6287        use axum::{Router, body::Body, routing::get};
6288        use futures::StreamExt as _;
6289
6290        let _lock = crate::circuit_breaker::TEST_LOCK
6291            .lock()
6292            .unwrap_or_else(std::sync::PoisonError::into_inner);
6293        crate::circuit_breaker::global_registry().clear();
6294
6295        let app = Router::new()
6296            .route(
6297                "/slow",
6298                get(|| async {
6299                    let stream = futures::stream::once(async {
6300                        tokio::time::sleep(Duration::from_millis(300)).await;
6301                        Ok::<_, std::io::Error>(Bytes::from_static(b"late"))
6302                    });
6303                    Body::from_stream(stream)
6304                }),
6305            )
6306            .route(
6307                "/broken",
6308                get(|| async {
6309                    // Headers and one chunk go out, then the body fails.
6310                    let stream = futures::stream::iter([
6311                        Ok(Bytes::from_static(b"partial")),
6312                        Err(std::io::Error::other("cut")),
6313                    ])
6314                    .then(|item| async move {
6315                        tokio::time::sleep(Duration::from_millis(50)).await;
6316                        item
6317                    });
6318                    Body::from_stream(stream)
6319                }),
6320            );
6321        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6322        let addr = listener.local_addr().unwrap();
6323        tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
6324
6325        for custom_path in [false, true] {
6326            let (fields, durations, _guard) = capture_client_spans();
6327            let mut slow = Client::new().get(format!("http://{addr}/slow"));
6328            let mut broken = Client::new().get(format!("http://{addr}/broken"));
6329            if custom_path {
6330                slow = slow.no_redirect();
6331                broken = broken.no_redirect();
6332            }
6333            assert_eq!(slow.send().await.unwrap().text(), "late");
6334            let closed = durations.closed.lock().unwrap().clone();
6335            assert_eq!(closed.len(), 1, "custom={custom_path}: {closed:?}");
6336            assert!(
6337                closed[0] >= Duration::from_millis(250),
6338                "custom={custom_path}: the span ended before the body: {closed:?}"
6339            );
6340
6341            fields.spans.lock().unwrap().clear();
6342            assert!(broken.send().await.is_err());
6343            let spans: Vec<_> = fields.spans.lock().unwrap().values().cloned().collect();
6344            assert_eq!(spans.len(), 1, "custom={custom_path}: {spans:?}");
6345            assert_eq!(
6346                spans[0].get("error.type").map(String::as_str),
6347                Some("body"),
6348                "custom={custom_path}: {spans:?}"
6349            );
6350            assert_eq!(
6351                spans[0].get("otel.status_code").map(String::as_str),
6352                Some("ERROR"),
6353                "custom={custom_path}: {spans:?}"
6354            );
6355
6356            // A 4xx response is an error span too.
6357            fields.spans.lock().unwrap().clear();
6358            let mut missing = Client::new().get(format!("http://{addr}/missing"));
6359            if custom_path {
6360                missing = missing.no_redirect();
6361            }
6362            assert_eq!(missing.send().await.unwrap().status().as_u16(), 404);
6363            let spans: Vec<_> = fields.spans.lock().unwrap().values().cloned().collect();
6364            assert_eq!(spans.len(), 1, "custom={custom_path}: {spans:?}");
6365            assert_eq!(
6366                spans[0].get("otel.status_code").map(String::as_str),
6367                Some("ERROR"),
6368                "custom={custom_path}: {spans:?}"
6369            );
6370        }
6371
6372        crate::circuit_breaker::global_registry().clear();
6373    }
6374
6375    // A sim attempt opens a CLIENT span too, one per attempt.
6376    #[tokio::test]
6377    async fn sim_attempts_open_client_spans() {
6378        use axum::{Router, http::StatusCode, routing::get};
6379        use std::sync::atomic::{AtomicUsize, Ordering};
6380
6381        let (fields, _durations, _guard) = capture_client_spans();
6382        let calls = std::sync::Arc::new(AtomicUsize::new(0));
6383        let payments = Router::new().route(
6384            "/charge",
6385            get(move || {
6386                let calls = std::sync::Arc::clone(&calls);
6387                async move {
6388                    if calls.fetch_add(1, Ordering::SeqCst) == 0 {
6389                        StatusCode::SERVICE_UNAVAILABLE
6390                    } else {
6391                        StatusCode::OK
6392                    }
6393                }
6394            }),
6395        );
6396        let mut client = Client::new();
6397        client.sim_net = Some(Arc::new(
6398            crate::sim::SimNet::new().host("payments", payments),
6399        ));
6400
6401        let response = client.get("http://payments/charge").send().await.unwrap();
6402        assert_eq!(response.status().as_u16(), 200);
6403
6404        let mut attempts: Vec<_> = fields.spans.lock().unwrap().values().cloned().collect();
6405        attempts.sort_by_key(|f| f.get("http.request.resend_count").cloned());
6406        assert_eq!(attempts.len(), 2, "{attempts:?}");
6407        assert_eq!(
6408            attempts[0].get("server.address").map(String::as_str),
6409            Some("payments")
6410        );
6411        assert_eq!(
6412            attempts[0]
6413                .get("http.response.status_code")
6414                .map(String::as_str),
6415            Some("503")
6416        );
6417        assert_eq!(
6418            attempts[1]
6419                .get("http.response.status_code")
6420                .map(String::as_str),
6421            Some("200")
6422        );
6423        // A 5xx attempt is an error span; a 2xx attempt has no status.
6424        assert_eq!(
6425            attempts[0].get("otel.status_code").map(String::as_str),
6426            Some("ERROR"),
6427            "{attempts:?}"
6428        );
6429        assert_eq!(attempts[1].get("otel.status_code"), None, "{attempts:?}");
6430    }
6431
6432    // A 429 attempt span ends before the `Retry-After` sleep, so its duration
6433    // is the HTTP exchange only, on the plain and the custom send paths.
6434    #[tokio::test]
6435    #[allow(clippy::await_holding_lock)]
6436    async fn attempt_span_ends_before_the_retry_after_sleep() {
6437        use axum::{Router, http::StatusCode, response::IntoResponse as _, routing::get};
6438        use std::sync::atomic::{AtomicUsize, Ordering};
6439        use tracing_subscriber::layer::SubscriberExt as _;
6440
6441        let _lock = crate::circuit_breaker::TEST_LOCK
6442            .lock()
6443            .unwrap_or_else(std::sync::PoisonError::into_inner);
6444        crate::circuit_breaker::global_registry().clear();
6445
6446        for custom_path in [false, true] {
6447            let durations = ClientSpanDurations::default();
6448            let _guard = tracing::subscriber::set_default(
6449                tracing_subscriber::registry().with(durations.clone()),
6450            );
6451            drop(client_attempt_span(&Method::GET, "http://warm.up/", 0));
6452            tracing::callsite::rebuild_interest_cache();
6453            durations.closed.lock().unwrap().clear();
6454
6455            // The first call answers 429 with `Retry-After: 1`, the next 200.
6456            let calls = std::sync::Arc::new(AtomicUsize::new(0));
6457            let app = Router::new().route(
6458                "/limited",
6459                get(move || {
6460                    let calls = std::sync::Arc::clone(&calls);
6461                    async move {
6462                        if calls.fetch_add(1, Ordering::SeqCst) == 0 {
6463                            (StatusCode::TOO_MANY_REQUESTS, [("retry-after", "1")]).into_response()
6464                        } else {
6465                            StatusCode::OK.into_response()
6466                        }
6467                    }
6468                }),
6469            );
6470            let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6471            let addr = listener.local_addr().unwrap();
6472            tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
6473
6474            let mut request = Client::new().get(format!("http://{addr}/limited"));
6475            if custom_path {
6476                request = request.no_redirect();
6477            }
6478            assert_eq!(request.send().await.unwrap().status().as_u16(), 200);
6479
6480            let closed = durations.closed.lock().unwrap().clone();
6481            assert_eq!(closed.len(), 2, "custom={custom_path}: {closed:?}");
6482            assert!(
6483                closed[0] < Duration::from_millis(500),
6484                "custom={custom_path}: the 429 attempt span held the 1s sleep: {closed:?}"
6485            );
6486        }
6487
6488        crate::circuit_breaker::global_registry().clear();
6489    }
6490
6491    // One CLIENT span per outbound attempt (issue #3064), on the plain and
6492    // the custom send paths.
6493    #[tokio::test]
6494    #[allow(clippy::await_holding_lock)]
6495    async fn each_attempt_opens_one_client_span() {
6496        use axum::{Router, http::StatusCode, routing::get};
6497        use std::sync::atomic::{AtomicUsize, Ordering};
6498        use tracing_subscriber::layer::SubscriberExt as _;
6499
6500        let _lock = crate::circuit_breaker::TEST_LOCK
6501            .lock()
6502            .unwrap_or_else(std::sync::PoisonError::into_inner);
6503        crate::circuit_breaker::global_registry().clear();
6504
6505        for custom_path in [false, true] {
6506            let capture = ClientSpanCapture::default();
6507            let _guard = tracing::subscriber::set_default(
6508                tracing_subscriber::registry().with(capture.clone()),
6509            );
6510            // Another test can register this callsite on a thread with no
6511            // subscriber, and cache it as "never". Register it here first,
6512            // rebuild the cache, then discard the warm-up span.
6513            drop(client_attempt_span(&Method::GET, "http://warm.up/", 0));
6514            tracing::callsite::rebuild_interest_cache();
6515            capture.spans.lock().unwrap().clear();
6516
6517            // The first call answers 503, the next 200: two attempts.
6518            let calls = std::sync::Arc::new(AtomicUsize::new(0));
6519            let app = Router::new().route(
6520                "/flaky",
6521                get(move || {
6522                    let calls = std::sync::Arc::clone(&calls);
6523                    async move {
6524                        if calls.fetch_add(1, Ordering::SeqCst) == 0 {
6525                            StatusCode::SERVICE_UNAVAILABLE
6526                        } else {
6527                            StatusCode::OK
6528                        }
6529                    }
6530                }),
6531            );
6532            let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6533            let addr = listener.local_addr().unwrap();
6534            tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
6535
6536            let mut request = Client::new().get(format!("http://{addr}/flaky"));
6537            if custom_path {
6538                request = request.no_redirect();
6539            }
6540            let response = request.send().await.unwrap();
6541            assert_eq!(response.status().as_u16(), 200);
6542
6543            let mut attempts: Vec<_> = capture.spans.lock().unwrap().values().cloned().collect();
6544            attempts.sort_by_key(|f| f.get("http.request.resend_count").cloned());
6545            assert_eq!(attempts.len(), 2, "custom={custom_path}: {attempts:?}");
6546            for (n, fields) in attempts.iter().enumerate() {
6547                assert_eq!(fields.get("otel.kind").map(String::as_str), Some("client"));
6548                assert_eq!(
6549                    fields.get("http.request.method").map(String::as_str),
6550                    Some("GET")
6551                );
6552                assert_eq!(
6553                    fields.get("server.address").map(String::as_str),
6554                    Some("127.0.0.1")
6555                );
6556                let resend = fields.get("http.request.resend_count");
6557                assert_eq!(resend, (n > 0).then(|| n.to_string()).as_ref());
6558            }
6559            assert_eq!(
6560                attempts[0]
6561                    .get("http.response.status_code")
6562                    .map(String::as_str),
6563                Some("503")
6564            );
6565            assert_eq!(
6566                attempts[1]
6567                    .get("http.response.status_code")
6568                    .map(String::as_str),
6569                Some("200")
6570            );
6571        }
6572
6573        crate::circuit_breaker::global_registry().clear();
6574    }
6575
6576    // TEST 36: Real POST with JSON body covers the body-sending code path.
6577    #[tokio::test]
6578    #[allow(clippy::await_holding_lock)]
6579    async fn real_post_with_json_body_covers_body_path() {
6580        use axum::{Json, Router, routing::post};
6581        use serde_json::Value;
6582
6583        let _lock = crate::circuit_breaker::TEST_LOCK
6584            .lock()
6585            .unwrap_or_else(std::sync::PoisonError::into_inner);
6586        crate::circuit_breaker::global_registry().clear();
6587
6588        let app = Router::new().route(
6589            "/echo",
6590            post(|Json(body): Json<Value>| async move { Json(body) }),
6591        );
6592        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6593        let addr = listener.local_addr().unwrap();
6594        tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
6595
6596        let client = Client::new();
6597        let resp = client
6598            .post(format!("http://127.0.0.1:{}/echo", addr.port()))
6599            .json(&serde_json::json!({"hello": "world"}))
6600            .send()
6601            .await
6602            .unwrap();
6603
6604        assert_eq!(resp.status().as_u16(), 200);
6605        let body: Value = resp.json().unwrap();
6606        assert_eq!(body["hello"], "world");
6607
6608        crate::circuit_breaker::global_registry().clear();
6609    }
6610
6611    // TEST 37: GET with one 503 then 200 covers the retry-sleep and 5xx-retry paths.
6612    #[tokio::test]
6613    #[allow(clippy::await_holding_lock)]
6614    async fn real_get_retries_on_503_then_succeeds() {
6615        use axum::{Router, routing::get};
6616        use std::sync::Arc;
6617        use std::sync::atomic::{AtomicU32, Ordering as SeqOrdering};
6618
6619        let _lock = crate::circuit_breaker::TEST_LOCK
6620            .lock()
6621            .unwrap_or_else(std::sync::PoisonError::into_inner);
6622        crate::circuit_breaker::global_registry().clear();
6623
6624        let hit = Arc::new(AtomicU32::new(0));
6625        let hit2 = hit.clone();
6626        let app = Router::new().route(
6627            "/flaky",
6628            get(move || {
6629                let c = hit2.clone();
6630                async move {
6631                    if c.fetch_add(1, SeqOrdering::SeqCst) == 0 {
6632                        axum::http::StatusCode::SERVICE_UNAVAILABLE
6633                    } else {
6634                        axum::http::StatusCode::OK
6635                    }
6636                }
6637            }),
6638        );
6639        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6640        let addr = listener.local_addr().unwrap();
6641        tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
6642
6643        // retries(1): 2 total attempts, a 0-100 ms jittered sleep between them.
6644        let resp = Client::new()
6645            .get(format!("http://127.0.0.1:{}/flaky", addr.port()))
6646            .retries(1)
6647            .send()
6648            .await
6649            .unwrap();
6650
6651        assert_eq!(resp.status().as_u16(), 200);
6652        assert_eq!(hit.load(SeqOrdering::SeqCst), 2);
6653
6654        crate::circuit_breaker::global_registry().clear();
6655    }
6656
6657    /// Serve `503` + `Retry-After: 1` once, then `200`. Returns the address
6658    /// and the hit counter.
6659    async fn serve_down_once_with_retry_after() -> (SocketAddr, Arc<AtomicUsize>) {
6660        use axum::{Router, response::IntoResponse as _, routing::get};
6661        let hits = Arc::new(AtomicUsize::new(0));
6662        let counter = hits.clone();
6663        let app = Router::new().route(
6664            "/down",
6665            get(move || {
6666                let counter = counter.clone();
6667                async move {
6668                    if counter.fetch_add(1, Ordering::SeqCst) == 0 {
6669                        (
6670                            axum::http::StatusCode::SERVICE_UNAVAILABLE,
6671                            [("retry-after", "1")],
6672                        )
6673                            .into_response()
6674                    } else {
6675                        axum::http::StatusCode::OK.into_response()
6676                    }
6677                }
6678            }),
6679        );
6680        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6681        let addr = listener.local_addr().unwrap();
6682        tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
6683        (addr, hits)
6684    }
6685
6686    /// Issue #3054 on the real network path (`send_inner`) and the custom
6687    /// path (`send_one`, via `no_redirect`): a 503 with `Retry-After: 1`
6688    /// waits at least 1 s.
6689    #[tokio::test]
6690    #[allow(clippy::await_holding_lock)]
6691    async fn real_paths_honour_retry_after_on_503() {
6692        let _lock = crate::circuit_breaker::TEST_LOCK
6693            .lock()
6694            .unwrap_or_else(std::sync::PoisonError::into_inner);
6695        crate::circuit_breaker::global_registry().clear();
6696
6697        for custom_path in [false, true] {
6698            let (addr, hits) = serve_down_once_with_retry_after().await;
6699            let mut request = Client::new()
6700                .get(format!("http://127.0.0.1:{}/down", addr.port()))
6701                .retries(1);
6702            if custom_path {
6703                request = request.no_redirect();
6704            }
6705            let start = std::time::Instant::now();
6706            let response = request.send().await.unwrap();
6707            let waited = start.elapsed();
6708            assert_eq!(response.status().as_u16(), 200, "custom_path={custom_path}");
6709            assert_eq!(hits.load(Ordering::SeqCst), 2);
6710            assert!(
6711                waited >= Duration::from_secs(1),
6712                "custom_path={custom_path}: waited only {waited:?}"
6713            );
6714        }
6715
6716        crate::circuit_breaker::global_registry().clear();
6717    }
6718
6719    #[test]
6720    fn from_config_reads_max_backoff_ms() {
6721        let config = HttpClientConfig {
6722            max_backoff_ms: 1_500,
6723            ..Default::default()
6724        };
6725        assert_eq!(
6726            Client::from_config(&config).retry_policy.max_backoff,
6727            Duration::from_millis(1_500)
6728        );
6729        assert_eq!(RetryPolicy::default().max_backoff, Duration::from_secs(20));
6730    }
6731
6732    // TEST 38: text_body sets a plain-text body.
6733    #[test]
6734    fn text_body_sets_body() {
6735        let client = Client::new();
6736        let builder = client.post("https://example.com").text_body("hello");
6737        assert_eq!(builder.body, Some(bytes::Bytes::from_static(b"hello")));
6738    }
6739
6740    // TEST 39: ClientError::NoMock displays correctly.
6741    #[test]
6742    fn client_error_display() {
6743        let err = ClientError::NoMock("GET".to_owned(), "/path".to_owned());
6744        assert!(err.to_string().contains("GET"));
6745        assert!(err.to_string().contains("/path"));
6746    }
6747
6748    /// Capsule replay is offline by construction (issue #1598, AC4): a capsule
6749    /// records the request, the clock and the database, and nothing about the
6750    /// services the handler called. The block is process-wide and one-way, so
6751    /// this checks the predicate's default and the message an operator sees —
6752    /// the wiring itself is pinned by `app::tests::replay_reaches_nothing_outside_the_capsule`,
6753    /// which cannot be expressed here without blocking the whole test binary.
6754    #[test]
6755    fn outbound_is_open_until_a_replay_blocks_it() {
6756        assert!(
6757            !outbound_blocked_for_replay(),
6758            "a serving process must never start out blocked"
6759        );
6760        let error = ClientError::BlockedDuringReplay(
6761            "GET".to_owned(),
6762            "https://payments.example/charge".to_owned(),
6763        )
6764        .to_string();
6765        assert!(
6766            error.contains("blocked during replay") && error.contains("not recorded"),
6767            "the error must say why the call did not happen, got {error}"
6768        );
6769        assert!(
6770            error.contains("GET") && error.contains("https://payments.example/charge"),
6771            "and which call it was, got {error}"
6772        );
6773    }
6774
6775    // TEST 40: Outbound circuit breaker integration trips and fails fast.
6776    #[tokio::test]
6777    #[allow(clippy::await_holding_lock)]
6778    async fn test_http_client_circuit_breaker_integration() {
6779        use axum::{Router, routing::get};
6780        use std::sync::atomic::{AtomicU32, Ordering as SeqOrdering};
6781
6782        let _lock = crate::circuit_breaker::TEST_LOCK
6783            .lock()
6784            .unwrap_or_else(std::sync::PoisonError::into_inner);
6785        crate::circuit_breaker::global_registry().clear();
6786
6787        let hit = Arc::new(AtomicU32::new(0));
6788        let hit2 = hit.clone();
6789        let app = Router::new().route(
6790            "/flaky",
6791            get(move || {
6792                let c = hit2.clone();
6793                async move {
6794                    c.fetch_add(1, SeqOrdering::SeqCst);
6795                    axum::http::StatusCode::INTERNAL_SERVER_ERROR
6796                }
6797            }),
6798        );
6799        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6800        let addr = listener.local_addr().unwrap();
6801        tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
6802
6803        // Build a resilience config with custom thresholds
6804        let mut rc = crate::config::ResilienceConfig::default();
6805        rc.circuit_breaker.defaults.failure_ratio_threshold = Some(0.5);
6806        rc.circuit_breaker.defaults.minimum_sample_count = Some(3);
6807        rc.circuit_breaker.defaults.open_duration_secs = Some(10);
6808
6809        let client = Client::new();
6810        // Attach the resilience config
6811        let client = Client {
6812            resilience_config: Some(Arc::new(rc)),
6813            ..client
6814        };
6815
6816        let url = format!("http://127.0.0.1:{}/flaky", addr.port());
6817
6818        // Send 3 requests (all fail with 500)
6819        for _ in 0..3 {
6820            let res = client.get(&url).send().await;
6821            let res = res.unwrap();
6822            assert_eq!(res.status().as_u16(), 500);
6823        }
6824
6825        // Now the breaker for 127.0.0.1 should be OPEN, and next request should fail fast
6826        let res = client.get(&url).send().await;
6827        assert!(matches!(res, Err(ClientError::CircuitBreakerOpen)));
6828
6829        // Assert that the server was only hit 3 times
6830        assert_eq!(hit.load(SeqOrdering::SeqCst), 3);
6831        crate::circuit_breaker::global_registry().clear();
6832    }
6833
6834    // RED-PHASE TEST 41: SharedReqwestClient round-trips through AppState extensions.
6835    #[test]
6836    fn shared_reqwest_client_ext_round_trips() {
6837        let ext = SharedReqwestClient {
6838            client: reqwest::Client::new(),
6839            timeout_secs: 30,
6840        };
6841        let state = crate::AppState::for_test();
6842        state.insert_extension(ext);
6843        let retrieved = state.extension::<SharedReqwestClient>();
6844        assert!(retrieved.is_some());
6845    }
6846
6847    // RED-PHASE TEST 42: Client::head() compiles and builds a HEAD RequestBuilder.
6848    #[test]
6849    fn client_head_method_builds_request_builder() {
6850        let client = Client::new();
6851        let _builder = client.head("https://example.com/resource");
6852    }
6853
6854    // RED-PHASE TEST 43: from_state reuses the SharedReqwestClient when registered.
6855    // Spins up a local echo server that returns the User-Agent header as the body,
6856    // then asserts the extracted Client carries the distinctive user-agent we set
6857    // on the shared inner client — proving from_state cloned it rather than
6858    // building a fresh default.
6859    #[tokio::test]
6860    #[allow(clippy::await_holding_lock)]
6861    async fn from_state_reuses_shared_client() {
6862        use axum::{Router, routing::get};
6863
6864        let _lock = crate::circuit_breaker::TEST_LOCK
6865            .lock()
6866            .unwrap_or_else(std::sync::PoisonError::into_inner);
6867        crate::circuit_breaker::global_registry().clear();
6868
6869        let app = Router::new().route(
6870            "/ua",
6871            get(|req: axum::http::Request<axum::body::Body>| async move {
6872                req.headers()
6873                    .get("user-agent")
6874                    .and_then(|v| v.to_str().ok())
6875                    .unwrap_or("")
6876                    .to_owned()
6877            }),
6878        );
6879        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6880        let addr = listener.local_addr().unwrap();
6881        tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
6882
6883        let distinctive_inner = reqwest::ClientBuilder::new()
6884            .user_agent("autumn-shared-pool-test")
6885            .build()
6886            .expect("failed to build inner client");
6887        let state = crate::AppState::for_test();
6888        state.insert_extension(SharedReqwestClient {
6889            client: distinctive_inner,
6890            timeout_secs: 30,
6891        });
6892
6893        let client = Client::from_state(&state);
6894        let resp = client
6895            .get(format!("http://127.0.0.1:{}/ua", addr.port()))
6896            .send()
6897            .await
6898            .expect("request should succeed");
6899
6900        assert_eq!(resp.text(), "autumn-shared-pool-test");
6901        crate::circuit_breaker::global_registry().clear();
6902    }
6903
6904    #[cfg(feature = "http-client")]
6905    #[test]
6906    fn from_state_falls_back_when_timeout_mismatches_shared_client() {
6907        // SharedReqwestClient was built with 5s; config says 10s → mismatch
6908        // → from_state must not reuse the shared inner (falls through to
6909        //   from_config which builds a fresh reqwest::Client).
6910        use crate::config::{AutumnConfig, HttpClientConfig};
6911        use std::sync::Arc;
6912
6913        let mut config = AutumnConfig::default();
6914        config.http.client = HttpClientConfig {
6915            timeout_secs: 10,
6916            ..Default::default()
6917        };
6918
6919        let state = crate::AppState::for_test();
6920        state.insert_extension(SharedReqwestClient {
6921            client: reqwest::Client::new(),
6922            timeout_secs: 5, // deliberately different from config
6923        });
6924        state.insert_extension(Arc::new(config));
6925
6926        // Should not panic — falls back to building a fresh client.
6927        let _client = Client::from_state(&state);
6928    }
6929
6930    #[cfg(feature = "http-client")]
6931    #[test]
6932    fn from_state_reuses_shared_client_when_no_config() {
6933        // No HttpConfig/AutumnConfig in state, but SharedReqwestClient is
6934        // present → hits the (None, Some(inner)) arm → with_inner.
6935        let state = crate::AppState::for_test();
6936        // Default timeout_secs from HttpClientConfig matches the default used
6937        // in effective_timeout_secs, so the shared client is reused.
6938        let default_timeout = crate::config::HttpClientConfig::default().timeout_secs;
6939        state.insert_extension(SharedReqwestClient {
6940            client: reqwest::Client::new(),
6941            timeout_secs: default_timeout,
6942        });
6943
6944        let _client = Client::from_state(&state);
6945    }
6946
6947    // ── Security-hardening tests (#1238 redirects, #1239 SSRF/pinning) ────────
6948
6949    use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
6950
6951    // TEST 44: SSRF address policy — blocked ranges.
6952    #[test]
6953    fn ssrf_policy_blocks_private_and_reserved_ipv4() {
6954        let blocked = [
6955            "0.0.0.0",
6956            "10.1.2.3",
6957            "100.64.0.1",      // CGNAT
6958            "127.0.0.1",       // loopback
6959            "169.254.169.254", // cloud metadata
6960            "172.16.5.4",      // private
6961            "192.0.0.1",       // IETF
6962            "192.0.2.5",       // TEST-NET-1
6963            "192.88.99.1",     // 6to4 anycast relay
6964            "192.168.1.1",     // private
6965            "198.18.0.1",      // benchmarking
6966            "198.51.100.7",    // TEST-NET-2
6967            "203.0.113.9",     // TEST-NET-3
6968            "224.0.0.1",       // multicast
6969            "240.0.0.1",       // reserved
6970            "255.255.255.255", // broadcast
6971        ];
6972        for s in blocked {
6973            let ip: IpAddr = s.parse().unwrap();
6974            assert!(is_blocked_ip(ip), "{s} should be blocked");
6975            assert!(!is_public_ip(ip), "{s} should not be public");
6976        }
6977    }
6978
6979    // TEST 45: SSRF address policy — public IPv4 is allowed.
6980    #[test]
6981    fn ssrf_policy_allows_public_ipv4() {
6982        for s in ["1.1.1.1", "8.8.8.8", "93.184.216.34"] {
6983            let ip: IpAddr = s.parse().unwrap();
6984            assert!(is_public_ip(ip), "{s} should be public");
6985            assert!(!is_blocked_ip(ip), "{s} should not be blocked");
6986        }
6987    }
6988
6989    // TEST 46: SSRF address policy — IPv6 blocked ranges, mapped/compatible forms.
6990    #[test]
6991    fn ssrf_policy_ipv6_and_mapped_forms() {
6992        let blocked = [
6993            "::",                     // unspecified
6994            "::1",                    // loopback
6995            "fe80::1",                // link-local
6996            "fc00::1",                // ULA
6997            "ff02::1",                // multicast
6998            "2001:db8::1",            // documentation
6999            "fec0::1",                // deprecated site-local
7000            "::ffff:169.254.169.254", // IPv4-mapped metadata
7001            "::ffff:127.0.0.1",       // IPv4-mapped loopback
7002            // Remaining IANA special-purpose prefixes (Globally Reachable = False).
7003            "100::1",          // Discard-Only 100::/64 (RFC 6666)
7004            "100::dead:beef",  // Discard-Only 100::/64 (RFC 6666)
7005            "2001:2::1",       // Benchmarking 2001:2::/48 (RFC 5180)
7006            "2001:10::1",      // ORCHID 2001:10::/28 (RFC 4843)
7007            "2001:20::1",      // ORCHIDv2 2001:20::/28 (RFC 7343)
7008            "2001:20:abcd::1", // ORCHIDv2 within /28 (RFC 7343)
7009            "2001:2f::1",      // ORCHIDv2 top of /28 (s[1]=0x002f, RFC 7343)
7010            "3fff::1",         // Documentation 3fff::/20 (RFC 9637)
7011            "3fff:0fff::1",    // Documentation top of /20 (s[1]=0x0fff, RFC 9637)
7012            "5f00::1",         // SRv6 SIDs 5f00::/16 (RFC 9602)
7013            "5f00:1234::1",    // SRv6 SIDs within /16 (RFC 9602)
7014            "2620:4f:8000::1", // Direct Delegation AS112 (RFC 7534)
7015        ];
7016        for s in blocked {
7017            let ip: IpAddr = s.parse().unwrap();
7018            assert!(is_blocked_ip(ip), "{s} should be blocked");
7019        }
7020        // IPv4-compatible ::7f00:1 == 127.0.0.1 (deprecated form) is blocked.
7021        let compat = IpAddr::V6(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0x7f00, 0x0001));
7022        assert!(
7023            is_blocked_ip(compat),
7024            "::7f00:1 (127.0.0.1) should be blocked"
7025        );
7026
7027        // A real public IPv6 (Cloudflare DNS) is allowed.
7028        let public: IpAddr = "2606:4700:4700::1111".parse().unwrap();
7029        assert!(
7030            is_public_ip(public),
7031            "2606:4700:4700::1111 should be public"
7032        );
7033
7034        // Negative tests: real public addresses adjacent to the newly-blocked
7035        // special-purpose prefixes must stay allowed (no over-blocking).
7036        let public_addrs = [
7037            "2001:4860:4860::8888", // Google DNS
7038            "2606:4700:4700::1111", // Cloudflare DNS
7039            "2400:cb00:2048::1",    // public 2400 (Cloudflare)
7040            "2620:0:2d0:200::7",    // public 2620 NOT in AS112 2620:4f:8000::/48
7041            "2001:2:1::1",          // just outside benchmarking /48 (s[2]=1, not ORCHID)
7042            "3fff:abcd::1",         // outside documentation /20 (s[1]=0xabcd > 0x0fff)
7043            "4000::1",              // outside documentation 3fff::/20
7044            "5e00::1",              // outside SRv6 5f00::/16
7045            "6000::1",              // outside SRv6 5f00::/16
7046        ];
7047        for s in public_addrs {
7048            let ip: IpAddr = s.parse().unwrap();
7049            assert!(is_public_ip(ip), "{s} should be public");
7050            assert!(!is_blocked_ip(ip), "{s} should not be blocked");
7051        }
7052    }
7053
7054    // TEST 46b: SSRF policy blocks private / metadata IPv4 tunnelled inside an
7055    // IPv6 literal via NAT64 (64:ff9b::/96) and 6to4 (2002::/16), while leaving
7056    // a genuinely-public embedded IPv4 (and native public v6) public.
7057    #[test]
7058    fn ssrf_policy_blocks_tunnelled_ipv4() {
7059        let blocked = [
7060            "64:ff9b::a9fe:a9fe", // NAT64 → 169.254.169.254 (cloud metadata)
7061            "64:ff9b::7f00:1",    // NAT64 → 127.0.0.1 (loopback)
7062            // RFC 8215 local-use NAT64 `64:ff9b:1::/48` is denied outright.
7063            "64:ff9b:1::a9fe:a9fe", // local-use NAT64 → 169.254.169.254
7064            "64:ff9b:1::7f00:1",    // local-use NAT64 → 127.0.0.1
7065            "64:ff9b:1::808:808",   // local-use NAT64 embedding public 8.8.8.8: still blocked
7066            "2002:a9fe:a9fe::",     // 6to4  → 169.254.169.254
7067            "2002:7f00:1::",        // 6to4  → 127.0.0.1
7068            "2002:0a00:0001::",     // 6to4  → 10.0.0.1
7069            // SIIT IPv4-translated `::ffff:0:0:0/96` (RFC 6052): segment[4] ==
7070            // 0xffff, segment[5] == 0, IPv4 in the last 32 bits. Distinct from
7071            // IPv4-mapped `::ffff:0:0/96`, so must be decoded and re-checked.
7072            "::ffff:0:169.254.169.254", // SIIT → 169.254.169.254 (cloud metadata)
7073            "::ffff:0:127.0.0.1",       // SIIT → 127.0.0.1 (loopback)
7074        ];
7075        for s in blocked {
7076            let ip: IpAddr = s.parse().unwrap();
7077            assert!(is_blocked_ip(ip), "{s} should be blocked");
7078            assert!(!is_public_ip(ip), "{s} should not be public");
7079        }
7080
7081        // 192.88.99.0/24 (6to4 anycast relay) is blocked as a plain IPv4 literal.
7082        let anycast: IpAddr = "192.88.99.1".parse().unwrap();
7083        assert!(is_blocked_ip(anycast), "192.88.99.1 should be blocked");
7084
7085        // A 6to4 address embedding a genuinely-public IPv4 (8.8.8.8) stays
7086        // public (the embedded v4 is public, so it falls through to the native
7087        // v6 checks, which do not match 2002::/16). So does a native public v6.
7088        // A SIIT IPv4-translated address embedding a genuinely-public IPv4
7089        // (8.8.8.8) stays public — the decoded v4 is public, so it falls
7090        // through to the native v6 checks, which do not match it.
7091        for s in ["2002:0808:0808::", "2606:4700::1111", "::ffff:0:8.8.8.8"] {
7092            let ip: IpAddr = s.parse().unwrap();
7093            assert!(is_public_ip(ip), "{s} should be public");
7094            assert!(!is_blocked_ip(ip), "{s} should not be blocked");
7095        }
7096    }
7097
7098    // TEST 47: the `url` crate normalises decimal/hex IP-literal hosts to Ipv4,
7099    // so `http://2130706433/` (== 127.0.0.1) is recognised as a blocked IP.
7100    #[test]
7101    fn ssrf_policy_decimal_encoded_host_is_blocked() {
7102        for raw in [
7103            "http://2130706433/",
7104            "http://0x7f000001/",
7105            "http://127.0.0.1/",
7106        ] {
7107            let parsed = url::Url::parse(raw).unwrap();
7108            match parsed.host() {
7109                Some(url::Host::Ipv4(v4)) => {
7110                    assert_eq!(
7111                        v4,
7112                        Ipv4Addr::LOCALHOST,
7113                        "{raw} should normalise to 127.0.0.1"
7114                    );
7115                    assert!(
7116                        is_blocked_ip(IpAddr::V4(v4)),
7117                        "{raw} host should be blocked"
7118                    );
7119                }
7120                other => panic!("{raw} did not parse to an Ipv4 host: {other:?}"),
7121            }
7122        }
7123    }
7124
7125    // Small axum helper: spawn `app` on an ephemeral 127.0.0.1 port, return it.
7126    async fn spawn(app: axum::Router) -> std::net::SocketAddr {
7127        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
7128        let addr = listener.local_addr().unwrap();
7129        tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
7130        addr
7131    }
7132
7133    fn redirect_302(location: String) -> axum::response::Response {
7134        axum::response::Response::builder()
7135            .status(302)
7136            .header("location", location)
7137            .body(axum::body::Body::empty())
7138            .unwrap()
7139    }
7140
7141    fn redirect_307(location: String) -> axum::response::Response {
7142        axum::response::Response::builder()
7143            .status(307)
7144            .header("location", location)
7145            .body(axum::body::Body::empty())
7146            .unwrap()
7147    }
7148
7149    // Build a response with an arbitrary status that carries a `Location`
7150    // header — used to prove non-followable 3xx statuses (304/300/…) are NOT
7151    // treated as redirects even when they advertise a `Location`.
7152    fn response_with_location(status: u16, location: String) -> axum::response::Response {
7153        axum::response::Response::builder()
7154            .status(status)
7155            .header("location", location)
7156            .body(axum::body::Body::empty())
7157            .unwrap()
7158    }
7159
7160    // TEST 48: no_redirect returns the 3xx verbatim without following it.
7161    #[tokio::test]
7162    async fn no_redirect_returns_3xx_unfollowed() {
7163        use axum::{Router, routing::get};
7164        let addr = spawn(Router::new().route(
7165            "/start",
7166            get(|| async { redirect_302("http://127.0.0.1:1/never".to_owned()) }),
7167        ))
7168        .await;
7169
7170        let resp = Client::new()
7171            .get(format!("http://127.0.0.1:{}/start", addr.port()))
7172            .no_redirect()
7173            .send()
7174            .await
7175            .unwrap();
7176
7177        assert_eq!(resp.status().as_u16(), 302);
7178        assert_eq!(
7179            resp.headers().get("location").and_then(|v| v.to_str().ok()),
7180            Some("http://127.0.0.1:1/never")
7181        );
7182    }
7183
7184    // TEST 48b: the non-ssrf `no_redirect()` path returns the 3xx verbatim even
7185    // when `Location` is a MALFORMED URL. That path uses reqwest's
7186    // `Policy::none()` and never parses `Location`, so a bad target cannot turn
7187    // a `no_redirect()` fetch into an error. This guards the same "don't parse
7188    // Location when not following" contract that `send_ssrf_safe` now enforces
7189    // by returning the 3xx BEFORE calling `redirect_target`.
7190    //
7191    // NOTE: the SSRF-safe equivalent — `get_ssrf_safe(url).no_redirect()`
7192    // against a listener returning a malformed `Location` — cannot be exercised
7193    // end-to-end in-sandbox: the SSRF guard denies loopback (127.0.0.1), so the
7194    // request is rejected during resolve→validate before any response is
7195    // received. This testable-layer variant documents/guards the shared
7196    // contract instead.
7197    #[tokio::test]
7198    async fn no_redirect_returns_3xx_with_malformed_location() {
7199        use axum::{Router, routing::get};
7200        let addr = spawn(Router::new().route(
7201            "/start",
7202            // `ht!tp://\bad` is not a parseable absolute URL (invalid scheme),
7203            // but it is a valid HTTP header value, so the server can emit it.
7204            get(|| async { redirect_302("ht!tp://\\bad".to_owned()) }),
7205        ))
7206        .await;
7207
7208        let resp = Client::new()
7209            .get(format!("http://127.0.0.1:{}/start", addr.port()))
7210            .no_redirect()
7211            .send()
7212            .await
7213            .expect("no_redirect() must return the 3xx even with a malformed Location");
7214
7215        assert_eq!(resp.status().as_u16(), 302);
7216        assert_eq!(
7217            resp.headers().get("location").and_then(|v| v.to_str().ok()),
7218            Some("ht!tp://\\bad")
7219        );
7220    }
7221
7222    // TEST 49: follow_redirects follows a valid chain A→B and calls the validator.
7223    #[tokio::test]
7224    async fn follow_redirects_valid_chain_calls_validator() {
7225        use axum::{Router, routing::get};
7226
7227        let b_addr = spawn(Router::new().route("/final", get(|| async { "final-body" }))).await;
7228        let b_port = b_addr.port();
7229        let a_addr = spawn(Router::new().route(
7230            "/start",
7231            get(move || async move { redirect_302(format!("http://127.0.0.1:{b_port}/final")) }),
7232        ))
7233        .await;
7234
7235        let calls = Arc::new(AtomicUsize::new(0));
7236        let calls2 = calls.clone();
7237        let resp = Client::new()
7238            .get(format!("http://127.0.0.1:{}/start", a_addr.port()))
7239            .follow_redirects(5, move |_loc| {
7240                calls2.fetch_add(1, Ordering::SeqCst);
7241                true
7242            })
7243            .send()
7244            .await
7245            .unwrap();
7246
7247        assert_eq!(resp.status().as_u16(), 200);
7248        assert_eq!(resp.text(), "final-body");
7249        assert_eq!(
7250            calls.load(Ordering::SeqCst),
7251            1,
7252            "validator called once per hop"
7253        );
7254    }
7255
7256    // TEST 50: follow_redirects rejects a redirect to a private/blocked target,
7257    // and NEVER connects to that private address. Uses the SSRF IP policy as the
7258    // validator — structurally the same guard get_ssrf_safe applies per hop.
7259    #[tokio::test]
7260    async fn follow_redirects_rejects_private_target() {
7261        use axum::{Router, routing::get};
7262        use std::sync::atomic::AtomicBool;
7263
7264        // A "private" server that must never be reached.
7265        let touched = Arc::new(AtomicBool::new(false));
7266        let touched2 = touched.clone();
7267        let priv_addr = spawn(Router::new().route(
7268            "/secret",
7269            get(move || {
7270                let t = touched2.clone();
7271                async move {
7272                    t.store(true, Ordering::SeqCst);
7273                    "SECRET"
7274                }
7275            }),
7276        ))
7277        .await;
7278        let priv_port = priv_addr.port();
7279
7280        // Public-ish entrypoint that 302s to the private loopback target.
7281        let a_addr = spawn(Router::new().route(
7282            "/start",
7283            get(
7284                move || async move { redirect_302(format!("http://127.0.0.1:{priv_port}/secret")) },
7285            ),
7286        ))
7287        .await;
7288
7289        let validator = |u: &str| -> bool {
7290            let Ok(p) = url::Url::parse(u) else {
7291                return false;
7292            };
7293            match p.host() {
7294                Some(url::Host::Ipv4(v4)) => is_public_ip(IpAddr::V4(v4)),
7295                Some(url::Host::Ipv6(v6)) => is_public_ip(IpAddr::V6(v6)),
7296                _ => true,
7297            }
7298        };
7299
7300        let result = Client::new()
7301            .get(format!("http://127.0.0.1:{}/start", a_addr.port()))
7302            .follow_redirects(5, validator)
7303            .send()
7304            .await;
7305
7306        assert!(
7307            matches!(result, Err(ClientError::RedirectRejected(_))),
7308            "expected RedirectRejected, got {result:?}"
7309        );
7310        assert!(
7311            !touched.load(Ordering::SeqCst),
7312            "the private target must never be connected to"
7313        );
7314    }
7315
7316    // TEST 51: follow_redirects with a chain longer than `max` → TooManyRedirects.
7317    #[tokio::test]
7318    async fn follow_redirects_cap_exceeded() {
7319        use axum::{Router, routing::get};
7320
7321        // /loop always redirects back to itself → infinite chain.
7322        let addr =
7323            spawn(Router::new().route("/loop", get(|| async { redirect_302("/loop".to_owned()) })))
7324                .await;
7325
7326        let result = Client::new()
7327            .get(format!("http://127.0.0.1:{}/loop", addr.port()))
7328            .follow_redirects(2, |_| true)
7329            .send()
7330            .await;
7331
7332        assert!(
7333            matches!(result, Err(ClientError::TooManyRedirects(2))),
7334            "expected TooManyRedirects(2), got {result:?}"
7335        );
7336    }
7337
7338    // TEST 52: follow_redirects(0, ..) turns the first 3xx into TooManyRedirects.
7339    #[tokio::test]
7340    async fn follow_redirects_zero_max_errors_on_first_3xx() {
7341        use axum::{Router, routing::get};
7342        let addr = spawn(Router::new().route(
7343            "/start",
7344            get(|| async { redirect_302("http://127.0.0.1:1/x".to_owned()) }),
7345        ))
7346        .await;
7347
7348        let result = Client::new()
7349            .get(format!("http://127.0.0.1:{}/start", addr.port()))
7350            .follow_redirects(0, |_| true)
7351            .send()
7352            .await;
7353
7354        assert!(matches!(result, Err(ClientError::TooManyRedirects(0))));
7355    }
7356
7357    // TEST 53: pin_to bypasses DNS and connects to the URL's port.
7358    //
7359    // The host `pinned.invalid` is guaranteed non-resolvable (.invalid TLD), yet
7360    // pinning to 127.0.0.1 reaches the listener — proving DNS was bypassed. The
7361    // pinned SocketAddr uses port 1 (which nothing listens on) while the URL uses
7362    // the real listener port; reaching the listener proves reqwest IGNORES the
7363    // resolve SocketAddr's port and connects to the URL's port instead.
7364    #[tokio::test]
7365    async fn pin_to_bypasses_dns_and_uses_url_port() {
7366        use axum::{Router, routing::get};
7367        let addr = spawn(Router::new().route("/ping", get(|| async { "pong" }))).await;
7368        let listener_port = addr.port();
7369
7370        let resp = Client::new()
7371            .get(format!("http://pinned.invalid:{listener_port}/ping"))
7372            // Deliberately-wrong port (1) to probe reqwest's port handling.
7373            .pin_to(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 1))
7374            .send()
7375            .await
7376            .expect("pinned request should reach the loopback listener");
7377
7378        assert_eq!(resp.status().as_u16(), 200);
7379        assert_eq!(resp.text(), "pong");
7380    }
7381
7382    // `pin_to_addrs` hands the whole set to one request, so a refused first
7383    // address falls back inside reqwest instead of re-sending the body.
7384    #[tokio::test]
7385    async fn pin_to_addrs_falls_back_to_the_next_address_on_connect_failure() {
7386        use axum::{Router, routing::get};
7387        let addr = spawn(Router::new().route("/ping", get(|| async { "pong" }))).await;
7388        let port = addr.port();
7389
7390        let resp = Client::new()
7391            .get(format!("http://pinned.invalid:{port}/ping"))
7392            .pin_to_addrs([
7393                // The listener is IPv4 only, so this address refuses at once.
7394                // `127.0.0.2` is not portable: macOS does not route it.
7395                SocketAddr::new(IpAddr::V6(std::net::Ipv6Addr::LOCALHOST), port),
7396                SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), port),
7397            ])
7398            .send()
7399            .await
7400            .expect("the second pinned address must serve the request");
7401
7402        assert_eq!(resp.text(), "pong");
7403    }
7404
7405    #[tokio::test]
7406    async fn pin_to_addrs_with_no_address_fails_instead_of_sending_unpinned() {
7407        let result = Client::new()
7408            .get("http://pinned.invalid:9/ping")
7409            .pin_to_addrs([])
7410            .send()
7411            .await;
7412        assert!(
7413            matches!(result, Err(ClientError::InvalidUrl(_))),
7414            "{result:?}"
7415        );
7416    }
7417
7418    // TEST 54: get_ssrf_safe rejects a host that resolves to a blocked IP BEFORE
7419    // connecting. `localhost` resolves to 127.0.0.1 (and/or ::1), both blocked.
7420    // The listener's handler must never fire.
7421    #[tokio::test]
7422    async fn get_ssrf_safe_rejects_loopback_host_before_connecting() {
7423        use axum::{Router, routing::get};
7424        use std::sync::atomic::AtomicBool;
7425
7426        let touched = Arc::new(AtomicBool::new(false));
7427        let touched2 = touched.clone();
7428        let addr = spawn(Router::new().route(
7429            "/x",
7430            get(move || {
7431                let t = touched2.clone();
7432                async move {
7433                    t.store(true, Ordering::SeqCst);
7434                    "reached"
7435                }
7436            }),
7437        ))
7438        .await;
7439
7440        let result = Client::new()
7441            .get_ssrf_safe(format!("http://localhost:{}/x", addr.port()))
7442            .send()
7443            .await;
7444
7445        assert!(
7446            matches!(result, Err(ClientError::SsrfBlocked(_))),
7447            "expected SsrfBlocked, got {result:?}"
7448        );
7449        assert!(
7450            !touched.load(Ordering::SeqCst),
7451            "SSRF guard must reject before any connection"
7452        );
7453    }
7454
7455    // TEST 55: get_ssrf_safe rejects a decimal-encoded loopback IP literal
7456    // (http://2130706433/ == 127.0.0.1) with no DNS lookup.
7457    #[tokio::test]
7458    async fn get_ssrf_safe_rejects_decimal_encoded_loopback() {
7459        let result = Client::new()
7460            .get_ssrf_safe("http://2130706433/")
7461            .send()
7462            .await;
7463        assert!(
7464            matches!(result, Err(ClientError::SsrfBlocked(_))),
7465            "expected SsrfBlocked, got {result:?}"
7466        );
7467    }
7468
7469    // TEST 56: resolve_and_validate — the resolve→validate core of get_ssrf_safe.
7470    // A public IP literal validates (returning the URL's port); blocked literals
7471    // and decimal-encoded loopback are rejected with SsrfBlocked. Exercising the
7472    // actual network connect of the safe path against a public host is NOT
7473    // reproducible in-sandbox (no reachable public server); it is covered
7474    // structurally by the pin_to and follow_redirects tests. See the report.
7475    #[tokio::test]
7476    async fn resolve_and_validate_accepts_public_rejects_blocked() {
7477        // Public literal with an explicit port → Ok, single-element validated
7478        // set, port preserved.
7479        let ok = resolve_and_validate("http://8.8.8.8:8080/path")
7480            .await
7481            .unwrap();
7482        assert_eq!(
7483            ok,
7484            vec![SocketAddr::new(IpAddr::V4(Ipv4Addr::new(8, 8, 8, 8)), 8080)]
7485        );
7486
7487        // https default port is inferred.
7488        let ok_https = resolve_and_validate("https://1.1.1.1/").await.unwrap();
7489        assert_eq!(ok_https.len(), 1);
7490        assert_eq!(ok_https[0].port(), 443);
7491
7492        // Blocked literals and decimal-encoded loopback → SsrfBlocked.
7493        for raw in [
7494            "http://127.0.0.1/",
7495            "http://169.254.169.254/latest/meta-data/",
7496            "http://10.0.0.1/",
7497            "http://2130706433/",
7498        ] {
7499            let err = resolve_and_validate(raw).await;
7500            assert!(
7501                matches!(err, Err(ClientError::SsrfBlocked(_))),
7502                "{raw} should be SsrfBlocked, got {err:?}"
7503            );
7504        }
7505    }
7506
7507    // TEST 56b: validate_resolved_addrs — the pure validation core shared by the
7508    // resolve→validate step. A set of public addrs returns ALL of them in order;
7509    // a set mixing a public and a blocked addr is rejected with SsrfBlocked; an
7510    // all-public IPv6+IPv4 mix returns everything in order. This exercises the
7511    // multi-record path (pin to ALL validated addresses) without needing real
7512    // multi-record DNS.
7513    #[test]
7514    fn validate_resolved_addrs_returns_all_public_rejects_any_blocked() {
7515        let v4 = |a, b, c, d, p| SocketAddr::new(IpAddr::V4(Ipv4Addr::new(a, b, c, d)), p);
7516
7517        // Two public addrs → Ok with BOTH returned, order preserved.
7518        let two = vec![v4(1, 1, 1, 1, 443), v4(8, 8, 8, 8, 443)];
7519        assert_eq!(validate_resolved_addrs(two.clone()).unwrap(), two);
7520
7521        // Public + blocked (10.0.0.1, RFC1918) → Err(SsrfBlocked).
7522        let mixed = vec![v4(1, 1, 1, 1, 443), v4(10, 0, 0, 1, 443)];
7523        assert!(
7524            matches!(
7525                validate_resolved_addrs(mixed),
7526                Err(ClientError::SsrfBlocked(_))
7527            ),
7528            "a set containing a blocked address must be rejected"
7529        );
7530
7531        // All-public IPv6 (2606:4700:4700::1111, Cloudflare) + IPv4 mix → Ok,
7532        // all preserved in order.
7533        let v6 = SocketAddr::new(
7534            IpAddr::V6(Ipv6Addr::new(0x2606, 0x4700, 0x4700, 0, 0, 0, 0, 0x1111)),
7535            443,
7536        );
7537        let mix = vec![v6, v4(8, 8, 8, 8, 443)];
7538        assert_eq!(validate_resolved_addrs(mix.clone()).unwrap(), mix);
7539    }
7540
7541    // TEST 57: pin_to alone does NOT auto-follow a cross-host redirect. reqwest
7542    // would otherwise re-resolve the new host via normal DNS, silently escaping
7543    // the pin. The 302 must be returned verbatim and the onward target must
7544    // never be connected to.
7545    #[tokio::test]
7546    async fn pin_to_does_not_follow_redirect_unpinned() {
7547        use axum::{Router, routing::get};
7548        use std::sync::atomic::AtomicBool;
7549
7550        // The redirect target that must never be reached.
7551        let touched = Arc::new(AtomicBool::new(false));
7552        let touched2 = touched.clone();
7553        let onward = spawn(Router::new().route(
7554            "/onward",
7555            get(move || {
7556                let t = touched2.clone();
7557                async move {
7558                    t.store(true, Ordering::SeqCst);
7559                    "REACHED"
7560                }
7561            }),
7562        ))
7563        .await;
7564        let onward_port = onward.port();
7565
7566        let start =
7567            spawn(Router::new().route(
7568                "/start",
7569                get(move || async move {
7570                    redirect_302(format!("http://127.0.0.1:{onward_port}/onward"))
7571                }),
7572            ))
7573            .await;
7574        let start_port = start.port();
7575
7576        // Use a DOMAIN host (`pinned.invalid`) so the pin's resolve override is
7577        // actually consulted; pinning an IP-literal host is now rejected by the
7578        // PinRequiresDomainHost guard. The non-resolvable domain reaches the
7579        // loopback listener only via the pin.
7580        let resp = Client::new()
7581            .get(format!("http://pinned.invalid:{start_port}/start"))
7582            .pin_to(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), start_port))
7583            .send()
7584            .await
7585            .expect("pinned request should return the 302 unfollowed");
7586
7587        assert_eq!(
7588            resp.status().as_u16(),
7589            302,
7590            "pin_to must return the redirect unfollowed"
7591        );
7592        assert!(
7593            !touched.load(Ordering::SeqCst),
7594            "pin_to must not silently follow the redirect onward"
7595        );
7596    }
7597
7598    // TEST 58: get_ssrf_safe rejects a non-http(s) scheme up front with
7599    // InvalidUrl, before any DNS resolution or connection.
7600    #[tokio::test]
7601    async fn get_ssrf_safe_rejects_non_http_scheme() {
7602        for raw in ["ftp://public.example/resource", "gopher://public.example/"] {
7603            let result = Client::new().get_ssrf_safe(raw).send().await;
7604            assert!(
7605                matches!(result, Err(ClientError::InvalidUrl(_))),
7606                "{raw} should be rejected with InvalidUrl, got {result:?}"
7607            );
7608        }
7609    }
7610
7611    // TEST 59: a cross-origin redirect (different port ⇒ different origin) must
7612    // NOT forward credential-bearing request headers to the new origin. This is
7613    // the manual-redirect-loop version of reqwest's built-in strip-on-cross-host
7614    // behaviour, closing a credential-leak hole (Fix A).
7615    #[tokio::test]
7616    async fn follow_redirects_strips_sensitive_headers_cross_origin() {
7617        use axum::{Router, routing::get};
7618
7619        let seen: Arc<Mutex<Option<HeaderMap>>> = Arc::new(Mutex::new(None));
7620        let seen2 = seen.clone();
7621        // Listener B records the headers it received (different port = different
7622        // origin from A).
7623        let b_addr = spawn(Router::new().route(
7624            "/dst",
7625            get(move |headers: HeaderMap| {
7626                let slot = seen2.clone();
7627                async move {
7628                    *slot.lock().unwrap() = Some(headers);
7629                    "ok"
7630                }
7631            }),
7632        ))
7633        .await;
7634        let b_port = b_addr.port();
7635
7636        // Listener A 302-redirects onto B.
7637        let a_addr = spawn(Router::new().route(
7638            "/",
7639            get(move || async move { redirect_302(format!("http://127.0.0.1:{b_port}/dst")) }),
7640        ))
7641        .await;
7642
7643        let resp = Client::new()
7644            .get(format!("http://127.0.0.1:{}/", a_addr.port()))
7645            .header("authorization", "secret")
7646            .header("cookie", "session=abc")
7647            .header("proxy-authorization", "Basic zzz")
7648            .follow_redirects(3, |_| true)
7649            .send()
7650            .await
7651            .unwrap();
7652
7653        assert_eq!(resp.status().as_u16(), 200);
7654        let headers = seen
7655            .lock()
7656            .unwrap()
7657            .clone()
7658            .expect("listener B must have been reached");
7659        assert!(
7660            headers.get("authorization").is_none(),
7661            "authorization must be stripped on a cross-origin redirect"
7662        );
7663        assert!(
7664            headers.get("cookie").is_none(),
7665            "cookie must be stripped on a cross-origin redirect"
7666        );
7667        assert!(
7668            headers.get("proxy-authorization").is_none(),
7669            "proxy-authorization must be stripped on a cross-origin redirect"
7670        );
7671    }
7672
7673    // A Web Bot Auth signature is a credential: on the default send path a
7674    // cross-origin redirect must not carry it to the next host.
7675    #[tokio::test]
7676    async fn default_redirects_strip_message_signatures_cross_origin() {
7677        use axum::{Router, routing::get};
7678
7679        let seen: Arc<Mutex<Option<HeaderMap>>> = Arc::new(Mutex::new(None));
7680        let seen2 = seen.clone();
7681        let b_addr = spawn(Router::new().route(
7682            "/dst",
7683            get(move |headers: HeaderMap| {
7684                let slot = seen2.clone();
7685                async move {
7686                    *slot.lock().unwrap() = Some(headers);
7687                    "ok"
7688                }
7689            }),
7690        ))
7691        .await;
7692        let b_port = b_addr.port();
7693        let a_addr = spawn(Router::new().route(
7694            "/",
7695            get(move || async move { redirect_302(format!("http://127.0.0.1:{b_port}/dst")) }),
7696        ))
7697        .await;
7698
7699        let resp = Client::new()
7700            .get(format!("http://127.0.0.1:{}/", a_addr.port()))
7701            .header("signature-agent", "\"https://bot.example\"")
7702            .header("signature-input", "sig1=(\"@authority\")")
7703            .header("signature", "sig1=:AAAA:")
7704            .send()
7705            .await
7706            .unwrap();
7707
7708        assert_eq!(
7709            resp.status().as_u16(),
7710            200,
7711            "the redirect is still followed"
7712        );
7713        let headers = seen.lock().unwrap().clone().expect("B reached");
7714        for name in ["signature", "signature-input", "signature-agent"] {
7715            assert!(headers.get(name).is_none(), "{name} must be stripped");
7716        }
7717    }
7718
7719    // A Web Bot Auth signature is made when the request is sent, not when
7720    // the builder is configured, so a held builder never sends a stale one.
7721    #[tokio::test]
7722    async fn web_bot_auth_signs_at_send_time() {
7723        use axum::{Router, routing::get};
7724
7725        let seen: Arc<Mutex<Option<HeaderMap>>> = Arc::new(Mutex::new(None));
7726        let seen2 = seen.clone();
7727        let addr = spawn(Router::new().route(
7728            "/",
7729            get(move |headers: HeaderMap| {
7730                let slot = seen2.clone();
7731                async move {
7732                    *slot.lock().unwrap() = Some(headers);
7733                    "ok"
7734                }
7735            }),
7736        ))
7737        .await;
7738        let key = crate::aeo::web_bot_auth::WebBotAuthKey::from_seed_b64(
7739            "nWGxne_9WmC6hEr0kuwsxERJxWl7MmkZcDusAxyuf2A",
7740        )
7741        .unwrap();
7742        let signer = crate::aeo::web_bot_auth::WebBotAuthSigner::new(key, "https://bot.example");
7743        let created = |input: &str| -> u64 {
7744            input
7745                .split(";created=")
7746                .nth(1)
7747                .and_then(|r| r.split(';').next())
7748                .and_then(|v| v.parse().ok())
7749                .unwrap()
7750        };
7751        let url = format!("http://127.0.0.1:{}/", addr.port());
7752        let first = created(&signer.sign_url(&url).unwrap().signature_input);
7753        let builder = Client::new().get(&url).sign_web_bot_auth(&signer);
7754        tokio::time::sleep(Duration::from_millis(1100)).await;
7755        builder.send().await.unwrap();
7756
7757        let headers = seen.lock().unwrap().clone().expect("reached");
7758        let sent = headers.get("signature-input").unwrap().to_str().unwrap();
7759        assert!(created(sent) > first, "{sent}");
7760        assert_eq!(headers.get_all("signature").iter().count(), 1);
7761    }
7762
7763    // A failure capsule matches a replayed call on the headers the handler
7764    // chose. A Web Bot Auth signature is made new for each send, so it is
7765    // left out, or no replay of a signed call would ever match.
7766    #[cfg(feature = "reporting")]
7767    #[test]
7768    fn capsule_matching_skips_the_web_bot_auth_signature() {
7769        let key = crate::aeo::web_bot_auth::WebBotAuthKey::from_seed_b64(
7770            "nWGxne_9WmC6hEr0kuwsxERJxWl7MmkZcDusAxyuf2A",
7771        )
7772        .unwrap();
7773        let signer = crate::aeo::web_bot_auth::WebBotAuthSigner::new(key, "https://bot.example");
7774        let builder = Client::new()
7775            .get("https://api.example/x")
7776            .header("x-kept", "1")
7777            .sign_web_bot_auth(&signer);
7778        let names: Vec<String> = caller_headers(&builder)
7779            .into_iter()
7780            .map(|(name, _)| name)
7781            .collect();
7782        assert!(names.contains(&"x-kept".to_owned()), "{names:?}");
7783        assert!(names.contains(&"signature-agent".to_owned()), "{names:?}");
7784        assert!(!names.contains(&"signature".to_owned()), "{names:?}");
7785        assert!(!names.contains(&"signature-input".to_owned()), "{names:?}");
7786
7787        // A handler's own `Signature` header, with no signer, still counts.
7788        let plain = Client::new()
7789            .get("https://api.example/x")
7790            .header("signature", "mine");
7791        assert_eq!(
7792            caller_headers(&plain),
7793            vec![("signature".to_owned(), "mine".to_owned())]
7794        );
7795    }
7796
7797    // TEST 60: a SAME-origin redirect (relative `Location`, same host:port) must
7798    // keep credential-bearing headers — stripping only applies across origins.
7799    #[tokio::test]
7800    async fn follow_redirects_keeps_sensitive_headers_same_origin() {
7801        use axum::{Router, routing::get};
7802
7803        let seen: Arc<Mutex<Option<HeaderMap>>> = Arc::new(Mutex::new(None));
7804        let seen2 = seen.clone();
7805        let addr = spawn(
7806            Router::new()
7807                .route("/", get(|| async { redirect_302("/next".to_owned()) }))
7808                .route(
7809                    "/next",
7810                    get(move |headers: HeaderMap| {
7811                        let slot = seen2.clone();
7812                        async move {
7813                            *slot.lock().unwrap() = Some(headers);
7814                            "ok"
7815                        }
7816                    }),
7817                ),
7818        )
7819        .await;
7820
7821        let resp = Client::new()
7822            .get(format!("http://127.0.0.1:{}/", addr.port()))
7823            .header("authorization", "secret")
7824            .follow_redirects(3, |_| true)
7825            .send()
7826            .await
7827            .unwrap();
7828
7829        assert_eq!(resp.status().as_u16(), 200);
7830        let headers = seen
7831            .lock()
7832            .unwrap()
7833            .clone()
7834            .expect("/next must have been reached");
7835        assert_eq!(
7836            headers.get("authorization").and_then(|v| v.to_str().ok()),
7837            Some("secret"),
7838            "authorization must be preserved on a same-origin redirect"
7839        );
7840    }
7841
7842    // TEST 61: RFC 7231 §6.4.3 — a 302 in response to a POST rewrites the next
7843    // hop to a bodyless GET (Fix B).
7844    #[tokio::test]
7845    async fn follow_redirects_302_post_becomes_get() {
7846        use axum::{
7847            Router,
7848            routing::{any, post},
7849        };
7850
7851        let seen: Arc<Mutex<Option<(String, HeaderMap, Bytes)>>> = Arc::new(Mutex::new(None));
7852        let seen2 = seen.clone();
7853        // B records the method + headers + body it actually received, for any verb.
7854        let b_addr = spawn(Router::new().route(
7855            "/dst",
7856            any(move |method: Method, headers: HeaderMap, body: Bytes| {
7857                let slot = seen2.clone();
7858                async move {
7859                    *slot.lock().unwrap() = Some((method.to_string(), headers, body));
7860                    "ok"
7861                }
7862            }),
7863        ))
7864        .await;
7865        let b_port = b_addr.port();
7866
7867        let a_addr = spawn(Router::new().route(
7868            "/",
7869            post(move || async move { redirect_302(format!("http://127.0.0.1:{b_port}/dst")) }),
7870        ))
7871        .await;
7872
7873        // `.json(..)` sets a request body AND `Content-Type: application/json`.
7874        // On the 302 POST→GET rewrite the body is dropped, and the payload
7875        // headers must be dropped with it (Fix B) — otherwise the followed GET
7876        // would carry a misleading `Content-Type` for a body it no longer has.
7877        let resp = Client::new()
7878            .post(format!("http://127.0.0.1:{}/", a_addr.port()))
7879            .json(&serde_json::json!({"payload": true}))
7880            .follow_redirects(3, |_| true)
7881            .send()
7882            .await
7883            .unwrap();
7884
7885        assert_eq!(resp.status().as_u16(), 200);
7886        let (method, headers, body) = seen
7887            .lock()
7888            .unwrap()
7889            .clone()
7890            .expect("listener B must have been reached");
7891        assert_eq!(method, "GET", "302 must rewrite POST → GET");
7892        assert!(body.is_empty(), "302 POST→GET must drop the request body");
7893        assert!(
7894            !headers.contains_key(reqwest::header::CONTENT_TYPE),
7895            "302 POST→GET must drop the Content-Type payload header"
7896        );
7897        assert!(
7898            !headers.contains_key(reqwest::header::CONTENT_LENGTH),
7899            "302 POST→GET must drop the Content-Length payload header"
7900        );
7901    }
7902
7903    // TEST 62: RFC 7231 §6.4.7 — a 307 preserves BOTH the method and the body
7904    // across the redirect (the review bot's drop-body-every-hop snippet was
7905    // wrong here) (Fix B).
7906    #[tokio::test]
7907    async fn follow_redirects_307_preserves_method_and_body() {
7908        use axum::{
7909            Router,
7910            routing::{any, post},
7911        };
7912
7913        let seen: Arc<Mutex<Option<(String, Bytes)>>> = Arc::new(Mutex::new(None));
7914        let seen2 = seen.clone();
7915        let b_addr = spawn(Router::new().route(
7916            "/dst",
7917            any(move |method: Method, body: Bytes| {
7918                let slot = seen2.clone();
7919                async move {
7920                    *slot.lock().unwrap() = Some((method.to_string(), body));
7921                    "ok"
7922                }
7923            }),
7924        ))
7925        .await;
7926        let b_port = b_addr.port();
7927
7928        let a_addr = spawn(Router::new().route(
7929            "/",
7930            post(move || async move { redirect_307(format!("http://127.0.0.1:{b_port}/dst")) }),
7931        ))
7932        .await;
7933
7934        let resp = Client::new()
7935            .post(format!("http://127.0.0.1:{}/", a_addr.port()))
7936            .text_body("payload")
7937            .follow_redirects(3, |_| true)
7938            .send()
7939            .await
7940            .unwrap();
7941
7942        assert_eq!(resp.status().as_u16(), 200);
7943        let (method, body) = seen
7944            .lock()
7945            .unwrap()
7946            .clone()
7947            .expect("listener B must have been reached");
7948        assert_eq!(method, "POST", "307 must preserve the POST method");
7949        assert_eq!(
7950            &body[..],
7951            b"payload",
7952            "307 must preserve the request body verbatim"
7953        );
7954    }
7955
7956    // TEST 63: get_ssrf_safe derives its redirect follow/cap from the chained
7957    // builder mode via ssrf_redirect_plan, so `no_redirect()` /
7958    // `follow_redirects(max, ..)` override the SSRF-safe default hop cap.
7959    //
7960    // NOTE: the network-level follow behaviour of get_ssrf_safe cannot be
7961    // exercised in-sandbox — the only reachable address here is loopback, which
7962    // the SSRF guard blocks before connecting — so this deterministic unit test
7963    // on the factored `ssrf_redirect_plan` helper stands in for it.
7964    #[test]
7965    fn ssrf_redirect_plan_honours_chained_override() {
7966        let client = Client::new();
7967
7968        // Default get_ssrf_safe → follow up to the SSRF-safe hop cap.
7969        let default = client.get_ssrf_safe("https://example.com/");
7970        assert_eq!(
7971            default.ssrf_redirect_plan(),
7972            (true, SSRF_SAFE_MAX_REDIRECTS),
7973            "default SSRF-safe path follows up to SSRF_SAFE_MAX_REDIRECTS"
7974        );
7975
7976        // no_redirect() → do NOT follow.
7977        let none = client.get_ssrf_safe("https://example.com/").no_redirect();
7978        let (follow, _max) = none.ssrf_redirect_plan();
7979        assert!(
7980            !follow,
7981            "no_redirect() must disable following on the safe path"
7982        );
7983
7984        // follow_redirects(3, ..) → follow up to the caller's max.
7985        let follow3 = client
7986            .get_ssrf_safe("https://example.com/")
7987            .follow_redirects(3, |_| true);
7988        assert_eq!(
7989            follow3.ssrf_redirect_plan(),
7990            (true, 3),
7991            "follow_redirects(3, ..) must cap the safe path at 3 hops"
7992        );
7993    }
7994
7995    // TEST 64: pin_to + follow_redirects is rejected at send time with
7996    // IncompatiblePinRedirect — deterministically, without touching the network.
7997    // A single pinned SocketAddr only covers hop 0; later redirect hops re-resolve
7998    // via normal DNS, so following would silently escape the pin.
7999    #[tokio::test]
8000    async fn pin_then_follow_redirects_is_rejected() {
8001        let result = Client::new()
8002            .get("http://example.com/")
8003            .pin_to(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 8080))
8004            .follow_redirects(2, |_| true)
8005            .send()
8006            .await;
8007
8008        assert!(
8009            matches!(result, Err(ClientError::IncompatiblePinRedirect(_))),
8010            "expected IncompatiblePinRedirect, got {result:?}"
8011        );
8012    }
8013
8014    // TEST 65: the rejection is order-independent — chaining follow_redirects
8015    // before pin_to produces the same IncompatiblePinRedirect error.
8016    #[tokio::test]
8017    async fn follow_redirects_then_pin_is_rejected() {
8018        let result = Client::new()
8019            .get("http://example.com/")
8020            .follow_redirects(2, |_| true)
8021            .pin_to(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 8080))
8022            .send()
8023            .await;
8024
8025        assert!(
8026            matches!(result, Err(ClientError::IncompatiblePinRedirect(_))),
8027            "expected IncompatiblePinRedirect, got {result:?}"
8028        );
8029    }
8030
8031    // TEST 66: pin_to combined with no_redirect is NOT affected by the guard —
8032    // the request proceeds and returns the 3xx verbatim (RedirectMode::None).
8033    #[tokio::test]
8034    async fn pin_with_no_redirect_is_allowed() {
8035        use axum::{Router, routing::get};
8036
8037        let addr = spawn(Router::new().route(
8038            "/start",
8039            get(|| async { redirect_302("http://127.0.0.1:1/onward".to_owned()) }),
8040        ))
8041        .await;
8042        let port = addr.port();
8043
8044        // Use a DOMAIN host so the pin's resolve override is consulted; pinning
8045        // an IP-literal host is now rejected by the PinRequiresDomainHost guard.
8046        let resp = Client::new()
8047            .get(format!("http://pinned.invalid:{port}/start"))
8048            .pin_to(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), port))
8049            .no_redirect()
8050            .send()
8051            .await
8052            .expect("pin_to + no_redirect must return the 3xx unfollowed, not error");
8053
8054        assert_eq!(
8055            resp.status().as_u16(),
8056            302,
8057            "pin_to + no_redirect returns the redirect verbatim"
8058        );
8059    }
8060
8061    // TEST 67: pin_to on an IPv4-literal URL host is rejected at send time with
8062    // PinRequiresDomainHost — deterministically, without touching the network.
8063    // reqwest/hyper treat an IP-literal host as already-resolved and skip the
8064    // resolve override that installs the pin, so the socket would connect to the
8065    // literal in the URL (198.51.100.1), NOT the pinned 127.0.0.1 — silently
8066    // bypassing the pin. The guard rejects it instead.
8067    #[tokio::test]
8068    async fn pin_to_ipv4_literal_host_is_rejected() {
8069        let result = Client::new()
8070            .get("http://198.51.100.1:8080/")
8071            .pin_to(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 8080))
8072            .send()
8073            .await;
8074
8075        assert!(
8076            matches!(result, Err(ClientError::PinRequiresDomainHost(_))),
8077            "expected PinRequiresDomainHost, got {result:?}"
8078        );
8079    }
8080
8081    // TEST 68: the same rejection applies to an IPv6-literal URL host.
8082    #[tokio::test]
8083    async fn pin_to_ipv6_literal_host_is_rejected() {
8084        let result = Client::new()
8085            .get("http://[2606:4700::1111]/")
8086            .pin_to(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 8080))
8087            .send()
8088            .await;
8089
8090        assert!(
8091            matches!(result, Err(ClientError::PinRequiresDomainHost(_))),
8092            "expected PinRequiresDomainHost, got {result:?}"
8093        );
8094    }
8095
8096    // TEST 68b: get_ssrf_safe combined with pin_to is rejected at send time with
8097    // PinNotAllowedWithSsrfSafe — deterministically, without touching the
8098    // network. The SSRF-safe path runs its own per-hop resolve/validate/pin and
8099    // never reads the pin_to address, so an explicit pin would be silently
8100    // ignored; the guard fails loudly instead.
8101    #[tokio::test]
8102    async fn get_ssrf_safe_with_pin_to_is_rejected() {
8103        let result = Client::new()
8104            .get_ssrf_safe("http://example.com/")
8105            .pin_to(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 8080))
8106            .send()
8107            .await;
8108
8109        assert!(
8110            matches!(result, Err(ClientError::PinNotAllowedWithSsrfSafe(_))),
8111            "expected PinNotAllowedWithSsrfSafe, got {result:?}"
8112        );
8113    }
8114
8115    // TEST 69: a `304 Not Modified` that happens to carry a `Location` header is
8116    // NOT a followable redirect. reqwest only follows 301/302/303/307/308, so
8117    // `redirect_target` must return the 304 to the caller verbatim rather than
8118    // issuing a second request against the `Location` target.
8119    #[tokio::test]
8120    async fn follow_redirects_does_not_follow_304_with_location() {
8121        use axum::{Router, routing::get};
8122        use std::sync::atomic::AtomicBool;
8123
8124        // Listener B must never be reached.
8125        let touched = Arc::new(AtomicBool::new(false));
8126        let touched2 = touched.clone();
8127        let b_addr = spawn(Router::new().route(
8128            "/dst",
8129            get(move || {
8130                let t = touched2.clone();
8131                async move {
8132                    t.store(true, Ordering::SeqCst);
8133                    "SHOULD-NOT-BE-HIT"
8134                }
8135            }),
8136        ))
8137        .await;
8138        let b_port = b_addr.port();
8139
8140        // Listener A returns 304 + Location pointing at B.
8141        let a_addr = spawn(Router::new().route(
8142            "/start",
8143            get(move || async move {
8144                response_with_location(304, format!("http://127.0.0.1:{b_port}/dst"))
8145            }),
8146        ))
8147        .await;
8148
8149        let resp = Client::new()
8150            .get(format!("http://127.0.0.1:{}/start", a_addr.port()))
8151            .follow_redirects(5, |_| true)
8152            .send()
8153            .await
8154            .unwrap();
8155
8156        assert_eq!(
8157            resp.status().as_u16(),
8158            304,
8159            "a 304 with a Location header must be returned verbatim, not followed"
8160        );
8161        assert!(
8162            !touched.load(Ordering::SeqCst),
8163            "the 304 Location target must never be requested"
8164        );
8165    }
8166
8167    // TEST 70: a `300 Multiple Choices` that carries a `Location` header is
8168    // likewise not a followable redirect (only 301/302/303/307/308 are), so the
8169    // 300 is returned to the caller and the `Location` target is never hit.
8170    #[tokio::test]
8171    async fn follow_redirects_does_not_follow_300_with_location() {
8172        use axum::{Router, routing::get};
8173        use std::sync::atomic::AtomicBool;
8174
8175        let touched = Arc::new(AtomicBool::new(false));
8176        let touched2 = touched.clone();
8177        let b_addr = spawn(Router::new().route(
8178            "/dst",
8179            get(move || {
8180                let t = touched2.clone();
8181                async move {
8182                    t.store(true, Ordering::SeqCst);
8183                    "SHOULD-NOT-BE-HIT"
8184                }
8185            }),
8186        ))
8187        .await;
8188        let b_port = b_addr.port();
8189
8190        let a_addr = spawn(Router::new().route(
8191            "/start",
8192            get(move || async move {
8193                response_with_location(300, format!("http://127.0.0.1:{b_port}/dst"))
8194            }),
8195        ))
8196        .await;
8197
8198        let resp = Client::new()
8199            .get(format!("http://127.0.0.1:{}/start", a_addr.port()))
8200            .follow_redirects(5, |_| true)
8201            .send()
8202            .await
8203            .unwrap();
8204
8205        assert_eq!(
8206            resp.status().as_u16(),
8207            300,
8208            "a 300 with a Location header must be returned verbatim, not followed"
8209        );
8210        assert!(
8211            !touched.load(Ordering::SeqCst),
8212            "the 300 Location target must never be requested"
8213        );
8214    }
8215
8216    // ── Issue #3058: deadline and retry budget on the real send paths ────────
8217    #[allow(clippy::large_futures, reason = "test futures, awaited once")]
8218    mod deadline_and_budget {
8219        use super::super::*;
8220        use std::sync::atomic::AtomicU32;
8221
8222        /// A server that counts hits on `/x`. It answers with `status` and
8223        /// `headers`, or never when `status` is `None`.
8224        async fn counting(
8225            status: Option<u16>,
8226            headers: &'static [(&'static str, &'static str)],
8227        ) -> (String, Arc<AtomicU32>) {
8228            let hits = Arc::new(AtomicU32::new(0));
8229            let counter = Arc::clone(&hits);
8230            let app = axum::Router::new().route(
8231                "/x",
8232                axum::routing::get(move || {
8233                    counter.fetch_add(1, Ordering::SeqCst);
8234                    async move {
8235                        let Some(code) = status else {
8236                            return std::future::pending().await;
8237                        };
8238                        let mut response = axum::response::Response::builder().status(code);
8239                        for (name, value) in headers {
8240                            response = response.header(*name, *value);
8241                        }
8242                        response.body(axum::body::Body::empty()).unwrap()
8243                    }
8244                }),
8245            );
8246            let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8247            let addr = listener.local_addr().unwrap();
8248            tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
8249            (format!("http://127.0.0.1:{}/x", addr.port()), hits)
8250        }
8251
8252        /// A client whose budget allows one transient retry and never refills.
8253        fn one_retry_client() -> Client {
8254            let mut config = crate::config::HttpClientConfig::default();
8255            config.retry_budget.capacity = 14;
8256            config.retry_budget.retry_ratio = 0.0;
8257            Client::from_config(&config)
8258        }
8259
8260        /// Run `send` under a deadline `after` from now, with a 5 s safety
8261        /// limit.
8262        async fn with_deadline(
8263            after: Duration,
8264            send: impl std::future::Future<Output = Result<Response, ClientError>>,
8265        ) -> Result<Response, ClientError> {
8266            tokio::time::timeout(Duration::from_secs(5), Deadline::after(after).scope(send))
8267                .await
8268                .expect("the deadline must stop the call")
8269        }
8270
8271        fn lock() -> std::sync::MutexGuard<'static, ()> {
8272            let guard = crate::circuit_breaker::TEST_LOCK
8273                .lock()
8274                .unwrap_or_else(std::sync::PoisonError::into_inner);
8275            crate::circuit_breaker::global_registry().clear();
8276            guard
8277        }
8278
8279        /// A server where `/r` waits `delay`, then redirects to `/t`. Both
8280        /// record the deadline header they get.
8281        async fn redirecting(delay: Duration) -> (String, Arc<Mutex<Vec<(String, u64)>>>) {
8282            let seen = Arc::new(Mutex::new(Vec::new()));
8283            let record = |path: &'static str, seen: Arc<Mutex<Vec<(String, u64)>>>| {
8284                move |headers: HeaderMap| {
8285                    let millis = headers
8286                        .get(DEADLINE_HEADER)
8287                        .and_then(|value| value.to_str().ok()?.parse().ok())
8288                        .unwrap_or(u64::MAX);
8289                    seen.lock().unwrap().push((path.to_owned(), millis));
8290                }
8291            };
8292            let at_r = record("/r", Arc::clone(&seen));
8293            let at_t = record("/t", Arc::clone(&seen));
8294            let app = axum::Router::new()
8295                .route(
8296                    "/r",
8297                    axum::routing::get(move |headers: HeaderMap| async move {
8298                        at_r(headers);
8299                        tokio::time::sleep(delay).await;
8300                        axum::response::Redirect::temporary("/t")
8301                    }),
8302                )
8303                .route(
8304                    "/t",
8305                    axum::routing::get(move |headers: HeaderMap| async move {
8306                        at_t(headers);
8307                        "done"
8308                    }),
8309                );
8310            let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8311            let addr = listener.local_addr().unwrap();
8312            tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
8313            (format!("http://127.0.0.1:{}/r", addr.port()), seen)
8314        }
8315
8316        #[tokio::test]
8317        async fn plain_path_sends_a_fresh_deadline_header_after_a_redirect() {
8318            let (url, seen) = redirecting(Duration::from_millis(500)).await;
8319            let response = with_deadline(Duration::from_secs(3), Client::new().get(&url).send())
8320                .await
8321                .unwrap();
8322            assert_eq!(response.status().as_u16(), 200);
8323            let seen = seen.lock().unwrap().clone();
8324            let [(first, at_origin), (second, at_target)] = seen.as_slice() else {
8325                panic!("two hops: {seen:?}");
8326            };
8327            assert_eq!((first.as_str(), second.as_str()), ("/r", "/t"));
8328            assert!(*at_origin <= 3_000, "{seen:?}");
8329            assert!(
8330                *at_target + 400 <= *at_origin,
8331                "the target gets the time left after the redirect: {seen:?}"
8332            );
8333        }
8334
8335        #[tokio::test]
8336        async fn a_redirected_connect_error_charges_the_destination_budget() {
8337            // A port with nothing listening on it.
8338            let closed = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8339            let dead_port = closed.local_addr().unwrap().port();
8340            drop(closed);
8341            let target = format!("http://127.0.0.1:{dead_port}/t");
8342            let app = axum::Router::new().route(
8343                "/r",
8344                axum::routing::get(move || {
8345                    let target = target.clone();
8346                    async move { axum::response::Redirect::temporary(&target) }
8347                }),
8348            );
8349            let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8350            let origin_port = listener.local_addr().unwrap().port();
8351            tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
8352
8353            let client = Client::from_config(&crate::config::HttpClientConfig::default());
8354            let budgets = client.retry.budgets.clone().unwrap();
8355            let result = client
8356                .get(format!("http://127.0.0.1:{origin_port}/r"))
8357                .retries(1)
8358                .send()
8359                .await;
8360            assert!(result.is_err(), "{result:?}");
8361            let full = f64::from(RetryBudgetConfig::default().capacity);
8362            let origin = budgets.for_host(&format!("127.0.0.1:{origin_port}"));
8363            let destination = budgets.for_host(&format!("127.0.0.1:{dead_port}"));
8364            assert!((origin.available() - full).abs() < f64::EPSILON, "origin");
8365            assert!(destination.available() < full, "the destination paid");
8366        }
8367
8368        #[tokio::test]
8369        async fn a_redirect_under_a_deadline_sets_referer() {
8370            let seen = Arc::new(Mutex::new(None));
8371            let record = Arc::clone(&seen);
8372            let app = axum::Router::new()
8373                .route(
8374                    "/r",
8375                    axum::routing::get(|| async { axum::response::Redirect::temporary("/t") }),
8376                )
8377                .route(
8378                    "/t",
8379                    axum::routing::get(move |headers: HeaderMap| async move {
8380                        *record.lock().unwrap() = headers
8381                            .get(reqwest::header::REFERER)
8382                            .map(|value| value.to_str().unwrap().to_owned());
8383                    }),
8384                );
8385            let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8386            let port = listener.local_addr().unwrap().port();
8387            tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
8388            let origin = format!("http://127.0.0.1:{port}/r");
8389            let response = with_deadline(Duration::from_secs(3), Client::new().get(&origin).send())
8390                .await
8391                .unwrap();
8392            assert_eq!(response.status().as_u16(), 200);
8393            assert_eq!(seen.lock().unwrap().as_deref(), Some(origin.as_str()));
8394        }
8395
8396        #[test]
8397        fn referer_drops_credentials_and_is_not_sent_on_a_downgrade() {
8398            let mut headers = HeaderMap::new();
8399            set_referer(&mut headers, "https://b/x", "https://u:p@a/y#frag");
8400            assert_eq!(headers[reqwest::header::REFERER], "https://a/y");
8401            set_referer(&mut headers, "http://c/z", "https://b/x");
8402            assert!(!headers.contains_key(reqwest::header::REFERER));
8403        }
8404
8405        #[test]
8406        fn every_host_of_a_redirect_chain_is_refilled() {
8407            let budgets = Arc::new(RetryBudgets::new(&RetryBudgetConfig::default()));
8408            let middle = budgets.for_host("b:443");
8409            let last = budgets.for_host("c:443");
8410            while middle.try_acquire(RetryKind::Transient) {}
8411            while last.try_acquire(RetryKind::Transient) {}
8412            let (middle_before, last_before) = (middle.available(), last.available());
8413            let gate =
8414                RetryGate::with_deadline(None, Some(Arc::clone(&budgets)), Some("a:443"), true);
8415            let mut seen = std::collections::HashSet::new();
8416            gate.record_destinations(
8417                &[
8418                    "https://b/1".to_owned(),
8419                    "https://c/2".to_owned(),
8420                    "https://c/3".to_owned(),
8421                ],
8422                &mut seen,
8423            );
8424            assert!(middle.available() > middle_before, "the middle host");
8425            let one_refill = middle.available() - middle_before;
8426            assert!(
8427                (last.available() - last_before - one_refill).abs() < f64::EPSILON,
8428                "one refill per host"
8429            );
8430        }
8431
8432        /// A server whose `/r` answers 302 with `location`, and whose `/t`
8433        /// answers 200. With `endless`, the 302 body never ends.
8434        async fn redirect_server(location: &'static str, endless: bool) -> String {
8435            let app = axum::Router::new()
8436                .route(
8437                    "/r",
8438                    axum::routing::get(move || async move {
8439                        let body = if endless {
8440                            axum::body::Body::from_stream(futures::stream::pending::<
8441                                Result<Bytes, std::io::Error>,
8442                            >())
8443                        } else {
8444                            axum::body::Body::empty()
8445                        };
8446                        axum::response::Response::builder()
8447                            .status(302)
8448                            .header("location", location)
8449                            .body(body)
8450                            .unwrap()
8451                    }),
8452                )
8453                .route("/t", axum::routing::get(|| async { "done" }));
8454            let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8455            let port = listener.local_addr().unwrap().port();
8456            tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
8457            format!("http://127.0.0.1:{port}/r")
8458        }
8459
8460        #[tokio::test]
8461        async fn a_redirect_under_a_deadline_does_not_read_the_redirect_body() {
8462            let url = redirect_server("/t", true).await;
8463            let response = with_deadline(Duration::from_secs(3), Client::new().get(&url).send())
8464                .await
8465                .unwrap();
8466            assert_eq!(response.status().as_u16(), 200);
8467            assert_eq!(response.text(), "done");
8468        }
8469
8470        #[tokio::test]
8471        async fn a_bad_location_under_a_deadline_returns_the_redirect() {
8472            let url = redirect_server("http://[::1", false).await;
8473            let response = with_deadline(Duration::from_secs(3), Client::new().get(&url).send())
8474                .await
8475                .unwrap();
8476            assert_eq!(response.status().as_u16(), 302);
8477            let plain = Client::new().get(&url).send().await.unwrap();
8478            assert_eq!(
8479                plain.status().as_u16(),
8480                302,
8481                "the same as without a deadline"
8482            );
8483        }
8484
8485        #[tokio::test]
8486        async fn a_redirect_chain_under_a_deadline_shares_one_retry_count() {
8487            use axum::response::IntoResponse;
8488            let origin_hits = Arc::new(AtomicU32::new(0));
8489            let target_hits = Arc::new(AtomicU32::new(0));
8490            let (origin, target) = (Arc::clone(&origin_hits), Arc::clone(&target_hits));
8491            // Each hop fails once, then works.
8492            let app = axum::Router::new()
8493                .route(
8494                    "/r",
8495                    axum::routing::get(move || async move {
8496                        if origin.fetch_add(1, Ordering::SeqCst) == 0 {
8497                            axum::http::StatusCode::SERVICE_UNAVAILABLE.into_response()
8498                        } else {
8499                            axum::response::Redirect::temporary("/t").into_response()
8500                        }
8501                    }),
8502                )
8503                .route(
8504                    "/t",
8505                    axum::routing::get(move || async move {
8506                        if target.fetch_add(1, Ordering::SeqCst) == 0 {
8507                            axum::http::StatusCode::SERVICE_UNAVAILABLE.into_response()
8508                        } else {
8509                            "done".into_response()
8510                        }
8511                    }),
8512                );
8513            let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8514            let port = listener.local_addr().unwrap().port();
8515            tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
8516            let url = format!("http://127.0.0.1:{port}/r");
8517            let response = with_deadline(
8518                Duration::from_secs(3),
8519                Client::new().get(&url).retries(1).send(),
8520            )
8521            .await
8522            .unwrap();
8523            assert_eq!(response.status().as_u16(), 503, "the one retry is spent");
8524            assert_eq!(origin_hits.load(Ordering::SeqCst), 2);
8525            assert_eq!(target_hits.load(Ordering::SeqCst), 1);
8526        }
8527
8528        /// Entropy that always draws one value.
8529        #[derive(Debug)]
8530        struct FixedDraw(u64);
8531
8532        impl crate::entropy::Entropy for FixedDraw {
8533            fn next_u64(&self) -> u64 {
8534                self.0
8535            }
8536            fn fill_bytes(&self, dest: &mut [u8]) {
8537                dest.fill(0);
8538            }
8539        }
8540
8541        #[tokio::test]
8542        async fn a_cross_origin_redirect_under_a_deadline_strips_what_reqwest_strips() {
8543            let seen: Arc<std::sync::Mutex<Option<HeaderMap>>> =
8544                Arc::new(std::sync::Mutex::new(None));
8545            let slot = Arc::clone(&seen);
8546            let target = super::spawn(axum::Router::new().route(
8547                "/dst",
8548                axum::routing::get(move |headers: HeaderMap| {
8549                    let slot = Arc::clone(&slot);
8550                    async move {
8551                        *slot.lock().unwrap() = Some(headers);
8552                        "ok"
8553                    }
8554                }),
8555            ))
8556            .await;
8557            let target_port = target.port();
8558            let origin = super::spawn(axum::Router::new().route(
8559                "/",
8560                axum::routing::get(move || async move {
8561                    super::redirect_302(format!("http://127.0.0.1:{target_port}/dst"))
8562                }),
8563            ))
8564            .await;
8565            let response = with_deadline(
8566                Duration::from_secs(3),
8567                Client::new()
8568                    .get(format!("http://127.0.0.1:{}/", origin.port()))
8569                    .header("authorization", "secret")
8570                    .header("cookie", "a=b")
8571                    .header("cookie2", "c=d")
8572                    .header("proxy-authorization", "Basic zzz")
8573                    .header("www-authenticate", "Basic realm=x")
8574                    .send(),
8575            )
8576            .await
8577            .unwrap();
8578            assert_eq!(response.status().as_u16(), 200);
8579            let headers = seen.lock().unwrap().clone().expect("target reached");
8580            for name in [
8581                "authorization",
8582                "cookie",
8583                "cookie2",
8584                "proxy-authorization",
8585                "www-authenticate",
8586            ] {
8587                assert!(headers.get(name).is_none(), "{name} reached the target");
8588            }
8589        }
8590
8591        #[tokio::test]
8592        async fn a_retry_cancelled_during_its_backoff_gives_its_tokens_back() {
8593            let (url, hits) = counting(Some(502), &[]).await;
8594            let host = url_host(&url).unwrap();
8595            // The first retry waits 100 % 101 = 100 ms.
8596            let mut client = Client::new();
8597            client.entropy = Arc::new(FixedDraw(100));
8598            let budget = client.retry.budgets.clone().unwrap().for_host(&host);
8599            let cancelled = tokio::time::timeout(
8600                Duration::from_millis(50),
8601                client.get(&url).retries(3).send(),
8602            )
8603            .await;
8604            assert!(cancelled.is_err(), "cancelled in the backoff");
8605            assert_eq!(hits.load(Ordering::SeqCst), 1);
8606            let full = f64::from(RetryBudgetConfig::default().capacity);
8607            assert!(
8608                (budget.available() - full).abs() < f64::EPSILON,
8609                "no retry ran, so none is paid for: {}",
8610                budget.available()
8611            );
8612        }
8613
8614        #[tokio::test]
8615        #[allow(clippy::await_holding_lock)]
8616        async fn a_deadline_stop_on_the_custom_path_is_not_a_throttle_reject() {
8617            let _lock = crate::circuit_breaker::TEST_LOCK
8618                .lock()
8619                .unwrap_or_else(std::sync::PoisonError::into_inner);
8620            crate::circuit_breaker::global_registry().clear();
8621            let (url, _hits) = counting(None, &[]).await;
8622            let throttle = Arc::new(crate::admission::AdaptiveThrottle::new(
8623                2.0,
8624                Duration::from_secs(120),
8625            ));
8626            let client = Client {
8627                resilience_config: Some(Arc::new(crate::config::ResilienceConfig::default())),
8628                throttle: Some(Arc::clone(&throttle)),
8629                ..Client::new()
8630            };
8631            for _ in 0..10 {
8632                let result = with_deadline(
8633                    Duration::from_millis(30),
8634                    client.get(&url).no_redirect().breaker_scoped().send(),
8635                )
8636                .await;
8637                assert!(
8638                    matches!(result, Err(ClientError::DeadlineExceeded)),
8639                    "{result:?}"
8640                );
8641            }
8642            let host = throttle_host(&url).unwrap();
8643            assert!(
8644                throttle.reject_probability(&host, crate::time::ambient_instant()) < f64::EPSILON,
8645                "the caller's deadline is not the host's reject"
8646            );
8647            crate::circuit_breaker::global_registry().clear();
8648        }
8649
8650        #[tokio::test]
8651        async fn the_throttle_applies_under_a_deadline() {
8652            let (url, _hits) = counting(Some(503), &[]).await;
8653            let throttle = Arc::new(crate::admission::AdaptiveThrottle::new(
8654                2.0,
8655                Duration::from_secs(120),
8656            ));
8657            let client = Client {
8658                throttle: Some(Arc::clone(&throttle)),
8659                ..Client::new()
8660            };
8661            let mut throttled = 0;
8662            for _ in 0..100 {
8663                let result =
8664                    with_deadline(Duration::from_secs(5), client.get(&url).no_retry().send()).await;
8665                if matches!(result, Err(ClientError::ThrottledLocally { .. })) {
8666                    throttled += 1;
8667                }
8668            }
8669            assert!(
8670                throttled > 20,
8671                "the deadline path must use the throttle: {throttled}"
8672            );
8673        }
8674
8675        #[tokio::test]
8676        async fn each_retry_under_a_deadline_is_a_throttle_attempt() {
8677            use axum::response::IntoResponse;
8678            // Each call gets 503, 503, then 200.
8679            let hits = Arc::new(AtomicU32::new(0));
8680            let counter = Arc::clone(&hits);
8681            let upstream = super::spawn(axum::Router::new().route(
8682                "/x",
8683                axum::routing::get(move || {
8684                    let status = if counter.fetch_add(1, Ordering::SeqCst) % 3 < 2 {
8685                        axum::http::StatusCode::SERVICE_UNAVAILABLE
8686                    } else {
8687                        axum::http::StatusCode::OK
8688                    };
8689                    async move { status.into_response() }
8690                }),
8691            ))
8692            .await;
8693            let url = format!("http://127.0.0.1:{}/x", upstream.port());
8694            let throttle = Arc::new(crate::admission::AdaptiveThrottle::new(
8695                2.0,
8696                Duration::from_secs(120),
8697            ));
8698            // The highest draw: the throttle never rejects, so every call
8699            // runs its three attempts.
8700            let mut client = Client {
8701                throttle: Some(Arc::clone(&throttle)),
8702                ..Client::new()
8703            };
8704            client.entropy = Arc::new(FixedDraw(u64::MAX));
8705            for _ in 0..10 {
8706                let response =
8707                    with_deadline(Duration::from_secs(4), client.get(&url).retries(2).send())
8708                        .await
8709                        .unwrap();
8710                assert_eq!(response.status().as_u16(), 200);
8711            }
8712            assert_eq!(hits.load(Ordering::SeqCst), 30);
8713            // 30 attempts, 10 accepts: below 1/K, as on the path without a
8714            // deadline. One count per call would hide the 503s.
8715            let host = throttle_host(&url).unwrap();
8716            assert!(
8717                throttle.reject_probability(&host, crate::time::ambient_instant()) > 0.3,
8718                "each attempt counts"
8719            );
8720        }
8721
8722        #[tokio::test]
8723        async fn a_retry_sends_what_is_left_of_the_callers_deadline_header() {
8724            use axum::response::IntoResponse;
8725            // The first attempt takes 60 ms and fails; the retry works.
8726            let seen: Arc<std::sync::Mutex<Vec<u64>>> = Arc::default();
8727            let slot = Arc::clone(&seen);
8728            let upstream = super::spawn(axum::Router::new().route(
8729                "/x",
8730                axum::routing::get(move |headers: HeaderMap| {
8731                    let slot = Arc::clone(&slot);
8732                    async move {
8733                        let value = headers
8734                            .get(DEADLINE_HEADER)
8735                            .and_then(|v| v.to_str().ok()?.parse().ok())
8736                            .unwrap_or(u64::MAX);
8737                        let first = {
8738                            let mut seen = slot.lock().unwrap();
8739                            seen.push(value);
8740                            seen.len() == 1
8741                        };
8742                        if first {
8743                            tokio::time::sleep(Duration::from_millis(60)).await;
8744                            axum::http::StatusCode::BAD_GATEWAY.into_response()
8745                        } else {
8746                            axum::http::StatusCode::OK.into_response()
8747                        }
8748                    }
8749                }),
8750            ))
8751            .await;
8752            // A zero draw: the retry starts at once.
8753            let mut client = Client::new();
8754            client.entropy = Arc::new(FixedDraw(0));
8755            let response = with_deadline(
8756                Duration::from_secs(4),
8757                client
8758                    .get(format!("http://127.0.0.1:{}/x", upstream.port()))
8759                    .header(DEADLINE_HEADER, "200")
8760                    .retries(1)
8761                    .send(),
8762            )
8763            .await
8764            .unwrap();
8765            assert_eq!(response.status().as_u16(), 200);
8766            let seen = seen.lock().unwrap().clone();
8767            assert_eq!(seen.len(), 2);
8768            assert!(seen[0] <= 200, "{seen:?}");
8769            // 60 ms of the caller's 200 ms are gone.
8770            assert!(
8771                seen[1] <= 140,
8772                "the retry re-extended the deadline: {seen:?}"
8773            );
8774        }
8775
8776        #[tokio::test]
8777        async fn a_redirect_chain_under_a_deadline_keeps_the_client_timeout() {
8778            use axum::response::IntoResponse;
8779            // Each hop takes 200 ms: 400 ms in all, over the 300 ms timeout.
8780            let app = axum::Router::new()
8781                .route(
8782                    "/a",
8783                    axum::routing::get(|| async {
8784                        tokio::time::sleep(Duration::from_millis(200)).await;
8785                        axum::response::Redirect::temporary("/b").into_response()
8786                    }),
8787                )
8788                .route(
8789                    "/b",
8790                    axum::routing::get(|| async {
8791                        tokio::time::sleep(Duration::from_millis(200)).await;
8792                        "ok"
8793                    }),
8794                );
8795            let upstream = super::spawn(app).await;
8796            let mut client = Client::new();
8797            client.retry_policy.request_timeout = Some(Duration::from_millis(300));
8798            let result = with_deadline(
8799                Duration::from_secs(4),
8800                client
8801                    .get(format!("http://127.0.0.1:{}/a", upstream.port()))
8802                    .no_retry()
8803                    .send(),
8804            )
8805            .await;
8806            // Without a deadline, reqwest's one timeout covers the redirects.
8807            assert!(
8808                matches!(&result, Err(ClientError::Request(e)) if e.is_timeout()),
8809                "the chain must not get a new timeout per hop: {result:?}"
8810            );
8811        }
8812
8813        #[tokio::test]
8814        async fn without_a_deadline_a_retry_sends_what_is_left_of_the_callers_header() {
8815            use axum::response::IntoResponse;
8816            // The first attempt takes 60 ms and fails; the retry works.
8817            let seen: Arc<std::sync::Mutex<Vec<u64>>> = Arc::default();
8818            let slot = Arc::clone(&seen);
8819            let upstream = super::spawn(axum::Router::new().route(
8820                "/x",
8821                axum::routing::get(move |headers: HeaderMap| {
8822                    let slot = Arc::clone(&slot);
8823                    async move {
8824                        let first = {
8825                            let mut seen = slot.lock().unwrap();
8826                            seen.push(header_millis(&headers));
8827                            seen.len() == 1
8828                        };
8829                        if first {
8830                            tokio::time::sleep(Duration::from_millis(60)).await;
8831                            axum::http::StatusCode::BAD_GATEWAY.into_response()
8832                        } else {
8833                            axum::http::StatusCode::OK.into_response()
8834                        }
8835                    }
8836                }),
8837            ))
8838            .await;
8839            let mut client = Client::new();
8840            client.entropy = Arc::new(FixedDraw(0));
8841            // No task deadline: only the caller's header.
8842            let response = client
8843                .get(format!("http://127.0.0.1:{}/x", upstream.port()))
8844                .header(DEADLINE_HEADER, "200")
8845                .retries(1)
8846                .send()
8847                .await
8848                .unwrap();
8849            assert_eq!(response.status().as_u16(), 200);
8850            let seen = seen.lock().unwrap().clone();
8851            assert_eq!(seen[0], 200, "{seen:?}");
8852            assert!(seen[1] <= 140, "the retry re-extended the header: {seen:?}");
8853        }
8854
8855        #[tokio::test]
8856        async fn without_a_deadline_a_redirect_hop_sends_what_is_left_of_the_callers_header() {
8857            use axum::response::IntoResponse;
8858            let seen: Arc<std::sync::Mutex<Vec<u64>>> = Arc::default();
8859            let (at_a, at_b) = (Arc::clone(&seen), Arc::clone(&seen));
8860            let app = axum::Router::new()
8861                .route(
8862                    "/a",
8863                    axum::routing::get(move |headers: HeaderMap| async move {
8864                        at_a.lock().unwrap().push(header_millis(&headers));
8865                        tokio::time::sleep(Duration::from_millis(60)).await;
8866                        axum::response::Redirect::temporary("/b").into_response()
8867                    }),
8868                )
8869                .route(
8870                    "/b",
8871                    axum::routing::get(move |headers: HeaderMap| async move {
8872                        at_b.lock().unwrap().push(header_millis(&headers));
8873                        "ok"
8874                    }),
8875                );
8876            let upstream = super::spawn(app).await;
8877            let response = Client::new()
8878                .get(format!("http://127.0.0.1:{}/a", upstream.port()))
8879                .header(DEADLINE_HEADER, "200")
8880                .send()
8881                .await
8882                .unwrap();
8883            assert_eq!(response.status().as_u16(), 200);
8884            let seen = seen.lock().unwrap().clone();
8885            assert_eq!(seen.len(), 2, "{seen:?}");
8886            assert_eq!(seen[0], 200, "{seen:?}");
8887            assert!(seen[1] <= 140, "the hop re-extended the header: {seen:?}");
8888        }
8889
8890        /// The [`DEADLINE_HEADER`] value of a request, or `u64::MAX`.
8891        fn header_millis(headers: &HeaderMap) -> u64 {
8892            headers
8893                .get(DEADLINE_HEADER)
8894                .and_then(|v| v.to_str().ok()?.parse().ok())
8895                .unwrap_or(u64::MAX)
8896        }
8897
8898        #[tokio::test]
8899        async fn a_redirect_hop_continues_the_chain_backoff() {
8900            use axum::response::IntoResponse;
8901            let origin_hits = Arc::new(AtomicU32::new(0));
8902            let target_hits = Arc::new(AtomicU32::new(0));
8903            let (origin, target) = (Arc::clone(&origin_hits), Arc::clone(&target_hits));
8904            // Each hop fails once, then works.
8905            let app = axum::Router::new()
8906                .route(
8907                    "/r",
8908                    axum::routing::get(move || async move {
8909                        if origin.fetch_add(1, Ordering::SeqCst) == 0 {
8910                            axum::http::StatusCode::BAD_GATEWAY.into_response()
8911                        } else {
8912                            axum::response::Redirect::temporary("/t").into_response()
8913                        }
8914                    }),
8915                )
8916                .route(
8917                    "/t",
8918                    axum::routing::get(move || async move {
8919                        if target.fetch_add(1, Ordering::SeqCst) == 0 {
8920                            axum::http::StatusCode::BAD_GATEWAY.into_response()
8921                        } else {
8922                            "done".into_response()
8923                        }
8924                    }),
8925                );
8926            let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8927            let port = listener.local_addr().unwrap().port();
8928            tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
8929            let url = format!("http://127.0.0.1:{port}/r");
8930            // 20_099 = 101 × 199: the first retry waits 20_099 % 101 = 0 ms,
8931            // the second 20_099 % 201 = 200 ms.
8932            let mut client = Client::new();
8933            client.entropy = Arc::new(FixedDraw(20_099));
8934            let start = std::time::Instant::now();
8935            let response =
8936                with_deadline(Duration::from_secs(5), client.get(&url).retries(2).send())
8937                    .await
8938                    .unwrap();
8939            assert_eq!(response.status().as_u16(), 200);
8940            assert_eq!(origin_hits.load(Ordering::SeqCst), 2);
8941            assert_eq!(target_hits.load(Ordering::SeqCst), 2);
8942            assert!(
8943                start.elapsed() >= Duration::from_millis(200),
8944                "the hop's retry is the chain's second: {:?}",
8945                start.elapsed()
8946            );
8947        }
8948
8949        #[tokio::test]
8950        async fn a_post_redirected_to_get_under_a_deadline_gets_no_retry() {
8951            use axum::response::IntoResponse;
8952            let target_hits = Arc::new(AtomicU32::new(0));
8953            let target = Arc::clone(&target_hits);
8954            let app = axum::Router::new()
8955                .route(
8956                    "/r",
8957                    axum::routing::post(|| async { axum::response::Redirect::to("/t") }),
8958                )
8959                .route(
8960                    "/t",
8961                    axum::routing::get(move || async move {
8962                        if target.fetch_add(1, Ordering::SeqCst) == 0 {
8963                            axum::http::StatusCode::SERVICE_UNAVAILABLE.into_response()
8964                        } else {
8965                            "done".into_response()
8966                        }
8967                    }),
8968                );
8969            let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8970            let port = listener.local_addr().unwrap().port();
8971            tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
8972            let url = format!("http://127.0.0.1:{port}/r");
8973            let response = with_deadline(
8974                Duration::from_secs(3),
8975                Client::new().post(&url).retries(3).send(),
8976            )
8977            .await
8978            .unwrap();
8979            assert_eq!(response.status().as_u16(), 503, "as without a deadline");
8980            assert_eq!(target_hits.load(Ordering::SeqCst), 1);
8981        }
8982
8983        #[tokio::test]
8984        async fn a_backoff_past_the_hop_deadline_returns_the_response() {
8985            let (url, hits) = counting(Some(502), &[]).await;
8986            let policy = RetryPolicy {
8987                max_retries: 20,
8988                ..RetryPolicy::default()
8989            };
8990            let gate = RetryGate::with_deadline(None, None, None, true);
8991            // Warm the connection. A cold first connect on a slow runner
8992            // must not use up the hop deadline.
8993            let client = reqwest::Client::new();
8994            let _ = client.get(&url).send().await;
8995            let result = send_one(
8996                &client,
8997                &Method::GET,
8998                &url,
8999                &HeaderMap::new(),
9000                None,
9001                &policy,
9002                // A fixed draw gives the backoffs 97, 199, 400 and 400 ms. The
9003                // fourth passes the 1 s deadline, and the last retry starts
9004                // with about 300 ms left. A random draw can start a last retry
9005                // with only `MIN_ATTEMPT` left, and a Windows timer can use
9006                // all of it.
9007                &FixedDraw(400),
9008                false,
9009                Some(crate::time::ambient_instant() + Duration::from_secs(1)),
9010                false,
9011                &gate,
9012                false,
9013                None,
9014                None,
9015            )
9016            .await;
9017            assert_eq!(
9018                result.map(|response| response.status().as_u16()).ok(),
9019                Some(502),
9020                "the last response, not a deadline error"
9021            );
9022            // The warm-up request, the first attempt and at least one retry.
9023            assert!(hits.load(Ordering::SeqCst) >= 3, "no retry ran");
9024        }
9025
9026        #[tokio::test]
9027        async fn a_transport_retry_past_the_hop_deadline_is_not_charged() {
9028            let app = axum::Router::new().route(
9029                "/hang",
9030                axum::routing::get(|| async {
9031                    std::future::pending::<()>().await;
9032                    ""
9033                }),
9034            );
9035            let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
9036            let port = listener.local_addr().unwrap().port();
9037            tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
9038            let host = format!("127.0.0.1:{port}");
9039            let budgets = Arc::new(RetryBudgets::new(&RetryBudgetConfig::default()));
9040            let gate =
9041                RetryGate::with_deadline(None, Some(Arc::clone(&budgets)), Some(&host), true);
9042            // The attempt uses up the hop: no retry can follow it.
9043            let result = send_one(
9044                &reqwest::Client::new(),
9045                &Method::GET,
9046                &format!("http://{host}/hang"),
9047                &HeaderMap::new(),
9048                None,
9049                &RetryPolicy::default(),
9050                &crate::entropy::SeededEntropy::new(1),
9051                false,
9052                Some(crate::time::ambient_instant() + Duration::from_millis(200)),
9053                false,
9054                &gate,
9055                false,
9056                None,
9057                None,
9058            )
9059            .await;
9060            assert!(result.is_err(), "{result:?}");
9061            let full = f64::from(RetryBudgetConfig::default().capacity);
9062            let available = budgets.for_host(&host).available();
9063            assert!(
9064                (full - available).abs() < f64::EPSILON,
9065                "no tokens for a retry that cannot run: {available}"
9066            );
9067        }
9068
9069        #[tokio::test]
9070        async fn a_retry_needs_a_minimum_attempt_before_the_hop_deadline() {
9071            let (url, hits) = counting(Some(502), &[]).await;
9072            let gate = RetryGate::with_deadline(None, None, None, true);
9073            // The 100 ms backoff ends before the hop deadline, but leaves less
9074            // than `MIN_ATTEMPT` for the retry.
9075            let result = send_one(
9076                &reqwest::Client::new(),
9077                &Method::GET,
9078                &url,
9079                &HeaderMap::new(),
9080                None,
9081                &RetryPolicy::default(),
9082                &FixedDraw(100),
9083                false,
9084                Some(crate::time::ambient_instant() + Duration::from_millis(105)),
9085                false,
9086                &gate,
9087                false,
9088                None,
9089                None,
9090            )
9091            .await;
9092            assert_eq!(
9093                result.map(|response| response.status().as_u16()).ok(),
9094                Some(502),
9095                "the response, not a retry that cannot run"
9096            );
9097            assert_eq!(hits.load(Ordering::SeqCst), 1);
9098        }
9099
9100        #[tokio::test]
9101        async fn an_origin_failure_after_a_redirect_charges_the_origin() {
9102            // The first request redirects to a dead port; later ones hang.
9103            let closed = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
9104            let dead_port = closed.local_addr().unwrap().port();
9105            drop(closed);
9106            let target = format!("http://127.0.0.1:{dead_port}/t");
9107            let hits = Arc::new(AtomicU32::new(0));
9108            let counter = Arc::clone(&hits);
9109            let app = axum::Router::new().route(
9110                "/r",
9111                axum::routing::get(move || {
9112                    let target = target.clone();
9113                    let first = counter.fetch_add(1, Ordering::SeqCst) == 0;
9114                    async move {
9115                        if !first {
9116                            std::future::pending::<()>().await;
9117                        }
9118                        axum::response::Redirect::temporary(&target)
9119                    }
9120                }),
9121            );
9122            let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
9123            let origin_port = listener.local_addr().unwrap().port();
9124            tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
9125
9126            let client = Client::with_timeout(Duration::from_millis(300));
9127            let budgets = client.retry.budgets.clone().unwrap();
9128            let result = client
9129                .get(format!("http://127.0.0.1:{origin_port}/r"))
9130                .retries(2)
9131                .send()
9132                .await;
9133            assert!(result.is_err(), "{result:?}");
9134            let full = f64::from(RetryBudgetConfig::default().capacity);
9135            let cost = f64::from(RetryBudgetConfig::default().transient_cost);
9136            let origin = budgets.for_host(&format!("127.0.0.1:{origin_port}"));
9137            let dead = budgets.for_host(&format!("127.0.0.1:{dead_port}"));
9138            assert!(
9139                (full - dead.available() - cost).abs() < f64::EPSILON,
9140                "the dead target pays its one retry: {}",
9141                dead.available()
9142            );
9143            assert!(
9144                (full - origin.available() - cost).abs() < f64::EPSILON,
9145                "the origin pays the retry after its own timeout: {}",
9146                origin.available()
9147            );
9148        }
9149
9150        #[tokio::test]
9151        async fn a_redirect_host_is_refilled_when_the_final_body_fails() {
9152            // The target's body fails after the head.
9153            let target_app = axum::Router::new().route(
9154                "/t",
9155                axum::routing::get(|| async {
9156                    axum::body::Body::from_stream(futures::stream::once(async {
9157                        Err::<Bytes, _>(std::io::Error::other("broken body"))
9158                    }))
9159                }),
9160            );
9161            let target_listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
9162            let target_port = target_listener.local_addr().unwrap().port();
9163            tokio::spawn(async move { axum::serve(target_listener, target_app).await.unwrap() });
9164            let target = format!("http://127.0.0.1:{target_port}/t");
9165            let origin_app = axum::Router::new().route(
9166                "/r",
9167                axum::routing::get(move || {
9168                    let target = target.clone();
9169                    async move { axum::response::Redirect::temporary(&target) }
9170                }),
9171            );
9172            let origin_listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
9173            let origin_port = origin_listener.local_addr().unwrap().port();
9174            tokio::spawn(async move { axum::serve(origin_listener, origin_app).await.unwrap() });
9175
9176            let client = Client::new();
9177            let budgets = client.retry.budgets.clone().unwrap();
9178            let drained = budgets.for_host(&format!("127.0.0.1:{target_port}"));
9179            while drained.try_acquire(RetryKind::Transient) {}
9180            let before = drained.available();
9181            let result = client
9182                .get(format!("http://127.0.0.1:{origin_port}/r"))
9183                .send()
9184                .await;
9185            assert!(result.is_err(), "{result:?}");
9186            assert!(drained.available() > before, "the target got its refill");
9187        }
9188
9189        #[tokio::test]
9190        async fn ten_redirects_are_followed_with_and_without_a_deadline() {
9191            // `/r/{n}` redirects to `/r/{n - 1}`; `/r/0` answers.
9192            let app = axum::Router::new().route(
9193                "/r/{n}",
9194                axum::routing::get(
9195                    |axum::extract::Path(n): axum::extract::Path<u32>| async move {
9196                        use axum::response::IntoResponse;
9197                        if n == 0 {
9198                            "done".into_response()
9199                        } else {
9200                            axum::response::Redirect::temporary(&format!("/r/{}", n - 1))
9201                                .into_response()
9202                        }
9203                    },
9204                ),
9205            );
9206            let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
9207            let port = listener.local_addr().unwrap().port();
9208            tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
9209            let ten = format!("http://127.0.0.1:{port}/r/10");
9210            let eleven = format!("http://127.0.0.1:{port}/r/11");
9211
9212            let plain = Client::new().get(&ten).send().await.unwrap();
9213            assert_eq!(plain.status().as_u16(), 200, "as reqwest's own limit");
9214            let timed = with_deadline(Duration::from_secs(3), Client::new().get(&ten).send())
9215                .await
9216                .unwrap();
9217            assert_eq!(timed.status().as_u16(), 200);
9218
9219            assert!(Client::new().get(&eleven).send().await.is_err());
9220            assert!(
9221                with_deadline(Duration::from_secs(3), Client::new().get(&eleven).send())
9222                    .await
9223                    .is_err()
9224            );
9225        }
9226
9227        #[tokio::test]
9228        async fn a_caller_deadline_header_is_capped_at_the_time_left() {
9229            let seen = Arc::new(Mutex::new(Vec::new()));
9230            let record = Arc::clone(&seen);
9231            let app = axum::Router::new().route(
9232                "/x",
9233                axum::routing::get(move |headers: HeaderMap| async move {
9234                    let values: Vec<String> = headers
9235                        .get_all(DEADLINE_HEADER)
9236                        .iter()
9237                        .map(|value| value.to_str().unwrap().to_owned())
9238                        .collect();
9239                    record.lock().unwrap().push(values);
9240                }),
9241            );
9242            let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
9243            let port = listener.local_addr().unwrap().port();
9244            tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
9245            let url = format!("http://127.0.0.1:{port}/x");
9246
9247            // A larger caller value is cut to the time left.
9248            let send = Client::new()
9249                .get(&url)
9250                .header(DEADLINE_HEADER, "5000")
9251                .send();
9252            with_deadline(Duration::from_secs(1), send).await.unwrap();
9253            // A smaller caller value is kept.
9254            let send = Client::new()
9255                .get(&url)
9256                .header(DEADLINE_HEADER, "200")
9257                .send();
9258            with_deadline(Duration::from_secs(3), send).await.unwrap();
9259            // With no deadline, the caller's value goes as is.
9260            Client::new()
9261                .get(&url)
9262                .header(DEADLINE_HEADER, "5000")
9263                .send()
9264                .await
9265                .unwrap();
9266
9267            let seen = seen.lock().unwrap().clone();
9268            assert_eq!(seen.len(), 3);
9269            assert_eq!(seen[0].len(), 1, "one header: {seen:?}");
9270            assert!(seen[0][0].parse::<u64>().unwrap() <= 1_000, "{seen:?}");
9271            assert_eq!(seen[1], vec!["200".to_owned()]);
9272            assert_eq!(seen[2], vec!["5000".to_owned()]);
9273        }
9274
9275        #[tokio::test]
9276        async fn plain_path_follows_a_redirect_without_a_deadline() {
9277            let (url, seen) = redirecting(Duration::ZERO).await;
9278            let response = Client::new().get(&url).send().await.unwrap();
9279            assert_eq!(response.status().as_u16(), 200);
9280            assert_eq!(seen.lock().unwrap().len(), 2);
9281        }
9282
9283        #[tokio::test]
9284        #[allow(clippy::await_holding_lock)]
9285        async fn plain_path_stops_a_hanging_call_at_the_deadline() {
9286            let _lock = lock();
9287            let (url, hits) = counting(None, &[]).await;
9288            let result =
9289                with_deadline(Duration::from_millis(300), Client::new().get(&url).send()).await;
9290            assert!(
9291                matches!(result, Err(ClientError::DeadlineExceeded)),
9292                "{result:?}"
9293            );
9294            assert_eq!(
9295                hits.load(Ordering::SeqCst),
9296                1,
9297                "no retry after the deadline"
9298            );
9299        }
9300
9301        #[tokio::test]
9302        #[allow(clippy::await_holding_lock)]
9303        async fn plain_path_does_not_start_a_retry_that_cannot_fit() {
9304            let _lock = lock();
9305            let (url, hits) = counting(Some(503), &[]).await;
9306            // 50 retries with jittered backoff need far more than 700 ms; the
9307            // deadline stops them, and the last 503 comes back.
9308            let response = with_deadline(
9309                Duration::from_millis(700),
9310                Client::new().get(&url).retries(50).send(),
9311            )
9312            .await
9313            .unwrap();
9314            assert_eq!(response.status().as_u16(), 503);
9315            let made = hits.load(Ordering::SeqCst);
9316            assert!(
9317                (2..51).contains(&made),
9318                "retries stop at the deadline: {made}"
9319            );
9320        }
9321
9322        #[tokio::test]
9323        #[allow(clippy::await_holding_lock)]
9324        async fn plain_path_skips_a_retry_after_wait_past_the_deadline() {
9325            let _lock = lock();
9326            let (url, hits) = counting(Some(429), &[("retry-after", "10")]).await;
9327            let response = with_deadline(Duration::from_secs(2), Client::new().get(&url).send())
9328                .await
9329                .unwrap();
9330            assert_eq!(
9331                response.status().as_u16(),
9332                429,
9333                "the 429 comes back at once"
9334            );
9335            assert_eq!(hits.load(Ordering::SeqCst), 1);
9336        }
9337
9338        #[tokio::test]
9339        #[allow(clippy::await_holding_lock)]
9340        async fn plain_path_obeys_the_retry_budget() {
9341            let _lock = lock();
9342            let (url, hits) = counting(Some(503), &[]).await;
9343            let client = one_retry_client();
9344            let first = client.get(&url).send().await.unwrap();
9345            assert_eq!(first.status().as_u16(), 503);
9346            assert_eq!(hits.load(Ordering::SeqCst), 2, "one retry from the budget");
9347            let second = client.get(&url).send().await.unwrap();
9348            assert_eq!(second.status().as_u16(), 503);
9349            assert_eq!(
9350                hits.load(Ordering::SeqCst),
9351                3,
9352                "the empty budget blocks retries"
9353            );
9354        }
9355
9356        #[tokio::test]
9357        async fn custom_path_obeys_the_deadline_and_the_budget() {
9358            let (url, hits) = counting(Some(503), &[]).await;
9359            let response = with_deadline(
9360                Duration::from_millis(700),
9361                Client::new().get(&url).retries(50).no_redirect().send(),
9362            )
9363            .await
9364            .unwrap();
9365            assert_eq!(response.status().as_u16(), 503);
9366            let made = hits.load(Ordering::SeqCst);
9367            assert!(
9368                (2..51).contains(&made),
9369                "retries stop at the deadline: {made}"
9370            );
9371
9372            let client = one_retry_client();
9373            client.get(&url).no_redirect().send().await.unwrap();
9374            assert_eq!(
9375                hits.load(Ordering::SeqCst),
9376                made + 2,
9377                "one retry from the budget"
9378            );
9379            client.get(&url).no_redirect().send().await.unwrap();
9380            assert_eq!(
9381                hits.load(Ordering::SeqCst),
9382                made + 3,
9383                "the empty budget blocks retries"
9384            );
9385        }
9386
9387        #[tokio::test]
9388        async fn custom_path_reports_a_deadline_timeout_as_deadline_exceeded() {
9389            let (url, hits) = counting(None, &[]).await;
9390            let result = with_deadline(
9391                Duration::from_millis(300),
9392                Client::new().get(&url).no_redirect().send(),
9393            )
9394            .await;
9395            assert!(
9396                matches!(result, Err(ClientError::DeadlineExceeded)),
9397                "{result:?}"
9398            );
9399            assert_eq!(
9400                hits.load(Ordering::SeqCst),
9401                1,
9402                "no retry after the deadline"
9403            );
9404        }
9405
9406        /// A server whose response head arrives at once and whose body never
9407        /// ends.
9408        async fn stalled_body() -> String {
9409            let app = axum::Router::new().route(
9410                "/x",
9411                axum::routing::get(|| async {
9412                    let stream = futures::stream::pending::<Result<bytes::Bytes, std::io::Error>>();
9413                    axum::response::Response::new(axum::body::Body::from_stream(stream))
9414                }),
9415            );
9416            let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
9417            let addr = listener.local_addr().unwrap();
9418            tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
9419            format!("http://127.0.0.1:{}/x", addr.port())
9420        }
9421
9422        #[tokio::test]
9423        #[allow(clippy::await_holding_lock)]
9424        async fn a_body_stalled_past_the_deadline_is_deadline_exceeded() {
9425            let _lock = lock();
9426            let url = stalled_body().await;
9427            let plain =
9428                with_deadline(Duration::from_millis(300), Client::new().get(&url).send()).await;
9429            assert!(
9430                matches!(plain, Err(ClientError::DeadlineExceeded)),
9431                "{plain:?}"
9432            );
9433            let custom = with_deadline(
9434                Duration::from_millis(300),
9435                Client::new().get(&url).no_redirect().send(),
9436            )
9437            .await;
9438            assert!(
9439                matches!(custom, Err(ClientError::DeadlineExceeded)),
9440                "{custom:?}"
9441            );
9442        }
9443
9444        #[tokio::test]
9445        async fn ssrf_safe_path_fails_at_once_when_the_deadline_has_passed() {
9446            let result = Deadline::at(tokio::time::Instant::now())
9447                .scope(Client::new().get_ssrf_safe("https://example.com/").send())
9448                .await;
9449            assert!(
9450                matches!(result, Err(ClientError::DeadlineExceeded)),
9451                "{result:?}"
9452            );
9453        }
9454
9455        #[test]
9456        fn a_redirect_hop_uses_the_budget_of_its_own_host() {
9457            let budgets = Arc::new(RetryBudgets::new(&RetryBudgetConfig::default()));
9458            let origin = RetryGate::with_deadline(
9459                None,
9460                Some(Arc::clone(&budgets)),
9461                Some("origin:443"),
9462                true,
9463            );
9464            let hop = origin.for_hop("https://target/x", &mut std::collections::HashSet::new());
9465            assert!(Arc::ptr_eq(
9466                hop.budget.as_ref().unwrap(),
9467                &budgets.for_host("target:443")
9468            ));
9469            assert!(!Arc::ptr_eq(
9470                hop.budget.as_ref().unwrap(),
9471                origin.budget.as_ref().unwrap()
9472            ));
9473        }
9474
9475        #[test]
9476        fn a_redirect_chain_refills_each_host_once() {
9477            let budgets = Arc::new(RetryBudgets::new(&RetryBudgetConfig::default()));
9478            let origin_budget = budgets.for_host("a:443");
9479            let target_budget = budgets.for_host("b:443");
9480            while origin_budget.try_acquire(RetryKind::Transient) {}
9481            while target_budget.try_acquire(RetryKind::Transient) {}
9482            let origin =
9483                RetryGate::with_deadline(None, Some(Arc::clone(&budgets)), Some("a:443"), true);
9484            let mut refilled = std::collections::HashSet::from(["a:443".to_owned()]);
9485            // a -> b -> a -> b
9486            origin.check().unwrap();
9487            let after_first = origin_budget.available();
9488            let before_b = target_budget.available();
9489            origin
9490                .for_hop("https://b/1", &mut refilled)
9491                .check()
9492                .unwrap();
9493            let after_b = target_budget.available();
9494            assert!(after_b > before_b, "the first visit to b refills it");
9495            origin
9496                .for_hop("https://a/2", &mut refilled)
9497                .check()
9498                .unwrap();
9499            origin
9500                .for_hop("https://b/3", &mut refilled)
9501                .check()
9502                .unwrap();
9503            assert!(
9504                (origin_budget.available() - after_first).abs() < f64::EPSILON,
9505                "the origin is not credited again"
9506            );
9507            assert!(
9508                (target_budget.available() - after_b).abs() < f64::EPSILON,
9509                "the target is credited once"
9510            );
9511        }
9512
9513        #[test]
9514        fn rekey_moves_to_the_budget_of_the_answering_host() {
9515            let budgets = Arc::new(RetryBudgets::new(&RetryBudgetConfig::default()));
9516            let mut gate = RetryGate::with_deadline(
9517                None,
9518                Some(Arc::clone(&budgets)),
9519                Some("origin:443"),
9520                true,
9521            );
9522            let origin = Arc::clone(gate.budget.as_ref().unwrap());
9523            gate.rekey("https://origin/x");
9524            assert!(
9525                Arc::ptr_eq(gate.budget.as_ref().unwrap(), &origin),
9526                "same host"
9527            );
9528            gate.rekey("https://target/x");
9529            assert!(Arc::ptr_eq(
9530                gate.budget.as_ref().unwrap(),
9531                &budgets.for_host("target:443")
9532            ));
9533        }
9534
9535        #[test]
9536        fn a_redirected_answer_refills_the_destination_budget() {
9537            let budgets = Arc::new(RetryBudgets::new(&RetryBudgetConfig::default()));
9538            let target = budgets.for_host("target:443");
9539            while target.try_acquire(RetryKind::Transient) {}
9540            let gate = RetryGate::with_deadline(
9541                None,
9542                Some(Arc::clone(&budgets)),
9543                Some("origin:443"),
9544                true,
9545            );
9546            let origin = Arc::clone(gate.budget.as_ref().unwrap());
9547            let (before, origin_before) = (target.available(), origin.available());
9548
9549            gate.record_destination("https://origin/x");
9550            assert!(
9551                (target.available() - before).abs() < f64::EPSILON,
9552                "same host"
9553            );
9554
9555            gate.record_destination("https://target/x");
9556            assert!(target.available() > before, "the destination is refilled");
9557            assert!(
9558                (origin.available() - origin_before).abs() < f64::EPSILON,
9559                "the origin budget is unchanged"
9560            );
9561            assert!(
9562                Arc::ptr_eq(gate.budget.as_ref().unwrap(), &origin),
9563                "no rekey"
9564            );
9565        }
9566
9567        #[tokio::test(start_paused = true)]
9568        async fn an_expired_deadline_reclassifies_a_hop_error() {
9569            let gate = Deadline::at(tokio::time::Instant::now())
9570                .scope(async { RetryGate::start(None, None, true) })
9571                .await;
9572            let error = gate.classify(ClientError::InvalidUrl("dns stalled".into()));
9573            assert!(matches!(error, ClientError::DeadlineExceeded));
9574            let open = RetryGate::start(None, None, true);
9575            let error = open.classify(ClientError::InvalidUrl("dns".into()));
9576            assert!(matches!(error, ClientError::InvalidUrl(_)));
9577        }
9578
9579        #[tokio::test]
9580        async fn a_rejected_ssrf_safe_host_is_not_refilled() {
9581            let client = Client::new();
9582            let budget = client
9583                .retry
9584                .budgets
9585                .clone()
9586                .unwrap()
9587                .for_host("127.0.0.1:80");
9588            while budget.try_acquire(RetryKind::Transient) {}
9589            let empty = budget.available();
9590            // Loopback is refused before any HTTP attempt.
9591            let result = client.get_ssrf_safe("http://127.0.0.1/").send().await;
9592            assert!(result.is_err(), "{result:?}");
9593            assert!(
9594                (budget.available() - empty).abs() < f64::EPSILON,
9595                "no refill without an attempt: {}",
9596                budget.available()
9597            );
9598        }
9599
9600        #[tokio::test(start_paused = true)]
9601        async fn a_followed_redirect_host_is_refilled_after_the_deadline() {
9602            let budgets = Arc::new(RetryBudgets::new(&RetryBudgetConfig::default()));
9603            let target = budgets.for_host("t:80");
9604            while target.try_acquire(RetryKind::Transient) {}
9605            let empty = target.available();
9606            let gate = RetryGate::with_deadline(
9607                Some(Deadline::after(Duration::from_millis(1))),
9608                Some(budgets),
9609                Some("o:80"),
9610                true,
9611            );
9612            // The redirect was followed, then the deadline passed.
9613            tokio::time::advance(Duration::from_millis(5)).await;
9614            let mut seen = std::collections::HashSet::from(["o:80".to_owned()]);
9615            gate.record_destinations(&["http://t/".to_owned()], &mut seen);
9616            assert!(
9617                target.available() > empty,
9618                "the target's attempt started, so it is refilled"
9619            );
9620        }
9621
9622        #[tokio::test(start_paused = true)]
9623        async fn a_request_past_its_deadline_does_not_refill_the_budget() {
9624            let budgets = Arc::new(RetryBudgets::new(&RetryBudgetConfig::default()));
9625            let budget = budgets.for_host("h:80");
9626            while budget.try_acquire(RetryKind::Transient) {}
9627            let empty = budget.available();
9628            let late = RetryGate::with_deadline(
9629                Some(Deadline::at(tokio::time::Instant::now())),
9630                Some(Arc::clone(&budgets)),
9631                Some("h:80"),
9632                true,
9633            );
9634            assert!(late.check().is_err());
9635            assert!(
9636                (budget.available() - empty).abs() < f64::EPSILON,
9637                "no refill"
9638            );
9639
9640            // Live when built, expired before the first attempt.
9641            let short = RetryGate::with_deadline(
9642                Some(Deadline::after(Duration::from_millis(1))),
9643                Some(Arc::clone(&budgets)),
9644                Some("h:80"),
9645                true,
9646            );
9647            tokio::time::advance(Duration::from_millis(5)).await;
9648            assert!(short.check().is_err());
9649            assert!(
9650                (budget.available() - empty).abs() < f64::EPSILON,
9651                "no refill without an attempt"
9652            );
9653
9654            let live = RetryGate::with_deadline(None, Some(budgets), Some("h:80"), true);
9655            assert!(
9656                (budget.available() - empty).abs() < f64::EPSILON,
9657                "not when built"
9658            );
9659            live.check().unwrap();
9660            let once = budget.available();
9661            assert!(once > empty, "a live request refills at its first attempt");
9662            live.check().unwrap();
9663            assert!((budget.available() - once).abs() < f64::EPSILON, "once");
9664        }
9665
9666        #[test]
9667        fn budget_keys_keep_the_port() {
9668            assert_eq!(url_host("http://svc:8001/a").as_deref(), Some("svc:8001"));
9669            assert_eq!(url_host("https://svc/a").as_deref(), Some("svc:443"));
9670            assert_eq!(url_host("http://svc/a").as_deref(), Some("svc:80"));
9671            assert_eq!(url_host("/relative"), None);
9672        }
9673
9674        #[test]
9675        fn a_retry_ending_in_a_client_or_server_error_keeps_its_tokens_spent() {
9676            let budget = Arc::new(RetryBudget::new(&RetryBudgetConfig::default()));
9677            let gate = RetryGate {
9678                deadline: None,
9679                budgets: None,
9680                budget: Some(Arc::clone(&budget)),
9681                host: None,
9682                send_header: true,
9683                refill_pending: AtomicBool::new(false),
9684                pending_retry: std::sync::Mutex::new(None),
9685                caller_deadline: std::sync::OnceLock::new(),
9686                signer: None,
9687            };
9688            assert!(gate.allow(RetryKind::Transient, Duration::ZERO));
9689            // The retry starts.
9690            gate.check().unwrap();
9691            gate.finish(Some(RetryKind::Transient), 500);
9692            assert!((budget.available() - 486.0).abs() < f64::EPSILON);
9693            gate.finish(Some(RetryKind::Transient), 200);
9694            assert!((budget.available() - 500.0).abs() < f64::EPSILON);
9695        }
9696    }
9697}