Skip to main content

tensorlake/
read_retry.rs

1//! Bounded retries for finite, explicitly read-only metadata requests.
2//!
3//! Direct connections use current DNS answers and pools scoped to each
4//! origin/address, with address-family fallback before request delivery.
5//! A timed-out HTTP/2 stream therefore cannot pin every
6//! retry to the same multiplexed connection. Mutations and streams never enter
7//! this path. System proxy settings remain in force; a forwarding proxy owns
8//! origin DNS/routing, so address selection cannot control its upstream leg.
9
10use super::*;
11use rand::Rng;
12use std::{collections::HashSet, net::IpAddr, net::SocketAddr, sync::Mutex};
13use tokio::time::Instant;
14
15const MAX_POOLS: usize = 32;
16const POOL_IDLE: Duration = Duration::from_secs(60);
17const RETRY_CAPACITY: f64 = 20.0;
18const DNS_FAMILY_GRACE: Duration = Duration::from_millis(50);
19const MAX_CONNECT_TIMEOUT: Duration = Duration::from_secs(2);
20
21/// Retry limits for sandbox metadata reads. All attempts, DNS lookups, body
22/// reads and backoff share `total_timeout`, further capped by request timeout.
23/// Long-running operations and response streams do not use this policy.
24#[derive(Clone, Debug)]
25pub struct ReadRetryConfig {
26    pub max_attempts: usize,
27    pub attempt_timeout: Duration,
28    pub total_timeout: Duration,
29}
30
31impl Default for ReadRetryConfig {
32    fn default() -> Self {
33        Self {
34            max_attempts: 3,
35            attempt_timeout: Duration::from_secs(3),
36            total_timeout: Duration::from_secs(10),
37        }
38    }
39}
40
41impl ReadRetryConfig {
42    pub(super) fn validate(&self) -> Result<(), SdkError> {
43        if !(1..=10).contains(&self.max_attempts)
44            || self.attempt_timeout.is_zero()
45            || self.total_timeout.is_zero()
46        {
47            return Err(SdkError::ClientError(
48                "read retry limits require 1..=10 attempts and nonzero timeouts".into(),
49            ));
50        }
51        Ok(())
52    }
53}
54
55#[derive(Clone, Debug, PartialEq, Eq)]
56struct Endpoint {
57    origin: String,
58    address: Option<SocketAddr>,
59}
60
61struct Pool {
62    endpoint: Endpoint,
63    connect_timeout: Duration,
64    client: Arc<ClientWithMiddleware>,
65    used_at: Instant,
66}
67
68struct State {
69    pools: Vec<Pool>,
70    tokens: f64,
71    refilled_at: Instant,
72    suspect: Vec<(Endpoint, Instant)>,
73}
74
75#[derive(Debug)]
76struct ReadResolvers {
77    ipv4: hickory_resolver::TokioResolver,
78    ipv6: hickory_resolver::TokioResolver,
79}
80
81impl ReadResolvers {
82    fn new(
83        builder: impl Fn() -> Result<
84            hickory_resolver::ResolverBuilder<hickory_resolver::net::runtime::TokioRuntimeProvider>,
85            hickory_resolver::net::NetError,
86        >,
87    ) -> Result<Self, hickory_resolver::net::NetError> {
88        use hickory_resolver::config::{LookupIpStrategy, ResolveHosts};
89        let resolver = |strategy| {
90            let mut builder = builder()?;
91            builder.options_mut().cache_size = 64;
92            builder.options_mut().use_hosts_file = ResolveHosts::Always;
93            builder.options_mut().ip_strategy = strategy;
94            builder.build()
95        };
96        Ok(Self {
97            ipv4: resolver(LookupIpStrategy::Ipv4Only)?,
98            ipv6: resolver(LookupIpStrategy::Ipv6Only)?,
99        })
100    }
101
102    async fn lookup(&self, host: &str, deadline: Instant) -> Vec<IpAddr> {
103        use futures::future::{Either, select};
104        let ipv4 = Box::pin(self.ipv4.lookup_ip(host));
105        let ipv6 = Box::pin(self.ipv6.lookup_ip(host));
106        let Ok(first_completed) = tokio::time::timeout_at(deadline, select(ipv4, ipv6)).await
107        else {
108            return Vec::new();
109        };
110        let (first, remaining) = match first_completed {
111            Either::Left(pair) | Either::Right(pair) => pair,
112        };
113        let mut addresses = first
114            .map(|result| result.iter().collect::<Vec<_>>())
115            .unwrap_or_default();
116        // A missing answer is not a reason to abandon the other query. Once
117        // one family is usable, collect the other only if it arrives promptly.
118        // Hickory's combined strategy instead waits through its DNS timeout.
119        let other_deadline = if addresses.is_empty() {
120            deadline
121        } else {
122            deadline.min(Instant::now() + DNS_FAMILY_GRACE)
123        };
124        if let Ok(Ok(other)) = tokio::time::timeout_at(other_deadline, remaining).await {
125            addresses.extend(other.iter());
126        }
127        addresses
128    }
129}
130
131pub(super) struct ReadTransport {
132    config: ReadRetryConfig,
133    headers: HeaderMap,
134    user_agent: String,
135    middlewares: Vec<Arc<dyn Middleware>>,
136    follow_redirects: bool,
137    system_proxy: hyper_util::client::proxy::matcher::Matcher,
138    state: Mutex<State>,
139    resolver: tokio::sync::OnceCell<Option<ReadResolvers>>,
140    #[cfg(test)]
141    test_options: Mutex<TestOptions>,
142}
143
144#[cfg(test)]
145#[derive(Default)]
146struct TestOptions {
147    addresses: Option<Vec<SocketAddr>>,
148    certificate: Option<reqwest::tls::Certificate>,
149}
150
151impl ReadTransport {
152    pub(super) fn new(
153        config: ReadRetryConfig,
154        headers: HeaderMap,
155        user_agent: String,
156        middlewares: Vec<Arc<dyn Middleware>>,
157        follow_redirects: bool,
158    ) -> Self {
159        Self {
160            config,
161            headers,
162            user_agent,
163            middlewares,
164            follow_redirects,
165            system_proxy: hyper_util::client::proxy::matcher::Matcher::from_system(),
166            resolver: tokio::sync::OnceCell::new(),
167            state: Mutex::new(State {
168                pools: Vec::new(),
169                tokens: RETRY_CAPACITY,
170                refilled_at: Instant::now(),
171                suspect: Vec::new(),
172            }),
173            #[cfg(test)]
174            test_options: Mutex::new(TestOptions::default()),
175        }
176    }
177
178    // Hickory caches records only for their DNS TTL and uses the machine's
179    // configured nameservers and hosts file. If unavailable, retain the system
180    // resolver path (including platform-specific name services).
181    async fn addresses(&self, url: &url::Url, timeout: Duration) -> Vec<SocketAddr> {
182        #[cfg(test)]
183        if let Some(addresses) = self.test_options.lock().unwrap().addresses.clone() {
184            return addresses;
185        }
186        if let Ok(uri) = url.as_str().parse()
187            && self.system_proxy.intercept(&uri).is_some()
188        {
189            // Origin DNS belongs to a forwarding proxy. Do not require local
190            // resolution, or bypass a proxy to reach a selected address.
191            return Vec::new();
192        }
193        let Some(host) = url.host_str() else {
194            return Vec::new();
195        };
196        let Some(port) = url.port_or_known_default() else {
197            return Vec::new();
198        };
199        match url.host() {
200            Some(url::Host::Ipv4(ip)) => return vec![SocketAddr::new(ip.into(), port)],
201            Some(url::Host::Ipv6(ip)) => return vec![SocketAddr::new(ip.into(), port)],
202            _ => {}
203        }
204        let resolver = self
205            .resolver
206            .get_or_init(|| async {
207                ReadResolvers::new(hickory_resolver::TokioResolver::builder_tokio).ok()
208            })
209            .await;
210        // Reserve time for the platform resolver and connection establishment
211        // even when the configured DNS servers never answer either query.
212        let dns_budget = (timeout / 2).min(Duration::from_secs(1));
213        let dns_deadline = Instant::now() + dns_budget;
214        let mut addresses = Vec::new();
215        if let Some(resolver) = resolver {
216            let result = resolver.lookup(host, Instant::now() + dns_budget / 2).await;
217            addresses.extend(result.into_iter().map(|ip| SocketAddr::new(ip, port)));
218        }
219        if addresses.is_empty()
220            && let Ok(Ok(result)) =
221                tokio::time::timeout_at(dns_deadline, tokio::net::lookup_host((host, port))).await
222        {
223            addresses.extend(result);
224        }
225        // A system HTTP proxy can resolve a name the local host cannot.
226        // An empty result leaves origin resolution to the normal proxy path.
227        addresses.sort_unstable();
228        addresses.dedup();
229        if !addresses.is_empty() {
230            let start = rand::rng().random_range(0..addresses.len());
231            addresses.rotate_left(start);
232        }
233        addresses
234    }
235
236    fn select(
237        &self,
238        origin: &str,
239        addresses: &[SocketAddr],
240        tried: &HashSet<SocketAddr>,
241    ) -> Option<SocketAddr> {
242        if addresses.is_empty() {
243            return None;
244        }
245        let mut state = self.state.lock().unwrap();
246        state.suspect.retain(|(_, until)| *until > Instant::now());
247        let healthy = |address: &&SocketAddr| {
248            !state
249                .suspect
250                .iter()
251                .any(|(e, _)| e.origin == origin && e.address == Some(**address))
252        };
253        // Prefer an untried healthy peer, but never block every address. A
254        // briefly suspect endpoint remains usable when no alternative exists.
255        addresses
256            .iter()
257            .filter(|a| !tried.contains(*a))
258            .find(healthy)
259            .or_else(|| addresses.iter().find(|a| !tried.contains(*a)))
260            .or_else(|| addresses.iter().find(healthy))
261            .or(addresses.first())
262            .copied()
263    }
264
265    async fn resolve_other_family(
266        &self,
267        url: &url::Url,
268        failed: SocketAddr,
269        remaining: Duration,
270    ) -> Vec<SocketAddr> {
271        let (Some(url::Host::Domain(host)), Some(port), Some(Some(resolvers))) =
272            (url.host(), url.port_or_known_default(), self.resolver.get())
273        else {
274            return Vec::new();
275        };
276        let resolver = if failed.is_ipv4() {
277            &resolvers.ipv6
278        } else {
279            &resolvers.ipv4
280        };
281        // The other family's initial query may have exceeded the short grace
282        // period. If the first family cannot connect, give that answer another
283        // bounded chance instead of repeatedly choosing the fast, broken one.
284        match tokio::time::timeout(
285            (remaining / 2).min(Duration::from_secs(1)),
286            resolver.lookup_ip(host),
287        )
288        .await
289        {
290            Ok(Ok(result)) => result.iter().map(|ip| SocketAddr::new(ip, port)).collect(),
291            _ => Vec::new(),
292        }
293    }
294
295    fn pool(
296        &self,
297        endpoint: &Endpoint,
298        host: &str,
299        connect_budget: Duration,
300    ) -> Result<Arc<ClientWithMiddleware>, SdkError> {
301        // Reqwest's connection timeout belongs to the pooled client. Round
302        // down to a shared timeout bucket so similar caller deadlines reuse
303        // connections instead of creating a pool for every elapsed nanosecond.
304        let mut connect_timeout = MAX_CONNECT_TIMEOUT;
305        while connect_timeout > connect_budget {
306            connect_timeout /= 2;
307        }
308        let mut state = self.state.lock().unwrap();
309        state
310            .pools
311            .retain(|pool| pool.used_at.elapsed() < POOL_IDLE);
312        if let Some(pool) = state
313            .pools
314            .iter_mut()
315            .find(|p| p.endpoint == *endpoint && p.connect_timeout == connect_timeout)
316        {
317            pool.used_at = Instant::now();
318            return Ok(pool.client.clone());
319        }
320        let mut builder = crate::http_transport::https_builder()
321            .default_headers(self.headers.clone())
322            .user_agent(&self.user_agent)
323            .redirect(if self.follow_redirects {
324                reqwest::redirect::Policy::default()
325            } else {
326                reqwest::redirect::Policy::none()
327            })
328            .retry(reqwest::retry::never())
329            .connect_timeout(connect_timeout);
330        if let Some(address) = endpoint.address {
331            builder = builder.resolve(host, address);
332        }
333        #[cfg(test)]
334        {
335            let options = self.test_options.lock().unwrap();
336            if options.addresses.is_some() {
337                builder = builder.no_proxy();
338            }
339            if let Some(cert) = &options.certificate {
340                builder = builder.tls_certs_merge([cert.clone()]);
341            }
342        }
343        let mut builder = ReqwestClientBuilder::new(builder.build()?);
344        for middleware in &self.middlewares {
345            builder = builder.with_arc(middleware.clone());
346        }
347        let client = Arc::new(builder.build());
348        if state.pools.len() >= MAX_POOLS {
349            let oldest = state
350                .pools
351                .iter()
352                .enumerate()
353                .min_by_key(|(_, p)| p.used_at)
354                .unwrap()
355                .0;
356            state.pools.swap_remove(oldest);
357        }
358        state.pools.push(Pool {
359            endpoint: endpoint.clone(),
360            connect_timeout,
361            client: client.clone(),
362            used_at: Instant::now(),
363        });
364        Ok(client)
365    }
366
367    fn retire(&self, endpoint: &Endpoint, client: &Arc<ClientWithMiddleware>) {
368        // Dropping the cache's handle does not cancel other in-flight streams.
369        // Their handles live until their own requests/deadlines finish.
370        let mut state = self.state.lock().unwrap();
371        let Some(index) = state
372            .pools
373            .iter()
374            .position(|p| p.endpoint == *endpoint && Arc::ptr_eq(&p.client, client))
375        else {
376            // A different request already retired this pool. Its replacement
377            // must survive late failures from the old connection, including
378            // their attempts to extend the endpoint's suspect interval.
379            return;
380        };
381        state.pools.swap_remove(index);
382        state
383            .suspect
384            .retain(|(e, until)| e != endpoint && *until > Instant::now());
385        if state.suspect.len() >= MAX_POOLS {
386            state.suspect.remove(0);
387        }
388        state
389            .suspect
390            .push((endpoint.clone(), Instant::now() + Duration::from_secs(10)));
391    }
392
393    fn take_retry_token(&self) -> bool {
394        let mut state = self.state.lock().unwrap();
395        // A small burst, replenished by successes and at one retry per second
396        // during an outage. Cloned clients share this budget.
397        state.tokens =
398            (state.tokens + state.refilled_at.elapsed().as_secs_f64()).min(RETRY_CAPACITY);
399        state.refilled_at = Instant::now();
400        if state.tokens < 1.0 {
401            return false;
402        }
403        state.tokens -= 1.0;
404        true
405    }
406
407    fn success(&self) {
408        let mut state = self.state.lock().unwrap();
409        state.tokens = (state.tokens + 1.0).min(RETRY_CAPACITY);
410    }
411}
412
413fn jitter(attempt: usize) -> Duration {
414    let cap_ms = 100_u64 * (1_u64 << attempt.min(4));
415    Duration::from_millis(rand::rng().random_range(0..=cap_ms))
416}
417
418fn retryable_status(status: StatusCode) -> bool {
419    matches!(status.as_u16(), 429 | 502 | 503 | 504)
420}
421
422fn retryable(error: &SdkError) -> bool {
423    if let SdkError::ServerError { status, .. } = error {
424        return retryable_status(*status) || error.never_reached_server();
425    }
426    if matches!(error, SdkError::StreamTimeout { .. }) {
427        return true;
428    }
429    let Some(error) = error.as_reqwest() else {
430        return false;
431    };
432    // Never reinterpret invalid certificates or other TLS policy failures as
433    // endpoint impairment. Inspection uses typed causes, not error strings.
434    let mut cause = std::error::Error::source(error);
435    while let Some(current) = cause {
436        if current.downcast_ref::<rustls::Error>().is_some() {
437            return false;
438        }
439        // io::Error::source() can skip its inner error. hyper-rustls wraps
440        // tokio-rustls' io::Error again, so walk get_ref() at every layer.
441        cause = current
442            .downcast_ref::<std::io::Error>()
443            .and_then(std::io::Error::get_ref)
444            .map(|inner| inner as &(dyn std::error::Error + 'static))
445            .or_else(|| current.source());
446    }
447
448    !error.is_builder()
449        && (error.is_timeout()
450            || error.is_connect()
451            || error.is_request()
452            || error.is_body()
453            || error.is_decode())
454}
455
456fn exhausted(error: SdkError) -> SdkError {
457    SdkError::ReadRetryExhausted(Box::new(error))
458}
459
460impl Client {
461    /// Execute a finite, side-effect-free metadata GET with bounded retries.
462    /// The body is fully received before anything is exposed to the caller.
463    /// Use `execute_json` for mutations, long polls and large/streaming reads.
464    pub async fn execute_read_json<T: DeserializeOwned>(
465        &self,
466        request: Request,
467    ) -> Result<Traced<T>, SdkError> {
468        self.execute_metadata_read(request, self.reads.config.max_attempts)
469            .await
470    }
471
472    /// Readiness polling owns its own deadline and retry loop. Give each poll
473    /// address selection and pool retirement, without nesting another loop or
474    /// hiding the final observed state behind a retry-boundary error.
475    pub(crate) async fn execute_read_json_once<T: DeserializeOwned>(
476        &self,
477        request: Request,
478    ) -> Result<Traced<T>, SdkError> {
479        self.execute_metadata_read(request, 1)
480            .await
481            .map_err(SdkError::into_original)
482    }
483
484    async fn execute_metadata_read<T: DeserializeOwned>(
485        &self,
486        request: Request,
487        max_attempts: usize,
488    ) -> Result<Traced<T>, SdkError> {
489        if request.method() != Method::GET || request.body().is_some() {
490            return Err(SdkError::ClientError(
491                "resilient reads require a bodyless GET".into(),
492            ));
493        }
494        let policy = &self.reads.config;
495        let total = request
496            .timeout()
497            .or(self.timeout.as_ref())
498            .copied()
499            .map_or(policy.total_timeout, |timeout| {
500                timeout.min(policy.total_timeout)
501            });
502        let started = Instant::now();
503        let (_, read_id) = generate_traceparent();
504        let deadline = started + total;
505        let mut tried = HashSet::new();
506        let mut last_error = None;
507        for attempt in 1..=max_attempts {
508            let now = Instant::now();
509            if now >= deadline {
510                break;
511            }
512            // Charge only retries that actually start, after backoff and the
513            // deadline check. An unusable Retry-After or cancelled wait must
514            // not consume the quota shared with other reads.
515            if attempt > 1 && !self.reads.take_retry_token() {
516                break;
517            }
518            let timeout = policy.attempt_timeout.min(deadline - now);
519            let attempt_deadline = now + timeout;
520            let mut endpoint = None;
521            let mut attempted_pool = None;
522            let mut retry_after = None;
523            let mut response_status = None;
524            let mut stage = "dns";
525            let mut trace_id = String::new();
526            let result = tokio::time::timeout(timeout, async {
527                let addresses = self.reads.addresses(request.url(), timeout).await;
528                let origin = request.url().origin().ascii_serialization();
529                let address = self.reads.select(&origin, &addresses, &tried);
530                let other_family: Vec<_> = addresses
531                    .iter()
532                    .copied()
533                    .filter(|a| address.is_some_and(|first| a.is_ipv4() != first.is_ipv4()))
534                    .collect();
535                let mut fallback = self.reads.select(&origin, &other_family, &tried);
536                let mut candidate = address;
537                let mut response = None;
538                for connection_attempt in 0..2 {
539                    if let Some(address) = candidate {
540                        tried.insert(address);
541                    }
542                    let selected = Endpoint {
543                        origin: origin.clone(),
544                        address: candidate,
545                    };
546                    let remaining = attempt_deadline.saturating_duration_since(Instant::now());
547                    // A pinned address cannot use the connector's normal
548                    // family racing. Reserve half the remaining time for a
549                    // fallback connection, even for short caller deadlines.
550                    // Limit connection establishment, not HTTP delivery: a
551                    // slow response must still spend an HTTP retry attempt.
552                    let connect_budget = if connection_attempt == 0 && candidate.is_some() {
553                        remaining / 2
554                    } else {
555                        remaining
556                    };
557                    if connect_budget.is_zero() {
558                        return Err(SdkError::StreamTimeout {
559                            waiting_for: "a sandbox metadata read",
560                            timeout,
561                        });
562                    }
563                    let transport = self
564                        .reads
565                        .pool(&selected, request.url().host_str().unwrap_or(""), connect_budget)?;
566                    endpoint = Some(selected);
567                    attempted_pool = Some(transport.clone());
568                    let mut current = request.try_clone().expect("bodyless GET is replayable");
569                    trace_id = self.prepare_traced_request(&mut current);
570                    let remaining = attempt_deadline.saturating_duration_since(Instant::now());
571                    if remaining.is_zero() {
572                        return Err(SdkError::StreamTimeout {
573                            waiting_for: "a sandbox metadata read",
574                            timeout,
575                        });
576                    }
577                    *current.timeout_mut() = Some(remaining);
578                    current.headers_mut().insert(
579                        REQUEST_TIMEOUT_HEADER,
580                        HeaderValue::from(remaining.as_millis() as u64),
581                    );
582                    stage = "response_headers";
583                    match transport.execute(current).await {
584                        Ok(received) => {
585                            response = Some(received);
586                            break;
587                        }
588                        Err(error) => {
589                            let error = SdkError::from(error);
590                            // Like the default connector's family fallback,
591                            // this is still one HTTP attempt: no request was
592                            // delivered. Never fall back on a TLS policy error
593                            // or after HTTP transmission/body failure.
594                            if connection_attempt == 0
595                                && let Some(failed) = candidate
596                                && error.as_reqwest().is_some_and(|e| e.is_connect())
597                                && retryable(&error)
598                            {
599                                if fallback.is_none() {
600                                    let late = self.reads.resolve_other_family(request.url(), failed, attempt_deadline.saturating_duration_since(Instant::now())).await;
601                                    fallback = self.reads.select(&origin, &late, &tried);
602                                }
603                                if fallback.is_some() {
604                                    self.reads.retire(endpoint.as_ref().unwrap(), &transport);
605                                    tracing::debug!(attempt, %read_id, %trace_id, selected_address = ?candidate, fallback_address = ?fallback, "sandbox metadata connection trying alternate address family");
606                                    candidate = fallback;
607                                    continue;
608                                }
609                            }
610                            return Err(error);
611                        }
612                    }
613                }
614                let response = response.expect(
615                    "connection attempts either return an error or receive a response",
616                );
617                response_status = Some(response.status());
618                retry_after = response
619                    .headers()
620                    .get(reqwest::header::RETRY_AFTER)
621                    .and_then(|value| value.to_str().ok())
622                    .and_then(|value| {
623                        value
624                            .parse::<u64>()
625                            .ok()
626                            .map(Duration::from_secs)
627                            .or_else(|| {
628                                httpdate::parse_http_date(value).ok().map(|date| {
629                                    date.duration_since(std::time::SystemTime::now())
630                                        .unwrap_or_default()
631                                })
632                            })
633                    });
634                stage = "response_body";
635                let response = self.handle_response(response).await?;
636                Ok::<_, SdkError>(response.bytes().await?)
637            })
638            .await
639            .unwrap_or_else(|_| {
640                Err(SdkError::StreamTimeout {
641                    waiting_for: "a sandbox metadata read",
642                    timeout,
643                })
644            });
645            match result {
646                Ok(bytes) => {
647                    let jd = &mut serde_json::Deserializer::from_slice(&bytes);
648                    let value = serde_path_to_error::deserialize(jd)?;
649                    self.reads.success();
650                    if attempt > 1 {
651                        tracing::debug!(attempt, %read_id, %trace_id, selected_address = ?endpoint.as_ref().and_then(|e| e.address), elapsed_ms = started.elapsed().as_millis() as u64, "sandbox metadata read recovered");
652                    }
653                    return Ok(Traced::new(trace_id, value));
654                }
655                Err(error) => {
656                    // Once a definitive status arrived, a stalled/truncated
657                    // error body must not turn a 401/403/404 into a replay.
658                    if !matches!(error, SdkError::ServerError { .. })
659                        && retryable(&error)
660                        && let Some(status) =
661                            response_status.filter(|s| !s.is_success() && !retryable_status(*s))
662                    {
663                        let message = "response body did not complete".to_string();
664                        return Err(match status {
665                            StatusCode::UNAUTHORIZED => SdkError::Authentication(message),
666                            StatusCode::FORBIDDEN => SdkError::Authorization(message),
667                            _ => SdkError::ServerError { status, message },
668                        });
669                    }
670                    if !retryable(&error) {
671                        return Err(if error.never_reached_server() {
672                            exhausted(error)
673                        } else {
674                            error
675                        });
676                    }
677                    let transport_failure = !matches!(error, SdkError::ServerError { .. });
678                    if transport_failure
679                        && let (Some(endpoint), Some(pool)) = (&endpoint, &attempted_pool)
680                    {
681                        self.reads.retire(endpoint, pool);
682                    }
683                    let failure = match error.transport_failure() {
684                        Some(crate::error::TransportFailure::Timeout) => "timeout",
685                        Some(crate::error::TransportFailure::Connect) => "connect",
686                        None if response_status.is_some_and(|s| !s.is_success()) => "http_status",
687                        None => "connection_or_body",
688                    };
689                    tracing::debug!(attempt, %read_id, %trace_id, stage, failure, status = ?response_status.map(|s| s.as_u16()), selected_address = ?endpoint.as_ref().and_then(|e| e.address), elapsed_ms = started.elapsed().as_millis() as u64, "sandbox metadata read attempt failed");
690                    last_error = Some(error);
691                    if attempt == max_attempts {
692                        break;
693                    }
694                    let wait = retry_after.unwrap_or_else(|| jitter(attempt));
695                    if wait >= deadline.saturating_duration_since(Instant::now()) {
696                        break;
697                    }
698                    tokio::time::sleep(wait).await;
699                }
700            }
701        }
702        Err(exhausted(last_error.unwrap_or(SdkError::StreamTimeout {
703            waiting_for: "a sandbox metadata read",
704            timeout: total,
705        })))
706    }
707}
708
709#[cfg(test)]
710#[path = "read_retry_tests.rs"]
711mod tests;