1use super::*;
11use rand::Rng;
12use std::{collections::HashSet, net::IpAddr, net::SocketAddr, sync::Mutex};
13use tokio::time::Instant;
14
15const MAX_POOLS: usize = 32;
16const POOL_IDLE: Duration = Duration::from_secs(60);
17const RETRY_CAPACITY: f64 = 20.0;
18const DNS_FAMILY_GRACE: Duration = Duration::from_millis(50);
19const MAX_CONNECT_TIMEOUT: Duration = Duration::from_secs(2);
20
21#[derive(Clone, Debug)]
25pub struct ReadRetryConfig {
26 pub max_attempts: usize,
27 pub attempt_timeout: Duration,
28 pub total_timeout: Duration,
29}
30
31impl Default for ReadRetryConfig {
32 fn default() -> Self {
33 Self {
34 max_attempts: 3,
35 attempt_timeout: Duration::from_secs(3),
36 total_timeout: Duration::from_secs(10),
37 }
38 }
39}
40
41impl ReadRetryConfig {
42 pub(super) fn validate(&self) -> Result<(), SdkError> {
43 if !(1..=10).contains(&self.max_attempts)
44 || self.attempt_timeout.is_zero()
45 || self.total_timeout.is_zero()
46 {
47 return Err(SdkError::ClientError(
48 "read retry limits require 1..=10 attempts and nonzero timeouts".into(),
49 ));
50 }
51 Ok(())
52 }
53}
54
55#[derive(Clone, Debug, PartialEq, Eq)]
56struct Endpoint {
57 origin: String,
58 address: Option<SocketAddr>,
59}
60
61struct Pool {
62 endpoint: Endpoint,
63 connect_timeout: Duration,
64 client: Arc<ClientWithMiddleware>,
65 used_at: Instant,
66}
67
68struct State {
69 pools: Vec<Pool>,
70 tokens: f64,
71 refilled_at: Instant,
72 suspect: Vec<(Endpoint, Instant)>,
73}
74
75#[derive(Debug)]
76struct ReadResolvers {
77 ipv4: hickory_resolver::TokioResolver,
78 ipv6: hickory_resolver::TokioResolver,
79}
80
81impl ReadResolvers {
82 fn new(
83 builder: impl Fn() -> Result<
84 hickory_resolver::ResolverBuilder<hickory_resolver::net::runtime::TokioRuntimeProvider>,
85 hickory_resolver::net::NetError,
86 >,
87 ) -> Result<Self, hickory_resolver::net::NetError> {
88 use hickory_resolver::config::{LookupIpStrategy, ResolveHosts};
89 let resolver = |strategy| {
90 let mut builder = builder()?;
91 builder.options_mut().cache_size = 64;
92 builder.options_mut().use_hosts_file = ResolveHosts::Always;
93 builder.options_mut().ip_strategy = strategy;
94 builder.build()
95 };
96 Ok(Self {
97 ipv4: resolver(LookupIpStrategy::Ipv4Only)?,
98 ipv6: resolver(LookupIpStrategy::Ipv6Only)?,
99 })
100 }
101
102 async fn lookup(&self, host: &str, deadline: Instant) -> Vec<IpAddr> {
103 use futures::future::{Either, select};
104 let ipv4 = Box::pin(self.ipv4.lookup_ip(host));
105 let ipv6 = Box::pin(self.ipv6.lookup_ip(host));
106 let Ok(first_completed) = tokio::time::timeout_at(deadline, select(ipv4, ipv6)).await
107 else {
108 return Vec::new();
109 };
110 let (first, remaining) = match first_completed {
111 Either::Left(pair) | Either::Right(pair) => pair,
112 };
113 let mut addresses = first
114 .map(|result| result.iter().collect::<Vec<_>>())
115 .unwrap_or_default();
116 let other_deadline = if addresses.is_empty() {
120 deadline
121 } else {
122 deadline.min(Instant::now() + DNS_FAMILY_GRACE)
123 };
124 if let Ok(Ok(other)) = tokio::time::timeout_at(other_deadline, remaining).await {
125 addresses.extend(other.iter());
126 }
127 addresses
128 }
129}
130
131pub(super) struct ReadTransport {
132 config: ReadRetryConfig,
133 headers: HeaderMap,
134 user_agent: String,
135 middlewares: Vec<Arc<dyn Middleware>>,
136 follow_redirects: bool,
137 system_proxy: hyper_util::client::proxy::matcher::Matcher,
138 state: Mutex<State>,
139 resolver: tokio::sync::OnceCell<Option<ReadResolvers>>,
140 #[cfg(test)]
141 test_options: Mutex<TestOptions>,
142}
143
144#[cfg(test)]
145#[derive(Default)]
146struct TestOptions {
147 addresses: Option<Vec<SocketAddr>>,
148 certificate: Option<reqwest::tls::Certificate>,
149}
150
151impl ReadTransport {
152 pub(super) fn new(
153 config: ReadRetryConfig,
154 headers: HeaderMap,
155 user_agent: String,
156 middlewares: Vec<Arc<dyn Middleware>>,
157 follow_redirects: bool,
158 ) -> Self {
159 Self {
160 config,
161 headers,
162 user_agent,
163 middlewares,
164 follow_redirects,
165 system_proxy: hyper_util::client::proxy::matcher::Matcher::from_system(),
166 resolver: tokio::sync::OnceCell::new(),
167 state: Mutex::new(State {
168 pools: Vec::new(),
169 tokens: RETRY_CAPACITY,
170 refilled_at: Instant::now(),
171 suspect: Vec::new(),
172 }),
173 #[cfg(test)]
174 test_options: Mutex::new(TestOptions::default()),
175 }
176 }
177
178 async fn addresses(&self, url: &url::Url, timeout: Duration) -> Vec<SocketAddr> {
182 #[cfg(test)]
183 if let Some(addresses) = self.test_options.lock().unwrap().addresses.clone() {
184 return addresses;
185 }
186 if let Ok(uri) = url.as_str().parse()
187 && self.system_proxy.intercept(&uri).is_some()
188 {
189 return Vec::new();
192 }
193 let Some(host) = url.host_str() else {
194 return Vec::new();
195 };
196 let Some(port) = url.port_or_known_default() else {
197 return Vec::new();
198 };
199 match url.host() {
200 Some(url::Host::Ipv4(ip)) => return vec![SocketAddr::new(ip.into(), port)],
201 Some(url::Host::Ipv6(ip)) => return vec![SocketAddr::new(ip.into(), port)],
202 _ => {}
203 }
204 let resolver = self
205 .resolver
206 .get_or_init(|| async {
207 ReadResolvers::new(hickory_resolver::TokioResolver::builder_tokio).ok()
208 })
209 .await;
210 let dns_budget = (timeout / 2).min(Duration::from_secs(1));
213 let dns_deadline = Instant::now() + dns_budget;
214 let mut addresses = Vec::new();
215 if let Some(resolver) = resolver {
216 let result = resolver.lookup(host, Instant::now() + dns_budget / 2).await;
217 addresses.extend(result.into_iter().map(|ip| SocketAddr::new(ip, port)));
218 }
219 if addresses.is_empty()
220 && let Ok(Ok(result)) =
221 tokio::time::timeout_at(dns_deadline, tokio::net::lookup_host((host, port))).await
222 {
223 addresses.extend(result);
224 }
225 addresses.sort_unstable();
228 addresses.dedup();
229 if !addresses.is_empty() {
230 let start = rand::rng().random_range(0..addresses.len());
231 addresses.rotate_left(start);
232 }
233 addresses
234 }
235
236 fn select(
237 &self,
238 origin: &str,
239 addresses: &[SocketAddr],
240 tried: &HashSet<SocketAddr>,
241 ) -> Option<SocketAddr> {
242 if addresses.is_empty() {
243 return None;
244 }
245 let mut state = self.state.lock().unwrap();
246 state.suspect.retain(|(_, until)| *until > Instant::now());
247 let healthy = |address: &&SocketAddr| {
248 !state
249 .suspect
250 .iter()
251 .any(|(e, _)| e.origin == origin && e.address == Some(**address))
252 };
253 addresses
256 .iter()
257 .filter(|a| !tried.contains(*a))
258 .find(healthy)
259 .or_else(|| addresses.iter().find(|a| !tried.contains(*a)))
260 .or_else(|| addresses.iter().find(healthy))
261 .or(addresses.first())
262 .copied()
263 }
264
265 async fn resolve_other_family(
266 &self,
267 url: &url::Url,
268 failed: SocketAddr,
269 remaining: Duration,
270 ) -> Vec<SocketAddr> {
271 let (Some(url::Host::Domain(host)), Some(port), Some(Some(resolvers))) =
272 (url.host(), url.port_or_known_default(), self.resolver.get())
273 else {
274 return Vec::new();
275 };
276 let resolver = if failed.is_ipv4() {
277 &resolvers.ipv6
278 } else {
279 &resolvers.ipv4
280 };
281 match tokio::time::timeout(
285 (remaining / 2).min(Duration::from_secs(1)),
286 resolver.lookup_ip(host),
287 )
288 .await
289 {
290 Ok(Ok(result)) => result.iter().map(|ip| SocketAddr::new(ip, port)).collect(),
291 _ => Vec::new(),
292 }
293 }
294
295 fn pool(
296 &self,
297 endpoint: &Endpoint,
298 host: &str,
299 connect_budget: Duration,
300 ) -> Result<Arc<ClientWithMiddleware>, SdkError> {
301 let mut connect_timeout = MAX_CONNECT_TIMEOUT;
305 while connect_timeout > connect_budget {
306 connect_timeout /= 2;
307 }
308 let mut state = self.state.lock().unwrap();
309 state
310 .pools
311 .retain(|pool| pool.used_at.elapsed() < POOL_IDLE);
312 if let Some(pool) = state
313 .pools
314 .iter_mut()
315 .find(|p| p.endpoint == *endpoint && p.connect_timeout == connect_timeout)
316 {
317 pool.used_at = Instant::now();
318 return Ok(pool.client.clone());
319 }
320 let mut builder = crate::http_transport::https_builder()
321 .default_headers(self.headers.clone())
322 .user_agent(&self.user_agent)
323 .redirect(if self.follow_redirects {
324 reqwest::redirect::Policy::default()
325 } else {
326 reqwest::redirect::Policy::none()
327 })
328 .retry(reqwest::retry::never())
329 .connect_timeout(connect_timeout);
330 if let Some(address) = endpoint.address {
331 builder = builder.resolve(host, address);
332 }
333 #[cfg(test)]
334 {
335 let options = self.test_options.lock().unwrap();
336 if options.addresses.is_some() {
337 builder = builder.no_proxy();
338 }
339 if let Some(cert) = &options.certificate {
340 builder = builder.tls_certs_merge([cert.clone()]);
341 }
342 }
343 let mut builder = ReqwestClientBuilder::new(builder.build()?);
344 for middleware in &self.middlewares {
345 builder = builder.with_arc(middleware.clone());
346 }
347 let client = Arc::new(builder.build());
348 if state.pools.len() >= MAX_POOLS {
349 let oldest = state
350 .pools
351 .iter()
352 .enumerate()
353 .min_by_key(|(_, p)| p.used_at)
354 .unwrap()
355 .0;
356 state.pools.swap_remove(oldest);
357 }
358 state.pools.push(Pool {
359 endpoint: endpoint.clone(),
360 connect_timeout,
361 client: client.clone(),
362 used_at: Instant::now(),
363 });
364 Ok(client)
365 }
366
367 fn retire(&self, endpoint: &Endpoint, client: &Arc<ClientWithMiddleware>) {
368 let mut state = self.state.lock().unwrap();
371 let Some(index) = state
372 .pools
373 .iter()
374 .position(|p| p.endpoint == *endpoint && Arc::ptr_eq(&p.client, client))
375 else {
376 return;
380 };
381 state.pools.swap_remove(index);
382 state
383 .suspect
384 .retain(|(e, until)| e != endpoint && *until > Instant::now());
385 if state.suspect.len() >= MAX_POOLS {
386 state.suspect.remove(0);
387 }
388 state
389 .suspect
390 .push((endpoint.clone(), Instant::now() + Duration::from_secs(10)));
391 }
392
393 fn take_retry_token(&self) -> bool {
394 let mut state = self.state.lock().unwrap();
395 state.tokens =
398 (state.tokens + state.refilled_at.elapsed().as_secs_f64()).min(RETRY_CAPACITY);
399 state.refilled_at = Instant::now();
400 if state.tokens < 1.0 {
401 return false;
402 }
403 state.tokens -= 1.0;
404 true
405 }
406
407 fn success(&self) {
408 let mut state = self.state.lock().unwrap();
409 state.tokens = (state.tokens + 1.0).min(RETRY_CAPACITY);
410 }
411}
412
413fn jitter(attempt: usize) -> Duration {
414 let cap_ms = 100_u64 * (1_u64 << attempt.min(4));
415 Duration::from_millis(rand::rng().random_range(0..=cap_ms))
416}
417
418fn retryable_status(status: StatusCode) -> bool {
419 matches!(status.as_u16(), 429 | 502 | 503 | 504)
420}
421
422fn retryable(error: &SdkError) -> bool {
423 if let SdkError::ServerError { status, .. } = error {
424 return retryable_status(*status) || error.never_reached_server();
425 }
426 if matches!(error, SdkError::StreamTimeout { .. }) {
427 return true;
428 }
429 let Some(error) = error.as_reqwest() else {
430 return false;
431 };
432 let mut cause = std::error::Error::source(error);
435 while let Some(current) = cause {
436 if current.downcast_ref::<rustls::Error>().is_some() {
437 return false;
438 }
439 cause = current
442 .downcast_ref::<std::io::Error>()
443 .and_then(std::io::Error::get_ref)
444 .map(|inner| inner as &(dyn std::error::Error + 'static))
445 .or_else(|| current.source());
446 }
447
448 !error.is_builder()
449 && (error.is_timeout()
450 || error.is_connect()
451 || error.is_request()
452 || error.is_body()
453 || error.is_decode())
454}
455
456fn exhausted(error: SdkError) -> SdkError {
457 SdkError::ReadRetryExhausted(Box::new(error))
458}
459
460impl Client {
461 pub async fn execute_read_json<T: DeserializeOwned>(
465 &self,
466 request: Request,
467 ) -> Result<Traced<T>, SdkError> {
468 self.execute_metadata_read(request, self.reads.config.max_attempts)
469 .await
470 }
471
472 pub(crate) async fn execute_read_json_once<T: DeserializeOwned>(
476 &self,
477 request: Request,
478 ) -> Result<Traced<T>, SdkError> {
479 self.execute_metadata_read(request, 1)
480 .await
481 .map_err(SdkError::into_original)
482 }
483
484 async fn execute_metadata_read<T: DeserializeOwned>(
485 &self,
486 request: Request,
487 max_attempts: usize,
488 ) -> Result<Traced<T>, SdkError> {
489 if request.method() != Method::GET || request.body().is_some() {
490 return Err(SdkError::ClientError(
491 "resilient reads require a bodyless GET".into(),
492 ));
493 }
494 let policy = &self.reads.config;
495 let total = request
496 .timeout()
497 .or(self.timeout.as_ref())
498 .copied()
499 .map_or(policy.total_timeout, |timeout| {
500 timeout.min(policy.total_timeout)
501 });
502 let started = Instant::now();
503 let (_, read_id) = generate_traceparent();
504 let deadline = started + total;
505 let mut tried = HashSet::new();
506 let mut last_error = None;
507 for attempt in 1..=max_attempts {
508 let now = Instant::now();
509 if now >= deadline {
510 break;
511 }
512 if attempt > 1 && !self.reads.take_retry_token() {
516 break;
517 }
518 let timeout = policy.attempt_timeout.min(deadline - now);
519 let attempt_deadline = now + timeout;
520 let mut endpoint = None;
521 let mut attempted_pool = None;
522 let mut retry_after = None;
523 let mut response_status = None;
524 let mut stage = "dns";
525 let mut trace_id = String::new();
526 let result = tokio::time::timeout(timeout, async {
527 let addresses = self.reads.addresses(request.url(), timeout).await;
528 let origin = request.url().origin().ascii_serialization();
529 let address = self.reads.select(&origin, &addresses, &tried);
530 let other_family: Vec<_> = addresses
531 .iter()
532 .copied()
533 .filter(|a| address.is_some_and(|first| a.is_ipv4() != first.is_ipv4()))
534 .collect();
535 let mut fallback = self.reads.select(&origin, &other_family, &tried);
536 let mut candidate = address;
537 let mut response = None;
538 for connection_attempt in 0..2 {
539 if let Some(address) = candidate {
540 tried.insert(address);
541 }
542 let selected = Endpoint {
543 origin: origin.clone(),
544 address: candidate,
545 };
546 let remaining = attempt_deadline.saturating_duration_since(Instant::now());
547 let connect_budget = if connection_attempt == 0 && candidate.is_some() {
553 remaining / 2
554 } else {
555 remaining
556 };
557 if connect_budget.is_zero() {
558 return Err(SdkError::StreamTimeout {
559 waiting_for: "a sandbox metadata read",
560 timeout,
561 });
562 }
563 let transport = self
564 .reads
565 .pool(&selected, request.url().host_str().unwrap_or(""), connect_budget)?;
566 endpoint = Some(selected);
567 attempted_pool = Some(transport.clone());
568 let mut current = request.try_clone().expect("bodyless GET is replayable");
569 trace_id = self.prepare_traced_request(&mut current);
570 let remaining = attempt_deadline.saturating_duration_since(Instant::now());
571 if remaining.is_zero() {
572 return Err(SdkError::StreamTimeout {
573 waiting_for: "a sandbox metadata read",
574 timeout,
575 });
576 }
577 *current.timeout_mut() = Some(remaining);
578 current.headers_mut().insert(
579 REQUEST_TIMEOUT_HEADER,
580 HeaderValue::from(remaining.as_millis() as u64),
581 );
582 stage = "response_headers";
583 match transport.execute(current).await {
584 Ok(received) => {
585 response = Some(received);
586 break;
587 }
588 Err(error) => {
589 let error = SdkError::from(error);
590 if connection_attempt == 0
595 && let Some(failed) = candidate
596 && error.as_reqwest().is_some_and(|e| e.is_connect())
597 && retryable(&error)
598 {
599 if fallback.is_none() {
600 let late = self.reads.resolve_other_family(request.url(), failed, attempt_deadline.saturating_duration_since(Instant::now())).await;
601 fallback = self.reads.select(&origin, &late, &tried);
602 }
603 if fallback.is_some() {
604 self.reads.retire(endpoint.as_ref().unwrap(), &transport);
605 tracing::debug!(attempt, %read_id, %trace_id, selected_address = ?candidate, fallback_address = ?fallback, "sandbox metadata connection trying alternate address family");
606 candidate = fallback;
607 continue;
608 }
609 }
610 return Err(error);
611 }
612 }
613 }
614 let response = response.expect(
615 "connection attempts either return an error or receive a response",
616 );
617 response_status = Some(response.status());
618 retry_after = response
619 .headers()
620 .get(reqwest::header::RETRY_AFTER)
621 .and_then(|value| value.to_str().ok())
622 .and_then(|value| {
623 value
624 .parse::<u64>()
625 .ok()
626 .map(Duration::from_secs)
627 .or_else(|| {
628 httpdate::parse_http_date(value).ok().map(|date| {
629 date.duration_since(std::time::SystemTime::now())
630 .unwrap_or_default()
631 })
632 })
633 });
634 stage = "response_body";
635 let response = self.handle_response(response).await?;
636 Ok::<_, SdkError>(response.bytes().await?)
637 })
638 .await
639 .unwrap_or_else(|_| {
640 Err(SdkError::StreamTimeout {
641 waiting_for: "a sandbox metadata read",
642 timeout,
643 })
644 });
645 match result {
646 Ok(bytes) => {
647 let jd = &mut serde_json::Deserializer::from_slice(&bytes);
648 let value = serde_path_to_error::deserialize(jd)?;
649 self.reads.success();
650 if attempt > 1 {
651 tracing::debug!(attempt, %read_id, %trace_id, selected_address = ?endpoint.as_ref().and_then(|e| e.address), elapsed_ms = started.elapsed().as_millis() as u64, "sandbox metadata read recovered");
652 }
653 return Ok(Traced::new(trace_id, value));
654 }
655 Err(error) => {
656 if !matches!(error, SdkError::ServerError { .. })
659 && retryable(&error)
660 && let Some(status) =
661 response_status.filter(|s| !s.is_success() && !retryable_status(*s))
662 {
663 let message = "response body did not complete".to_string();
664 return Err(match status {
665 StatusCode::UNAUTHORIZED => SdkError::Authentication(message),
666 StatusCode::FORBIDDEN => SdkError::Authorization(message),
667 _ => SdkError::ServerError { status, message },
668 });
669 }
670 if !retryable(&error) {
671 return Err(if error.never_reached_server() {
672 exhausted(error)
673 } else {
674 error
675 });
676 }
677 let transport_failure = !matches!(error, SdkError::ServerError { .. });
678 if transport_failure
679 && let (Some(endpoint), Some(pool)) = (&endpoint, &attempted_pool)
680 {
681 self.reads.retire(endpoint, pool);
682 }
683 let failure = match error.transport_failure() {
684 Some(crate::error::TransportFailure::Timeout) => "timeout",
685 Some(crate::error::TransportFailure::Connect) => "connect",
686 None if response_status.is_some_and(|s| !s.is_success()) => "http_status",
687 None => "connection_or_body",
688 };
689 tracing::debug!(attempt, %read_id, %trace_id, stage, failure, status = ?response_status.map(|s| s.as_u16()), selected_address = ?endpoint.as_ref().and_then(|e| e.address), elapsed_ms = started.elapsed().as_millis() as u64, "sandbox metadata read attempt failed");
690 last_error = Some(error);
691 if attempt == max_attempts {
692 break;
693 }
694 let wait = retry_after.unwrap_or_else(|| jitter(attempt));
695 if wait >= deadline.saturating_duration_since(Instant::now()) {
696 break;
697 }
698 tokio::time::sleep(wait).await;
699 }
700 }
701 }
702 Err(exhausted(last_error.unwrap_or(SdkError::StreamTimeout {
703 waiting_for: "a sandbox metadata read",
704 timeout: total,
705 })))
706 }
707}
708
709#[cfg(test)]
710#[path = "read_retry_tests.rs"]
711mod tests;