1use std::collections::BTreeSet;
2use std::fmt;
3use std::future::Future;
4use std::net::IpAddr;
5use std::pin::Pin;
6use std::sync::Arc;
7use std::time::Duration;
8
9use futures_util::StreamExt as _;
10use reqwest::header::{
11 ACCEPT, ACCEPT_ENCODING, CONTENT_ENCODING, CONTENT_LENGTH, CONTENT_TYPE, LOCATION,
12};
13use tea_control::CancellationScope;
14use tea_provider_http::ProviderHttpConfig;
15
16use super::{
17 FetchAddressPolicy, FetchProviderError, FetchProviderErrorCode, FetchRedirect, FetchUrlPolicy,
18 MAX_FETCH_REDIRECTS, ValidatedFetchAddresses, ValidatedFetchUrl,
19};
20
21pub(crate) const FETCH_ACCEPT_VALUE: &str =
22 "text/html, application/xhtml+xml, application/json, text/plain";
23pub(crate) const FETCH_ACCEPT_ENCODING_VALUE: &str = "identity";
24
25pub const DEFAULT_FETCH_RESPONSE_BYTES: usize = 2 * 1024 * 1024;
27pub const MAX_FETCH_RESPONSE_BYTES: usize = 8 * 1024 * 1024;
29
30const MAX_RESPONSE_HEADER_VALUE_BYTES: usize = 1024;
31const MAX_HTTP_TIMEOUT: Duration = Duration::from_mins(2);
32
33#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
35pub struct FetchHttpTimeouts {
36 dns: Duration,
37 connect: Duration,
38 first_byte: Duration,
39 total: Duration,
40 low_speed_window: Duration,
41 low_speed_min_bytes: usize,
42}
43
44impl FetchHttpTimeouts {
45 pub fn new(
52 dns: Duration,
53 connect: Duration,
54 first_byte: Duration,
55 total: Duration,
56 low_speed_window: Duration,
57 low_speed_min_bytes: usize,
58 ) -> Result<Self, FetchProviderError> {
59 let durations = [dns, connect, first_byte, total, low_speed_window];
60 if durations
61 .iter()
62 .any(|duration| duration.is_zero() || *duration > MAX_HTTP_TIMEOUT)
63 || total < dns
64 || total < connect
65 || total < first_byte
66 || total < low_speed_window
67 || low_speed_min_bytes == 0
68 {
69 return Err(FetchProviderError::new(
70 FetchProviderErrorCode::InvalidConfiguration,
71 ));
72 }
73 Ok(Self {
74 dns,
75 connect,
76 first_byte,
77 total,
78 low_speed_window,
79 low_speed_min_bytes,
80 })
81 }
82
83 #[must_use]
85 pub const fn dns(self) -> Duration {
86 self.dns
87 }
88
89 #[must_use]
91 pub const fn connect(self) -> Duration {
92 self.connect
93 }
94
95 #[must_use]
97 pub const fn first_byte(self) -> Duration {
98 self.first_byte
99 }
100
101 #[must_use]
103 pub const fn total(self) -> Duration {
104 self.total
105 }
106
107 #[must_use]
109 pub const fn low_speed_window(self) -> Duration {
110 self.low_speed_window
111 }
112
113 #[must_use]
115 pub const fn low_speed_min_bytes(self) -> usize {
116 self.low_speed_min_bytes
117 }
118}
119
120impl Default for FetchHttpTimeouts {
121 fn default() -> Self {
122 Self {
123 dns: Duration::from_secs(5),
124 connect: Duration::from_secs(10),
125 first_byte: Duration::from_secs(15),
126 total: Duration::from_secs(30),
127 low_speed_window: Duration::from_secs(5),
128 low_speed_min_bytes: 1024,
129 }
130 }
131}
132
133#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
135pub struct FetchHttpLimits {
136 max_redirects: usize,
137 max_response_bytes: usize,
138}
139
140impl FetchHttpLimits {
141 pub fn new(
148 max_redirects: usize,
149 max_response_bytes: usize,
150 ) -> Result<Self, FetchProviderError> {
151 if max_redirects > MAX_FETCH_REDIRECTS
152 || !(1..=MAX_FETCH_RESPONSE_BYTES).contains(&max_response_bytes)
153 {
154 return Err(FetchProviderError::new(
155 FetchProviderErrorCode::InvalidConfiguration,
156 ));
157 }
158 Ok(Self {
159 max_redirects,
160 max_response_bytes,
161 })
162 }
163
164 #[must_use]
166 pub const fn max_redirects(self) -> usize {
167 self.max_redirects
168 }
169
170 #[must_use]
172 pub const fn max_response_bytes(self) -> usize {
173 self.max_response_bytes
174 }
175}
176
177impl Default for FetchHttpLimits {
178 fn default() -> Self {
179 Self {
180 max_redirects: 5,
181 max_response_bytes: DEFAULT_FETCH_RESPONSE_BYTES,
182 }
183 }
184}
185
186#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
188pub struct FetchHttpConfig {
189 url_policy: FetchUrlPolicy,
190 address_policy: FetchAddressPolicy,
191 timeouts: FetchHttpTimeouts,
192 limits: FetchHttpLimits,
193}
194
195impl FetchHttpConfig {
196 #[must_use]
198 pub const fn new(
199 url_policy: FetchUrlPolicy,
200 address_policy: FetchAddressPolicy,
201 timeouts: FetchHttpTimeouts,
202 limits: FetchHttpLimits,
203 ) -> Self {
204 Self {
205 url_policy,
206 address_policy,
207 timeouts,
208 limits,
209 }
210 }
211
212 #[must_use]
214 pub fn production() -> Self {
215 Self::new(
216 FetchUrlPolicy::production(),
217 FetchAddressPolicy::public_network(),
218 FetchHttpTimeouts::default(),
219 FetchHttpLimits::default(),
220 )
221 }
222
223 #[must_use]
225 pub fn loopback_tests(timeouts: FetchHttpTimeouts, limits: FetchHttpLimits) -> Self {
226 Self::new(
227 FetchUrlPolicy::loopback_tests(),
228 FetchAddressPolicy::loopback_tests(),
229 timeouts,
230 limits,
231 )
232 }
233
234 #[must_use]
236 pub const fn url_policy(self) -> FetchUrlPolicy {
237 self.url_policy
238 }
239
240 #[must_use]
242 pub const fn address_policy(self) -> FetchAddressPolicy {
243 self.address_policy
244 }
245
246 #[must_use]
248 pub const fn timeouts(self) -> FetchHttpTimeouts {
249 self.timeouts
250 }
251
252 #[must_use]
254 pub const fn limits(self) -> FetchHttpLimits {
255 self.limits
256 }
257}
258
259impl Default for FetchHttpConfig {
260 fn default() -> Self {
261 Self::production()
262 }
263}
264
265pub type FetchResolveFuture<'a> =
267 Pin<Box<dyn Future<Output = Result<Vec<IpAddr>, FetchProviderError>> + Send + 'a>>;
268
269pub trait FetchDnsResolver: fmt::Debug + Send + Sync {
271 fn resolve(&self, host: &str, cancellation: CancellationScope) -> FetchResolveFuture<'_>;
273}
274
275#[derive(Debug, Clone, Copy, Default)]
277pub struct SystemFetchDnsResolver;
278
279impl FetchDnsResolver for SystemFetchDnsResolver {
280 fn resolve(&self, host: &str, cancellation: CancellationScope) -> FetchResolveFuture<'_> {
281 let host = host.to_owned();
282 Box::pin(async move {
283 if let Ok(address) = host.parse::<IpAddr>() {
284 return Ok(vec![address]);
285 }
286 let lookup = tokio::net::lookup_host((host.as_str(), 0));
287 let addresses = tokio::select! {
288 biased;
289 () = cancellation.cancelled() => return Err(FetchProviderError::cancelled()),
290 result = lookup => result.map_err(|_| FetchProviderError::new(FetchProviderErrorCode::DnsResolution))?,
291 };
292 Ok(addresses.map(|address| address.ip()).collect())
293 })
294 }
295}
296
297#[derive(Clone, PartialEq, Eq)]
299pub struct FetchHttpHeaders {
300 mime: Option<String>,
301 encoding: Option<String>,
302 length: Option<u64>,
303}
304
305impl FetchHttpHeaders {
306 pub fn new(
312 content_type: Option<String>,
313 content_encoding: Option<String>,
314 content_length: Option<u64>,
315 ) -> Result<Self, FetchProviderError> {
316 let mime = content_type.map(validate_header_value).transpose()?;
317 let encoding = content_encoding.map(validate_header_value).transpose()?;
318 Ok(Self {
319 mime,
320 encoding,
321 length: content_length,
322 })
323 }
324
325 #[must_use]
327 pub fn content_type(&self) -> Option<&str> {
328 self.mime.as_deref()
329 }
330
331 #[must_use]
333 pub fn content_encoding(&self) -> Option<&str> {
334 self.encoding.as_deref()
335 }
336
337 #[must_use]
339 pub const fn content_length(&self) -> Option<u64> {
340 self.length
341 }
342}
343
344impl fmt::Debug for FetchHttpHeaders {
345 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
346 formatter
347 .debug_struct("FetchHttpHeaders")
348 .field("content_type", &self.mime)
349 .field("content_encoding", &self.encoding)
350 .field("content_length", &self.length)
351 .finish()
352 }
353}
354
355#[derive(Clone, PartialEq, Eq)]
357pub struct FetchHttpResponse {
358 requested_url: String,
359 final_url: String,
360 status: u16,
361 headers: FetchHttpHeaders,
362 body: Vec<u8>,
363 redirects: Vec<FetchRedirect>,
364}
365
366impl FetchHttpResponse {
367 #[must_use]
369 pub fn requested_url(&self) -> &str {
370 &self.requested_url
371 }
372
373 #[must_use]
375 pub fn final_url(&self) -> &str {
376 &self.final_url
377 }
378
379 #[must_use]
381 pub const fn status(&self) -> u16 {
382 self.status
383 }
384
385 #[must_use]
387 pub const fn headers(&self) -> &FetchHttpHeaders {
388 &self.headers
389 }
390
391 #[must_use]
393 pub fn body(&self) -> &[u8] {
394 &self.body
395 }
396
397 #[must_use]
399 pub fn redirects(&self) -> &[FetchRedirect] {
400 &self.redirects
401 }
402}
403
404impl fmt::Debug for FetchHttpResponse {
405 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
406 formatter
407 .debug_struct("FetchHttpResponse")
408 .field("status", &self.status)
409 .field("headers", &self.headers)
410 .field("body_bytes", &self.body.len())
411 .field("redirect_count", &self.redirects.len())
412 .finish_non_exhaustive()
413 }
414}
415
416#[derive(Clone)]
418pub struct FetchHttpTransport {
419 config: FetchHttpConfig,
420 resolver: Arc<dyn FetchDnsResolver>,
421 http: ProviderHttpConfig,
422}
423
424impl FetchHttpTransport {
425 #[must_use]
427 pub fn new(
428 config: FetchHttpConfig,
429 resolver: Arc<dyn FetchDnsResolver>,
430 http: ProviderHttpConfig,
431 ) -> Self {
432 Self {
433 config,
434 resolver,
435 http,
436 }
437 }
438
439 #[must_use]
441 pub const fn url_policy(&self) -> FetchUrlPolicy {
442 self.config.url_policy()
443 }
444
445 #[must_use]
447 pub const fn config(&self) -> FetchHttpConfig {
448 self.config
449 }
450
451 #[must_use]
453 pub fn production(http: ProviderHttpConfig) -> Self {
454 Self::new(
455 FetchHttpConfig::production(),
456 Arc::new(SystemFetchDnsResolver),
457 http,
458 )
459 }
460
461 pub async fn get(
468 &self,
469 url: &str,
470 cancellation: CancellationScope,
471 ) -> Result<FetchHttpResponse, FetchProviderError> {
472 if cancellation.is_cancelled() {
473 return Err(FetchProviderError::cancelled());
474 }
475 let request = self
476 .config
477 .url_policy()
478 .validate(url)
479 .map_err(|_| FetchProviderError::new(FetchProviderErrorCode::InvalidRequest))?;
480 let operation = self.get_inner(request, cancellation.child());
481 tokio::select! {
482 biased;
483 () = cancellation.cancelled() => Err(FetchProviderError::cancelled()),
484 result = tokio::time::timeout(self.config.timeouts().total(), operation) => {
485 result.unwrap_or_else(|_| Err(FetchProviderError::new(FetchProviderErrorCode::Timeout)))
486 }
487 }
488 }
489
490 async fn get_inner(
491 &self,
492 requested: ValidatedFetchUrl,
493 cancellation: CancellationScope,
494 ) -> Result<FetchHttpResponse, FetchProviderError> {
495 let mut current = requested.clone();
496 let mut visited = BTreeSet::from([requested.as_str().to_owned()]);
497 let mut redirects = Vec::new();
498
499 loop {
500 let addresses = self.resolve(¤t, cancellation.child()).await?;
501 let response = self
502 .send_to_validated_peer(¤t, &addresses, cancellation.child())
503 .await?;
504 if is_redirect(response.status()) {
505 if redirects.len() >= self.config.limits().max_redirects() {
506 return Err(FetchProviderError::new(
507 FetchProviderErrorCode::RedirectRejected,
508 ));
509 }
510 let location = response
511 .headers()
512 .get(LOCATION)
513 .ok_or_else(|| {
514 FetchProviderError::new(FetchProviderErrorCode::RedirectRejected)
515 })?
516 .to_str()
517 .map_err(|_| {
518 FetchProviderError::new(FetchProviderErrorCode::RedirectRejected)
519 })?;
520 let next = current
521 .resolve_redirect(location, &self.config.url_policy())
522 .map_err(|_| {
523 FetchProviderError::new(FetchProviderErrorCode::RedirectRejected)
524 })?;
525 if current.is_https() && !next.is_https() {
526 return Err(FetchProviderError::new(
527 FetchProviderErrorCode::RedirectRejected,
528 ));
529 }
530 if !visited.insert(next.as_str().to_owned()) {
531 return Err(FetchProviderError::new(
532 FetchProviderErrorCode::RedirectRejected,
533 ));
534 }
535 redirects.push(FetchRedirect::new_with_policy(
536 current.as_str(),
537 next.as_str(),
538 response.status().as_u16(),
539 &self.config.url_policy(),
540 )?);
541 current = next;
542 continue;
543 }
544
545 let status = response.status().as_u16();
546 let headers = response_headers(&response)?;
547 let body = read_raw_body(
548 response,
549 self.config.timeouts(),
550 self.config.limits().max_response_bytes(),
551 cancellation.child(),
552 )
553 .await?;
554 return Ok(FetchHttpResponse {
555 requested_url: requested.as_str().to_owned(),
556 final_url: current.as_str().to_owned(),
557 status,
558 headers,
559 body,
560 redirects,
561 });
562 }
563 }
564
565 async fn resolve(
566 &self,
567 url: &ValidatedFetchUrl,
568 cancellation: CancellationScope,
569 ) -> Result<ValidatedFetchAddresses, FetchProviderError> {
570 let resolution = self.resolver.resolve(url.host(), cancellation.child());
571 let addresses = tokio::select! {
572 biased;
573 () = cancellation.cancelled() => return Err(FetchProviderError::cancelled()),
574 result = tokio::time::timeout(self.config.timeouts().dns(), resolution) => {
575 result
576 .map_err(|_| FetchProviderError::new(FetchProviderErrorCode::Timeout))??
577 }
578 };
579 self.config
580 .address_policy()
581 .validate(addresses)
582 .map_err(|_| FetchProviderError::new(FetchProviderErrorCode::DnsResolution))
583 }
584
585 async fn send_to_validated_peer(
586 &self,
587 url: &ValidatedFetchUrl,
588 addresses: &ValidatedFetchAddresses,
589 cancellation: CancellationScope,
590 ) -> Result<reqwest::Response, FetchProviderError> {
591 let mut last_error = FetchProviderError::explicit_retry(FetchProviderErrorCode::Transport);
592 for address in addresses.socket_addresses(url.port()) {
593 let client = self
594 .http
595 .build_pinned_client_without_redirects(
596 self.config.timeouts().total(),
597 self.config.timeouts().connect(),
598 url.host(),
599 address,
600 )
601 .map_err(|_| {
602 FetchProviderError::new(FetchProviderErrorCode::InvalidConfiguration)
603 })?;
604 let send = client
605 .get(url.as_str())
606 .header(ACCEPT, FETCH_ACCEPT_VALUE)
607 .header(ACCEPT_ENCODING, FETCH_ACCEPT_ENCODING_VALUE)
608 .send();
609 let response = tokio::select! {
610 biased;
611 () = cancellation.cancelled() => return Err(FetchProviderError::cancelled()),
612 result = tokio::time::timeout(self.config.timeouts().first_byte(), send) => {
613 match result {
614 Ok(Ok(response)) => response,
615 Ok(Err(error)) => {
616 last_error = map_pre_response_error(&error);
617 continue;
618 }
619 Err(_) => {
620 last_error = FetchProviderError::explicit_retry(
621 FetchProviderErrorCode::Timeout,
622 );
623 continue;
624 }
625 }
626 }
627 };
628 if response.remote_addr() != Some(address) {
629 return Err(FetchProviderError::new(
630 FetchProviderErrorCode::ForbiddenDestination,
631 ));
632 }
633 return Ok(response);
634 }
635 Err(last_error)
636 }
637}
638
639impl fmt::Debug for FetchHttpTransport {
640 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
641 formatter
642 .debug_struct("FetchHttpTransport")
643 .field("config", &self.config)
644 .field("resolver", &self.resolver)
645 .finish_non_exhaustive()
646 }
647}
648
649fn response_headers(response: &reqwest::Response) -> Result<FetchHttpHeaders, FetchProviderError> {
650 let content_type = bounded_header(response.headers().get(CONTENT_TYPE))?;
651 let content_encoding = bounded_header(response.headers().get(CONTENT_ENCODING))?;
652 let content_length = response
653 .headers()
654 .get(CONTENT_LENGTH)
655 .map(|value| {
656 value
657 .to_str()
658 .ok()
659 .and_then(|value| value.parse::<u64>().ok())
660 .ok_or_else(|| FetchProviderError::new(FetchProviderErrorCode::MalformedResponse))
661 })
662 .transpose()?;
663 FetchHttpHeaders::new(content_type, content_encoding, content_length)
664}
665
666fn bounded_header(
667 value: Option<&reqwest::header::HeaderValue>,
668) -> Result<Option<String>, FetchProviderError> {
669 value
670 .map(|value| {
671 let value = value
672 .to_str()
673 .map_err(|_| FetchProviderError::new(FetchProviderErrorCode::MalformedResponse))?;
674 if value.len() > MAX_RESPONSE_HEADER_VALUE_BYTES {
675 return Err(FetchProviderError::new(
676 FetchProviderErrorCode::MalformedResponse,
677 ));
678 }
679 Ok(value.to_owned())
680 })
681 .transpose()
682}
683
684fn validate_header_value(value: String) -> Result<String, FetchProviderError> {
685 if value.len() > MAX_RESPONSE_HEADER_VALUE_BYTES || value.chars().any(char::is_control) {
686 Err(FetchProviderError::new(
687 FetchProviderErrorCode::MalformedResponse,
688 ))
689 } else {
690 Ok(value)
691 }
692}
693
694async fn read_raw_body(
695 response: reqwest::Response,
696 timeouts: FetchHttpTimeouts,
697 max_bytes: usize,
698 cancellation: CancellationScope,
699) -> Result<Vec<u8>, FetchProviderError> {
700 if response
701 .content_length()
702 .is_some_and(|length| length > max_bytes as u64)
703 {
704 return Err(FetchProviderError::new(
705 FetchProviderErrorCode::ResponseTooLarge,
706 ));
707 }
708 let mut stream = response.bytes_stream();
709 let mut bytes = Vec::new();
710 let mut window_started = tokio::time::Instant::now();
711 let mut window_bytes = 0_usize;
712 loop {
713 let remaining = timeouts
714 .low_speed_window()
715 .saturating_sub(window_started.elapsed());
716 if remaining.is_zero() {
717 if window_bytes < timeouts.low_speed_min_bytes() {
718 return Err(FetchProviderError::new(FetchProviderErrorCode::Timeout));
719 }
720 window_started = tokio::time::Instant::now();
721 window_bytes = 0;
722 continue;
723 }
724 let next = tokio::select! {
725 biased;
726 () = cancellation.cancelled() => return Err(FetchProviderError::cancelled()),
727 result = tokio::time::timeout(remaining, stream.next()) => {
728 result.map_err(|_| FetchProviderError::new(FetchProviderErrorCode::Timeout))?
729 }
730 };
731 let Some(chunk) = next else {
732 break;
733 };
734 let chunk = chunk.map_err(|error| map_response_body_error(&error))?;
735 if bytes.len().saturating_add(chunk.len()) > max_bytes {
736 return Err(FetchProviderError::new(
737 FetchProviderErrorCode::ResponseTooLarge,
738 ));
739 }
740 window_bytes = window_bytes.saturating_add(chunk.len());
741 bytes.extend_from_slice(&chunk);
742 }
743 Ok(bytes)
744}
745
746fn is_redirect(status: reqwest::StatusCode) -> bool {
747 matches!(status.as_u16(), 301 | 302 | 303 | 307 | 308)
748}
749
750fn map_pre_response_error(error: &reqwest::Error) -> FetchProviderError {
751 if error.is_timeout() {
752 FetchProviderError::explicit_retry(FetchProviderErrorCode::Timeout)
753 } else {
754 FetchProviderError::explicit_retry(FetchProviderErrorCode::Transport)
755 }
756}
757
758fn map_response_body_error(error: &reqwest::Error) -> FetchProviderError {
759 if error.is_timeout() {
760 FetchProviderError::new(FetchProviderErrorCode::Timeout)
761 } else {
762 FetchProviderError::new(FetchProviderErrorCode::Transport)
763 }
764}