1use std::future::Future;
2use std::io;
3use std::net::{IpAddr, SocketAddr};
4use std::pin::Pin;
5use std::sync::{Arc, OnceLock};
6use std::time::{Duration, Instant};
7
8use bytes::Bytes;
9use http::header::{self, HeaderMap, HeaderName, HeaderValue};
10use http::{Method, StatusCode};
11use tokio::net::TcpStream;
12use url::Url;
13
14use rustls_platform_verifier::BuilderVerifierExt;
15
16use crate::policy::Sources;
17use crate::{Error, Policy};
18
19pub trait Resolve: Send + Sync + 'static {
25 fn resolve<'a>(
27 &'a self,
28 host: &'a str,
29 port: u16,
30 ) -> Pin<Box<dyn Future<Output = io::Result<Vec<SocketAddr>>> + Send + 'a>>;
31}
32
33struct System;
34
35impl Resolve for System {
36 fn resolve<'a>(
37 &'a self,
38 host: &'a str,
39 port: u16,
40 ) -> Pin<Box<dyn Future<Output = io::Result<Vec<SocketAddr>>> + Send + 'a>> {
41 Box::pin(async move { Ok(tokio::net::lookup_host((host, port)).await?.collect()) })
42 }
43}
44
45struct Unpinned;
50
51impl reqwest::dns::Resolve for Unpinned {
52 fn resolve(&self, name: reqwest::dns::Name) -> reqwest::dns::Resolving {
53 let name = name.as_str().to_string();
54 Box::pin(async move { Err(format!("{name} was not checked against the policy").into()) })
55 }
56}
57
58const REFUSED_HEADERS: &[&str] = &[
63 "connection",
64 "content-length",
65 "expect",
66 "host",
67 "keep-alive",
68 "proxy-authorization",
69 "proxy-connection",
70 "te",
71 "trailer",
72 "transfer-encoding",
73 "upgrade",
74];
75
76#[derive(Clone)]
78#[must_use]
79pub struct Request {
80 method: Method,
81 url: Url,
82 headers: HeaderMap,
83 body: Option<Vec<u8>>,
84 refused: Option<HeaderName>,
85}
86
87impl std::fmt::Debug for Request {
90 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
91 f.debug_struct("Request")
92 .field("method", &self.method)
93 .field("origin", &self.url.origin().ascii_serialization())
94 .field("headers", &self.headers.keys().collect::<Vec<_>>())
95 .field("body_len", &self.body.as_ref().map(Vec::len))
96 .finish()
97 }
98}
99
100impl Request {
101 pub fn new(method: Method, url: Url) -> Request {
103 Request {
104 method,
105 url,
106 headers: HeaderMap::new(),
107 body: None,
108 refused: None,
109 }
110 }
111
112 pub fn header(mut self, name: HeaderName, mut value: HeaderValue) -> Request {
120 if REFUSED_HEADERS.contains(&name.as_str()) {
121 self.refused.get_or_insert(name);
122 return self;
123 }
124 if name == header::AUTHORIZATION || name == header::COOKIE {
125 value.set_sensitive(true);
126 }
127 self.headers.append(name, value);
128 self
129 }
130
131 pub fn body(mut self, body: impl Into<Vec<u8>>) -> Request {
135 self.body = Some(body.into());
136 self
137 }
138}
139
140pub struct Response {
142 inner: reqwest::Response,
143 url: Url,
144 timeout: Duration,
145 max_body: usize,
146 read: usize,
147}
148
149impl std::fmt::Debug for Response {
151 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
152 f.debug_struct("Response")
153 .field("status", &self.status())
154 .field("origin", &self.url.origin().ascii_serialization())
155 .field("headers", &self.headers().keys().collect::<Vec<_>>())
156 .field("remote_addr", &self.remote_addr())
157 .field("read", &self.read)
158 .finish()
159 }
160}
161
162impl Response {
163 pub fn status(&self) -> StatusCode {
165 self.inner.status()
166 }
167
168 pub fn headers(&self) -> &HeaderMap {
170 self.inner.headers()
171 }
172
173 pub fn url(&self) -> &Url {
175 &self.url
176 }
177
178 pub fn remote_addr(&self) -> Option<SocketAddr> {
180 self.inner.remote_addr()
181 }
182
183 pub async fn chunk(&mut self) -> Result<Option<Bytes>, Error> {
187 let Some(chunk) = self.next_chunk().await? else {
188 return Ok(None);
189 };
190 if self.read + chunk.len() > self.max_body {
191 return Err(Error::BodyTooLarge(self.max_body));
192 }
193 self.read += chunk.len();
194 Ok(Some(chunk))
195 }
196
197 pub async fn bytes(mut self) -> Result<Vec<u8>, Error> {
200 let mut body = Vec::new();
201 while let Some(chunk) = self.chunk().await? {
202 body.extend_from_slice(&chunk);
203 }
204 Ok(body)
205 }
206
207 pub async fn bytes_truncated(mut self) -> Result<(Vec<u8>, bool), Error> {
210 let mut body = Vec::new();
211 while let Some(chunk) = self.next_chunk().await? {
212 let room = self.max_body - body.len();
213 if chunk.len() > room {
214 body.extend_from_slice(&chunk[..room]);
215 return Ok((body, true));
216 }
217 body.extend_from_slice(&chunk);
218 }
219 Ok((body, false))
220 }
221
222 pub async fn text(self) -> Result<String, Error> {
224 let body = self.bytes().await?;
225 Ok(String::from_utf8(body)
226 .unwrap_or_else(|e| String::from_utf8_lossy(e.as_bytes()).into_owned()))
227 }
228
229 async fn next_chunk(&mut self) -> Result<Option<Bytes>, Error> {
230 self.inner
231 .chunk()
232 .await
233 .map_err(|e| http_error(e, self.timeout))
234 }
235}
236
237#[derive(Clone)]
243#[must_use]
244pub struct Fetcher {
245 policy: Arc<Policy>,
246 resolver: Arc<dyn Resolve>,
247 user_agent: Option<HeaderValue>,
248}
249
250impl std::fmt::Debug for Fetcher {
251 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
252 f.debug_struct("Fetcher")
253 .field("policy", &self.policy)
254 .field("user_agent", &self.user_agent)
255 .finish()
256 }
257}
258
259impl Default for Fetcher {
260 fn default() -> Fetcher {
261 Fetcher::new(Policy::default())
262 }
263}
264
265const MIN_ATTEMPT: Duration = Duration::from_secs(1);
268
269impl Fetcher {
270 pub fn new(policy: Policy) -> Fetcher {
272 Fetcher {
273 policy: Arc::new(policy),
274 resolver: Arc::new(System),
275 user_agent: None,
276 }
277 }
278
279 pub fn with_resolver(mut self, resolver: impl Resolve) -> Fetcher {
281 self.resolver = Arc::new(resolver);
282 self
283 }
284
285 pub fn user_agent(mut self, value: HeaderValue) -> Fetcher {
287 self.user_agent = Some(value);
288 self
289 }
290
291 pub fn policy(&self) -> &Policy {
293 &self.policy
294 }
295
296 pub async fn get(&self, url: &str) -> Result<Response, Error> {
298 let url = Url::parse(url).map_err(|e| Error::InvalidUrl(format!("not a URL: {e}")))?;
300 self.send(Request::new(Method::GET, url)).await
301 }
302
303 pub async fn send(&self, request: Request) -> Result<Response, Error> {
309 if let Some(name) = &request.refused {
310 return Err(Error::RefusedHeader(name.to_string()));
311 }
312 let started = Instant::now();
313 let budget = self.policy.timeout;
314 match tokio::time::timeout(budget, self.follow(request, started)).await {
315 Ok(result) => result,
316 Err(_) => Err(Error::Timeout(budget)),
317 }
318 }
319
320 async fn follow(&self, request: Request, started: Instant) -> Result<Response, Error> {
321 let Request {
322 mut method,
323 mut url,
324 mut headers,
325 body,
326 refused: _,
327 } = request;
328 let budget = self.policy.timeout;
329 let max = self.policy.max_redirects;
330 let (mut host, mut port) = target(&url)?;
331 let mut send_body = body.is_some();
332 let mut trusted = true;
334 for hop in 0..=max {
335 let addrs = self.resolve(&host, port).await?;
336 let remaining = budget.saturating_sub(started.elapsed());
337 let client = self.client(&host, addrs, remaining)?;
338
339 let mut req = client.request(method.clone(), url.clone());
340 if trusted {
341 req = req.headers(headers.clone());
342 }
343 if send_body {
344 req = req.body(body.clone().unwrap_or_default());
345 }
346 let resp = req.send().await.map_err(|e| http_error(e, budget))?;
347
348 let status = resp.status().as_u16();
349 let follow = matches!(status, 301 | 302 | 303 | 307 | 308) && max > 0;
350 let location = resp.headers().get(header::LOCATION).filter(|_| follow);
351 let Some(location) = location else {
352 return Ok(Response {
353 inner: resp,
354 url,
355 timeout: budget,
356 max_body: self.policy.max_body,
357 read: 0,
358 });
359 };
360 if hop == max {
361 return Err(Error::TooManyRedirects(max));
362 }
363 let location = std::str::from_utf8(location.as_bytes()).map_err(|_| {
366 Error::BadRedirect(format!("HTTP {status} with a Location that isn't UTF-8"))
367 })?;
368 let next = url.join(location).map_err(|_| {
369 Error::BadRedirect(format!("HTTP {status} to an unreadable Location"))
370 })?;
371 (host, port) = target(&next).map_err(|e| Error::BadRedirect(e.to_string()))?;
372 trusted = trusted && same_origin(&url, &next);
373 let (next_method, keep_body) = redirect_method(&method, status);
374 let keep_body = send_body && keep_body && trusted;
377 if send_body && !keep_body {
378 for name in [
379 header::CONTENT_TYPE,
380 header::CONTENT_ENCODING,
381 header::CONTENT_LANGUAGE,
382 header::CONTENT_LOCATION,
383 ] {
384 headers.remove(name);
385 }
386 }
387 send_body = keep_body;
388 method = next_method;
389 url = next;
390 }
391 Err(Error::TooManyRedirects(max))
392 }
393
394 fn client(
396 &self,
397 host: &str,
398 addrs: Vec<SocketAddr>,
399 remaining: Duration,
400 ) -> Result<reqwest::Client, Error> {
401 let from = self.policy.sources.for_peer(addrs[0].ip()).ok().flatten();
402 let pinned: Vec<SocketAddr> = match from {
403 Some(from) => addrs
406 .into_iter()
407 .filter(|a| a.is_ipv4() == from.is_ipv4())
408 .collect(),
409 None => addrs,
410 };
411 let builder = || {
412 let mut b = reqwest::Client::builder()
413 .redirect(reqwest::redirect::Policy::none())
414 .no_proxy()
417 .dns_resolver(Unpinned)
418 .connect_timeout(remaining)
419 .timeout(remaining)
420 .local_address(from);
421 if let Some(ua) = &self.user_agent {
422 b = b.user_agent(ua.clone());
423 }
424 if host.parse::<IpAddr>().is_err() {
425 b = b.resolve_to_addrs(host, &pinned);
426 }
427 b
428 };
429 builder()
434 .tls_backend_preconfigured(tls_config()?)
435 .build()
436 .map_err(|e| Error::Tls(e.without_url().to_string()))
437 }
438
439 pub async fn resolve(&self, host: &str, port: u16) -> Result<Vec<SocketAddr>, Error> {
447 let bare = host
448 .strip_prefix('[')
449 .and_then(|h| h.strip_suffix(']'))
450 .unwrap_or(host);
451 let mut addrs = match bare.parse::<IpAddr>() {
452 Ok(ip) => vec![SocketAddr::new(ip, port)],
453 Err(_) => self
454 .resolver
455 .resolve(bare, port)
456 .await
457 .map_err(|source| Error::Resolve {
458 host: host.to_string(),
459 source,
460 })?,
461 };
462 if addrs.is_empty() {
463 let source = io::Error::new(io::ErrorKind::NotFound, "no addresses");
464 return Err(Error::Resolve {
465 host: host.to_string(),
466 source,
467 });
468 }
469 for a in &mut addrs {
472 a.set_port(port);
473 }
474 if let Some(bad) = addrs.iter().find(|a| !self.policy.allows(a.ip())) {
475 return Err(Error::Denied {
476 host: host.to_string(),
477 addr: bad.ip(),
478 });
479 }
480 let reachable = self.policy.sources.reachable(addrs);
481 if reachable.is_empty() {
482 return Err(Error::NoSource {
483 host: host.to_string(),
484 });
485 }
486 Ok(reachable)
487 }
488
489 pub async fn connect(&self, host: &str, port: u16) -> Result<TcpStream, Error> {
494 let budget = self.policy.timeout;
495 let started = Instant::now();
496 let work = async {
497 let addrs = self.resolve(host, port).await?;
498 let count = addrs.len();
499 let mut last = None;
500 for (i, addr) in addrs.into_iter().enumerate() {
501 let left = budget.saturating_sub(started.elapsed());
502 let share = (left / (count - i) as u32).max(MIN_ATTEMPT.min(left));
503 match tokio::time::timeout(share, connect_with(self.policy.sources, addr)).await {
504 Ok(Ok(stream)) => return Ok(stream),
505 Ok(Err(e)) => last = Some(e),
506 Err(_) => {
507 let msg = format!("{addr} didn't answer within {share:?}");
508 last = Some(io::Error::new(io::ErrorKind::TimedOut, msg));
509 }
510 }
511 }
512 let source = last.unwrap_or_else(|| io::Error::other("no address to try"));
513 Err(Error::Connect {
514 host: host.to_string(),
515 source,
516 })
517 };
518 tokio::time::timeout(budget, work)
519 .await
520 .unwrap_or(Err(Error::Timeout(budget)))
521 }
522}
523
524async fn connect_with(sources: Sources, to: SocketAddr) -> io::Result<TcpStream> {
525 let from = sources
526 .for_peer(to.ip())
527 .map_err(|_| io::Error::other(format!("no source address in the family of {}", to.ip())))?;
528 let Some(from) = from else {
529 return TcpStream::connect(to).await;
530 };
531 let socket = match to {
532 SocketAddr::V4(_) => tokio::net::TcpSocket::new_v4()?,
533 SocketAddr::V6(_) => tokio::net::TcpSocket::new_v6()?,
534 };
535 socket.bind(SocketAddr::new(from, 0))?;
536 socket.connect(to).await
537}
538
539fn target(url: &Url) -> Result<(String, u16), Error> {
544 if !url.username().is_empty() || url.password().is_some() {
547 return Err(Error::InvalidUrl(
548 "the URL has a user name or password in it".to_string(),
549 ));
550 }
551 if !matches!(url.scheme(), "http" | "https") {
552 return Err(Error::InvalidUrl(format!(
553 "{}: URLs can't be fetched, only http and https",
554 url.scheme()
555 )));
556 }
557 let host = match url.host_str() {
558 Some(h) if !h.is_empty() => h.trim_start_matches('[').trim_end_matches(']').to_string(),
559 _ => return Err(Error::InvalidUrl("the URL has no host".to_string())),
560 };
561 let port = url.port_or_known_default().unwrap_or(80);
562 Ok((host, port))
563}
564
565fn http_error(e: reqwest::Error, budget: Duration) -> Error {
566 if e.is_timeout() {
567 Error::Timeout(budget)
568 } else {
569 Error::Http(Box::new(e.without_url()))
571 }
572}
573
574fn redirect_method(method: &Method, status: u16) -> (Method, bool) {
578 let get = (Method::GET, false);
579 match status {
580 303 if *method != Method::HEAD => get,
581 301 | 302 if *method == Method::POST => get,
582 _ => (method.clone(), true),
583 }
584}
585
586fn same_origin(a: &Url, b: &Url) -> bool {
590 a.scheme() == b.scheme()
591 && a.host_str() == b.host_str()
592 && a.port_or_known_default() == b.port_or_known_default()
593}
594
595fn crypto_provider() -> Arc<rustls::crypto::CryptoProvider> {
596 match rustls::crypto::CryptoProvider::get_default() {
597 Some(p) => p.clone(),
598 None => Arc::new(rustls::crypto::aws_lc_rs::default_provider()),
599 }
600}
601
602fn tls_config() -> Result<rustls::ClientConfig, Error> {
613 static TLS: OnceLock<rustls::ClientConfig> = OnceLock::new();
614 if let Some(tls) = TLS.get() {
615 return Ok(tls.clone());
616 }
617 let mut tls = rustls::ClientConfig::builder_with_provider(crypto_provider())
618 .with_safe_default_protocol_versions()
619 .map_err(|e| Error::Tls(e.to_string()))?
620 .with_platform_verifier()
621 .map_err(|e| Error::Tls(format!("no trust store to verify against: {e}")))?
622 .with_no_client_auth();
623 tls.alpn_protocols = vec![b"http/1.1".to_vec()];
625 tls.resumption = rustls::client::Resumption::disabled();
626 Ok(TLS.get_or_init(|| tls).clone())
627}
628
629#[cfg(test)]
630mod tests {
631 use super::*;
632
633 #[test]
634 fn reqwest_takes_the_shared_tls_config() {
635 reqwest::Client::builder()
638 .tls_backend_preconfigured(tls_config().unwrap())
639 .build()
640 .unwrap();
641 }
642
643 #[test]
644 fn request_debug_hides_secrets() {
645 let url = Url::parse("https://example.com/hooks/s-789?token=q-012").unwrap();
646 let req = Request::new(Method::POST, url)
647 .header(
648 header::AUTHORIZATION,
649 HeaderValue::from_static("Bearer t-456"),
650 )
651 .header(
652 HeaderName::from_static("x-api-key"),
653 HeaderValue::from_static("k-123"),
654 )
655 .body("hunter2");
656 let shown = format!("{req:?}");
657 assert!(shown.contains("x-api-key"), "{shown}");
658 assert!(shown.contains("https://example.com"), "{shown}");
659 for secret in ["t-456", "k-123", "hunter2", "s-789", "q-012"] {
660 assert!(!shown.contains(secret), "{shown}");
661 }
662 assert!(req.headers[header::AUTHORIZATION].is_sensitive());
663 }
664
665 #[test]
666 fn redirect_methods() {
667 assert_eq!(redirect_method(&Method::POST, 303), (Method::GET, false));
668 assert_eq!(redirect_method(&Method::PUT, 303), (Method::GET, false));
669 assert_eq!(redirect_method(&Method::HEAD, 303), (Method::HEAD, true));
670 assert_eq!(redirect_method(&Method::POST, 301), (Method::GET, false));
671 assert_eq!(redirect_method(&Method::POST, 302), (Method::GET, false));
672 assert_eq!(redirect_method(&Method::PUT, 302), (Method::PUT, true));
673 assert_eq!(redirect_method(&Method::POST, 307), (Method::POST, true));
674 assert_eq!(redirect_method(&Method::PATCH, 308), (Method::PATCH, true));
675 assert_eq!(redirect_method(&Method::GET, 301), (Method::GET, true));
676 }
677
678 #[test]
679 fn origins() {
680 let u = |s: &str| Url::parse(s).unwrap();
681 assert!(same_origin(
682 &u("https://example.com/a"),
683 &u("https://example.com:443/b?c")
684 ));
685 assert!(!same_origin(
686 &u("https://example.com/a"),
687 &u("https://evil.example/a")
688 ));
689 assert!(
690 !same_origin(&u("https://example.com/a"), &u("http://example.com/a")),
691 "a downgrade is another origin"
692 );
693 assert!(!same_origin(
694 &u("https://example.com/a"),
695 &u("https://example.com:8443/a")
696 ));
697 assert!(!same_origin(
698 &u("https://example.com/a"),
699 &u("https://www.example.com/a")
700 ));
701 assert!(
702 same_origin(&u("https://example.com/a"), &u("https://EXAMPLE.com/b")),
703 "host case is not an origin"
704 );
705 }
706
707 #[test]
708 fn targets() {
709 let t = |s: &str| target(&Url::parse(s).unwrap());
710 assert_eq!(
711 t("https://example.com/x").unwrap(),
712 ("example.com".into(), 443)
713 );
714 assert_eq!(
715 t("http://[2001:db8::1]:8080/").unwrap(),
716 ("2001:db8::1".into(), 8080)
717 );
718 assert!(t("ftp://example.com/").is_err());
719 assert!(t("https://user:pw@example.com/").is_err());
720 assert!(t("https://user@example.com/").is_err());
721 assert!(t("file:///etc/passwd").is_err());
722 }
723
724 #[test]
725 fn url_errors_dont_repeat_the_url() {
726 let t = |s: &str| target(&Url::parse(s).unwrap()).unwrap_err().to_string();
727 for (url, secret) in [
728 ("https://user:pw-345@example.com/", "pw-345"),
729 ("ftp://user:pw-345@example.com/", "pw-345"),
730 ("ftp://example.com/?token=q-012", "q-012"),
731 ] {
732 let shown = t(url);
733 assert!(!shown.contains(secret), "{shown}");
734 }
735 }
736}