use std::error::Error as StdError;
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr};
use std::sync::Arc;
use std::time::Duration;
use reqwest::dns::{Addrs, Name, Resolve, Resolving};
use reqwest::redirect::Policy;
use thiserror::Error;
const MAX_REDIRECTS: usize = 3;
const PRIVATE_ADDRESS_MARKER: &str = "not globally routable";
#[derive(Debug, Error)]
pub enum RemoteFetchError {
#[error("invalid remote url: {0}")]
InvalidUrl(String),
#[error("only http:// and https:// urls are supported")]
UnsupportedScheme,
#[error("remote host resolves to a non-public address")]
PrivateAddress,
#[error("too many redirects")]
TooManyRedirects,
#[error("remote connection failed: {0}")]
ConnectionFailed(String),
#[error("remote request timed out")]
Timeout,
#[error("remote server returned not found")]
NotFound,
#[error("remote server returned a server error (status {0})")]
ServerError(u16),
#[error("remote response exceeded the size limit")]
ResponseTooLarge,
}
#[derive(Debug, Clone)]
pub struct RemoteFetchResult {
pub data: Vec<u8>,
pub content_type: String,
pub status_code: u16,
}
pub struct RemoteFetcher {
max_size: usize,
http: reqwest::Client,
enforce_private_address_guard: bool,
}
impl RemoteFetcher {
pub fn new(timeout_ms: u32, max_size: usize) -> Self {
Self::build(timeout_ms, max_size, true)
}
#[cfg(test)]
fn new_without_private_address_guard_for_tests(timeout_ms: u32, max_size: usize) -> Self {
Self::build(timeout_ms, max_size, false)
}
fn build(timeout_ms: u32, max_size: usize, enforce_private_address_guard: bool) -> Self {
let redirect_policy = Policy::custom(move |attempt| {
if attempt.previous().len() >= MAX_REDIRECTS {
return attempt.error("too many redirects");
}
if !matches!(attempt.url().scheme(), "http" | "https") {
return attempt.error("redirect to unsupported scheme");
}
if enforce_private_address_guard && literal_ip_host_is_private(attempt.url()) {
return attempt.error(PRIVATE_ADDRESS_MARKER);
}
attempt.follow()
});
let mut builder = reqwest::Client::builder()
.timeout(Duration::from_millis(timeout_ms as u64))
.user_agent("imgx/1.0")
.redirect(redirect_policy);
if enforce_private_address_guard {
builder = builder.dns_resolver(Arc::new(SafeResolver));
}
let http = builder.build().expect("reqwest client builder");
Self {
max_size,
http,
enforce_private_address_guard,
}
}
pub async fn fetch(&self, url: &str) -> Result<RemoteFetchResult, RemoteFetchError> {
let parsed =
url::Url::parse(url).map_err(|e| RemoteFetchError::InvalidUrl(e.to_string()))?;
if !matches!(parsed.scheme(), "http" | "https") {
return Err(RemoteFetchError::UnsupportedScheme);
}
if self.enforce_private_address_guard && literal_ip_host_is_private(&parsed) {
return Err(RemoteFetchError::PrivateAddress);
}
let mut resp = self
.http
.get(parsed)
.send()
.await
.map_err(classify_send_error)?;
let status_code = resp.status().as_u16();
if (300..400).contains(&status_code) {
return Err(RemoteFetchError::UnsupportedScheme);
}
if status_code == 404 {
return Err(RemoteFetchError::NotFound);
}
if status_code >= 500 {
return Err(RemoteFetchError::ServerError(status_code));
}
let content_type = resp
.headers()
.get(reqwest::header::CONTENT_TYPE)
.and_then(|v| v.to_str().ok())
.unwrap_or("application/octet-stream")
.to_string();
if resp
.content_length()
.is_some_and(|len| len as usize > self.max_size)
{
return Err(RemoteFetchError::ResponseTooLarge);
}
let mut data = Vec::new();
while let Some(chunk) = resp
.chunk()
.await
.map_err(|e| RemoteFetchError::ConnectionFailed(e.to_string()))?
{
data.extend_from_slice(&chunk);
if data.len() > self.max_size {
return Err(RemoteFetchError::ResponseTooLarge);
}
}
Ok(RemoteFetchResult {
data,
content_type,
status_code,
})
}
}
fn literal_ip_host_is_private(url: &url::Url) -> bool {
match url.host() {
Some(url::Host::Ipv4(ip)) => !is_globally_routable(IpAddr::V4(ip)),
Some(url::Host::Ipv6(ip)) => !is_globally_routable(IpAddr::V6(ip)),
_ => false,
}
}
fn classify_send_error(e: reqwest::Error) -> RemoteFetchError {
if e.is_timeout() {
return RemoteFetchError::Timeout;
}
if error_chain_contains(&e, PRIVATE_ADDRESS_MARKER) {
return RemoteFetchError::PrivateAddress;
}
if e.is_redirect() {
if error_chain_contains(&e, "too many redirects") {
return RemoteFetchError::TooManyRedirects;
}
return RemoteFetchError::UnsupportedScheme;
}
RemoteFetchError::ConnectionFailed(e.to_string())
}
fn error_chain_contains(e: &reqwest::Error, needle: &str) -> bool {
if e.to_string().contains(needle) {
return true;
}
let mut source: Option<&(dyn StdError + 'static)> = e.source();
while let Some(s) = source {
if s.to_string().contains(needle) {
return true;
}
source = s.source();
}
false
}
struct SafeResolver;
impl Resolve for SafeResolver {
fn resolve(&self, name: Name) -> Resolving {
let host = name.as_str().to_string();
Box::pin(async move {
let lookup_target = format!("{host}:0");
let resolved: Vec<SocketAddr> = tokio::net::lookup_host(lookup_target)
.await
.map_err(|e| -> Box<dyn std::error::Error + Send + Sync> { Box::new(e) })?
.collect();
let safe: Vec<SocketAddr> = resolved
.into_iter()
.filter(|addr| is_globally_routable(addr.ip()))
.collect();
if safe.is_empty() {
return Err("resolved address is not globally routable".into());
}
let addrs: Addrs = Box::new(safe.into_iter());
Ok(addrs)
})
}
}
pub fn is_globally_routable(ip: IpAddr) -> bool {
match ip {
IpAddr::V4(v4) => is_v4_globally_routable(v4),
IpAddr::V6(v6) => match v6.to_ipv4_mapped() {
Some(mapped) => is_v4_globally_routable(mapped),
None => is_v6_globally_routable(v6),
},
}
}
fn is_v4_globally_routable(ip: Ipv4Addr) -> bool {
if ip.is_private()
|| ip.is_loopback()
|| ip.is_link_local()
|| ip.is_broadcast()
|| ip.is_multicast()
|| ip.is_unspecified()
|| ip.is_documentation()
{
return false;
}
let o = ip.octets();
if o[0] == 0 {
return false; }
if o[0] == 100 && (o[1] & 0xC0) == 64 {
return false; }
true
}
fn is_v6_globally_routable(ip: Ipv6Addr) -> bool {
if ip.is_loopback() || ip.is_unspecified() || ip.is_multicast() || ip.is_unique_local() {
return false;
}
let seg0 = ip.segments()[0];
if seg0 & 0xffc0 == 0xfe80 {
return false; }
true
}
#[cfg(test)]
mod tests {
use super::*;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::TcpListener;
use wiremock::matchers::{method as wm_method, path as wm_path};
use wiremock::{Mock, MockServer, ResponseTemplate};
#[test]
fn rejects_ipv4_loopback() {
assert!(!is_globally_routable("127.0.0.1".parse().unwrap()));
}
#[test]
fn rejects_ipv4_rfc1918_private_ranges() {
assert!(!is_globally_routable("10.0.0.1".parse().unwrap()));
assert!(!is_globally_routable("172.16.0.1".parse().unwrap()));
assert!(!is_globally_routable("192.168.1.1".parse().unwrap()));
}
#[test]
fn rejects_ipv4_link_local() {
assert!(!is_globally_routable("169.254.1.1".parse().unwrap()));
}
#[test]
fn rejects_ipv4_cgnat_range() {
assert!(!is_globally_routable("100.64.0.1".parse().unwrap()));
assert!(!is_globally_routable("100.127.255.255".parse().unwrap()));
}
#[test]
fn accepts_ipv4_just_outside_cgnat_range() {
assert!(is_globally_routable("100.63.255.255".parse().unwrap()));
assert!(is_globally_routable("100.128.0.0".parse().unwrap()));
}
#[test]
fn rejects_ipv4_this_network_and_broadcast_and_multicast() {
assert!(!is_globally_routable("0.0.0.0".parse().unwrap()));
assert!(!is_globally_routable("0.1.2.3".parse().unwrap()));
assert!(!is_globally_routable("255.255.255.255".parse().unwrap()));
assert!(!is_globally_routable("224.0.0.1".parse().unwrap()));
}
#[test]
fn rejects_ipv6_loopback_and_link_local_and_unique_local() {
assert!(!is_globally_routable("::1".parse().unwrap()));
assert!(!is_globally_routable("fe80::1".parse().unwrap()));
assert!(!is_globally_routable("fc00::1".parse().unwrap()));
assert!(!is_globally_routable("fd12:3456:789a::1".parse().unwrap()));
}
#[test]
fn rejects_ipv4_mapped_ipv6_private_address() {
assert!(!is_globally_routable("::ffff:127.0.0.1".parse().unwrap()));
assert!(!is_globally_routable("::ffff:10.0.0.1".parse().unwrap()));
}
#[test]
fn accepts_public_ipv4_and_ipv6_addresses() {
assert!(is_globally_routable("8.8.8.8".parse().unwrap()));
assert!(is_globally_routable("1.1.1.1".parse().unwrap()));
assert!(is_globally_routable(
"2606:4700:4700::1111".parse().unwrap()
));
}
#[tokio::test]
async fn fetch_rejects_non_http_scheme_without_a_network_call() {
let fetcher = RemoteFetcher::new(5000, 10 * 1024 * 1024);
let result = fetcher.fetch("ftp://example.com/file.jpg").await;
assert!(matches!(result, Err(RemoteFetchError::UnsupportedScheme)));
}
#[tokio::test]
async fn fetch_rejects_file_scheme() {
let fetcher = RemoteFetcher::new(5000, 10 * 1024 * 1024);
let result = fetcher.fetch("file:///etc/passwd").await;
assert!(matches!(result, Err(RemoteFetchError::UnsupportedScheme)));
}
#[tokio::test]
async fn fetch_rejects_gopher_scheme() {
let fetcher = RemoteFetcher::new(5000, 10 * 1024 * 1024);
let result = fetcher.fetch("gopher://example.com/file.jpg").await;
assert!(matches!(result, Err(RemoteFetchError::UnsupportedScheme)));
}
#[tokio::test]
async fn fetch_rejects_invalid_url() {
let fetcher = RemoteFetcher::new(5000, 10 * 1024 * 1024);
let result = fetcher.fetch("not a url").await;
assert!(matches!(result, Err(RemoteFetchError::InvalidUrl(_))));
}
#[tokio::test]
async fn fetch_rejects_url_resolving_to_loopback() {
let server = MockServer::start().await;
Mock::given(wm_method("GET"))
.and(wm_path("/photo.jpg"))
.respond_with(ResponseTemplate::new(200).set_body_bytes(b"nope".to_vec()))
.mount(&server)
.await;
let fetcher = RemoteFetcher::new(5000, 10 * 1024 * 1024);
let url = format!("{}/photo.jpg", server.uri());
let result = fetcher.fetch(&url).await;
assert!(matches!(result, Err(RemoteFetchError::PrivateAddress)));
}
#[tokio::test]
async fn fetch_succeeds_against_a_real_server() {
let server = MockServer::start().await;
Mock::given(wm_method("GET"))
.and(wm_path("/photo.webp"))
.respond_with(
ResponseTemplate::new(200)
.set_body_bytes(b"remote-bytes".to_vec())
.insert_header("content-type", "image/webp"),
)
.mount(&server)
.await;
let fetcher =
RemoteFetcher::new_without_private_address_guard_for_tests(5000, 10 * 1024 * 1024);
let url = format!("{}/photo.webp", server.uri());
let result = fetcher.fetch(&url).await.unwrap();
assert_eq!(result.data, b"remote-bytes");
assert_eq!(result.content_type, "image/webp");
assert_eq!(result.status_code, 200);
}
#[tokio::test]
async fn fetch_returns_not_found_on_real_404() {
let server = MockServer::start().await;
Mock::given(wm_method("GET"))
.and(wm_path("/missing.jpg"))
.respond_with(ResponseTemplate::new(404))
.mount(&server)
.await;
let fetcher =
RemoteFetcher::new_without_private_address_guard_for_tests(5000, 10 * 1024 * 1024);
let url = format!("{}/missing.jpg", server.uri());
let result = fetcher.fetch(&url).await;
assert!(matches!(result, Err(RemoteFetchError::NotFound)));
}
#[tokio::test]
async fn fetch_returns_server_error_on_real_5xx() {
let server = MockServer::start().await;
Mock::given(wm_method("GET"))
.and(wm_path("/broken.jpg"))
.respond_with(ResponseTemplate::new(503))
.mount(&server)
.await;
let fetcher =
RemoteFetcher::new_without_private_address_guard_for_tests(5000, 10 * 1024 * 1024);
let url = format!("{}/broken.jpg", server.uri());
let result = fetcher.fetch(&url).await;
assert!(matches!(result, Err(RemoteFetchError::ServerError(503))));
}
async fn serve_once(response: &'static str) -> String {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.unwrap();
let mut buf = [0u8; 1024];
let _ = socket.read(&mut buf).await;
let _ = socket.write_all(response.as_bytes()).await;
let _ = socket.shutdown().await;
});
format!("http://{addr}")
}
#[tokio::test]
async fn fetch_rejects_declared_content_length_over_max_size_without_reading_body() {
let base_url = serve_once(
"HTTP/1.1 200 OK\r\nContent-Type: image/png\r\nContent-Length: 1000\r\n\r\n",
)
.await;
let fetcher = RemoteFetcher::new_without_private_address_guard_for_tests(5000, 10);
let url = format!("{base_url}/photo.png");
let result = fetcher.fetch(&url).await;
assert!(matches!(result, Err(RemoteFetchError::ResponseTooLarge)));
}
#[tokio::test]
async fn fetch_rejects_streamed_body_over_max_size_even_without_content_length() {
let base_url = serve_once(
"HTTP/1.1 200 OK\r\nContent-Type: image/png\r\nTransfer-Encoding: chunked\r\n\r\n\
a\r\n0123456789\r\n0\r\n\r\n",
)
.await;
let fetcher = RemoteFetcher::new_without_private_address_guard_for_tests(5000, 5);
let url = format!("{base_url}/photo.png");
let result = fetcher.fetch(&url).await;
assert!(matches!(result, Err(RemoteFetchError::ResponseTooLarge)));
}
#[tokio::test]
async fn fetch_accepts_body_within_max_size() {
let base_url = serve_once(
"HTTP/1.1 200 OK\r\nContent-Type: image/png\r\nContent-Length: 5\r\n\r\nhello",
)
.await;
let fetcher = RemoteFetcher::new_without_private_address_guard_for_tests(5000, 10);
let url = format!("{base_url}/photo.png");
let result = fetcher.fetch(&url).await.unwrap();
assert_eq!(result.data, b"hello");
}
#[tokio::test]
async fn fetch_follows_a_redirect_within_the_cap() {
let server = MockServer::start().await;
Mock::given(wm_method("GET"))
.and(wm_path("/start.jpg"))
.respond_with(ResponseTemplate::new(302).insert_header("location", "/final.jpg"))
.mount(&server)
.await;
Mock::given(wm_method("GET"))
.and(wm_path("/final.jpg"))
.respond_with(ResponseTemplate::new(200).set_body_bytes(b"final-bytes".to_vec()))
.mount(&server)
.await;
let fetcher =
RemoteFetcher::new_without_private_address_guard_for_tests(5000, 10 * 1024 * 1024);
let url = format!("{}/start.jpg", server.uri());
let result = fetcher.fetch(&url).await.unwrap();
assert_eq!(result.data, b"final-bytes");
}
#[tokio::test]
async fn fetch_rejects_a_redirect_chain_longer_than_the_cap() {
let server = MockServer::start().await;
for i in 0..4 {
let next = format!("/hop{}", i + 1);
Mock::given(wm_method("GET"))
.and(wm_path(format!("/hop{i}")))
.respond_with(ResponseTemplate::new(302).insert_header("location", next.as_str()))
.mount(&server)
.await;
}
Mock::given(wm_method("GET"))
.and(wm_path("/hop4"))
.respond_with(ResponseTemplate::new(200).set_body_bytes(b"unreachable".to_vec()))
.mount(&server)
.await;
let fetcher =
RemoteFetcher::new_without_private_address_guard_for_tests(5000, 10 * 1024 * 1024);
let url = format!("{}/hop0", server.uri());
let result = fetcher.fetch(&url).await;
assert!(matches!(result, Err(RemoteFetchError::TooManyRedirects)));
}
#[tokio::test]
async fn fetch_rejects_a_redirect_to_a_non_http_scheme() {
let server = MockServer::start().await;
Mock::given(wm_method("GET"))
.and(wm_path("/start.jpg"))
.respond_with(
ResponseTemplate::new(302).insert_header("location", "file:///etc/passwd"),
)
.mount(&server)
.await;
let fetcher =
RemoteFetcher::new_without_private_address_guard_for_tests(5000, 10 * 1024 * 1024);
let url = format!("{}/start.jpg", server.uri());
let result = fetcher.fetch(&url).await;
assert!(matches!(result, Err(RemoteFetchError::UnsupportedScheme)));
}
#[tokio::test]
async fn fetch_rejects_a_redirect_to_a_private_address_when_guard_is_enabled() {
let server = MockServer::start().await;
Mock::given(wm_method("GET"))
.and(wm_path("/start.jpg"))
.respond_with(ResponseTemplate::new(302).insert_header("location", "/final.jpg"))
.mount(&server)
.await;
let fetcher = RemoteFetcher::new(5000, 10 * 1024 * 1024);
let url = format!("{}/start.jpg", server.uri());
let result = fetcher.fetch(&url).await;
assert!(matches!(result, Err(RemoteFetchError::PrivateAddress)));
}
}