codoseo_crawler/
guard.rs

1//! Keeps the cloud crawler away from private and internal addresses.
2//!
3//! Two layers, because reqwest never calls the resolver for IP-literal hosts:
4//! [`check_url`] rejects blocked IP literals before a request is made, and
5//! [`GuardedResolver`] drops blocked addresses from DNS answers. The fetcher
6//! re-runs [`check_url`] on every redirect hop.
7
8use std::future::Future;
9use std::io;
10use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr};
11use std::sync::Arc;
12
13use codoseo_core::crawl::AddressPolicy;
14use reqwest::dns::{Addrs, Name, Resolve, Resolving};
15use url::{Host, Url};
16
17#[derive(Debug, thiserror::Error)]
18pub enum GuardError {
19    #[error("{0} is a private or internal address")]
20    BlockedAddress(IpAddr),
21    #[error("{host} only resolves to private or internal addresses")]
22    OnlyBlockedAddresses { host: String },
23    #[error("could not resolve {host}: {reason}")]
24    Lookup { host: String, reason: String },
25}
26
27/// True for addresses the cloud crawler must never connect to.
28pub fn is_blocked(ip: IpAddr) -> bool {
29    match ip {
30        IpAddr::V4(v4) => is_blocked_v4(v4),
31        IpAddr::V6(v6) => is_blocked_v6(v6),
32    }
33}
34
35fn is_blocked_v4(ip: Ipv4Addr) -> bool {
36    let [a, b, c, _] = ip.octets();
37    a == 0 // "this network", including 0.0.0.0
38        || ip.is_private() // 10/8, 172.16/12, 192.168/16
39        || ip.is_loopback() // 127/8
40        || ip.is_link_local() // 169.254/16, includes cloud metadata
41        || (a == 100 && (b & 0xC0) == 64) // 100.64/10 carrier-grade NAT
42        || (a == 192 && b == 0 && c == 0) // 192.0.0/24 protocol assignments
43        || ip.is_documentation()
44        || (a == 198 && (b == 18 || b == 19)) // 198.18/15 benchmarking
45        || ip.is_multicast() // 224/4
46        || a >= 240 // 240/4 reserved, includes broadcast
47}
48
49fn is_blocked_v6(ip: Ipv6Addr) -> bool {
50    if let Some(v4) = embedded_v4(ip) {
51        return is_blocked_v4(v4);
52    }
53    let s = ip.segments();
54    ip.is_unspecified()
55        || ip.is_loopback()
56        || ip.is_multicast()
57        || ip.is_unique_local() // fc00::/7
58        || ip.is_unicast_link_local() // fe80::/10
59        || (s[0] & 0xffc0) == 0xfec0 // fec0::/10 deprecated site-local
60        || (s[0] == 0x2001 && s[1] == 0x0db8) // 2001:db8::/32 documentation
61        || (s[0] == 0x2001 && s[1] == 0x0000) // 2001::/32 Teredo
62        || (s[0] == 0x0064 && s[1] == 0xff9b && s[2] == 0x0001) // 64:ff9b:1::/48 local NAT64
63        || (s[0] == 0x0100 && s[1] == 0 && s[2] == 0 && s[3] == 0) // 100::/64 discard
64}
65
66/// IPv4 addresses carried inside IPv6 ones: mapped, compatible, NAT64 and 6to4.
67fn embedded_v4(ip: Ipv6Addr) -> Option<Ipv4Addr> {
68    let s = ip.segments();
69    let tail =
70        |hi: u16, lo: u16| Ipv4Addr::new((hi >> 8) as u8, hi as u8, (lo >> 8) as u8, lo as u8);
71    if let Some(v4) = ip.to_ipv4_mapped() {
72        return Some(v4);
73    }
74    if s[..6].iter().all(|&x| x == 0) && !(s[6] == 0 && s[7] <= 1) {
75        return Some(tail(s[6], s[7])); // ::a.b.c.d (but not :: or ::1)
76    }
77    if s[0] == 0x0064 && s[1] == 0xff9b && s[2..6].iter().all(|&x| x == 0) {
78        return Some(tail(s[6], s[7])); // 64:ff9b::/96
79    }
80    if s[0] == 0x2002 {
81        return Some(tail(s[1], s[2])); // 2002::/16 6to4
82    }
83    None
84}
85
86/// Rejects URLs whose host is a blocked IP literal. Domain names are checked
87/// later, by [`GuardedResolver`].
88pub fn check_url(url: &Url, policy: AddressPolicy) -> Result<(), GuardError> {
89    if policy == AddressPolicy::AllowPrivate {
90        return Ok(());
91    }
92    let ip = match url.host() {
93        Some(Host::Ipv4(v4)) => IpAddr::V4(v4),
94        Some(Host::Ipv6(v6)) => IpAddr::V6(v6),
95        _ => return Ok(()),
96    };
97    if is_blocked(ip) {
98        return Err(GuardError::BlockedAddress(ip));
99    }
100    Ok(())
101}
102
103/// Turns a host name into IP addresses.
104pub trait Lookup: Send + Sync + 'static {
105    fn lookup(&self, host: &str) -> impl Future<Output = io::Result<Vec<IpAddr>>> + Send;
106}
107
108/// The operating system's resolver.
109#[derive(Debug, Default, Clone, Copy)]
110pub struct SystemLookup;
111
112impl Lookup for SystemLookup {
113    fn lookup(&self, host: &str) -> impl Future<Output = io::Result<Vec<IpAddr>>> + Send {
114        let host = host.to_owned();
115        async move {
116            let addrs = tokio::net::lookup_host((host.as_str(), 0)).await?;
117            Ok(addrs.map(|a| a.ip()).collect())
118        }
119    }
120}
121
122/// A reqwest resolver that only returns addresses allowed by [`is_blocked`].
123pub struct GuardedResolver<L: Lookup> {
124    lookup: Arc<L>,
125}
126
127impl<L: Lookup> GuardedResolver<L> {
128    pub fn new(lookup: L) -> Self {
129        GuardedResolver {
130            lookup: Arc::new(lookup),
131        }
132    }
133
134    pub async fn lookup_checked(&self, host: &str) -> Result<Vec<SocketAddr>, GuardError> {
135        checked(self.lookup.as_ref(), host).await
136    }
137}
138
139async fn checked<L: Lookup>(lookup: &L, host: &str) -> Result<Vec<SocketAddr>, GuardError> {
140    let ips = lookup.lookup(host).await.map_err(|e| GuardError::Lookup {
141        host: host.to_owned(),
142        reason: e.to_string(),
143    })?;
144    let allowed: Vec<SocketAddr> = ips
145        .into_iter()
146        .filter(|ip| !is_blocked(*ip))
147        .map(|ip| SocketAddr::new(ip, 0))
148        .collect();
149    if allowed.is_empty() {
150        return Err(GuardError::OnlyBlockedAddresses {
151            host: host.to_owned(),
152        });
153    }
154    Ok(allowed)
155}
156
157impl<L: Lookup> Resolve for GuardedResolver<L> {
158    fn resolve(&self, name: Name) -> Resolving {
159        let lookup = Arc::clone(&self.lookup);
160        Box::pin(async move {
161            let addrs = checked(lookup.as_ref(), name.as_str()).await?;
162            Ok(Box::new(addrs.into_iter()) as Addrs)
163        })
164    }
165}
codoseo_crawler/guard.rs

codoseo_crawler/
guard.rs
