reserve_core/lookup/

registry.rs

//! Where to ask about an extension, and how to read the answer.

use std::collections::HashMap;
use std::path::Path;
use std::time::{Duration, SystemTime};

use serde::Deserialize;

use crate::error::{Error, Result};

/// @docgen The published list is a few hundred kilobytes, so this leaves generous headroom while still bounding a hostile reply.
const MAX_BOOTSTRAP_BYTES: usize = 8 * 1024 * 1024;

const CACHE_MAX_AGE: Duration = Duration::from_secs(7 * 24 * 60 * 60);

/// @docgen Published so the diagnostic report can name where the tool fetches its registry list from.
pub const BOOTSTRAP_URL: &str = "https://data.iana.org/rdap/dns.json";

/// @docgen Absence from the published list does not mean absence of a service; every extension here runs one.
const UNLISTED_SERVICES: &[(&str, &str)] = &[
    ("de", "https://rdap.denic.de/"),
    ("io", "https://rdap.identitydigital.services/rdap/"),
    ("us", "https://rdap.nic.us/"),
    ("co", "https://rdap.nic.co/"),
    ("me", "https://rdap.identitydigital.services/rdap/"),
    ("sh", "https://rdap.identitydigital.services/rdap/"),
    ("tv", "https://tld-rdap.verisign.com/tv/v1/"),
    ("cc", "https://tld-rdap.verisign.com/cc/v1/"),
];

#[derive(Debug, Deserialize)]
struct BootstrapFile {
    /// @docgen IANA's own shape: `[[["com","net"], ["https://rdap.verisign.com/com/v1/"]], ...]`.
    services: Vec<Vec<Vec<String>>>,
}

#[derive(Debug, Default, Clone)]
pub struct ServiceMap {
    by_suffix: HashMap<String, Vec<String>>,
}

async fn read_cache(cache: &Path) -> std::io::Result<String> {
    let path = cache.to_path_buf();
    tokio::task::spawn_blocking(move || {
        crate::lookup::read_capped(&path, crate::lookup::MAX_TABLE_BYTES)
    })
    .await
    .map_err(std::io::Error::other)?
}

/// @docgen The cache decides which hosts the tool queries, so on a shared machine no other user may write into its directory.
async fn create_owner_only(dir: &Path) -> std::io::Result<()> {
    let dir = dir.to_path_buf();
    tokio::task::spawn_blocking(move || {
        #[cfg(unix)]
        {
            use std::os::unix::fs::DirBuilderExt as _;
            std::fs::DirBuilder::new()
                .recursive(true)
                .mode(0o700)
                .create(&dir)
        }
        #[cfg(not(unix))]
        {
            std::fs::create_dir_all(&dir)
        }
    })
    .await
    .map_err(std::io::Error::other)?
}

/// @docgen Staged beside the target and renamed, so a run killed mid-write leaves the old list rather than a torn one a later run would trust.
async fn store_cache(cache: &Path, text: &str) -> std::io::Result<()> {
    let Some(parent) = cache.parent() else {
        return Err(std::io::Error::other("the cache path has no directory"));
    };
    create_owner_only(parent).await?;

    let staging = parent.join(format!(
        ".{}.staging",
        cache
            .file_name()
            .and_then(|name| name.to_str())
            .unwrap_or("cache")
    ));
    tokio::fs::write(&staging, text).await?;
    match tokio::fs::rename(&staging, cache).await {
        Ok(()) => Ok(()),
        Err(error) => {
            let _ = tokio::fs::remove_file(&staging).await;
            Err(error)
        }
    }
}

impl ServiceMap {
    /// @docgen A failed download falls back to the cached copy however stale, because a week-old list beats no list.
    pub async fn load(
        client: &reqwest::Client,
        cache: &Path,
        refresh: bool,
    ) -> Result<(Self, Freshness)> {
        let cache_is_fresh = !refresh
            && cache
                .metadata()
                .and_then(|meta| meta.modified())
                .is_ok_and(|at| {
                    SystemTime::now()
                        .duration_since(at)
                        .is_ok_and(|age| age < CACHE_MAX_AGE)
                });

        if cache_is_fresh
            && let Ok(text) = read_cache(cache).await
            && let Ok(services) = Self::parse(&text)
        {
            return Ok((services, Freshness::Cached));
        }

        match Self::download(client).await {
            Ok(text) => {
                let services = Self::parse(&text)?;
                if let Err(error) = store_cache(cache, &text).await {
                    // @docgen A cache that cannot be written costs only a re-download, so the run continues and says so once.
                    tracing::warn!(path = %cache.display(), %error, "the registry list could not be cached");
                }
                Ok((services, Freshness::Fresh))
            }
            Err(error) => {
                if let Ok(text) = read_cache(cache).await
                    && let Ok(services) = Self::parse(&text)
                {
                    return Ok((services, Freshness::Stale));
                }
                Err(error)
            }
        }
    }

    async fn download(client: &reqwest::Client) -> Result<String> {
        let mut response = client
            .get(BOOTSTRAP_URL)
            .send()
            .await
            .map_err(|source| Error::BootstrapUnavailable {
                source: Box::new(source),
            })?
            .error_for_status()
            .map_err(|source| Error::BootstrapUnavailable {
                source: Box::new(source),
            })?;

        // @docgen The body is gzip-decoded, so an unbounded read would let one endpoint expand into all available memory.
        let mut body = Vec::new();
        while let Some(chunk) =
            response
                .chunk()
                .await
                .map_err(|source| Error::BootstrapUnavailable {
                    source: Box::new(source),
                })?
        {
            if body.len().saturating_add(chunk.len()) > MAX_BOOTSTRAP_BYTES {
                return Err(Error::BootstrapUnavailable {
                    source: format!(
                        "the registry list exceeded the {MAX_BOOTSTRAP_BYTES} byte limit"
                    )
                    .into(),
                });
            }
            body.extend_from_slice(&chunk);
        }

        String::from_utf8(body).map_err(|_| Error::BootstrapUnavailable {
            source: "the registry list was not valid text".into(),
        })
    }

    pub fn parse(text: &str) -> Result<Self> {
        let file: BootstrapFile =
            serde_json::from_str(text).map_err(|source| Error::CatalogMalformed {
                source: Box::new(source),
            })?;

        let mut by_suffix: HashMap<String, Vec<String>> = HashMap::new();
        for service in file.services {
            let (suffixes, urls) = match (service.first(), service.get(1)) {
                (Some(s), Some(u)) if !s.is_empty() && !u.is_empty() => (s, u),
                _ => continue,
            };
            let urls: Vec<String> = urls
                .iter()
                .filter(|url| is_usable_service(url))
                .cloned()
                .map(with_trailing_slash)
                .collect();
            if urls.is_empty() {
                continue;
            }
            for suffix in suffixes {
                by_suffix.insert(suffix.to_lowercase(), urls.clone());
            }
        }

        for (suffix, url) in UNLISTED_SERVICES {
            by_suffix
                .entry((*suffix).to_owned())
                .or_insert_with(|| vec![(*url).to_owned()]);
        }

        Ok(Self { by_suffix })
    }

    pub fn from_file(path: &Path) -> Result<Self> {
        let text =
            crate::lookup::read_capped(path, crate::lookup::MAX_TABLE_BYTES).map_err(|source| {
                Error::FileUnreadable {
                    path: path.to_path_buf(),
                    source,
                }
            })?;
        let parsed = Self::parse(&text)?;
        if parsed.by_suffix.is_empty() {
            return Err(Error::CatalogEmptySelection);
        }
        Ok(parsed)
    }

    pub fn merge(&mut self, other: Self) {
        self.by_suffix.extend(other.by_suffix);
    }

    /// @docgen Matching longest first lets a multi-label suffix resolve through its parent when it has no service of its own.
    #[must_use]
    pub fn for_suffix(&self, suffix: &str) -> Option<&[String]> {
        let suffix = suffix.trim_matches('.').to_lowercase();
        let mut rest = suffix.as_str();
        loop {
            if let Some(urls) = self.by_suffix.get(rest) {
                return Some(urls);
            }
            match rest.split_once('.') {
                Some((_, tail)) if !tail.is_empty() => rest = tail,
                _ => return None,
            }
        }
    }

    #[must_use]
    pub fn len(&self) -> usize {
        self.by_suffix.len()
    }

    #[must_use]
    pub fn is_empty(&self) -> bool {
        self.by_suffix.is_empty()
    }
}

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Freshness {
    Fresh,
    Cached,
    Stale,
}

impl Freshness {
    #[must_use]
    pub const fn label(self) -> &'static str {
        match self {
            Self::Fresh => "downloaded",
            Self::Cached => "cached",
            Self::Stale => "cached, out of date",
        }
    }
}

/// @docgen Service base URLs are joined with `domain/<name>`, so they need the trailing slash.
fn with_trailing_slash(url: String) -> String {
    if url.ends_with('/') {
        url
    } else {
        format!("{url}/")
    }
}

#[must_use]
/// @docgen Userinfo is dropped so a credentialed URL never becomes the pacing key or reaches the output.
/// @docgen A cache or a supplied list decides what the tool fetches, so a plaintext or internal address is refused here.
fn is_usable_service(url: &str) -> bool {
    if !url.starts_with("https://") {
        return false;
    }
    let host = host_of(url);
    if host.is_empty() || url.contains('@') {
        return false;
    }
    is_public_host(host)
}

/// @docgen A host taken from a cleartext reply decides where the next query goes, so an internal address is refused before it is dialled.
pub(crate) fn is_public_host(host: &str) -> bool {
    let lowered = host.trim().trim_end_matches('.').to_lowercase();
    if lowered.is_empty() || lowered.contains(char::is_whitespace) || lowered.contains('@') {
        return false;
    }
    if lowered == "localhost" || lowered.ends_with(".localhost") {
        return false;
    }
    match lowered.parse::<std::net::IpAddr>() {
        Ok(ip) => is_public_ip(ip),
        Err(_) => true,
    }
}

/// @docgen A name is only a promise until it resolves, so the address it answers with is judged too.
pub(crate) fn is_public_ip(ip: std::net::IpAddr) -> bool {
    match ip {
        std::net::IpAddr::V4(ip) => is_public_v4(ip),
        std::net::IpAddr::V6(ip) => {
            if ip.is_loopback() || ip.is_unspecified() || ip.is_multicast() {
                return false;
            }
            // @docgen An IPv4 address written in v6 form fails every v6 test, so ::ffff:127.0.0.1 would otherwise pass.
            if let Some(written_as_v6) = ip.to_ipv4() {
                return is_public_v4(written_as_v6);
            }
            // @docgen Rust still gates is_unique_local, so fc00::/7 and fe80::/10 are matched on their prefix here.
            let first = ip.segments().first().copied().unwrap_or(0);
            let unique_local = (first & 0xfe00) == 0xfc00;
            let link_local = (first & 0xffc0) == 0xfe80;
            !(unique_local || link_local)
        }
    }
}

/// @docgen Carrier-grade NAT, multicast, and broadcast are reachable internal targets that no registry ever answers from.
fn is_public_v4(ip: std::net::Ipv4Addr) -> bool {
    let [first, second, ..] = ip.octets();
    let carrier_grade_nat = first == 100 && (64..=127).contains(&second);
    !(ip.is_loopback()
        || ip.is_private()
        || ip.is_link_local()
        || ip.is_unspecified()
        || ip.is_multicast()
        || ip.is_broadcast()
        || carrier_grade_nat)
}

pub(crate) fn host_of(url: &str) -> &str {
    let rest = url.split_once("://").map_or(url, |(_, rest)| rest);
    let authority_end = rest.find('/').unwrap_or(rest.len());
    let authority = rest.get(..authority_end).unwrap_or(rest);
    let host = authority
        .rsplit_once('@')
        .map_or(authority, |(_, host)| host);
    // @docgen An IPv6 host is bracketed and full of colons, so cutting at the first one leaves a stub that matches nothing.
    if let Some(rest) = host.strip_prefix('[') {
        return rest.split_once(']').map_or(host, |(inside, _)| inside);
    }
    let end = host.find([':', '?']).unwrap_or(host.len());
    host.get(..end).unwrap_or(host)
}

#[cfg(test)]
mod tests {

    #[test]
    fn a_referral_host_naming_an_internal_address_is_refused() {
        for host in [
            "127.0.0.1",
            "169.254.169.254",
            "10.0.0.1",
            "192.168.1.1",
            "localhost",
            "whois.internal.localhost",
            "::1",
            "fe80::1",
            "fd00::1",
            "::ffff:127.0.0.1",
            "::ffff:169.254.169.254",
            "::ffff:10.0.0.1",
            "::ffff:192.168.1.1",
            "ff02::1",
            "100.64.0.1",
            "100.127.255.254",
            "224.0.0.1",
            "255.255.255.255",
            "",
            "whois example com",
        ] {
            assert!(
                !is_public_host(host),
                "{host} must never be dialled from a cleartext referral"
            );
        }
    }

    #[test]
    fn a_real_registry_host_still_passes() {
        for host in ["whois.btcl.net.bd", "whois.nic.example", "203.0.113.10"] {
            assert!(is_public_host(host), "{host} is a normal public host");
        }
    }
    use std::sync::Arc;

    use tempfile::tempdir;

    use super::*;
    use crate::error::ErrorId;

    #[test]
    fn a_bracketed_ipv6_host_is_read_whole_rather_than_cut_at_its_first_colon() {
        assert_eq!(host_of("https://[::1]/rdap/"), "::1");
        assert_eq!(host_of("https://[fd00::1]:8443/rdap/"), "fd00::1");
        assert_eq!(host_of("https://rdap.example/x"), "rdap.example");
        assert_eq!(host_of("https://user:pass@rdap.example/x"), "rdap.example");
    }

    #[test]
    fn an_internal_service_address_is_refused_in_either_address_family() {
        for bad in [
            "https://[::1]/rdap/",
            "https://[fd00::1]/rdap/",
            "https://127.0.0.1/rdap/",
            "https://10.0.0.5/rdap/",
            "https://169.254.169.254/rdap/",
            "https://localhost/rdap/",
            "http://rdap.example/",
            "https://user:key@rdap.example/",
        ] {
            assert!(!is_usable_service(bad), "{bad} must not be fetched");
        }
        assert!(is_usable_service("https://rdap.verisign.com/com/v1/"));
    }

    const SAMPLE: &str = r#"{"services":[
        [["com","net"],["https://rdap.verisign.com/com/v1"]],
        [["uk"],["https://rdap.nominet.uk/uk/"]]
    ]}"#;

    /// @docgen A resolver that answers nothing keeps the download path in the test offline and instant.
    #[derive(Debug)]
    struct NeverResolves;

    impl reqwest::dns::Resolve for NeverResolves {
        fn resolve(&self, _name: reqwest::dns::Name) -> reqwest::dns::Resolving {
            Box::pin(async {
                Err(Box::<dyn std::error::Error + Send + Sync>::from(
                    "this test never leaves the machine",
                ))
            })
        }
    }

    fn grounded_client() -> reqwest::Client {
        reqwest::Client::builder()
            .no_proxy()
            .dns_resolver(Arc::new(NeverResolves))
            .build()
            .expect("a client that can reach nothing")
    }

    fn age_by_days(path: &Path, days: u64) {
        let when = SystemTime::now()
            .checked_sub(Duration::from_secs(days * 24 * 60 * 60))
            .expect("a moment inside the epoch");
        let file = std::fs::File::options()
            .write(true)
            .open(path)
            .expect("the cache opens for writing");
        file.set_times(std::fs::FileTimes::new().set_modified(when))
            .expect("the cache takes a new modified time");
    }

    fn cache_holding(dir: &Path, text: &str) -> std::path::PathBuf {
        let path = dir.join("servers.json");
        std::fs::write(&path, text).expect("the cache is written");
        path
    }

    #[tokio::test]
    async fn a_cache_written_today_is_read_instead_of_downloaded() {
        let dir = tempdir().expect("temp dir");
        let cache = cache_holding(dir.path(), SAMPLE);

        let (services, freshness) = ServiceMap::load(&grounded_client(), &cache, false)
            .await
            .expect("a fresh cache needs no download");

        assert_eq!(freshness, Freshness::Cached);
        assert!(services.for_suffix("com").is_some());
    }

    #[tokio::test]
    async fn a_cache_older_than_a_week_is_still_used_when_the_download_fails() {
        let dir = tempdir().expect("temp dir");
        let cache = cache_holding(dir.path(), SAMPLE);
        age_by_days(&cache, 8);

        let (services, freshness) = ServiceMap::load(&grounded_client(), &cache, false)
            .await
            .expect("a week-old list beats no list");

        assert_eq!(freshness, Freshness::Stale);
        assert!(services.for_suffix("com").is_some());
    }

    #[tokio::test]
    async fn asking_for_a_refresh_still_falls_back_to_the_cache_it_skipped() {
        let dir = tempdir().expect("temp dir");
        let cache = cache_holding(dir.path(), SAMPLE);

        let (services, freshness) = ServiceMap::load(&grounded_client(), &cache, true)
            .await
            .expect("a failed refresh falls back rather than failing");

        assert_eq!(freshness, Freshness::Stale);
        assert!(services.for_suffix("com").is_some());
    }

    #[tokio::test]
    async fn no_cache_and_no_download_is_an_error_rather_than_an_empty_list() {
        let dir = tempdir().expect("temp dir");
        let missing = dir.path().join("never-written").join("servers.json");

        let error = ServiceMap::load(&grounded_client(), &missing, false)
            .await
            .expect_err("an empty service map would read every extension as unserved");

        assert_eq!(error.id(), ErrorId::BootstrapUnavailable);
    }

    #[tokio::test]
    async fn a_corrupt_cache_is_not_read_as_a_list_with_nothing_in_it() {
        let dir = tempdir().expect("temp dir");
        let cache = cache_holding(dir.path(), "half a file, no json");

        let error = ServiceMap::load(&grounded_client(), &cache, false)
            .await
            .expect_err("a corrupt cache must not stand in for a real list");

        assert_eq!(error.id(), ErrorId::BootstrapUnavailable);
    }

    #[tokio::test]
    async fn a_failed_download_never_overwrites_the_cache_it_fell_back_to() {
        let dir = tempdir().expect("temp dir");
        let cache = cache_holding(dir.path(), SAMPLE);
        age_by_days(&cache, 8);

        let _ = ServiceMap::load(&grounded_client(), &cache, false).await;

        assert_eq!(
            std::fs::read_to_string(&cache).expect("the cache survives"),
            SAMPLE
        );
    }

    #[test]
    fn a_base_url_always_ends_in_a_slash() {
        let services = ServiceMap::parse(SAMPLE).unwrap();
        assert_eq!(
            services.for_suffix("com"),
            Some(&["https://rdap.verisign.com/com/v1/".to_owned()][..])
        );
    }

    #[test]
    fn a_multi_label_suffix_resolves_through_its_parent() {
        let services = ServiceMap::parse(SAMPLE).unwrap();
        assert!(services.for_suffix("co.uk").is_some());
        assert_eq!(services.for_suffix("co.uk"), services.for_suffix("uk"));
    }

    #[test]
    fn an_extension_with_no_service_reports_none() {
        let services = ServiceMap::parse(SAMPLE).unwrap();
        assert!(services.for_suffix("bd").is_none());
        assert!(services.for_suffix("com.bd").is_none());
    }

    #[test]
    fn services_missing_from_the_published_list_are_still_reachable() {
        // @docgen The sample carries none of these, yet all of them answer, so the fallback must fill them in.
        let services = ServiceMap::parse(SAMPLE).unwrap();
        for suffix in ["de", "io", "us"] {
            assert!(
                services.for_suffix(suffix).is_some(),
                ".{suffix} has a working service and must not read as having none"
            );
        }
    }

    #[test]
    fn a_published_entry_wins_over_the_unlisted_fallback() {
        let text = r#"{"services":[[["io"],["https://published.example/"]]]}"#;
        let services = ServiceMap::parse(text).unwrap();
        assert_eq!(
            services.for_suffix("io"),
            Some(&["https://published.example/".to_owned()][..])
        );
    }

    #[test]
    fn a_custom_list_overlays_the_published_one() {
        let mut services = ServiceMap::parse(SAMPLE).unwrap();
        let custom = ServiceMap::parse(r#"{"services":[[["com"],["https://mine/"]]]}"#).unwrap();
        services.merge(custom);
        assert_eq!(
            services.for_suffix("com"),
            Some(&["https://mine/".to_owned()][..])
        );
        assert!(services.for_suffix("uk").is_some());
    }

    #[test]
    fn rubbish_is_refused() {
        assert!(ServiceMap::parse("not json").is_err());
        assert!(ServiceMap::parse(r#"{"services":"nope"}"#).is_err());
    }

    #[test]
    fn an_empty_service_entry_is_skipped_rather_than_stored() {
        let services = ServiceMap::parse(r#"{"services":[[[],[]],[["com"],[]]]}"#).unwrap();
        assert!(services.for_suffix("com").is_none());
    }

    #[test]
    fn hosts_come_out_of_urls() {
        assert_eq!(
            host_of("https://rdap.verisign.com/com/v1/"),
            "rdap.verisign.com"
        );
        assert_eq!(host_of("http://a.b.c:8080/x"), "a.b.c");
        assert_eq!(host_of("whois.nic.io"), "whois.nic.io");
    }

    #[test]
    fn each_freshness_describes_itself() {
        assert_eq!(Freshness::Fresh.label(), "downloaded");
        assert!(Freshness::Stale.label().contains("out of date"));
    }

    #[test]
    fn an_address_a_name_resolved_to_is_judged_the_same_way_the_name_was() {
        for raw in [
            "127.0.0.1",
            "10.0.0.1",
            "169.254.169.254",
            "100.64.0.1",
            "224.0.0.1",
            "255.255.255.255",
            "::1",
            "fd00::1",
            "fe80::1",
            "::ffff:127.0.0.1",
            "::ffff:10.0.0.1",
        ] {
            let ip: std::net::IpAddr = raw.parse().expect("a literal address parses");
            assert!(
                !is_public_ip(ip),
                "{raw} must never be dialled, however the name reached it"
            );
        }

        for raw in ["203.0.113.10", "2606:4700:4700::1111"] {
            let ip: std::net::IpAddr = raw.parse().expect("a literal address parses");
            assert!(is_public_ip(ip), "{raw} is a normal public address");
        }
    }
}