reserve_core/lookup/
registry.rs

1//! Where to ask about an extension, and how to read the answer.
2
3use std::collections::HashMap;
4use std::path::Path;
5use std::time::{Duration, SystemTime};
6
7use serde::Deserialize;
8
9use crate::error::{Error, Result};
10
11/// @docgen The published list is a few hundred kilobytes, so this leaves generous headroom while still bounding a hostile reply.
12const MAX_BOOTSTRAP_BYTES: usize = 8 * 1024 * 1024;
13
14const CACHE_MAX_AGE: Duration = Duration::from_secs(7 * 24 * 60 * 60);
15
16/// @docgen Published so the diagnostic report can name where the tool fetches its registry list from.
17pub const BOOTSTRAP_URL: &str = "https://data.iana.org/rdap/dns.json";
18
19/// @docgen Absence from the published list does not mean absence of a service; every extension here runs one.
20const UNLISTED_SERVICES: &[(&str, &str)] = &[
21    ("de", "https://rdap.denic.de/"),
22    ("io", "https://rdap.identitydigital.services/rdap/"),
23    ("us", "https://rdap.nic.us/"),
24    ("co", "https://rdap.nic.co/"),
25    ("me", "https://rdap.identitydigital.services/rdap/"),
26    ("sh", "https://rdap.identitydigital.services/rdap/"),
27    ("tv", "https://tld-rdap.verisign.com/tv/v1/"),
28    ("cc", "https://tld-rdap.verisign.com/cc/v1/"),
29];
30
31#[derive(Debug, Deserialize)]
32struct BootstrapFile {
33    /// @docgen IANA's own shape: `[[["com","net"], ["https://rdap.verisign.com/com/v1/"]], ...]`.
34    services: Vec<Vec<Vec<String>>>,
35}
36
37#[derive(Debug, Default, Clone)]
38pub struct ServiceMap {
39    by_suffix: HashMap<String, Vec<String>>,
40}
41
42async fn read_cache(cache: &Path) -> std::io::Result<String> {
43    let path = cache.to_path_buf();
44    tokio::task::spawn_blocking(move || {
45        crate::lookup::read_capped(&path, crate::lookup::MAX_TABLE_BYTES)
46    })
47    .await
48    .map_err(std::io::Error::other)?
49}
50
51/// @docgen The cache decides which hosts the tool queries, so on a shared machine no other user may write into its directory.
52async fn create_owner_only(dir: &Path) -> std::io::Result<()> {
53    let dir = dir.to_path_buf();
54    tokio::task::spawn_blocking(move || {
55        #[cfg(unix)]
56        {
57            use std::os::unix::fs::DirBuilderExt as _;
58            std::fs::DirBuilder::new()
59                .recursive(true)
60                .mode(0o700)
61                .create(&dir)
62        }
63        #[cfg(not(unix))]
64        {
65            std::fs::create_dir_all(&dir)
66        }
67    })
68    .await
69    .map_err(std::io::Error::other)?
70}
71
72/// @docgen Staged beside the target and renamed, so a run killed mid-write leaves the old list rather than a torn one a later run would trust.
73async fn store_cache(cache: &Path, text: &str) -> std::io::Result<()> {
74    let Some(parent) = cache.parent() else {
75        return Err(std::io::Error::other("the cache path has no directory"));
76    };
77    create_owner_only(parent).await?;
78
79    let staging = parent.join(format!(
80        ".{}.staging",
81        cache
82            .file_name()
83            .and_then(|name| name.to_str())
84            .unwrap_or("cache")
85    ));
86    tokio::fs::write(&staging, text).await?;
87    match tokio::fs::rename(&staging, cache).await {
88        Ok(()) => Ok(()),
89        Err(error) => {
90            let _ = tokio::fs::remove_file(&staging).await;
91            Err(error)
92        }
93    }
94}
95
96impl ServiceMap {
97    /// @docgen A failed download falls back to the cached copy however stale, because a week-old list beats no list.
98    pub async fn load(
99        client: &reqwest::Client,
100        cache: &Path,
101        refresh: bool,
102    ) -> Result<(Self, Freshness)> {
103        let cache_is_fresh = !refresh
104            && cache
105                .metadata()
106                .and_then(|meta| meta.modified())
107                .is_ok_and(|at| {
108                    SystemTime::now()
109                        .duration_since(at)
110                        .is_ok_and(|age| age < CACHE_MAX_AGE)
111                });
112
113        if cache_is_fresh
114            && let Ok(text) = read_cache(cache).await
115            && let Ok(services) = Self::parse(&text)
116        {
117            return Ok((services, Freshness::Cached));
118        }
119
120        match Self::download(client).await {
121            Ok(text) => {
122                let services = Self::parse(&text)?;
123                if let Err(error) = store_cache(cache, &text).await {
124                    // @docgen A cache that cannot be written costs only a re-download, so the run continues and says so once.
125                    tracing::warn!(path = %cache.display(), %error, "the registry list could not be cached");
126                }
127                Ok((services, Freshness::Fresh))
128            }
129            Err(error) => {
130                if let Ok(text) = read_cache(cache).await
131                    && let Ok(services) = Self::parse(&text)
132                {
133                    return Ok((services, Freshness::Stale));
134                }
135                Err(error)
136            }
137        }
138    }
139
140    async fn download(client: &reqwest::Client) -> Result<String> {
141        let mut response = client
142            .get(BOOTSTRAP_URL)
143            .send()
144            .await
145            .map_err(|source| Error::BootstrapUnavailable {
146                source: Box::new(source),
147            })?
148            .error_for_status()
149            .map_err(|source| Error::BootstrapUnavailable {
150                source: Box::new(source),
151            })?;
152
153        // @docgen The body is gzip-decoded, so an unbounded read would let one endpoint expand into all available memory.
154        let mut body = Vec::new();
155        while let Some(chunk) =
156            response
157                .chunk()
158                .await
159                .map_err(|source| Error::BootstrapUnavailable {
160                    source: Box::new(source),
161                })?
162        {
163            if body.len().saturating_add(chunk.len()) > MAX_BOOTSTRAP_BYTES {
164                return Err(Error::BootstrapUnavailable {
165                    source: format!(
166                        "the registry list exceeded the {MAX_BOOTSTRAP_BYTES} byte limit"
167                    )
168                    .into(),
169                });
170            }
171            body.extend_from_slice(&chunk);
172        }
173
174        String::from_utf8(body).map_err(|_| Error::BootstrapUnavailable {
175            source: "the registry list was not valid text".into(),
176        })
177    }
178
179    pub fn parse(text: &str) -> Result<Self> {
180        let file: BootstrapFile =
181            serde_json::from_str(text).map_err(|source| Error::CatalogMalformed {
182                source: Box::new(source),
183            })?;
184
185        let mut by_suffix: HashMap<String, Vec<String>> = HashMap::new();
186        for service in file.services {
187            let (suffixes, urls) = match (service.first(), service.get(1)) {
188                (Some(s), Some(u)) if !s.is_empty() && !u.is_empty() => (s, u),
189                _ => continue,
190            };
191            let urls: Vec<String> = urls
192                .iter()
193                .filter(|url| is_usable_service(url))
194                .cloned()
195                .map(with_trailing_slash)
196                .collect();
197            if urls.is_empty() {
198                continue;
199            }
200            for suffix in suffixes {
201                by_suffix.insert(suffix.to_lowercase(), urls.clone());
202            }
203        }
204
205        for (suffix, url) in UNLISTED_SERVICES {
206            by_suffix
207                .entry((*suffix).to_owned())
208                .or_insert_with(|| vec![(*url).to_owned()]);
209        }
210
211        Ok(Self { by_suffix })
212    }
213
214    pub fn from_file(path: &Path) -> Result<Self> {
215        let text =
216            crate::lookup::read_capped(path, crate::lookup::MAX_TABLE_BYTES).map_err(|source| {
217                Error::FileUnreadable {
218                    path: path.to_path_buf(),
219                    source,
220                }
221            })?;
222        let parsed = Self::parse(&text)?;
223        if parsed.by_suffix.is_empty() {
224            return Err(Error::CatalogEmptySelection);
225        }
226        Ok(parsed)
227    }
228
229    pub fn merge(&mut self, other: Self) {
230        self.by_suffix.extend(other.by_suffix);
231    }
232
233    /// @docgen Matching longest first lets a multi-label suffix resolve through its parent when it has no service of its own.
234    #[must_use]
235    pub fn for_suffix(&self, suffix: &str) -> Option<&[String]> {
236        let suffix = suffix.trim_matches('.').to_lowercase();
237        let mut rest = suffix.as_str();
238        loop {
239            if let Some(urls) = self.by_suffix.get(rest) {
240                return Some(urls);
241            }
242            match rest.split_once('.') {
243                Some((_, tail)) if !tail.is_empty() => rest = tail,
244                _ => return None,
245            }
246        }
247    }
248
249    #[must_use]
250    pub fn len(&self) -> usize {
251        self.by_suffix.len()
252    }
253
254    #[must_use]
255    pub fn is_empty(&self) -> bool {
256        self.by_suffix.is_empty()
257    }
258}
259
260#[derive(Debug, Clone, Copy, PartialEq, Eq)]
261pub enum Freshness {
262    Fresh,
263    Cached,
264    Stale,
265}
266
267impl Freshness {
268    #[must_use]
269    pub const fn label(self) -> &'static str {
270        match self {
271            Self::Fresh => "downloaded",
272            Self::Cached => "cached",
273            Self::Stale => "cached, out of date",
274        }
275    }
276}
277
278/// @docgen Service base URLs are joined with `domain/<name>`, so they need the trailing slash.
279fn with_trailing_slash(url: String) -> String {
280    if url.ends_with('/') {
281        url
282    } else {
283        format!("{url}/")
284    }
285}
286
287#[must_use]
288/// @docgen Userinfo is dropped so a credentialed URL never becomes the pacing key or reaches the output.
289/// @docgen A cache or a supplied list decides what the tool fetches, so a plaintext or internal address is refused here.
290fn is_usable_service(url: &str) -> bool {
291    if !url.starts_with("https://") {
292        return false;
293    }
294    let host = host_of(url);
295    if host.is_empty() || url.contains('@') {
296        return false;
297    }
298    is_public_host(host)
299}
300
301/// @docgen A host taken from a cleartext reply decides where the next query goes, so an internal address is refused before it is dialled.
302pub(crate) fn is_public_host(host: &str) -> bool {
303    let lowered = host.trim().trim_end_matches('.').to_lowercase();
304    if lowered.is_empty() || lowered.contains(char::is_whitespace) || lowered.contains('@') {
305        return false;
306    }
307    if lowered == "localhost" || lowered.ends_with(".localhost") {
308        return false;
309    }
310    match lowered.parse::<std::net::IpAddr>() {
311        Ok(ip) => is_public_ip(ip),
312        Err(_) => true,
313    }
314}
315
316/// @docgen A name is only a promise until it resolves, so the address it answers with is judged too.
317pub(crate) fn is_public_ip(ip: std::net::IpAddr) -> bool {
318    match ip {
319        std::net::IpAddr::V4(ip) => is_public_v4(ip),
320        std::net::IpAddr::V6(ip) => {
321            if ip.is_loopback() || ip.is_unspecified() || ip.is_multicast() {
322                return false;
323            }
324            // @docgen An IPv4 address written in v6 form fails every v6 test, so ::ffff:127.0.0.1 would otherwise pass.
325            if let Some(written_as_v6) = ip.to_ipv4() {
326                return is_public_v4(written_as_v6);
327            }
328            // @docgen Rust still gates is_unique_local, so fc00::/7 and fe80::/10 are matched on their prefix here.
329            let first = ip.segments().first().copied().unwrap_or(0);
330            let unique_local = (first & 0xfe00) == 0xfc00;
331            let link_local = (first & 0xffc0) == 0xfe80;
332            !(unique_local || link_local)
333        }
334    }
335}
336
337/// @docgen Carrier-grade NAT, multicast, and broadcast are reachable internal targets that no registry ever answers from.
338fn is_public_v4(ip: std::net::Ipv4Addr) -> bool {
339    let [first, second, ..] = ip.octets();
340    let carrier_grade_nat = first == 100 && (64..=127).contains(&second);
341    !(ip.is_loopback()
342        || ip.is_private()
343        || ip.is_link_local()
344        || ip.is_unspecified()
345        || ip.is_multicast()
346        || ip.is_broadcast()
347        || carrier_grade_nat)
348}
349
350pub(crate) fn host_of(url: &str) -> &str {
351    let rest = url.split_once("://").map_or(url, |(_, rest)| rest);
352    let authority_end = rest.find('/').unwrap_or(rest.len());
353    let authority = rest.get(..authority_end).unwrap_or(rest);
354    let host = authority
355        .rsplit_once('@')
356        .map_or(authority, |(_, host)| host);
357    // @docgen An IPv6 host is bracketed and full of colons, so cutting at the first one leaves a stub that matches nothing.
358    if let Some(rest) = host.strip_prefix('[') {
359        return rest.split_once(']').map_or(host, |(inside, _)| inside);
360    }
361    let end = host.find([':', '?']).unwrap_or(host.len());
362    host.get(..end).unwrap_or(host)
363}
364
365#[cfg(test)]
366mod tests {
367
368    #[test]
369    fn a_referral_host_naming_an_internal_address_is_refused() {
370        for host in [
371            "127.0.0.1",
372            "169.254.169.254",
373            "10.0.0.1",
374            "192.168.1.1",
375            "localhost",
376            "whois.internal.localhost",
377            "::1",
378            "fe80::1",
379            "fd00::1",
380            "::ffff:127.0.0.1",
381            "::ffff:169.254.169.254",
382            "::ffff:10.0.0.1",
383            "::ffff:192.168.1.1",
384            "ff02::1",
385            "100.64.0.1",
386            "100.127.255.254",
387            "224.0.0.1",
388            "255.255.255.255",
389            "",
390            "whois example com",
391        ] {
392            assert!(
393                !is_public_host(host),
394                "{host} must never be dialled from a cleartext referral"
395            );
396        }
397    }
398
399    #[test]
400    fn a_real_registry_host_still_passes() {
401        for host in ["whois.btcl.net.bd", "whois.nic.example", "203.0.113.10"] {
402            assert!(is_public_host(host), "{host} is a normal public host");
403        }
404    }
405    use std::sync::Arc;
406
407    use tempfile::tempdir;
408
409    use super::*;
410    use crate::error::ErrorId;
411
412    #[test]
413    fn a_bracketed_ipv6_host_is_read_whole_rather_than_cut_at_its_first_colon() {
414        assert_eq!(host_of("https://[::1]/rdap/"), "::1");
415        assert_eq!(host_of("https://[fd00::1]:8443/rdap/"), "fd00::1");
416        assert_eq!(host_of("https://rdap.example/x"), "rdap.example");
417        assert_eq!(host_of("https://user:pass@rdap.example/x"), "rdap.example");
418    }
419
420    #[test]
421    fn an_internal_service_address_is_refused_in_either_address_family() {
422        for bad in [
423            "https://[::1]/rdap/",
424            "https://[fd00::1]/rdap/",
425            "https://127.0.0.1/rdap/",
426            "https://10.0.0.5/rdap/",
427            "https://169.254.169.254/rdap/",
428            "https://localhost/rdap/",
429            "http://rdap.example/",
430            "https://user:key@rdap.example/",
431        ] {
432            assert!(!is_usable_service(bad), "{bad} must not be fetched");
433        }
434        assert!(is_usable_service("https://rdap.verisign.com/com/v1/"));
435    }
436
437    const SAMPLE: &str = r#"{"services":[
438        [["com","net"],["https://rdap.verisign.com/com/v1"]],
439        [["uk"],["https://rdap.nominet.uk/uk/"]]
440    ]}"#;
441
442    /// @docgen A resolver that answers nothing keeps the download path in the test offline and instant.
443    #[derive(Debug)]
444    struct NeverResolves;
445
446    impl reqwest::dns::Resolve for NeverResolves {
447        fn resolve(&self, _name: reqwest::dns::Name) -> reqwest::dns::Resolving {
448            Box::pin(async {
449                Err(Box::<dyn std::error::Error + Send + Sync>::from(
450                    "this test never leaves the machine",
451                ))
452            })
453        }
454    }
455
456    fn grounded_client() -> reqwest::Client {
457        reqwest::Client::builder()
458            .no_proxy()
459            .dns_resolver(Arc::new(NeverResolves))
460            .build()
461            .expect("a client that can reach nothing")
462    }
463
464    fn age_by_days(path: &Path, days: u64) {
465        let when = SystemTime::now()
466            .checked_sub(Duration::from_secs(days * 24 * 60 * 60))
467            .expect("a moment inside the epoch");
468        let file = std::fs::File::options()
469            .write(true)
470            .open(path)
471            .expect("the cache opens for writing");
472        file.set_times(std::fs::FileTimes::new().set_modified(when))
473            .expect("the cache takes a new modified time");
474    }
475
476    fn cache_holding(dir: &Path, text: &str) -> std::path::PathBuf {
477        let path = dir.join("servers.json");
478        std::fs::write(&path, text).expect("the cache is written");
479        path
480    }
481
482    #[tokio::test]
483    async fn a_cache_written_today_is_read_instead_of_downloaded() {
484        let dir = tempdir().expect("temp dir");
485        let cache = cache_holding(dir.path(), SAMPLE);
486
487        let (services, freshness) = ServiceMap::load(&grounded_client(), &cache, false)
488            .await
489            .expect("a fresh cache needs no download");
490
491        assert_eq!(freshness, Freshness::Cached);
492        assert!(services.for_suffix("com").is_some());
493    }
494
495    #[tokio::test]
496    async fn a_cache_older_than_a_week_is_still_used_when_the_download_fails() {
497        let dir = tempdir().expect("temp dir");
498        let cache = cache_holding(dir.path(), SAMPLE);
499        age_by_days(&cache, 8);
500
501        let (services, freshness) = ServiceMap::load(&grounded_client(), &cache, false)
502            .await
503            .expect("a week-old list beats no list");
504
505        assert_eq!(freshness, Freshness::Stale);
506        assert!(services.for_suffix("com").is_some());
507    }
508
509    #[tokio::test]
510    async fn asking_for_a_refresh_still_falls_back_to_the_cache_it_skipped() {
511        let dir = tempdir().expect("temp dir");
512        let cache = cache_holding(dir.path(), SAMPLE);
513
514        let (services, freshness) = ServiceMap::load(&grounded_client(), &cache, true)
515            .await
516            .expect("a failed refresh falls back rather than failing");
517
518        assert_eq!(freshness, Freshness::Stale);
519        assert!(services.for_suffix("com").is_some());
520    }
521
522    #[tokio::test]
523    async fn no_cache_and_no_download_is_an_error_rather_than_an_empty_list() {
524        let dir = tempdir().expect("temp dir");
525        let missing = dir.path().join("never-written").join("servers.json");
526
527        let error = ServiceMap::load(&grounded_client(), &missing, false)
528            .await
529            .expect_err("an empty service map would read every extension as unserved");
530
531        assert_eq!(error.id(), ErrorId::BootstrapUnavailable);
532    }
533
534    #[tokio::test]
535    async fn a_corrupt_cache_is_not_read_as_a_list_with_nothing_in_it() {
536        let dir = tempdir().expect("temp dir");
537        let cache = cache_holding(dir.path(), "half a file, no json");
538
539        let error = ServiceMap::load(&grounded_client(), &cache, false)
540            .await
541            .expect_err("a corrupt cache must not stand in for a real list");
542
543        assert_eq!(error.id(), ErrorId::BootstrapUnavailable);
544    }
545
546    #[tokio::test]
547    async fn a_failed_download_never_overwrites_the_cache_it_fell_back_to() {
548        let dir = tempdir().expect("temp dir");
549        let cache = cache_holding(dir.path(), SAMPLE);
550        age_by_days(&cache, 8);
551
552        let _ = ServiceMap::load(&grounded_client(), &cache, false).await;
553
554        assert_eq!(
555            std::fs::read_to_string(&cache).expect("the cache survives"),
556            SAMPLE
557        );
558    }
559
560    #[test]
561    fn a_base_url_always_ends_in_a_slash() {
562        let services = ServiceMap::parse(SAMPLE).unwrap();
563        assert_eq!(
564            services.for_suffix("com"),
565            Some(&["https://rdap.verisign.com/com/v1/".to_owned()][..])
566        );
567    }
568
569    #[test]
570    fn a_multi_label_suffix_resolves_through_its_parent() {
571        let services = ServiceMap::parse(SAMPLE).unwrap();
572        assert!(services.for_suffix("co.uk").is_some());
573        assert_eq!(services.for_suffix("co.uk"), services.for_suffix("uk"));
574    }
575
576    #[test]
577    fn an_extension_with_no_service_reports_none() {
578        let services = ServiceMap::parse(SAMPLE).unwrap();
579        assert!(services.for_suffix("bd").is_none());
580        assert!(services.for_suffix("com.bd").is_none());
581    }
582
583    #[test]
584    fn services_missing_from_the_published_list_are_still_reachable() {
585        // @docgen The sample carries none of these, yet all of them answer, so the fallback must fill them in.
586        let services = ServiceMap::parse(SAMPLE).unwrap();
587        for suffix in ["de", "io", "us"] {
588            assert!(
589                services.for_suffix(suffix).is_some(),
590                ".{suffix} has a working service and must not read as having none"
591            );
592        }
593    }
594
595    #[test]
596    fn a_published_entry_wins_over_the_unlisted_fallback() {
597        let text = r#"{"services":[[["io"],["https://published.example/"]]]}"#;
598        let services = ServiceMap::parse(text).unwrap();
599        assert_eq!(
600            services.for_suffix("io"),
601            Some(&["https://published.example/".to_owned()][..])
602        );
603    }
604
605    #[test]
606    fn a_custom_list_overlays_the_published_one() {
607        let mut services = ServiceMap::parse(SAMPLE).unwrap();
608        let custom = ServiceMap::parse(r#"{"services":[[["com"],["https://mine/"]]]}"#).unwrap();
609        services.merge(custom);
610        assert_eq!(
611            services.for_suffix("com"),
612            Some(&["https://mine/".to_owned()][..])
613        );
614        assert!(services.for_suffix("uk").is_some());
615    }
616
617    #[test]
618    fn rubbish_is_refused() {
619        assert!(ServiceMap::parse("not json").is_err());
620        assert!(ServiceMap::parse(r#"{"services":"nope"}"#).is_err());
621    }
622
623    #[test]
624    fn an_empty_service_entry_is_skipped_rather_than_stored() {
625        let services = ServiceMap::parse(r#"{"services":[[[],[]],[["com"],[]]]}"#).unwrap();
626        assert!(services.for_suffix("com").is_none());
627    }
628
629    #[test]
630    fn hosts_come_out_of_urls() {
631        assert_eq!(
632            host_of("https://rdap.verisign.com/com/v1/"),
633            "rdap.verisign.com"
634        );
635        assert_eq!(host_of("http://a.b.c:8080/x"), "a.b.c");
636        assert_eq!(host_of("whois.nic.io"), "whois.nic.io");
637    }
638
639    #[test]
640    fn each_freshness_describes_itself() {
641        assert_eq!(Freshness::Fresh.label(), "downloaded");
642        assert!(Freshness::Stale.label().contains("out of date"));
643    }
644
645    #[test]
646    fn an_address_a_name_resolved_to_is_judged_the_same_way_the_name_was() {
647        for raw in [
648            "127.0.0.1",
649            "10.0.0.1",
650            "169.254.169.254",
651            "100.64.0.1",
652            "224.0.0.1",
653            "255.255.255.255",
654            "::1",
655            "fd00::1",
656            "fe80::1",
657            "::ffff:127.0.0.1",
658            "::ffff:10.0.0.1",
659        ] {
660            let ip: std::net::IpAddr = raw.parse().expect("a literal address parses");
661            assert!(
662                !is_public_ip(ip),
663                "{raw} must never be dialled, however the name reached it"
664            );
665        }
666
667        for raw in ["203.0.113.10", "2606:4700:4700::1111"] {
668            let ip: std::net::IpAddr = raw.parse().expect("a literal address parses");
669            assert!(is_public_ip(ip), "{raw} is a normal public address");
670        }
671    }
672}
reserve_core/lookup/registry.rs

reserve_core/lookup/
registry.rs
