nym_http_api_client/

dns.rs

// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: Apache-2.0

//! DNS resolver configuration for internal lookups.
//!
//! The resolver itself is the set combination of the cloudflare, and quad9 endpoints supporting DoH
//! and DoT.
//!
//! ```rust
//! use nym_http_api_client::HickoryDnsResolver;
//! # use nym_http_api_client::ResolveError;
//! # type Err = ResolveError;
//! # async fn run() -> Result<(), Err> {
//! let resolver = HickoryDnsResolver::default();
//! resolver.resolve_str("example.com").await?;
//! # Ok(())
//! # }
//! ```
//!
//! ## Fallbacks
//!
//! **System Resolver --** This resolver supports an optional fallback mechanism where, should the
//! DNS-over-TLS resolution fail, a followup resolution will be done using the hosts configured
//! default (e.g. `/etc/resolve.conf` on linux).
//!
//! This is disabled by default and can be enabled using `enable_system_fallback`.
//!
//! **Static Table --**  There is also a second optional fallback mechanism that allows a static map
//! to be used as a last resort. This can help when DNS encounters errors due to blocked resolvers
//! or unknown conditions. This is enabled by default, and can be customized if building a new
//! resolver.
//!
//! ## IPv4 / IPv6
//!
//! By default the resolver uses only IPv4 nameservers, and is configured to do `A` lookups first,
//! and only do `AAAA` if no `A` record is available.
//!
//! ---
//!
//! Requires the `dns-over-https-rustls`, `webpki-roots` feature for the `hickory-resolver` crate
#![deny(missing_docs)]

use crate::ClientBuilder;

use std::{
    collections::HashMap,
    net::{IpAddr, SocketAddr},
    str::FromStr,
    sync::{
        Arc, LazyLock,
        atomic::{AtomicBool, Ordering::Relaxed},
    },
    time::Duration,
};

use hickory_resolver::{
    TokioResolver,
    config::{CLOUDFLARE, NameServerConfig, QUAD9, ResolverConfig, ResolverOpts},
    net::{NetError, runtime::TokioRuntimeProvider},
};
use once_cell::sync::OnceCell;
use reqwest::dns::{Addrs, Name, Resolve, Resolving};
use tracing::*;

mod constants;
mod static_resolver;
pub(crate) use static_resolver::*;

pub(crate) const DEFAULT_POSITIVE_LOOKUP_CACHE_TTL: Duration = Duration::from_secs(1800);
pub(crate) const DEFAULT_OVERALL_LOOKUP_TIMEOUT: Duration = Duration::from_secs(10);
pub(crate) const DEFAULT_QUERY_TIMEOUT: Duration = Duration::from_secs(5);

impl ClientBuilder {
    /// Override the DNS resolver implementation used by the underlying http client.
    /// This forces the use of an independent request executor (via [`Self::non_shared`]).
    pub fn dns_resolver<R: Resolve + 'static>(mut self, resolver: Arc<R>) -> Self {
        self = self.non_shared();
        // because of the call to non-shared this conditional should always run.
        if let Some(rb) = self.reqwest_client_builder {
            self.reqwest_client_builder = Some(rb.dns_resolver(resolver));
        }
        self.use_secure_dns = false;
        self
    }

    /// Override the DNS resolver implementation used by the underlying http client. If
    /// [`Self::dns_resolver`] is called directly that will take priority over this, there is no
    /// need to call both.
    /// This forces the use of an independent request executor (via [`Self::non_shared`]).
    pub fn no_hickory_dns(mut self) -> Self {
        self = self.non_shared();
        self.use_secure_dns = false;
        self
    }
}

// n.b. static items do not call [`Drop`] on program termination, so this won't be deallocated.
// this is fine, as the OS can deallocate the terminated program faster than we can free memory
// but tools like valgrind might report "memory leaks" as it isn't obvious this is intentional.
static SHARED_RESOLVER: LazyLock<HickoryDnsResolver> = LazyLock::new(|| {
    tracing::debug!("Initializing shared DNS resolver");
    HickoryDnsResolver {
        use_shared: false, // prevent infinite recursion
        ..Default::default()
    }
});

#[derive(Debug, thiserror::Error)]
#[allow(missing_docs)]
/// Error occurring while resolving a hostname into an IP address.
pub enum ResolveError {
    #[error("invalid name: {0}")]
    InvalidNameError(String),
    #[error("hickory-dns resolver error: {0}")]
    ResolveError(#[from] NetError),
    #[error("high level lookup timed out")]
    Timeout,
    #[error("hostname not found in static lookup table")]
    StaticLookupMiss,
}

impl ResolveError {
    /// Returns true if the error is a timeout.
    pub fn is_timeout(&self) -> bool {
        matches!(
            self,
            ResolveError::Timeout | ResolveError::ResolveError(NetError::Timeout)
        )
    }
}

/// Wrapper around an `AsyncResolver`, which implements the `Resolve` trait.
///
/// Typical use involves instantiating using the `Default` implementation and then resolving using
/// methods or trait implementations.
///
/// The default initialization uses a shared underlying `AsyncResolver`. If a thread local resolver
/// is required use `thread_resolver()` to build a resolver with an independently instantiated
/// internal `AsyncResolver`.
#[derive(Debug, Clone)]
pub struct HickoryDnsResolver {
    // Since we might not have been called in the context of a
    // Tokio Runtime in initialization, so we must delay the actual
    // construction of the resolver.
    state: Arc<OnceCell<TokioResolver>>,
    use_system: Arc<AtomicBool>,
    system_resolver: Arc<OnceCell<TokioResolver>>,
    static_base: Option<Arc<OnceCell<StaticResolver>>>,
    use_shared: bool,
    /// Overall timeout for dns lookup associated with any individual host resolution. For example,
    /// use of retries, server_ordering_strategy, etc. ends absolutely if this timeout is reached.
    overall_dns_timeout: Duration,
}

impl Default for HickoryDnsResolver {
    fn default() -> Self {
        Self {
            state: Default::default(),
            use_system: Arc::new(AtomicBool::new(false)),
            system_resolver: Default::default(),
            static_base: Some(Default::default()),
            use_shared: true,
            overall_dns_timeout: DEFAULT_OVERALL_LOOKUP_TIMEOUT,
        }
    }
}

impl Resolve for HickoryDnsResolver {
    fn resolve(&self, name: Name) -> Resolving {
        let use_system = self.use_system.load(std::sync::atomic::Ordering::Relaxed);
        let use_shared = self.use_shared;
        let result = if use_system {
            self.system_resolver
                .get_or_try_init(|| HickoryDnsResolver::new_resolver_system(use_shared))
        } else {
            self.state
                .get_or_try_init(|| HickoryDnsResolver::new_resolver(use_shared))
        };

        let resolver = match result {
            Ok(r) => r.clone(),
            Err(err) => return Box::pin(return_err(err)),
        };

        let maybe_static = self.static_base.clone();
        let overall_dns_timeout = self.overall_dns_timeout;
        Box::pin(async move {
            resolve(
                name,
                resolver,
                maybe_static,
                use_shared,
                overall_dns_timeout,
            )
            .await
            .map_err(|e| Box::new(e) as Box<dyn std::error::Error + Send + Sync>)
        })
    }
}

async fn return_err(e: ResolveError) -> Result<Addrs, Box<dyn std::error::Error + Send + Sync>> {
    Err(Box::new(e) as Box<dyn std::error::Error + Send + Sync>)
}

async fn resolve(
    name: Name,
    resolver: TokioResolver,
    maybe_static: Option<Arc<OnceCell<StaticResolver>>>,
    independent: bool,
    overall_dns_timeout: Duration,
) -> Result<Addrs, ResolveError> {
    // try checking the static table to see if any of the addresses in the table have been
    // looked up previously within the timeout to where we are not yet ready to try the
    // default resolver yet again.
    if let Some(ref static_resolver) = maybe_static {
        let resolver =
            static_resolver.get_or_init(|| HickoryDnsResolver::new_static_fallback(independent));

        if let Some(addrs) = resolver.pre_resolve(name.as_str()) {
            let addrs: Addrs =
                Box::new(addrs.into_iter().map(|ip_addr| SocketAddr::new(ip_addr, 0)));
            return Ok(addrs);
        }
    }

    // Attempt a lookup using the primary resolver
    let resolve_fut = tokio::time::timeout(overall_dns_timeout, resolver.lookup_ip(name.as_str()));
    let primary_err = match resolve_fut.await {
        Err(_) => ResolveError::Timeout,
        Ok(Ok(lookup)) => {
            // Shuffle so that successive connection attempts cycle through all
            // returned IPs rather than always hitting the same first address.
            let mut ips = Vec::from_iter(lookup.iter());
            fastrand::shuffle(&mut ips);
            let addrs: Addrs = Box::new(ips.into_iter().map(|ip| SocketAddr::new(ip, 0)));
            return Ok(addrs);
        }
        Ok(Err(e)) => {
            // on failure use the fall back system configured DNS resolver
            if !e.is_no_records_found() {
                warn!("primary DNS failed w/ error: {e}");
            }
            e.into()
        }
    };

    // If no record has been found and a static map of fallback addresses is configured
    // check the table for our entry
    if let Some(ref static_resolver) = maybe_static {
        debug!("checking static");
        let resolver =
            static_resolver.get_or_init(|| HickoryDnsResolver::new_static_fallback(independent));

        if let Ok(addrs) = resolver.resolve(name).await {
            return Ok(addrs);
        }
    }

    Err(primary_err)
}

impl HickoryDnsResolver {
    /// Returns an instance of the shared resolver.
    pub fn shared() -> Self {
        SHARED_RESOLVER.clone()
    }

    /// Attempt to resolve a domain name to a set of ['IpAddr']s
    pub async fn resolve_str(
        &self,
        name: &str,
    ) -> Result<impl Iterator<Item = IpAddr> + use<>, ResolveError> {
        let n =
            Name::from_str(name).map_err(|_| ResolveError::InvalidNameError(name.to_string()))?;
        let use_system = self.use_system.load(std::sync::atomic::Ordering::Relaxed);
        let resolver = if use_system {
            self.system_resolver
                .get_or_try_init(|| HickoryDnsResolver::new_resolver_system(self.use_shared))?
                .clone()
        } else {
            self.state
                .get_or_try_init(|| HickoryDnsResolver::new_resolver(self.use_shared))?
                .clone()
        };

        resolve(
            n,
            resolver,
            self.static_base.clone(),
            self.use_shared,
            self.overall_dns_timeout,
        )
        .await
        .map(|addrs| addrs.map(|socket_addr| socket_addr.ip()))
    }

    /// Create a (lazy-initialized) resolver that is not shared across threads.
    pub fn thread_resolver() -> Self {
        Self {
            use_shared: false,
            ..Default::default()
        }
    }

    fn new_resolver(use_shared: bool) -> Result<TokioResolver, ResolveError> {
        // using a closure here is slightly gross, but this makes sure that if the
        // lazy-init returns an error it can be handled by the client
        if use_shared {
            SHARED_RESOLVER.state.get_or_try_init(new_resolver).cloned()
        } else {
            new_resolver()
        }
    }

    fn new_resolver_system(use_shared: bool) -> Result<TokioResolver, ResolveError> {
        // using a closure here is slightly gross, but this makes sure that if the
        // lazy-init returns an error it can be handled by the client
        if !use_shared {
            new_resolver_system()
        } else {
            Ok(SHARED_RESOLVER
                .system_resolver
                .get_or_try_init(new_resolver_system)?
                .clone())
        }
    }

    fn new_static_fallback(use_shared: bool) -> StaticResolver {
        if use_shared && let Some(ref shared_resolver) = SHARED_RESOLVER.static_base {
            shared_resolver
                .get_or_init(new_default_static_fallback)
                .clone()
        } else {
            new_default_static_fallback()
        }
    }

    /// Swap the primary internal resolver to the system resolver rather than the
    /// configured custom resolver.
    pub fn use_system_resolver(&self) {
        self.use_system.store(true, Relaxed);

        if self.use_shared {
            SHARED_RESOLVER.use_system_resolver();
        }
    }

    /// Swap the primary internal resolver to the configured custom resolver rather than the
    /// system resolver.
    pub fn use_configured_resolver(&self) {
        self.use_system.store(false, Relaxed);

        if self.use_shared {
            SHARED_RESOLVER.use_configured_resolver();
        }
    }

    /// Clear entries from the static table that would return entries during the pre-resolve stage.
    /// This means that all lookups will attempt to use the network resolver again before the static
    /// table is consulted.
    ///
    /// Entries elevated to pre-resolve from fallback (added from default or using
    /// [`set_fallback`]`) will have their cache timeout cleared. Entries added directly to
    /// pre-resolve (using [`Self::set_static_preresolve`]) will be removed.
    pub fn clear_preresolve(&self) {
        debug!("clearing pre-resolve table");
        if let Some(cell) = &self.static_base
            && let Some(static_base) = cell.get()
        {
            static_base.clear_preresolve()
        }
    }

    /// Get the current map of hostnames to addresses used in the fallback static lookup stage if one
    /// exists.
    pub fn get_static_fallbacks(&self) -> Option<HashMap<String, Vec<IpAddr>>> {
        Some(self.static_base.as_ref()?.get()?.get_fallback_addrs())
    }

    /// Set (or overwrite) the map of addresses used in the fallback static hostname lookup
    pub fn set_fallback_addrs(&mut self, addrs: HashMap<String, Vec<IpAddr>>) {
        debug!("setting fallback entries for {:?}", addrs.keys());
        if self.static_base.is_none() {
            let cell = OnceCell::new();
            self.static_base = Some(Arc::new(cell));
        }
        self.static_base
            .as_ref()
            .unwrap()
            .get_or_init(|| Self::new_static_fallback(self.use_shared))
            .set_fallback(addrs);
    }

    /// Get the current map of hostnames to addresses used in the preresolve static lookup stage
    /// if one exists.
    pub fn get_static_preresolve(&self) -> Option<HashMap<String, Vec<IpAddr>>> {
        Some(self.static_base.as_ref()?.get()?.get_preresolve_addrs())
    }

    /// Set (or overwrite) the map of addresses used in the preresolve static hostname lookup
    pub fn set_static_preresolve(&mut self, addrs: HashMap<String, Vec<IpAddr>>) {
        debug!("setting pre-resolve entries for {:?}", addrs.keys());
        if self.static_base.is_none() {
            let cell = OnceCell::new();
            self.static_base = Some(Arc::new(cell));
        }
        self.static_base
            .as_ref()
            .unwrap()
            .get_or_init(|| Self::new_static_fallback(self.use_shared))
            .set_preresolve(addrs);
    }

    /// Successfully resolved addresses are cached for a minimum of 30 minutes
    /// Individual lookup Timeouts are set to 3 seconds
    /// Number of retries after lookup failure before giving up is set to (default) to 2
    /// Lookup order is set to (default) A then AAAA
    /// Number or parallel lookup is set to (default) 2
    /// Nameserver selection uses the (default) EWMA statistics / performance based strategy
    fn default_options() -> ResolverOpts {
        let mut opts = ResolverOpts::default();
        // Always cache successful responses for queries received by this resolver for 30 min minimum.
        opts.positive_min_ttl = Some(DEFAULT_POSITIVE_LOOKUP_CACHE_TTL);
        opts.timeout = DEFAULT_QUERY_TIMEOUT;
        opts.attempts = 0;

        opts
    }

    /// Get the list of currently available nameserver configs.
    pub fn all_configured_name_servers(&self) -> Vec<NameServerConfig> {
        default_nameserver_group()
    }

    /// Do a trial resolution using each nameserver individually to test which are working and which
    /// fail to complete a lookup. This will always try the full set of default configured resolvers.
    pub async fn trial_nameservers(&self) {
        let nameservers = default_nameserver_group();
        for (ns, result) in trial_nameservers_inner(&nameservers).await {
            if let Err(e) = result {
                warn!("trial {ns:?} errored: {e}");
            } else {
                info!("trial {ns:?} succeeded");
            }
        }
    }
}

/// Create a new resolver with a custom DoT based configuration. The options are overridden to look
/// up for both IPv4 and IPv6 addresses to work with "happy eyeballs" algorithm.
///
/// Individual lookup Timeouts are set to 3 seconds
/// Number of retries after lookup failure before giving up Defaults to 2
/// Lookup order is set to (default) A then AAAA
///
/// Caches successfully resolved addresses for 30 minutes to prevent continual use of remote lookup.
/// This resolver is intended to be used for OUR API endpoints that do not rapidly rotate IPs.
fn new_resolver() -> Result<TokioResolver, ResolveError> {
    let name_servers = default_nameserver_group_ipv4_only();

    configure_and_build_resolver(name_servers)
}

fn configure_and_build_resolver(
    name_servers: Vec<NameServerConfig>,
) -> Result<TokioResolver, ResolveError> {
    let options = HickoryDnsResolver::default_options();
    info!("building new configured resolver");
    debug!("configuring resolver with {options:?}, {name_servers:?}");

    let config = ResolverConfig::from_parts(None, Vec::new(), name_servers);
    let mut resolver_builder =
        TokioResolver::builder_with_config(config, TokioRuntimeProvider::default());

    resolver_builder = resolver_builder.with_options(options);

    Ok(resolver_builder.build()?)
}

fn filter_ipv4(nameservers: impl IntoIterator<Item = NameServerConfig>) -> Vec<NameServerConfig> {
    nameservers
        .into_iter()
        .filter(|ns| ns.ip.is_ipv4())
        .collect()
}

#[allow(unused)]
fn filter_ipv6(nameservers: impl IntoIterator<Item = NameServerConfig>) -> Vec<NameServerConfig> {
    nameservers
        .into_iter()
        .filter(|ns| ns.ip.is_ipv6())
        .collect()
}

#[allow(unused)]
fn default_nameserver_group() -> Vec<NameServerConfig> {
    QUAD9
        .tls()
        .chain(QUAD9.https())
        .chain(CLOUDFLARE.tls())
        .chain(CLOUDFLARE.https())
        .collect()
}

fn default_nameserver_group_ipv4_only() -> Vec<NameServerConfig> {
    filter_ipv4(default_nameserver_group())
}

#[allow(unused)]
fn default_nameserver_group_ipv6_only() -> Vec<NameServerConfig> {
    filter_ipv6(default_nameserver_group())
}

/// Create a new resolver with the default configuration, which reads from the system DNS config
/// (i.e. `/etc/resolve.conf` in unix). The options are overridden to look up for both IPv4 and IPv6
/// addresses to work with "happy eyeballs" algorithm.
fn new_resolver_system() -> Result<TokioResolver, ResolveError> {
    let mut resolver_builder = TokioResolver::builder_tokio()?;

    let options = HickoryDnsResolver::default_options();
    info!("building new fallback system resolver");
    debug!("fallback system resolver with {options:?}");

    resolver_builder = resolver_builder.with_options(options);

    Ok(resolver_builder.build()?)
}

fn new_default_static_fallback() -> StaticResolver {
    StaticResolver::new().with_fallback(constants::default_static_addrs())
}

/// Do a trial resolution using each nameserver individually to test which are working and which
/// fail to complete a lookup.
async fn trial_nameservers_inner(
    name_servers: &[NameServerConfig],
) -> Vec<(NameServerConfig, Result<(), ResolveError>)> {
    let mut trial_lookups = tokio::task::JoinSet::new();

    for name_server in name_servers {
        let ns = name_server.clone();
        trial_lookups.spawn(async { (ns.clone(), trial_lookup(ns, "example.com").await) });
    }

    trial_lookups.join_all().await
}

/// Create an independent resolver that has only the provided nameserver and do one lookup for the
/// provided query target.
async fn trial_lookup(name_server: NameServerConfig, query: &str) -> Result<(), ResolveError> {
    debug!("running ns trial {name_server:?} query={query}");

    let resolver = configure_and_build_resolver(vec![name_server])?;

    match tokio::time::timeout(DEFAULT_OVERALL_LOOKUP_TIMEOUT, resolver.ipv4_lookup(query)).await {
        Ok(Ok(_)) => Ok(()),
        Ok(Err(e)) => Err(e.into()),
        Err(_) => Err(ResolveError::Timeout),
    }
}

#[cfg(test)]
mod test {
    use super::*;
    use itertools::Itertools;
    use std::{
        collections::HashMap,
        net::{IpAddr, Ipv4Addr, Ipv6Addr},
    };

    /// IP addresses guaranteed to fail attempts to resolve
    ///
    /// Addresses drawn from blocks set off by RFC5737 (ipv4) and RFC3849 (ipv6)
    const GUARANTEED_BROKEN_IPS_1: &[IpAddr] = &[
        IpAddr::V4(Ipv4Addr::new(192, 0, 2, 1)),
        IpAddr::V4(Ipv4Addr::new(198, 51, 100, 1)),
        IpAddr::V6(Ipv6Addr::new(0x2001, 0x0db8, 0, 0, 0, 0, 0, 0x1111)),
        IpAddr::V6(Ipv6Addr::new(0x2001, 0x0db8, 0, 0, 0, 0, 0, 0x1001)),
    ];

    #[tokio::test]
    async fn reqwest_with_custom_dns() {
        let var_name = HickoryDnsResolver::default();
        let resolver = var_name;
        let client = reqwest::ClientBuilder::new()
            .dns_resolver(resolver)
            .build()
            .unwrap();

        let resp = client
            .get("http://ifconfig.me:80")
            .send()
            .await
            .unwrap()
            .bytes()
            .await
            .unwrap();

        assert!(!resp.is_empty());
    }

    #[tokio::test]
    async fn dns_lookup() -> Result<(), ResolveError> {
        let resolver = HickoryDnsResolver::default();

        let domain = "ifconfig.me";
        let addrs = resolver.resolve_str(domain).await?;

        assert!(addrs.into_iter().next().is_some());

        Ok(())
    }

    #[tokio::test]
    async fn static_resolver_as_fallback() -> Result<(), ResolveError> {
        let example_domain = "non-existent.nymvpn.com";
        let mut resolver = HickoryDnsResolver {
            ..Default::default()
        };

        let result = resolver.resolve_str(example_domain).await;
        assert!(result.is_err()); // should be NXDomain

        resolver.static_base = Some(Default::default());

        let mut addr_map = HashMap::new();
        let example_ip4: IpAddr = "10.10.10.10".parse().unwrap();
        let example_ip6: IpAddr = "dead::beef".parse().unwrap();
        addr_map.insert(example_domain.to_string(), vec![example_ip4, example_ip6]);

        resolver.set_fallback_addrs(addr_map);

        let mut addrs = resolver.resolve_str(example_domain).await?;
        assert!(addrs.contains(&example_ip4));
        assert!(addrs.contains(&example_ip6));
        Ok(())
    }

    // Test the nameserver trial functionality with mostly nameservers guaranteed to be broken and
    // one that should work.
    #[tokio::test]
    async fn trial_nameservers() {
        let good_cf_ip = IpAddr::V4(Ipv4Addr::new(1, 1, 1, 1));

        let mut ns_ips = GUARANTEED_BROKEN_IPS_1.to_vec();
        ns_ips.push(good_cf_ip);

        let domain = Arc::<str>::from("cloudflare-dns.com");
        let path = Arc::<str>::from("/dns-query");
        let broken_ns_https = GUARANTEED_BROKEN_IPS_1
            .iter()
            .chain([&good_cf_ip])
            .map(|ip| NameServerConfig::https(*ip, domain.clone(), Some(path.clone())))
            .collect::<Vec<_>>();

        for (ns, result) in trial_nameservers_inner(&broken_ns_https).await {
            if ns.ip == good_cf_ip {
                assert!(result.is_ok())
            } else {
                assert!(result.is_err())
            }
        }
    }

    mod failure_test {
        use super::*;

        // Create a resolver that behaves the same as the custom configured router, except for the fact
        // that it is guaranteed to fail.
        fn build_broken_resolver() -> Result<TokioResolver, ResolveError> {
            info!("building new faulty resolver");

            let domain = Arc::<str>::from("cloudflare-dns.com");
            let path = Arc::<str>::from("/dns-query");
            let broken_ns_group = GUARANTEED_BROKEN_IPS_1
                .iter()
                .map(|ip| NameServerConfig::tls(*ip, domain.clone()))
                .chain(
                    GUARANTEED_BROKEN_IPS_1
                        .iter()
                        .map(|ip| NameServerConfig::https(*ip, domain.clone(), Some(path.clone())))
                        .collect::<Vec<_>>(),
                )
                .collect::<Vec<_>>();

            configure_and_build_resolver(broken_ns_group)
        }

        #[tokio::test]
        async fn dns_lookup_failures() -> Result<(), ResolveError> {
            let time_start = std::time::Instant::now();

            let r = OnceCell::new();
            r.set(build_broken_resolver().expect("failed to build resolver"))
                .expect("broken resolver init error");

            // create a new resolver that won't mess with the shared resolver used by other tests
            let resolver = HickoryDnsResolver {
                use_shared: false,
                state: Arc::new(r),
                overall_dns_timeout: Duration::from_secs(5),
                ..Default::default()
            };
            build_broken_resolver()?;
            let domain = "ifconfig.me";
            let result = resolver.resolve_str(domain).await;
            assert!(result.is_err_and(|e| e.is_timeout()));

            let duration = time_start.elapsed();
            assert!(duration < resolver.overall_dns_timeout + Duration::from_secs(1));

            Ok(())
        }

        #[tokio::test]
        async fn fallback_to_static() -> Result<(), ResolveError> {
            let r = OnceCell::new();
            r.set(build_broken_resolver().expect("failed to build resolver"))
                .expect("broken resolver init error");

            // create a new resolver that won't mess with the shared resolver used by other tests
            let resolver = HickoryDnsResolver {
                use_shared: false,
                state: Arc::new(r),
                static_base: Some(Default::default()),
                overall_dns_timeout: Duration::from_secs(5),
                ..Default::default()
            };
            build_broken_resolver()?;

            // successful lookup using fallback to static resolver
            let domain = "nymvpn.com";
            let _ = resolver
                .resolve_str(domain)
                .await
                .expect("failed to resolve address in static lookup");

            // unsuccessful lookup - primary times out, and not in static table
            let domain = "non-existent.nymtech.net";
            let result = resolver.resolve_str(domain).await;
            assert!(result.is_err_and(|e| e.is_timeout()));

            Ok(())
        }

        #[tokio::test]
        #[cfg(any())] // #[ignore] we run --ignore in CI/CD assuming it just means slow -_-
        // This test impacts the state of the shared resolver and as such is disabled to avoid
        // interference with other tests.
        //
        // this test is dependent of external network setup -- i.e. blocking all traffic to the
        // default resolvers. Otherwise the default resolvers will succeed without using the static
        // fallback, making the test pointless
        async fn dns_lookup_failure_on_shared() -> Result<(), ResolveError> {
            let resolver1 = HickoryDnsResolver::shared();

            let time_start = std::time::Instant::now();
            // create a new resolver that uses the shared resolver
            let resolver = HickoryDnsResolver::shared();

            // successful lookup using fallback to static resolver
            let domain = "rpc.nymtech.net";
            let _ = resolver
                .resolve_str(domain)
                .await
                .expect("failed to resolve address in static lookup");

            let lookup_dur = Instant::now() - time_start;
            assert!(
                lookup_dur > resolver.overall_dns_timeout,
                "expected lookup timeout - took {}ms",
                (lookup_dur).as_millis()
            );

            let time_start = std::time::Instant::now();
            // successful lookup using pre-resolve entry promoted from fallback
            let domain = "rpc.nymtech.net";
            let _ = resolver1
                .resolve_str(domain)
                .await
                .expect("domain expected to be in pre-resolve");

            // this lookup should basically be instant as we are using pre-resolve
            let lookup_dur = std::time::Instant::now() - time_start;
            assert!(
                lookup_dur < Duration::from_millis(10),
                "expected instant - took {}ms",
                (lookup_dur).as_millis()
            );

            // unsuccessful lookup - primary times out, and not in static table
            let domain = "non-existent.nymtech.net";
            let result = resolver.resolve_str(domain).await;
            assert!(result.is_err());
            // assert!(result.is_err_and(|e| matches!(e, ResolveError::Timeout)));
            // assert!(result.is_err_and(|e| matches!(e, ResolveError::ResolveError(e) if e.is_nx_domain())));
            Ok(())
        }

        #[tokio::test]
        #[cfg(any())] // #[ignore] we run --ignore in CI/CD assuming it just means slow -_-
        // This test impacts the state of the shared resolver and as such is disabled to avoid
        // interference with other tests.
        async fn setting_dns_fallbacks_with_shared_resolver() -> Result<(), ResolveError> {
            let resolver1 = HickoryDnsResolver::shared();

            // create a new resolver that uses the shared resolver
            let mut resolver = HickoryDnsResolver::shared();

            let example_domains = [
                String::from("static1.nymvpn.com"),
                String::from("static2.nymvpn.com"),
            ];
            let mut addr_map1 = HashMap::new();
            addr_map1.insert(
                example_domains[0].clone(),
                vec![Ipv4Addr::new(10, 10, 10, 10).into()],
            );
            addr_map1.insert(
                example_domains[1].clone(),
                vec![Ipv4Addr::new(1, 1, 1, 1).into()],
            );

            resolver.set_static_preresolve(addr_map1);

            let time_start = std::time::Instant::now();
            // successful lookup using pre-resolve entry promoted from fallback
            let _ = resolver1
                .resolve_str(&example_domains[0])
                .await
                .expect("domain expected to be in pre-resolve");

            // this lookup should basically be instant as we are using pre-resolve
            let lookup_dur = std::time::Instant::now() - time_start;
            assert!(
                lookup_dur < Duration::from_millis(10),
                "expected instant - took {}ms",
                (lookup_dur).as_millis()
            );

            // After clearing the pre-resolve in one instance of the shared resolver ...
            resolver.clear_preresolve();

            // ... other instances have their pre-resolve entries cleared.
            let prereslve_lookup = resolver1
                .static_base
                .as_ref()
                .unwrap()
                .get()
                .unwrap()
                .pre_resolve(&example_domains[0]);
            assert!(prereslve_lookup.is_none());

            Ok(())
        }
    }
}