holger_intel/

net.rs

//! The network gate. Every byte this crate could ever pull from outside goes
//! through [`IntelNet::get`], and an offline gate refuses every URL before any
//! socket exists.

use std::sync::atomic::{AtomicU64, Ordering};

/// A network refusal or failure.
#[derive(Debug, thiserror::Error)]
pub enum NetError {
    /// The gate is offline (`intel.offline = true`, or a build with no fetcher):
    /// the URL was refused, and nothing was sent.
    #[error("network refused: intel is offline, and {url} was not fetched")]
    Refused { url: String },
    /// The fetch was attempted and failed: no route, DNS, TLS, a non-200.
    #[error("network unavailable for {url}: {why}")]
    Unavailable { url: String, why: String },
}

impl NetError {
    pub fn is_refused(&self) -> bool {
        matches!(self, NetError::Refused { .. })
    }
}

/// Something that can GET a URL.
pub trait Fetch: Send + Sync {
    fn get(&self, url: &str) -> Result<Vec<u8>, NetError>;
    /// `get`, refusing a body past `max` bytes. A fetcher with no limit of its own
    /// may take the default.
    fn get_capped(&self, url: &str, max: u64) -> Result<Vec<u8>, NetError> {
        let _ = max;
        self.get(url)
    }
}

/// The gate itself: offline, or online over one [`Fetch`].
pub struct IntelNet {
    inner: Option<Box<dyn Fetch>>,
    refused: AtomicU64,
}

impl IntelNet {
    /// A gate that refuses every URL. There is no fetcher behind it at all, so
    /// a refusal is not a policy check that could be skipped: there is nothing
    /// to call.
    pub fn offline() -> Self {
        Self {
            inner: None,
            refused: AtomicU64::new(0),
        }
    }

    /// A gate over `fetch`.
    pub fn online(fetch: Box<dyn Fetch>) -> Self {
        Self {
            inner: Some(fetch),
            refused: AtomicU64::new(0),
        }
    }

    /// The online gate over `ureq` (feature `fetch`).
    #[cfg(feature = "fetch")]
    pub fn http() -> Self {
        Self::online(Box::new(UreqFetch::new()))
    }

    pub fn is_offline(&self) -> bool {
        self.inner.is_none()
    }

    /// How many URLs this gate has refused.
    pub fn refused_count(&self) -> u64 {
        self.refused.load(Ordering::Relaxed)
    }

    pub fn get(&self, url: &str) -> Result<Vec<u8>, NetError> {
        match &self.inner {
            None => {
                self.refused.fetch_add(1, Ordering::Relaxed);
                Err(NetError::Refused {
                    url: url.to_string(),
                })
            }
            Some(f) => f.get(url),
        }
    }

    /// **One feed, under that feed's own cap** ([`crate::FeedSpec::max_bytes`],
    /// HOL-DOGFOOD-2): an OSV dump may be several GiB (Chainguard's passed 1 GiB in
    /// 2026), NVD and KEV keep the 1 GiB bound.
    pub fn get_feed(&self, spec: &crate::FeedSpec) -> Result<Vec<u8>, NetError> {
        match &self.inner {
            None => {
                self.refused.fetch_add(1, Ordering::Relaxed);
                Err(NetError::Refused {
                    url: spec.uri.clone(),
                })
            }
            Some(f) => f.get_capped(&spec.uri, spec.max_bytes()),
        }
    }
}

/// The online fetcher: https only, bounded time and size, no redirects, no
/// proxy from the environment.
#[cfg(feature = "fetch")]
pub struct UreqFetch {
    agent: ureq::Agent,
}

#[cfg(feature = "fetch")]
impl UreqFetch {
    /// The largest body [`Fetch::get`] accepts; a feed's own cap is
    /// [`crate::FeedSpec::max_bytes`].
    pub const MAX_BODY: u64 = 1024 * 1024 * 1024;

    pub fn new() -> Self {
        use std::time::Duration;
        let agent = ureq::Agent::config_builder()
            .timeout_connect(Some(Duration::from_secs(15)))
            .timeout_recv_response(Some(Duration::from_secs(60)))
            .timeout_recv_body(Some(Duration::from_secs(60 * 60)))
            .max_redirects(0)
            .http_status_as_error(false)
            .proxy(None)
            .build()
            .new_agent();
        Self { agent }
    }
}

#[cfg(feature = "fetch")]
impl Default for UreqFetch {
    fn default() -> Self {
        Self::new()
    }
}

#[cfg(feature = "fetch")]
impl Fetch for UreqFetch {
    fn get(&self, url: &str) -> Result<Vec<u8>, NetError> {
        self.get_capped(url, Self::MAX_BODY)
    }

    fn get_capped(&self, url: &str, max: u64) -> Result<Vec<u8>, NetError> {
        let unavailable = |why: String| NetError::Unavailable {
            url: url.to_string(),
            why,
        };
        if !url.starts_with("https://") && !is_loopback_http(url) {
            return Err(unavailable(
                "only https:// feeds are fetched (plain http on loopback only)".into(),
            ));
        }
        let mut resp = self
            .agent
            .get(url)
            .call()
            .map_err(|e| unavailable(e.to_string()))?;
        let status = resp.status().as_u16();
        if status != 200 {
            return Err(unavailable(format!("HTTP {status}")));
        }
        resp.body_mut()
            .with_config()
            .limit(max)
            .read_to_vec()
            .map_err(|e| unavailable(format!("{e} (this feed is capped at {max} bytes)")))
    }
}

/// `http://127.0.0.1…`, `http://[::1]…` or `http://localhost…`: a feed mirror on
/// the box itself (`intel.feed_mirror`), the one plain-http source fetched.
pub fn is_loopback_http(url: &str) -> bool {
    let Some(rest) = url.strip_prefix("http://") else {
        return false;
    };
    let host = rest.split('/').next().unwrap_or("");
    let host = host
        .rsplit_once(':')
        .map(|(h, p)| {
            if p.chars().all(|c| c.is_ascii_digit()) {
                h
            } else {
                host
            }
        })
        .unwrap_or(host);
    matches!(host, "127.0.0.1" | "[::1]" | "localhost")
}

#[cfg(test)]
mod loopback_tests {
    #[test]
    fn only_a_loopback_host_may_be_plain_http() {
        for ok in [
            "http://127.0.0.1:8080/index.txt",
            "http://localhost/kev.json",
            "http://[::1]:9/x",
        ] {
            assert!(super::is_loopback_http(ok), "{ok}");
        }
        for no in [
            "http://127.0.0.1.evil.example/x",
            "http://10.0.0.1/x",
            "https://127.0.0.1/x",
            "http://localhost.example:80/x",
            "ftp://localhost/x",
        ] {
            assert!(!super::is_loopback_http(no), "{no}");
        }
    }
}

#[cfg(test)]
mod per_feed_cap_tests {
    use super::*;
    use crate::feeds::FeedSpec;
    use std::sync::{Arc, Mutex};

    /// Records the cap each GET was asked with.
    struct Caps(Arc<Mutex<Vec<u64>>>);
    impl Fetch for Caps {
        fn get(&self, _url: &str) -> Result<Vec<u8>, NetError> {
            self.get_capped(_url, u64::MAX)
        }
        fn get_capped(&self, _url: &str, max: u64) -> Result<Vec<u8>, NetError> {
            self.0.lock().unwrap().push(max);
            Ok(Vec::new())
        }
    }

    /// HOL-DOGFOOD-2: the Chainguard OSV dump was 1 116 562 155 bytes on 2026-11-03, past the
    /// one 1 GiB body limit, and the whole default refresh failed on it. Each feed declares
    /// its own cap: an OSV dump has room to grow past 4 GiB; NVD and KEV keep 1 GiB.
    #[test]
    fn each_feed_is_fetched_under_its_own_cap() {
        let seen = Arc::new(Mutex::new(Vec::new()));
        let net = IntelNet::online(Box::new(Caps(Arc::clone(&seen))));
        net.get_feed(&FeedSpec::osv("Chainguard")).unwrap();
        net.get_feed(&FeedSpec::nvd("2024")).unwrap();
        net.get_feed(&FeedSpec::kev()).unwrap();
        let seen = seen.lock().unwrap().clone();
        assert!(
            seen[0] >= 4 * 1024 * 1024 * 1024,
            "the OSV dump cap: {seen:?}"
        );
        assert_eq!(seen[1], 1024 * 1024 * 1024, "{seen:?}");
        assert_eq!(seen[2], 1024 * 1024 * 1024, "{seen:?}");
    }
}