holger-intel 3.0.0

//! The three upstream feeds and their normalization into one [`AdvisorySet`].
//!
//! * **OSV** is the package-match authority: one `all.zip` per ecosystem, one
//!   OSV JSON record per member. Parsed range-native by `modgunn::osv`.
//! * **NVD** (CVE JSON 2.0, usually gzip) enriches by CVE id or alias: CVSS
//!   vector, CWE ids, a stronger severity. It never changes which versions match.
//! * **CISA KEV** flags known-exploited CVEs; modgunn's policy escalates a KEV
//!   hit to Critical.
//!
//! The parsers are modgunn's. This module only reads bytes, bounds them, and
//! folds them, so the same feed bytes always give the same set.

use std::collections::{HashMap, HashSet};
use std::io::Read;
use std::path::Path;

use anyhow::{bail, Context, Result};
use chrono::{DateTime, Utc};
use holger_update_bundle::digest::sha256_hex;
use modgunn::scan::Advisory;

use crate::set::{AdvisorySet, SourceRecord};

/// The OSV bucket that publishes the per-ecosystem dumps.
pub const OSV_BUCKET: &str = "https://osv-vulnerabilities.storage.googleapis.com";
/// NVD's CVE JSON 2.0 feed directory.
pub const NVD_FEEDS: &str = "https://nvd.nist.gov/feeds/json/cve/2.0";
/// CISA's Known Exploited Vulnerabilities catalog.
pub const KEV_URL: &str =
    "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json";

/// The ecosystems a default build fetches: what Holger serves, and the
/// distributions of the images it pulls through (Chainguard's are Wolfi and
/// Chainguard; SANDBOX-SCAN's image CVE scanner matches their apk packages).
pub const DEFAULT_ECOSYSTEMS: &[&str] = &[
    "PyPI",
    "crates.io",
    "npm",
    "Maven",
    "Go",
    "NuGet",
    "RubyGems",
    "Packagist",
    "Wolfi",
    "Chainguard",
];

/// Bound on one OSV record inside a zip (a real one is a few KiB).
const MAX_OSV_RECORD: usize = 16 * 1024 * 1024;
/// Bound on a decompressed NVD feed (a year is about 200 MiB unpacked).
const MAX_NVD_BYTES: u64 = 2 * 1024 * 1024 * 1024;

/// What a feed is.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum FeedKind {
    /// An OSV dump for one ecosystem: `all.zip`, or a single / batch JSON doc.
    Osv { ecosystem: String },
    /// An NVD CVE JSON 2.0 feed, plain or gzip. `name` is `2024`, `modified`, …
    Nvd { name: String },
    /// The CISA KEV catalog.
    Kev,
}

impl FeedKind {
    pub fn label(&self) -> String {
        match self {
            FeedKind::Osv { ecosystem } => format!("osv:{ecosystem}"),
            FeedKind::Nvd { name } => format!("nvd:{name}"),
            FeedKind::Kev => "kev".into(),
        }
    }
}

/// A feed and where to get it.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct FeedSpec {
    pub kind: FeedKind,
    pub uri: String,
}

impl FeedSpec {
    pub fn osv(ecosystem: &str) -> Self {
        Self {
            kind: FeedKind::Osv {
                ecosystem: ecosystem.to_string(),
            },
            uri: format!("{OSV_BUCKET}/{ecosystem}/all.zip"),
        }
    }

    pub fn nvd(name: &str) -> Self {
        Self {
            kind: FeedKind::Nvd {
                name: name.to_string(),
            },
            uri: format!("{NVD_FEEDS}/nvdcve-2.0-{name}.json.gz"),
        }
    }

    pub fn kev() -> Self {
        Self {
            kind: FeedKind::Kev,
            uri: KEV_URL.to_string(),
        }
    }

    /// The default online feed list: OSV for `ecosystems`, NVD for every year
    /// from 2002 to `this_year` plus `modified`, and KEV.
    pub fn default_set(ecosystems: &[String], this_year: i32) -> Vec<FeedSpec> {
        let mut out: Vec<FeedSpec> = ecosystems.iter().map(|e| FeedSpec::osv(e)).collect();
        for y in 2002..=this_year {
            out.push(FeedSpec::nvd(&y.to_string()));
        }
        out.push(FeedSpec::nvd("modified"));
        out.push(FeedSpec::kev());
        out
    }

    /// The feeds found in a directory of already-downloaded files, by name:
    /// `osv-<Ecosystem>.zip` / `osv-<Ecosystem>.json`, `nvd-<name>.json[.gz]`,
    /// `kev.json`. Sorted, so the set is the same however the disk lists it.
    pub fn from_dir(dir: &Path) -> Result<Vec<FeedSpec>> {
        let mut names: Vec<String> = std::fs::read_dir(dir)
            .with_context(|| format!("read feeds dir {}", dir.display()))?
            .filter_map(|e| e.ok())
            .filter(|e| e.path().is_file())
            .map(|e| e.file_name().to_string_lossy().into_owned())
            .collect();
        names.sort();
        let mut out = Vec::new();
        for n in names {
            let uri = format!("file:{}", dir.join(&n).display());
            let stem = n.strip_suffix(".gz").unwrap_or(&n);
            let stem = stem
                .strip_suffix(".zip")
                .or_else(|| stem.strip_suffix(".json"))
                .unwrap_or(stem);
            let kind = if let Some(eco) = stem.strip_prefix("osv-") {
                FeedKind::Osv {
                    ecosystem: eco.to_string(),
                }
            } else if let Some(name) = stem.strip_prefix("nvd-") {
                FeedKind::Nvd {
                    name: name.to_string(),
                }
            } else if stem == "kev" {
                FeedKind::Kev
            } else {
                continue;
            };
            out.push(FeedSpec { kind, uri });
        }
        if out.is_empty() {
            bail!(
                "no feed files in {} (expected osv-<Ecosystem>.zip|.json, nvd-<name>.json[.gz], kev.json)",
                dir.display()
            );
        }
        Ok(out)
    }
}

/// A feed's bytes, as fetched.
#[derive(Debug, Clone)]
pub struct RawFeed {
    pub spec: FeedSpec,
    pub bytes: Vec<u8>,
    pub fetched_at: DateTime<Utc>,
}

impl RawFeed {
    /// Read a `file:` feed from disk.
    pub fn read_file(spec: FeedSpec) -> Result<Self> {
        let path = spec
            .uri
            .strip_prefix("file:")
            .ok_or_else(|| anyhow::anyhow!("{} is not a file: feed", spec.uri))?;
        let bytes = std::fs::read(path).with_context(|| format!("read feed {path}"))?;
        Ok(Self {
            spec,
            bytes,
            fetched_at: Utc::now(),
        })
    }
}

fn is_zip(b: &[u8]) -> bool {
    b.starts_with(b"PK\x03\x04") || b.starts_with(b"PK\x05\x06")
}

fn is_gzip(b: &[u8]) -> bool {
    b.starts_with(&[0x1f, 0x8b])
}

fn gunzip_bounded(bytes: &[u8]) -> Result<Vec<u8>> {
    let mut out = Vec::new();
    flate2::read::MultiGzDecoder::new(bytes)
        .take(MAX_NVD_BYTES + 1)
        .read_to_end(&mut out)
        .context("gunzip")?;
    if out.len() as u64 > MAX_NVD_BYTES {
        bail!("decompressed feed exceeds {MAX_NVD_BYTES} bytes");
    }
    Ok(out)
}

/// Every advisory in one OSV feed (zip dump or JSON document).
pub fn parse_osv_feed(bytes: &[u8]) -> Result<Vec<Advisory>> {
    if !is_zip(bytes) {
        let text = std::str::from_utf8(bytes).context("OSV feed is not UTF-8")?;
        return modgunn::osv::parse_doc(text).context("OSV feed is not an OSV document");
    }
    let mut out = Vec::new();
    for m in zip_entry::members(bytes).context("OSV dump is not a readable zip")? {
        if !m.name.ends_with(".json") {
            continue;
        }
        let body = zip_entry::read_member_capped(bytes, &m, MAX_OSV_RECORD)
            .with_context(|| format!("OSV dump member {}", m.name))?;
        let text = std::str::from_utf8(&body)
            .with_context(|| format!("OSV dump member {} is not UTF-8", m.name))?;
        let advisories = modgunn::osv::parse_doc(text)
            .with_context(|| format!("OSV dump member {} is not an OSV record", m.name))?;
        out.extend(advisories);
    }
    Ok(out)
}

/// Fold every feed into one set. OSV makes the advisories, NVD and KEV enrich
/// them. A feed that does not parse fails the whole build: a set that silently
/// lacks an ecosystem would scan clean and say nothing.
pub fn normalize(feeds: &[RawFeed], generated_at: DateTime<Utc>) -> Result<AdvisorySet> {
    let mut advisories: Vec<Advisory> = Vec::new();
    let mut nvd = Vec::new();
    let mut kev: HashSet<String> = HashSet::new();
    let mut sources = Vec::new();
    for f in feeds {
        let label = f.spec.kind.label();
        let records = match &f.spec.kind {
            FeedKind::Osv { .. } => {
                let a = parse_osv_feed(&f.bytes).with_context(|| format!("feed {label}"))?;
                let n = a.len();
                advisories.extend(a);
                n
            }
            FeedKind::Nvd { .. } => {
                let plain = if is_gzip(&f.bytes) {
                    gunzip_bounded(&f.bytes)?
                } else {
                    f.bytes.clone()
                };
                let text = std::str::from_utf8(&plain)
                    .with_context(|| format!("feed {label} is not UTF-8"))?;
                let entries =
                    modgunn::nvd::parse_nvd(text).with_context(|| format!("feed {label}"))?;
                let n = entries.len();
                nvd.extend(entries);
                n
            }
            FeedKind::Kev => {
                let text = std::str::from_utf8(&f.bytes)
                    .with_context(|| format!("feed {label} is not UTF-8"))?;
                let ids = modgunn::feeds::parse_kev_json(text)
                    .with_context(|| format!("feed {label}"))?;
                let n = ids.len();
                kev.extend(ids);
                n
            }
        };
        sources.push(SourceRecord {
            feed: label,
            uri: f.spec.uri.clone(),
            sha256: sha256_hex(&f.bytes),
            bytes: f.bytes.len() as u64,
            fetched_at: f.fetched_at,
            records: records as u64,
        });
    }
    if !feeds
        .iter()
        .any(|f| matches!(f.spec.kind, FeedKind::Osv { .. }))
    {
        bail!("no OSV feed: NVD and KEV only enrich, so a set without OSV matches nothing");
    }
    let mut advisories = modgunn::scan::merge_by_alias(advisories);
    modgunn::nvd::enrich_with_nvd(&mut advisories, &nvd);
    modgunn::feeds::enrich(&mut advisories, &HashMap::new(), &kev);
    Ok(AdvisorySet::from_advisories(
        generated_at,
        sources,
        &advisories,
    ))
}