holger-intel 3.1.2

//! The three upstream feeds and their normalization into one [`AdvisorySet`].
//!
//! * **OSV** is the package-match authority: one `all.zip` per ecosystem, one
//!   OSV JSON record per member. Parsed range-native by `modgunn::osv`.
//! * **NVD** (CVE JSON 2.0, usually gzip) enriches by CVE id or alias: CVSS
//!   vector, CWE ids, a stronger severity. It never changes which versions match.
//! * **CISA KEV** flags known-exploited CVEs; modgunn's policy escalates a KEV
//!   hit to Critical.
//!
//! The parsers are modgunn's. This module only reads bytes, bounds them, and
//! folds them, so the same feed bytes always give the same set.

use std::collections::{HashMap, HashSet};
use std::io::Read;
use std::path::Path;

use anyhow::{bail, Context, Result};
use chrono::{DateTime, Utc};
use holger_update_bundle::digest::sha256_hex;
use modgunn::scan::Advisory;

use crate::set::{AdvisorySet, SourceRecord};

/// The OSV bucket that publishes the per-ecosystem dumps.
pub const OSV_BUCKET: &str = "https://osv-vulnerabilities.storage.googleapis.com";
/// NVD's CVE JSON 2.0 feed directory.
pub const NVD_FEEDS: &str = "https://nvd.nist.gov/feeds/json/cve/2.0";
/// CISA's Known Exploited Vulnerabilities catalog.
pub const KEV_URL: &str =
    "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json";

/// The ecosystems a default build fetches: what Holger serves, and the
/// distributions of the images it pulls through (Chainguard's are Wolfi and
/// Chainguard; SANDBOX-SCAN's image CVE scanner matches their apk packages).
pub const DEFAULT_ECOSYSTEMS: &[&str] = &[
    "PyPI",
    "crates.io",
    "npm",
    "Maven",
    "Go",
    "NuGet",
    "RubyGems",
    "Packagist",
    "Wolfi",
    "Chainguard",
];

/// Bound on one OSV record inside a zip (a real one is a few KiB).
const MAX_OSV_RECORD: usize = 16 * 1024 * 1024;
/// The most records one OSV dump may hold: `npm/all.zip` held 230 994 on 2026-10-31, past the
/// 200 000 of a package's zip (`zip_entry::MAX_MEMBERS`); twenty times that leaves room to grow
/// and still bounds a crafted central directory (lane HOL-FIELD-TEST).
const MAX_OSV_MEMBERS: usize = 4_000_000;
/// Bound on a decompressed NVD feed (a year is about 200 MiB unpacked).
const MAX_NVD_BYTES: u64 = 2 * 1024 * 1024 * 1024;

/// What a feed is.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum FeedKind {
    /// An OSV dump for one ecosystem: `all.zip`, or a single / batch JSON doc.
    Osv { ecosystem: String },
    /// An NVD CVE JSON 2.0 feed, plain or gzip. `name` is `2024`, `modified`, …
    Nvd { name: String },
    /// The CISA KEV catalog.
    Kev,
}

impl FeedKind {
    pub fn label(&self) -> String {
        match self {
            FeedKind::Osv { ecosystem } => format!("osv:{ecosystem}"),
            FeedKind::Nvd { name } => format!("nvd:{name}"),
            FeedKind::Kev => "kev".into(),
        }
    }
}

/// A feed and where to get it.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct FeedSpec {
    pub kind: FeedKind,
    pub uri: String,
}

/// The cap on one fetched OSV dump (`all.zip`): Chainguard's was 1 116 562 155 bytes on
/// 2026-11-03 (HOL-DOGFOOD-2), past the old single 1 GiB limit; room to grow.
pub const MAX_OSV_DUMP_BYTES: u64 = 4 * 1024 * 1024 * 1024;
/// The cap on one fetched NVD or KEV feed (a year of NVD gzipped is tens of MiB).
pub const MAX_FEED_BYTES: u64 = 1024 * 1024 * 1024;

impl FeedSpec {
    /// The most bytes one fetch of this feed may bring (HOL-DOGFOOD-2): an OSV dump
    /// [`MAX_OSV_DUMP_BYTES`], NVD and KEV [`MAX_FEED_BYTES`].
    pub fn max_bytes(&self) -> u64 {
        match self.kind {
            FeedKind::Osv { .. } => MAX_OSV_DUMP_BYTES,
            FeedKind::Nvd { .. } | FeedKind::Kev => MAX_FEED_BYTES,
        }
    }

    pub fn osv(ecosystem: &str) -> Self {
        Self {
            kind: FeedKind::Osv {
                ecosystem: ecosystem.to_string(),
            },
            uri: format!("{OSV_BUCKET}/{ecosystem}/all.zip"),
        }
    }

    pub fn nvd(name: &str) -> Self {
        Self {
            kind: FeedKind::Nvd {
                name: name.to_string(),
            },
            uri: format!("{NVD_FEEDS}/nvdcve-2.0-{name}.json.gz"),
        }
    }

    pub fn kev() -> Self {
        Self {
            kind: FeedKind::Kev,
            uri: KEV_URL.to_string(),
        }
    }

    /// The default online feed list: OSV for `ecosystems`, NVD for every year
    /// from 2002 to `this_year` plus `modified`, and KEV.
    pub fn default_set(ecosystems: &[String], this_year: i32) -> Vec<FeedSpec> {
        let mut out: Vec<FeedSpec> = ecosystems.iter().map(|e| FeedSpec::osv(e)).collect();
        for y in 2002..=this_year {
            out.push(FeedSpec::nvd(&y.to_string()));
        }
        out.push(FeedSpec::nvd("modified"));
        out.push(FeedSpec::kev());
        out
    }

    /// The feeds found in a directory of already-downloaded files, by name:
    /// `osv-<Ecosystem>.zip` / `osv-<Ecosystem>.json`, `nvd-<name>.json[.gz]`,
    /// `kev.json`. Sorted, so the set is the same however the disk lists it.
    pub fn from_dir(dir: &Path) -> Result<Vec<FeedSpec>> {
        let mut names: Vec<String> = std::fs::read_dir(dir)
            .with_context(|| format!("read feeds dir {}", dir.display()))?
            .filter_map(|e| e.ok())
            .filter(|e| e.path().is_file())
            .map(|e| e.file_name().to_string_lossy().into_owned())
            .collect();
        names.sort();
        let mut out = Vec::new();
        for n in names {
            let uri = format!("file:{}", dir.join(&n).display());
            let stem = n.strip_suffix(".gz").unwrap_or(&n);
            let stem = stem
                .strip_suffix(".zip")
                .or_else(|| stem.strip_suffix(".json"))
                .unwrap_or(stem);
            let kind = if let Some(eco) = stem.strip_prefix("osv-") {
                FeedKind::Osv {
                    ecosystem: eco.to_string(),
                }
            } else if let Some(name) = stem.strip_prefix("nvd-") {
                FeedKind::Nvd {
                    name: name.to_string(),
                }
            } else if stem == "kev" {
                FeedKind::Kev
            } else {
                continue;
            };
            out.push(FeedSpec { kind, uri });
        }
        if out.is_empty() {
            bail!(
                "no feed files in {} (expected osv-<Ecosystem>.zip|.json, nvd-<name>.json[.gz], kev.json)",
                dir.display()
            );
        }
        Ok(out)
    }
}

/// A feed's bytes, as fetched.
#[derive(Debug, Clone)]
pub struct RawFeed {
    pub spec: FeedSpec,
    pub bytes: Vec<u8>,
    pub fetched_at: DateTime<Utc>,
}

impl RawFeed {
    /// Read a `file:` feed from disk.
    pub fn read_file(spec: FeedSpec) -> Result<Self> {
        let path = spec
            .uri
            .strip_prefix("file:")
            .ok_or_else(|| anyhow::anyhow!("{} is not a file: feed", spec.uri))?;
        let bytes = std::fs::read(path).with_context(|| format!("read feed {path}"))?;
        Ok(Self {
            spec,
            bytes,
            fetched_at: Utc::now(),
        })
    }
}

fn is_zip(b: &[u8]) -> bool {
    b.starts_with(b"PK\x03\x04") || b.starts_with(b"PK\x05\x06")
}

fn is_gzip(b: &[u8]) -> bool {
    b.starts_with(&[0x1f, 0x8b])
}

fn gunzip_bounded(bytes: &[u8]) -> Result<Vec<u8>> {
    let mut out = Vec::new();
    flate2::read::MultiGzDecoder::new(bytes)
        .take(MAX_NVD_BYTES + 1)
        .read_to_end(&mut out)
        .context("gunzip")?;
    if out.len() as u64 > MAX_NVD_BYTES {
        bail!("decompressed feed exceeds {MAX_NVD_BYTES} bytes");
    }
    Ok(out)
}

/// **A Debian release's package name in the advisory set** (lane HOL-FIELD-2 P17):
/// `debian-<n>/<source>`, what server-lib's deb scan asks for (`scanner::osv::deb_names`).
pub const DEBIAN_RELEASE_PREFIX: &str = "debian-";

/// `Debian:<n>` → the affected entry's package is named `debian-<n>/<name>`.
fn debian_by_release(affected: &mut serde_json::Value) {
    let Some(pkg) = affected.get_mut("package") else {
        return;
    };
    let Some(release) = pkg
        .get("ecosystem")
        .and_then(|e| e.as_str())
        .and_then(|e| e.strip_prefix("Debian:"))
        .map(|r| r.trim().to_string())
    else {
        return;
    };
    if release.is_empty() {
        return;
    }
    if let Some(name) = pkg.get("name").and_then(|n| n.as_str()).map(str::to_string) {
        pkg["name"] = serde_json::Value::String(format!("{DEBIAN_RELEASE_PREFIX}{release}/{name}"));
    }
}

/// Every advisory in one OSV feed (zip dump or JSON document).
pub fn parse_osv_feed(bytes: &[u8]) -> Result<Vec<Advisory>> {
    if !is_zip(bytes) {
        let text = std::str::from_utf8(bytes).context("OSV feed is not UTF-8")?;
        return parse_osv_doc(text).context("OSV feed is not an OSV document");
    }
    let mut out = Vec::new();
    for m in zip_entry::members_up_to(bytes, MAX_OSV_MEMBERS)
        .context("OSV dump is not a readable zip")?
    {
        if !m.name.ends_with(".json") {
            continue;
        }
        let body = zip_entry::read_member_capped(bytes, &m, MAX_OSV_RECORD)
            .with_context(|| format!("OSV dump member {}", m.name))?;
        let text = std::str::from_utf8(&body)
            .with_context(|| format!("OSV dump member {} is not UTF-8", m.name))?;
        let advisories = parse_osv_doc(text)
            .with_context(|| format!("OSV dump member {} is not an OSV record", m.name))?;
        out.extend(advisories);
    }
    Ok(out)
}

/// One OSV document (a record or an array of them). The OSV schema makes `affected[].package`
/// optional: an entry without one names git ranges only (Debian's dump has them, e.g.
/// CVE-2024-38998) and can match no package, so it is dropped and the record kept; anything
/// else that does not parse is still refused (lane HOL-FIELD-TEST).
fn parse_osv_doc(text: &str) -> Result<Vec<Advisory>, serde_json::Error> {
    // P17 (lane HOL-FIELD-2): Debian's records give each release its own range, and the
    // advisory set has one `deb` ecosystem; each release's package is named
    // `debian-<n>/<source>` so a bookworm package is never judged by trixie's range.
    if text.contains("\"Debian:") {
        if let Ok(mut doc) = serde_json::from_str::<serde_json::Value>(text) {
            let mut each = |rec: &mut serde_json::Value| {
                if let Some(affected) = rec.get_mut("affected").and_then(|a| a.as_array_mut()) {
                    affected.retain(|a| a.get("package").is_some());
                    for a in affected.iter_mut() {
                        debian_by_release(a);
                    }
                }
            };
            match &mut doc {
                serde_json::Value::Array(recs) => recs.iter_mut().for_each(&mut each),
                rec => each(rec),
            }
            return modgunn::osv::parse_doc(&doc.to_string());
        }
    }
    let first = match modgunn::osv::parse_doc(text) {
        Ok(v) => return Ok(v),
        Err(e) => e,
    };
    let Ok(mut doc) = serde_json::from_str::<serde_json::Value>(text) else {
        return Err(first);
    };
    let mut dropped = false;
    let mut drop_unpackaged = |rec: &mut serde_json::Value| {
        if let Some(affected) = rec.get_mut("affected").and_then(|a| a.as_array_mut()) {
            let before = affected.len();
            affected.retain(|a| a.get("package").is_some());
            dropped |= affected.len() != before;
        }
    };
    match &mut doc {
        serde_json::Value::Array(recs) => recs.iter_mut().for_each(&mut drop_unpackaged),
        rec => drop_unpackaged(rec),
    }
    if !dropped {
        return Err(first);
    }
    modgunn::osv::parse_doc(&doc.to_string())
}

/// `modgunn::scan::merge_by_alias`, one package at a time. It merges only records of the same
/// ecosystem and package, and compares each record with every group so far: over the real feeds
/// (about 300 000 records) that is quadratic and never ended (lane HOL-FIELD-TEST). Bucketed by
/// (ecosystem, package) it does the same merges, and the order inside a bucket is the feeds'
/// order, which is all [`AdvisorySet::from_advisories`]'s stable sort keeps.
fn merge_by_alias_per_package(advisories: Vec<Advisory>) -> Vec<Advisory> {
    let mut buckets: HashMap<(String, String), Vec<Advisory>> = HashMap::new();
    let mut order: Vec<(String, String)> = Vec::new();
    for a in advisories {
        let key = (a.ecosystem.clone(), a.package.clone());
        let bucket = buckets.entry(key.clone()).or_default();
        if bucket.is_empty() {
            order.push(key);
        }
        bucket.push(a);
    }
    order
        .into_iter()
        .flat_map(|k| modgunn::scan::merge_by_alias(buckets.remove(&k).unwrap_or_default()))
        .collect()
}

/// Fold every feed into one set. OSV makes the advisories, NVD and KEV enrich
/// them. A feed that does not parse fails the whole build: a set that silently
/// lacks an ecosystem would scan clean and say nothing.
pub fn normalize(feeds: &[RawFeed], generated_at: DateTime<Utc>) -> Result<AdvisorySet> {
    let mut advisories: Vec<Advisory> = Vec::new();
    let mut nvd = Vec::new();
    let mut kev: HashSet<String> = HashSet::new();
    let mut sources = Vec::new();
    for f in feeds {
        let label = f.spec.kind.label();
        let records = match &f.spec.kind {
            FeedKind::Osv { .. } => {
                let a = parse_osv_feed(&f.bytes).with_context(|| format!("feed {label}"))?;
                let n = a.len();
                advisories.extend(a);
                n
            }
            FeedKind::Nvd { .. } => {
                let plain = if is_gzip(&f.bytes) {
                    gunzip_bounded(&f.bytes)?
                } else {
                    f.bytes.clone()
                };
                let text = std::str::from_utf8(&plain)
                    .with_context(|| format!("feed {label} is not UTF-8"))?;
                let entries =
                    modgunn::nvd::parse_nvd(text).with_context(|| format!("feed {label}"))?;
                let n = entries.len();
                nvd.extend(entries);
                n
            }
            FeedKind::Kev => {
                let text = std::str::from_utf8(&f.bytes)
                    .with_context(|| format!("feed {label} is not UTF-8"))?;
                let ids = modgunn::feeds::parse_kev_json(text)
                    .with_context(|| format!("feed {label}"))?;
                let n = ids.len();
                kev.extend(ids);
                n
            }
        };
        sources.push(SourceRecord {
            feed: label,
            uri: f.spec.uri.clone(),
            sha256: sha256_hex(&f.bytes),
            bytes: f.bytes.len() as u64,
            fetched_at: f.fetched_at,
            records: records as u64,
        });
    }
    if !feeds
        .iter()
        .any(|f| matches!(f.spec.kind, FeedKind::Osv { .. }))
    {
        bail!("no OSV feed: NVD and KEV only enrich, so a set without OSV matches nothing");
    }
    let mut advisories = merge_by_alias_per_package(advisories);
    modgunn::nvd::enrich_with_nvd(&mut advisories, &nvd);
    modgunn::feeds::enrich(&mut advisories, &HashMap::new(), &kev);
    Ok(AdvisorySet::from_advisories(
        generated_at,
        sources,
        &advisories,
    ))
}