holger_intel/
feeds.rs

1//! The three upstream feeds and their normalization into one [`AdvisorySet`].
2//!
3//! * **OSV** is the package-match authority: one `all.zip` per ecosystem, one
4//!   OSV JSON record per member. Parsed range-native by `modgunn::osv`.
5//! * **NVD** (CVE JSON 2.0, usually gzip) enriches by CVE id or alias: CVSS
6//!   vector, CWE ids, a stronger severity. It never changes which versions match.
7//! * **CISA KEV** flags known-exploited CVEs; modgunn's policy escalates a KEV
8//!   hit to Critical.
9//!
10//! The parsers are modgunn's. This module only reads bytes, bounds them, and
11//! folds them, so the same feed bytes always give the same set.
12
13use std::collections::{HashMap, HashSet};
14use std::io::Read;
15use std::path::Path;
16
17use anyhow::{bail, Context, Result};
18use chrono::{DateTime, Utc};
19use holger_update_bundle::digest::sha256_hex;
20use modgunn::scan::Advisory;
21
22use crate::set::{AdvisorySet, SourceRecord};
23
24/// The OSV bucket that publishes the per-ecosystem dumps.
25pub const OSV_BUCKET: &str = "https://osv-vulnerabilities.storage.googleapis.com";
26/// NVD's CVE JSON 2.0 feed directory.
27pub const NVD_FEEDS: &str = "https://nvd.nist.gov/feeds/json/cve/2.0";
28/// CISA's Known Exploited Vulnerabilities catalog.
29pub const KEV_URL: &str =
30    "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json";
31
32/// The ecosystems a default build fetches: what Holger serves, and the
33/// distributions of the images it pulls through (Chainguard's are Wolfi and
34/// Chainguard; SANDBOX-SCAN's image CVE scanner matches their apk packages).
35pub const DEFAULT_ECOSYSTEMS: &[&str] =
36    &["PyPI", "crates.io", "npm", "Maven", "Go", "NuGet", "RubyGems", "Packagist", "Wolfi", "Chainguard"];
37
38/// Bound on one OSV record inside a zip (a real one is a few KiB).
39const MAX_OSV_RECORD: usize = 16 * 1024 * 1024;
40/// Bound on a decompressed NVD feed (a year is about 200 MiB unpacked).
41const MAX_NVD_BYTES: u64 = 2 * 1024 * 1024 * 1024;
42
43/// What a feed is.
44#[derive(Debug, Clone, PartialEq, Eq)]
45pub enum FeedKind {
46    /// An OSV dump for one ecosystem: `all.zip`, or a single / batch JSON doc.
47    Osv { ecosystem: String },
48    /// An NVD CVE JSON 2.0 feed, plain or gzip. `name` is `2024`, `modified`, …
49    Nvd { name: String },
50    /// The CISA KEV catalog.
51    Kev,
52}
53
54impl FeedKind {
55    pub fn label(&self) -> String {
56        match self {
57            FeedKind::Osv { ecosystem } => format!("osv:{ecosystem}"),
58            FeedKind::Nvd { name } => format!("nvd:{name}"),
59            FeedKind::Kev => "kev".into(),
60        }
61    }
62}
63
64/// A feed and where to get it.
65#[derive(Debug, Clone, PartialEq, Eq)]
66pub struct FeedSpec {
67    pub kind: FeedKind,
68    pub uri: String,
69}
70
71impl FeedSpec {
72    pub fn osv(ecosystem: &str) -> Self {
73        Self {
74            kind: FeedKind::Osv { ecosystem: ecosystem.to_string() },
75            uri: format!("{OSV_BUCKET}/{ecosystem}/all.zip"),
76        }
77    }
78
79    pub fn nvd(name: &str) -> Self {
80        Self {
81            kind: FeedKind::Nvd { name: name.to_string() },
82            uri: format!("{NVD_FEEDS}/nvdcve-2.0-{name}.json.gz"),
83        }
84    }
85
86    pub fn kev() -> Self {
87        Self { kind: FeedKind::Kev, uri: KEV_URL.to_string() }
88    }
89
90    /// The default online feed list: OSV for `ecosystems`, NVD for every year
91    /// from 2002 to `this_year` plus `modified`, and KEV.
92    pub fn default_set(ecosystems: &[String], this_year: i32) -> Vec<FeedSpec> {
93        let mut out: Vec<FeedSpec> = ecosystems.iter().map(|e| FeedSpec::osv(e)).collect();
94        for y in 2002..=this_year {
95            out.push(FeedSpec::nvd(&y.to_string()));
96        }
97        out.push(FeedSpec::nvd("modified"));
98        out.push(FeedSpec::kev());
99        out
100    }
101
102    /// The feeds found in a directory of already-downloaded files, by name:
103    /// `osv-<Ecosystem>.zip` / `osv-<Ecosystem>.json`, `nvd-<name>.json[.gz]`,
104    /// `kev.json`. Sorted, so the set is the same however the disk lists it.
105    pub fn from_dir(dir: &Path) -> Result<Vec<FeedSpec>> {
106        let mut names: Vec<String> = std::fs::read_dir(dir)
107            .with_context(|| format!("read feeds dir {}", dir.display()))?
108            .filter_map(|e| e.ok())
109            .filter(|e| e.path().is_file())
110            .map(|e| e.file_name().to_string_lossy().into_owned())
111            .collect();
112        names.sort();
113        let mut out = Vec::new();
114        for n in names {
115            let uri = format!("file:{}", dir.join(&n).display());
116            let stem = n.strip_suffix(".gz").unwrap_or(&n);
117            let stem = stem
118                .strip_suffix(".zip")
119                .or_else(|| stem.strip_suffix(".json"))
120                .unwrap_or(stem);
121            let kind = if let Some(eco) = stem.strip_prefix("osv-") {
122                FeedKind::Osv { ecosystem: eco.to_string() }
123            } else if let Some(name) = stem.strip_prefix("nvd-") {
124                FeedKind::Nvd { name: name.to_string() }
125            } else if stem == "kev" {
126                FeedKind::Kev
127            } else {
128                continue;
129            };
130            out.push(FeedSpec { kind, uri });
131        }
132        if out.is_empty() {
133            bail!(
134                "no feed files in {} (expected osv-<Ecosystem>.zip|.json, nvd-<name>.json[.gz], kev.json)",
135                dir.display()
136            );
137        }
138        Ok(out)
139    }
140}
141
142/// A feed's bytes, as fetched.
143#[derive(Debug, Clone)]
144pub struct RawFeed {
145    pub spec: FeedSpec,
146    pub bytes: Vec<u8>,
147    pub fetched_at: DateTime<Utc>,
148}
149
150impl RawFeed {
151    /// Read a `file:` feed from disk.
152    pub fn read_file(spec: FeedSpec) -> Result<Self> {
153        let path = spec
154            .uri
155            .strip_prefix("file:")
156            .ok_or_else(|| anyhow::anyhow!("{} is not a file: feed", spec.uri))?;
157        let bytes = std::fs::read(path).with_context(|| format!("read feed {path}"))?;
158        Ok(Self { spec, bytes, fetched_at: Utc::now() })
159    }
160}
161
162fn is_zip(b: &[u8]) -> bool {
163    b.starts_with(b"PK\x03\x04") || b.starts_with(b"PK\x05\x06")
164}
165
166fn is_gzip(b: &[u8]) -> bool {
167    b.starts_with(&[0x1f, 0x8b])
168}
169
170fn gunzip_bounded(bytes: &[u8]) -> Result<Vec<u8>> {
171    let mut out = Vec::new();
172    flate2::read::MultiGzDecoder::new(bytes)
173        .take(MAX_NVD_BYTES + 1)
174        .read_to_end(&mut out)
175        .context("gunzip")?;
176    if out.len() as u64 > MAX_NVD_BYTES {
177        bail!("decompressed feed exceeds {MAX_NVD_BYTES} bytes");
178    }
179    Ok(out)
180}
181
182/// Every advisory in one OSV feed (zip dump or JSON document).
183pub fn parse_osv_feed(bytes: &[u8]) -> Result<Vec<Advisory>> {
184    if !is_zip(bytes) {
185        let text = std::str::from_utf8(bytes).context("OSV feed is not UTF-8")?;
186        return modgunn::osv::parse_doc(text).context("OSV feed is not an OSV document");
187    }
188    let mut out = Vec::new();
189    for m in zip_entry::members(bytes).context("OSV dump is not a readable zip")? {
190        if !m.name.ends_with(".json") {
191            continue;
192        }
193        let body = zip_entry::read_member_capped(bytes, &m, MAX_OSV_RECORD)
194            .with_context(|| format!("OSV dump member {}", m.name))?;
195        let text = std::str::from_utf8(&body)
196            .with_context(|| format!("OSV dump member {} is not UTF-8", m.name))?;
197        let advisories = modgunn::osv::parse_doc(text)
198            .with_context(|| format!("OSV dump member {} is not an OSV record", m.name))?;
199        out.extend(advisories);
200    }
201    Ok(out)
202}
203
204/// Fold every feed into one set. OSV makes the advisories, NVD and KEV enrich
205/// them. A feed that does not parse fails the whole build: a set that silently
206/// lacks an ecosystem would scan clean and say nothing.
207pub fn normalize(feeds: &[RawFeed], generated_at: DateTime<Utc>) -> Result<AdvisorySet> {
208    let mut advisories: Vec<Advisory> = Vec::new();
209    let mut nvd = Vec::new();
210    let mut kev: HashSet<String> = HashSet::new();
211    let mut sources = Vec::new();
212    for f in feeds {
213        let label = f.spec.kind.label();
214        let records = match &f.spec.kind {
215            FeedKind::Osv { .. } => {
216                let a = parse_osv_feed(&f.bytes).with_context(|| format!("feed {label}"))?;
217                let n = a.len();
218                advisories.extend(a);
219                n
220            }
221            FeedKind::Nvd { .. } => {
222                let plain = if is_gzip(&f.bytes) { gunzip_bounded(&f.bytes)? } else { f.bytes.clone() };
223                let text = std::str::from_utf8(&plain).with_context(|| format!("feed {label} is not UTF-8"))?;
224                let entries = modgunn::nvd::parse_nvd(text).with_context(|| format!("feed {label}"))?;
225                let n = entries.len();
226                nvd.extend(entries);
227                n
228            }
229            FeedKind::Kev => {
230                let text = std::str::from_utf8(&f.bytes).with_context(|| format!("feed {label} is not UTF-8"))?;
231                let ids = modgunn::feeds::parse_kev_json(text).with_context(|| format!("feed {label}"))?;
232                let n = ids.len();
233                kev.extend(ids);
234                n
235            }
236        };
237        sources.push(SourceRecord {
238            feed: label,
239            uri: f.spec.uri.clone(),
240            sha256: sha256_hex(&f.bytes),
241            bytes: f.bytes.len() as u64,
242            fetched_at: f.fetched_at,
243            records: records as u64,
244        });
245    }
246    if !feeds.iter().any(|f| matches!(f.spec.kind, FeedKind::Osv { .. })) {
247        bail!("no OSV feed: NVD and KEV only enrich, so a set without OSV matches nothing");
248    }
249    let mut advisories = modgunn::scan::merge_by_alias(advisories);
250    modgunn::nvd::enrich_with_nvd(&mut advisories, &nvd);
251    modgunn::feeds::enrich(&mut advisories, &HashMap::new(), &kev);
252    Ok(AdvisorySet::from_advisories(generated_at, sources, &advisories))
253}
holger_intel/feeds.rs

holger_intel/
feeds.rs
