holger_intel/
feeds.rs

1//! The three upstream feeds and their normalization into one [`AdvisorySet`].
2//!
3//! * **OSV** is the package-match authority: one `all.zip` per ecosystem, one
4//!   OSV JSON record per member. Parsed range-native by `modgunn::osv`.
5//! * **NVD** (CVE JSON 2.0, usually gzip) enriches by CVE id or alias: CVSS
6//!   vector, CWE ids, a stronger severity. It never changes which versions match.
7//! * **CISA KEV** flags known-exploited CVEs; modgunn's policy escalates a KEV
8//!   hit to Critical.
9//!
10//! The parsers are modgunn's. This module only reads bytes, bounds them, and
11//! folds them, so the same feed bytes always give the same set.
12
13use std::collections::{HashMap, HashSet};
14use std::io::Read;
15use std::path::Path;
16
17use anyhow::{bail, Context, Result};
18use chrono::{DateTime, Utc};
19use holger_update_bundle::digest::sha256_hex;
20use modgunn::scan::Advisory;
21
22use crate::set::{AdvisorySet, SourceRecord};
23
24/// The OSV bucket that publishes the per-ecosystem dumps.
25pub const OSV_BUCKET: &str = "https://osv-vulnerabilities.storage.googleapis.com";
26/// NVD's CVE JSON 2.0 feed directory.
27pub const NVD_FEEDS: &str = "https://nvd.nist.gov/feeds/json/cve/2.0";
28/// CISA's Known Exploited Vulnerabilities catalog.
29pub const KEV_URL: &str =
30    "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json";
31
32/// The ecosystems a default build fetches: what Holger serves, and the
33/// distributions of the images it pulls through (Chainguard's are Wolfi and
34/// Chainguard; SANDBOX-SCAN's image CVE scanner matches their apk packages).
35pub const DEFAULT_ECOSYSTEMS: &[&str] = &[
36    "PyPI",
37    "crates.io",
38    "npm",
39    "Maven",
40    "Go",
41    "NuGet",
42    "RubyGems",
43    "Packagist",
44    "Wolfi",
45    "Chainguard",
46];
47
48/// Bound on one OSV record inside a zip (a real one is a few KiB).
49const MAX_OSV_RECORD: usize = 16 * 1024 * 1024;
50/// The most records one OSV dump may hold: `npm/all.zip` held 230 994 on 2026-10-31, past the
51/// 200 000 of a package's zip (`zip_entry::MAX_MEMBERS`); twenty times that leaves room to grow
52/// and still bounds a crafted central directory (lane HOL-FIELD-TEST).
53const MAX_OSV_MEMBERS: usize = 4_000_000;
54/// Bound on a decompressed NVD feed (a year is about 200 MiB unpacked).
55const MAX_NVD_BYTES: u64 = 2 * 1024 * 1024 * 1024;
56
57/// What a feed is.
58#[derive(Debug, Clone, PartialEq, Eq)]
59pub enum FeedKind {
60    /// An OSV dump for one ecosystem: `all.zip`, or a single / batch JSON doc.
61    Osv { ecosystem: String },
62    /// An NVD CVE JSON 2.0 feed, plain or gzip. `name` is `2024`, `modified`, …
63    Nvd { name: String },
64    /// The CISA KEV catalog.
65    Kev,
66}
67
68impl FeedKind {
69    pub fn label(&self) -> String {
70        match self {
71            FeedKind::Osv { ecosystem } => format!("osv:{ecosystem}"),
72            FeedKind::Nvd { name } => format!("nvd:{name}"),
73            FeedKind::Kev => "kev".into(),
74        }
75    }
76}
77
78/// A feed and where to get it.
79#[derive(Debug, Clone, PartialEq, Eq)]
80pub struct FeedSpec {
81    pub kind: FeedKind,
82    pub uri: String,
83}
84
85/// The cap on one fetched OSV dump (`all.zip`): Chainguard's was 1 116 562 155 bytes on
86/// 2026-11-03 (HOL-DOGFOOD-2), past the old single 1 GiB limit; room to grow.
87pub const MAX_OSV_DUMP_BYTES: u64 = 4 * 1024 * 1024 * 1024;
88/// The cap on one fetched NVD or KEV feed (a year of NVD gzipped is tens of MiB).
89pub const MAX_FEED_BYTES: u64 = 1024 * 1024 * 1024;
90
91impl FeedSpec {
92    /// The most bytes one fetch of this feed may bring (HOL-DOGFOOD-2): an OSV dump
93    /// [`MAX_OSV_DUMP_BYTES`], NVD and KEV [`MAX_FEED_BYTES`].
94    pub fn max_bytes(&self) -> u64 {
95        match self.kind {
96            FeedKind::Osv { .. } => MAX_OSV_DUMP_BYTES,
97            FeedKind::Nvd { .. } | FeedKind::Kev => MAX_FEED_BYTES,
98        }
99    }
100
101    pub fn osv(ecosystem: &str) -> Self {
102        Self {
103            kind: FeedKind::Osv {
104                ecosystem: ecosystem.to_string(),
105            },
106            uri: format!("{OSV_BUCKET}/{ecosystem}/all.zip"),
107        }
108    }
109
110    pub fn nvd(name: &str) -> Self {
111        Self {
112            kind: FeedKind::Nvd {
113                name: name.to_string(),
114            },
115            uri: format!("{NVD_FEEDS}/nvdcve-2.0-{name}.json.gz"),
116        }
117    }
118
119    pub fn kev() -> Self {
120        Self {
121            kind: FeedKind::Kev,
122            uri: KEV_URL.to_string(),
123        }
124    }
125
126    /// The default online feed list: OSV for `ecosystems`, NVD for every year
127    /// from 2002 to `this_year` plus `modified`, and KEV.
128    pub fn default_set(ecosystems: &[String], this_year: i32) -> Vec<FeedSpec> {
129        let mut out: Vec<FeedSpec> = ecosystems.iter().map(|e| FeedSpec::osv(e)).collect();
130        for y in 2002..=this_year {
131            out.push(FeedSpec::nvd(&y.to_string()));
132        }
133        out.push(FeedSpec::nvd("modified"));
134        out.push(FeedSpec::kev());
135        out
136    }
137
138    /// The feeds found in a directory of already-downloaded files, by name:
139    /// `osv-<Ecosystem>.zip` / `osv-<Ecosystem>.json`, `nvd-<name>.json[.gz]`,
140    /// `kev.json`. Sorted, so the set is the same however the disk lists it.
141    pub fn from_dir(dir: &Path) -> Result<Vec<FeedSpec>> {
142        let mut names: Vec<String> = std::fs::read_dir(dir)
143            .with_context(|| format!("read feeds dir {}", dir.display()))?
144            .filter_map(|e| e.ok())
145            .filter(|e| e.path().is_file())
146            .map(|e| e.file_name().to_string_lossy().into_owned())
147            .collect();
148        names.sort();
149        let mut out = Vec::new();
150        for n in names {
151            let uri = format!("file:{}", dir.join(&n).display());
152            let stem = n.strip_suffix(".gz").unwrap_or(&n);
153            let stem = stem
154                .strip_suffix(".zip")
155                .or_else(|| stem.strip_suffix(".json"))
156                .unwrap_or(stem);
157            let kind = if let Some(eco) = stem.strip_prefix("osv-") {
158                FeedKind::Osv {
159                    ecosystem: eco.to_string(),
160                }
161            } else if let Some(name) = stem.strip_prefix("nvd-") {
162                FeedKind::Nvd {
163                    name: name.to_string(),
164                }
165            } else if stem == "kev" {
166                FeedKind::Kev
167            } else {
168                continue;
169            };
170            out.push(FeedSpec { kind, uri });
171        }
172        if out.is_empty() {
173            bail!(
174                "no feed files in {} (expected osv-<Ecosystem>.zip|.json, nvd-<name>.json[.gz], kev.json)",
175                dir.display()
176            );
177        }
178        Ok(out)
179    }
180}
181
182/// A feed's bytes, as fetched.
183#[derive(Debug, Clone)]
184pub struct RawFeed {
185    pub spec: FeedSpec,
186    pub bytes: Vec<u8>,
187    pub fetched_at: DateTime<Utc>,
188}
189
190impl RawFeed {
191    /// Read a `file:` feed from disk.
192    pub fn read_file(spec: FeedSpec) -> Result<Self> {
193        let path = spec
194            .uri
195            .strip_prefix("file:")
196            .ok_or_else(|| anyhow::anyhow!("{} is not a file: feed", spec.uri))?;
197        let bytes = std::fs::read(path).with_context(|| format!("read feed {path}"))?;
198        Ok(Self {
199            spec,
200            bytes,
201            fetched_at: Utc::now(),
202        })
203    }
204}
205
206fn is_zip(b: &[u8]) -> bool {
207    b.starts_with(b"PK\x03\x04") || b.starts_with(b"PK\x05\x06")
208}
209
210fn is_gzip(b: &[u8]) -> bool {
211    b.starts_with(&[0x1f, 0x8b])
212}
213
214fn gunzip_bounded(bytes: &[u8]) -> Result<Vec<u8>> {
215    let mut out = Vec::new();
216    flate2::read::MultiGzDecoder::new(bytes)
217        .take(MAX_NVD_BYTES + 1)
218        .read_to_end(&mut out)
219        .context("gunzip")?;
220    if out.len() as u64 > MAX_NVD_BYTES {
221        bail!("decompressed feed exceeds {MAX_NVD_BYTES} bytes");
222    }
223    Ok(out)
224}
225
226/// **A Debian release's package name in the advisory set** (lane HOL-FIELD-2 P17):
227/// `debian-<n>/<source>`, what server-lib's deb scan asks for (`scanner::osv::deb_names`).
228pub const DEBIAN_RELEASE_PREFIX: &str = "debian-";
229
230/// `Debian:<n>` → the affected entry's package is named `debian-<n>/<name>`.
231fn debian_by_release(affected: &mut serde_json::Value) {
232    let Some(pkg) = affected.get_mut("package") else {
233        return;
234    };
235    let Some(release) = pkg
236        .get("ecosystem")
237        .and_then(|e| e.as_str())
238        .and_then(|e| e.strip_prefix("Debian:"))
239        .map(|r| r.trim().to_string())
240    else {
241        return;
242    };
243    if release.is_empty() {
244        return;
245    }
246    if let Some(name) = pkg.get("name").and_then(|n| n.as_str()).map(str::to_string) {
247        pkg["name"] = serde_json::Value::String(format!("{DEBIAN_RELEASE_PREFIX}{release}/{name}"));
248    }
249}
250
251/// Every advisory in one OSV feed (zip dump or JSON document).
252pub fn parse_osv_feed(bytes: &[u8]) -> Result<Vec<Advisory>> {
253    if !is_zip(bytes) {
254        let text = std::str::from_utf8(bytes).context("OSV feed is not UTF-8")?;
255        return parse_osv_doc(text).context("OSV feed is not an OSV document");
256    }
257    let mut out = Vec::new();
258    for m in zip_entry::members_up_to(bytes, MAX_OSV_MEMBERS)
259        .context("OSV dump is not a readable zip")?
260    {
261        if !m.name.ends_with(".json") {
262            continue;
263        }
264        let body = zip_entry::read_member_capped(bytes, &m, MAX_OSV_RECORD)
265            .with_context(|| format!("OSV dump member {}", m.name))?;
266        let text = std::str::from_utf8(&body)
267            .with_context(|| format!("OSV dump member {} is not UTF-8", m.name))?;
268        let advisories = parse_osv_doc(text)
269            .with_context(|| format!("OSV dump member {} is not an OSV record", m.name))?;
270        out.extend(advisories);
271    }
272    Ok(out)
273}
274
275/// One OSV document (a record or an array of them). The OSV schema makes `affected[].package`
276/// optional: an entry without one names git ranges only (Debian's dump has them, e.g.
277/// CVE-2024-38998) and can match no package, so it is dropped and the record kept; anything
278/// else that does not parse is still refused (lane HOL-FIELD-TEST).
279fn parse_osv_doc(text: &str) -> Result<Vec<Advisory>, serde_json::Error> {
280    // P17 (lane HOL-FIELD-2): Debian's records give each release its own range, and the
281    // advisory set has one `deb` ecosystem; each release's package is named
282    // `debian-<n>/<source>` so a bookworm package is never judged by trixie's range.
283    if text.contains("\"Debian:") {
284        if let Ok(mut doc) = serde_json::from_str::<serde_json::Value>(text) {
285            let mut each = |rec: &mut serde_json::Value| {
286                if let Some(affected) = rec.get_mut("affected").and_then(|a| a.as_array_mut()) {
287                    affected.retain(|a| a.get("package").is_some());
288                    for a in affected.iter_mut() {
289                        debian_by_release(a);
290                    }
291                }
292            };
293            match &mut doc {
294                serde_json::Value::Array(recs) => recs.iter_mut().for_each(&mut each),
295                rec => each(rec),
296            }
297            return modgunn::osv::parse_doc(&doc.to_string());
298        }
299    }
300    let first = match modgunn::osv::parse_doc(text) {
301        Ok(v) => return Ok(v),
302        Err(e) => e,
303    };
304    let Ok(mut doc) = serde_json::from_str::<serde_json::Value>(text) else {
305        return Err(first);
306    };
307    let mut dropped = false;
308    let mut drop_unpackaged = |rec: &mut serde_json::Value| {
309        if let Some(affected) = rec.get_mut("affected").and_then(|a| a.as_array_mut()) {
310            let before = affected.len();
311            affected.retain(|a| a.get("package").is_some());
312            dropped |= affected.len() != before;
313        }
314    };
315    match &mut doc {
316        serde_json::Value::Array(recs) => recs.iter_mut().for_each(&mut drop_unpackaged),
317        rec => drop_unpackaged(rec),
318    }
319    if !dropped {
320        return Err(first);
321    }
322    modgunn::osv::parse_doc(&doc.to_string())
323}
324
325/// `modgunn::scan::merge_by_alias`, one package at a time. It merges only records of the same
326/// ecosystem and package, and compares each record with every group so far: over the real feeds
327/// (about 300 000 records) that is quadratic and never ended (lane HOL-FIELD-TEST). Bucketed by
328/// (ecosystem, package) it does the same merges, and the order inside a bucket is the feeds'
329/// order, which is all [`AdvisorySet::from_advisories`]'s stable sort keeps.
330fn merge_by_alias_per_package(advisories: Vec<Advisory>) -> Vec<Advisory> {
331    let mut buckets: HashMap<(String, String), Vec<Advisory>> = HashMap::new();
332    let mut order: Vec<(String, String)> = Vec::new();
333    for a in advisories {
334        let key = (a.ecosystem.clone(), a.package.clone());
335        let bucket = buckets.entry(key.clone()).or_default();
336        if bucket.is_empty() {
337            order.push(key);
338        }
339        bucket.push(a);
340    }
341    order
342        .into_iter()
343        .flat_map(|k| modgunn::scan::merge_by_alias(buckets.remove(&k).unwrap_or_default()))
344        .collect()
345}
346
347/// Fold every feed into one set. OSV makes the advisories, NVD and KEV enrich
348/// them. A feed that does not parse fails the whole build: a set that silently
349/// lacks an ecosystem would scan clean and say nothing.
350pub fn normalize(feeds: &[RawFeed], generated_at: DateTime<Utc>) -> Result<AdvisorySet> {
351    let mut advisories: Vec<Advisory> = Vec::new();
352    let mut nvd = Vec::new();
353    let mut kev: HashSet<String> = HashSet::new();
354    let mut sources = Vec::new();
355    for f in feeds {
356        let label = f.spec.kind.label();
357        let records = match &f.spec.kind {
358            FeedKind::Osv { .. } => {
359                let a = parse_osv_feed(&f.bytes).with_context(|| format!("feed {label}"))?;
360                let n = a.len();
361                advisories.extend(a);
362                n
363            }
364            FeedKind::Nvd { .. } => {
365                let plain = if is_gzip(&f.bytes) {
366                    gunzip_bounded(&f.bytes)?
367                } else {
368                    f.bytes.clone()
369                };
370                let text = std::str::from_utf8(&plain)
371                    .with_context(|| format!("feed {label} is not UTF-8"))?;
372                let entries =
373                    modgunn::nvd::parse_nvd(text).with_context(|| format!("feed {label}"))?;
374                let n = entries.len();
375                nvd.extend(entries);
376                n
377            }
378            FeedKind::Kev => {
379                let text = std::str::from_utf8(&f.bytes)
380                    .with_context(|| format!("feed {label} is not UTF-8"))?;
381                let ids = modgunn::feeds::parse_kev_json(text)
382                    .with_context(|| format!("feed {label}"))?;
383                let n = ids.len();
384                kev.extend(ids);
385                n
386            }
387        };
388        sources.push(SourceRecord {
389            feed: label,
390            uri: f.spec.uri.clone(),
391            sha256: sha256_hex(&f.bytes),
392            bytes: f.bytes.len() as u64,
393            fetched_at: f.fetched_at,
394            records: records as u64,
395        });
396    }
397    if !feeds
398        .iter()
399        .any(|f| matches!(f.spec.kind, FeedKind::Osv { .. }))
400    {
401        bail!("no OSV feed: NVD and KEV only enrich, so a set without OSV matches nothing");
402    }
403    let mut advisories = merge_by_alias_per_package(advisories);
404    modgunn::nvd::enrich_with_nvd(&mut advisories, &nvd);
405    modgunn::feeds::enrich(&mut advisories, &HashMap::new(), &kev);
406    Ok(AdvisorySet::from_advisories(
407        generated_at,
408        sources,
409        &advisories,
410    ))
411}
holger_intel/feeds.rs

holger_intel/
feeds.rs
