use std::collections::BTreeSet;
use std::fmt;
use std::io::BufRead;
use serde::de::{DeserializeSeed, IgnoredAny, MapAccess, SeqAccess, Visitor};
use serde::{Deserialize, Deserializer, Serialize};
use crate::cve::{Catalogue, CatalogueError, MAX_DOCUMENT_BYTES};
use crate::fingerprint::SignatureDb;
pub const NVD_ID: &str = "nvd:cve";
pub const MAX_FEED_BYTES: u64 = 100 * MAX_DOCUMENT_BYTES;
const MAX_TITLE_BYTES: usize = 160;
#[non_exhaustive]
#[derive(Debug, thiserror::Error)]
pub enum NvdError {
#[error("the NVD feed could not be read: {0}")]
Io(#[from] std::io::Error),
#[error("the NVD feed is malformed: {0}")]
Malformed(String),
#[error("the NVD feed is longer than the {limit} byte limit")]
TooLarge {
limit: u64,
},
#[error("{entries} entries convert to more than the {limit} byte document limit")]
TooManyEntries {
entries: usize,
limit: u64,
},
#[error("the converted catalogue was refused: {0}")]
Rejected(#[from] CatalogueError),
}
pub fn to_document(input: &mut dyn BufRead) -> Result<String, NvdError> {
to_document_for(input, SignatureDb::global().versioned_products())
}
pub fn to_document_for(
input: &mut dyn BufRead,
products: &BTreeSet<String>,
) -> Result<String, NvdError> {
let bounded = std::io::Read::take(input, MAX_FEED_BYTES.saturating_add(1));
let counted = Counting::new(bounded);
let read = counted.read.clone();
let mut deserializer = serde_json::Deserializer::from_reader(counted);
let feed = Feed { products }
.deserialize(&mut deserializer)
.map_err(|error| NvdError::Malformed(error.to_string()))?;
if read.get() > MAX_FEED_BYTES {
return Err(NvdError::TooLarge {
limit: MAX_FEED_BYTES,
});
}
let document = Document {
id: NVD_ID.to_string(),
version: feed_version(&feed.timestamp),
vulnerability: feed.vulnerability,
};
let toml =
toml::to_string(&document).map_err(|error| NvdError::Malformed(error.to_string()))?;
if toml.len() as u64 > MAX_DOCUMENT_BYTES {
return Err(NvdError::TooManyEntries {
entries: document.vulnerability.len(),
limit: MAX_DOCUMENT_BYTES,
});
}
Ok(toml)
}
pub fn read(input: &mut dyn BufRead) -> Result<Catalogue, NvdError> {
let document = to_document(input)?;
Ok(Catalogue::read(&mut document.as_bytes())?)
}
fn feed_version(timestamp: &str) -> String {
let date = timestamp.split('T').next().unwrap_or_default();
let mut parts = date.split('-');
let mut number = || -> Option<u32> { parts.next()?.parse().ok() };
match (number(), number(), number()) {
(Some(year), Some(month), Some(day)) => format!("{year}.{month}.{day}"),
_ => "0.0.0".to_string(),
}
}
struct Counting<R> {
inner: R,
read: std::rc::Rc<std::cell::Cell<u64>>,
}
impl<R> Counting<R> {
fn new(inner: R) -> Self {
Self {
inner,
read: std::rc::Rc::new(std::cell::Cell::new(0)),
}
}
}
impl<R: std::io::Read> std::io::Read for Counting<R> {
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
let n = self.inner.read(buf)?;
self.read.set(self.read.get().saturating_add(n as u64));
Ok(n)
}
}
struct Feed<'a> {
products: &'a BTreeSet<String>,
}
struct Converted {
timestamp: String,
vulnerability: Vec<Entry>,
}
impl<'de> DeserializeSeed<'de> for Feed<'_> {
type Value = Converted;
fn deserialize<D: Deserializer<'de>>(self, deserializer: D) -> Result<Converted, D::Error> {
deserializer.deserialize_map(self)
}
}
impl<'de> Visitor<'de> for Feed<'_> {
type Value = Converted;
fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str("an NVD feed object carrying `cve_items`")
}
fn visit_map<A: MapAccess<'de>>(self, mut map: A) -> Result<Converted, A::Error> {
let mut timestamp = String::new();
let mut vulnerability = Vec::new();
let mut seen_items = false;
while let Some(key) = map.next_key::<String>()? {
match key.as_str() {
"timestamp" => timestamp = map.next_value()?,
"cve_items" | "vulnerabilities" => {
seen_items = true;
vulnerability = map.next_value_seed(Records {
products: self.products,
})?;
}
_ => {
map.next_value::<IgnoredAny>()?;
}
}
}
if !seen_items {
return Err(serde::de::Error::missing_field("cve_items"));
}
Ok(Converted {
timestamp,
vulnerability,
})
}
}
struct Records<'a> {
products: &'a BTreeSet<String>,
}
impl<'de> DeserializeSeed<'de> for Records<'_> {
type Value = Vec<Entry>;
fn deserialize<D: Deserializer<'de>>(self, deserializer: D) -> Result<Vec<Entry>, D::Error> {
deserializer.deserialize_seq(self)
}
}
impl<'de> Visitor<'de> for Records<'_> {
type Value = Vec<Entry>;
fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str("an array of NVD records")
}
fn visit_seq<A: SeqAccess<'de>>(self, mut seq: A) -> Result<Vec<Entry>, A::Error> {
let mut out = Vec::new();
while let Some(record) = seq.next_element::<Record>()? {
entries_from(&record, self.products, &mut out);
}
Ok(out)
}
}
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
struct Record {
cve: Option<Box<Record>>,
#[serde(default)]
id: String,
#[serde(default)]
vuln_status: String,
#[serde(default)]
descriptions: Vec<Description>,
#[serde(default)]
metrics: Metrics,
#[serde(default)]
weaknesses: Vec<Weakness>,
#[serde(default)]
configurations: Vec<Configuration>,
}
#[derive(Deserialize)]
struct Description {
lang: String,
value: String,
}
#[derive(Default, Deserialize)]
#[serde(rename_all = "camelCase")]
struct Metrics {
#[serde(default)]
cvss_metric_v31: Vec<Metric>,
#[serde(default)]
cvss_metric_v30: Vec<Metric>,
#[serde(default)]
cvss_metric_v2: Vec<Metric>,
}
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
struct Metric {
#[serde(default)]
base_severity: Option<String>,
#[serde(default)]
cvss_data: CvssData,
}
#[derive(Default, Deserialize)]
#[serde(rename_all = "camelCase")]
struct CvssData {
#[serde(default)]
base_severity: Option<String>,
}
#[derive(Deserialize)]
struct Weakness {
#[serde(default)]
description: Vec<Description>,
}
#[derive(Deserialize)]
struct Configuration {
#[serde(default)]
nodes: Vec<Node>,
}
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
struct Node {
#[serde(default)]
cpe_match: Vec<CpeMatch>,
}
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
struct CpeMatch {
#[serde(default = "yes")]
vulnerable: bool,
#[serde(default)]
criteria: String,
version_start_including: Option<String>,
version_start_excluding: Option<String>,
version_end_including: Option<String>,
version_end_excluding: Option<String>,
}
fn yes() -> bool {
true
}
#[derive(Serialize)]
struct Document {
id: String,
version: String,
vulnerability: Vec<Entry>,
}
#[derive(Serialize)]
struct Entry {
cve: String,
title: String,
severity: String,
vendor: String,
product: String,
affected: String,
#[serde(skip_serializing_if = "Option::is_none")]
cwe: Option<u32>,
}
fn entries_from(record: &Record, products: &BTreeSet<String>, out: &mut Vec<Entry>) {
let record = record.cve.as_deref().unwrap_or(record);
if record.vuln_status.eq_ignore_ascii_case("rejected") {
return;
}
let (Some(severity), Some(title)) = (severity_of(record), title_of(record)) else {
return;
};
let cwe = cwe_of(record);
let mut seen = BTreeSet::new();
for configuration in &record.configurations {
for node in &configuration.nodes {
for matched in &node.cpe_match {
let Some((vendor, product, affected)) = affected_range(matched, products) else {
continue;
};
if !seen.insert((vendor.clone(), product.clone(), affected.clone())) {
continue;
}
out.push(Entry {
cve: record.id.clone(),
title: title.clone(),
severity: severity.clone(),
vendor,
product,
affected,
cwe,
});
}
}
}
}
fn affected_range(
matched: &CpeMatch,
products: &BTreeSet<String>,
) -> Option<(String, String, String)> {
if !matched.vulnerable {
return None;
}
let mut parts = matched.criteria.split(':');
if parts.next()? != "cpe" || parts.next()? != "2.3" || parts.next()? != "a" {
return None;
}
let vendor = parts.next()?;
let product = parts.next()?;
let version = parts.next()?;
let key = format!("{vendor}:{product}");
if !products.contains(&key) {
return None;
}
let clauses: Vec<String> = [
(">=", &matched.version_start_including),
(">", &matched.version_start_excluding),
("<=", &matched.version_end_including),
("<", &matched.version_end_excluding),
]
.iter()
.filter_map(|(op, bound)| bound.as_ref().map(|value| format!("{op} {value}")))
.collect();
let affected = match clauses.is_empty() {
true if matches!(version, "*" | "-" | "") => return None,
true => format!("== {version}"),
false => clauses.join(", "),
};
Some((vendor.to_string(), product.to_string(), affected))
}
fn severity_of(record: &Record) -> Option<String> {
[
&record.metrics.cvss_metric_v31,
&record.metrics.cvss_metric_v30,
&record.metrics.cvss_metric_v2,
]
.into_iter()
.flatten()
.find_map(|metric| {
metric
.cvss_data
.base_severity
.as_deref()
.or(metric.base_severity.as_deref())
})
.map(str::to_ascii_lowercase)
}
fn title_of(record: &Record) -> Option<String> {
let english = record
.descriptions
.iter()
.find(|description| description.lang == "en")?;
let mut title = String::with_capacity(MAX_TITLE_BYTES);
let mut whitespace = false;
for character in english.value.trim().chars() {
if character.is_whitespace() {
whitespace = !title.is_empty();
continue;
}
if whitespace {
if title.ends_with('.') || title.ends_with('!') || title.ends_with('?') {
break;
}
title.push(' ');
whitespace = false;
}
if title.len() + character.len_utf8() > MAX_TITLE_BYTES {
break;
}
title.push(character);
}
let title = title.trim_end_matches(['.', ' ']).to_string();
(!title.is_empty()).then_some(title)
}
fn cwe_of(record: &Record) -> Option<u32> {
record
.weaknesses
.iter()
.flat_map(|weakness| weakness.description.iter())
.filter(|description| description.lang == "en")
.find_map(|description| description.value.strip_prefix("CWE-")?.parse().ok())
}
#[cfg(test)]
mod tests {
use super::*;
fn products(names: &[&str]) -> BTreeSet<String> {
names.iter().map(|name| (*name).to_string()).collect()
}
const EXIM: &str = r#"{
"timestamp": "2026-09-08T00:00:09+00:00",
"cve_count": 1,
"cve_items": [{
"id": "CVE-2019-10149",
"vulnStatus": "Analyzed",
"descriptions": [
{"lang": "es", "value": "Se descubrio un fallo."},
{"lang": "en", "value": "A flaw was found in Exim 4.87 to 4.91. Improper validation of recipient address."}
],
"metrics": {"cvssMetricV31": [{"cvssData": {"baseSeverity": "CRITICAL"}}]},
"weaknesses": [{"description": [{"lang": "en", "value": "CWE-78"}]}],
"configurations": [{"nodes": [{"cpeMatch": [
{"vulnerable": true, "criteria": "cpe:2.3:a:exim:exim:*:*:*:*:*:*:*:*",
"versionStartIncluding": "4.87", "versionEndIncluding": "4.91"},
{"vulnerable": true, "criteria": "cpe:2.3:o:debian:debian_linux:9.0:*:*:*:*:*:*:*"}
]}]}]
}]
}"#;
#[test]
fn a_record_converts_to_the_entry_a_person_wrote_by_hand() {
let document = to_document_for(&mut EXIM.as_bytes(), &products(&["exim:exim"]))
.expect("the feed converts");
assert!(document.contains(r#"cve = "CVE-2019-10149""#), "{document}");
assert!(
document.contains(r#"affected = ">= 4.87, <= 4.91""#),
"{document}"
);
assert!(document.contains(r#"severity = "critical""#), "{document}");
assert!(document.contains(r#"vendor = "exim""#), "{document}");
assert!(document.contains(r#"product = "exim""#), "{document}");
assert!(document.contains("cwe = 78"), "{document}");
assert!(
document.contains(r#"title = "A flaw was found in Exim 4.87 to 4.91""#),
"{document}"
);
}
#[test]
fn an_operating_system_cpe_is_left_alone() {
let document = to_document_for(
&mut EXIM.as_bytes(),
&products(&["exim:exim", "debian:debian_linux"]),
)
.expect("the feed converts");
assert!(!document.contains("debian"), "{document}");
assert_eq!(document.matches("[[vulnerability]]").count(), 1);
}
#[test]
fn a_product_the_corpus_cannot_version_is_dropped() {
let document = to_document_for(&mut EXIM.as_bytes(), &products(&["nothing:at_all"]))
.expect("converts");
assert_eq!(
document.matches("[[vulnerability]]").count(),
0,
"{document}"
);
assert!(document.contains(r#"id = "nvd:cve""#), "{document}");
assert!(document.contains(r#"version = "2026.9.8""#), "{document}");
}
#[test]
fn disjoint_ranges_become_separate_entries() {
const REGRESSHION: &str = r#"{
"timestamp": "2026-09-08T00:00:00+00:00",
"cve_items": [{
"id": "CVE-2024-6387",
"descriptions": [{"lang": "en", "value": "A signal handler race condition in OpenSSH."}],
"metrics": {"cvssMetricV31": [{"cvssData": {"baseSeverity": "HIGH"}}]},
"configurations": [{"nodes": [{"cpeMatch": [
{"vulnerable": true, "criteria": "cpe:2.3:a:openbsd:openssh:*:*:*:*:*:*:*:*",
"versionEndExcluding": "4.4"},
{"vulnerable": true, "criteria": "cpe:2.3:a:openbsd:openssh:*:*:*:*:*:*:*:*",
"versionStartIncluding": "8.5", "versionEndExcluding": "9.8"}
]}]}]
}]
}"#;
let document =
to_document_for(&mut REGRESSHION.as_bytes(), &products(&["openbsd:openssh"]))
.expect("converts");
assert_eq!(
document.matches("[[vulnerability]]").count(),
2,
"{document}"
);
assert!(document.contains(r#"affected = "< 4.4""#), "{document}");
assert!(
document.contains(r#"affected = ">= 8.5, < 9.8""#),
"{document}"
);
}
#[test]
fn a_repeated_range_is_recorded_once() {
const REPEATED: &str = r#"{
"timestamp": "2026-01-02T00:00:00+00:00",
"cve_items": [{
"id": "CVE-2024-0001",
"descriptions": [{"lang": "en", "value": "Something in Tomcat."}],
"metrics": {"cvssMetricV31": [{"cvssData": {"baseSeverity": "MEDIUM"}}]},
"configurations": [
{"nodes": [{"cpeMatch": [{"vulnerable": true,
"criteria": "cpe:2.3:a:apache:tomcat:9.0.1:*:*:*:*:*:*:*"}]}]},
{"nodes": [{"cpeMatch": [{"vulnerable": true,
"criteria": "cpe:2.3:a:apache:tomcat:9.0.1:*:*:*:*:*:*:*"}]}]}
]
}]
}"#;
let document = to_document_for(&mut REPEATED.as_bytes(), &products(&["apache:tomcat"]))
.expect("converts");
assert_eq!(
document.matches("[[vulnerability]]").count(),
1,
"{document}"
);
assert!(document.contains(r#"affected = "== 9.0.1""#), "{document}");
}
#[test]
fn a_cvss_2_record_is_still_scored() {
const OLD: &str = r#"{
"timestamp": "2026-01-02T00:00:00+00:00",
"cve_items": [{
"id": "CVE-2009-0001",
"descriptions": [{"lang": "en", "value": "An old flaw in ProFTPD."}],
"metrics": {"cvssMetricV2": [{"baseSeverity": "HIGH", "cvssData": {"baseScore": 7.5}}]},
"configurations": [{"nodes": [{"cpeMatch": [{"vulnerable": true,
"criteria": "cpe:2.3:a:proftpd:proftpd:1.3.1:*:*:*:*:*:*:*"}]}]}]
}]
}"#;
let document = to_document_for(&mut OLD.as_bytes(), &products(&["proftpd:proftpd"]))
.expect("converts");
assert!(document.contains(r#"severity = "high""#), "{document}");
}
#[test]
fn a_record_with_nothing_to_say_yields_nothing() {
const NOTHING: &str = r#"{
"timestamp": "2026-01-02T00:00:00+00:00",
"cve_items": [
{"id": "CVE-2024-1", "vulnStatus": "Rejected",
"descriptions": [{"lang": "en", "value": "Withdrawn."}],
"metrics": {"cvssMetricV31": [{"cvssData": {"baseSeverity": "HIGH"}}]},
"configurations": [{"nodes": [{"cpeMatch": [{"vulnerable": true,
"criteria": "cpe:2.3:a:apache:tomcat:9.0.1:*:*:*:*:*:*:*"}]}]}]},
{"id": "CVE-2024-2",
"descriptions": [{"lang": "en", "value": "Not yet scored."}],
"configurations": [{"nodes": [{"cpeMatch": [{"vulnerable": true,
"criteria": "cpe:2.3:a:apache:tomcat:9.0.2:*:*:*:*:*:*:*"}]}]}]},
{"id": "CVE-2024-3",
"descriptions": [{"lang": "en", "value": "Needs something present."}],
"metrics": {"cvssMetricV31": [{"cvssData": {"baseSeverity": "HIGH"}}]},
"configurations": [{"nodes": [{"cpeMatch": [
{"vulnerable": false, "criteria": "cpe:2.3:a:apache:tomcat:9.0.3:*:*:*:*:*:*:*"},
{"vulnerable": true, "criteria": "cpe:2.3:a:apache:tomcat:*:*:*:*:*:*:*:*"}
]}]}]}
]
}"#;
let document = to_document_for(&mut NOTHING.as_bytes(), &products(&["apache:tomcat"]))
.expect("converts");
assert_eq!(
document.matches("[[vulnerability]]").count(),
0,
"{document}"
);
}
#[test]
fn the_api_shape_is_read_as_well_as_the_feed_shape() {
const NESTED: &str = r#"{
"timestamp": "2026-01-02T00:00:00+00:00",
"vulnerabilities": [{"cve": {
"id": "CVE-2024-0002",
"descriptions": [{"lang": "en", "value": "Something in Tomcat."}],
"metrics": {"cvssMetricV31": [{"cvssData": {"baseSeverity": "LOW"}}]},
"configurations": [{"nodes": [{"cpeMatch": [{"vulnerable": true,
"criteria": "cpe:2.3:a:apache:tomcat:10.1.0:*:*:*:*:*:*:*"}]}]}]
}}]
}"#;
let document = to_document_for(&mut NESTED.as_bytes(), &products(&["apache:tomcat"]))
.expect("converts");
assert!(document.contains(r#"cve = "CVE-2024-0002""#), "{document}");
assert!(document.contains(r#"affected = "== 10.1.0""#), "{document}");
}
#[test]
fn a_converted_range_tells_two_versions_apart() {
use crate::model::host::Host;
use crate::model::port::{Port, PortState, Protocol, Service};
let catalogue = read(&mut EXIM.as_bytes()).expect("the feed reads as a catalogue");
assert_eq!(catalogue.id(), NVD_ID);
assert_eq!(catalogue.len(), 1);
let matched = |version: &str| {
let mut host = Host::new("192.0.2.1".parse().expect("an address"));
let service = Service::new("smtp", 90).with_cpe(format!("cpe:/a:exim:exim:{version}"));
host.add_port(Port::new(25, Protocol::Tcp, PortState::Open).with_service(service));
crate::cve::correlate_with(&mut host, &catalogue);
host.ports()
.find(|port| port.number() == 25)
.expect("the port")
.findings()
.any(|finding| finding.detection().id() == NVD_ID)
};
assert!(matched("4.90"), "4.90 is inside >= 4.87, <= 4.91");
assert!(
!matched("4.92"),
"4.92 is past the range and must not match"
);
}
#[test]
fn an_undated_feed_takes_a_version_that_says_so() {
assert_eq!(feed_version("2026-09-08T00:00:09+00:00"), "2026.9.8");
assert_eq!(feed_version("2024-01-02"), "2024.1.2");
assert_eq!(feed_version(""), "0.0.0");
assert_eq!(feed_version("last Tuesday"), "0.0.0");
}
#[test]
fn a_title_is_a_sentence_and_never_longer_than_the_cap() {
let record = Record {
cve: None,
id: String::new(),
vuln_status: String::new(),
descriptions: vec![Description {
lang: "en".into(),
value: format!("A flaw in version 1.2.3 of the thing {}", "x".repeat(400)),
}],
metrics: Metrics::default(),
weaknesses: Vec::new(),
configurations: Vec::new(),
};
let title = title_of(&record).expect("a title");
assert!(title.len() <= MAX_TITLE_BYTES, "{}", title.len());
assert!(
title.starts_with("A flaw in version 1.2.3 of the thing"),
"the dots in a version number do not end the sentence: {title}"
);
}
}