use std::fmt;
use std::io::BufRead;
use std::net::Ipv6Addr;
use serde::de::{self, DeserializeSeed, IgnoredAny, MapAccess, SeqAccess, Visitor};
use serde::{Deserialize, Deserializer};
use crate::format::{ENGINE_NAME, SCHEMA_VERSION};
use crate::import::{ImportError, ImportLimits, ImportOrigin, Importer, TargetSink};
const FORMAT: &str = "JSON";
const LINES_FORMAT: &str = "JSON Lines";
#[derive(Debug, Deserialize)]
struct HostRecord {
primary_ip: String,
#[serde(default)]
ips: Vec<String>,
#[serde(default)]
zone: Option<String>,
#[serde(default)]
ports: Vec<PortRecord>,
}
#[derive(Debug, Deserialize)]
struct PortRecord {
port: u16,
protocol: String,
}
#[derive(Debug, Deserialize)]
#[serde(tag = "type")]
enum LineRecord {
#[serde(rename = "report")]
Report(HeaderRecord),
#[serde(rename = "host")]
Host(HostRecord),
#[serde(other)]
Unknown,
}
#[derive(Debug, Deserialize)]
struct HeaderRecord {
schema_version: u32,
}
struct Emitter<'a> {
sink: &'a mut dyn TargetSink,
token: String,
ports: String,
failure: Option<ImportError>,
versioned: bool,
max_addresses: u128,
hosted: bool,
}
impl<'a> Emitter<'a> {
fn new(sink: &'a mut dyn TargetSink, limits: ImportLimits) -> Self {
Self {
sink,
token: String::new(),
ports: String::new(),
failure: None,
max_addresses: limits.max_addresses,
versioned: false,
hosted: false,
}
}
fn accept_version(&mut self, version: u32, format: &'static str, origin: ImportOrigin) -> bool {
if version > SCHEMA_VERSION {
self.failure = Some(ImportError::Malformed {
format,
origin,
message: format!(
"schema version {version} is newer than this build understands ({SCHEMA_VERSION}); \
its fields do not mean what they used to"
),
});
return false;
}
self.versioned = true;
true
}
fn emit(&mut self, host: &HostRecord, format: &'static str, origin: ImportOrigin) -> bool {
self.ports.clear();
for port in &host.ports {
let name = port.protocol.to_ascii_lowercase();
let Some(protocol) = crate::record::wire::protocol(&name) else {
self.failure = Some(ImportError::Malformed {
format,
origin,
message: format!(
"'{}' names transport '{name}', which this build cannot probe",
host.primary_ip
),
});
return false;
};
let prefix = protocol.spec_prefix();
if !self.ports.is_empty() {
self.ports.push(',');
}
self.ports.push_str(prefix);
self.ports.push_str(&port.port.to_string());
}
let addresses: &[String] = if host.ips.is_empty() {
std::slice::from_ref(&host.primary_ip)
} else {
&host.ips
};
if addresses.len() as u128 > self.max_addresses {
self.failure = Some(ImportError::TooManyAddresses {
origin,
token: host.primary_ip.clone(),
limit: self.max_addresses,
});
return false;
}
for address in addresses {
let scoped = host
.zone
.as_deref()
.filter(|_| is_link_local(address))
.map(|zone| format!("{address}%{zone}"));
let address = scoped.as_deref().unwrap_or(address);
crate::import::expression(&mut self.token, address, &self.ports);
if let Err(error) = self.sink.accept(&self.token, origin) {
self.failure = Some(error);
return false;
}
}
true
}
}
fn is_link_local(address: &str) -> bool {
address
.parse::<Ipv6Addr>()
.is_ok_and(|address| address.is_unicast_link_local())
}
#[derive(Debug, Clone, Copy, Default)]
pub struct JsonImporter {
limits: ImportLimits,
}
impl JsonImporter {
pub fn new(limits: ImportLimits) -> Self {
Self { limits }
}
}
impl Importer for JsonImporter {
fn import(
&self,
input: &mut dyn BufRead,
sink: &mut dyn TargetSink,
) -> Result<(), ImportError> {
crate::import::skip_bom(input)?;
let mut emitter = Emitter::new(sink, self.limits);
let mut deserializer = serde_json::Deserializer::from_reader(input);
let outcome = Document {
emitter: &mut emitter,
}
.deserialize(&mut deserializer);
if let Some(failure) = emitter.failure.take() {
return Err(failure);
}
outcome.map_err(|error| ImportError::Malformed {
format: FORMAT,
origin: ImportOrigin::unknown(),
message: error.to_string(),
})?;
if !emitter.versioned {
return Err(ImportError::Malformed {
format: FORMAT,
origin: ImportOrigin::unknown(),
message: format!("no 'schema_version': this is not a document {ENGINE_NAME} wrote"),
});
}
if !emitter.hosted {
return Err(ImportError::Malformed {
format: FORMAT,
origin: ImportOrigin::unknown(),
message: "no 'hosts': a document with no host list names no targets, and a \
record-per-line export is read as JSON Lines"
.to_string(),
});
}
Ok(())
}
}
struct Document<'a, 'e> {
emitter: &'e mut Emitter<'a>,
}
impl<'de, 'a, 'e> DeserializeSeed<'de> for Document<'a, 'e> {
type Value = ();
fn deserialize<D: Deserializer<'de>>(self, deserializer: D) -> Result<(), D::Error> {
deserializer.deserialize_map(self)
}
}
impl<'de, 'a, 'e> Visitor<'de> for Document<'a, 'e> {
type Value = ();
fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str("a scan report object")
}
fn visit_map<A: MapAccess<'de>>(self, mut map: A) -> Result<(), A::Error> {
while let Some(key) = map.next_key::<String>()? {
match key.as_str() {
"schema_version" => {
let version: u32 = map.next_value()?;
if !self
.emitter
.accept_version(version, FORMAT, ImportOrigin::unknown())
{
return Err(de::Error::custom("unsupported schema version"));
}
}
"hosts" => {
self.emitter.hosted = true;
map.next_value_seed(Hosts {
emitter: &mut *self.emitter,
})?;
}
_ => {
map.next_value::<IgnoredAny>()?;
}
}
}
Ok(())
}
}
struct Hosts<'a, 'e> {
emitter: &'e mut Emitter<'a>,
}
impl<'de, 'a, 'e> DeserializeSeed<'de> for Hosts<'a, 'e> {
type Value = ();
fn deserialize<D: Deserializer<'de>>(self, deserializer: D) -> Result<(), D::Error> {
deserializer.deserialize_seq(self)
}
}
impl<'de, 'a, 'e> Visitor<'de> for Hosts<'a, 'e> {
type Value = ();
fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str("an array of hosts")
}
fn visit_seq<A: SeqAccess<'de>>(self, mut seq: A) -> Result<(), A::Error> {
while let Some(host) = seq.next_element::<HostRecord>()? {
if !self.emitter.emit(&host, FORMAT, ImportOrigin::unknown()) {
return Err(de::Error::custom("import stopped"));
}
}
Ok(())
}
}
#[derive(Debug, Clone, Copy, Default)]
pub struct JsonLinesImporter {
limits: ImportLimits,
}
impl JsonLinesImporter {
pub fn new(limits: ImportLimits) -> Self {
Self { limits }
}
}
impl Importer for JsonLinesImporter {
fn import(
&self,
input: &mut dyn BufRead,
sink: &mut dyn TargetSink,
) -> Result<(), ImportError> {
crate::import::skip_bom(input)?;
let mut emitter = Emitter::new(sink, self.limits);
let mut buffer = Vec::new();
let mut line_number = 0u64;
loop {
buffer.clear();
line_number += 1;
let origin = ImportOrigin::line(line_number);
if !crate::import::list::read_line(
input,
&mut buffer,
self.limits.max_line_bytes,
origin,
)? {
break;
}
let text =
std::str::from_utf8(&buffer).map_err(|_| ImportError::InvalidUtf8 { origin })?;
if text.trim().is_empty() {
continue;
}
let record: LineRecord =
serde_json::from_str(text).map_err(|error| ImportError::Malformed {
format: LINES_FORMAT,
origin,
message: error.to_string(),
})?;
let carried_on = match record {
LineRecord::Report(header) => {
emitter.accept_version(header.schema_version, LINES_FORMAT, origin)
}
LineRecord::Host(host) => emitter.emit(&host, LINES_FORMAT, origin),
LineRecord::Unknown => true,
};
if !carried_on {
return Err(emitter.failure.take().expect("a refusal records why"));
}
}
if !emitter.versioned {
return Err(ImportError::Malformed {
format: LINES_FORMAT,
origin: ImportOrigin::unknown(),
message: format!("no 'report' record: this is not output {ENGINE_NAME} wrote"),
});
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::import::{ImportFormat, ImportOptions, Imported};
use crate::model::port::PortSet;
use std::io::Cursor;
const DEFAULT_PORT: u16 = 80;
fn options() -> ImportOptions<'static> {
ImportOptions::new(PortSet::try_from("80").unwrap())
}
fn read(format: ImportFormat, input: &str) -> Result<Imported, ImportError> {
format.read(&mut Cursor::new(input), &options())
}
fn document(hosts: &str) -> String {
format!(r#"{{"schema_version":1,"engine":{{"name":"zond-engine"}},"hosts":[{hosts}]}}"#)
}
#[test]
fn a_host_is_rescanned_on_the_ports_it_was_found_on() {
let file = document(
r#"{"primary_ip":"10.0.0.1","ips":["10.0.0.1"],
"ports":[{"port":22,"protocol":"tcp"},{"port":53,"protocol":"udp"}]}"#,
);
let imported = read(ImportFormat::Json, &file).expect("imports");
assert_eq!(imported.addresses, 1);
let ports = imported.map.units[0].ports();
assert!(ports.has_tcp(22), "the TCP port came back as TCP");
assert!(ports.has_udp(53), "the UDP port came back as UDP");
assert!(!ports.has_tcp(53), "and not as both");
}
#[test]
fn a_host_with_no_ports_takes_the_default_ports() {
let file = document(r#"{"primary_ip":"10.0.0.1"},{"primary_ip":"10.0.0.2"}"#);
let imported = read(ImportFormat::Json, &file).expect("imports");
assert_eq!(imported.addresses, 2);
assert!(imported.map.units[0].ports().has_tcp(80));
}
#[test]
fn every_address_of_a_host_becomes_a_target() {
let file = document(
r#"{"primary_ip":"10.0.0.1","ips":["10.0.0.1","2001:db8::1"],
"ports":[{"port":443,"protocol":"tcp"}]}"#,
);
let imported = read(ImportFormat::Json, &file).expect("imports");
assert_eq!(imported.addresses, 2);
}
#[test]
fn a_link_local_host_keeps_the_interface_that_makes_it_reachable() {
fn zones(name: &str) -> Option<u32> {
(name == "en0").then_some(7)
}
let options = options()
.with_context(crate::model::parse::target::TargetContext::new().with_zones(&zones));
let file = document(
r#"{"primary_ip":"fe80::aa","ips":["fe80::aa","10.0.0.1"],"zone":"en0",
"ports":[{"port":22,"protocol":"tcp"}]}"#,
);
let imported = ImportFormat::Json
.read(&mut Cursor::new(file), &options)
.expect("imports");
let v6 = imported.map.units[0].ips().v6();
assert_eq!(v6.len(), 1);
assert_eq!(v6[0].zone(), Some(7), "the zone was dropped on the way in");
assert_eq!(imported.map.units[0].ips().v4().len(), 1);
}
#[test]
fn fields_this_build_does_not_know_are_ignored() {
let file = r#"{
"schema_version":1,
"engine":{"name":"zond-engine","version":"9.9.9"},
"summary":{"hosts_total":1,"anything":{"nested":[1,2,3]}},
"hosts":[{"primary_ip":"10.0.0.1","telemetry":{"rtt_median_us":1234},
"invented_field":true,
"ports":[{"port":22,"protocol":"tcp","state":"open","service":{"name":"ssh"}}]}],
"trailing_unknown":[{"a":1}]
}"#;
let imported = read(ImportFormat::Json, file).expect("imports");
assert_eq!(imported.addresses, 1);
assert!(imported.map.units[0].ports().has_tcp(22));
}
#[test]
fn an_sctp_port_reads_back_as_an_sctp_port() {
let file = document(r#"{"primary_ip":"10.0.0.1","ports":[{"port":9,"protocol":"sctp"}]}"#);
let imported = read(ImportFormat::Json, &file).expect("imports");
let ports = imported.map.units[0].ports();
assert!(ports.has_sctp(9) && !ports.has_tcp(9));
}
#[test]
fn an_unknown_transport_is_refused_rather_than_assumed() {
let file = document(r#"{"primary_ip":"10.0.0.1","ports":[{"port":9,"protocol":"dccp"}]}"#);
let err = read(ImportFormat::Json, &file).expect_err("dccp cannot be probed");
match err {
ImportError::Malformed { message, .. } => {
assert!(
message.contains("dccp"),
"the error has to name it: {message}"
);
}
other => panic!("expected a malformed document, got {other:?}"),
}
}
#[test]
fn a_record_per_line_file_read_as_a_document_is_refused() {
let file = concat!(
r#"{"type":"report","schema_version":1,"engine":{"name":"zond-engine"}}"#,
"\n",
r#"{"type":"host","primary_ip":"10.0.0.1","ports":[{"port":22,"protocol":"tcp"}]}"#,
"\n",
);
let error =
read(ImportFormat::Json, file).expect_err("a stream of records is not one document");
match error {
ImportError::Malformed { message, .. } => assert!(
message.contains("hosts"),
"the refusal should name what was missing, said: {message}"
),
other => panic!("expected a malformed document, got {other:?}"),
}
}
#[test]
fn a_newer_schema_version_is_refused_and_a_missing_one_is_not_a_report() {
let newer = r#"{"schema_version":9999,"hosts":[{"primary_ip":"10.0.0.1"}]}"#;
assert!(matches!(
read(ImportFormat::Json, newer),
Err(ImportError::Malformed { .. })
));
let anonymous = r#"{"hosts":[{"primary_ip":"10.0.0.1"}]}"#;
let err = read(ImportFormat::Json, anonymous)
.expect_err("JSON with a hosts key is not necessarily a report");
assert!(matches!(err, ImportError::Malformed { .. }));
}
#[test]
fn a_record_per_line_report_reads_the_same_hosts() {
let file = concat!(
r#"{"type":"report","schema_version":1,"engine":{"name":"zond-engine"}}"#,
"\n",
r#"{"type":"host","primary_ip":"10.0.0.1","ports":[{"port":22,"protocol":"tcp"}]}"#,
"\n",
r#"{"type":"host","primary_ip":"10.0.0.2","ports":[{"port":22,"protocol":"tcp"}]}"#,
"\n",
);
let imported = read(ImportFormat::JsonLines, file).expect("imports");
assert_eq!(imported.addresses, 2);
assert_eq!(imported.map.units.len(), 1, "both on 22/tcp");
}
#[test]
fn a_truncated_record_per_line_report_still_reads_what_survived() {
let file = concat!(
r#"{"type":"report","schema_version":1}"#,
"\n",
r#"{"type":"host","primary_ip":"10.0.0.1"}"#,
"\n",
);
let imported = read(ImportFormat::JsonLines, file).expect("imports");
assert_eq!(imported.addresses, 1);
}
#[test]
fn an_unknown_record_kind_is_skipped() {
let file = concat!(
r#"{"type":"report","schema_version":1}"#,
"\n",
r#"{"type":"finding","severity":"high","note":"something new"}"#,
"\n",
r#"{"type":"host","primary_ip":"10.0.0.1"}"#,
"\n",
);
let imported = read(ImportFormat::JsonLines, file).expect("imports");
assert_eq!(imported.addresses, 1);
}
#[cfg(feature = "export-json")]
#[test]
fn every_host_and_port_in_a_report_survives_the_round_trip() {
use crate::export::{ExportOptions, Exporter, JsonExporter};
use std::collections::BTreeSet;
let report = crate::export::fixture::report();
let mut expected: BTreeSet<(String, u16)> = BTreeSet::new();
let mut expected_addresses: BTreeSet<String> = BTreeSet::new();
for host in report.hosts() {
for ip in host.ips() {
expected_addresses.insert(ip.to_string());
if host.port_count() == 0 {
expected.insert((ip.to_string(), DEFAULT_PORT));
continue;
}
for port in host.ports() {
expected.insert((ip.to_string(), port.number()));
}
}
}
let mut document = Vec::new();
JsonExporter::new(ExportOptions::new())
.export(&report, &mut document)
.expect("the fixture exports");
let imported = ImportFormat::Json
.read(&mut Cursor::new(document), &options())
.expect("its own document reads back");
let mut found: BTreeSet<(String, u16)> = BTreeSet::new();
let mut found_addresses: BTreeSet<String> = BTreeSet::new();
for target in imported.map.iter() {
found_addresses.insert(target.ip.to_string());
found.insert((target.ip.to_string(), target.port));
}
assert_eq!(
found_addresses, expected_addresses,
"the addresses the scan found are not the addresses that came back"
);
assert_eq!(
found, expected,
"the host and port pairs the scan found are not the ones that came back"
);
assert!(
!expected.is_empty(),
"a fixture with no ports proves nothing"
);
}
#[cfg(all(feature = "export-json", feature = "export-jsonl"))]
#[test]
fn the_two_report_formats_read_back_as_the_same_targets() {
use crate::export::{ExportOptions, Exporter, JsonExporter, JsonLinesExporter};
let report = crate::export::fixture::report();
let options = ExportOptions::new();
let mut document = Vec::new();
JsonExporter::new(options.clone())
.export(&report, &mut document)
.expect("exports");
let mut lines = Vec::new();
JsonLinesExporter::new(options)
.export(&report, &mut lines)
.expect("exports");
let from_document = ImportFormat::Json
.read(&mut Cursor::new(document), &super::tests::options())
.expect("imports");
let from_lines = ImportFormat::JsonLines
.read(&mut Cursor::new(lines), &super::tests::options())
.expect("imports");
assert_eq!(from_document.addresses, from_lines.addresses);
assert_eq!(
from_document.map.units.len(),
from_lines.map.units.len(),
"the same report produced different targets in two formats"
);
}
#[test]
fn a_line_that_is_not_a_record_names_its_line() {
let file = concat!(
r#"{"type":"report","schema_version":1}"#,
"\n",
"not json at all\n",
);
match read(ImportFormat::JsonLines, file).expect_err("line 2 is not a record") {
ImportError::Malformed { origin, .. } => assert_eq!(origin, ImportOrigin::line(2)),
other => panic!("expected a malformed record, got {other:?}"),
}
}
}