pub mod list;
#[cfg(feature = "import-csv")]
pub mod csv;
#[cfg(feature = "import-json")]
pub mod json;
#[cfg(feature = "import-nmap")]
pub mod nmap;
#[cfg(feature = "import-settings")]
pub mod settings;
use std::fmt;
use std::io::BufRead;
use std::path::Path;
use crate::model::parse::target::{TargetContext, TargetMapBuilder, TargetParseError};
use crate::model::port::PortSet;
use crate::model::target::{TargetMap, TargetSet};
pub use list::ListImporter;
#[cfg(feature = "import-csv")]
pub use csv::{CsvColumn, CsvImporter};
#[cfg(feature = "import-json")]
pub use json::{JsonImporter, JsonLinesImporter};
#[cfg(feature = "import-nmap")]
pub use nmap::NmapXmlImporter;
#[cfg(feature = "import-settings")]
pub use settings::{Settings, SettingsDocument, SettingsError, SettingsWarning};
#[non_exhaustive]
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)]
pub struct Origin {
pub line: Option<u64>,
}
impl Origin {
pub fn line(line: u64) -> Self {
Self { line: Some(line) }
}
pub fn unknown() -> Self {
Self::default()
}
}
impl fmt::Display for Origin {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self.line {
Some(line) => write!(f, "line {line}"),
None => f.write_str("input"),
}
}
}
#[non_exhaustive]
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct ImportLimits {
pub max_line_bytes: usize,
pub max_tokens: u64,
pub max_addresses: u128,
}
impl Default for ImportLimits {
fn default() -> Self {
Self {
max_line_bytes: 64 * 1024,
max_tokens: 16_777_216,
max_addresses: 1u128 << 32,
}
}
}
impl ImportLimits {
pub fn new() -> Self {
Self::default()
}
pub fn with_max_line_bytes(mut self, bytes: usize) -> Self {
self.max_line_bytes = bytes;
self
}
pub fn with_max_tokens(mut self, tokens: u64) -> Self {
self.max_tokens = tokens;
self
}
pub fn with_max_addresses(mut self, addresses: u128) -> Self {
self.max_addresses = addresses;
self
}
pub fn none() -> Self {
Self {
max_line_bytes: usize::MAX,
max_tokens: u64::MAX,
max_addresses: u128::MAX,
}
}
}
#[non_exhaustive]
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)]
pub enum OnRefusal {
#[default]
Abort,
Collect,
}
#[derive(Debug)]
pub struct Refusal {
pub origin: Origin,
pub token: String,
pub reason: TargetParseError,
}
impl Imported {
pub fn into_ip_set(self) -> crate::model::ip::set::IpSet {
self.map
.units
.into_iter()
.map(TargetSet::into_ips)
.collect()
}
}
impl fmt::Display for Refusal {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{}: {}", self.origin, self.reason)
}
}
#[non_exhaustive]
#[derive(Debug, Clone)]
pub struct ImportOptions<'a> {
pub default_ports: PortSet,
pub context: TargetContext<'a>,
pub limits: ImportLimits,
pub on_refusal: OnRefusal,
}
impl<'a> ImportOptions<'a> {
pub fn new(default_ports: PortSet) -> Self {
Self {
default_ports,
context: TargetContext::new(),
limits: ImportLimits::default(),
on_refusal: OnRefusal::default(),
}
}
pub fn with_context(mut self, context: TargetContext<'a>) -> Self {
self.context = context;
self
}
pub fn with_limits(mut self, limits: ImportLimits) -> Self {
self.limits = limits;
self
}
pub fn with_refusal_policy(mut self, on_refusal: OnRefusal) -> Self {
self.on_refusal = on_refusal;
self
}
}
#[non_exhaustive]
#[derive(Debug)]
pub struct Imported {
pub map: TargetMap,
pub refusals: Vec<Refusal>,
pub tokens: u64,
pub addresses: u128,
}
#[non_exhaustive]
#[derive(Debug, thiserror::Error)]
pub enum ImportError {
#[error("reading the input failed: {0}")]
Io(#[from] std::io::Error),
#[error("{origin}: longer than the {limit} byte line limit")]
LineTooLong {
origin: Origin,
limit: usize,
},
#[error("{origin}: not valid UTF-8")]
InvalidUtf8 {
origin: Origin,
},
#[error("more than {limit} target expressions; use a range instead of a list")]
TooManyTokens {
limit: u64,
},
#[error("{origin}: '{token}' takes the scan past {limit} addresses")]
TooManyAddresses {
origin: Origin,
token: String,
limit: u128,
},
#[error("{origin}: {source}")]
Target {
origin: Origin,
token: String,
#[source]
source: TargetParseError,
},
#[error("{origin}: not valid {format}: {message}")]
Malformed {
format: &'static str,
origin: Origin,
message: String,
},
}
pub trait TargetSink {
fn accept(&mut self, token: &str, origin: Origin) -> Result<(), ImportError>;
}
pub trait Importer {
fn import(&self, input: &mut dyn BufRead, sink: &mut dyn TargetSink)
-> Result<(), ImportError>;
}
#[derive(Debug)]
pub struct TargetCollector<'a> {
builder: TargetMapBuilder,
options: ImportOptions<'a>,
refusals: Vec<Refusal>,
tokens: u64,
gross_addresses: u128,
}
impl<'a> TargetCollector<'a> {
pub fn new(options: ImportOptions<'a>) -> Self {
Self {
builder: TargetMapBuilder::new(options.default_ports.clone()),
options,
refusals: Vec::new(),
tokens: 0,
gross_addresses: 0,
}
}
pub fn finish(mut self) -> Imported {
let addresses = self.builder.address_count();
Imported {
map: self.builder.build(),
refusals: self.refusals,
tokens: self.tokens,
addresses,
}
}
}
impl TargetSink for TargetCollector<'_> {
fn accept(&mut self, token: &str, origin: Origin) -> Result<(), ImportError> {
self.tokens = self.tokens.saturating_add(1);
if self.tokens > self.options.limits.max_tokens {
return Err(ImportError::TooManyTokens {
limit: self.options.limits.max_tokens,
});
}
let before = self.builder.gross_address_count();
match self.builder.push(token, &self.options.context) {
Ok(()) => {}
Err(reason) => {
return match self.options.on_refusal {
OnRefusal::Abort => Err(ImportError::Target {
origin,
token: token.trim().to_string(),
source: reason,
}),
OnRefusal::Collect => {
self.refusals.push(Refusal {
origin,
token: token.trim().to_string(),
reason,
});
Ok(())
}
};
}
}
let added = self.builder.gross_address_count().saturating_sub(before);
self.gross_addresses = self.gross_addresses.saturating_add(added);
if self.gross_addresses > self.options.limits.max_addresses {
return Err(ImportError::TooManyAddresses {
origin,
token: token.trim().to_string(),
limit: self.options.limits.max_addresses,
});
}
Ok(())
}
}
#[non_exhaustive]
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum ImportFormat {
List,
#[cfg(feature = "import-csv")]
Csv,
#[cfg(feature = "import-json")]
Json,
#[cfg(feature = "import-json")]
JsonLines,
#[cfg(feature = "import-nmap")]
NmapXml,
}
impl ImportFormat {
pub fn from_extension(extension: &str) -> Option<Self> {
match extension.to_ascii_lowercase().as_str() {
"txt" | "list" | "lst" => Some(ImportFormat::List),
#[cfg(feature = "import-csv")]
"csv" => Some(ImportFormat::Csv),
#[cfg(feature = "import-json")]
"json" => Some(ImportFormat::Json),
#[cfg(feature = "import-json")]
"jsonl" | "ndjson" => Some(ImportFormat::JsonLines),
#[cfg(feature = "import-nmap")]
"xml" => Some(ImportFormat::NmapXml),
_ => None,
}
}
pub fn sniff(input: &mut dyn BufRead) -> Result<Self, ImportError> {
const UTF8_BOM: [u8; 3] = [0xEF, 0xBB, 0xBF];
#[cfg(feature = "import-json")]
const REPORT_TAG: &[u8] = br#""type":"report""#;
let buffered = input.fill_buf()?;
let prefix = buffered.strip_prefix(&UTF8_BOM).unwrap_or(buffered);
let prefix = prefix.trim_ascii_start();
let _ = &prefix;
#[cfg(feature = "import-csv")]
{
let header = crate::format::csv::COLUMNS.join(",");
let overlap = prefix.len().min(header.len());
if overlap >= 16 && prefix[..overlap] == header.as_bytes()[..overlap] {
return Ok(ImportFormat::Csv);
}
}
#[cfg(feature = "import-nmap")]
if prefix.first() == Some(&b'<') {
return Ok(ImportFormat::NmapXml);
}
#[cfg(feature = "import-json")]
if prefix.first() == Some(&b'{') {
let head = &prefix[..prefix.len().min(256)];
let tagged = head
.windows(REPORT_TAG.len())
.any(|window| window == REPORT_TAG);
return Ok(if tagged {
ImportFormat::JsonLines
} else {
ImportFormat::Json
});
}
Ok(ImportFormat::List)
}
pub fn resolve(path: Option<&Path>, input: &mut dyn BufRead) -> Result<Self, ImportError> {
match path.and_then(Self::from_path) {
Some(format) => Ok(format),
None => Self::sniff(input),
}
}
pub fn from_path(path: &Path) -> Option<Self> {
path.extension()
.and_then(|extension| extension.to_str())
.and_then(Self::from_extension)
}
pub fn extension(self) -> &'static str {
match self {
ImportFormat::List => "txt",
#[cfg(feature = "import-csv")]
ImportFormat::Csv => "csv",
#[cfg(feature = "import-json")]
ImportFormat::Json => "json",
#[cfg(feature = "import-json")]
ImportFormat::JsonLines => "jsonl",
#[cfg(feature = "import-nmap")]
ImportFormat::NmapXml => "xml",
}
}
pub fn all() -> &'static [ImportFormat] {
&[
ImportFormat::List,
#[cfg(feature = "import-csv")]
ImportFormat::Csv,
#[cfg(feature = "import-json")]
ImportFormat::Json,
#[cfg(feature = "import-json")]
ImportFormat::JsonLines,
#[cfg(feature = "import-nmap")]
ImportFormat::NmapXml,
]
}
pub fn importer(self, options: &ImportOptions<'_>) -> Box<dyn Importer> {
match self {
ImportFormat::List => Box::new(ListImporter::new(options.limits)),
#[cfg(feature = "import-csv")]
ImportFormat::Csv => Box::new(CsvImporter::new(options.limits)),
#[cfg(feature = "import-json")]
ImportFormat::Json => Box::new(JsonImporter::new(options.limits)),
#[cfg(feature = "import-json")]
ImportFormat::JsonLines => Box::new(JsonLinesImporter::new(options.limits)),
#[cfg(feature = "import-nmap")]
ImportFormat::NmapXml => Box::new(NmapXmlImporter::new(options.limits)),
}
}
pub fn read(
self,
input: &mut dyn BufRead,
options: &ImportOptions<'_>,
) -> Result<Imported, ImportError> {
let importer = self.importer(options);
let mut collector = TargetCollector::new(options.clone());
importer.import(input, &mut collector)?;
Ok(collector.finish())
}
}
impl fmt::Display for ImportFormat {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(match self {
ImportFormat::List => "list",
#[cfg(feature = "import-csv")]
ImportFormat::Csv => "csv",
#[cfg(feature = "import-json")]
ImportFormat::Json => "json",
#[cfg(feature = "import-json")]
ImportFormat::JsonLines => "jsonl",
#[cfg(feature = "import-nmap")]
ImportFormat::NmapXml => "nmap-xml",
})
}
}
pub fn read_from(
path: &Path,
input: &mut dyn BufRead,
options: &ImportOptions<'_>,
) -> Option<Result<Imported, ImportError>> {
let format = ImportFormat::from_path(path)?;
Some(format.read(input, options))
}
#[cfg(test)]
mod tests {
use super::*;
use std::io::Cursor;
fn options(ports: &str) -> ImportOptions<'static> {
ImportOptions::new(PortSet::try_from(ports).expect("test ports parse"))
}
fn read(input: &str, options: &ImportOptions<'_>) -> Result<Imported, ImportError> {
ImportFormat::List.read(&mut Cursor::new(input), options)
}
#[test]
fn every_advertised_format_resolves_from_its_own_extension() {
for format in ImportFormat::all() {
assert_eq!(
ImportFormat::from_extension(format.extension()),
Some(*format),
"{format} does not resolve from its own extension"
);
}
assert_eq!(ImportFormat::from_path(Path::new("/tmp/targets")), None);
assert_eq!(ImportFormat::from_extension("pdf"), None);
}
#[test]
fn reading_by_path_matches_reading_by_format() {
let opts = options("80");
let mut input = Cursor::new("10.0.0.1\n");
let imported = read_from(Path::new("targets.txt"), &mut input, &opts)
.expect("the extension names a format")
.expect("the import succeeds");
assert_eq!(imported.addresses, 1);
assert!(
read_from(Path::new("targets.pdf"), &mut Cursor::new(""), &opts).is_none(),
"an unsupported extension must not quietly produce targets"
);
}
#[test]
fn a_refused_expression_aborts_and_names_its_line() {
let err = read("10.0.0.1\n10.0.0.300\n10.0.0.2\n", &options("80"))
.expect_err("the second line is not an address");
match err {
ImportError::Target { origin, token, .. } => {
assert_eq!(origin, Origin::line(2));
assert_eq!(token, "10.0.0.300");
}
other => panic!("expected a refused target, got {other:?}"),
}
}
#[test]
fn collecting_keeps_the_good_targets_and_reports_the_bad_ones() {
let opts = options("80").with_refusal_policy(OnRefusal::Collect);
let imported = read("10.0.0.1\n10.0.0.300\nnot-an-address\n10.0.0.2\n", &opts)
.expect("collecting does not fail the import");
assert_eq!(imported.addresses, 2, "both good targets survived");
assert_eq!(imported.refusals.len(), 2);
assert_eq!(imported.refusals[0].origin, Origin::line(2));
assert_eq!(imported.refusals[1].token, "not-an-address");
assert_eq!(imported.tokens, 4, "refused expressions are still counted");
}
#[test]
fn a_range_past_the_address_limit_is_refused() {
let err = read("::/0\n", &options("80")).expect_err("the whole of IPv6 is not a scan");
assert!(matches!(err, ImportError::TooManyAddresses { .. }));
let imported = read("0.0.0.0/0\n", &options("80")).expect("the whole of IPv4 is a scan");
assert_eq!(imported.addresses, 1u128 << 32);
}
#[test]
fn limits_can_be_lifted_and_tightened() {
let permissive = options("80").with_limits(ImportLimits::none());
assert!(read("::/0\n", &permissive).is_ok());
let strict = options("80").with_limits(ImportLimits {
max_addresses: 100,
..ImportLimits::default()
});
assert!(matches!(
read("10.0.0.0/24\n", &strict),
Err(ImportError::TooManyAddresses { .. })
));
let few = options("80").with_limits(ImportLimits {
max_tokens: 2,
..ImportLimits::default()
});
assert!(matches!(
read("10.0.0.1\n10.0.0.2\n10.0.0.3\n", &few),
Err(ImportError::TooManyTokens { limit: 2 })
));
}
#[test]
fn the_reported_address_count_merges_overlapping_targets() {
let imported = read("10.0.0.0/24\n10.0.0.5\n", &options("80")).expect("imports");
assert_eq!(imported.addresses, 256, "the same block, named twice");
assert_eq!(imported.tokens, 2);
}
#[test]
fn a_hand_written_list_of_addresses_feeds_both_entry_points() {
let list = "\
192.168.0.1
192.168.0.100
192.168.0.20
192.168.0.53
192.168.0.151
";
let imported = read(list, &options("22,80")).expect("a list of addresses imports");
assert_eq!(imported.tokens, 5);
assert_eq!(imported.addresses, 5);
assert_eq!(imported.refusals.len(), 0);
assert_eq!(imported.map.units.len(), 1);
let map = imported.map.clone();
assert_eq!(map.gross_targets().unwrap(), 10);
let targets = imported.into_ip_set();
assert_eq!(targets.len(), 5);
for address in ["192.168.0.1", "192.168.0.20", "192.168.0.151"] {
assert!(
targets.contains(&address.parse().unwrap()),
"{address} did not survive"
);
}
}
#[test]
fn converting_to_addresses_merges_what_the_units_kept_apart() {
let imported =
read("10.0.0.1:22\n10.0.0.1:443\n10.0.0.2:22\n", &options("80")).expect("imports");
assert_eq!(imported.map.units.len(), 2, "two port specifications");
assert_eq!(imported.addresses, 3, "counted once per unit");
assert_eq!(
imported.into_ip_set().len(),
2,
"but only two hosts to sweep"
);
}
#[test]
fn sniffing_leaves_the_input_where_it_found_it() {
let file = "10.0.0.1\n10.0.0.2\n10.0.0.3\n";
let mut input = Cursor::new(file);
let format = ImportFormat::sniff(&mut input).expect("sniffs");
assert_eq!(format, ImportFormat::List);
let imported = format.read(&mut input, &options("80")).expect("imports");
assert_eq!(imported.addresses, 3, "sniffing consumed part of the input");
}
#[test]
fn an_ambiguous_document_is_read_as_a_list() {
for file in [
"[2001:db8::1]:443\n",
"192.168.1.1,192.168.1.2\n",
"Server,Location\nweb01,rack 4\n",
"",
"# just a comment\n",
] {
assert_eq!(
ImportFormat::sniff(&mut Cursor::new(file)).expect("sniffs"),
ImportFormat::List,
"{file:?}"
);
}
}
#[cfg(all(feature = "import-csv", feature = "export-csv"))]
#[test]
fn a_report_this_engine_wrote_is_recognised_and_read_back() {
use crate::export::{CsvExporter, ExportOptions, Exporter};
let report = crate::export::fixture::report();
let mut document = Vec::new();
CsvExporter::new(ExportOptions::new())
.export(&report, &mut document)
.expect("the fixture exports");
let mut input = Cursor::new(document);
let format = ImportFormat::sniff(&mut input).expect("sniffs");
assert_eq!(
format,
ImportFormat::Csv,
"this crate must recognise its own CSV"
);
let imported = format.read(&mut input, &options("80")).expect("imports");
assert!(
imported.addresses > 0,
"a report with hosts in it read back as no targets"
);
assert_eq!(
imported.refusals.len(),
0,
"every row of our own output has to parse as a target"
);
}
#[cfg(all(
feature = "import-json",
feature = "export-json",
feature = "export-jsonl"
))]
#[test]
fn the_two_report_formats_are_told_apart_by_what_they_call_themselves() {
use crate::export::{ExportOptions, Exporter, JsonExporter, JsonLinesExporter};
let report = crate::export::fixture::report();
for (exporter, expected) in [
(
Box::new(JsonExporter::new(ExportOptions::new())) as Box<dyn Exporter>,
ImportFormat::Json,
),
(
Box::new(JsonExporter::new(ExportOptions::new()).compact()),
ImportFormat::Json,
),
(
Box::new(JsonLinesExporter::new(ExportOptions::new())),
ImportFormat::JsonLines,
),
] {
let mut document = Vec::new();
exporter.export(&report, &mut document).expect("exports");
assert_eq!(
ImportFormat::sniff(&mut Cursor::new(document)).expect("sniffs"),
expected,
);
}
}
#[test]
fn a_path_decides_the_format_and_a_silent_one_defers_to_the_input() {
let mut input = Cursor::new("10.0.0.1\n");
assert_eq!(
ImportFormat::resolve(Some(Path::new("scope.txt")), &mut input).unwrap(),
ImportFormat::List
);
assert_eq!(
ImportFormat::resolve(Some(Path::new("scope.dat")), &mut input).unwrap(),
ImportFormat::List,
"an extension that names no format is not an error"
);
assert_eq!(
ImportFormat::resolve(None, &mut input).unwrap(),
ImportFormat::List
);
}
}