use std::collections::HashSet;
use std::fs;
use std::path::{Path, PathBuf};
use std::time::{Duration, SystemTime};
use super::cursor::Checkpoint;
use super::file::{
append_existing, claim_directory_for_invoking_user, create_private as create_private_file,
create_private_directory, exists, kinds, names, open_existing, open_to_read,
remove as remove_file, remove_directory, replace,
};
use super::format::JournalError;
use super::lock::{Lock, LockRefused, LockState};
use super::manifest::{JobOptions, JournalManifest, Plan, PlanChanged};
use super::ownership::Kind;
use super::settle::Settlements;
use crate::detect::compute::{DetectionLine, DetectionRunRecord, PortRunsRecord};
use crate::model::host::Host;
use crate::model::ip::scoped::ScopedIp;
use crate::model::port::Protocol;
use crate::record::{HostRecord, PhaseRecord};
use crate::report::{ScanKind, ScanPhase, ScanReport, Unheard};
use crate::system::privilege::Privilege;
const MANIFEST: &str = "manifest.json";
const CURSOR: &str = "cursor.json";
const HOSTS: &str = "hosts.jsonl";
const PHASES: &str = "phases.jsonl";
const DETECTIONS: &str = "detections.jsonl";
const FINISHED: &str = "finished.jsonl";
const LOCK: &str = "LOCK";
const SITTING: &str = "sitting-";
const OPTIONS: &str = "options.json";
#[non_exhaustive]
#[derive(Debug, thiserror::Error)]
pub enum OpenError {
#[error("{0}")]
PlanChanged(#[from] PlanChanged),
#[error("this journal records a {held} and cannot be continued as a {asked}")]
WrongPhase {
held: &'static str,
asked: &'static str,
},
#[error(
"this journal was written in format {found} and this build continues \
format {understood}; its findings still read, but it cannot be resumed"
)]
VersionTooOld {
found: u32,
understood: u32,
},
#[error("{0}")]
Locked(LockRefused),
#[error("{0}")]
Journal(#[from] JournalError),
}
impl From<LockRefused> for OpenError {
fn from(refused: LockRefused) -> Self {
OpenError::Locked(refused)
}
}
#[derive(Debug)]
pub struct Journal {
directory: PathBuf,
manifest: JournalManifest,
lock: Lock,
resume_point: Checkpoint,
restored: Vec<Host>,
earlier: Vec<ScanPhase>,
created: bool,
options: Option<JobOptions>,
written: Written,
length: u64,
compact_after: u64,
sitting: PathBuf,
}
impl Journal {
pub fn create(
root: &Path,
plan: &Plan,
privilege: Privilege,
summary: impl Into<String>,
) -> Result<Self, OpenError> {
let (id, directory) = claim_directory(root)?;
let manifest = JournalManifest::new(id, plan, privilege, summary);
match Self::furnish(&directory, manifest) {
Ok(journal) => Ok(journal),
Err(error) => {
let _ = remove_directory(&directory);
Err(error)
}
}
}
fn furnish(directory: &Path, manifest: JournalManifest) -> Result<Self, OpenError> {
write_private(
&directory.join(MANIFEST),
&serde_json::to_vec(&manifest).map_err(JournalError::json)?,
)?;
let lock = Lock::acquire(&directory.join(LOCK))?;
let sitting = sitting_file(directory, &lock);
let mut journal = Self {
directory: directory.to_path_buf(),
manifest,
lock,
resume_point: Checkpoint::default(),
restored: Vec::new(),
earlier: Vec::new(),
created: true,
options: None,
written: Written::default(),
length: 0,
compact_after: 0,
sitting,
};
journal.open_findings()?;
journal.length = findings_length(directory)?;
Ok(journal)
}
pub fn resume(
directory: &Path,
plan: &Plan,
privilege: Privilege,
) -> Result<(Self, Checkpoint), OpenError> {
Self::resume_locking(directory, plan, privilege, Lock::acquire)
}
fn resume_locking(
directory: &Path,
plan: &Plan,
privilege: Privilege,
lock: fn(&Path) -> Result<Lock, LockRefused>,
) -> Result<(Self, Checkpoint), OpenError> {
let manifest = read_manifest(directory)?;
if manifest.journal_version < super::JOURNAL_VERSION {
return Err(OpenError::VersionTooOld {
found: manifest.journal_version,
understood: super::JOURNAL_VERSION,
});
}
if manifest.kind() != plan.kind() {
return Err(OpenError::WrongPhase {
held: phase_name(manifest.kind()),
asked: phase_name(plan.kind()),
});
}
manifest.covers(plan, privilege)?;
let lock = lock(&directory.join(LOCK))?;
let sitting = sitting_file(directory, &lock);
let checkpoint = read_checkpoint(directory)?;
let earlier = read_phases(directory)?;
let unheard = Unheard::decided(&earlier);
let mut restored = read_findings(directory)?;
restored.retain(|host| !unheard.drops(host));
let length = findings_length(directory)?;
let written = Written::holding(&restored, length)?;
Ok((
Self {
directory: directory.to_path_buf(),
manifest,
lock,
resume_point: checkpoint.clone(),
restored,
earlier,
created: false,
options: read_options(directory)?,
written,
length,
compact_after: 0,
sitting,
},
checkpoint,
))
}
pub fn reopen(
directory: &Path,
privilege: Privilege,
) -> Result<(Self, Checkpoint, Plan), OpenError> {
let plan = read_manifest(directory)?.recorded();
let (journal, checkpoint) = Self::resume(directory, &plan, privilege)?;
Ok((journal, checkpoint, plan))
}
pub fn take_over(
directory: &Path,
privilege: Privilege,
) -> Result<(Self, Checkpoint, Plan), OpenError> {
let plan = read_manifest(directory)?.recorded();
let (journal, checkpoint) =
Self::resume_locking(directory, &plan, privilege, Lock::take_over)?;
Ok((journal, checkpoint, plan))
}
pub fn restored(&self) -> &[Host] {
&self.restored
}
pub fn record_hosts(&mut self, hosts: &[Host]) -> Result<(), JournalError> {
let mut records = Vec::with_capacity(hosts.len());
for host in hosts {
records.extend(self.written.delta(host)?);
}
if records.is_empty() {
return Ok(());
}
let file = open_for_append(&self.directory.join(HOSTS))?;
let measured = file.try_clone()?;
let mut deltas = Vec::with_capacity(records.len());
let mut writer = crate::journal::format::Writer::append(std::io::BufWriter::new(file));
for (record, delta) in records {
writer.write(&record)?;
deltas.push(delta);
}
writer.flush()?;
drop(writer);
for delta in deltas {
self.written.update(delta);
}
self.length = measured.metadata()?.len();
Ok(())
}
pub fn record_detections(&mut self, runs: &[DetectionRunRecord]) -> Result<(), JournalError> {
if runs.is_empty() {
return Ok(());
}
let path = self.directory.join(DETECTIONS);
let (file, mut writer) = match open_for_append(&path) {
Ok(file) => {
let written = file.try_clone()?;
let writer =
crate::journal::format::Writer::append(std::io::BufWriter::new(written));
(file, writer)
}
Err(JournalError::Io(error)) if error.kind() == std::io::ErrorKind::NotFound => {
let file = create_private_file(&path)?;
let written = file.try_clone()?;
let writer =
crate::journal::format::Writer::create(std::io::BufWriter::new(written))?;
(file, writer)
}
Err(error) => return Err(error),
};
writer.flush()?;
let began = file.metadata()?.len();
let outcome = PortRunsRecord::grouping(runs)
.iter()
.try_for_each(|line| writer.write(line))
.and_then(|()| writer.flush());
if outcome.is_err() {
drop(writer);
let _ = file.set_len(began);
}
outcome
}
pub(crate) fn record_finished(
&mut self,
hosts: impl IntoIterator<Item = String>,
) -> Result<(), JournalError> {
let record = FinishedRecord {
hosts: hosts.into_iter().collect(),
};
if record.hosts.is_empty() {
return Ok(());
}
let path = self.directory.join(FINISHED);
let mut writer = match open_for_append(&path) {
Ok(file) => crate::journal::format::Writer::append(std::io::BufWriter::new(file)),
Err(JournalError::Io(error)) if error.kind() == std::io::ErrorKind::NotFound => {
let file = create_private_file(&path)?;
crate::journal::format::Writer::create(std::io::BufWriter::new(file))?
}
Err(error) => return Err(error),
};
writer.write(&record)?;
writer.flush()
}
pub(crate) fn finished_hosts(&self) -> Result<HashSet<String>, JournalError> {
let file = match open_to_read(&self.directory.join(FINISHED)) {
Ok(file) => file,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(HashSet::new()),
Err(e) => return Err(e.into()),
};
let mut reader = match crate::journal::format::Reader::open(std::io::BufReader::new(file)) {
Ok(reader) => reader,
Err(JournalError::NotAJournal) => return Ok(HashSet::new()),
Err(e) => return Err(e),
};
let mut hosts = HashSet::new();
while let Some(record) = reader.read::<FinishedRecord>()? {
hosts.extend(record.hosts);
}
Ok(hosts)
}
pub fn should_compact(&self) -> bool {
outgrown(self.length, self.written.superseded)
&& self.written.superseded >= self.compact_after
}
pub fn compact(&mut self, all: &[Host]) -> Result<(), JournalError> {
let compacted = self.write_whole(all);
if compacted.is_err() {
let superseded = self.written.superseded;
let live = self.length.saturating_sub(superseded);
self.compact_after = superseded.saturating_add(COMPACT_FLOOR.max(live));
}
compacted
}
fn write_whole(&mut self, all: &[Host]) -> Result<(), JournalError> {
let destination = self.directory.join(HOSTS);
let (written, length) = replace(
&destination,
&destination.with_extension("jsonl-tmp"),
|file| {
let measured = file.try_clone()?;
let mut written = Written::default();
let mut writer =
crate::journal::format::Writer::create(std::io::BufWriter::new(file))?;
for host in all {
if let Some((record, delta)) = Written::default().delta(host)? {
writer.write(&record)?;
written.update(delta);
}
}
writer.flush()?;
drop(writer);
Ok::<_, JournalError>((written, measured.metadata()?.len()))
},
)?;
self.written = written;
self.length = length;
self.compact_after = 0;
Ok(())
}
pub fn earlier_phases(&self) -> &[ScanPhase] {
&self.earlier
}
pub fn record_phases(&mut self, phases: &[ScanPhase]) -> Result<(), JournalError> {
if phases.is_empty() {
return Ok(());
}
let file = open_for_append(&self.directory.join(PHASES))?;
let mut writer = crate::journal::format::Writer::append(std::io::BufWriter::new(file));
for phase in phases {
writer.write(&PhaseRecord::from(phase))?;
}
writer.flush()
}
pub(crate) fn record_standing(&mut self, phases: &[ScanPhase]) -> Result<(), JournalError> {
if phases.is_empty() {
return Ok(());
}
replace(
&self.sitting,
&self.sitting.with_extension("jsonl-tmp"),
|file| {
let mut writer =
crate::journal::format::Writer::create(std::io::BufWriter::new(file))?;
for phase in phases {
writer.write(&PhaseRecord::from(phase))?;
}
writer.flush()
},
)
}
pub(crate) fn end_sitting(&mut self, phases: &[ScanPhase]) -> Result<(), JournalError> {
if phases.is_empty() {
return Ok(());
}
self.record_phases(phases)?;
match remove_file(&self.sitting) {
Ok(()) => Ok(()),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),
Err(e) => Err(e.into()),
}
}
fn open_findings(&mut self) -> Result<(), JournalError> {
for name in [HOSTS, PHASES] {
let path = self.directory.join(name);
let file = create_private_file(&path)?;
crate::journal::format::Writer::create(std::io::BufWriter::new(file))?.flush()?;
}
Ok(())
}
pub fn resume_point(&self) -> &Checkpoint {
&self.resume_point
}
pub fn checkpoint(&mut self, settlements: &Settlements) -> Result<(), JournalError> {
self.write_cursor(&settlements.checkpoint())
}
pub(crate) fn write_cursor(&mut self, cursor: &Checkpoint) -> Result<(), JournalError> {
cursor.write_atomically(&self.directory.join(CURSOR))?;
self.lock.beat()
}
pub fn record(
&mut self,
hosts: &[Host],
settlements: &Settlements,
) -> Result<(), JournalError> {
self.record_hosts(hosts)?;
self.checkpoint(settlements)
}
pub fn options(&self) -> Option<&JobOptions> {
self.options.as_ref()
}
pub(crate) fn record_options(&mut self, options: JobOptions) -> Result<(), JournalError> {
if self.options.is_some() || !self.is_untouched() {
return Ok(());
}
write_private(
&self.directory.join(OPTIONS),
&serde_json::to_vec(&options).map_err(JournalError::json)?,
)?;
let numbered = self
.manifest
.recorded()
.numbered_targets(&options.excluded_ports());
self.options = Some(options);
if self.created && numbered != self.manifest.total_targets {
self.manifest.total_targets = numbered;
write_private(
&self.directory.join(MANIFEST),
&serde_json::to_vec(&self.manifest).map_err(JournalError::json)?,
)?;
}
Ok(())
}
fn is_untouched(&self) -> bool {
self.earlier.is_empty()
&& self.restored.is_empty()
&& !exists(&self.directory.join(CURSOR)).unwrap_or(true)
}
pub fn manifest(&self) -> &JournalManifest {
&self.manifest
}
pub fn directory(&self) -> &Path {
&self.directory
}
pub(crate) fn withdraw(self) {
if self.created && self.is_untouched() {
let _ = remove_directory(&self.directory);
}
}
pub fn close(self) -> Result<(), JournalError> {
self.lock.release()
}
}
fn outgrown(length: u64, superseded: u64) -> bool {
superseded > COMPACT_FLOOR.max(length.saturating_sub(superseded))
}
const COMPACT_FLOOR: u64 = 4 * 1024 * 1024;
#[derive(Debug, Default)]
struct Written {
hosts: std::collections::HashMap<ScopedIp, HeldHost>,
superseded: u64,
}
#[derive(Debug, Default)]
struct HeldHost {
rest: Option<Mark>,
ports: std::collections::HashMap<PortKey, Mark>,
}
type PortKey = (u16, Protocol);
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
struct Mark {
digest: u64,
length: u32,
}
#[derive(Debug)]
struct Delta {
key: ScopedIp,
rest: Mark,
ports: Vec<(PortKey, Mark)>,
}
impl Delta {
fn length(&self) -> u64 {
u64::from(self.rest.length)
+ self
.ports
.iter()
.map(|(_, mark)| u64::from(mark.length))
.sum::<u64>()
}
}
impl Written {
fn holding(hosts: &[Host], length: u64) -> Result<Self, JournalError> {
let mut written = Self::default();
let mut live = 0;
for host in hosts {
if let Some((_, delta)) = Self::default().delta(host)? {
live += delta.length();
written.update(delta);
}
}
written.superseded = length.saturating_sub(live);
Ok(written)
}
fn delta(&self, host: &Host) -> Result<Option<(HostRecord, Delta)>, JournalError> {
let held = self.hosts.get(&host.scoped_ip());
let mut record = HostRecord::from(host);
let ports = std::mem::take(&mut record.ports);
let rest = mark(&record)?;
let mut changed = Vec::new();
for (port, written) in host.ports().zip(ports) {
debug_assert_eq!(
port.number(),
written.port,
"a record lists its host's ports"
);
let key = (port.number(), port.protocol());
let mark = mark(&written)?;
let unchanged = held
.and_then(|held| held.ports.get(&key))
.is_some_and(|was| was.digest == mark.digest);
if !unchanged {
changed.push((key, mark));
record.ports.push(written);
}
}
let unchanged = changed.is_empty()
&& held.and_then(|held| held.rest).map(|was| was.digest) == Some(rest.digest);
if unchanged {
return Ok(None);
}
let delta = Delta {
key: host.scoped_ip(),
rest,
ports: changed,
};
Ok(Some((record, delta)))
}
fn update(&mut self, delta: Delta) {
let held = match self.hosts.entry(delta.key) {
std::collections::hash_map::Entry::Occupied(slot) => {
let held = slot.into_mut();
self.superseded += held.rest.map_or(0, |was| u64::from(was.length));
held
}
std::collections::hash_map::Entry::Vacant(slot) => slot.insert(HeldHost::default()),
};
held.rest = Some(delta.rest);
for (key, mark) in delta.ports {
if let Some(was) = held.ports.insert(key, mark) {
self.superseded += u64::from(was.length);
}
}
}
}
fn mark(record: &impl serde::Serialize) -> Result<Mark, JournalError> {
use std::hash::Hasher;
struct Hashing {
hasher: std::collections::hash_map::DefaultHasher,
length: u64,
}
impl std::io::Write for Hashing {
fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
self.hasher.write(bytes);
self.length += bytes.len() as u64;
Ok(bytes.len())
}
fn flush(&mut self) -> std::io::Result<()> {
Ok(())
}
}
let mut hashing = Hashing {
hasher: std::collections::hash_map::DefaultHasher::new(),
length: 0,
};
serde_json::to_writer(&mut hashing, record).map_err(JournalError::json)?;
Ok(Mark {
digest: hashing.hasher.finish(),
length: u32::try_from(hashing.length + 1).unwrap_or(u32::MAX),
})
}
#[non_exhaustive]
#[derive(Debug, Clone)]
pub struct Entry {
pub directory: PathBuf,
pub manifest: JournalManifest,
pub checkpoint: Option<Checkpoint>,
pub lock: LockState,
}
impl Entry {
pub fn new(
directory: PathBuf,
manifest: JournalManifest,
checkpoint: Option<Checkpoint>,
lock: LockState,
) -> Self {
Self {
directory,
manifest,
checkpoint,
lock,
}
}
pub fn is_complete(&self) -> bool {
if self.kind() == ScanKind::Listen {
return false;
}
self.settled()
.is_some_and(|settled| settled >= self.manifest.total_targets)
}
pub fn kind(&self) -> ScanKind {
self.manifest.kind()
}
pub fn settled(&self) -> Option<u128> {
self.checkpoint
.as_ref()
.map(|checkpoint| u128::from(checkpoint.settled_count()))
}
}
pub fn prepare_root(root: &Path) -> std::io::Result<()> {
let own = super::paths::root().as_deref() == Some(root);
#[cfg(unix)]
let home = super::ownership::invoking().map(|user| user.home.as_path());
#[cfg(not(unix))]
let home = None;
prepare_root_with(root, own, home, |path, hand| match hand {
Hand::Give => claim_directory_for_invoking_user(path),
Hand::Reclaim => super::ownership::reclaim(path),
})
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum Hand {
Give,
Reclaim,
}
fn prepare_root_with(
root: &Path,
own: bool,
home: Option<&Path>,
mut claim: impl FnMut(&Path, Hand),
) -> std::io::Result<()> {
let created = super::ownership::create_missing(root, None)?;
if own && let Some(above) = root.parent() {
let mut between: Vec<&Path> = above
.ancestors()
.skip(1)
.take_while(|directory| {
home.is_some_and(|home| *directory != home && directory.starts_with(home))
})
.collect();
between.reverse();
for directory in between {
let hand = if created.iter().any(|made| made == directory) {
Hand::Give
} else {
Hand::Reclaim
};
claim(directory, hand);
}
claim(above, Hand::Give);
}
claim(root, Hand::Give);
Ok(())
}
pub fn list(root: &Path) -> Result<Vec<Entry>, JournalError> {
let held = match kinds(root) {
Ok(held) => held,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()),
Err(e) => return Err(e.into()),
};
let mut found = Vec::new();
for (name, kind) in held {
let directory = root.join(&name);
match kind {
Kind::Directory => {}
Kind::Link => {
crate::warn!(
"{} not listed: a link, not a journal (not followed)",
name.to_string_lossy()
);
continue;
}
Kind::Other => continue,
}
let manifest = match read_manifest(&directory) {
Ok(manifest) => manifest,
Err(JournalError::Io(e)) if e.kind() == std::io::ErrorKind::NotFound => continue,
Err(JournalError::Io(e)) => {
crate::warn!("{} not listed: {e}", name.to_string_lossy());
continue;
}
Err(e) => {
crate::warn!("{} not listed: {e}", name.to_string_lossy());
continue;
}
};
found.push(Entry {
checkpoint: read_checkpoint(&directory).ok(),
lock: super::lock::inspect(&directory.join(LOCK)),
manifest,
directory,
});
}
found.sort_by_key(|entry| std::cmp::Reverse(entry.manifest.created_at));
Ok(found)
}
pub fn report(directory: &Path) -> Result<ScanReport, JournalError> {
let manifest = read_manifest(directory)?;
Ok(ScanReport::recorded(
manifest.engine_version,
read_phases(directory)?,
read_findings(directory)?,
))
}
fn phase_name(kind: ScanKind) -> &'static str {
match kind {
ScanKind::Discovery => "host-discovery sweep",
ScanKind::PortScan => "port scan",
ScanKind::Listen => "listening phase",
}
}
pub fn remove(directory: &Path) -> Result<(), OpenError> {
let state = super::lock::inspect(&directory.join(LOCK));
if !state.is_resumable() {
return Err(OpenError::Locked(LockRefused::Held(state)));
}
remove_directory(directory).map_err(JournalError::from)?;
Ok(())
}
#[derive(Debug, serde::Serialize, serde::Deserialize)]
struct FinishedRecord {
hosts: Vec<String>,
}
fn read_findings(directory: &Path) -> Result<Vec<Host>, JournalError> {
let file = match open_to_read(&directory.join(HOSTS)) {
Ok(file) => file,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()),
Err(e) => return Err(e.into()),
};
let mut reader = match crate::journal::format::Reader::open(std::io::BufReader::new(file)) {
Ok(reader) => reader,
Err(JournalError::NotAJournal) => return Ok(Vec::new()),
Err(e) => return Err(e),
};
let mut hosts: Vec<Option<Host>> = Vec::new();
let mut slot_of: std::collections::BTreeMap<ScopedIp, usize> =
std::collections::BTreeMap::new();
while let Some(record) = reader.read::<HostRecord>()? {
let host = Host::from(&record);
let mut matched: Vec<usize> = scoped_ips(&host)
.filter_map(|ip| slot_of.get(&ip).copied())
.collect();
matched.sort_unstable();
matched.dedup();
let slot = match matched.split_first() {
Some((&keep, absorb)) => {
for &other in absorb {
if let Some(other) = hosts[other].take() {
merge_into(&mut hosts, keep, other, Host::merge);
}
}
merge_into(&mut hosts, keep, host, Host::merge_later_account);
keep
}
None => {
hosts.push(Some(host));
hosts.len() - 1
}
};
let Some(settled) = hosts[slot].as_ref() else {
continue;
};
for ip in scoped_ips(settled) {
slot_of.insert(ip, slot);
}
}
Ok(hosts.into_iter().flatten().collect())
}
fn scoped_ips(host: &Host) -> impl Iterator<Item = ScopedIp> + '_ {
host.ips().iter().map(|ip| match host.zone() {
Some(zone) => ScopedIp::scoped(*ip, zone.clone()),
None => ScopedIp::unscoped(*ip),
})
}
fn merge_into(hosts: &mut [Option<Host>], slot: usize, host: Host, fold: fn(&mut Host, Host)) {
match hosts[slot].as_mut() {
Some(existing) => fold(existing, host),
None => hosts[slot] = Some(host),
}
}
fn findings_length(directory: &Path) -> Result<u64, JournalError> {
match open_to_read(&directory.join(HOSTS)) {
Ok(file) => Ok(file.metadata()?.len()),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(0),
Err(e) => Err(e.into()),
}
}
fn read_phases(directory: &Path) -> Result<Vec<ScanPhase>, JournalError> {
let file = match open_to_read(&directory.join(PHASES)) {
Ok(file) => file,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()),
Err(e) => return Err(e.into()),
};
let mut reader = match crate::journal::format::Reader::open(std::io::BufReader::new(file)) {
Ok(reader) => reader,
Err(JournalError::NotAJournal) => return Ok(Vec::new()),
Err(e) => return Err(e),
};
let mut phases = Vec::new();
while let Some(record) = reader.read::<PhaseRecord>()? {
phases.push(ScanPhase::from(&record));
}
for standing in standing_phases(directory) {
let recorded = phases.iter().any(|phase| {
phase.kind() == standing.kind() && phase.started_at() == standing.started_at()
});
if !recorded {
phases.push(standing);
}
}
phases.sort_by_key(ScanPhase::started_at);
Ok(phases)
}
fn sitting_file(directory: &Path, lock: &Lock) -> PathBuf {
let started = lock
.record()
.started_at
.duration_since(SystemTime::UNIX_EPOCH)
.unwrap_or_default()
.as_nanos();
directory.join(format!("{SITTING}{started:020}.jsonl"))
}
fn standing_phases(directory: &Path) -> Vec<ScanPhase> {
let Ok(held) = names(directory) else {
return Vec::new();
};
let mut files: Vec<PathBuf> = held
.into_iter()
.filter(|name| {
name.to_str()
.is_some_and(|name| name.starts_with(SITTING) && name.ends_with(".jsonl"))
})
.map(|name| directory.join(name))
.collect();
files.sort();
let mut phases = Vec::new();
for path in files {
let Ok(file) = open_to_read(&path) else {
continue;
};
let Ok(mut reader) = crate::journal::format::Reader::open(std::io::BufReader::new(file))
else {
continue;
};
while let Ok(Some(record)) = reader.read::<PhaseRecord>() {
phases.push(ScanPhase::from(&record));
}
}
phases
}
pub fn read_detections(directory: &Path) -> Result<Vec<DetectionRunRecord>, JournalError> {
let file = match open_to_read(&directory.join(DETECTIONS)) {
Ok(file) => file,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()),
Err(e) => return Err(e.into()),
};
let mut reader = match crate::journal::format::Reader::open(std::io::BufReader::new(file)) {
Ok(reader) => reader,
Err(JournalError::NotAJournal) => return Ok(Vec::new()),
Err(e) => return Err(e),
};
let mut runs = Vec::new();
while let Some(line) = reader.read::<DetectionLine>()? {
runs.extend(line.into_runs());
}
Ok(runs)
}
fn open_for_append(path: &Path) -> Result<fs::File, JournalError> {
mend(path)?;
Ok(append_existing(path)?)
}
fn mend(path: &Path) -> Result<(), JournalError> {
use std::io::{Read, Seek, SeekFrom};
let mut file = open_existing(path)?;
let length = file.metadata()?.len();
if length == 0 {
super::format::Writer::create(&mut file)?.flush()?;
return Ok(());
}
super::format::Reader::open(std::io::BufReader::new(&file))?;
file.seek(SeekFrom::End(-1))?;
let mut last = [0u8; 1];
file.read_exact(&mut last)?;
if last[0] == b'\n' {
return Ok(());
}
let keep = last_whole_line(&mut file, length)?;
file.set_len(keep)?;
if keep == 0 {
file.seek(SeekFrom::Start(0))?;
super::format::Writer::create(&mut file)?.flush()?;
}
Ok(())
}
fn last_whole_line(file: &mut fs::File, length: u64) -> Result<u64, JournalError> {
use std::io::{Read, Seek, SeekFrom};
const WINDOW: u64 = 8 * 1024;
let mut end = length;
while end > 0 {
let size = WINDOW.min(end);
let start = end - size;
file.seek(SeekFrom::Start(start))?;
let mut window = vec![0u8; size as usize];
file.read_exact(&mut window)?;
if let Some(at) = window.iter().rposition(|byte| *byte == b'\n') {
return Ok(start + at as u64 + 1);
}
end = start;
}
Ok(0)
}
fn read_options(directory: &Path) -> Result<Option<JobOptions>, JournalError> {
let text = match read_bounded(&directory.join(OPTIONS), "a journal's options") {
Ok(text) => text,
Err(JournalError::Io(e)) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None),
Err(e) => return Err(e),
};
Ok(Some(
serde_json::from_str(&text).map_err(JournalError::json)?,
))
}
fn read_manifest(directory: &Path) -> Result<JournalManifest, JournalError> {
let text = read_bounded(&directory.join(MANIFEST), "a journal manifest")?;
let manifest: JournalManifest = serde_json::from_str(&text).map_err(JournalError::json)?;
if manifest.journal_version > super::JOURNAL_VERSION {
return Err(JournalError::VersionTooNew {
found: manifest.journal_version,
understood: super::JOURNAL_VERSION,
});
}
Ok(manifest)
}
fn read_checkpoint(directory: &Path) -> Result<Checkpoint, JournalError> {
match Checkpoint::read(&directory.join(CURSOR)) {
Ok(checkpoint) => Ok(checkpoint),
Err(JournalError::Io(e)) if e.kind() == std::io::ErrorKind::NotFound => {
Ok(Checkpoint::default())
}
Err(e) => Err(e),
}
}
const ALPHABET: &[u8; 32] = b"0123456789ABCDEFGHJKMNPQRSTVWXYZ";
const ID_CHARS: usize = 16;
const ID_TIME_BITS: u32 = 48;
fn mint_id() -> String {
let millis = SystemTime::now()
.duration_since(SystemTime::UNIX_EPOCH)
.map_or(0, |since| since.as_millis())
& ((1u128 << ID_TIME_BITS) - 1);
let random_bits = ID_CHARS as u32 * 5 - ID_TIME_BITS;
let entropy = u128::from(rand::random::<u64>()) & ((1u128 << random_bits) - 1);
let mut value = (millis << random_bits) | entropy;
let mut out = [b'0'; ID_CHARS];
for slot in out.iter_mut().rev() {
*slot = ALPHABET[(value & 0x1F) as usize];
value >>= 5;
}
String::from_utf8(out.to_vec()).expect("base32 alphabet is ASCII")
}
fn claim_directory(root: &Path) -> Result<(String, PathBuf), JournalError> {
const ATTEMPTS: usize = 8;
for _ in 0..ATTEMPTS {
let id = mint_id();
let directory = root.join(&id);
match create_private_directory(&directory) {
Ok(()) => {
claim_directory_for_invoking_user(&directory);
return Ok((id, directory));
}
Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => continue,
Err(e) => return Err(e.into()),
}
}
Err(std::io::Error::new(
std::io::ErrorKind::AlreadyExists,
"could not find an unused id for this scan",
)
.into())
}
pub(super) fn read_bounded(path: &Path, what: &str) -> Result<String, JournalError> {
use std::io::Read;
let mut text = String::new();
let read = open_to_read(path)?
.take(super::format::MAX_READ_BYTES + 1)
.read_to_string(&mut text)?;
if read as u64 > super::format::MAX_READ_BYTES {
return Err(super::format::too_large(what));
}
Ok(text)
}
fn write_private(path: &Path, bytes: &[u8]) -> Result<(), JournalError> {
use std::io::Write;
replace(path, &path.with_extension("tmp"), |mut file| {
file.write_all(bytes)
})?;
Ok(())
}
#[non_exhaustive]
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Retention {
pub completed_for: Option<Duration>,
pub incomplete_for: Option<Duration>,
pub keep_at_most: Option<usize>,
}
impl Default for Retention {
fn default() -> Self {
Self {
completed_for: Some(Duration::from_secs(30 * 24 * 60 * 60)),
incomplete_for: None,
keep_at_most: Some(200),
}
}
}
impl Retention {
pub fn keep_everything() -> Self {
Self {
completed_for: None,
incomplete_for: None,
keep_at_most: None,
}
}
pub fn expired(&self, entries: &[Entry], now: SystemTime) -> Vec<usize> {
let age_of = |entry: &Entry| {
now.duration_since(entry.manifest.created_at)
.unwrap_or_default()
};
let mut removing: Vec<usize> = entries
.iter()
.enumerate()
.filter(|(_, entry)| entry.lock.is_resumable())
.filter(|(_, entry)| {
let limit = if entry.is_complete() {
self.completed_for
} else {
self.incomplete_for
};
limit.is_some_and(|limit| age_of(entry) > limit)
})
.map(|(index, _)| index)
.collect();
let Some(cap) = self.keep_at_most else {
return removing;
};
let mut surviving: Vec<usize> = (0..entries.len())
.filter(|index| !removing.contains(index))
.collect();
surviving.sort_by_key(|&index| {
let entry = &entries[index];
(
entry.is_complete(),
std::cmp::Reverse(entry.manifest.created_at),
)
});
while surviving.len() > cap {
let Some(index) = surviving.pop() else { break };
if entries[index].lock.is_resumable() {
removing.push(index);
}
}
removing.sort_unstable();
removing
}
}
#[non_exhaustive]
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct Pruned {
pub removed: Vec<String>,
pub held: Vec<Held>,
}
#[non_exhaustive]
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Held {
pub id: String,
pub reason: String,
}
impl Held {
pub fn new(id: impl Into<String>, reason: impl Into<String>) -> Self {
Self {
id: id.into(),
reason: reason.into(),
}
}
}
pub fn prune(root: &Path, retention: &Retention) -> Result<Pruned, JournalError> {
let entries = list(root)?;
let mut pruned = Pruned::default();
for index in retention.expired(&entries, SystemTime::now()) {
let entry = &entries[index];
match remove(&entry.directory) {
Ok(()) => pruned.removed.push(entry.manifest.id.clone()),
Err(error) => pruned.held.push(Held {
id: entry.manifest.id.clone(),
reason: error.to_string(),
}),
}
}
Ok(pruned)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::journal::settle::Outcome;
use crate::model::exclusion::Exclusions;
use crate::model::ip::set::IpSet;
use crate::model::port::PortSet;
use crate::model::target::{TargetMap, TargetSet};
use crate::model::technique::TcpScanTechnique;
use crate::scanner::checkpoint::spawn_checkpoints;
fn scratch(name: &str) -> PathBuf {
let dir = std::env::temp_dir().join(format!("zond-store-{}-{name}", std::process::id()));
let _ = fs::remove_dir_all(&dir);
fs::create_dir_all(&dir).expect("scratch root");
dir
}
fn plan(range: &str, ports: &str) -> TargetMap {
let mut map = TargetMap::new();
map.add_unit(TargetSet::new(
range.parse::<IpSet>().expect("a range"),
ports.parse::<PortSet>().expect("ports"),
));
map
}
fn ports(map: &TargetMap) -> Plan {
Plan::port_scan(map, &Exclusions::none(), TcpScanTechnique::Syn)
}
fn begin(root: &Path, map: &TargetMap) -> Journal {
Journal::create(root, &ports(map), Privilege::Raw, "test").expect("creates")
}
fn standing_in(silent: &str) -> ScanPhase {
use crate::config::ZondConfig;
use crate::model::ip::range::IpRange;
use crate::report::{LivenessSkip, PhaseParts, ScanSettings, TargetScope};
let silent: IpSet = silent.parse().expect("a range");
ScanPhase::from_parts(PhaseParts {
open: false,
attachments: Vec::new(),
kind: ScanKind::PortScan,
started_at: SystemTime::UNIX_EPOCH,
elapsed: Duration::from_secs(1),
privilege: Some(Privilege::Raw),
targets: TargetScope::from_ip_set(&mut IpSet::new(), &Exclusions::none()),
settings: ScanSettings::from(&ZondConfig::default()),
failures: Vec::new(),
refusals: Vec::new(),
unroutable: Vec::new(),
refused_by_route: Vec::new(),
timed_out: Vec::new(),
icmp_rate_limited: Vec::new(),
reached_by_connect: Vec::new(),
undecided: Vec::new(),
liveness_skipped: Some(LivenessSkip::PortsNoDearer),
silent: silent.v4().iter().copied().map(IpRange::V4).collect(),
stopped: None,
passes_cut: Vec::new(),
unreached: 0,
unheard_probes: 0,
probes: Vec::new(),
origin: None,
})
}
fn unheard(address: &str) -> Host {
use crate::model::port::{Port, PortState, Protocol};
let mut host = Host::new(address.parse().expect("an address"));
host.add_port(Port::new(80, Protocol::Tcp, PortState::Filtered));
host
}
fn heard(address: &str) -> Host {
let mut host = Host::new(address.parse().expect("an address"));
host.set_status(crate::model::host::HostStatus::Up);
host
}
#[test]
fn a_record_an_earlier_sitting_heard_nothing_from_is_not_restored() {
let root = scratch("unheard-restore");
let map = plan("192.0.2.1-192.0.2.8", "80");
let directory = {
let mut journal = begin(&root, &map);
journal
.record_hosts(&[heard("192.0.2.1"), unheard("192.0.2.5")])
.expect("records");
journal
.record_phases(&[standing_in("192.0.2.5")])
.expect("records");
let directory = journal.directory().to_path_buf();
journal.close().expect("closes");
directory
};
let (journal, _) =
Journal::resume(&directory, &ports(&map), Privilege::Raw).expect("resumes");
let restored: Vec<_> = journal.restored().iter().map(Host::primary_ip).collect();
assert_eq!(
restored,
["192.0.2.1".parse::<std::net::IpAddr>().expect("an address")]
);
journal.close().expect("closes");
std::fs::remove_dir_all(&root).ok();
}
#[tokio::test]
async fn a_sitting_that_heard_nothing_from_an_address_leaves_no_record_of_it() {
let root = scratch("unheard-close");
let map = plan("192.0.2.1-192.0.2.8", "80");
let mut journal = begin(&root, &map);
let directory = journal.directory().to_path_buf();
journal
.record_hosts(&[heard("192.0.2.1"), unheard("192.0.2.5")])
.expect("records");
let (_session, ctx) = crate::scanner::session::ScanSession::new();
ctx.restore_hosts(&[heard("192.0.2.1")]);
spawn_checkpoints(journal, ctx.progress())
.finish(&[standing_in("192.0.2.5")])
.await;
let file = fs::read_to_string(directory.join(HOSTS)).expect("reads");
assert!(
!file.contains("192.0.2.5"),
"a record of the silent address is still on disk:\n{file}"
);
let kept: Vec<_> = read_findings(&directory)
.expect("reads")
.iter()
.map(Host::primary_ip)
.collect();
assert_eq!(
kept,
["192.0.2.1".parse::<std::net::IpAddr>().expect("an address")]
);
std::fs::remove_dir_all(&root).ok();
}
#[tokio::test]
async fn a_sitting_killed_before_its_verdicts_leaves_its_undecided_records_to_the_next() {
use crate::config::ZondConfig;
use crate::journal::settle::Outcome;
use crate::model::port::{Port, PortState, Protocol};
use crate::model::target::TargetIndex;
use crate::report::{LivenessSkip, TargetScope};
use crate::scanner::recorder::PhaseRecorder;
let root = scratch("unheard-killed");
let map = plan("192.0.2.1-192.0.2.8", "80,443");
let journal = begin(&root, &map);
let directory = journal.directory().to_path_buf();
let address = |address: &str| address.parse::<std::net::IpAddr>().expect("an address");
let position = |at: &str, port: u16| {
map.iter()
.position(|target| target.ip == address(at) && target.port == port)
.expect("a planned target") as u64
};
let (_session, ctx) = crate::scanner::session::ScanSession::new();
ctx.number_targets(TargetIndex::of(&map));
let _phase = PhaseRecorder::start(
ScanKind::PortScan,
Privilege::Raw,
TargetScope::from_ip_set(
&mut "192.0.2.1-192.0.2.8".parse().expect("a range"),
&Exclusions::none(),
),
&ZondConfig::default(),
)
.skipping_liveness(LivenessSkip::PortsNoDearer)
.opening_in(&ctx);
ctx.await_verdicts();
ctx.update_host(address("192.0.2.1"), |host| {
host.set_status(crate::model::host::HostStatus::Up);
});
ctx.write_host(address("192.0.2.5"), |host| {
*host = unheard("192.0.2.5");
true
});
ctx.record_outcome(Outcome::Exhausted {
position: position("192.0.2.5", 80),
});
ctx.write_host(address("192.0.2.6"), |host| {
*host = unheard("192.0.2.6");
host.add_port(Port::new(443, Protocol::Tcp, PortState::Filtered));
true
});
for port in [80, 443] {
ctx.record_outcome(Outcome::Exhausted {
position: position("192.0.2.6", port),
});
}
let mut ticker = spawn_checkpoints(journal, ctx.progress());
ticker.checkpointed().await;
ticker.kill().await;
let hosts = report(&directory)
.expect("reads")
.hosts()
.map(Host::primary_ip)
.collect::<Vec<_>>();
assert_eq!(
hosts,
[address("192.0.2.1")],
"the report lists an undecided address"
);
let (journal, _) =
Journal::resume(&directory, &ports(&map), Privilege::Raw).expect("resumes");
let restored: Vec<_> = journal
.restored()
.iter()
.map(|host| (host.primary_ip(), host.ports().count()))
.collect();
assert_eq!(
restored,
[(address("192.0.2.1"), 0), (address("192.0.2.5"), 1)],
"the undecided record, and only it beside the host, is the next sitting's"
);
let unheard: u128 = journal
.earlier_phases()
.iter()
.map(ScanPhase::unheard_probes)
.sum();
assert_eq!(unheard, 2, "the silent address's probes went uncounted");
journal.close().expect("closes");
std::fs::remove_dir_all(&root).ok();
}
#[tokio::test]
async fn a_sitting_killed_before_its_verdicts_still_names_what_it_heard_nothing_from() {
use crate::config::ZondConfig;
use crate::journal::settle::Outcome;
use crate::model::target::TargetIndex;
use crate::report::TargetScope;
use crate::scanner::recorder::PhaseRecorder;
let root = scratch("unheard-named");
let map = plan("192.0.2.1-192.0.2.8", "80");
let journal = begin(&root, &map);
let directory = journal.directory().to_path_buf();
let (_session, ctx) = crate::scanner::session::ScanSession::new();
ctx.number_targets(TargetIndex::of(&map));
let _phase = PhaseRecorder::start(
ScanKind::PortScan,
Privilege::Raw,
TargetScope::from_ip_set(&mut IpSet::new(), &Exclusions::none()),
&ZondConfig::default(),
)
.opening_in(&ctx);
ctx.await_verdicts();
for address in ["192.0.2.5", "192.0.2.6"] {
ctx.write_host(
address.parse::<std::net::IpAddr>().expect("an address"),
|host| {
*host = unheard(address);
true
},
);
}
ctx.record_outcome(Outcome::Exhausted { position: 4 });
let mut ticker = spawn_checkpoints(journal, ctx.progress());
ticker.checkpointed().await;
ticker.kill().await;
let (journal, _) =
Journal::resume(&directory, &ports(&map), Privilege::Raw).expect("resumes");
let silent: Vec<_> = journal
.earlier_phases()
.iter()
.flat_map(|phase| phase.silent().iter().copied())
.collect();
let named: IpSet = "192.0.2.5".parse().expect("an address");
assert_eq!(
silent,
named
.v4()
.iter()
.copied()
.map(crate::model::ip::range::IpRange::V4)
.collect::<Vec<_>>(),
"the settled silent address is named, and only it"
);
journal.close().expect("closes");
std::fs::remove_dir_all(&root).ok();
}
#[tokio::test]
async fn a_record_held_for_its_verdict_is_written_once_the_phase_keeps_it() {
let root = scratch("unheard-kept");
let map = plan("192.0.2.1-192.0.2.8", "80");
let journal = begin(&root, &map);
let directory = journal.directory().to_path_buf();
let (_session, ctx) = crate::scanner::session::ScanSession::new();
ctx.await_verdicts();
ctx.write_host(
"192.0.2.5".parse::<std::net::IpAddr>().expect("an address"),
|host| {
*host = unheard("192.0.2.5");
true
},
);
let mut ticker = spawn_checkpoints(journal, ctx.progress());
ticker.checkpointed().await;
ctx.verdicts_reached();
ticker.finish(&[]).await;
let kept: Vec<_> = read_findings(&directory)
.expect("reads")
.iter()
.map(Host::primary_ip)
.collect();
assert_eq!(
kept,
["192.0.2.5".parse::<std::net::IpAddr>().expect("an address")]
);
std::fs::remove_dir_all(&root).ok();
}
#[test]
fn a_scan_resumes_from_where_it_stopped() {
let root = scratch("resume");
let map = plan("192.0.2.1-192.0.2.4", "80,443");
let directory = {
let mut journal = begin(&root, &map);
let settlements = Settlements::default();
for position in [0, 1, 2, 5] {
settlements.record(Outcome::Answered { position });
}
journal.checkpoint(&settlements).expect("checkpoints");
let directory = journal.directory().to_path_buf();
journal.close().expect("closes");
directory
};
let (_journal, checkpoint) =
Journal::resume(&directory, &ports(&map), Privilege::Raw).expect("resumes");
assert_eq!(checkpoint.watermark, 3);
let remaining: Vec<_> = checkpoint.remaining(map.iter()).collect();
assert_eq!(remaining.len(), 4, "eight targets, four settled");
}
#[test]
fn a_jobs_options_are_written_by_its_first_sitting_and_kept() {
use crate::config::ZondConfig;
let root = scratch("options");
let map = plan("192.0.2.1-192.0.2.4", "80,443");
let first = ZondConfig {
assume_up: true,
traceroute: true,
..ZondConfig::default()
};
let directory = {
let mut journal = begin(&root, &map);
assert!(journal.options().is_none(), "no sitting has started");
journal
.record_options(JobOptions::of(&first))
.expect("records");
let directory = journal.directory().to_path_buf();
journal.close().expect("closes");
directory
};
let (mut journal, _) =
Journal::resume(&directory, &ports(&map), Privilege::Raw).expect("resumes");
assert_eq!(journal.options(), Some(&JobOptions::of(&first)));
journal
.record_options(JobOptions::of(&ZondConfig::default()))
.expect("a second record is a no-op");
journal.close().expect("closes");
let (journal, _) =
Journal::resume(&directory, &ports(&map), Privilege::Raw).expect("resumes");
assert_eq!(journal.options(), Some(&JobOptions::of(&first)));
}
#[test]
fn a_journal_older_than_its_options_is_left_without_them() {
use crate::config::ZondConfig;
let root = scratch("options-older");
let map = plan("192.0.2.1-192.0.2.4", "80,443");
let directory = {
let mut journal = begin(&root, &map);
let settlements = Settlements::default();
settlements.record(Outcome::Answered { position: 0 });
journal.checkpoint(&settlements).expect("checkpoints");
let directory = journal.directory().to_path_buf();
journal.close().expect("closes");
directory
};
let (mut journal, checkpoint) = Journal::resume(&directory, &ports(&map), Privilege::Raw)
.expect("an older journal resumes");
assert_eq!(checkpoint.watermark, 1);
journal
.record_options(JobOptions::of(&ZondConfig::default()))
.expect("records nothing");
assert!(journal.options().is_none());
assert!(!directory.join(OPTIONS).exists());
}
#[test]
fn a_changed_plan_is_refused() {
let root = scratch("changed-plan");
let map = plan("192.0.2.1-192.0.2.4", "80,443");
let journal = begin(&root, &map);
let directory = journal.directory().to_path_buf();
journal.close().expect("closes");
let widened = plan("192.0.2.1-192.0.2.9", "80,443");
let refused = Journal::resume(&directory, &ports(&widened), Privilege::Raw)
.expect_err("the plan moved");
assert!(matches!(refused, OpenError::PlanChanged(_)), "{refused:?}");
}
#[test]
fn a_journal_whose_lock_went_stale_can_be_taken_over() {
let root = scratch("stale-lock");
let map = plan("192.0.2.1", "80");
let directory = {
let journal = begin(&root, &map);
journal.directory().to_path_buf()
};
let stale = super::super::lock::LockRecord {
pid: std::process::id(),
boot: super::super::lock::boot_identity(),
started_at: SystemTime::now() - Duration::from_secs(3_600),
heartbeat: SystemTime::now() - Duration::from_secs(600),
};
fs::write(
directory.join(LOCK),
serde_json::to_string(&stale).expect("json"),
)
.expect("writes");
assert!(matches!(
Journal::reopen(&directory, Privilege::Raw),
Err(OpenError::Locked(LockRefused::Held(
LockState::Stale { .. }
)))
));
let (journal, _, _) =
Journal::take_over(&directory, Privilege::Raw).expect("takes the journal over");
journal.close().expect("closes");
std::fs::remove_dir_all(&root).ok();
}
#[test]
fn a_live_journal_is_not_resumable() {
let root = scratch("live");
let map = plan("192.0.2.1", "80");
let journal = begin(&root, &map);
let directory = journal.directory().to_path_buf();
let refused =
Journal::resume(&directory, &ports(&map), Privilege::Raw).expect_err("it is held");
assert!(matches!(refused, OpenError::Locked(_)), "{refused:?}");
journal.close().expect("closes");
Journal::resume(&directory, &ports(&map), Privilege::Raw).expect("now free");
}
#[test]
fn listing_describes_journals_without_locking_them() {
let root = scratch("list");
let map = plan("192.0.2.1-192.0.2.4", "80,443");
let mut journal = begin(&root, &map);
let settlements = Settlements::default();
for position in 0..4 {
settlements.record(Outcome::Answered { position });
}
journal.checkpoint(&settlements).expect("checkpoints");
let listed = list(&root).expect("lists");
assert_eq!(listed.len(), 1);
let entry = &listed[0];
assert_eq!(entry.manifest.total_targets, 8);
assert_eq!(entry.settled(), Some(4));
assert!(!entry.is_complete());
assert!(
matches!(entry.lock, LockState::Held { .. }),
"the scan is still running: {:?}",
entry.lock
);
journal
.checkpoint(&settlements)
.expect("still holds its lock");
}
#[test]
fn a_finished_journal_reads_as_complete() {
let root = scratch("complete");
let map = plan("192.0.2.1-192.0.2.4", "80,443");
let mut journal = begin(&root, &map);
let settlements = Settlements::default();
for position in 0..8 {
settlements.record(Outcome::Answered { position });
}
journal.checkpoint(&settlements).expect("checkpoints");
journal.close().expect("closes");
let listed = list(&root).expect("lists");
assert!(listed[0].is_complete());
assert_eq!(listed[0].settled(), Some(8));
}
#[test]
fn a_journal_with_no_checkpoint_resumes_from_the_beginning() {
let root = scratch("no-checkpoint");
let map = plan("192.0.2.1-192.0.2.4", "80,443");
let journal = begin(&root, &map);
let directory = journal.directory().to_path_buf();
journal.close().expect("closes");
let (_journal, checkpoint) =
Journal::resume(&directory, &ports(&map), Privilege::Raw).expect("resumes");
assert_eq!(checkpoint, Checkpoint::default());
assert_eq!(checkpoint.remaining(map.iter()).count(), 8);
}
#[test]
fn ids_are_sortable_and_distinct() {
let mut ids: Vec<String> = (0..64).map(|_| mint_id()).collect();
assert!(ids.iter().all(|id| id.len() == ID_CHARS));
let distinct: std::collections::BTreeSet<&String> = ids.iter().collect();
assert_eq!(distinct.len(), ids.len(), "ids collided");
let sorted = {
ids.sort();
ids.clone()
};
assert_eq!(ids, sorted, "ids minted in one burst lost their order");
let first = mint_id();
std::thread::sleep(std::time::Duration::from_millis(3));
let second = mint_id();
assert!(first < second, "{first} should sort before {second}");
}
#[test]
fn an_unreadable_journal_does_not_hide_the_others() {
let root = scratch("damaged");
let map = plan("192.0.2.1", "80");
begin(&root, &map).close().expect("closes");
let damaged = root.join("01JUNREADABLEJUNREADABLE00");
fs::create_dir_all(&damaged).expect("directory");
fs::write(damaged.join(MANIFEST), "{not json").expect("writes");
let listed = list(&root).expect("lists");
assert_eq!(listed.len(), 1, "the readable one is still there");
}
#[test]
fn pruning_refuses_a_journal_that_is_being_written() {
let root = scratch("prune");
let map = plan("192.0.2.1", "80");
let journal = begin(&root, &map);
let directory = journal.directory().to_path_buf();
assert!(matches!(remove(&directory), Err(OpenError::Locked(_))));
journal.close().expect("closes");
remove(&directory).expect("removes a free journal");
assert!(list(&root).expect("lists").is_empty());
}
#[tokio::test]
async fn the_checkpoint_task_writes_a_final_cursor_and_releases() {
let root = scratch("ticker");
let map = plan("192.0.2.1-192.0.2.4", "80,443");
let journal = begin(&root, &map);
let directory = journal.directory().to_path_buf();
let (_session, ctx) = crate::scanner::session::ScanSession::new();
for position in 0..5 {
ctx.record_outcome(Outcome::Answered { position });
}
spawn_checkpoints(journal, ctx.progress()).finish(&[]).await;
let checkpoint = Checkpoint::read(&directory.join(CURSOR)).expect("a cursor was written");
assert_eq!(checkpoint.watermark, 5);
assert_eq!(
super::super::lock::inspect(&directory.join(LOCK)),
LockState::Free,
"the lock outlived the scan and was then released"
);
}
#[test]
fn a_resumed_cursor_carries_the_earlier_sittings_progress() {
let root = scratch("carry-forward");
let map = plan("192.0.2.1-192.0.2.4", "80,443");
let directory = {
let mut journal = begin(&root, &map);
let settlements = Settlements::default();
for position in 0..3 {
settlements.record(Outcome::Answered { position });
}
journal.checkpoint(&settlements).expect("checkpoints");
let directory = journal.directory().to_path_buf();
journal.close().expect("closes");
directory
};
let (mut journal, checkpoint) =
Journal::resume(&directory, &ports(&map), Privilege::Raw).expect("resumes");
let settlements = Settlements::resuming(&checkpoint);
journal
.checkpoint(&settlements)
.expect("checkpoints having settled nothing new");
journal.close().expect("closes");
let carried = Checkpoint::read(&directory.join(CURSOR)).expect("reads");
assert_eq!(
carried.watermark, 3,
"the second sitting must not roll the cursor back"
);
}
#[test]
fn what_a_sitting_found_comes_back() {
use crate::model::host::HostStatus;
use crate::model::port::{Port, PortState, Protocol};
let root = scratch("findings");
let map = plan("192.0.2.1-192.0.2.4", "80,443");
let directory = {
let mut journal = begin(&root, &map);
let mut host = Host::new("192.0.2.1".parse().expect("an address"));
host.set_status(HostStatus::Up);
host.set_hostname(Some("router.example".to_string()));
host.add_port(Port::new(80, Protocol::Tcp, PortState::Open));
journal.record_hosts(&[host]).expect("records");
let directory = journal.directory().to_path_buf();
journal.close().expect("closes");
directory
};
let (journal, _) =
Journal::resume(&directory, &ports(&map), Privilege::Raw).expect("resumes");
let restored = journal.restored();
assert_eq!(restored.len(), 1);
assert_eq!(restored[0].hostname(), Some("router.example"));
assert_eq!(restored[0].port_count(), 1);
assert!(restored[0].is_alive());
}
#[test]
fn detection_run_tapes_survive_a_journal_round_trip() {
use crate::detect::compute::{CapTape, CapTapeRecord, DetectionRunRecord, SpeakExchange};
use crate::model::finding::{DetectionId, Version};
use crate::record::DetectionIdRecord;
let root = scratch("detections");
let map = plan("192.0.2.1", "80");
let tape = CapTape {
speaks: vec![SpeakExchange {
sent: b"PING\r\n".to_vec(),
reply: Ok(b"+PONG\r\n".to_vec()),
}],
nows: vec![7],
..CapTape::default()
};
let detection = DetectionId::new("redis-unauth", Version::new(1, 0, 0), "abc").unwrap();
let run = DetectionRunRecord {
host: "192.0.2.1".to_string(),
host_name: None,
port: 80,
protocol: "tcp".to_string(),
detection: DetectionIdRecord::from(&detection),
responses: vec!["+PONG\r\n".to_string()],
tape: CapTapeRecord::from(&tape),
};
let directory = {
let mut journal = begin(&root, &map);
journal
.record_detections(std::slice::from_ref(&run))
.expect("records the run");
let directory = journal.directory().to_path_buf();
journal.close().expect("closes");
directory
};
let read = read_detections(&directory).expect("reads the runs back");
assert_eq!(
read,
vec![run],
"a detection run did not survive the journal"
);
assert_eq!(
read[0].tape.rebuild(),
tape,
"the tape did not rebuild to what was recorded"
);
}
fn passive_run(
id: &str,
port: u16,
responses: &[&str],
) -> crate::detect::compute::DetectionRunRecord {
use crate::detect::compute::{CapTape, CapTapeRecord, DetectionRunRecord};
use crate::model::finding::{DetectionId, Version};
use crate::record::DetectionIdRecord;
let detection = DetectionId::new(id, Version::new(1, 0, 0), "abc").expect("an id");
DetectionRunRecord {
host: "192.0.2.1".to_string(),
host_name: None,
port,
protocol: "tcp".to_string(),
detection: DetectionIdRecord::from(&detection),
responses: responses
.iter()
.map(|response| response.to_string())
.collect(),
tape: CapTapeRecord::from(&CapTape::default()),
}
}
#[test]
fn a_ports_responses_are_written_once_however_many_runs_read_them() {
let root = scratch("tapes-once");
let map = plan("192.0.2.1", "80");
let page = "HTTP/1.1 200 OK\r\nServer: a-distinctive-server\r\n\r\n";
let runs: Vec<_> = (0..13)
.map(|n| passive_run(&format!("http-{n}"), 80, &[page]))
.chain([passive_run("ssh-banner", 22, &["SSH-2.0-x\r\n"])])
.collect();
let directory = {
let mut journal = begin(&root, &map);
journal.record_detections(&runs).expect("records the runs");
let directory = journal.directory().to_path_buf();
journal.close().expect("closes");
directory
};
let written = fs::read_to_string(directory.join(DETECTIONS)).expect("reads the file");
assert_eq!(written.matches("a-distinctive-server").count(), 1);
assert_eq!(read_detections(&directory).expect("reads back"), runs);
}
#[test]
fn tapes_written_one_run_to_a_line_still_read_back() {
use std::io::Write;
let root = scratch("tapes-legacy");
let map = plan("192.0.2.1", "80");
let grouped = passive_run("http-title", 80, &["HTTP/1.1 200 OK\r\n\r\n"]);
let alone = passive_run("ssh-banner", 22, &["SSH-2.0-x\r\n"]);
let directory = {
let mut journal = begin(&root, &map);
journal
.record_detections(std::slice::from_ref(&grouped))
.expect("records the run");
let directory = journal.directory().to_path_buf();
journal.close().expect("closes");
directory
};
let mut file = fs::OpenOptions::new()
.append(true)
.open(directory.join(DETECTIONS))
.expect("opens the file");
let line = serde_json::to_string(&alone).expect("serialises");
writeln!(file, "{line}").expect("appends a run on its own");
assert_eq!(
read_detections(&directory).expect("reads back"),
[grouped, alone]
);
}
#[test]
fn repeated_records_for_one_host_are_folded_together() {
use crate::model::host::HostStatus;
use crate::model::port::{Port, PortState, Protocol};
let root = scratch("folded");
let map = plan("192.0.2.1", "80");
let ip: std::net::IpAddr = "192.0.2.1".parse().expect("an address");
let directory = {
let mut journal = begin(&root, &map);
let mut found = Host::new(ip);
found.set_status(HostStatus::Up);
journal.record_hosts(&[found]).expect("the host answered");
let mut classified = Host::new(ip);
classified.add_port(Port::new(80, Protocol::Tcp, PortState::Open));
journal
.record_hosts(&[classified])
.expect("and then its ports");
let directory = journal.directory().to_path_buf();
journal.close().expect("closes");
directory
};
let (journal, _) =
Journal::resume(&directory, &ports(&map), Privilege::Raw).expect("resumes");
let restored = journal.restored();
assert_eq!(restored.len(), 1, "one address is one host");
assert!(restored[0].is_alive(), "the first record's status survived");
assert_eq!(restored[0].port_count(), 1, "the second record's port too");
}
#[test]
fn link_locals_on_two_interfaces_come_back_as_two_hosts() {
use crate::model::host::HostStatus;
use crate::model::ip::scoped::Zone;
let root = scratch("zones");
let map = plan("192.0.2.1", "80");
let ip: std::net::IpAddr = "fe80::1".parse().expect("an address");
let directory = {
let mut journal = begin(&root, &map);
let hosts: Vec<Host> = [Zone::new(4, "en0"), Zone::new(9, "en7")]
.into_iter()
.map(|zone| {
let mut host = Host::new(ip);
host.set_status(HostStatus::Up);
host.set_zone(zone);
host
})
.collect();
journal.record_hosts(&hosts).expect("records both");
let directory = journal.directory().to_path_buf();
journal.close().expect("closes");
directory
};
assert_eq!(report(&directory).expect("reads").host_count(), 2);
let (journal, _) =
Journal::resume(&directory, &ports(&map), Privilege::Raw).expect("resumes");
let mut zones: Vec<&str> = journal
.restored()
.iter()
.filter_map(|host| host.zone().map(Zone::name))
.collect();
zones.sort_unstable();
assert_eq!(zones, ["en0", "en7"], "each restored on its own link");
}
#[test]
fn a_journal_with_no_findings_restores_nothing() {
let root = scratch("no-findings");
let map = plan("192.0.2.1", "80");
let journal = begin(&root, &map);
let directory = journal.directory().to_path_buf();
journal.close().expect("closes");
let (journal, _) =
Journal::resume(&directory, &ports(&map), Privilege::Raw).expect("resumes");
assert!(journal.restored().is_empty());
}
#[test]
fn a_journal_reads_back_as_the_report_its_scan_produced() {
let root = scratch("replay");
let map = plan("192.0.2.1-192.0.2.4", "80,443");
let original = crate::export::fixture::report();
let directory = {
let mut journal = begin(&root, &map);
let hosts: Vec<Host> = original.hosts().cloned().collect();
journal.record_hosts(&hosts).expect("records hosts");
journal
.record_phases(original.phases())
.expect("records phases");
let directory = journal.directory().to_path_buf();
journal.close().expect("closes");
directory
};
let replayed = report(&directory).expect("reads");
assert_eq!(replayed.host_count(), original.host_count());
assert_eq!(replayed.phases().len(), original.phases().len());
assert_eq!(
replayed.summary().hosts_alive,
original.summary().hosts_alive
);
assert_eq!(replayed.summary().ports_open, original.summary().ports_open);
let (before, after) = (&original.phases()[0], &replayed.phases()[0]);
assert_eq!(after.kind(), before.kind());
assert_eq!(after.privilege(), before.privilege());
assert_eq!(after.failures().len(), before.failures().len());
assert_eq!(
replayed.is_partial(),
original.is_partial(),
"a scan that left ground uncovered has to still say so"
);
}
#[test]
fn a_replayed_report_names_the_engine_that_ran_the_scan() {
let root = scratch("replay-version");
let map = plan("192.0.2.1", "80");
let directory = {
let journal = begin(&root, &map);
let directory = journal.directory().to_path_buf();
journal.close().expect("closes");
directory
};
let path = directory.join(MANIFEST);
let mut manifest: JournalManifest =
serde_json::from_str(&fs::read_to_string(&path).expect("reads")).expect("parses");
manifest.engine_version = "0.1.0".to_string();
fs::write(&path, serde_json::to_vec(&manifest).expect("encodes")).expect("writes");
let replayed = report(&directory).expect("reads");
assert_eq!(replayed.engine_version(), "0.1.0");
assert_ne!(
replayed.engine_version(),
crate::report::ENGINE_VERSION,
"this build's version must not have been stamped over the record's"
);
}
#[cfg(unix)]
#[test]
fn a_cursor_that_cannot_be_read_is_not_a_scan_that_settled_nothing() {
use std::os::unix::fs::PermissionsExt;
let root = scratch("unreadable-cursor");
let map = plan("192.0.2.1-192.0.2.4", "80");
let directory = {
let mut journal = begin(&root, &map);
let settlements = Settlements::default();
settlements.record(Outcome::Answered { position: 0 });
journal.checkpoint(&settlements).expect("checkpoints");
let directory = journal.directory().to_path_buf();
journal.close().expect("closes");
directory
};
let listed = list(&root).expect("lists");
assert_eq!(listed[0].settled(), Some(1));
fs::set_permissions(directory.join(CURSOR), fs::Permissions::from_mode(0o000))
.expect("removes every permission");
let listed = list(&root).expect("a journal it cannot read must not hide the rest");
assert_eq!(listed.len(), 1, "the journal still lists");
assert_eq!(
listed[0].settled(),
None,
"unreadable is not zero: zero is a claim about the scan"
);
assert!(
!listed[0].is_complete(),
"and nothing that cannot be read may be called finished"
);
let _ = fs::set_permissions(directory.join(CURSOR), fs::Permissions::from_mode(0o600));
}
#[tokio::test]
async fn what_a_scan_learns_after_a_checkpoint_still_reaches_the_file() {
use crate::model::host::{HostStatus, StatusProtocol, StatusReason};
let root = scratch("late-findings");
let map = plan("192.0.2.1", "80");
let journal = begin(&root, &map);
let directory = journal.directory().to_path_buf();
let (_session, ctx) = crate::scanner::session::ScanSession::new();
let mut ticker = spawn_checkpoints(journal, ctx.progress());
let ip: std::net::IpAddr = "192.0.2.1".parse().expect("an address");
ctx.update_host(ip, |host| {
host.set_status(HostStatus::Up);
host.record_evidence(
HostStatus::Up,
StatusReason::new(StatusProtocol::Arp, "answered"),
);
});
ticker.checkpointed().await;
ctx.update_host(ip, |host| {
host.record_evidence(
HostStatus::Up,
StatusReason::new(StatusProtocol::IcmpEcho, "echo answered"),
);
});
ticker.finish(&[]).await;
let restored = read_findings(&directory).expect("reads");
assert_eq!(restored.len(), 1, "one host was scanned");
let protocols: Vec<_> = restored[0]
.reasons()
.iter()
.map(|reason| reason.protocol.clone())
.collect();
assert!(
protocols.contains(&StatusProtocol::IcmpEcho),
"what the scan learned last was lost: {protocols:?}"
);
}
fn aged(id: &str, created_at: SystemTime, settled: u64, total: u128) -> Entry {
Entry {
directory: PathBuf::from(id),
manifest: JournalManifest {
links: Vec::new(),
journal_version: crate::journal::JOURNAL_VERSION,
id: id.to_string(),
engine_version: "0.0.0".to_string(),
created_at,
kind: crate::record::wire::scan_kind_name(ScanKind::PortScan).to_owned(),
plan: crate::journal::manifest::PlanFingerprint::of(
&ports(&plan("192.0.2.1", "80")),
Privilege::Raw,
),
targets: crate::record::PlanRecord::from(&plan("192.0.2.1", "80")),
technique: TcpScanTechnique::Syn.name().to_owned(),
sweep: false,
privilege: Privilege::Raw,
total_targets: total,
order_seed: None,
summary: String::new(),
},
checkpoint: Some(Checkpoint {
watermark: settled,
settled_above: Vec::new(),
walked: None,
}),
lock: LockState::Free,
}
}
fn ago(seconds: u64) -> SystemTime {
now() - Duration::from_secs(seconds)
}
fn now() -> SystemTime {
SystemTime::UNIX_EPOCH + Duration::from_secs(1_000_000)
}
#[test]
fn age_removes_finished_journals_and_keeps_unfinished_ones() {
let entries = vec![
aged("finished-old", ago(100), 1, 1),
aged("unfinished-old", ago(100), 0, 1),
aged("finished-new", ago(1), 1, 1),
];
let retention = Retention {
completed_for: Some(Duration::from_secs(50)),
incomplete_for: None,
keep_at_most: None,
};
assert_eq!(retention.expired(&entries, now()), vec![0]);
}
#[test]
fn a_live_journal_is_never_pruned() {
let mut entries = vec![aged("held", ago(10_000), 1, 1)];
entries[0].lock = LockState::Held {
pid: 1,
last_beat: Duration::from_secs(1),
};
let retention = Retention {
completed_for: Some(Duration::from_secs(1)),
incomplete_for: Some(Duration::from_secs(1)),
keep_at_most: Some(0),
};
assert!(retention.expired(&entries, now()).is_empty());
}
#[test]
fn the_cap_removes_duplicates_before_unfinished_work() {
let entries = vec![
aged("finished-new", ago(1), 1, 1),
aged("unfinished-mid", ago(2), 0, 1),
aged("finished-old", ago(3), 1, 1),
];
let retention = Retention {
completed_for: None,
incomplete_for: None,
keep_at_most: Some(2),
};
let removed = retention.expired(&entries, now());
assert_eq!(removed, vec![2], "the older of the two finished ones");
let retention = Retention {
keep_at_most: Some(1),
..retention
};
let removed = retention.expired(&entries, now());
assert_eq!(
removed,
vec![0, 2],
"both finished ones, the unfinished kept"
);
}
#[test]
fn keeping_everything_removes_nothing() {
let entries = vec![
aged("ancient", ago(900_000), 1, 1),
aged("also-ancient", ago(900_000), 0, 1),
];
assert!(
Retention::keep_everything()
.expired(&entries, now())
.is_empty()
);
}
#[test]
fn the_default_favours_unfinished_work() {
let two_months = Duration::from_secs(60 * 24 * 60 * 60);
let created = now() - two_months;
let entries = vec![
aged("finished", created, 1, 1),
aged("unfinished", created, 0, 1),
];
assert_eq!(Retention::default().expired(&entries, now()), vec![0]);
}
#[test]
fn pruning_removes_what_the_policy_selected() {
let root = scratch("retention");
let map = plan("192.0.2.1", "80");
let mut journal = begin(&root, &map);
let settlements = Settlements::default();
settlements.record(Outcome::Answered { position: 0 });
journal.checkpoint(&settlements).expect("checkpoints");
let id = journal.manifest().id.clone();
journal.close().expect("closes");
let untouched = prune(&root, &Retention::default()).expect("prunes");
assert!(untouched.removed.is_empty());
assert_eq!(list(&root).expect("lists").len(), 1);
let swept = prune(
&root,
&Retention {
completed_for: Some(Duration::ZERO),
..Retention::default()
},
)
.expect("prunes");
assert_eq!(swept.removed, vec![id]);
assert!(swept.held.is_empty());
assert!(list(&root).expect("lists").is_empty());
std::fs::remove_dir_all(&root).ok();
}
#[test]
fn a_host_written_under_two_addresses_comes_back_as_one() {
use crate::model::host::HostStatus;
let root = scratch("promoted");
let map = plan("192.0.2.1", "80");
let link_local: std::net::IpAddr = "fe80::1".parse().expect("an address");
let global: std::net::IpAddr = "2001:db8::1".parse().expect("an address");
let directory = {
let mut journal = begin(&root, &map);
let mut early = Host::new(link_local);
early.set_status(HostStatus::Up);
journal.record_hosts(&[early]).expect("records");
let mut promoted = Host::new(link_local);
promoted.add_ip(global);
assert!(
promoted.consider_primary_ip(global),
"the global address should lead"
);
promoted.set_hostname(Some("router.example".to_string()));
journal.record_hosts(&[promoted]).expect("records again");
let directory = journal.directory().to_path_buf();
journal.close().expect("closes");
directory
};
let (journal, _) =
Journal::resume(&directory, &ports(&map), Privilege::Raw).expect("resumes");
let restored = journal.restored();
assert_eq!(
restored.len(),
1,
"one machine, written twice: {:?}",
restored.iter().map(Host::primary_ip).collect::<Vec<_>>()
);
assert!(restored[0].is_alive(), "the first record's status survived");
assert_eq!(restored[0].hostname(), Some("router.example"));
}
#[cfg(unix)]
#[test]
fn every_file_a_journal_writes_is_private() {
use std::os::unix::fs::PermissionsExt;
let root = scratch("modes");
let map = plan("192.0.2.1", "80");
let mut journal = begin(&root, &map);
let settlements = Settlements::default();
settlements.record(Outcome::Answered { position: 0 });
journal
.record(
&[Host::new("192.0.2.1".parse().expect("an address"))],
&settlements,
)
.expect("records");
let directory = journal.directory().to_path_buf();
let mode_of = |path: &Path| {
fs::metadata(path)
.unwrap_or_else(|e| panic!("{path:?}: {e}"))
.permissions()
.mode()
& 0o777
};
assert_eq!(mode_of(&directory), 0o700, "the directory");
for name in [MANIFEST, CURSOR, HOSTS, PHASES, LOCK] {
assert_eq!(mode_of(&directory.join(name)), 0o600, "{name}");
}
journal.checkpoint(&settlements).expect("beats");
assert_eq!(mode_of(&directory.join(LOCK)), 0o600, "after a heartbeat");
journal.close().expect("closes");
std::fs::remove_dir_all(&root).ok();
}
#[cfg(unix)]
#[test]
fn appending_refuses_a_link_where_a_journal_file_should_be() {
let root = scratch("nofollow");
let map = plan("192.0.2.1", "80");
let mut journal = begin(&root, &map);
let directory = journal.directory().to_path_buf();
let elsewhere = root.join("not-the-journals-to-touch");
fs::write(&elsewhere, b"untouched").expect("writes");
for name in [HOSTS, PHASES] {
fs::remove_file(directory.join(name)).expect("removes");
std::os::unix::fs::symlink(&elsewhere, directory.join(name)).expect("links");
}
let phases = crate::export::fixture::report().phases().to_vec();
assert!(
journal
.record_hosts(&[Host::new("192.0.2.1".parse().expect("an address"))])
.is_err(),
"a link was appended to as the findings file"
);
assert!(
journal.record_phases(&phases).is_err(),
"a link was appended to as the phases file"
);
assert_eq!(
fs::read(&elsewhere).expect("reads"),
b"untouched",
"the file behind the link was written through"
);
journal.close().expect("closes");
std::fs::remove_dir_all(&root).ok();
}
#[test]
fn a_host_recorded_twice_keeps_the_round_trips_its_last_record_holds() {
use std::time::Duration;
let root = scratch("latency");
let mut journal = begin(&root, &plan("192.0.2.1", "80"));
let mut host = Host::new("192.0.2.1".parse().expect("an address"));
host.add_rtts([Duration::from_millis(1), Duration::from_millis(2)]);
journal.record_hosts(&[host.clone()]).expect("records");
host.add_rtts([Duration::from_millis(6), Duration::from_millis(7)]);
journal
.record_hosts(&[host.clone()])
.expect("records again");
let directory = journal.directory().to_path_buf();
journal.close().expect("closes");
let read = report(&directory).expect("reads");
std::fs::remove_dir_all(&root).ok();
let [restored] = read.hosts().collect::<Vec<_>>()[..] else {
panic!("one host on record");
};
let figures = |host: &Host| (host.min_rtt(), host.average_rtt(), host.max_rtt());
assert_eq!(figures(restored), figures(&host), "min, average, max");
assert_eq!(
restored.telemetry().history().len(),
host.telemetry().history().len(),
"round trips held"
);
}
#[test]
fn a_host_recorded_at_every_checkpoint_reads_back_as_the_host_it_is() {
use crate::model::confidence::Confidence;
use crate::model::finding::{DetectionClass, DetectionId, Finding, Severity, Version};
use crate::model::host::{OsEvidence, OsSource};
let root = scratch("repeated");
let mut journal = begin(&root, &plan("203.0.113.1-203.0.113.9", "22,80,443,445"));
let mut hosts: Vec<Host> = crate::export::fixture::report().hosts().cloned().collect();
for checkpoint in 0..3u64 {
let finding = Finding::new(
DetectionId::new(format!("check-{checkpoint}"), Version::new(1, 0, 0), "")
.expect("an id"),
"A check",
Severity::Low,
Confidence::Certain,
DetectionClass::Passive,
)
.expect("a finding");
for host in &mut hosts {
host.add_rtt(Duration::from_micros(700 + 100 * checkpoint));
host.record_os_evidence(OsEvidence {
source: OsSource::ServiceBanner,
family: Some("Linux".to_string()),
device: None,
vendor: None,
product: None,
version: None,
kernel: None,
arch: None,
cpe: None,
confidence: 0.5,
evidence: format!("banner read at checkpoint {checkpoint}"),
});
host.add_finding(finding.clone());
let first = host
.ports()
.next()
.map(|port| (port.number(), port.protocol()));
if let Some((number, protocol)) = first {
host.add_port_finding(number, protocol, finding.clone());
}
}
journal.record_hosts(&hosts).expect("records");
}
let directory = journal.directory().to_path_buf();
journal.close().expect("closes");
let mut read = read_findings(&directory).expect("reads");
std::fs::remove_dir_all(&root).ok();
read.sort_by_key(Host::primary_ip);
hosts.sort_by_key(Host::primary_ip);
let as_recorded = |hosts: &[Host]| {
hosts
.iter()
.map(|host| serde_json::to_value(HostRecord::from(host)).expect("serialises"))
.collect::<Vec<_>>()
};
assert_eq!(as_recorded(&read), as_recorded(&hosts));
}
#[cfg(unix)]
#[test]
fn a_link_in_a_root_of_journals_is_not_listed() {
let root = scratch("listed-link");
let elsewhere = scratch("listed-link-elsewhere");
let journal = begin(&elsewhere, &plan("192.0.2.1", "80"));
let directory = journal.directory().to_path_buf();
journal.close().expect("closes");
std::os::unix::fs::symlink(&directory, root.join("06LINKED00000000")).expect("links");
let listed = list(&root).expect("lists").len();
let where_it_is = list(&elsewhere).expect("lists").len();
std::fs::remove_dir_all(&root).ok();
std::fs::remove_dir_all(&elsewhere).ok();
assert_eq!(listed, 0, "the link was listed as a journal");
assert_eq!(where_it_is, 1, "the journal itself lists");
}
#[cfg(unix)]
#[test]
fn reading_refuses_a_link_where_a_journal_file_should_be() {
let root = scratch("read-nofollow");
let mut ours = begin(&root, &plan("192.0.2.1", "80"));
ours.record_hosts(&[Host::new("192.0.2.1".parse().expect("an address"))])
.expect("records");
ours.write_cursor(&Checkpoint::default())
.expect("checkpoints");
let directory = ours.directory().to_path_buf();
ours.close().expect("closes");
let mut theirs = begin(&root, &plan("198.51.100.7", "80"));
theirs
.record_hosts(&[Host::new("198.51.100.7".parse().expect("an address"))])
.expect("records");
theirs
.write_cursor(&Checkpoint::new(1, []))
.expect("checkpoints");
let elsewhere = theirs.directory().to_path_buf();
theirs.close().expect("closes");
let text = root.join("not-a-journal-file");
fs::write(&text, b"somebody else's bytes").expect("writes");
let refused = |name: &str, error: String| {
assert!(
error.contains("is a link"),
"{name}: refused, but not as a link: {error}"
);
};
for name in [MANIFEST, CURSOR, HOSTS, PHASES] {
let kept = directory.join(format!("{name}.kept"));
fs::rename(directory.join(name), &kept).expect("moves aside");
std::os::unix::fs::symlink(elsewhere.join(name), directory.join(name)).expect("links");
match Journal::reopen(&directory, Privilege::Raw) {
Ok(_) => panic!("{name}: resumed through a link"),
Err(error) => refused(name, error.to_string()),
}
if matches!(name, MANIFEST | HOSTS | PHASES) {
match report(&directory) {
Ok(_) => panic!("{name}: reported through a link"),
Err(error) => refused(name, error.to_string()),
}
}
fs::remove_file(directory.join(name)).expect("removes the link");
fs::rename(&kept, directory.join(name)).expect("restores");
}
let journal = begin(&root, &plan("192.0.2.2", "80"));
for name in [DETECTIONS, FINISHED] {
std::os::unix::fs::symlink(&text, journal.directory().join(name)).expect("links");
}
match read_detections(journal.directory()) {
Ok(runs) => panic!("detections read through a link: {runs:?}"),
Err(error) => refused(DETECTIONS, error.to_string()),
}
match journal.finished_hosts() {
Ok(hosts) => panic!("finished hosts read through a link: {hosts:?}"),
Err(error) => refused(FINISHED, error.to_string()),
}
journal.close().expect("closes");
std::fs::remove_dir_all(&root).ok();
}
#[test]
fn a_cursor_repeating_settled_positions_does_not_inflate_the_count() {
let root = scratch("inflated-cursor");
let map = plan("192.0.2.1-192.0.2.4", "80,443");
let directory = {
let journal = begin(&root, &map);
let directory = journal.directory().to_path_buf();
journal.close().expect("closes");
directory
};
let checkpoint = Checkpoint {
watermark: 4,
settled_above: vec![0, 1, 2, 6],
walked: None,
};
checkpoint
.write_atomically(&directory.join(CURSOR))
.expect("writes");
let entry = list(&root)
.expect("lists")
.into_iter()
.next()
.expect("one journal");
assert_eq!(
entry.settled(),
Some(5),
"the watermark plus the one position genuinely above it"
);
assert!(
!entry.is_complete(),
"eight targets, five settled: a retention sweep must not take this"
);
}
#[test]
fn an_append_after_a_torn_tail_leaves_the_journal_readable() {
let root = scratch("torn-tail-append");
let map = plan("192.0.2.1-192.0.2.4", "80");
let first: std::net::IpAddr = "192.0.2.1".parse().expect("an address");
let second: std::net::IpAddr = "192.0.2.2".parse().expect("an address");
let directory = {
let mut journal = begin(&root, &map);
journal.record_hosts(&[Host::new(first)]).expect("records");
let directory = journal.directory().to_path_buf();
journal.close().expect("closes");
directory
};
{
use std::io::Write;
let mut file = fs::OpenOptions::new()
.append(true)
.open(directory.join(HOSTS))
.expect("opens");
file.write_all(br#"{"ip":"192.0.2.9","stat"#)
.expect("tears");
}
let (mut journal, _) = Journal::resume(&directory, &ports(&map), Privilege::Raw)
.expect("resumes past the tear");
journal.record_hosts(&[Host::new(second)]).expect("records");
journal.close().expect("closes");
let restored = read_findings(&directory).expect("the journal still reads");
let addresses: Vec<_> = restored.iter().map(Host::primary_ip).collect();
assert_eq!(
addresses,
vec![first, second],
"the whole record before the tear and the one after it both survive"
);
}
#[test]
fn an_append_to_a_findings_file_whose_header_never_landed_mends_it() {
let root = scratch("headerless-append");
let map = plan("192.0.2.1", "80");
let address: std::net::IpAddr = "192.0.2.1".parse().expect("an address");
let mut journal = begin(&root, &map);
let directory = journal.directory().to_path_buf();
fs::write(directory.join(HOSTS), b"").expect("empties");
journal
.record_hosts(&[Host::new(address)])
.expect("records");
journal.close().expect("closes");
let restored = read_findings(&directory).expect("reads");
assert_eq!(
restored.len(),
1,
"a record appended under no header is a record nothing reads back"
);
}
#[test]
fn an_append_to_a_file_this_engine_did_not_write_is_refused() {
let root = scratch("foreign-append");
let map = plan("192.0.2.1", "80");
let mut journal = begin(&root, &map);
let directory = journal.directory().to_path_buf();
let foreign: &[u8] = b"{\"not\":\"a journal\"}\n{\"still\":\"not\"}";
fs::write(directory.join(HOSTS), foreign).expect("plants");
assert!(
journal
.record_hosts(&[Host::new("192.0.2.1".parse().expect("an address"))])
.is_err(),
"a file with no header was appended to"
);
assert_eq!(
fs::read(directory.join(HOSTS)).expect("reads"),
foreign,
"the file was edited before it was refused"
);
journal.close().expect("closes");
}
#[test]
fn a_journal_from_an_older_format_reads_but_does_not_resume() {
let root = scratch("older-format");
let map = plan("192.0.2.1-192.0.2.4", "80");
let directory = {
let mut journal = begin(&root, &map);
journal
.record_hosts(&[Host::new("192.0.2.1".parse().expect("an address"))])
.expect("records");
let directory = journal.directory().to_path_buf();
journal.close().expect("closes");
directory
};
let path = directory.join(MANIFEST);
let mut manifest: JournalManifest =
serde_json::from_str(&fs::read_to_string(&path).expect("reads")).expect("parses");
manifest.journal_version = super::super::JOURNAL_VERSION - 1;
fs::write(&path, serde_json::to_vec(&manifest).expect("encodes")).expect("writes");
assert_eq!(
report(&directory).expect("reads back").host_count(),
1,
"an older journal still reads as the report its scan produced"
);
assert_eq!(
list(&root).expect("lists").len(),
1,
"and still appears in a listing"
);
match Journal::resume(&directory, &ports(&map), Privilege::Raw) {
Err(OpenError::VersionTooOld { found, understood }) => {
assert_eq!(found, super::super::JOURNAL_VERSION - 1);
assert_eq!(understood, super::super::JOURNAL_VERSION);
}
other => panic!("expected a refusal naming the format, got {other:?}"),
}
std::fs::remove_dir_all(&root).ok();
}
#[test]
fn a_journal_that_cannot_be_locked_leaves_no_directory() {
let root = scratch("unlockable");
let map = plan("192.0.2.1", "80");
let before = list(&root).expect("lists").len();
assert_eq!(before, 0);
let journal = begin(&root, &map);
let directory = journal.directory().to_path_buf();
assert!(matches!(
Journal::resume(&directory, &ports(&map), Privilege::Raw),
Err(OpenError::Locked(_))
));
assert!(directory.exists(), "the held journal is untouched");
journal.close().expect("closes");
std::fs::remove_dir_all(&root).ok();
}
#[test]
fn a_host_written_again_carries_only_what_changed_in_it() {
use crate::model::port::{Port, PortState, Protocol};
let root = scratch("delta");
let map = plan("192.0.2.1", "1-400");
let mut journal = begin(&root, &map);
let directory = journal.directory().to_path_buf();
let length = || fs::metadata(directory.join(HOSTS)).expect("stats").len();
let ip: std::net::IpAddr = "192.0.2.1".parse().expect("an address");
let mut host = Host::new(ip);
for number in 1..=400 {
host.add_port(Port::new(number, Protocol::Tcp, PortState::Closed));
}
let before = length();
journal
.record_hosts(std::slice::from_ref(&host))
.expect("records");
let first = length() - before;
host.add_port(Port::new(22, Protocol::Tcp, PortState::Open));
let before = length();
journal
.record_hosts(std::slice::from_ref(&host))
.expect("records");
let second = length() - before;
assert!(
second * 20 < first,
"one port opening appended {second} bytes against the host's {first}"
);
journal.close().expect("closes");
let restored = read_findings(&directory).expect("reads");
assert_eq!(restored.len(), 1);
assert_eq!(restored[0].port_count(), 400, "every port read back");
let open: Vec<u16> = restored[0]
.ports()
.filter(|port| port.state() == PortState::Open)
.map(|port| port.number())
.collect();
assert_eq!(open, [22], "the change read back over what it changed");
std::fs::remove_dir_all(&root).ok();
}
#[test]
fn a_record_counts_what_it_supersedes_and_not_what_it_adds() {
use crate::model::port::{Port, PortState, Protocol};
let ip: std::net::IpAddr = "192.0.2.1".parse().expect("an address");
let mut host = Host::new(ip);
host.add_port(Port::new(22, Protocol::Tcp, PortState::Closed));
let mut written = Written::default();
let (_, first) = written.delta(&host).expect("measures").expect("new");
let (rest, port) = (
u64::from(first.rest.length),
u64::from(first.ports[0].1.length),
);
written.update(first);
assert_eq!(written.superseded, 0, "a first record supersedes nothing");
host.add_port(Port::new(80, Protocol::Tcp, PortState::Closed));
let (_, second) = written.delta(&host).expect("measures").expect("a new port");
assert_eq!(second.ports.len(), 1, "only the new port is written");
written.update(second);
assert_eq!(written.superseded, rest, "the host's own fields, again");
host.add_port(Port::new(22, Protocol::Tcp, PortState::Open));
let (_, third) = written
.delta(&host)
.expect("measures")
.expect("a moved port");
let rest_again = written.hosts[&host.scoped_ip()]
.rest
.map_or(0, |mark| u64::from(mark.length));
written.update(third);
assert_eq!(
written.superseded,
rest + rest_again + port,
"and port 22's closed record under its open one"
);
}
#[test]
fn a_failed_compaction_leaves_nothing_behind_and_waits_before_trying_again() {
let root = scratch("failed-compaction");
let map = plan("192.0.2.1", "80");
let mut journal = begin(&root, &map);
let directory = journal.directory().to_path_buf();
let host = Host::new("192.0.2.1".parse().expect("an address"));
journal
.record_hosts(std::slice::from_ref(&host))
.expect("records");
journal.written.superseded = 3 * COMPACT_FLOOR;
journal.length = 4 * COMPACT_FLOOR;
assert!(journal.should_compact());
fs::remove_file(directory.join(HOSTS)).expect("removes the findings");
fs::create_dir_all(directory.join(HOSTS).join("occupied")).expect("blocks the name");
assert!(journal.compact(std::slice::from_ref(&host)).is_err());
assert!(
!directory.join(HOSTS).with_extension("jsonl-tmp").exists(),
"the partial copy is left holding the disk's room"
);
assert!(
!journal.should_compact(),
"a compaction that just failed is due again at once"
);
journal.written.superseded += 2 * COMPACT_FLOOR;
assert!(
journal.should_compact(),
"and again once as much is superseded"
);
drop(journal);
std::fs::remove_dir_all(&root).ok();
}
#[test]
fn a_findings_file_is_compacted_by_what_it_superseded() {
const MIB: u64 = 1024 * 1024;
assert!(
!outgrown(60 * MIB, 0),
"a file that grew by what the scan found holds nothing to drop"
);
assert!(!outgrown(6 * MIB, 3 * MIB), "a small file is left alone");
assert!(outgrown(9 * MIB, 5 * MIB), "past the floor it is rewritten");
assert!(
!outgrown(50 * MIB, 20 * MIB),
"twenty superseded beside thirty live is not yet worth it"
);
assert!(
outgrown(50 * MIB, 26 * MIB),
"twenty-six beside twenty-four is"
);
}
#[test]
fn compaction_bounds_the_findings_file_without_losing_a_host() {
use crate::model::host::HostStatus;
use crate::model::port::{Port, PortState, Protocol};
let root = scratch("compaction");
let map = plan("192.0.2.1-192.0.2.4", "80");
let addresses: Vec<std::net::IpAddr> = (1..=4)
.map(|last| format!("192.0.2.{last}").parse().expect("an address"))
.collect();
let directory = {
let mut journal = begin(&root, &map);
let mut live: Vec<Host> = addresses.iter().map(|ip| Host::new(*ip)).collect();
for round in 0..70 {
for host in &mut live {
host.set_status(HostStatus::Up);
host.add_port(Port::new(1000 + round, Protocol::Tcp, PortState::Open));
}
journal.record_hosts(&live).expect("records");
}
journal.compact(&live).expect("compacts");
assert!(!journal.should_compact(), "a file just written whole");
let directory = journal.directory().to_path_buf();
journal.close().expect("closes");
directory
};
let lines = std::fs::read_to_string(directory.join(HOSTS))
.expect("reads")
.lines()
.count();
assert_eq!(lines, 1 + 4, "a header and one record each");
let (journal, _) =
Journal::resume(&directory, &ports(&map), Privilege::Raw).expect("resumes");
let restored = journal.restored();
assert_eq!(restored.len(), 4);
for host in restored {
assert!(host.is_alive());
assert_eq!(host.port_count(), 70, "every round's port survived");
}
std::fs::remove_dir_all(&root).ok();
}
#[test]
fn a_journal_gives_back_the_plan_it_recorded() {
let root = scratch("reopen");
let map = plan("192.0.2.1-192.0.2.4", "80,443,u:53");
let directory = {
let journal = Journal::create(
&root,
&Plan::port_scan(&map, &Exclusions::none(), TcpScanTechnique::Fin),
Privilege::Raw,
"test",
)
.expect("creates");
let directory = journal.directory().to_path_buf();
journal.close().expect("closes");
directory
};
let (journal, _checkpoint, recovered) =
Journal::reopen(&directory, Privilege::Raw).expect("reopens");
let recovered = recovered.targets().expect("a port scan's plan");
assert_eq!(
recovered.iter().collect::<Vec<_>>(),
map.iter().collect::<Vec<_>>(),
"the same targets, in the same order, so positions still mean what they did"
);
assert_eq!(journal.manifest().technique(), TcpScanTechnique::Fin);
}
#[test]
fn a_journal_will_not_be_continued_under_different_privileges() {
let root = scratch("reopen-privilege");
let map = plan("192.0.2.1", "80");
let journal =
Journal::create(&root, &ports(&map), Privilege::Raw, "test").expect("creates");
let directory = journal.directory().to_path_buf();
journal.close().expect("closes");
let refused =
Journal::reopen(&directory, Privilege::Connect).expect_err("privileges differ");
assert!(matches!(refused, OpenError::PlanChanged(_)), "{refused:?}");
}
#[test]
fn preparing_a_root_creates_the_whole_path_to_it() {
let root = scratch("prepare-root")
.join("state")
.join("zond")
.join("journals");
assert!(!root.exists());
prepare_root(&root).expect("the path is created");
assert!(root.is_dir());
prepare_root(&root).expect("an existing root is not an error");
let plan = Plan::listen(vec![crate::model::ip::scoped::Zone::new(3, "en0")]);
Journal::create(&root, &plan, Privilege::Connect, "listening")
.expect("a journal is created inside it")
.close()
.expect("it closes");
}
#[test]
fn preparing_a_root_gives_away_what_it_created_and_repairs_what_root_left() {
use Hand::{Give, Reclaim};
let home = scratch("prepare-root-home");
let root = home.join(".local/state/zond/journals");
let prepared = |root: &Path, own: bool, home: Option<&Path>| {
let mut claimed = Vec::new();
prepare_root_with(root, own, home, |path, hand| {
claimed.push((path.to_path_buf(), hand));
})
.expect("the path is created");
claimed
};
let under = |pairs: &[(&str, Hand)]| -> Vec<(PathBuf, Hand)> {
pairs
.iter()
.map(|(below, hand)| (home.join(below), *hand))
.collect()
};
assert_eq!(
prepared(&root, true, Some(&home)),
under(&[
(".local", Give),
(".local/state", Give),
(".local/state/zond", Give),
(".local/state/zond/journals", Give),
]),
"a directory created for the journal was left to root"
);
fs::remove_dir_all(home.join(".local/state/zond")).expect("removes");
assert_eq!(
prepared(&root, true, Some(&home)),
under(&[
(".local", Reclaim),
(".local/state", Reclaim),
(".local/state/zond", Give),
(".local/state/zond/journals", Give),
])
);
assert_eq!(
prepared(&root, true, None),
under(&[
(".local/state/zond", Give),
(".local/state/zond/journals", Give)
])
);
let named = home.join("elsewhere/journals");
assert_eq!(
prepared(&named, false, Some(&home)),
[(named.clone(), Give)]
);
let _ = fs::remove_dir_all(&home);
}
#[test]
fn a_watch_is_never_complete_however_long_it_ran() {
use crate::model::ip::scoped::Zone;
let root = scratch("watch-never-complete");
let plan = Plan::listen(vec![Zone::new(3, "en0")]);
let journal = Journal::create(&root, &plan, Privilege::Raw, "listening on en0")
.expect("a journal is created");
journal.close().expect("it closes");
let entries = list(&root).expect("the root lists");
let entry = entries.first().expect("the watch is there");
assert_eq!(entry.kind(), ScanKind::Listen);
assert!(
!entry.is_complete(),
"another sitting can always be appended to a watch"
);
}
}