use crate::journal::Journal;
use crate::journal::cursor::Checkpoint;
use crate::journal::format::JournalError;
use crate::model::host::Host;
use crate::model::ip::set::IpSet;
use crate::report::{ScanKind, ScanPhase, ScannerKind, Unheard};
use crate::scanner::session::{ScanProgress, SoFar};
pub const CHECKPOINT_EVERY: std::time::Duration = std::time::Duration::from_secs(3);
#[derive(Debug)]
pub struct Checkpointing {
done: tokio::sync::oneshot::Sender<Vec<ScanPhase>>,
task: tokio::task::JoinHandle<()>,
#[cfg(test)]
written: tokio::sync::watch::Receiver<usize>,
}
impl Checkpointing {
pub async fn finish(self, phases: &[ScanPhase]) {
let _ = self.done.send(phases.to_vec());
let _ = self.task.await;
}
#[cfg(test)]
pub(crate) async fn kill(self) {
self.task.abort();
let _ = self.task.await;
}
#[cfg(test)]
pub(crate) async fn checkpointed(&mut self) {
self.written
.changed()
.await
.expect("the writer is still running");
}
}
pub fn spawn_checkpoints(journal: Journal, ctx: ScanProgress) -> Checkpointing {
let (done, mut stop) = tokio::sync::oneshot::channel::<Vec<ScanPhase>>();
#[cfg(test)]
let (counted, written) = tokio::sync::watch::channel(0usize);
let task = tokio::spawn(async move {
let mut writer = Writer::new(journal);
let phases = loop {
tokio::select! {
_ = tokio::time::sleep(CHECKPOINT_EVERY) => {
let ctx = ctx.clone();
let written = tokio::task::spawn_blocking(move || {
writer.checkpoint(&ctx);
writer
});
let Ok(returned) = written.await else {
return;
};
writer = returned;
#[cfg(test)]
counted.send_modify(|count| *count += 1);
}
finished = &mut stop => break finished.unwrap_or_default(),
}
};
let _ = tokio::task::spawn_blocking(move || {
let _ = writer.journal.end_sitting(&phases);
writer.close(&ctx, &phases);
})
.await;
});
Checkpointing {
done,
task,
#[cfg(test)]
written,
}
}
struct Writer {
journal: Journal,
failing: bool,
so_far: SoFar,
}
impl Writer {
fn new(journal: Journal) -> Self {
Self {
journal,
failing: false,
so_far: SoFar::default(),
}
}
fn checkpoint(&mut self, ctx: &ScanProgress) {
let cut = Cut::take(ctx);
self.write(ctx, cut);
}
fn write(&mut self, ctx: &ScanProgress, cut: Cut) {
let journal = &mut self.journal;
let outcome = if journal.should_compact() {
journal
.compact(&ctx.findings_snapshot())
.or_else(|_| journal.record_hosts(&cut.changed))
} else {
journal.record_hosts(&cut.changed)
}
.and_then(|()| journal.write_cursor(&cut.cursor));
if outcome.is_err() {
ctx.hand_back(&cut.changed);
}
let mut awaiting = cut.so_far.awaiting;
if outcome.is_ok() {
self.so_far = SoFar {
awaiting: IpSet::new(),
..cut.so_far
};
}
let mut named = IpSet::new();
for range in &self.so_far.silent {
named.insert_range(*range);
}
named.canonicalize();
awaiting.subtract(&named);
let standing = SoFar {
awaiting,
..self.so_far.clone()
};
match outcome {
Err(error) if !self.failing => {
self.failing = true;
ctx.record_failure(
ScannerKind::Journal,
format!(
"checkpoint failed: {} (resume replays more)",
reason(&error)
),
);
}
Err(_) => {}
Ok(()) => self.failing = false,
}
let tapes = ctx.take_tapes();
if self.journal.record_detections(&tapes).is_err() {
ctx.hand_back_tapes(tapes);
}
let _ = self
.journal
.record_standing(&ctx.standing_phases(&standing));
}
fn close(mut self, ctx: &ScanProgress, phases: &[ScanPhase]) {
let journal = &mut self.journal;
let unheard = Unheard::of(journal.earlier_phases().iter().chain(phases));
let cut = Cut::take(ctx);
let _ = if unheard.is_empty() {
journal.record_hosts(&cut.changed)
} else {
let mut kept = ctx.findings_snapshot();
kept.retain(|host| !unheard.drops(host));
journal.compact(&kept)
}
.and_then(|()| journal.write_cursor(&cut.cursor));
let _ = journal.record_detections(&ctx.take_tapes());
let _ = journal.record_finished(finished_hosts(ctx, phases));
let _ = self.journal.close();
}
}
struct Cut {
cursor: Checkpoint,
changed: Vec<Host>,
so_far: SoFar,
}
impl Cut {
fn take(ctx: &ScanProgress) -> Self {
Self::taking(ctx, || {})
}
fn taking(ctx: &ScanProgress, between: impl FnOnce()) -> Self {
let cursor = ctx.settlements().checkpoint();
between();
let changed = ctx.take_changed_findings();
let so_far = ctx.verdicts_so_far(&cursor);
Self {
cursor,
changed,
so_far,
}
}
}
fn finished_hosts(ctx: &ScanProgress, phases: &[ScanPhase]) -> Vec<String> {
let ran_to_its_end = !phases.is_empty()
&& phases
.iter()
.all(|phase| phase.kind() != ScanKind::Listen && phase.stopped().is_none());
if !ran_to_its_end {
return Vec::new();
}
let timed_out: std::collections::HashSet<std::net::IpAddr> = phases
.iter()
.flat_map(|phase| phase.timed_out().iter().copied())
.collect();
ctx.host_keys()
.into_iter()
.filter(|key| !timed_out.contains(&key.addr()))
.map(|key| key.to_string())
.collect()
}
fn reason(error: &JournalError) -> String {
let JournalError::Io(io) = error else {
return error.to_string();
};
let said = io.to_string();
let words = match io.raw_os_error() {
Some(code) => said
.strip_suffix(&format!(" (os error {code})"))
.unwrap_or(&said),
None => &said,
};
let mut letters = words.chars();
letters.next().map_or_else(String::new, |first| {
first.to_lowercase().chain(letters).collect()
})
}
#[cfg(test)]
mod tests {
use super::*;
use crate::journal::manifest::Plan;
use crate::model::exclusion::Exclusions;
use crate::model::target::{TargetMap, TargetSet};
use crate::model::technique::TcpScanTechnique;
use crate::system::privilege::Privilege;
fn scratch(name: &str) -> std::path::PathBuf {
let root =
std::env::temp_dir().join(format!("zond-checkpoint-{}-{name}", std::process::id()));
let _ = std::fs::remove_dir_all(&root);
std::fs::create_dir_all(&root).expect("scratch root");
root
}
fn one_target() -> Plan {
let mut map = TargetMap::new();
map.add_unit(TargetSet::new(
"192.0.2.1".parse().expect("an address"),
"80".parse().expect("ports"),
));
Plan::port_scan(&map, &Exclusions::none(), TcpScanTechnique::Syn)
}
fn holds_the_open_port(hosts: &[Host]) -> bool {
hosts.iter().any(|host| {
host.ports().any(|port| {
port.number() == 80 && port.state() == crate::model::port::PortState::Open
})
})
}
#[test]
fn a_target_settled_while_a_checkpoint_is_cut_is_not_skipped_without_its_finding() {
use crate::journal::settle::Outcome;
use crate::model::port::{Port, PortState, Protocol};
let root = scratch("cut");
let plan = one_target();
let journal = Journal::create(&root, &plan, Privilege::Raw, "test").expect("creates");
let directory = journal.directory().to_path_buf();
let (_session, ctx) = crate::scanner::session::ScanSession::new();
let progress = ctx.progress();
let ip: std::net::IpAddr = "192.0.2.1".parse().expect("an address");
let mut writer = Writer::new(journal);
let cut = Cut::taking(&progress, || {
ctx.update_host(ip, |host| {
host.add_port(Port::new(80, Protocol::Tcp, PortState::Open));
});
ctx.record_outcome(Outcome::Answered { position: 0 });
});
writer.write(&progress, cut);
drop(writer);
let (resumed, checkpoint) =
Journal::resume(&directory, &plan, Privilege::Raw).expect("resumes");
assert!(
holds_the_open_port(resumed.restored()) || !checkpoint.is_settled(0),
"the resume skips port 80 and restores no finding for it: {checkpoint:?}"
);
drop(resumed);
std::fs::remove_dir_all(&root).ok();
}
#[test]
fn findings_a_failed_checkpoint_took_are_written_by_the_next_one() {
use crate::journal::settle::Outcome;
use crate::model::port::{Port, PortState, Protocol};
let root = scratch("handed-back");
let plan = one_target();
let journal = Journal::create(&root, &plan, Privilege::Raw, "test").expect("creates");
let directory = journal.directory().to_path_buf();
let findings = directory.join("hosts.jsonl");
let aside = directory.join("hosts.jsonl-aside");
let (_session, ctx) = crate::scanner::session::ScanSession::new();
let progress = ctx.progress();
let ip: std::net::IpAddr = "192.0.2.1".parse().expect("an address");
ctx.update_host(ip, |host| {
host.add_port(Port::new(80, Protocol::Tcp, PortState::Open));
});
ctx.record_outcome(Outcome::Answered { position: 0 });
let mut writer = Writer::new(journal);
std::fs::rename(&findings, &aside).expect("moves the findings aside");
writer.checkpoint(&progress);
std::fs::rename(&aside, &findings).expect("puts them back");
writer.checkpoint(&progress);
drop(writer);
let (resumed, checkpoint) =
Journal::resume(&directory, &plan, Privilege::Raw).expect("resumes");
assert!(
holds_the_open_port(resumed.restored()) || !checkpoint.is_settled(0),
"the resume skips port 80 and restores no finding for it: {checkpoint:?}"
);
drop(resumed);
std::fs::remove_dir_all(&root).ok();
}
#[test]
fn a_pass_that_changed_nothing_writes_no_record() {
use crate::model::port::{Port, PortState, Protocol};
let root = scratch("unchanged");
let journal =
Journal::create(&root, &one_target(), Privilege::Raw, "test").expect("creates");
let findings = journal.directory().join("hosts.jsonl");
let (_session, ctx) = crate::scanner::session::ScanSession::new();
let progress = ctx.progress();
let ip: std::net::IpAddr = "192.0.2.1".parse().expect("an address");
ctx.update_host(ip, |host| {
host.add_port(Port::new(80, Protocol::Tcp, PortState::Open));
});
let mut writer = Writer::new(journal);
writer.checkpoint(&progress);
let written = std::fs::metadata(&findings).expect("written").len();
ctx.write_host(ip, |_| false);
writer.checkpoint(&progress);
assert_eq!(
std::fs::metadata(&findings).expect("written").len(),
written,
"the findings file grew for a pass that changed nothing"
);
drop(writer);
std::fs::remove_dir_all(&root).ok();
}
#[test]
fn a_checkpoint_takes_the_ports_that_changed_and_not_the_whole_host() {
use crate::model::port::{Port, PortState, Protocol};
const WIDE: u16 = 2_000;
let mut map = TargetMap::new();
map.add_unit(TargetSet::new(
"192.0.2.1".parse().expect("an address"),
format!("1-{WIDE}").parse().expect("ports"),
));
let plan = Plan::port_scan(&map, &Exclusions::none(), TcpScanTechnique::Syn);
let root = scratch("touched");
let journal = Journal::create(&root, &plan, Privilege::Raw, "test").expect("creates");
let directory = journal.directory().to_path_buf();
let (_session, ctx) = crate::scanner::session::ScanSession::new();
let progress = ctx.progress();
let ip: std::net::IpAddr = "192.0.2.1".parse().expect("an address");
ctx.update_host(ip, |host| {
for number in 1..=WIDE {
host.add_port(Port::new(number, Protocol::Tcp, PortState::Closed));
}
});
let mut writer = Writer::new(journal);
writer.checkpoint(&progress);
ctx.update_host(ip, |host| {
host.add_port(Port::new(80, Protocol::Tcp, PortState::Open));
});
let taken = progress.take_changed_findings();
assert_eq!(taken.len(), 1);
assert_eq!(
taken[0].port_count(),
1,
"a checkpoint copied every port of a host one port of which changed"
);
progress.hand_back(&taken);
writer.checkpoint(&progress);
drop(writer);
let (resumed, _) = Journal::resume(&directory, &plan, Privilege::Raw).expect("resumes");
let host = &resumed.restored()[0];
assert_eq!(
host.port_count(),
usize::from(WIDE),
"every port reads back"
);
assert!(
holds_the_open_port(resumed.restored()),
"with the one that changed"
);
drop(resumed);
std::fs::remove_dir_all(&root).ok();
}
#[test]
fn tapes_a_failed_checkpoint_took_are_written_by_the_next_one() {
use crate::detect::compute::{CapTape, CapTapeRecord, DetectionRunRecord};
use crate::record::DetectionIdRecord;
let root = scratch("tapes-handed-back");
let journal =
Journal::create(&root, &one_target(), Privilege::Raw, "test").expect("creates");
let directory = journal.directory().to_path_buf();
let tapes = directory.join("detections.jsonl");
let (_session, ctx) = crate::scanner::session::ScanSession::new();
let progress = ctx.progress();
let run = |detection: &str| DetectionRunRecord {
host: "192.0.2.1".to_string(),
host_name: None,
port: 80,
protocol: "tcp".to_string(),
detection: DetectionIdRecord {
id: detection.to_string(),
version: "1.0.0".to_string(),
content_hash: "0".repeat(64),
},
responses: vec!["HTTP/1.1 200 OK\r\n\r\n".to_string()],
tape: CapTapeRecord::from(&CapTape::default()),
};
ctx.tapes.record(|| run("first"));
let mut writer = Writer::new(journal);
std::fs::create_dir(&tapes).expect("stands a directory in the way");
writer.checkpoint(&progress);
std::fs::remove_dir(&tapes).expect("clears the way");
ctx.tapes.record(|| run("second"));
writer.checkpoint(&progress);
drop(writer);
let written: Vec<String> = crate::journal::store::read_detections(&directory)
.expect("reads")
.into_iter()
.map(|run| run.detection.id)
.collect();
assert_eq!(written, ["first", "second"], "in the order they ran");
std::fs::remove_dir_all(&root).ok();
}
#[cfg(unix)]
#[tokio::test]
async fn a_checkpoint_that_blocks_does_not_hold_the_runtime() {
use std::io::Read;
use std::os::unix::ffi::OsStrExt;
use std::time::{Duration, Instant};
const RELEASED_AFTER: Duration = Duration::from_secs(20);
let root = scratch("blocking");
let plan = one_target();
let journal = Journal::create(&root, &plan, Privilege::Raw, "test").expect("creates");
let findings = journal.directory().join("hosts.jsonl");
std::fs::remove_file(&findings).expect("removes the findings file");
let name = std::ffi::CString::new(findings.as_os_str().as_bytes()).expect("a path");
assert_eq!(
unsafe { libc::mkfifo(name.as_ptr(), 0o600) },
0,
"makes a pipe"
);
let (_session, ctx) = crate::scanner::session::ScanSession::new();
let ip: std::net::IpAddr = "192.0.2.1".parse().expect("an address");
ctx.update_host(ip, |host| {
host.set_status(crate::model::host::HostStatus::Up);
});
let (release, released) = std::sync::mpsc::channel::<()>();
let reader = std::thread::spawn(move || {
let _ = released.recv_timeout(RELEASED_AFTER);
let mut read = Vec::new();
std::fs::File::open(&findings)
.and_then(|mut pipe| pipe.read_to_end(&mut read))
.expect("reads the pipe");
read
});
let ticker = spawn_checkpoints(journal, ctx.progress());
let started = Instant::now();
let across = CHECKPOINT_EVERY + Duration::from_millis(500);
tokio::time::sleep(across).await;
let late = started.elapsed() - across;
let _ = release.send(());
let read = tokio::task::spawn_blocking(move || reader.join().expect("the reader"))
.await
.expect("joins");
ticker.kill().await;
std::fs::remove_dir_all(&root).ok();
assert!(
String::from_utf8_lossy(&read).contains("192.0.2.1"),
"the checkpoint never reached the pipe, so it proves nothing"
);
assert!(
late < RELEASED_AFTER / 2,
"a timer across the checkpoint ran {late:?} late"
);
}
fn open_a_port_phase(
ctx: &crate::scanner::session::ScanContext,
) -> crate::scanner::recorder::PhaseRecorder {
use crate::report::{ScanKind, TargetScope};
let mut addresses: crate::model::ip::set::IpSet = "192.0.2.1".parse().expect("an address");
let scope = TargetScope::from_ip_set(&mut addresses, &Exclusions::none());
crate::scanner::recorder::PhaseRecorder::start(
ScanKind::PortScan,
Privilege::Raw,
scope,
&crate::config::ZondConfig::default(),
)
.opening_in(ctx)
}
#[test]
fn a_killed_sitting_leaves_a_record_of_its_phase() {
let root = scratch("killed-phase");
let plan = one_target();
let journal = Journal::create(&root, &plan, Privilege::Raw, "test").expect("creates");
let directory = journal.directory().to_path_buf();
let (_session, ctx) = crate::scanner::session::ScanSession::new();
let _recorder = open_a_port_phase(&ctx);
ctx.record_failure(ScannerKind::SynPort, "the capture closed".to_string());
let mut writer = Writer::new(journal);
writer.checkpoint(&ctx.progress());
drop(writer);
let report = crate::journal::store::report(&directory).expect("reads");
assert_eq!(report.phases().len(), 1, "the killed sitting's phase");
assert_eq!(report.phases()[0].kind(), crate::report::ScanKind::PortScan);
assert_eq!(report.failures().count(), 1, "and what failed in it");
let (resumed, _) = Journal::resume(&directory, &plan, Privilege::Raw).expect("resumes");
assert_eq!(resumed.earlier_phases().len(), 1, "a resume carries it too");
drop(resumed);
std::fs::remove_dir_all(&root).ok();
}
#[test]
fn a_killed_sitting_marks_its_open_phase_and_the_report_partial() {
let root = scratch("killed-open");
let plan = one_target();
let journal = Journal::create(&root, &plan, Privilege::Raw, "test").expect("creates");
let directory = journal.directory().to_path_buf();
let (_session, ctx) = crate::scanner::session::ScanSession::new();
let _recorder = open_a_port_phase(&ctx);
let mut writer = Writer::new(journal);
writer.checkpoint(&ctx.progress());
drop(writer);
let report = crate::journal::store::report(&directory).expect("reads");
assert!(
report.phases()[0].is_open(),
"the killed phase never closed"
);
assert!(
report.is_partial(),
"a job whose only sitting was killed reads as complete"
);
let (mut resumed, _) = Journal::resume(&directory, &plan, Privilege::Raw).expect("resumes");
let (_session, ctx) = crate::scanner::session::ScanSession::new();
let closed = open_a_port_phase(&ctx).finish(&ctx);
resumed.end_sitting(closed.phases()).expect("records");
drop(resumed);
let report = crate::journal::store::report(&directory).expect("reads");
let open: Vec<bool> = report.phases().iter().map(ScanPhase::is_open).collect();
assert_eq!(open, [true, false], "each sitting's phase as it ended");
assert!(
!report.is_partial(),
"the resume finished what the kill left"
);
std::fs::remove_dir_all(&root).ok();
}
#[test]
fn a_sitting_that_ends_is_recorded_once() {
let root = scratch("ended-phase");
let plan = one_target();
let journal = Journal::create(&root, &plan, Privilege::Raw, "test").expect("creates");
let directory = journal.directory().to_path_buf();
let (_session, ctx) = crate::scanner::session::ScanSession::new();
let recorder = open_a_port_phase(&ctx);
let mut writer = Writer::new(journal);
writer.checkpoint(&ctx.progress());
let report = recorder.finish(&ctx);
writer
.journal
.end_sitting(report.phases())
.expect("records");
writer.close(&ctx.progress(), report.phases());
let phases = crate::journal::store::report(&directory).expect("reads");
assert_eq!(phases.phases().len(), 1);
let standing = std::fs::read_dir(&directory)
.expect("lists")
.filter_map(Result::ok)
.filter(|entry| entry.file_name().to_string_lossy().starts_with("sitting-"))
.count();
assert_eq!(standing, 0, "the standing record outlived the sitting");
std::fs::remove_dir_all(&root).ok();
}
#[test]
fn a_journal_that_cannot_be_written_is_told_once_and_short() {
let root = scratch("told-once");
let mut map = TargetMap::new();
map.add_unit(TargetSet::new(
"192.0.2.1-192.0.2.4".parse().expect("a range"),
"80".parse().expect("ports"),
));
let plan = Plan::port_scan(&map, &Exclusions::none(), TcpScanTechnique::Syn);
let journal = Journal::create(&root, &plan, Privilege::Raw, "test").expect("creates");
std::fs::remove_dir_all(journal.directory()).expect("removes");
let (_session, ctx) = crate::scanner::session::ScanSession::new();
let mut writer = Writer::new(journal);
for _ in 0..3 {
writer.checkpoint(&ctx.progress());
}
let told: Vec<_> = ctx
.failures_snapshot()
.into_iter()
.filter(|failure| failure.scanner() == ScannerKind::Journal)
.map(|failure| failure.reason().to_owned())
.collect();
assert_eq!(told.len(), 1, "{told:#?}");
assert!(
told[0].len() <= 80,
"{:?} is {} long",
told[0],
told[0].len()
);
std::fs::remove_dir_all(&root).ok();
}
}