1use std::collections::HashSet;
55use std::fs;
56use std::path::{Path, PathBuf};
57use std::time::{Duration, SystemTime};
58
59use super::cursor::Checkpoint;
60use super::file::{
61 append_existing, claim_directory_for_invoking_user, create_private as create_private_file,
62 create_private_directory, exists, kinds, names, open_existing, open_to_read,
63 remove as remove_file, remove_directory, replace,
64};
65use super::format::JournalError;
66use super::lock::{Lock, LockRefused, LockState};
67use super::manifest::{JobOptions, JournalManifest, Plan, PlanChanged};
68use super::ownership::Kind;
69use super::paths::{ID_ALPHABET, ID_CHARS};
70use super::settle::Settlements;
71use crate::detect::compute::{DetectionLine, DetectionRunRecord, PortRunsRecord};
72use crate::model::host::Host;
73use crate::model::ip::scoped::ScopedIp;
74use crate::model::port::Protocol;
75use crate::record::{HostRecord, PhaseRecord};
76use crate::report::{ScanKind, ScanPhase, ScanReport, Unheard};
77use crate::system::privilege::Privilege;
78
79const MANIFEST: &str = "manifest.json";
80const CURSOR: &str = "cursor.json";
81const HOSTS: &str = "hosts.jsonl";
82const PHASES: &str = "phases.jsonl";
83const DETECTIONS: &str = "detections.jsonl";
84const FINISHED: &str = "finished.jsonl";
87const LOCK: &str = "LOCK";
88const SITTING: &str = "sitting-";
90const OPTIONS: &str = "options.json";
93
94#[non_exhaustive]
96#[derive(Debug, thiserror::Error)]
97pub enum OpenError {
98 #[error("{0}")]
100 PlanChanged(#[from] PlanChanged),
101
102 #[error("this journal records a {held} and cannot be continued as a {asked}")]
107 WrongPhase {
108 held: &'static str,
110 asked: &'static str,
112 },
113
114 #[error("{0}")]
116 Locked(LockRefused),
117
118 #[error("{0}")]
120 Journal(#[from] JournalError),
121}
122
123impl From<LockRefused> for OpenError {
124 fn from(refused: LockRefused) -> Self {
125 OpenError::Locked(refused)
126 }
127}
128
129#[derive(Debug)]
133pub struct Journal {
134 directory: PathBuf,
135 manifest: JournalManifest,
136 lock: Lock,
137 resume_point: Checkpoint,
138 restored: Vec<Host>,
139 earlier: Vec<ScanPhase>,
140 created: bool,
143 options: Option<JobOptions>,
144 written: Written,
147 length: u64,
149 compact_after: u64,
152 sitting: PathBuf,
155}
156
157impl Journal {
158 pub fn create(
163 root: &Path,
164 plan: &Plan,
165 privilege: Privilege,
166 summary: impl Into<String>,
167 ) -> Result<Self, OpenError> {
168 let (id, directory) = claim_directory(root)?;
169 let manifest = JournalManifest::new(id, plan, privilege, summary);
170
171 match Self::furnish(&directory, manifest) {
175 Ok(journal) => Ok(journal),
176 Err(error) => {
177 let _ = remove_directory(&directory);
178 Err(error)
179 }
180 }
181 }
182
183 fn furnish(directory: &Path, manifest: JournalManifest) -> Result<Self, OpenError> {
187 write_private(
188 &directory.join(MANIFEST),
189 &serde_json::to_vec(&manifest).map_err(JournalError::json)?,
190 )?;
191
192 let lock = Lock::acquire(&directory.join(LOCK))?;
193 let sitting = sitting_file(directory, &lock);
194
195 let mut journal = Self {
196 directory: directory.to_path_buf(),
197 manifest,
198 lock,
199 resume_point: Checkpoint::default(),
200 restored: Vec::new(),
201 earlier: Vec::new(),
202 created: true,
203 options: None,
204 written: Written::default(),
205 length: 0,
206 compact_after: 0,
207 sitting,
208 };
209 journal.open_findings()?;
210 journal.length = findings_length(directory)?;
211 Ok(journal)
212 }
213
214 pub fn resume(
220 directory: &Path,
221 plan: &Plan,
222 privilege: Privilege,
223 ) -> Result<(Self, Checkpoint), OpenError> {
224 Self::resume_locking(directory, plan, privilege, Lock::acquire)
225 }
226
227 fn resume_locking(
229 directory: &Path,
230 plan: &Plan,
231 privilege: Privilege,
232 lock: fn(&Path) -> Result<Lock, LockRefused>,
233 ) -> Result<(Self, Checkpoint), OpenError> {
234 let manifest = read_manifest(directory)?;
235 if manifest.kind() != plan.kind() {
238 return Err(OpenError::WrongPhase {
239 held: phase_name(manifest.kind()),
240 asked: phase_name(plan.kind()),
241 });
242 }
243 manifest.covers(plan, privilege)?;
245
246 let lock = lock(&directory.join(LOCK))?;
247 let sitting = sitting_file(directory, &lock);
248 let checkpoint = read_checkpoint(directory)?;
249 let earlier = read_phases(directory)?;
250 let unheard = Unheard::decided(&earlier);
254 let mut restored = read_findings(directory)?;
255 restored.retain(|host| !unheard.drops(host));
256 let length = findings_length(directory)?;
260 let written = Written::holding(&restored, length)?;
261
262 Ok((
263 Self {
264 directory: directory.to_path_buf(),
265 manifest,
266 lock,
267 resume_point: checkpoint.clone(),
268 restored,
269 earlier,
270 created: false,
271 options: read_options(directory)?,
272 written,
273 length,
274 compact_after: 0,
275 sitting,
276 },
277 checkpoint,
278 ))
279 }
280
281 pub fn reopen(
291 directory: &Path,
292 privilege: Privilege,
293 ) -> Result<(Self, Checkpoint, Plan), OpenError> {
294 let plan = read_manifest(directory)?.recorded();
295 let (journal, checkpoint) = Self::resume(directory, &plan, privilege)?;
296 Ok((journal, checkpoint, plan))
297 }
298
299 pub fn take_over(
308 directory: &Path,
309 privilege: Privilege,
310 ) -> Result<(Self, Checkpoint, Plan), OpenError> {
311 let plan = read_manifest(directory)?.recorded();
312 let (journal, checkpoint) =
313 Self::resume_locking(directory, &plan, privilege, Lock::take_over)?;
314 Ok((journal, checkpoint, plan))
315 }
316
317 pub fn restored(&self) -> &[Host] {
327 &self.restored
328 }
329
330 pub fn record_hosts(&mut self, hosts: &[Host]) -> Result<(), JournalError> {
342 let mut records = Vec::with_capacity(hosts.len());
343 for host in hosts {
344 records.extend(self.written.delta(host)?);
345 }
346 if records.is_empty() {
347 return Ok(());
348 }
349
350 let file = open_for_append(&self.directory.join(HOSTS))?;
351 let measured = file.try_clone()?;
353
354 let mut deltas = Vec::with_capacity(records.len());
355 let mut writer = crate::journal::format::Writer::append(std::io::BufWriter::new(file));
356 for (record, delta) in records {
357 writer.write(&record)?;
358 deltas.push(delta);
359 }
360 writer.flush()?;
361 drop(writer);
362
363 for delta in deltas {
364 self.written.update(delta);
365 }
366 self.length = measured.metadata()?.len();
367 Ok(())
368 }
369
370 pub fn record_detections(&mut self, runs: &[DetectionRunRecord]) -> Result<(), JournalError> {
382 if runs.is_empty() {
383 return Ok(());
384 }
385
386 let path = self.directory.join(DETECTIONS);
389 let (file, mut writer) = match open_for_append(&path) {
390 Ok(file) => {
391 let written = file.try_clone()?;
392 let writer =
393 crate::journal::format::Writer::append(std::io::BufWriter::new(written));
394 (file, writer)
395 }
396 Err(JournalError::Io(error)) if error.kind() == std::io::ErrorKind::NotFound => {
397 let file = create_private_file(&path)?;
398 let written = file.try_clone()?;
399 let writer =
400 crate::journal::format::Writer::create(std::io::BufWriter::new(written))?;
401 (file, writer)
402 }
403 Err(error) => return Err(error),
404 };
405 writer.flush()?;
408 let began = file.metadata()?.len();
409 let outcome = PortRunsRecord::grouping(runs)
410 .iter()
411 .try_for_each(|line| writer.write(line))
412 .and_then(|()| writer.flush());
413 if outcome.is_err() {
414 drop(writer);
417 let _ = file.set_len(began);
418 }
419 outcome
420 }
421
422 pub(crate) fn record_finished(
434 &mut self,
435 hosts: impl IntoIterator<Item = String>,
436 ) -> Result<(), JournalError> {
437 let record = FinishedRecord {
438 hosts: hosts.into_iter().collect(),
439 };
440 if record.hosts.is_empty() {
441 return Ok(());
442 }
443
444 let path = self.directory.join(FINISHED);
446 let mut writer = match open_for_append(&path) {
447 Ok(file) => crate::journal::format::Writer::append(std::io::BufWriter::new(file)),
448 Err(JournalError::Io(error)) if error.kind() == std::io::ErrorKind::NotFound => {
449 let file = create_private_file(&path)?;
450 crate::journal::format::Writer::create(std::io::BufWriter::new(file))?
451 }
452 Err(error) => return Err(error),
453 };
454 writer.write(&record)?;
455 writer.flush()
456 }
457
458 pub(crate) fn finished_hosts(&self) -> Result<HashSet<String>, JournalError> {
464 let file = match open_to_read(&self.directory.join(FINISHED)) {
465 Ok(file) => file,
466 Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(HashSet::new()),
467 Err(e) => return Err(e.into()),
468 };
469 let mut reader = match crate::journal::format::Reader::open(std::io::BufReader::new(file)) {
470 Ok(reader) => reader,
471 Err(JournalError::NotAJournal) => return Ok(HashSet::new()),
472 Err(e) => return Err(e),
473 };
474
475 let mut hosts = HashSet::new();
476 while let Some(record) = reader.read::<FinishedRecord>()? {
477 hosts.extend(record.hosts);
478 }
479 Ok(hosts)
480 }
481
482 pub fn should_compact(&self) -> bool {
494 outgrown(self.length, self.written.superseded)
495 && self.written.superseded >= self.compact_after
496 }
497
498 pub fn compact(&mut self, all: &[Host]) -> Result<(), JournalError> {
509 let compacted = self.write_whole(all);
510 if compacted.is_err() {
511 let superseded = self.written.superseded;
512 let live = self.length.saturating_sub(superseded);
513 self.compact_after = superseded.saturating_add(COMPACT_FLOOR.max(live));
514 }
515 compacted
516 }
517
518 fn write_whole(&mut self, all: &[Host]) -> Result<(), JournalError> {
521 let destination = self.directory.join(HOSTS);
522 let (written, length) = replace(
523 &destination,
524 &destination.with_extension("jsonl-tmp"),
525 |file| {
526 let measured = file.try_clone()?;
529 let mut written = Written::default();
530 let mut writer =
531 crate::journal::format::Writer::create(std::io::BufWriter::new(file))?;
532 for host in all {
533 if let Some((record, delta)) = Written::default().delta(host)? {
535 writer.write(&record)?;
536 written.update(delta);
537 }
538 }
539 writer.flush()?;
540 drop(writer);
541 Ok::<_, JournalError>((written, measured.metadata()?.len()))
542 },
543 )?;
544 self.written = written;
545 self.length = length;
546 self.compact_after = 0;
547 Ok(())
548 }
549
550 pub fn earlier_phases(&self) -> &[ScanPhase] {
555 &self.earlier
556 }
557
558 pub fn record_phases(&mut self, phases: &[ScanPhase]) -> Result<(), JournalError> {
560 if phases.is_empty() {
561 return Ok(());
562 }
563
564 let file = open_for_append(&self.directory.join(PHASES))?;
565
566 let mut writer = crate::journal::format::Writer::append(std::io::BufWriter::new(file));
567 for phase in phases {
568 writer.write(&PhaseRecord::from(phase))?;
569 }
570 writer.flush()
571 }
572
573 pub(crate) fn record_standing(&mut self, phases: &[ScanPhase]) -> Result<(), JournalError> {
579 if phases.is_empty() {
580 return Ok(());
581 }
582 replace(
583 &self.sitting,
584 &self.sitting.with_extension("jsonl-tmp"),
585 |file| {
586 let mut writer =
587 crate::journal::format::Writer::create(std::io::BufWriter::new(file))?;
588 for phase in phases {
589 writer.write(&PhaseRecord::from(phase))?;
590 }
591 writer.flush()
592 },
593 )
594 }
595
596 pub(crate) fn end_sitting(&mut self, phases: &[ScanPhase]) -> Result<(), JournalError> {
602 if phases.is_empty() {
603 return Ok(());
604 }
605 self.record_phases(phases)?;
606 match remove_file(&self.sitting) {
607 Ok(()) => Ok(()),
608 Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),
609 Err(e) => Err(e.into()),
610 }
611 }
612
613 fn open_findings(&mut self) -> Result<(), JournalError> {
616 for name in [HOSTS, PHASES] {
617 let path = self.directory.join(name);
618 let file = create_private_file(&path)?;
619 crate::journal::format::Writer::create(std::io::BufWriter::new(file))?.flush()?;
620 }
621 Ok(())
622 }
623
624 pub fn resume_point(&self) -> &Checkpoint {
629 &self.resume_point
630 }
631
632 pub fn checkpoint(&mut self, settlements: &Settlements) -> Result<(), JournalError> {
637 self.write_cursor(&settlements.checkpoint())
638 }
639
640 pub(crate) fn write_cursor(&mut self, cursor: &Checkpoint) -> Result<(), JournalError> {
645 cursor.write_atomically(&self.directory.join(CURSOR))?;
646 self.lock.beat()
647 }
648
649 pub fn record(
658 &mut self,
659 hosts: &[Host],
660 settlements: &Settlements,
661 ) -> Result<(), JournalError> {
662 self.record_hosts(hosts)?;
663 self.checkpoint(settlements)
664 }
665
666 pub fn options(&self) -> Option<&JobOptions> {
676 self.options.as_ref()
677 }
678
679 pub(crate) fn record_options(&mut self, options: JobOptions) -> Result<(), JournalError> {
690 if self.options.is_some() || !self.is_untouched() {
691 return Ok(());
692 }
693
694 write_private(
695 &self.directory.join(OPTIONS),
696 &serde_json::to_vec(&options).map_err(JournalError::json)?,
697 )?;
698 let numbered = self
699 .manifest
700 .recorded()
701 .numbered_targets(&options.excluded_ports());
702 self.options = Some(options);
703 if self.created && numbered != self.manifest.total_targets {
704 self.manifest.total_targets = numbered;
705 write_private(
706 &self.directory.join(MANIFEST),
707 &serde_json::to_vec(&self.manifest).map_err(JournalError::json)?,
708 )?;
709 }
710 Ok(())
711 }
712
713 fn is_untouched(&self) -> bool {
721 self.earlier.is_empty()
722 && self.restored.is_empty()
723 && !exists(&self.directory.join(CURSOR)).unwrap_or(true)
724 }
725
726 pub fn manifest(&self) -> &JournalManifest {
728 &self.manifest
729 }
730
731 pub fn directory(&self) -> &Path {
733 &self.directory
734 }
735
736 pub(crate) fn withdraw(self) {
750 if self.created && self.is_untouched() {
751 let _ = remove_directory(&self.directory);
753 }
754 }
755
756 pub fn close(self) -> Result<(), JournalError> {
758 self.lock.release()
759 }
760}
761
762fn outgrown(length: u64, superseded: u64) -> bool {
765 superseded > COMPACT_FLOOR.max(length.saturating_sub(superseded))
766}
767
768const COMPACT_FLOOR: u64 = 4 * 1024 * 1024;
770
771#[derive(Debug, Default)]
780struct Written {
781 hosts: std::collections::HashMap<ScopedIp, HeldHost>,
782 superseded: u64,
784}
785
786#[derive(Debug, Default)]
788struct HeldHost {
789 rest: Option<Mark>,
791 ports: std::collections::HashMap<PortKey, Mark>,
792}
793
794type PortKey = (u16, Protocol);
796
797#[derive(Debug, Clone, Copy, PartialEq, Eq)]
799struct Mark {
800 digest: u64,
801 length: u32,
802}
803
804#[derive(Debug)]
806struct Delta {
807 key: ScopedIp,
808 rest: Mark,
809 ports: Vec<(PortKey, Mark)>,
810}
811
812impl Delta {
813 fn length(&self) -> u64 {
815 u64::from(self.rest.length)
816 + self
817 .ports
818 .iter()
819 .map(|(_, mark)| u64::from(mark.length))
820 .sum::<u64>()
821 }
822}
823
824impl Written {
825 fn holding(hosts: &[Host], length: u64) -> Result<Self, JournalError> {
828 let mut written = Self::default();
829 let mut live = 0;
830 for host in hosts {
831 if let Some((_, delta)) = Self::default().delta(host)? {
833 live += delta.length();
834 written.update(delta);
835 }
836 }
837 written.superseded = length.saturating_sub(live);
838 Ok(written)
839 }
840
841 fn delta(&self, host: &Host) -> Result<Option<(HostRecord, Delta)>, JournalError> {
849 let held = self.hosts.get(&host.scoped_ip());
850 let mut record = HostRecord::from(host);
851 let ports = std::mem::take(&mut record.ports);
853 let rest = mark(&record)?;
854 let mut changed = Vec::new();
855 for (port, written) in host.ports().zip(ports) {
856 debug_assert_eq!(
857 port.number(),
858 written.port,
859 "a record lists its host's ports"
860 );
861 let key = (port.number(), port.protocol());
862 let mark = mark(&written)?;
863 let unchanged = held
864 .and_then(|held| held.ports.get(&key))
865 .is_some_and(|was| was.digest == mark.digest);
866 if !unchanged {
867 changed.push((key, mark));
868 record.ports.push(written);
869 }
870 }
871 let unchanged = changed.is_empty()
872 && held.and_then(|held| held.rest).map(|was| was.digest) == Some(rest.digest);
873 if unchanged {
874 return Ok(None);
875 }
876 let delta = Delta {
877 key: host.scoped_ip(),
878 rest,
879 ports: changed,
880 };
881 Ok(Some((record, delta)))
882 }
883
884 fn update(&mut self, delta: Delta) {
887 let held = match self.hosts.entry(delta.key) {
888 std::collections::hash_map::Entry::Occupied(slot) => {
889 let held = slot.into_mut();
890 self.superseded += held.rest.map_or(0, |was| u64::from(was.length));
891 held
892 }
893 std::collections::hash_map::Entry::Vacant(slot) => slot.insert(HeldHost::default()),
894 };
895 held.rest = Some(delta.rest);
896 for (key, mark) in delta.ports {
897 if let Some(was) = held.ports.insert(key, mark) {
898 self.superseded += u64::from(was.length);
899 }
900 }
901 }
902}
903
904fn mark(record: &impl serde::Serialize) -> Result<Mark, JournalError> {
907 use std::hash::Hasher;
908
909 struct Hashing {
910 hasher: std::collections::hash_map::DefaultHasher,
911 length: u64,
912 }
913 impl std::io::Write for Hashing {
914 fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
915 self.hasher.write(bytes);
916 self.length += bytes.len() as u64;
917 Ok(bytes.len())
918 }
919 fn flush(&mut self) -> std::io::Result<()> {
920 Ok(())
921 }
922 }
923
924 let mut hashing = Hashing {
925 hasher: std::collections::hash_map::DefaultHasher::new(),
926 length: 0,
927 };
928 serde_json::to_writer(&mut hashing, record).map_err(JournalError::json)?;
929 Ok(Mark {
930 digest: hashing.hasher.finish(),
931 length: u32::try_from(hashing.length + 1).unwrap_or(u32::MAX),
934 })
935}
936
937#[non_exhaustive]
940#[derive(Debug, Clone)]
941pub struct Entry {
942 pub directory: PathBuf,
944 pub manifest: JournalManifest,
946 pub checkpoint: Option<Checkpoint>,
952 pub lock: LockState,
954}
955
956impl Entry {
957 pub fn new(
960 directory: PathBuf,
961 manifest: JournalManifest,
962 checkpoint: Option<Checkpoint>,
963 lock: LockState,
964 ) -> Self {
965 Self {
966 directory,
967 manifest,
968 checkpoint,
969 lock,
970 }
971 }
972
973 pub fn is_complete(&self) -> bool {
979 if self.kind() == ScanKind::Listen {
982 return false;
983 }
984
985 self.settled()
986 .is_some_and(|settled| settled >= self.manifest.total_targets)
987 }
988
989 pub fn kind(&self) -> ScanKind {
994 self.manifest.kind()
995 }
996
997 pub fn settled(&self) -> Option<u128> {
1006 self.checkpoint
1007 .as_ref()
1008 .map(|checkpoint| u128::from(checkpoint.settled_count()))
1009 }
1010}
1011
1012pub fn prepare_root(root: &Path) -> std::io::Result<()> {
1042 let own = super::paths::root().as_deref() == Some(root);
1043 #[cfg(unix)]
1044 let home = super::ownership::invoking().map(|user| user.home.as_path());
1045 #[cfg(not(unix))]
1046 let home = None;
1047 prepare_root_with(root, own, home, |path, hand| match hand {
1048 Hand::Give => claim_directory_for_invoking_user(path),
1049 Hand::Reclaim => super::ownership::reclaim(path),
1050 })
1051}
1052
1053#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1055enum Hand {
1056 Give,
1058 Reclaim,
1060}
1061
1062fn prepare_root_with(
1072 root: &Path,
1073 own: bool,
1074 home: Option<&Path>,
1075 mut claim: impl FnMut(&Path, Hand),
1076) -> std::io::Result<()> {
1077 let created = super::ownership::create_missing(root, Some(0o700))?;
1078
1079 if own && let Some(above) = root.parent() {
1080 let mut between: Vec<&Path> = above
1082 .ancestors()
1083 .skip(1)
1084 .take_while(|directory| {
1085 home.is_some_and(|home| *directory != home && directory.starts_with(home))
1086 })
1087 .collect();
1088 between.reverse();
1089 for directory in between {
1090 let hand = if created.iter().any(|made| made == directory) {
1091 Hand::Give
1092 } else {
1093 Hand::Reclaim
1094 };
1095 claim(directory, hand);
1096 }
1097 claim(above, Hand::Give);
1098 }
1099 claim(root, Hand::Give);
1100
1101 Ok(())
1102}
1103
1104#[non_exhaustive]
1111#[derive(Debug, Clone, Default)]
1112pub struct Listing {
1113 pub entries: Vec<Entry>,
1115 pub passed_over: Vec<PassedOver>,
1117}
1118
1119#[non_exhaustive]
1121#[derive(Debug, Clone, PartialEq, Eq)]
1122pub struct PassedOver {
1123 pub name: String,
1125 pub directory: PathBuf,
1127 pub why: Unlisted,
1129}
1130
1131impl PassedOver {
1132 pub fn new(name: impl Into<String>, directory: PathBuf, why: Unlisted) -> Self {
1135 Self {
1136 name: name.into(),
1137 directory,
1138 why,
1139 }
1140 }
1141}
1142
1143#[non_exhaustive]
1149#[derive(Debug, Clone, PartialEq, Eq)]
1150pub enum Unlisted {
1151 Link,
1156 NewerFormat {
1158 found: u32,
1160 },
1161 Unreadable(String),
1164}
1165
1166impl std::fmt::Display for Unlisted {
1167 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1168 match self {
1169 Self::Link => f.write_str("a link, not a journal (not followed)"),
1170 Self::NewerFormat { found } => write!(
1171 f,
1172 "journal format {found} is newer than this build's ({})",
1173 super::JOURNAL_VERSION
1174 ),
1175 Self::Unreadable(reason) => f.write_str(reason),
1176 }
1177 }
1178}
1179
1180pub fn list(root: &Path) -> Result<Listing, JournalError> {
1189 let held = match kinds(root) {
1190 Ok(held) => held,
1191 Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Listing::default()),
1192 Err(e) => return Err(e.into()),
1193 };
1194
1195 let mut listing = Listing::default();
1196 for (name, kind) in held {
1197 let directory = root.join(&name);
1198 let name = name.to_string_lossy().into_owned();
1199 match kind {
1200 Kind::Directory => {}
1201 Kind::Link => {
1202 listing.passed_over.push(PassedOver {
1203 name,
1204 directory,
1205 why: Unlisted::Link,
1206 });
1207 continue;
1208 }
1209 Kind::Other => continue,
1210 }
1211 let manifest = match read_manifest(&directory) {
1212 Ok(manifest) => manifest,
1213 Err(JournalError::Io(e)) if e.kind() == std::io::ErrorKind::NotFound => continue,
1214 Err(e) => {
1215 let why = match e {
1216 JournalError::VersionTooNew { found, .. } => Unlisted::NewerFormat { found },
1217 JournalError::Io(e) => Unlisted::Unreadable(e.to_string()),
1219 other => Unlisted::Unreadable(other.to_string()),
1220 };
1221 listing.passed_over.push(PassedOver {
1222 name,
1223 directory,
1224 why,
1225 });
1226 continue;
1227 }
1228 };
1229 listing.entries.push(Entry {
1230 checkpoint: read_checkpoint(&directory).ok(),
1233 lock: super::lock::inspect(&directory.join(LOCK)),
1234 manifest,
1235 directory,
1236 });
1237 }
1238
1239 listing
1240 .entries
1241 .sort_by_key(|entry| std::cmp::Reverse(entry.manifest.created_at));
1242 listing.passed_over.sort_by(|a, b| a.name.cmp(&b.name));
1243 Ok(listing)
1244}
1245
1246pub fn report(directory: &Path) -> Result<ScanReport, JournalError> {
1258 let manifest = read_manifest(directory)?;
1259
1260 Ok(ScanReport::recorded(
1261 manifest.engine_version,
1262 read_phases(directory)?,
1263 read_findings(directory)?,
1264 ))
1265}
1266
1267fn phase_name(kind: ScanKind) -> &'static str {
1269 match kind {
1270 ScanKind::Discovery => "host-discovery sweep",
1271 ScanKind::PortScan => "port scan",
1272 ScanKind::Listen => "listening phase",
1275 }
1276}
1277
1278pub fn remove(directory: &Path) -> Result<(), OpenError> {
1288 let state = super::lock::inspect(&directory.join(LOCK));
1289 if !state.is_resumable() {
1290 return Err(OpenError::Locked(LockRefused::Held(state)));
1291 }
1292
1293 remove_directory(directory).map_err(JournalError::from)?;
1294 Ok(())
1295}
1296
1297#[derive(Debug, serde::Serialize, serde::Deserialize)]
1300struct FinishedRecord {
1301 hosts: Vec<String>,
1302}
1303
1304fn read_findings(directory: &Path) -> Result<Vec<Host>, JournalError> {
1322 let file = match open_to_read(&directory.join(HOSTS)) {
1323 Ok(file) => file,
1324 Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()),
1325 Err(e) => return Err(e.into()),
1326 };
1327
1328 let mut reader = match crate::journal::format::Reader::open(std::io::BufReader::new(file)) {
1329 Ok(reader) => reader,
1330 Err(JournalError::NotAJournal) => return Ok(Vec::new()),
1333 Err(e) => return Err(e),
1334 };
1335
1336 let mut hosts: Vec<Option<Host>> = Vec::new();
1339 let mut slot_of: std::collections::BTreeMap<ScopedIp, usize> =
1340 std::collections::BTreeMap::new();
1341
1342 while let Some(record) = reader.read::<HostRecord>()? {
1343 let host = Host::from(&record);
1344
1345 let mut matched: Vec<usize> = scoped_ips(&host)
1346 .filter_map(|ip| slot_of.get(&ip).copied())
1347 .collect();
1348 matched.sort_unstable();
1349 matched.dedup();
1350
1351 let slot = match matched.split_first() {
1352 Some((&keep, absorb)) => {
1353 for &other in absorb {
1355 if let Some(other) = hosts[other].take() {
1356 merge_into(&mut hosts, keep, other, Host::merge);
1357 }
1358 }
1359 merge_into(&mut hosts, keep, host, Host::merge_later_account);
1360 keep
1361 }
1362 None => {
1363 hosts.push(Some(host));
1364 hosts.len() - 1
1365 }
1366 };
1367
1368 let Some(settled) = hosts[slot].as_ref() else {
1369 continue;
1370 };
1371 for ip in scoped_ips(settled) {
1372 slot_of.insert(ip, slot);
1373 }
1374 }
1375
1376 Ok(hosts.into_iter().flatten().collect())
1377}
1378
1379fn scoped_ips(host: &Host) -> impl Iterator<Item = ScopedIp> + '_ {
1382 host.ips().iter().map(|ip| match host.zone() {
1383 Some(zone) => ScopedIp::scoped(*ip, zone.clone()),
1384 None => ScopedIp::unscoped(*ip),
1385 })
1386}
1387
1388fn merge_into(hosts: &mut [Option<Host>], slot: usize, host: Host, fold: fn(&mut Host, Host)) {
1395 match hosts[slot].as_mut() {
1396 Some(existing) => fold(existing, host),
1397 None => hosts[slot] = Some(host),
1398 }
1399}
1400
1401fn findings_length(directory: &Path) -> Result<u64, JournalError> {
1404 match open_to_read(&directory.join(HOSTS)) {
1405 Ok(file) => Ok(file.metadata()?.len()),
1406 Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(0),
1407 Err(e) => Err(e.into()),
1408 }
1409}
1410
1411fn read_phases(directory: &Path) -> Result<Vec<ScanPhase>, JournalError> {
1413 let file = match open_to_read(&directory.join(PHASES)) {
1414 Ok(file) => file,
1415 Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()),
1416 Err(e) => return Err(e.into()),
1417 };
1418
1419 let mut reader = match crate::journal::format::Reader::open(std::io::BufReader::new(file)) {
1420 Ok(reader) => reader,
1421 Err(JournalError::NotAJournal) => return Ok(Vec::new()),
1422 Err(e) => return Err(e),
1423 };
1424
1425 let mut phases = Vec::new();
1426 while let Some(record) = reader.read::<PhaseRecord>()? {
1427 phases.push(ScanPhase::from(&record));
1428 }
1429
1430 for standing in standing_phases(directory) {
1433 let recorded = phases.iter().any(|phase| {
1434 phase.kind() == standing.kind() && phase.started_at() == standing.started_at()
1435 });
1436 if !recorded {
1437 phases.push(standing);
1438 }
1439 }
1440 phases.sort_by_key(ScanPhase::started_at);
1443 Ok(phases)
1444}
1445
1446fn sitting_file(directory: &Path, lock: &Lock) -> PathBuf {
1449 let started = lock
1450 .record()
1451 .started_at
1452 .duration_since(SystemTime::UNIX_EPOCH)
1453 .unwrap_or_default()
1454 .as_nanos();
1455 directory.join(format!("{SITTING}{started:020}.jsonl"))
1456}
1457
1458fn standing_phases(directory: &Path) -> Vec<ScanPhase> {
1463 let Ok(held) = names(directory) else {
1464 return Vec::new();
1465 };
1466 let mut files: Vec<PathBuf> = held
1467 .into_iter()
1468 .filter(|name| {
1469 name.to_str()
1470 .is_some_and(|name| name.starts_with(SITTING) && name.ends_with(".jsonl"))
1471 })
1472 .map(|name| directory.join(name))
1473 .collect();
1474 files.sort();
1475
1476 let mut phases = Vec::new();
1477 for path in files {
1478 let Ok(file) = open_to_read(&path) else {
1479 continue;
1480 };
1481 let Ok(mut reader) = crate::journal::format::Reader::open(std::io::BufReader::new(file))
1482 else {
1483 continue;
1484 };
1485 while let Ok(Some(record)) = reader.read::<PhaseRecord>() {
1486 phases.push(ScanPhase::from(&record));
1487 }
1488 }
1489 phases
1490}
1491
1492pub fn read_detections(directory: &Path) -> Result<Vec<DetectionRunRecord>, JournalError> {
1496 let file = match open_to_read(&directory.join(DETECTIONS)) {
1497 Ok(file) => file,
1498 Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()),
1499 Err(e) => return Err(e.into()),
1500 };
1501
1502 let mut reader = match crate::journal::format::Reader::open(std::io::BufReader::new(file)) {
1503 Ok(reader) => reader,
1504 Err(JournalError::NotAJournal) => return Ok(Vec::new()),
1505 Err(e) => return Err(e),
1506 };
1507
1508 let mut runs = Vec::new();
1509 while let Some(line) = reader.read::<DetectionLine>()? {
1510 runs.extend(line.into_runs());
1511 }
1512 Ok(runs)
1513}
1514
1515fn open_for_append(path: &Path) -> Result<fs::File, JournalError> {
1536 mend(path)?;
1537 Ok(append_existing(path)?)
1538}
1539
1540fn mend(path: &Path) -> Result<(), JournalError> {
1548 use std::io::{Read, Seek, SeekFrom};
1549
1550 let mut file = open_existing(path)?;
1551 let length = file.metadata()?.len();
1552
1553 if length == 0 {
1556 super::format::Writer::create(&mut file)?.flush()?;
1557 return Ok(());
1558 }
1559
1560 super::format::Reader::open(std::io::BufReader::new(&file))?;
1561
1562 file.seek(SeekFrom::End(-1))?;
1563 let mut last = [0u8; 1];
1564 file.read_exact(&mut last)?;
1565 if last[0] == b'\n' {
1566 return Ok(());
1567 }
1568
1569 let keep = last_whole_line(&mut file, length)?;
1570 file.set_len(keep)?;
1571
1572 if keep == 0 {
1576 file.seek(SeekFrom::Start(0))?;
1577 super::format::Writer::create(&mut file)?.flush()?;
1578 }
1579
1580 Ok(())
1581}
1582
1583fn last_whole_line(file: &mut fs::File, length: u64) -> Result<u64, JournalError> {
1589 use std::io::{Read, Seek, SeekFrom};
1590
1591 const WINDOW: u64 = 8 * 1024;
1597
1598 let mut end = length;
1599 while end > 0 {
1600 let size = WINDOW.min(end);
1601 let start = end - size;
1602
1603 file.seek(SeekFrom::Start(start))?;
1604 let mut window = vec![0u8; size as usize];
1606 file.read_exact(&mut window)?;
1607
1608 if let Some(at) = window.iter().rposition(|byte| *byte == b'\n') {
1609 return Ok(start + at as u64 + 1);
1610 }
1611
1612 end = start;
1613 }
1614
1615 Ok(0)
1616}
1617
1618fn read_options(directory: &Path) -> Result<Option<JobOptions>, JournalError> {
1620 let text = match read_bounded(&directory.join(OPTIONS), "a journal's options") {
1621 Ok(text) => text,
1622 Err(JournalError::Io(e)) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None),
1623 Err(e) => return Err(e),
1624 };
1625 Ok(Some(
1626 serde_json::from_str(&text).map_err(JournalError::json)?,
1627 ))
1628}
1629
1630fn read_manifest(directory: &Path) -> Result<JournalManifest, JournalError> {
1633 let text = read_bounded(&directory.join(MANIFEST), "a journal manifest")?;
1634 let manifest: JournalManifest = serde_json::from_str(&text).map_err(JournalError::json)?;
1635
1636 if manifest.journal_version > super::JOURNAL_VERSION {
1637 return Err(JournalError::VersionTooNew {
1638 found: manifest.journal_version,
1639 understood: super::JOURNAL_VERSION,
1640 });
1641 }
1642
1643 Ok(manifest)
1644}
1645
1646fn read_checkpoint(directory: &Path) -> Result<Checkpoint, JournalError> {
1648 match Checkpoint::read(&directory.join(CURSOR)) {
1649 Ok(checkpoint) => Ok(checkpoint),
1650 Err(JournalError::Io(e)) if e.kind() == std::io::ErrorKind::NotFound => {
1653 Ok(Checkpoint::default())
1654 }
1655 Err(e) => Err(e),
1656 }
1657}
1658
1659const ID_TIME_BITS: u32 = 48;
1661
1662fn mint_id() -> String {
1668 let millis = SystemTime::now()
1669 .duration_since(SystemTime::UNIX_EPOCH)
1670 .map_or(0, |since| since.as_millis())
1671 & ((1u128 << ID_TIME_BITS) - 1);
1672
1673 let random_bits = ID_CHARS as u32 * 5 - ID_TIME_BITS;
1674 let entropy = u128::from(rand::random::<u64>()) & ((1u128 << random_bits) - 1);
1675 let mut value = (millis << random_bits) | entropy;
1676
1677 let mut out = [b'0'; ID_CHARS];
1678 for slot in out.iter_mut().rev() {
1679 *slot = ID_ALPHABET[(value & 0x1F) as usize];
1680 value >>= 5;
1681 }
1682
1683 String::from_utf8(out.to_vec()).expect("base32 alphabet is ASCII")
1685}
1686
1687fn claim_directory(root: &Path) -> Result<(String, PathBuf), JournalError> {
1693 const ATTEMPTS: usize = 8;
1696
1697 for _ in 0..ATTEMPTS {
1698 let id = mint_id();
1699 let directory = root.join(&id);
1700
1701 match create_private_directory(&directory) {
1702 Ok(()) => {
1703 claim_directory_for_invoking_user(&directory);
1704 return Ok((id, directory));
1705 }
1706 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => continue,
1707 Err(e) => return Err(e.into()),
1708 }
1709 }
1710
1711 Err(std::io::Error::new(
1712 std::io::ErrorKind::AlreadyExists,
1713 "could not find an unused id for this scan",
1714 )
1715 .into())
1716}
1717
1718pub(super) fn read_bounded(path: &Path, what: &str) -> Result<String, JournalError> {
1726 use std::io::Read;
1727
1728 let mut text = String::new();
1729 let read = open_to_read(path)?
1730 .take(super::format::MAX_READ_BYTES + 1)
1731 .read_to_string(&mut text)?;
1732
1733 if read as u64 > super::format::MAX_READ_BYTES {
1734 return Err(super::format::too_large(what));
1735 }
1736
1737 Ok(text)
1738}
1739
1740fn write_private(path: &Path, bytes: &[u8]) -> Result<(), JournalError> {
1746 use std::io::Write;
1747
1748 replace(path, &path.with_extension("tmp"), |mut file| {
1749 file.write_all(bytes)
1750 })?;
1751 Ok(())
1752}
1753
1754#[non_exhaustive]
1760#[derive(Debug, Clone, PartialEq, Eq)]
1761pub struct Retention {
1762 pub completed_for: Option<Duration>,
1764 pub incomplete_for: Option<Duration>,
1766 pub keep_at_most: Option<usize>,
1771 pub unreadable: bool,
1776}
1777
1778impl Default for Retention {
1779 fn default() -> Self {
1783 Self {
1784 completed_for: Some(Duration::from_secs(30 * 24 * 60 * 60)),
1785 incomplete_for: None,
1786 keep_at_most: Some(200),
1787 unreadable: false,
1788 }
1789 }
1790}
1791
1792impl Retention {
1793 pub fn keep_everything() -> Self {
1795 Self {
1796 completed_for: None,
1797 incomplete_for: None,
1798 keep_at_most: None,
1799 unreadable: false,
1800 }
1801 }
1802
1803 pub fn expired(&self, entries: &[Entry], now: SystemTime) -> Vec<usize> {
1812 let age_of = |entry: &Entry| {
1813 now.duration_since(entry.manifest.created_at)
1814 .unwrap_or_default()
1815 };
1816
1817 let mut removing: Vec<usize> = entries
1818 .iter()
1819 .enumerate()
1820 .filter(|(_, entry)| entry.lock.is_resumable())
1821 .filter(|(_, entry)| {
1822 let limit = if entry.is_complete() {
1823 self.completed_for
1824 } else {
1825 self.incomplete_for
1826 };
1827 limit.is_some_and(|limit| age_of(entry) > limit)
1828 })
1829 .map(|(index, _)| index)
1830 .collect();
1831
1832 let Some(cap) = self.keep_at_most else {
1833 return removing;
1834 };
1835
1836 let mut surviving: Vec<usize> = (0..entries.len())
1838 .filter(|index| !removing.contains(index))
1839 .collect();
1840
1841 surviving.sort_by_key(|&index| {
1844 let entry = &entries[index];
1845 (
1846 entry.is_complete(),
1847 std::cmp::Reverse(entry.manifest.created_at),
1848 )
1849 });
1850
1851 while surviving.len() > cap {
1852 let Some(index) = surviving.pop() else { break };
1853 if entries[index].lock.is_resumable() {
1854 removing.push(index);
1855 }
1856 }
1857
1858 removing.sort_unstable();
1859 removing
1860 }
1861}
1862
1863#[non_exhaustive]
1865#[derive(Debug, Clone, Default, PartialEq, Eq)]
1866pub struct Pruned {
1867 pub removed: Vec<String>,
1869 pub held: Vec<Held>,
1872}
1873
1874#[non_exhaustive]
1876#[derive(Debug, Clone, PartialEq, Eq)]
1877pub struct Held {
1878 pub id: String,
1880 pub reason: String,
1882}
1883
1884impl Held {
1885 pub fn new(id: impl Into<String>, reason: impl Into<String>) -> Self {
1887 Self {
1888 id: id.into(),
1889 reason: reason.into(),
1890 }
1891 }
1892}
1893
1894pub fn prune(root: &Path, retention: &Retention) -> Result<Pruned, JournalError> {
1904 let listing = list(root)?;
1905 let mut pruned = Pruned::default();
1906
1907 for index in retention.expired(&listing.entries, SystemTime::now()) {
1908 let entry = &listing.entries[index];
1909 match remove(&entry.directory) {
1910 Ok(()) => pruned.removed.push(entry.manifest.id.clone()),
1911 Err(error) => pruned.held.push(Held {
1912 id: entry.manifest.id.clone(),
1913 reason: error.to_string(),
1914 }),
1915 }
1916 }
1917
1918 if retention.unreadable {
1919 let journals = listing
1920 .passed_over
1921 .iter()
1922 .filter(|passed| passed.why != Unlisted::Link);
1923 for passed in journals {
1924 match remove(&passed.directory) {
1925 Ok(()) => pruned.removed.push(passed.name.clone()),
1926 Err(error) => pruned
1927 .held
1928 .push(Held::new(passed.name.clone(), error.to_string())),
1929 }
1930 }
1931 }
1932
1933 Ok(pruned)
1934}
1935
1936#[cfg(test)]
1946mod tests {
1947 use super::*;
1948 use crate::journal::settle::Outcome;
1949 use crate::model::exclusion::Exclusions;
1952 use crate::model::ip::set::IpSet;
1953 use crate::model::port::PortSet;
1954 use crate::model::target::{TargetMap, TargetSet};
1955 use crate::model::technique::TcpScanTechnique;
1956 use crate::scanner::checkpoint::spawn_checkpoints;
1957
1958 fn scratch(name: &str) -> PathBuf {
1959 let dir = std::env::temp_dir().join(format!("zond-store-{}-{name}", std::process::id()));
1960 let _ = fs::remove_dir_all(&dir);
1961 fs::create_dir_all(&dir).expect("scratch root");
1962 dir
1963 }
1964
1965 fn plan(range: &str, ports: &str) -> TargetMap {
1966 let mut map = TargetMap::new();
1967 map.add_unit(TargetSet::new(
1968 range.parse::<IpSet>().expect("a range"),
1969 ports.parse::<PortSet>().expect("ports"),
1970 ));
1971 map
1972 }
1973
1974 fn ports(map: &TargetMap) -> Plan {
1975 Plan::port_scan(map, &Exclusions::none(), TcpScanTechnique::Syn)
1976 }
1977
1978 fn begin(root: &Path, map: &TargetMap) -> Journal {
1979 Journal::create(root, &ports(map), Privilege::Raw, "test").expect("creates")
1980 }
1981
1982 fn standing_in(silent: &str) -> ScanPhase {
1985 use crate::config::ZondConfig;
1986 use crate::model::ip::range::IpRange;
1987 use crate::report::{LivenessSkip, PhaseParts, ScanSettings, TargetScope};
1988
1989 let silent: IpSet = silent.parse().expect("a range");
1990 ScanPhase::from_parts(PhaseParts {
1991 open: false,
1992 attachments: Vec::new(),
1993 kind: ScanKind::PortScan,
1994 started_at: SystemTime::UNIX_EPOCH,
1995 elapsed: Duration::from_secs(1),
1996 privilege: Some(Privilege::Raw),
1997 targets: TargetScope::from_ip_set(&mut IpSet::new(), &Exclusions::none()),
1998 settings: ScanSettings::from(&ZondConfig::default()),
1999 failures: Vec::new(),
2000 refusals: Vec::new(),
2001 unroutable: Vec::new(),
2002 refused_by_route: Vec::new(),
2003 timed_out: Vec::new(),
2004 icmp_rate_limited: Vec::new(),
2005 reached_by_connect: Vec::new(),
2006 undecided: Vec::new(),
2007 liveness_skipped: Some(LivenessSkip::PortsNoDearer),
2008 silent: silent.v4().iter().copied().map(IpRange::V4).collect(),
2009 stopped: None,
2010 passes_cut: Vec::new(),
2011 unreached: 0,
2012 unheard_probes: 0,
2013 probes: Vec::new(),
2014 origin: None,
2015 })
2016 }
2017
2018 fn unheard(address: &str) -> Host {
2021 use crate::model::port::{Port, PortState, Protocol};
2022
2023 let mut host = Host::new(address.parse().expect("an address"));
2024 host.add_port(Port::new(80, Protocol::Tcp, PortState::NoReply));
2025 host
2026 }
2027
2028 fn heard(address: &str) -> Host {
2030 let mut host = Host::new(address.parse().expect("an address"));
2031 host.set_status(crate::model::host::HostStatus::Up);
2032 host
2033 }
2034
2035 #[test]
2039 fn a_record_an_earlier_sitting_heard_nothing_from_is_not_restored() {
2040 let root = scratch("unheard-restore");
2041 let map = plan("192.0.2.1-192.0.2.8", "80");
2042
2043 let directory = {
2044 let mut journal = begin(&root, &map);
2045 journal
2046 .record_hosts(&[heard("192.0.2.1"), unheard("192.0.2.5")])
2047 .expect("records");
2048 journal
2049 .record_phases(&[standing_in("192.0.2.5")])
2050 .expect("records");
2051 let directory = journal.directory().to_path_buf();
2052 journal.close().expect("closes");
2053 directory
2054 };
2055
2056 let (journal, _) =
2057 Journal::resume(&directory, &ports(&map), Privilege::Raw).expect("resumes");
2058 let restored: Vec<_> = journal.restored().iter().map(Host::primary_ip).collect();
2059
2060 assert_eq!(
2061 restored,
2062 ["192.0.2.1".parse::<std::net::IpAddr>().expect("an address")]
2063 );
2064 journal.close().expect("closes");
2065 std::fs::remove_dir_all(&root).ok();
2066 }
2067
2068 #[tokio::test]
2072 async fn a_sitting_that_heard_nothing_from_an_address_leaves_no_record_of_it() {
2073 let root = scratch("unheard-close");
2074 let map = plan("192.0.2.1-192.0.2.8", "80");
2075 let mut journal = begin(&root, &map);
2076 let directory = journal.directory().to_path_buf();
2077
2078 journal
2081 .record_hosts(&[heard("192.0.2.1"), unheard("192.0.2.5")])
2082 .expect("records");
2083 let (_session, ctx) = crate::scanner::session::ScanSession::new();
2084 ctx.restore_hosts(&[heard("192.0.2.1")]);
2085
2086 spawn_checkpoints(journal, ctx.progress())
2087 .finish(&[standing_in("192.0.2.5")])
2088 .await;
2089
2090 let file = fs::read_to_string(directory.join(HOSTS)).expect("reads");
2091 assert!(
2092 !file.contains("192.0.2.5"),
2093 "a record of the silent address is still on disk:\n{file}"
2094 );
2095 let kept: Vec<_> = read_findings(&directory)
2096 .expect("reads")
2097 .iter()
2098 .map(Host::primary_ip)
2099 .collect();
2100 assert_eq!(
2101 kept,
2102 ["192.0.2.1".parse::<std::net::IpAddr>().expect("an address")]
2103 );
2104
2105 std::fs::remove_dir_all(&root).ok();
2106 }
2107
2108 #[tokio::test]
2117 async fn a_sitting_killed_before_its_verdicts_leaves_its_undecided_records_to_the_next() {
2118 use crate::config::ZondConfig;
2119 use crate::journal::settle::Outcome;
2120 use crate::model::port::{Port, PortState, Protocol};
2121 use crate::model::target::TargetIndex;
2122 use crate::report::{LivenessSkip, TargetScope};
2123 use crate::scanner::recorder::PhaseRecorder;
2124
2125 let root = scratch("unheard-killed");
2126 let map = plan("192.0.2.1-192.0.2.8", "80,443");
2127 let journal = begin(&root, &map);
2128 let directory = journal.directory().to_path_buf();
2129 let address = |address: &str| address.parse::<std::net::IpAddr>().expect("an address");
2130 let position = |at: &str, port: u16| {
2131 map.iter()
2132 .position(|target| target.ip == address(at) && target.port == port)
2133 .expect("a planned target") as u64
2134 };
2135
2136 let (_session, ctx) = crate::scanner::session::ScanSession::new();
2137 ctx.number_targets(TargetIndex::of(&map));
2138 let _phase = PhaseRecorder::start(
2139 ScanKind::PortScan,
2140 Privilege::Raw,
2141 TargetScope::from_ip_set(
2142 &mut "192.0.2.1-192.0.2.8".parse().expect("a range"),
2143 &Exclusions::none(),
2144 ),
2145 &ZondConfig::default(),
2146 )
2147 .skipping_liveness(LivenessSkip::PortsNoDearer)
2148 .opening_in(&ctx);
2149 ctx.await_verdicts();
2150 ctx.update_host(address("192.0.2.1"), |host| {
2153 host.set_status(crate::model::host::HostStatus::Up);
2154 });
2155 ctx.write_host(address("192.0.2.5"), |host| {
2156 *host = unheard("192.0.2.5");
2157 true
2158 });
2159 ctx.record_outcome(Outcome::Exhausted {
2160 position: position("192.0.2.5", 80),
2161 });
2162 ctx.write_host(address("192.0.2.6"), |host| {
2163 *host = unheard("192.0.2.6");
2164 host.add_port(Port::new(443, Protocol::Tcp, PortState::NoReply));
2165 true
2166 });
2167 for port in [80, 443] {
2168 ctx.record_outcome(Outcome::Exhausted {
2169 position: position("192.0.2.6", port),
2170 });
2171 }
2172
2173 let mut ticker = spawn_checkpoints(journal, ctx.progress());
2175 ticker.checkpointed().await;
2176 ticker.kill().await;
2177
2178 let hosts = report(&directory)
2179 .expect("reads")
2180 .hosts()
2181 .map(Host::primary_ip)
2182 .collect::<Vec<_>>();
2183 assert_eq!(
2184 hosts,
2185 [address("192.0.2.1")],
2186 "the report lists an undecided address"
2187 );
2188
2189 let (journal, _) =
2190 Journal::resume(&directory, &ports(&map), Privilege::Raw).expect("resumes");
2191 let restored: Vec<_> = journal
2192 .restored()
2193 .iter()
2194 .map(|host| (host.primary_ip(), host.ports().count()))
2195 .collect();
2196 assert_eq!(
2197 restored,
2198 [(address("192.0.2.1"), 0), (address("192.0.2.5"), 1)],
2199 "the undecided record, and only it beside the host, is the next sitting's"
2200 );
2201 let unheard: u128 = journal
2202 .earlier_phases()
2203 .iter()
2204 .map(ScanPhase::unheard_probes)
2205 .sum();
2206 assert_eq!(unheard, 2, "the silent address's probes went uncounted");
2207 journal.close().expect("closes");
2208 std::fs::remove_dir_all(&root).ok();
2209 }
2210
2211 #[tokio::test]
2215 async fn a_sitting_killed_before_its_verdicts_still_names_what_it_heard_nothing_from() {
2216 use crate::config::ZondConfig;
2217 use crate::journal::settle::Outcome;
2218 use crate::model::target::TargetIndex;
2219 use crate::report::TargetScope;
2220 use crate::scanner::recorder::PhaseRecorder;
2221
2222 let root = scratch("unheard-named");
2223 let map = plan("192.0.2.1-192.0.2.8", "80");
2224 let journal = begin(&root, &map);
2225 let directory = journal.directory().to_path_buf();
2226
2227 let (_session, ctx) = crate::scanner::session::ScanSession::new();
2228 ctx.number_targets(TargetIndex::of(&map));
2229 let _phase = PhaseRecorder::start(
2230 ScanKind::PortScan,
2231 Privilege::Raw,
2232 TargetScope::from_ip_set(&mut IpSet::new(), &Exclusions::none()),
2233 &ZondConfig::default(),
2234 )
2235 .opening_in(&ctx);
2236 ctx.await_verdicts();
2237 for address in ["192.0.2.5", "192.0.2.6"] {
2239 ctx.write_host(
2240 address.parse::<std::net::IpAddr>().expect("an address"),
2241 |host| {
2242 *host = unheard(address);
2243 true
2244 },
2245 );
2246 }
2247 ctx.record_outcome(Outcome::Exhausted { position: 4 });
2248
2249 let mut ticker = spawn_checkpoints(journal, ctx.progress());
2250 ticker.checkpointed().await;
2251 ticker.kill().await;
2252
2253 let (journal, _) =
2254 Journal::resume(&directory, &ports(&map), Privilege::Raw).expect("resumes");
2255 let silent: Vec<_> = journal
2256 .earlier_phases()
2257 .iter()
2258 .flat_map(|phase| phase.silent().iter().copied())
2259 .collect();
2260 let named: IpSet = "192.0.2.5".parse().expect("an address");
2261 assert_eq!(
2262 silent,
2263 named
2264 .v4()
2265 .iter()
2266 .copied()
2267 .map(crate::model::ip::range::IpRange::V4)
2268 .collect::<Vec<_>>(),
2269 "the settled silent address is named, and only it"
2270 );
2271 journal.close().expect("closes");
2272 std::fs::remove_dir_all(&root).ok();
2273 }
2274
2275 #[tokio::test]
2278 async fn a_record_held_for_its_verdict_is_written_once_the_phase_keeps_it() {
2279 let root = scratch("unheard-kept");
2280 let map = plan("192.0.2.1-192.0.2.8", "80");
2281 let journal = begin(&root, &map);
2282 let directory = journal.directory().to_path_buf();
2283
2284 let (_session, ctx) = crate::scanner::session::ScanSession::new();
2285 ctx.await_verdicts();
2286 ctx.write_host(
2287 "192.0.2.5".parse::<std::net::IpAddr>().expect("an address"),
2288 |host| {
2289 *host = unheard("192.0.2.5");
2290 true
2291 },
2292 );
2293 let mut ticker = spawn_checkpoints(journal, ctx.progress());
2294 ticker.checkpointed().await;
2295 ctx.verdicts_reached();
2296 ticker.finish(&[]).await;
2297
2298 let kept: Vec<_> = read_findings(&directory)
2299 .expect("reads")
2300 .iter()
2301 .map(Host::primary_ip)
2302 .collect();
2303 assert_eq!(
2304 kept,
2305 ["192.0.2.5".parse::<std::net::IpAddr>().expect("an address")]
2306 );
2307 std::fs::remove_dir_all(&root).ok();
2308 }
2309
2310 #[test]
2312 fn a_scan_resumes_from_where_it_stopped() {
2313 let root = scratch("resume");
2314 let map = plan("192.0.2.1-192.0.2.4", "80,443");
2315
2316 let directory = {
2317 let mut journal = begin(&root, &map);
2318 let settlements = Settlements::default();
2319 for position in [0, 1, 2, 5] {
2320 settlements.record(Outcome::Answered { position });
2321 }
2322 journal.checkpoint(&settlements).expect("checkpoints");
2323
2324 let directory = journal.directory().to_path_buf();
2325 journal.close().expect("closes");
2326 directory
2327 };
2328
2329 let (_journal, checkpoint) =
2330 Journal::resume(&directory, &ports(&map), Privilege::Raw).expect("resumes");
2331
2332 assert_eq!(checkpoint.watermark, 3);
2333 let remaining: Vec<_> = checkpoint.remaining(map.iter()).collect();
2334 assert_eq!(remaining.len(), 4, "eight targets, four settled");
2335 }
2336
2337 #[test]
2341 fn a_jobs_options_are_written_by_its_first_sitting_and_kept() {
2342 use crate::config::ZondConfig;
2343
2344 let root = scratch("options");
2345 let map = plan("192.0.2.1-192.0.2.4", "80,443");
2346 let first = ZondConfig {
2347 assume_up: true,
2348 traceroute: true,
2349 ..ZondConfig::default()
2350 };
2351
2352 let directory = {
2353 let mut journal = begin(&root, &map);
2354 assert!(journal.options().is_none(), "no sitting has started");
2355 journal
2356 .record_options(JobOptions::of(&first))
2357 .expect("records");
2358 let directory = journal.directory().to_path_buf();
2359 journal.close().expect("closes");
2360 directory
2361 };
2362
2363 let (mut journal, _) =
2364 Journal::resume(&directory, &ports(&map), Privilege::Raw).expect("resumes");
2365 assert_eq!(journal.options(), Some(&JobOptions::of(&first)));
2366
2367 journal
2368 .record_options(JobOptions::of(&ZondConfig::default()))
2369 .expect("a second record is a no-op");
2370 journal.close().expect("closes");
2371 let (journal, _) =
2372 Journal::resume(&directory, &ports(&map), Privilege::Raw).expect("resumes");
2373 assert_eq!(journal.options(), Some(&JobOptions::of(&first)));
2374 }
2375
2376 #[test]
2379 fn a_journal_older_than_its_options_is_left_without_them() {
2380 use crate::config::ZondConfig;
2381
2382 let root = scratch("options-older");
2383 let map = plan("192.0.2.1-192.0.2.4", "80,443");
2384
2385 let directory = {
2386 let mut journal = begin(&root, &map);
2387 let settlements = Settlements::default();
2388 settlements.record(Outcome::Answered { position: 0 });
2389 journal.checkpoint(&settlements).expect("checkpoints");
2390 let directory = journal.directory().to_path_buf();
2391 journal.close().expect("closes");
2392 directory
2393 };
2394
2395 let (mut journal, checkpoint) = Journal::resume(&directory, &ports(&map), Privilege::Raw)
2396 .expect("an older journal resumes");
2397 assert_eq!(checkpoint.watermark, 1);
2398 journal
2399 .record_options(JobOptions::of(&ZondConfig::default()))
2400 .expect("records nothing");
2401 assert!(journal.options().is_none());
2402 assert!(!directory.join(OPTIONS).exists());
2403 }
2404
2405 #[test]
2408 fn a_changed_plan_is_refused() {
2409 let root = scratch("changed-plan");
2410 let map = plan("192.0.2.1-192.0.2.4", "80,443");
2411
2412 let journal = begin(&root, &map);
2413 let directory = journal.directory().to_path_buf();
2414 journal.close().expect("closes");
2415
2416 let widened = plan("192.0.2.1-192.0.2.9", "80,443");
2417 let refused = Journal::resume(&directory, &ports(&widened), Privilege::Raw)
2418 .expect_err("the plan moved");
2419
2420 assert!(matches!(refused, OpenError::PlanChanged(_)), "{refused:?}");
2421 }
2422
2423 #[test]
2427 fn a_journal_whose_lock_went_stale_can_be_taken_over() {
2428 let root = scratch("stale-lock");
2429 let map = plan("192.0.2.1", "80");
2430 let directory = {
2431 let journal = begin(&root, &map);
2432 journal.directory().to_path_buf()
2433 };
2434 let stale = super::super::lock::LockRecord {
2435 pid: std::process::id(),
2436 boot: super::super::lock::boot_identity(),
2437 started_at: SystemTime::now() - Duration::from_secs(3_600),
2438 heartbeat: SystemTime::now() - Duration::from_secs(600),
2439 };
2440 fs::write(
2441 directory.join(LOCK),
2442 serde_json::to_string(&stale).expect("json"),
2443 )
2444 .expect("writes");
2445
2446 assert!(matches!(
2447 Journal::reopen(&directory, Privilege::Raw),
2448 Err(OpenError::Locked(LockRefused::Held(
2449 LockState::Stale { .. }
2450 )))
2451 ));
2452 let (journal, _, _) =
2453 Journal::take_over(&directory, Privilege::Raw).expect("takes the journal over");
2454 journal.close().expect("closes");
2455
2456 std::fs::remove_dir_all(&root).ok();
2457 }
2458
2459 #[test]
2460 fn a_live_journal_is_not_resumable() {
2461 let root = scratch("live");
2462 let map = plan("192.0.2.1", "80");
2463
2464 let journal = begin(&root, &map);
2465 let directory = journal.directory().to_path_buf();
2466
2467 let refused =
2468 Journal::resume(&directory, &ports(&map), Privilege::Raw).expect_err("it is held");
2469 assert!(matches!(refused, OpenError::Locked(_)), "{refused:?}");
2470
2471 journal.close().expect("closes");
2473 Journal::resume(&directory, &ports(&map), Privilege::Raw).expect("now free");
2474 }
2475
2476 #[test]
2478 fn listing_describes_journals_without_locking_them() {
2479 let root = scratch("list");
2480 let map = plan("192.0.2.1-192.0.2.4", "80,443");
2481
2482 let mut journal = begin(&root, &map);
2483 let settlements = Settlements::default();
2484 for position in 0..4 {
2485 settlements.record(Outcome::Answered { position });
2486 }
2487 journal.checkpoint(&settlements).expect("checkpoints");
2488
2489 let listed = list(&root).expect("lists").entries;
2490 assert_eq!(listed.len(), 1);
2491
2492 let entry = &listed[0];
2493 assert_eq!(entry.manifest.total_targets, 8);
2494 assert_eq!(entry.settled(), Some(4));
2495 assert!(!entry.is_complete());
2496 assert!(
2497 matches!(entry.lock, LockState::Held { .. }),
2498 "the scan is still running: {:?}",
2499 entry.lock
2500 );
2501
2502 journal
2504 .checkpoint(&settlements)
2505 .expect("still holds its lock");
2506 }
2507
2508 #[test]
2510 fn a_finished_journal_reads_as_complete() {
2511 let root = scratch("complete");
2512 let map = plan("192.0.2.1-192.0.2.4", "80,443");
2513
2514 let mut journal = begin(&root, &map);
2515 let settlements = Settlements::default();
2516 for position in 0..8 {
2517 settlements.record(Outcome::Answered { position });
2518 }
2519 journal.checkpoint(&settlements).expect("checkpoints");
2520 journal.close().expect("closes");
2521
2522 let listed = list(&root).expect("lists").entries;
2523 assert!(listed[0].is_complete());
2524 assert_eq!(listed[0].settled(), Some(8));
2525 }
2526
2527 #[test]
2530 fn a_journal_with_no_checkpoint_resumes_from_the_beginning() {
2531 let root = scratch("no-checkpoint");
2532 let map = plan("192.0.2.1-192.0.2.4", "80,443");
2533
2534 let journal = begin(&root, &map);
2535 let directory = journal.directory().to_path_buf();
2536 journal.close().expect("closes");
2537
2538 let (_journal, checkpoint) =
2539 Journal::resume(&directory, &ports(&map), Privilege::Raw).expect("resumes");
2540
2541 assert_eq!(checkpoint, Checkpoint::default());
2542 assert_eq!(checkpoint.remaining(map.iter()).count(), 8);
2543 }
2544
2545 #[test]
2547 fn ids_are_sortable_and_distinct() {
2548 let mut ids: Vec<String> = (0..64).map(|_| mint_id()).collect();
2549
2550 assert!(ids.iter().all(|id| id.len() == ID_CHARS));
2551 let distinct: std::collections::BTreeSet<&String> = ids.iter().collect();
2552 assert_eq!(distinct.len(), ids.len(), "ids collided");
2553
2554 let sorted = {
2555 ids.sort();
2556 ids.clone()
2557 };
2558 assert_eq!(ids, sorted, "ids minted in one burst lost their order");
2559
2560 let first = mint_id();
2562 std::thread::sleep(std::time::Duration::from_millis(3));
2563 let second = mint_id();
2564 assert!(first < second, "{first} should sort before {second}");
2565 }
2566
2567 #[test]
2570 fn an_unreadable_journal_does_not_hide_the_others() {
2571 let root = scratch("damaged");
2572 let map = plan("192.0.2.1", "80");
2573 begin(&root, &map).close().expect("closes");
2574
2575 let damaged = root.join("01JUNREADABLEJUNREADABLE00");
2576 fs::create_dir_all(&damaged).expect("directory");
2577 fs::write(damaged.join(MANIFEST), "{not json").expect("writes");
2578
2579 let listed = list(&root).expect("lists").entries;
2580 assert_eq!(listed.len(), 1, "the readable one is still there");
2581 }
2582
2583 #[test]
2585 fn pruning_refuses_a_journal_that_is_being_written() {
2586 let root = scratch("prune");
2587 let map = plan("192.0.2.1", "80");
2588
2589 let journal = begin(&root, &map);
2590 let directory = journal.directory().to_path_buf();
2591
2592 assert!(matches!(remove(&directory), Err(OpenError::Locked(_))));
2593
2594 journal.close().expect("closes");
2595 remove(&directory).expect("removes a free journal");
2596 assert!(list(&root).expect("lists").entries.is_empty());
2597 }
2598
2599 #[tokio::test]
2602 async fn the_checkpoint_task_writes_a_final_cursor_and_releases() {
2603 let root = scratch("ticker");
2604 let map = plan("192.0.2.1-192.0.2.4", "80,443");
2605 let journal = begin(&root, &map);
2606 let directory = journal.directory().to_path_buf();
2607
2608 let (_session, ctx) = crate::scanner::session::ScanSession::new();
2609 for position in 0..5 {
2610 ctx.record_outcome(Outcome::Answered { position });
2611 }
2612
2613 spawn_checkpoints(journal, ctx.progress()).finish(&[]).await;
2615
2616 let checkpoint = Checkpoint::read(&directory.join(CURSOR)).expect("a cursor was written");
2617 assert_eq!(checkpoint.watermark, 5);
2618 assert_eq!(
2619 super::super::lock::inspect(&directory.join(LOCK)),
2620 LockState::Free,
2621 "the lock outlived the scan and was then released"
2622 );
2623 }
2624
2625 #[test]
2629 fn a_resumed_cursor_carries_the_earlier_sittings_progress() {
2630 let root = scratch("carry-forward");
2631 let map = plan("192.0.2.1-192.0.2.4", "80,443");
2632
2633 let directory = {
2634 let mut journal = begin(&root, &map);
2635 let settlements = Settlements::default();
2636 for position in 0..3 {
2637 settlements.record(Outcome::Answered { position });
2638 }
2639 journal.checkpoint(&settlements).expect("checkpoints");
2640 let directory = journal.directory().to_path_buf();
2641 journal.close().expect("closes");
2642 directory
2643 };
2644
2645 let (mut journal, checkpoint) =
2646 Journal::resume(&directory, &ports(&map), Privilege::Raw).expect("resumes");
2647
2648 let settlements = Settlements::resuming(&checkpoint);
2650 journal
2651 .checkpoint(&settlements)
2652 .expect("checkpoints having settled nothing new");
2653 journal.close().expect("closes");
2654
2655 let carried = Checkpoint::read(&directory.join(CURSOR)).expect("reads");
2656 assert_eq!(
2657 carried.watermark, 3,
2658 "the second sitting must not roll the cursor back"
2659 );
2660 }
2661
2662 #[test]
2664 fn what_a_sitting_found_comes_back() {
2665 use crate::model::host::HostStatus;
2666 use crate::model::port::{Port, PortState, Protocol};
2667
2668 let root = scratch("findings");
2669 let map = plan("192.0.2.1-192.0.2.4", "80,443");
2670
2671 let directory = {
2672 let mut journal = begin(&root, &map);
2673
2674 let mut host = Host::new("192.0.2.1".parse().expect("an address"));
2675 host.set_status(HostStatus::Up);
2676 host.set_hostname(Some("router.example".to_string()));
2677 host.add_port(Port::new(80, Protocol::Tcp, PortState::Open));
2678
2679 journal.record_hosts(&[host]).expect("records");
2680
2681 let directory = journal.directory().to_path_buf();
2682 journal.close().expect("closes");
2683 directory
2684 };
2685
2686 let (journal, _) =
2687 Journal::resume(&directory, &ports(&map), Privilege::Raw).expect("resumes");
2688
2689 let restored = journal.restored();
2690 assert_eq!(restored.len(), 1);
2691 assert_eq!(restored[0].hostname(), Some("router.example"));
2692 assert_eq!(restored[0].port_count(), 1);
2693 assert!(restored[0].is_alive());
2694 }
2695
2696 #[test]
2699 fn detection_run_tapes_survive_a_journal_round_trip() {
2700 use crate::detect::compute::{CapTape, CapTapeRecord, DetectionRunRecord, SpeakExchange};
2701 use crate::model::finding::{DetectionId, Version};
2702 use crate::record::DetectionIdRecord;
2703
2704 let root = scratch("detections");
2705 let map = plan("192.0.2.1", "80");
2706
2707 let tape = CapTape {
2708 speaks: vec![SpeakExchange {
2709 sent: b"PING\r\n".to_vec(),
2710 reply: Ok(b"+PONG\r\n".to_vec()),
2711 }],
2712 nows: vec![7],
2713 ..CapTape::default()
2714 };
2715 let detection = DetectionId::new("redis-unauth", Version::new(1, 0, 0), "abc").unwrap();
2716 let run = DetectionRunRecord {
2717 host: "192.0.2.1".to_string(),
2718 host_name: None,
2719 port: 80,
2720 protocol: "tcp".to_string(),
2721 detection: DetectionIdRecord::from(&detection),
2722 responses: vec!["+PONG\r\n".to_string()],
2723 tape: CapTapeRecord::from(&tape),
2724 };
2725
2726 let directory = {
2727 let mut journal = begin(&root, &map);
2728 journal
2729 .record_detections(std::slice::from_ref(&run))
2730 .expect("records the run");
2731 let directory = journal.directory().to_path_buf();
2732 journal.close().expect("closes");
2733 directory
2734 };
2735
2736 let read = read_detections(&directory).expect("reads the runs back");
2737 assert_eq!(
2738 read,
2739 vec![run],
2740 "a detection run did not survive the journal"
2741 );
2742 assert_eq!(
2743 read[0].tape.rebuild(),
2744 tape,
2745 "the tape did not rebuild to what was recorded"
2746 );
2747 }
2748
2749 fn passive_run(
2752 id: &str,
2753 port: u16,
2754 responses: &[&str],
2755 ) -> crate::detect::compute::DetectionRunRecord {
2756 use crate::detect::compute::{CapTape, CapTapeRecord, DetectionRunRecord};
2757 use crate::model::finding::{DetectionId, Version};
2758 use crate::record::DetectionIdRecord;
2759
2760 let detection = DetectionId::new(id, Version::new(1, 0, 0), "abc").expect("an id");
2761 DetectionRunRecord {
2762 host: "192.0.2.1".to_string(),
2763 host_name: None,
2764 port,
2765 protocol: "tcp".to_string(),
2766 detection: DetectionIdRecord::from(&detection),
2767 responses: responses
2768 .iter()
2769 .map(|response| response.to_string())
2770 .collect(),
2771 tape: CapTapeRecord::from(&CapTape::default()),
2772 }
2773 }
2774
2775 #[test]
2779 fn a_ports_responses_are_written_once_however_many_runs_read_them() {
2780 let root = scratch("tapes-once");
2781 let map = plan("192.0.2.1", "80");
2782 let page = "HTTP/1.1 200 OK\r\nServer: a-distinctive-server\r\n\r\n";
2783
2784 let runs: Vec<_> = (0..13)
2785 .map(|n| passive_run(&format!("http-{n}"), 80, &[page]))
2786 .chain([passive_run("ssh-banner", 22, &["SSH-2.0-x\r\n"])])
2787 .collect();
2788
2789 let directory = {
2790 let mut journal = begin(&root, &map);
2791 journal.record_detections(&runs).expect("records the runs");
2792 let directory = journal.directory().to_path_buf();
2793 journal.close().expect("closes");
2794 directory
2795 };
2796
2797 let written = fs::read_to_string(directory.join(DETECTIONS)).expect("reads the file");
2798 assert_eq!(written.matches("a-distinctive-server").count(), 1);
2799 assert_eq!(read_detections(&directory).expect("reads back"), runs);
2800 }
2801
2802 #[test]
2805 fn tapes_written_one_run_to_a_line_still_read_back() {
2806 use std::io::Write;
2807
2808 let root = scratch("tapes-legacy");
2809 let map = plan("192.0.2.1", "80");
2810 let grouped = passive_run("http-title", 80, &["HTTP/1.1 200 OK\r\n\r\n"]);
2811 let alone = passive_run("ssh-banner", 22, &["SSH-2.0-x\r\n"]);
2812
2813 let directory = {
2814 let mut journal = begin(&root, &map);
2815 journal
2816 .record_detections(std::slice::from_ref(&grouped))
2817 .expect("records the run");
2818 let directory = journal.directory().to_path_buf();
2819 journal.close().expect("closes");
2820 directory
2821 };
2822 let mut file = fs::OpenOptions::new()
2823 .append(true)
2824 .open(directory.join(DETECTIONS))
2825 .expect("opens the file");
2826 let line = serde_json::to_string(&alone).expect("serialises");
2827 writeln!(file, "{line}").expect("appends a run on its own");
2828
2829 assert_eq!(
2830 read_detections(&directory).expect("reads back"),
2831 [grouped, alone]
2832 );
2833 }
2834
2835 #[test]
2838 fn repeated_records_for_one_host_are_folded_together() {
2839 use crate::model::host::HostStatus;
2840 use crate::model::port::{Port, PortState, Protocol};
2841
2842 let root = scratch("folded");
2843 let map = plan("192.0.2.1", "80");
2844 let ip: std::net::IpAddr = "192.0.2.1".parse().expect("an address");
2845
2846 let directory = {
2847 let mut journal = begin(&root, &map);
2848
2849 let mut found = Host::new(ip);
2850 found.set_status(HostStatus::Up);
2851 journal.record_hosts(&[found]).expect("the host answered");
2852
2853 let mut classified = Host::new(ip);
2854 classified.add_port(Port::new(80, Protocol::Tcp, PortState::Open));
2855 journal
2856 .record_hosts(&[classified])
2857 .expect("and then its ports");
2858
2859 let directory = journal.directory().to_path_buf();
2860 journal.close().expect("closes");
2861 directory
2862 };
2863
2864 let (journal, _) =
2865 Journal::resume(&directory, &ports(&map), Privilege::Raw).expect("resumes");
2866
2867 let restored = journal.restored();
2868 assert_eq!(restored.len(), 1, "one address is one host");
2869 assert!(restored[0].is_alive(), "the first record's status survived");
2870 assert_eq!(restored[0].port_count(), 1, "the second record's port too");
2871 }
2872
2873 #[test]
2876 fn link_locals_on_two_interfaces_come_back_as_two_hosts() {
2877 use crate::model::host::HostStatus;
2878 use crate::model::ip::scoped::Zone;
2879
2880 let root = scratch("zones");
2881 let map = plan("192.0.2.1", "80");
2882 let ip: std::net::IpAddr = "fe80::1".parse().expect("an address");
2883
2884 let directory = {
2885 let mut journal = begin(&root, &map);
2886 let hosts: Vec<Host> = [Zone::new(4, "en0"), Zone::new(9, "en7")]
2887 .into_iter()
2888 .map(|zone| {
2889 let mut host = Host::new(ip);
2890 host.set_status(HostStatus::Up);
2891 host.set_zone(zone);
2892 host
2893 })
2894 .collect();
2895 journal.record_hosts(&hosts).expect("records both");
2896
2897 let directory = journal.directory().to_path_buf();
2898 journal.close().expect("closes");
2899 directory
2900 };
2901
2902 assert_eq!(report(&directory).expect("reads").host_count(), 2);
2903
2904 let (journal, _) =
2905 Journal::resume(&directory, &ports(&map), Privilege::Raw).expect("resumes");
2906 let mut zones: Vec<&str> = journal
2907 .restored()
2908 .iter()
2909 .filter_map(|host| host.zone().map(Zone::name))
2910 .collect();
2911 zones.sort_unstable();
2912 assert_eq!(zones, ["en0", "en7"], "each restored on its own link");
2913 }
2914
2915 #[test]
2917 fn a_journal_with_no_findings_restores_nothing() {
2918 let root = scratch("no-findings");
2919 let map = plan("192.0.2.1", "80");
2920
2921 let journal = begin(&root, &map);
2922 let directory = journal.directory().to_path_buf();
2923 journal.close().expect("closes");
2924
2925 let (journal, _) =
2926 Journal::resume(&directory, &ports(&map), Privilege::Raw).expect("resumes");
2927 assert!(journal.restored().is_empty());
2928 }
2929
2930 #[test]
2933 fn a_journal_reads_back_as_the_report_its_scan_produced() {
2934 let root = scratch("replay");
2935 let map = plan("192.0.2.1-192.0.2.4", "80,443");
2936 let original = crate::export::fixture::report();
2937
2938 let directory = {
2939 let mut journal = begin(&root, &map);
2940 let hosts: Vec<Host> = original.hosts().cloned().collect();
2941 journal.record_hosts(&hosts).expect("records hosts");
2942 journal
2943 .record_phases(original.phases())
2944 .expect("records phases");
2945
2946 let directory = journal.directory().to_path_buf();
2947 journal.close().expect("closes");
2948 directory
2949 };
2950
2951 let replayed = report(&directory).expect("reads");
2952
2953 assert_eq!(replayed.host_count(), original.host_count());
2954 assert_eq!(replayed.phases().len(), original.phases().len());
2955 assert_eq!(
2956 replayed.summary().hosts_alive,
2957 original.summary().hosts_alive
2958 );
2959 assert_eq!(replayed.summary().ports_open, original.summary().ports_open);
2960
2961 let (before, after) = (&original.phases()[0], &replayed.phases()[0]);
2963 assert_eq!(after.kind(), before.kind());
2964 assert_eq!(after.privilege(), before.privilege());
2965 assert_eq!(after.failures().len(), before.failures().len());
2966 assert_eq!(
2967 replayed.is_partial(),
2968 original.is_partial(),
2969 "a scan that left ground uncovered has to still say so"
2970 );
2971 }
2972
2973 #[test]
2975 fn a_replayed_report_names_the_engine_that_ran_the_scan() {
2976 let root = scratch("replay-version");
2977 let map = plan("192.0.2.1", "80");
2978
2979 let directory = {
2980 let journal = begin(&root, &map);
2981 let directory = journal.directory().to_path_buf();
2982 journal.close().expect("closes");
2983 directory
2984 };
2985
2986 let path = directory.join(MANIFEST);
2988 let mut manifest: JournalManifest =
2989 serde_json::from_str(&fs::read_to_string(&path).expect("reads")).expect("parses");
2990 manifest.engine_version = "0.1.0".to_string();
2991 fs::write(&path, serde_json::to_vec(&manifest).expect("encodes")).expect("writes");
2992
2993 let replayed = report(&directory).expect("reads");
2994
2995 assert_eq!(replayed.engine_version(), "0.1.0");
2996 assert_ne!(
2997 replayed.engine_version(),
2998 crate::report::ENGINE_VERSION,
2999 "this build's version must not have been stamped over the record's"
3000 );
3001 }
3002
3003 #[cfg(unix)]
3009 #[test]
3010 fn a_cursor_that_cannot_be_read_is_not_a_scan_that_settled_nothing() {
3011 use std::os::unix::fs::PermissionsExt;
3012
3013 let root = scratch("unreadable-cursor");
3014 let map = plan("192.0.2.1-192.0.2.4", "80");
3015
3016 let directory = {
3017 let mut journal = begin(&root, &map);
3018 let settlements = Settlements::default();
3019 settlements.record(Outcome::Answered { position: 0 });
3020 journal.checkpoint(&settlements).expect("checkpoints");
3021
3022 let directory = journal.directory().to_path_buf();
3023 journal.close().expect("closes");
3024 directory
3025 };
3026
3027 let listed = list(&root).expect("lists").entries;
3029 assert_eq!(listed[0].settled(), Some(1));
3030
3031 fs::set_permissions(directory.join(CURSOR), fs::Permissions::from_mode(0o000))
3032 .expect("removes every permission");
3033
3034 let listed = list(&root)
3035 .expect("a journal it cannot read must not hide the rest")
3036 .entries;
3037 assert_eq!(listed.len(), 1, "the journal still lists");
3038 assert_eq!(
3039 listed[0].settled(),
3040 None,
3041 "unreadable is not zero: zero is a claim about the scan"
3042 );
3043 assert!(
3044 !listed[0].is_complete(),
3045 "and nothing that cannot be read may be called finished"
3046 );
3047
3048 let _ = fs::set_permissions(directory.join(CURSOR), fs::Permissions::from_mode(0o600));
3050 }
3051
3052 #[tokio::test]
3058 async fn what_a_scan_learns_after_a_checkpoint_still_reaches_the_file() {
3059 use crate::model::host::{HostStatus, StatusProtocol, StatusReason};
3060
3061 let root = scratch("late-findings");
3062 let map = plan("192.0.2.1", "80");
3063 let journal = begin(&root, &map);
3064 let directory = journal.directory().to_path_buf();
3065
3066 let (_session, ctx) = crate::scanner::session::ScanSession::new();
3067 let mut ticker = spawn_checkpoints(journal, ctx.progress());
3068
3069 let ip: std::net::IpAddr = "192.0.2.1".parse().expect("an address");
3071 ctx.update_host(ip, |host| {
3072 host.set_status(HostStatus::Up);
3073 host.record_evidence(
3074 HostStatus::Up,
3075 StatusReason::new(StatusProtocol::Arp, "answered"),
3076 );
3077 });
3078
3079 ticker.checkpointed().await;
3081
3082 ctx.update_host(ip, |host| {
3084 host.record_evidence(
3085 HostStatus::Up,
3086 StatusReason::new(StatusProtocol::IcmpEcho, "echo answered"),
3087 );
3088 });
3089
3090 ticker.finish(&[]).await;
3091
3092 let restored = read_findings(&directory).expect("reads");
3093 assert_eq!(restored.len(), 1, "one host was scanned");
3094
3095 let protocols: Vec<_> = restored[0]
3096 .reasons()
3097 .iter()
3098 .map(|reason| reason.protocol.clone())
3099 .collect();
3100 assert!(
3101 protocols.contains(&StatusProtocol::IcmpEcho),
3102 "what the scan learned last was lost: {protocols:?}"
3103 );
3104 }
3105
3106 fn aged(id: &str, created_at: SystemTime, settled: u64, total: u128) -> Entry {
3107 Entry {
3108 directory: PathBuf::from(id),
3109 manifest: JournalManifest {
3110 links: Vec::new(),
3111 journal_version: crate::journal::JOURNAL_VERSION,
3112 id: id.to_string(),
3113 engine_version: "0.0.0".to_string(),
3114 created_at,
3115 kind: crate::record::wire::scan_kind_name(ScanKind::PortScan).to_owned(),
3116 plan: crate::journal::manifest::PlanFingerprint::of(
3117 &ports(&plan("192.0.2.1", "80")),
3118 Privilege::Raw,
3119 ),
3120 targets: crate::record::PlanRecord::from(&plan("192.0.2.1", "80")),
3121 technique: TcpScanTechnique::Syn.name().to_owned(),
3122 sweep: false,
3123 privilege: Privilege::Raw,
3124 total_targets: total,
3125 order_seed: None,
3126 summary: String::new(),
3127 },
3128 checkpoint: Some(Checkpoint {
3129 watermark: settled,
3130 settled_above: Vec::new(),
3131 walked: None,
3132 }),
3133 lock: LockState::Free,
3134 }
3135 }
3136
3137 fn ago(seconds: u64) -> SystemTime {
3138 now() - Duration::from_secs(seconds)
3139 }
3140
3141 fn now() -> SystemTime {
3142 SystemTime::UNIX_EPOCH + Duration::from_secs(1_000_000)
3143 }
3144
3145 #[test]
3147 fn age_removes_finished_journals_and_keeps_unfinished_ones() {
3148 let entries = vec![
3149 aged("finished-old", ago(100), 1, 1),
3150 aged("unfinished-old", ago(100), 0, 1),
3151 aged("finished-new", ago(1), 1, 1),
3152 ];
3153
3154 let retention = Retention {
3155 completed_for: Some(Duration::from_secs(50)),
3156 incomplete_for: None,
3157 keep_at_most: None,
3158 unreadable: false,
3159 };
3160
3161 assert_eq!(retention.expired(&entries, now()), vec![0]);
3162 }
3163
3164 #[test]
3166 fn a_live_journal_is_never_pruned() {
3167 let mut entries = vec![aged("held", ago(10_000), 1, 1)];
3168 entries[0].lock = LockState::Held {
3169 pid: 1,
3170 last_beat: Duration::from_secs(1),
3171 };
3172
3173 let retention = Retention {
3174 completed_for: Some(Duration::from_secs(1)),
3175 incomplete_for: Some(Duration::from_secs(1)),
3176 keep_at_most: Some(0),
3177 unreadable: false,
3178 };
3179
3180 assert!(retention.expired(&entries, now()).is_empty());
3181 }
3182
3183 #[test]
3186 fn the_cap_removes_duplicates_before_unfinished_work() {
3187 let entries = vec![
3189 aged("finished-new", ago(1), 1, 1),
3190 aged("unfinished-mid", ago(2), 0, 1),
3191 aged("finished-old", ago(3), 1, 1),
3192 ];
3193
3194 let retention = Retention {
3195 completed_for: None,
3196 incomplete_for: None,
3197 keep_at_most: Some(2),
3198 unreadable: false,
3199 };
3200
3201 let removed = retention.expired(&entries, now());
3202 assert_eq!(removed, vec![2], "the older of the two finished ones");
3203
3204 let retention = Retention {
3206 keep_at_most: Some(1),
3207 ..retention
3208 };
3209 let removed = retention.expired(&entries, now());
3210 assert_eq!(
3211 removed,
3212 vec![0, 2],
3213 "both finished ones, the unfinished kept"
3214 );
3215 }
3216
3217 #[test]
3219 fn keeping_everything_removes_nothing() {
3220 let entries = vec![
3221 aged("ancient", ago(900_000), 1, 1),
3222 aged("also-ancient", ago(900_000), 0, 1),
3223 ];
3224
3225 assert!(
3226 Retention::keep_everything()
3227 .expired(&entries, now())
3228 .is_empty()
3229 );
3230 }
3231
3232 #[test]
3235 fn the_default_favours_unfinished_work() {
3236 let two_months = Duration::from_secs(60 * 24 * 60 * 60);
3237 let created = now() - two_months;
3238
3239 let entries = vec![
3240 aged("finished", created, 1, 1),
3241 aged("unfinished", created, 0, 1),
3242 ];
3243
3244 assert_eq!(Retention::default().expired(&entries, now()), vec![0]);
3245 }
3246
3247 #[test]
3249 fn pruning_removes_what_the_policy_selected() {
3250 let root = scratch("retention");
3251 let map = plan("192.0.2.1", "80");
3252
3253 let mut journal = begin(&root, &map);
3255 let settlements = Settlements::default();
3256 settlements.record(Outcome::Answered { position: 0 });
3257 journal.checkpoint(&settlements).expect("checkpoints");
3258 let id = journal.manifest().id.clone();
3259 journal.close().expect("closes");
3260
3261 let untouched = prune(&root, &Retention::default()).expect("prunes");
3263 assert!(untouched.removed.is_empty());
3264 assert_eq!(list(&root).expect("lists").entries.len(), 1);
3265
3266 let swept = prune(
3268 &root,
3269 &Retention {
3270 completed_for: Some(Duration::ZERO),
3271 ..Retention::default()
3272 },
3273 )
3274 .expect("prunes");
3275
3276 assert_eq!(swept.removed, vec![id]);
3277 assert!(swept.held.is_empty());
3278 assert!(list(&root).expect("lists").entries.is_empty());
3279
3280 std::fs::remove_dir_all(&root).ok();
3281 }
3282
3283 #[test]
3288 fn a_host_written_under_two_addresses_comes_back_as_one() {
3289 use crate::model::host::HostStatus;
3290
3291 let root = scratch("promoted");
3292 let map = plan("192.0.2.1", "80");
3293 let link_local: std::net::IpAddr = "fe80::1".parse().expect("an address");
3294 let global: std::net::IpAddr = "2001:db8::1".parse().expect("an address");
3295
3296 let directory = {
3297 let mut journal = begin(&root, &map);
3298
3299 let mut early = Host::new(link_local);
3301 early.set_status(HostStatus::Up);
3302 journal.record_hosts(&[early]).expect("records");
3303
3304 let mut promoted = Host::new(link_local);
3306 promoted.add_ip(global);
3307 assert!(
3308 promoted.consider_primary_ip(global),
3309 "the global address should lead"
3310 );
3311 promoted.set_hostname(Some("router.example".to_string()));
3312 journal.record_hosts(&[promoted]).expect("records again");
3313
3314 let directory = journal.directory().to_path_buf();
3315 journal.close().expect("closes");
3316 directory
3317 };
3318
3319 let (journal, _) =
3320 Journal::resume(&directory, &ports(&map), Privilege::Raw).expect("resumes");
3321
3322 let restored = journal.restored();
3323 assert_eq!(
3324 restored.len(),
3325 1,
3326 "one machine, written twice: {:?}",
3327 restored.iter().map(Host::primary_ip).collect::<Vec<_>>()
3328 );
3329 assert!(restored[0].is_alive(), "the first record's status survived");
3330 assert_eq!(restored[0].hostname(), Some("router.example"));
3331 }
3332
3333 #[cfg(unix)]
3335 #[test]
3336 fn every_file_a_journal_writes_is_private() {
3337 use std::os::unix::fs::PermissionsExt;
3338
3339 let root = scratch("modes");
3340 let map = plan("192.0.2.1", "80");
3341
3342 let mut journal = begin(&root, &map);
3343 let settlements = Settlements::default();
3344 settlements.record(Outcome::Answered { position: 0 });
3345 journal
3346 .record(
3347 &[Host::new("192.0.2.1".parse().expect("an address"))],
3348 &settlements,
3349 )
3350 .expect("records");
3351 let directory = journal.directory().to_path_buf();
3352
3353 let mode_of = |path: &Path| {
3354 fs::metadata(path)
3355 .unwrap_or_else(|e| panic!("{path:?}: {e}"))
3356 .permissions()
3357 .mode()
3358 & 0o777
3359 };
3360
3361 assert_eq!(mode_of(&directory), 0o700, "the directory");
3362 for name in [MANIFEST, CURSOR, HOSTS, PHASES, LOCK] {
3363 assert_eq!(mode_of(&directory.join(name)), 0o600, "{name}");
3364 }
3365
3366 journal.checkpoint(&settlements).expect("beats");
3368 assert_eq!(mode_of(&directory.join(LOCK)), 0o600, "after a heartbeat");
3369
3370 journal.close().expect("closes");
3371 std::fs::remove_dir_all(&root).ok();
3372 }
3373
3374 #[cfg(unix)]
3379 #[test]
3380 fn appending_refuses_a_link_where_a_journal_file_should_be() {
3381 let root = scratch("nofollow");
3382 let map = plan("192.0.2.1", "80");
3383 let mut journal = begin(&root, &map);
3384 let directory = journal.directory().to_path_buf();
3385
3386 let elsewhere = root.join("not-the-journals-to-touch");
3387 fs::write(&elsewhere, b"untouched").expect("writes");
3388
3389 for name in [HOSTS, PHASES] {
3390 fs::remove_file(directory.join(name)).expect("removes");
3391 std::os::unix::fs::symlink(&elsewhere, directory.join(name)).expect("links");
3392 }
3393
3394 let phases = crate::export::fixture::report().phases().to_vec();
3395 assert!(
3396 journal
3397 .record_hosts(&[Host::new("192.0.2.1".parse().expect("an address"))])
3398 .is_err(),
3399 "a link was appended to as the findings file"
3400 );
3401 assert!(
3402 journal.record_phases(&phases).is_err(),
3403 "a link was appended to as the phases file"
3404 );
3405 assert_eq!(
3406 fs::read(&elsewhere).expect("reads"),
3407 b"untouched",
3408 "the file behind the link was written through"
3409 );
3410
3411 journal.close().expect("closes");
3412 std::fs::remove_dir_all(&root).ok();
3413 }
3414
3415 #[test]
3420 fn a_host_recorded_twice_keeps_the_round_trips_its_last_record_holds() {
3421 use std::time::Duration;
3422
3423 let root = scratch("latency");
3424 let mut journal = begin(&root, &plan("192.0.2.1", "80"));
3425 let mut host = Host::new("192.0.2.1".parse().expect("an address"));
3426 host.add_rtts([Duration::from_millis(1), Duration::from_millis(2)]);
3427 journal.record_hosts(&[host.clone()]).expect("records");
3428 host.add_rtts([Duration::from_millis(6), Duration::from_millis(7)]);
3429 journal
3430 .record_hosts(&[host.clone()])
3431 .expect("records again");
3432 let directory = journal.directory().to_path_buf();
3433 journal.close().expect("closes");
3434
3435 let read = report(&directory).expect("reads");
3436 std::fs::remove_dir_all(&root).ok();
3437 let [restored] = read.hosts().collect::<Vec<_>>()[..] else {
3438 panic!("one host on record");
3439 };
3440 let figures = |host: &Host| (host.min_rtt(), host.average_rtt(), host.max_rtt());
3441 assert_eq!(figures(restored), figures(&host), "min, average, max");
3442 assert_eq!(
3443 restored.telemetry().history().len(),
3444 host.telemetry().history().len(),
3445 "round trips held"
3446 );
3447 }
3448
3449 #[test]
3454 fn a_host_recorded_at_every_checkpoint_reads_back_as_the_host_it_is() {
3455 use crate::model::confidence::Confidence;
3456 use crate::model::finding::{DetectionClass, DetectionId, Finding, Severity, Version};
3457 use crate::model::host::{OsEvidence, OsSource};
3458
3459 let root = scratch("repeated");
3460 let mut journal = begin(&root, &plan("203.0.113.1-203.0.113.9", "22,80,443,445"));
3461 let mut hosts: Vec<Host> = crate::export::fixture::report().hosts().cloned().collect();
3462 for checkpoint in 0..3u64 {
3463 let finding = Finding::new(
3464 DetectionId::new(format!("check-{checkpoint}"), Version::new(1, 0, 0), "")
3465 .expect("an id"),
3466 "A check",
3467 Severity::Low,
3468 Confidence::Certain,
3469 DetectionClass::Passive,
3470 )
3471 .expect("a finding");
3472 for host in &mut hosts {
3473 host.add_rtt(Duration::from_micros(700 + 100 * checkpoint));
3474 host.record_os_evidence(OsEvidence {
3475 source: OsSource::ServiceBanner,
3476 family: Some("Linux".to_string()),
3477 device: None,
3478 vendor: None,
3479 product: None,
3480 version: None,
3481 kernel: None,
3482 arch: None,
3483 cpe: None,
3484 confidence: 0.5,
3485 evidence: format!("banner read at checkpoint {checkpoint}"),
3486 });
3487 host.add_finding(finding.clone());
3488 let first = host
3489 .ports()
3490 .next()
3491 .map(|port| (port.number(), port.protocol()));
3492 if let Some((number, protocol)) = first {
3493 host.add_port_finding(number, protocol, finding.clone());
3494 }
3495 }
3496 journal.record_hosts(&hosts).expect("records");
3497 }
3498 let directory = journal.directory().to_path_buf();
3499 journal.close().expect("closes");
3500
3501 let mut read = read_findings(&directory).expect("reads");
3502 std::fs::remove_dir_all(&root).ok();
3503 read.sort_by_key(Host::primary_ip);
3504 hosts.sort_by_key(Host::primary_ip);
3505 let as_recorded = |hosts: &[Host]| {
3506 hosts
3507 .iter()
3508 .map(|host| serde_json::to_value(HostRecord::from(host)).expect("serialises"))
3509 .collect::<Vec<_>>()
3510 };
3511 assert_eq!(as_recorded(&read), as_recorded(&hosts));
3512 }
3513
3514 #[test]
3517 fn a_journal_from_a_newer_format_is_passed_over_by_name() {
3518 let root = scratch("passed-over");
3519 let map = plan("192.0.2.1", "80");
3520 let readable = begin(&root, &map);
3521 readable.close().expect("closes");
3522 let newer = begin(&root, &map);
3523 let newer_dir = newer.directory().to_path_buf();
3524 newer.close().expect("closes");
3525 age_forward(&newer_dir);
3526
3527 let listing = list(&root).expect("lists");
3528 std::fs::remove_dir_all(&root).ok();
3529
3530 assert_eq!(listing.entries.len(), 1, "the readable journal lists");
3531 let [passed] = listing.passed_over.as_slice() else {
3532 panic!("one passed over: {:?}", listing.passed_over);
3533 };
3534 assert_eq!(passed.directory, newer_dir);
3535 assert_eq!(
3536 Some(passed.name.as_str()),
3537 newer_dir.file_name().and_then(|name| name.to_str())
3538 );
3539 assert_eq!(
3540 passed.why,
3541 Unlisted::NewerFormat {
3542 found: super::super::JOURNAL_VERSION + 1
3543 }
3544 );
3545 }
3546
3547 #[cfg(unix)]
3550 #[test]
3551 fn a_prune_takes_unreadable_journals_only_when_asked_and_never_a_link() {
3552 let root = scratch("prune-unreadable");
3553 let elsewhere = scratch("prune-unreadable-elsewhere");
3554 let map = plan("192.0.2.1", "80");
3555 let newer = begin(&root, &map);
3556 let newer_dir = newer.directory().to_path_buf();
3557 newer.close().expect("closes");
3558 age_forward(&newer_dir);
3559 let target = begin(&elsewhere, &map);
3560 let target_dir = target.directory().to_path_buf();
3561 target.close().expect("closes");
3562 std::os::unix::fs::symlink(&target_dir, root.join("06LINKED00000000")).expect("links");
3563
3564 let ordinary = prune(&root, &Retention::default()).expect("prunes");
3565 assert!(ordinary.removed.is_empty() && ordinary.held.is_empty());
3566 assert!(newer_dir.exists(), "an ordinary sweep leaves it");
3567
3568 let mut asked = Retention::keep_everything();
3569 asked.unreadable = true;
3570 let pruned = prune(&root, &asked).expect("prunes");
3571 let newer_gone = !newer_dir.exists();
3572 let target_kept = target_dir.exists();
3573 let link_kept = root.join("06LINKED00000000").symlink_metadata().is_ok();
3574 std::fs::remove_dir_all(&root).ok();
3575 std::fs::remove_dir_all(&elsewhere).ok();
3576
3577 assert!(newer_gone, "asked for, it goes");
3578 assert_eq!(pruned.removed.len(), 1, "{pruned:?}");
3579 assert!(pruned.held.is_empty(), "a link is not selected: {pruned:?}");
3580 assert!(
3581 link_kept && target_kept,
3582 "neither the link nor what it names goes"
3583 );
3584 }
3585
3586 fn age_forward(directory: &Path) {
3588 let path = directory.join(MANIFEST);
3589 let mut manifest: serde_json::Value =
3590 serde_json::from_str(&fs::read_to_string(&path).expect("reads")).expect("parses");
3591 manifest["journal_version"] = serde_json::json!(super::super::JOURNAL_VERSION + 1);
3592 fs::write(&path, serde_json::to_vec(&manifest).expect("encodes")).expect("writes");
3593 }
3594
3595 #[cfg(unix)]
3598 #[test]
3599 fn a_link_in_a_root_of_journals_is_not_listed() {
3600 let root = scratch("listed-link");
3601 let elsewhere = scratch("listed-link-elsewhere");
3602 let journal = begin(&elsewhere, &plan("192.0.2.1", "80"));
3603 let directory = journal.directory().to_path_buf();
3604 journal.close().expect("closes");
3605 std::os::unix::fs::symlink(&directory, root.join("06LINKED00000000")).expect("links");
3606
3607 let listed = list(&root).expect("lists").entries.len();
3608 let where_it_is = list(&elsewhere).expect("lists").entries.len();
3609 std::fs::remove_dir_all(&root).ok();
3610 std::fs::remove_dir_all(&elsewhere).ok();
3611 assert_eq!(listed, 0, "the link was listed as a journal");
3612 assert_eq!(where_it_is, 1, "the journal itself lists");
3613 }
3614
3615 #[cfg(unix)]
3621 #[test]
3622 fn reading_refuses_a_link_where_a_journal_file_should_be() {
3623 let root = scratch("read-nofollow");
3624 let mut ours = begin(&root, &plan("192.0.2.1", "80"));
3625 ours.record_hosts(&[Host::new("192.0.2.1".parse().expect("an address"))])
3626 .expect("records");
3627 ours.write_cursor(&Checkpoint::default())
3628 .expect("checkpoints");
3629 let directory = ours.directory().to_path_buf();
3630 ours.close().expect("closes");
3631
3632 let mut theirs = begin(&root, &plan("198.51.100.7", "80"));
3633 theirs
3634 .record_hosts(&[Host::new("198.51.100.7".parse().expect("an address"))])
3635 .expect("records");
3636 theirs
3637 .write_cursor(&Checkpoint::new(1, []))
3638 .expect("checkpoints");
3639 let elsewhere = theirs.directory().to_path_buf();
3640 theirs.close().expect("closes");
3641 let text = root.join("not-a-journal-file");
3642 fs::write(&text, b"somebody else's bytes").expect("writes");
3643
3644 let refused = |name: &str, error: String| {
3645 assert!(
3646 error.contains("is a link"),
3647 "{name}: refused, but not as a link: {error}"
3648 );
3649 };
3650 for name in [MANIFEST, CURSOR, HOSTS, PHASES] {
3651 let kept = directory.join(format!("{name}.kept"));
3652 fs::rename(directory.join(name), &kept).expect("moves aside");
3653 std::os::unix::fs::symlink(elsewhere.join(name), directory.join(name)).expect("links");
3654
3655 match Journal::reopen(&directory, Privilege::Raw) {
3656 Ok(_) => panic!("{name}: resumed through a link"),
3657 Err(error) => refused(name, error.to_string()),
3658 }
3659 if matches!(name, MANIFEST | HOSTS | PHASES) {
3660 match report(&directory) {
3661 Ok(_) => panic!("{name}: reported through a link"),
3662 Err(error) => refused(name, error.to_string()),
3663 }
3664 }
3665
3666 fs::remove_file(directory.join(name)).expect("removes the link");
3667 fs::rename(&kept, directory.join(name)).expect("restores");
3668 }
3669
3670 let journal = begin(&root, &plan("192.0.2.2", "80"));
3672 for name in [DETECTIONS, FINISHED] {
3673 std::os::unix::fs::symlink(&text, journal.directory().join(name)).expect("links");
3674 }
3675 match read_detections(journal.directory()) {
3676 Ok(runs) => panic!("detections read through a link: {runs:?}"),
3677 Err(error) => refused(DETECTIONS, error.to_string()),
3678 }
3679 match journal.finished_hosts() {
3680 Ok(hosts) => panic!("finished hosts read through a link: {hosts:?}"),
3681 Err(error) => refused(FINISHED, error.to_string()),
3682 }
3683 journal.close().expect("closes");
3684
3685 std::fs::remove_dir_all(&root).ok();
3686 }
3687
3688 #[test]
3693 fn a_cursor_repeating_settled_positions_does_not_inflate_the_count() {
3694 let root = scratch("inflated-cursor");
3695 let map = plan("192.0.2.1-192.0.2.4", "80,443");
3696
3697 let directory = {
3698 let journal = begin(&root, &map);
3699 let directory = journal.directory().to_path_buf();
3700 journal.close().expect("closes");
3701 directory
3702 };
3703
3704 let checkpoint = Checkpoint {
3707 watermark: 4,
3708 settled_above: vec![0, 1, 2, 6],
3709 walked: None,
3710 };
3711 checkpoint
3712 .write_atomically(&directory.join(CURSOR))
3713 .expect("writes");
3714
3715 let entry = list(&root)
3716 .expect("lists")
3717 .entries
3718 .into_iter()
3719 .next()
3720 .expect("one journal");
3721
3722 assert_eq!(
3723 entry.settled(),
3724 Some(5),
3725 "the watermark plus the one position genuinely above it"
3726 );
3727 assert!(
3728 !entry.is_complete(),
3729 "eight targets, five settled: a retention sweep must not take this"
3730 );
3731 }
3732
3733 #[test]
3739 fn an_append_after_a_torn_tail_leaves_the_journal_readable() {
3740 let root = scratch("torn-tail-append");
3741 let map = plan("192.0.2.1-192.0.2.4", "80");
3742 let first: std::net::IpAddr = "192.0.2.1".parse().expect("an address");
3743 let second: std::net::IpAddr = "192.0.2.2".parse().expect("an address");
3744
3745 let directory = {
3746 let mut journal = begin(&root, &map);
3747 journal.record_hosts(&[Host::new(first)]).expect("records");
3748 let directory = journal.directory().to_path_buf();
3749 journal.close().expect("closes");
3750 directory
3751 };
3752
3753 {
3755 use std::io::Write;
3756 let mut file = fs::OpenOptions::new()
3757 .append(true)
3758 .open(directory.join(HOSTS))
3759 .expect("opens");
3760 file.write_all(br#"{"ip":"192.0.2.9","stat"#)
3761 .expect("tears");
3762 }
3763
3764 let (mut journal, _) = Journal::resume(&directory, &ports(&map), Privilege::Raw)
3765 .expect("resumes past the tear");
3766 journal.record_hosts(&[Host::new(second)]).expect("records");
3767 journal.close().expect("closes");
3768
3769 let restored = read_findings(&directory).expect("the journal still reads");
3770 let addresses: Vec<_> = restored.iter().map(Host::primary_ip).collect();
3771 assert_eq!(
3772 addresses,
3773 vec![first, second],
3774 "the whole record before the tear and the one after it both survive"
3775 );
3776 }
3777
3778 #[test]
3783 fn an_append_to_a_findings_file_whose_header_never_landed_mends_it() {
3784 let root = scratch("headerless-append");
3785 let map = plan("192.0.2.1", "80");
3786 let address: std::net::IpAddr = "192.0.2.1".parse().expect("an address");
3787
3788 let mut journal = begin(&root, &map);
3789 let directory = journal.directory().to_path_buf();
3790
3791 fs::write(directory.join(HOSTS), b"").expect("empties");
3792
3793 journal
3794 .record_hosts(&[Host::new(address)])
3795 .expect("records");
3796 journal.close().expect("closes");
3797
3798 let restored = read_findings(&directory).expect("reads");
3799 assert_eq!(
3800 restored.len(),
3801 1,
3802 "a record appended under no header is a record nothing reads back"
3803 );
3804 }
3805
3806 #[test]
3809 fn an_append_to_a_file_this_engine_did_not_write_is_refused() {
3810 let root = scratch("foreign-append");
3811 let map = plan("192.0.2.1", "80");
3812
3813 let mut journal = begin(&root, &map);
3814 let directory = journal.directory().to_path_buf();
3815
3816 let foreign: &[u8] = b"{\"not\":\"a journal\"}\n{\"still\":\"not\"}";
3818 fs::write(directory.join(HOSTS), foreign).expect("plants");
3819
3820 assert!(
3821 journal
3822 .record_hosts(&[Host::new("192.0.2.1".parse().expect("an address"))])
3823 .is_err(),
3824 "a file with no header was appended to"
3825 );
3826 assert_eq!(
3827 fs::read(directory.join(HOSTS)).expect("reads"),
3828 foreign,
3829 "the file was edited before it was refused"
3830 );
3831
3832 journal.close().expect("closes");
3833 }
3834
3835 #[test]
3837 fn a_journal_that_cannot_be_locked_leaves_no_directory() {
3838 let root = scratch("unlockable");
3839 let map = plan("192.0.2.1", "80");
3840
3841 let before = list(&root).expect("lists").entries.len();
3844 assert_eq!(before, 0);
3845
3846 let journal = begin(&root, &map);
3847 let directory = journal.directory().to_path_buf();
3848
3849 assert!(matches!(
3852 Journal::resume(&directory, &ports(&map), Privilege::Raw),
3853 Err(OpenError::Locked(_))
3854 ));
3855 assert!(directory.exists(), "the held journal is untouched");
3856
3857 journal.close().expect("closes");
3858 std::fs::remove_dir_all(&root).ok();
3859 }
3860
3861 #[test]
3863 fn a_host_written_again_carries_only_what_changed_in_it() {
3864 use crate::model::port::{Port, PortState, Protocol};
3865
3866 let root = scratch("delta");
3867 let map = plan("192.0.2.1", "1-400");
3868 let mut journal = begin(&root, &map);
3869 let directory = journal.directory().to_path_buf();
3870 let length = || fs::metadata(directory.join(HOSTS)).expect("stats").len();
3871
3872 let ip: std::net::IpAddr = "192.0.2.1".parse().expect("an address");
3873 let mut host = Host::new(ip);
3874 for number in 1..=400 {
3875 host.add_port(Port::new(number, Protocol::Tcp, PortState::Closed));
3876 }
3877 let before = length();
3878 journal
3879 .record_hosts(std::slice::from_ref(&host))
3880 .expect("records");
3881 let first = length() - before;
3882
3883 host.add_port(Port::new(22, Protocol::Tcp, PortState::Open));
3884 let before = length();
3885 journal
3886 .record_hosts(std::slice::from_ref(&host))
3887 .expect("records");
3888 let second = length() - before;
3889
3890 assert!(
3891 second * 20 < first,
3892 "one port opening appended {second} bytes against the host's {first}"
3893 );
3894 journal.close().expect("closes");
3895
3896 let restored = read_findings(&directory).expect("reads");
3897 assert_eq!(restored.len(), 1);
3898 assert_eq!(restored[0].port_count(), 400, "every port read back");
3899 let open: Vec<u16> = restored[0]
3900 .ports()
3901 .filter(|port| port.state() == PortState::Open)
3902 .map(|port| port.number())
3903 .collect();
3904 assert_eq!(open, [22], "the change read back over what it changed");
3905 std::fs::remove_dir_all(&root).ok();
3906 }
3907
3908 #[test]
3913 fn a_record_counts_what_it_supersedes_and_not_what_it_adds() {
3914 use crate::model::port::{Port, PortState, Protocol};
3915
3916 let ip: std::net::IpAddr = "192.0.2.1".parse().expect("an address");
3917 let mut host = Host::new(ip);
3918 host.add_port(Port::new(22, Protocol::Tcp, PortState::Closed));
3919 let mut written = Written::default();
3920
3921 let (_, first) = written.delta(&host).expect("measures").expect("new");
3922 let (rest, port) = (
3923 u64::from(first.rest.length),
3924 u64::from(first.ports[0].1.length),
3925 );
3926 written.update(first);
3927 assert_eq!(written.superseded, 0, "a first record supersedes nothing");
3928
3929 host.add_port(Port::new(80, Protocol::Tcp, PortState::Closed));
3930 let (_, second) = written.delta(&host).expect("measures").expect("a new port");
3931 assert_eq!(second.ports.len(), 1, "only the new port is written");
3932 written.update(second);
3933 assert_eq!(written.superseded, rest, "the host's own fields, again");
3934
3935 host.add_port(Port::new(22, Protocol::Tcp, PortState::Open));
3936 let (_, third) = written
3937 .delta(&host)
3938 .expect("measures")
3939 .expect("a moved port");
3940 let rest_again = written.hosts[&host.scoped_ip()]
3941 .rest
3942 .map_or(0, |mark| u64::from(mark.length));
3943 written.update(third);
3944 assert_eq!(
3945 written.superseded,
3946 rest + rest_again + port,
3947 "and port 22's closed record under its open one"
3948 );
3949 }
3950
3951 #[test]
3955 fn a_failed_compaction_leaves_nothing_behind_and_waits_before_trying_again() {
3956 let root = scratch("failed-compaction");
3957 let map = plan("192.0.2.1", "80");
3958 let mut journal = begin(&root, &map);
3959 let directory = journal.directory().to_path_buf();
3960 let host = Host::new("192.0.2.1".parse().expect("an address"));
3961 journal
3962 .record_hosts(std::slice::from_ref(&host))
3963 .expect("records");
3964
3965 journal.written.superseded = 3 * COMPACT_FLOOR;
3967 journal.length = 4 * COMPACT_FLOOR;
3968 assert!(journal.should_compact());
3969
3970 fs::remove_file(directory.join(HOSTS)).expect("removes the findings");
3971 fs::create_dir_all(directory.join(HOSTS).join("occupied")).expect("blocks the name");
3972 assert!(journal.compact(std::slice::from_ref(&host)).is_err());
3973
3974 assert!(
3975 !directory.join(HOSTS).with_extension("jsonl-tmp").exists(),
3976 "the partial copy is left holding the disk's room"
3977 );
3978 assert!(
3979 !journal.should_compact(),
3980 "a compaction that just failed is due again at once"
3981 );
3982 journal.written.superseded += 2 * COMPACT_FLOOR;
3983 assert!(
3984 journal.should_compact(),
3985 "and again once as much is superseded"
3986 );
3987
3988 drop(journal);
3989 std::fs::remove_dir_all(&root).ok();
3990 }
3991
3992 #[test]
3995 fn a_findings_file_is_compacted_by_what_it_superseded() {
3996 const MIB: u64 = 1024 * 1024;
3997
3998 assert!(
3999 !outgrown(60 * MIB, 0),
4000 "a file that grew by what the scan found holds nothing to drop"
4001 );
4002 assert!(!outgrown(6 * MIB, 3 * MIB), "a small file is left alone");
4003 assert!(outgrown(9 * MIB, 5 * MIB), "past the floor it is rewritten");
4004 assert!(
4005 !outgrown(50 * MIB, 20 * MIB),
4006 "twenty superseded beside thirty live is not yet worth it"
4007 );
4008 assert!(
4009 outgrown(50 * MIB, 26 * MIB),
4010 "twenty-six beside twenty-four is"
4011 );
4012 }
4013
4014 #[test]
4017 fn compaction_bounds_the_findings_file_without_losing_a_host() {
4018 use crate::model::host::HostStatus;
4019 use crate::model::port::{Port, PortState, Protocol};
4020
4021 let root = scratch("compaction");
4022 let map = plan("192.0.2.1-192.0.2.4", "80");
4023 let addresses: Vec<std::net::IpAddr> = (1..=4)
4024 .map(|last| format!("192.0.2.{last}").parse().expect("an address"))
4025 .collect();
4026
4027 let directory = {
4028 let mut journal = begin(&root, &map);
4029
4030 let mut live: Vec<Host> = addresses.iter().map(|ip| Host::new(*ip)).collect();
4032 for round in 0..70 {
4033 for host in &mut live {
4034 host.set_status(HostStatus::Up);
4035 host.add_port(Port::new(1000 + round, Protocol::Tcp, PortState::Open));
4036 }
4037 journal.record_hosts(&live).expect("records");
4038 }
4039
4040 journal.compact(&live).expect("compacts");
4041 assert!(!journal.should_compact(), "a file just written whole");
4042
4043 let directory = journal.directory().to_path_buf();
4044 journal.close().expect("closes");
4045 directory
4046 };
4047
4048 let lines = std::fs::read_to_string(directory.join(HOSTS))
4050 .expect("reads")
4051 .lines()
4052 .count();
4053 assert_eq!(lines, 1 + 4, "a header and one record each");
4054
4055 let (journal, _) =
4057 Journal::resume(&directory, &ports(&map), Privilege::Raw).expect("resumes");
4058
4059 let restored = journal.restored();
4060 assert_eq!(restored.len(), 4);
4061 for host in restored {
4062 assert!(host.is_alive());
4063 assert_eq!(host.port_count(), 70, "every round's port survived");
4064 }
4065
4066 std::fs::remove_dir_all(&root).ok();
4067 }
4068
4069 #[test]
4072 fn a_journal_gives_back_the_plan_it_recorded() {
4073 let root = scratch("reopen");
4074 let map = plan("192.0.2.1-192.0.2.4", "80,443,u:53");
4075
4076 let directory = {
4077 let journal = Journal::create(
4078 &root,
4079 &Plan::port_scan(&map, &Exclusions::none(), TcpScanTechnique::Fin),
4080 Privilege::Raw,
4081 "test",
4082 )
4083 .expect("creates");
4084 let directory = journal.directory().to_path_buf();
4085 journal.close().expect("closes");
4086 directory
4087 };
4088
4089 let (journal, _checkpoint, recovered) =
4090 Journal::reopen(&directory, Privilege::Raw).expect("reopens");
4091
4092 let recovered = recovered.targets().expect("a port scan's plan");
4093 assert_eq!(
4094 recovered.iter().collect::<Vec<_>>(),
4095 map.iter().collect::<Vec<_>>(),
4096 "the same targets, in the same order, so positions still mean what they did"
4097 );
4098 assert_eq!(journal.manifest().technique(), TcpScanTechnique::Fin);
4099 }
4100
4101 #[test]
4103 fn a_journal_will_not_be_continued_under_different_privileges() {
4104 let root = scratch("reopen-privilege");
4105 let map = plan("192.0.2.1", "80");
4106
4107 let journal =
4108 Journal::create(&root, &ports(&map), Privilege::Raw, "test").expect("creates");
4109 let directory = journal.directory().to_path_buf();
4110 journal.close().expect("closes");
4111
4112 let refused =
4113 Journal::reopen(&directory, Privilege::Connect).expect_err("privileges differ");
4114 assert!(matches!(refused, OpenError::PlanChanged(_)), "{refused:?}");
4115 }
4116
4117 #[test]
4121 fn preparing_a_root_creates_the_whole_path_to_it() {
4122 let root = scratch("prepare-root")
4123 .join("state")
4124 .join("zond")
4125 .join("journals");
4126 assert!(!root.exists());
4127
4128 prepare_root(&root).expect("the path is created");
4129 assert!(root.is_dir());
4130
4131 prepare_root(&root).expect("an existing root is not an error");
4133
4134 let plan = Plan::listen(vec![crate::model::ip::scoped::Zone::new(3, "en0")]);
4135 Journal::create(&root, &plan, Privilege::Connect, "listening")
4136 .expect("a journal is created inside it")
4137 .close()
4138 .expect("it closes");
4139 }
4140
4141 #[cfg(unix)]
4144 #[test]
4145 fn preparing_a_root_creates_private_directories() {
4146 use std::os::unix::fs::PermissionsExt;
4147
4148 let home = scratch("prepare-root-private");
4149 prepare_root_with(
4150 &home.join(".local/state/zond/journals"),
4151 true,
4152 None,
4153 |_, _| {},
4154 )
4155 .expect("the path is created");
4156
4157 for below in [
4158 ".local",
4159 ".local/state",
4160 ".local/state/zond",
4161 ".local/state/zond/journals",
4162 ] {
4163 let mode = fs::metadata(home.join(below))
4164 .expect("made")
4165 .permissions()
4166 .mode();
4167 assert_eq!(
4168 format!("{:o}", mode & 0o777),
4169 "700",
4170 "{below} was created readable by others"
4171 );
4172 }
4173 let _ = fs::remove_dir_all(&home);
4174 }
4175
4176 #[test]
4183 fn preparing_a_root_gives_away_what_it_created_and_repairs_what_root_left() {
4184 use Hand::{Give, Reclaim};
4185
4186 let home = scratch("prepare-root-home");
4187 let root = home.join(".local/state/zond/journals");
4188 let prepared = |root: &Path, own: bool, home: Option<&Path>| {
4189 let mut claimed = Vec::new();
4190 prepare_root_with(root, own, home, |path, hand| {
4191 claimed.push((path.to_path_buf(), hand));
4192 })
4193 .expect("the path is created");
4194 claimed
4195 };
4196 let under = |pairs: &[(&str, Hand)]| -> Vec<(PathBuf, Hand)> {
4197 pairs
4198 .iter()
4199 .map(|(below, hand)| (home.join(below), *hand))
4200 .collect()
4201 };
4202
4203 assert_eq!(
4204 prepared(&root, true, Some(&home)),
4205 under(&[
4206 (".local", Give),
4207 (".local/state", Give),
4208 (".local/state/zond", Give),
4209 (".local/state/zond/journals", Give),
4210 ]),
4211 "a directory created for the journal was left to root"
4212 );
4213
4214 fs::remove_dir_all(home.join(".local/state/zond")).expect("removes");
4217 assert_eq!(
4218 prepared(&root, true, Some(&home)),
4219 under(&[
4220 (".local", Reclaim),
4221 (".local/state", Reclaim),
4222 (".local/state/zond", Give),
4223 (".local/state/zond/journals", Give),
4224 ])
4225 );
4226
4227 assert_eq!(
4229 prepared(&root, true, None),
4230 under(&[
4231 (".local/state/zond", Give),
4232 (".local/state/zond/journals", Give)
4233 ])
4234 );
4235
4236 let named = home.join("elsewhere/journals");
4238 assert_eq!(
4239 prepared(&named, false, Some(&home)),
4240 [(named.clone(), Give)]
4241 );
4242
4243 let _ = fs::remove_dir_all(&home);
4244 }
4245
4246 #[test]
4249 fn a_watch_is_never_complete_however_long_it_ran() {
4250 use crate::model::ip::scoped::Zone;
4251
4252 let root = scratch("watch-never-complete");
4253 let plan = Plan::listen(vec![Zone::new(3, "en0")]);
4254 let journal = Journal::create(&root, &plan, Privilege::Raw, "listening on en0")
4255 .expect("a journal is created");
4256 journal.close().expect("it closes");
4257
4258 let entries = list(&root).expect("the root lists").entries;
4259 let entry = entries.first().expect("the watch is there");
4260
4261 assert_eq!(entry.kind(), ScanKind::Listen);
4262 assert!(
4263 !entry.is_complete(),
4264 "another sitting can always be appended to a watch"
4265 );
4266 }
4267}