Skip to main content

zond_engine/journal/
store.rs

1// Copyright (c) 2026 Erik Lening (hollowpointer) and Contributors
2//
3// This file is part of Zond Engine, licensed under the GNU Affero General
4// Public License, version 3 or later. See the LICENSE file for details, or
5// <https://www.gnu.org/licenses/agpl-3.0.html>.
6//
7// SPDX-License-Identifier: AGPL-3.0-or-later
8
9//! # One scan's journal on disk
10//!
11//! ```text
12//! <root>/<id>/
13//!     manifest.json   the plan, written as the job begins
14//!     cursor.json     how far the scan got, rewritten on a timer
15//!     hosts.jsonl     what it found, appended as it finds it
16//!     phases.jsonl    what each sitting did, appended as each one ends
17//!     sitting-*.jsonl what a sitting has done so far, until it ends
18//!     LOCK            who is writing, if anyone
19//! ```
20//!
21//! The cursor is rewritten because it describes one state; the findings are appended
22//! because they accumulate. A host that changes appears more than once and the records fold
23//! together on reading, later over earlier, so a torn tail costs at most one host's latest
24//! update.
25//!
26//! ## A host is written by what changed in it
27//!
28//! A record carries the whole host but only the ports that differ from what the file
29//! already holds. A host scanned on every port holds tens of thousands of them, and writing
30//! it whole on every change would be megabytes per checkpoint. Folding makes this safe: a
31//! port's later record merges over its earlier one, and a port a record leaves out keeps
32//! what the file says. What the file holds of each port is remembered as a digest, so
33//! finding what changed costs no copy of the host.
34//!
35//! ## A sitting is written down before it ends
36//!
37//! A sitting's phases are appended to `phases.jsonl` when it stops, and a sitting killed
38//! outright never stops. So each sitting also keeps its own file, named for when it started,
39//! holding its phases as they stand: those closed and the open one so far. It is rewritten
40//! whole at every checkpoint and removed once the phases are appended. One still present
41//! when the journal is read is a sitting that never ended, and its phases are read with the
42//! rest. A phase that never closed reports what it opened with, how long it ran and what
43//! failed, claims nothing only its close could establish, and is marked open. A phase found
44//! in both files, left by a sitting stopped between the append and the removal, is read
45//! once.
46//!
47//! [`Journal::create`] begins one, [`Journal::resume`] continues one, and [`list`]
48//! enumerates them for a caller offering a choice.
49//!
50//! A journal holds the addresses an engagement was pointed at, so everything is `0600`
51//! under a `0700` directory, and a scan that runs elevated leaves it owned by the user who
52//! invoked it. See [`paths`](super::paths).
53
54use std::collections::HashSet;
55use std::fs;
56use std::path::{Path, PathBuf};
57use std::time::{Duration, SystemTime};
58
59use super::cursor::Checkpoint;
60use super::file::{
61    append_existing, claim_directory_for_invoking_user, create_private as create_private_file,
62    create_private_directory, exists, kinds, names, open_existing, open_to_read,
63    remove as remove_file, remove_directory, replace,
64};
65use super::format::JournalError;
66use super::lock::{Lock, LockRefused, LockState};
67use super::manifest::{JobOptions, JournalManifest, Plan, PlanChanged};
68use super::ownership::Kind;
69use super::paths::{ID_ALPHABET, ID_CHARS};
70use super::settle::Settlements;
71use crate::detect::compute::{DetectionLine, DetectionRunRecord, PortRunsRecord};
72use crate::model::host::Host;
73use crate::model::ip::scoped::ScopedIp;
74use crate::model::port::Protocol;
75use crate::record::{HostRecord, PhaseRecord};
76use crate::report::{ScanKind, ScanPhase, ScanReport, Unheard};
77use crate::system::privilege::Privilege;
78
79const MANIFEST: &str = "manifest.json";
80const CURSOR: &str = "cursor.json";
81const HOSTS: &str = "hosts.jsonl";
82const PHASES: &str = "phases.jsonl";
83const DETECTIONS: &str = "detections.jsonl";
84/// The hosts each sitting that ran to its end had run every pass over. See
85/// [`Journal::record_finished`].
86const FINISHED: &str = "finished.jsonl";
87const LOCK: &str = "LOCK";
88/// Prefix of a sitting's own phases file, followed by when it started.
89const SITTING: &str = "sitting-";
90/// What the job runs under, written once, when its first sitting starts. See
91/// [`Journal::options`].
92const OPTIONS: &str = "options.json";
93
94/// Why a journal could not be opened for writing.
95#[non_exhaustive]
96#[derive(Debug, thiserror::Error)]
97pub enum OpenError {
98    /// The plan offered is not the plan this journal's positions are counted in.
99    #[error("{0}")]
100    PlanChanged(#[from] PlanChanged),
101
102    /// The journal records the other phase of a scan.
103    ///
104    /// A sweep's positions count addresses and a port scan's count address-and-port
105    /// pairs, so one continued as the other would skip targets nothing probed.
106    #[error("this journal records a {held} and cannot be continued as a {asked}")]
107    WrongPhase {
108        /// The phase the journal holds.
109        held: &'static str,
110        /// The phase it was asked to continue as.
111        asked: &'static str,
112    },
113
114    /// Somebody else holds it, or might.
115    #[error("{0}")]
116    Locked(LockRefused),
117
118    /// The journal could not be read or written.
119    #[error("{0}")]
120    Journal(#[from] JournalError),
121}
122
123impl From<LockRefused> for OpenError {
124    fn from(refused: LockRefused) -> Self {
125        OpenError::Locked(refused)
126    }
127}
128
129/// A journal open for writing.
130///
131/// Releases its lock on drop.
132#[derive(Debug)]
133pub struct Journal {
134    directory: PathBuf,
135    manifest: JournalManifest,
136    lock: Lock,
137    resume_point: Checkpoint,
138    restored: Vec<Host>,
139    earlier: Vec<ScanPhase>,
140    /// Whether this handle made the journal or reopened one. See
141    /// [`withdraw`](Journal::withdraw).
142    created: bool,
143    options: Option<JobOptions>,
144    /// What the findings file holds of each host, and how much of it later records
145    /// superseded. See [`record_hosts`](Journal::record_hosts).
146    written: Written,
147    /// How long the findings file is.
148    length: u64,
149    /// How much has to be superseded before a compaction is tried again after one
150    /// failed. See [`compact`](Journal::compact).
151    compact_after: u64,
152    /// Where this sitting writes its phases as they stand. See the module
153    /// documentation.
154    sitting: PathBuf,
155}
156
157impl Journal {
158    /// Begins a journal for `plan` under `root`, minting an id for it.
159    ///
160    /// The plan carries its phase, so a sweep and a port scan both come through here and
161    /// neither can be read back as the other.
162    pub fn create(
163        root: &Path,
164        plan: &Plan,
165        privilege: Privilege,
166        summary: impl Into<String>,
167    ) -> Result<Self, OpenError> {
168        let (id, directory) = claim_directory(root)?;
169        let manifest = JournalManifest::new(id, plan, privilege, summary);
170
171        // A scan that never started leaves no trace, whichever step failed, so every
172        // failure past the claim is caught here. A leftover directory would list as a scan that
173        // found nothing, or not list at all.
174        match Self::furnish(&directory, manifest) {
175            Ok(journal) => Ok(journal),
176            Err(error) => {
177                let _ = remove_directory(&directory);
178                Err(error)
179            }
180        }
181    }
182
183    /// Everything [`create`](Self::create) does once the directory is claimed.
184    ///
185    /// Split out so a failure past the claim is caught in one place.
186    fn furnish(directory: &Path, manifest: JournalManifest) -> Result<Self, OpenError> {
187        write_private(
188            &directory.join(MANIFEST),
189            &serde_json::to_vec(&manifest).map_err(JournalError::json)?,
190        )?;
191
192        let lock = Lock::acquire(&directory.join(LOCK))?;
193        let sitting = sitting_file(directory, &lock);
194
195        let mut journal = Self {
196            directory: directory.to_path_buf(),
197            manifest,
198            lock,
199            resume_point: Checkpoint::default(),
200            restored: Vec::new(),
201            earlier: Vec::new(),
202            created: true,
203            options: None,
204            written: Written::default(),
205            length: 0,
206            compact_after: 0,
207            sitting,
208        };
209        journal.open_findings()?;
210        journal.length = findings_length(directory)?;
211        Ok(journal)
212    }
213
214    /// Continues the journal at `directory`, refusing a plan that has moved or a
215    /// scan somebody else is running.
216    ///
217    /// Returns the checkpoint to subtract from the plan; feed it to
218    /// [`Checkpoint::remaining`] and to [`Settlements::resuming`].
219    pub fn resume(
220        directory: &Path,
221        plan: &Plan,
222        privilege: Privilege,
223    ) -> Result<(Self, Checkpoint), OpenError> {
224        Self::resume_locking(directory, plan, privilege, Lock::acquire)
225    }
226
227    /// [`resume`](Self::resume), taking the lock by `lock`.
228    fn resume_locking(
229        directory: &Path,
230        plan: &Plan,
231        privilege: Privilege,
232        lock: fn(&Path) -> Result<Lock, LockRefused>,
233    ) -> Result<(Self, Checkpoint), OpenError> {
234        let manifest = read_manifest(directory)?;
235        // The phase before the fingerprint, so continuing a sweep as a port scan is named
236        // as such and not reported as a moved plan.
237        if manifest.kind() != plan.kind() {
238            return Err(OpenError::WrongPhase {
239                held: phase_name(manifest.kind()),
240                asked: phase_name(plan.kind()),
241            });
242        }
243        // The plan next, before taking the lock, so a mistaken resume disturbs nothing.
244        manifest.covers(plan, privilege)?;
245
246        let lock = lock(&directory.join(LOCK))?;
247        let sitting = sitting_file(directory, &lock);
248        let checkpoint = read_checkpoint(directory)?;
249        let earlier = read_phases(directory)?;
250        // Drop the records earlier sittings heard nothing from, as a report of the job
251        // does, except those a sitting killed before its verdicts left for this one to decide;
252        // see `Unheard::decided`.
253        let unheard = Unheard::decided(&earlier);
254        let mut restored = read_findings(directory)?;
255        restored.retain(|host| !unheard.drops(host));
256        // What the file already holds, so a restored host this sitting changes is written
257        // by its changes. Anything in the file beyond what those fold to is superseded records. An
258        // empty file left by an earlier sitting counts as no length.
259        let length = findings_length(directory)?;
260        let written = Written::holding(&restored, length)?;
261
262        Ok((
263            Self {
264                directory: directory.to_path_buf(),
265                manifest,
266                lock,
267                resume_point: checkpoint.clone(),
268                restored,
269                earlier,
270                created: false,
271                options: read_options(directory)?,
272                written,
273                length,
274                compact_after: 0,
275                sitting,
276            },
277            checkpoint,
278        ))
279    }
280
281    /// Continues the journal at `directory`, scanning the plan it recorded.
282    ///
283    /// The counterpart of [`resume`](Self::resume) for a caller who has only the
284    /// scan's id. The plan comes back as recorded, phase included, so a hostname that has moved
285    /// does not change what is continued, and a caller can see which phase it has before
286    /// starting.
287    ///
288    /// Refused if this process holds different privileges than the scan did: the connect
289    /// fallback asks a different question than a raw technique.
290    pub fn reopen(
291        directory: &Path,
292        privilege: Privilege,
293    ) -> Result<(Self, Checkpoint, Plan), OpenError> {
294        let plan = read_manifest(directory)?.recorded();
295        let (journal, checkpoint) = Self::resume(directory, &plan, privilege)?;
296        Ok((journal, checkpoint, plan))
297    }
298
299    /// [`reopen`](Self::reopen), taking over a lock whose holder has stopped
300    /// checkpointing.
301    ///
302    /// For the refusal [`LockState::Stale`] names: a live process holds the recorded
303    /// number, and nothing here can tell whether it is the hung scan or an unrelated process the
304    /// number was reissued to. A caller who knows can continue the job here without removing
305    /// the lock by hand. A lock whose holder is checkpointing now is still refused. See
306    /// [`Lock::take_over`].
307    pub fn take_over(
308        directory: &Path,
309        privilege: Privilege,
310    ) -> Result<(Self, Checkpoint, Plan), OpenError> {
311        let plan = read_manifest(directory)?.recorded();
312        let (journal, checkpoint) =
313            Self::resume_locking(directory, &plan, privilege, Lock::take_over)?;
314        Ok((journal, checkpoint, plan))
315    }
316
317    /// What earlier sittings of this scan found.
318    ///
319    /// Empty for a journal just created. A scan seeds its store with these, so its
320    /// report describes the whole job.
321    ///
322    /// A record of an address an earlier sitting went on to hear nothing from is left out, as
323    /// the job's report leaves it out. One that a sitting killed before its port phase's
324    /// verdicts had yet to decide is included (the report leaves it out too): this sitting asks
325    /// what is left at the address and decides it.
326    pub fn restored(&self) -> &[Host] {
327        &self.restored
328    }
329
330    /// Appends what `hosts` currently hold that the file does not.
331    ///
332    /// Called with what
333    /// [`take_changed_hosts`](crate::scanner::session::ScanContext::take_changed_hosts) yields,
334    /// so a host is written once per change. Each record carries the host's own fields and only
335    /// the ports that changed since the file last held them; see the module documentation. A
336    /// host may carry only some of its ports, and those it leaves out stand as the file holds
337    /// them. A host the file already holds unchanged gets no record.
338    ///
339    /// What was written is remembered only after the write succeeds, so a failed write leaves
340    /// every port it carried to be written again.
341    pub fn record_hosts(&mut self, hosts: &[Host]) -> Result<(), JournalError> {
342        let mut records = Vec::with_capacity(hosts.len());
343        for host in hosts {
344            records.extend(self.written.delta(host)?);
345        }
346        if records.is_empty() {
347            return Ok(());
348        }
349
350        let file = open_for_append(&self.directory.join(HOSTS))?;
351        // Measured on the handle, as a compaction measures what it wrote.
352        let measured = file.try_clone()?;
353
354        let mut deltas = Vec::with_capacity(records.len());
355        let mut writer = crate::journal::format::Writer::append(std::io::BufWriter::new(file));
356        for (record, delta) in records {
357            writer.write(&record)?;
358            deltas.push(delta);
359        }
360        writer.flush()?;
361        drop(writer);
362
363        for delta in deltas {
364            self.written.update(delta);
365        }
366        self.length = measured.metadata()?.len();
367        Ok(())
368    }
369
370    /// Appends the tapes of detection runs, each recording what one detection read from
371    /// its capabilities so the run can be replayed offline.
372    ///
373    /// One port per line, the responses the port's runs read held once for all of them; see
374    /// `PortRunsRecord`. [`read_detections`] hands each run back whole.
375    ///
376    /// Its own file, created on the first run. The resume path never reads it: a tape is
377    /// evidence for later analysis, not a settled verdict.
378    ///
379    /// All or nothing: a write that fails part way is cut back to where it began, so a caller
380    /// that retries the same runs records each once.
381    pub fn record_detections(&mut self, runs: &[DetectionRunRecord]) -> Result<(), JournalError> {
382        if runs.is_empty() {
383            return Ok(());
384        }
385
386        // Append first and create only on `NotFound`. `create_private` refuses an existing
387        // name, so losing a race between two writers is reported, not a truncation.
388        let path = self.directory.join(DETECTIONS);
389        let (file, mut writer) = match open_for_append(&path) {
390            Ok(file) => {
391                let written = file.try_clone()?;
392                let writer =
393                    crate::journal::format::Writer::append(std::io::BufWriter::new(written));
394                (file, writer)
395            }
396            Err(JournalError::Io(error)) if error.kind() == std::io::ErrorKind::NotFound => {
397                let file = create_private_file(&path)?;
398                let written = file.try_clone()?;
399                let writer =
400                    crate::journal::format::Writer::create(std::io::BufWriter::new(written))?;
401                (file, writer)
402            }
403            Err(error) => return Err(error),
404        };
405        // Where this write begins: past the header a new file was just given, which a
406        // cut-back write keeps.
407        writer.flush()?;
408        let began = file.metadata()?.len();
409        let outcome = PortRunsRecord::grouping(runs)
410            .iter()
411            .try_for_each(|line| writer.write(line))
412            .and_then(|()| writer.flush());
413        if outcome.is_err() {
414            // Dropped first, since dropping flushes the buffer, then everything past the start
415            // is cut. A failed cut leaves a torn tail, which the next append mends.
416            drop(writer);
417            let _ = file.set_len(began);
418        }
419        outcome
420    }
421
422    /// Appends that a sitting ran every pass that follows its probes over
423    /// `hosts`, each named as its address and link are written.
424    ///
425    /// For a sitting that ran to its end, whose passes each finished over every host it
426    /// held. A later sitting reads these through [`finished_hosts`](Self::finished_hosts) and
427    /// reruns those passes only over what it asks something new of. Without this, a job resumed
428    /// after it was done would redo service identification, detection, TLS and route tracing
429    /// for every host.
430    ///
431    /// Its own file, created on the first record, so a journal from an older engine reads as one
432    /// no sitting finished, and an older engine ignores it.
433    pub(crate) fn record_finished(
434        &mut self,
435        hosts: impl IntoIterator<Item = String>,
436    ) -> Result<(), JournalError> {
437        let record = FinishedRecord {
438            hosts: hosts.into_iter().collect(),
439        };
440        if record.hosts.is_empty() {
441            return Ok(());
442        }
443
444        // Created on `NotFound`, as in `record_detections`.
445        let path = self.directory.join(FINISHED);
446        let mut writer = match open_for_append(&path) {
447            Ok(file) => crate::journal::format::Writer::append(std::io::BufWriter::new(file)),
448            Err(JournalError::Io(error)) if error.kind() == std::io::ErrorKind::NotFound => {
449                let file = create_private_file(&path)?;
450                crate::journal::format::Writer::create(std::io::BufWriter::new(file))?
451            }
452            Err(error) => return Err(error),
453        };
454        writer.write(&record)?;
455        writer.flush()
456    }
457
458    /// Every host an earlier sitting that ran to its end had finished every pass over,
459    /// named as [`record_finished`](Self::record_finished) wrote it.
460    ///
461    /// Empty for a journal no sitting finished or one without the record, which a resume reads
462    /// as owing every pass to every host.
463    pub(crate) fn finished_hosts(&self) -> Result<HashSet<String>, JournalError> {
464        let file = match open_to_read(&self.directory.join(FINISHED)) {
465            Ok(file) => file,
466            Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(HashSet::new()),
467            Err(e) => return Err(e.into()),
468        };
469        let mut reader = match crate::journal::format::Reader::open(std::io::BufReader::new(file)) {
470            Ok(reader) => reader,
471            Err(JournalError::NotAJournal) => return Ok(HashSet::new()),
472            Err(e) => return Err(e),
473        };
474
475        let mut hosts = HashSet::new();
476        while let Some(record) = reader.read::<FinishedRecord>()? {
477            hosts.extend(record.hosts);
478        }
479        Ok(hosts)
480    }
481
482    /// Whether the findings file holds enough superseded records to be worth rewriting.
483    ///
484    /// The dispatcher shuffles targets across the whole plan, so on a long scan most hosts
485    /// change in most intervals. Ports are written only as they change, but each record repeats
486    /// the host's own fields, so the file grows with the scan's duration. Compaction bounds it.
487    ///
488    /// Counted in bytes of superseded records, since that is what a rewrite costs and recovers;
489    /// a file growing by new findings holds nothing a rewrite would drop. The file is rewritten
490    /// once the superseded part outgrows the rest, so it stays under twice its live size and
491    /// each compaction is paid for by at least as much superseded since the last. Below a floor
492    /// it is never worth rewriting.
493    pub fn should_compact(&self) -> bool {
494        outgrown(self.length, self.written.superseded)
495            && self.written.superseded >= self.compact_after
496    }
497
498    /// Writes the findings file whole, replacing everything superseded.
499    ///
500    /// `all` must be every host the scan has found, since this replaces the file.
501    /// Written to a sibling and renamed over, so an interrupted compaction leaves the previous
502    /// file untouched.
503    ///
504    /// A failed compaction removes its sibling, and [`should_compact`](Self::should_compact)
505    /// does not ask again until as much again has been superseded. The usual cause, a disk with
506    /// no room for a second copy, would otherwise recur every checkpoint, rewriting the live
507    /// findings every few seconds into whatever room was left.
508    pub fn compact(&mut self, all: &[Host]) -> Result<(), JournalError> {
509        let compacted = self.write_whole(all);
510        if compacted.is_err() {
511            let superseded = self.written.superseded;
512            let live = self.length.saturating_sub(superseded);
513            self.compact_after = superseded.saturating_add(COMPACT_FLOOR.max(live));
514        }
515        compacted
516    }
517
518    /// [`compact`](Self::compact)'s writing, through a staged sibling that a
519    /// failure removes.
520    fn write_whole(&mut self, all: &[Host]) -> Result<(), JournalError> {
521        let destination = self.directory.join(HOSTS);
522        let (written, length) = replace(
523            &destination,
524            &destination.with_extension("jsonl-tmp"),
525            |file| {
526                // Measured on the handle, so the length is the written file's and not whatever
527                // the name holds by then.
528                let measured = file.try_clone()?;
529                let mut written = Written::default();
530                let mut writer =
531                    crate::journal::format::Writer::create(std::io::BufWriter::new(file))?;
532                for host in all {
533                    // Measured against nothing written, so the record is the whole host.
534                    if let Some((record, delta)) = Written::default().delta(host)? {
535                        writer.write(&record)?;
536                        written.update(delta);
537                    }
538                }
539                writer.flush()?;
540                drop(writer);
541                Ok::<_, JournalError>((written, measured.metadata()?.len()))
542            },
543        )?;
544        self.written = written;
545        self.length = length;
546        self.compact_after = 0;
547        Ok(())
548    }
549
550    /// What earlier sittings of this scan did.
551    ///
552    /// A resumed report carries these alongside its own, so it describes a job that ran
553    /// in several sittings.
554    pub fn earlier_phases(&self) -> &[ScanPhase] {
555        &self.earlier
556    }
557
558    /// Appends what one sitting did, once it has finished doing it.
559    pub fn record_phases(&mut self, phases: &[ScanPhase]) -> Result<(), JournalError> {
560        if phases.is_empty() {
561            return Ok(());
562        }
563
564        let file = open_for_append(&self.directory.join(PHASES))?;
565
566        let mut writer = crate::journal::format::Writer::append(std::io::BufWriter::new(file));
567        for phase in phases {
568            writer.write(&PhaseRecord::from(phase))?;
569        }
570        writer.flush()
571    }
572
573    /// Writes down this sitting's phases as they stand, over what it wrote last. See
574    /// the module documentation.
575    ///
576    /// Written to a sibling and renamed over, as the cursor is, so a sitting killed part way
577    /// leaves the previous record whole.
578    pub(crate) fn record_standing(&mut self, phases: &[ScanPhase]) -> Result<(), JournalError> {
579        if phases.is_empty() {
580            return Ok(());
581        }
582        replace(
583            &self.sitting,
584            &self.sitting.with_extension("jsonl-tmp"),
585            |file| {
586                let mut writer =
587                    crate::journal::format::Writer::create(std::io::BufWriter::new(file))?;
588                for phase in phases {
589                    writer.write(&PhaseRecord::from(phase))?;
590                }
591                writer.flush()
592            },
593        )
594    }
595
596    /// Appends what one sitting did once it has finished, and removes its standing
597    /// record of its phases.
598    ///
599    /// The removal waits for the append to land, since until then the standing record is the
600    /// only one.
601    pub(crate) fn end_sitting(&mut self, phases: &[ScanPhase]) -> Result<(), JournalError> {
602        if phases.is_empty() {
603            return Ok(());
604        }
605        self.record_phases(phases)?;
606        match remove_file(&self.sitting) {
607            Ok(()) => Ok(()),
608            Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),
609            Err(e) => Err(e.into()),
610        }
611    }
612
613    /// Writes the appended files' headers, so each is self-describing before
614    /// anything is added to it.
615    fn open_findings(&mut self) -> Result<(), JournalError> {
616        for name in [HOSTS, PHASES] {
617            let path = self.directory.join(name);
618            let file = create_private_file(&path)?;
619            crate::journal::format::Writer::create(std::io::BufWriter::new(file))?.flush()?;
620        }
621        Ok(())
622    }
623
624    /// What an earlier sitting settled, and this one may skip.
625    ///
626    /// Empty for a journal just created. A scan seeds both its dispatcher and its
627    /// settlements from this, so the second sitting's cursor continues the first's.
628    pub fn resume_point(&self) -> &Checkpoint {
629        &self.resume_point
630    }
631
632    /// Writes how far the scan has got, and reports the writer is alive.
633    ///
634    /// Cheap enough to call on a timer: the cursor is small however large the scan,
635    /// and the write is a rename over a small file. See [`Checkpoint::write_atomically`].
636    pub fn checkpoint(&mut self, settlements: &Settlements) -> Result<(), JournalError> {
637        self.write_cursor(&settlements.checkpoint())
638    }
639
640    /// Writes `cursor` as how far the scan has got, and reports the writer is alive.
641    ///
642    /// For a writer that read the cursor before taking the findings it writes beside it, the
643    /// order a running scan needs; see [`checkpoint`](crate::scanner::checkpoint).
644    pub(crate) fn write_cursor(&mut self, cursor: &Checkpoint) -> Result<(), JournalError> {
645        cursor.write_atomically(&self.directory.join(CURSOR))?;
646        self.lock.beat()
647    }
648
649    /// Writes down what the scan has found and how far it has got.
650    ///
651    /// Findings first: a cursor claiming a target settled beside a file missing its
652    /// finding loses that finding, while the other way round only probes a target twice.
653    ///
654    /// The cursor is read here, after `hosts` were taken, which suits a scan that has stopped
655    /// settling targets. While one is running a target can settle between the two readings
656    /// with its finding in neither, so the scanner's own checkpoints read the cursor first.
657    pub fn record(
658        &mut self,
659        hosts: &[Host],
660        settlements: &Settlements,
661    ) -> Result<(), JournalError> {
662        self.record_hosts(hosts)?;
663        self.checkpoint(settlements)
664    }
665
666    /// The options the job runs under, or `None` where they were never recorded.
667    ///
668    /// Recorded when the journal's first sitting starts, from its configuration, and never
669    /// rewritten. A later sitting is held to them, and a caller continuing a job by its id
670    /// restores them with [`JobOptions::apply_to`]; the type says which options those are.
671    ///
672    /// `None` for a journal whose first sitting ran on an engine that did not record them
673    /// (continued under whatever the caller passes), for one no sitting has started, and for a
674    /// watch, which has nothing for them to decide.
675    pub fn options(&self) -> Option<&JobOptions> {
676        self.options.as_ref()
677    }
678
679    /// Records the options the job runs under, if this is its first sitting.
680    ///
681    /// A journal an earlier sitting ran against without recording options ran under unknown
682    /// ones, and recording this sitting's would misdescribe it.
683    ///
684    /// Every sitting numbers the plan without the excluded ports, so the manifest's total,
685    /// written before they were known, is recounted here. Counted in, they would be a remainder
686    /// nothing reaches: a finished job would stay unfinished and a resume would announce probes
687    /// it will not send. Only rewritten in a manifest this handle wrote, since rewriting another
688    /// build's would drop fields this one does not know.
689    pub(crate) fn record_options(&mut self, options: JobOptions) -> Result<(), JournalError> {
690        if self.options.is_some() || !self.is_untouched() {
691            return Ok(());
692        }
693
694        write_private(
695            &self.directory.join(OPTIONS),
696            &serde_json::to_vec(&options).map_err(JournalError::json)?,
697        )?;
698        let numbered = self
699            .manifest
700            .recorded()
701            .numbered_targets(&options.excluded_ports());
702        self.options = Some(options);
703        if self.created && numbered != self.manifest.total_targets {
704            self.manifest.total_targets = numbered;
705            write_private(
706                &self.directory.join(MANIFEST),
707                &serde_json::to_vec(&self.manifest).map_err(JournalError::json)?,
708            )?;
709        }
710        Ok(())
711    }
712
713    /// Whether no sitting has run against this journal: no cursor written, no phase
714    /// and no finding recorded.
715    ///
716    /// A cursor that cannot be looked for counts as present. This decides whether to write the
717    /// job's options and whether to remove the journal, and wrongly calling it touched costs
718    /// only the options record, while wrongly calling it untouched could lose a sitting's
719    /// work.
720    fn is_untouched(&self) -> bool {
721        self.earlier.is_empty()
722            && self.restored.is_empty()
723            && !exists(&self.directory.join(CURSOR)).unwrap_or(true)
724    }
725
726    /// What this journal is a journal of.
727    pub fn manifest(&self) -> &JournalManifest {
728        &self.manifest
729    }
730
731    /// Where it lives.
732    pub fn directory(&self) -> &Path {
733        &self.directory
734    }
735
736    /// Gives up a journal handed to a scan that refused before it started, and removes
737    /// it if no sitting ever ran against it.
738    ///
739    /// A scan that never started leaves no trace, as in [`create`](Self::create). Kept, the
740    /// record would list as a resumable job with nothing done and count against a front end's
741    /// record limit, and the caller cannot tidy it since the journal was handed over by value.
742    ///
743    /// Only a journal this handle made and no sitting has touched (no cursor, phase or finding)
744    /// is removed. One an earlier sitting ran against holds its work, and one reopened was kept
745    /// by whoever made it; either is released for the next sitting.
746    ///
747    /// Removed while the lock is held, so nothing takes the journal up in between. The lock
748    /// file goes with the directory, and the following drop tolerates finding nothing.
749    pub(crate) fn withdraw(self) {
750        if self.created && self.is_untouched() {
751            // Best effort, as in `create`.
752            let _ = remove_directory(&self.directory);
753        }
754    }
755
756    /// Releases the lock, reporting a failure the drop would swallow.
757    pub fn close(self) -> Result<(), JournalError> {
758        self.lock.release()
759    }
760}
761
762/// Whether a findings file `length` bytes long, `superseded` of them superseded, is
763/// due to be rewritten. See [`Journal::should_compact`].
764fn outgrown(length: u64, superseded: u64) -> bool {
765    superseded > COMPACT_FLOOR.max(length.saturating_sub(superseded))
766}
767
768/// The size below which a findings file is not worth rewriting.
769const COMPACT_FLOOR: u64 = 4 * 1024 * 1024;
770
771/// What a journal's findings file holds of each host, and how many of its
772/// bytes later records superseded.
773///
774/// Each port is remembered as a digest of the record last written for it, a few
775/// bytes a port. Kept in memory only, so the hasher need not be stable across builds.
776///
777/// Sizes are of each piece as serialised, within a separator of the line; they feed a
778/// threshold, not an account.
779#[derive(Debug, Default)]
780struct Written {
781    hosts: std::collections::HashMap<ScopedIp, HeldHost>,
782    /// How many of the file's bytes hold what a later record superseded.
783    superseded: u64,
784}
785
786/// What the findings file holds of one host.
787#[derive(Debug, Default)]
788struct HeldHost {
789    /// Its last record, less its ports.
790    rest: Option<Mark>,
791    ports: std::collections::HashMap<PortKey, Mark>,
792}
793
794/// A port as a host keys it: its number and its transport.
795type PortKey = (u16, Protocol);
796
797/// A record as written: a digest of it, and how long it was.
798#[derive(Debug, Clone, Copy, PartialEq, Eq)]
799struct Mark {
800    digest: u64,
801    length: u32,
802}
803
804/// What one record wrote of one host, to remember once the write succeeds.
805#[derive(Debug)]
806struct Delta {
807    key: ScopedIp,
808    rest: Mark,
809    ports: Vec<(PortKey, Mark)>,
810}
811
812impl Delta {
813    /// How many bytes the record holds, near enough.
814    fn length(&self) -> u64 {
815        u64::from(self.rest.length)
816            + self
817                .ports
818                .iter()
819                .map(|(_, mark)| u64::from(mark.length))
820                .sum::<u64>()
821    }
822}
823
824impl Written {
825    /// What a file `length` bytes long holds, where `hosts` are what it folds to: those
826    /// whole, and the rest superseded.
827    fn holding(hosts: &[Host], length: u64) -> Result<Self, JournalError> {
828        let mut written = Self::default();
829        let mut live = 0;
830        for host in hosts {
831            // Against nothing written, the record is the whole host.
832            if let Some((_, delta)) = Self::default().delta(host)? {
833                live += delta.length();
834                written.update(delta);
835            }
836        }
837        written.superseded = length.saturating_sub(live);
838        Ok(written)
839    }
840
841    /// `host` as a record carrying only the ports whose record differs from what was last
842    /// written of them, with what writing it would hold; `None` where the file already holds
843    /// all of it.
844    ///
845    /// Compared, not trusted: a host is marked changed by whatever edited it, and an edit that
846    /// confirmed what was on record (a finding reached again, a service named as before)
847    /// changes nothing a record would carry.
848    fn delta(&self, host: &Host) -> Result<Option<(HostRecord, Delta)>, JournalError> {
849        let held = self.hosts.get(&host.scoped_ip());
850        let mut record = HostRecord::from(host);
851        // `HostRecord` lists the ports in the order the host yields them, so the two zip.
852        let ports = std::mem::take(&mut record.ports);
853        let rest = mark(&record)?;
854        let mut changed = Vec::new();
855        for (port, written) in host.ports().zip(ports) {
856            debug_assert_eq!(
857                port.number(),
858                written.port,
859                "a record lists its host's ports"
860            );
861            let key = (port.number(), port.protocol());
862            let mark = mark(&written)?;
863            let unchanged = held
864                .and_then(|held| held.ports.get(&key))
865                .is_some_and(|was| was.digest == mark.digest);
866            if !unchanged {
867                changed.push((key, mark));
868                record.ports.push(written);
869            }
870        }
871        let unchanged = changed.is_empty()
872            && held.and_then(|held| held.rest).map(|was| was.digest) == Some(rest.digest);
873        if unchanged {
874            return Ok(None);
875        }
876        let delta = Delta {
877            key: host.scoped_ip(),
878            rest,
879            ports: changed,
880        };
881        Ok(Some((record, delta)))
882    }
883
884    /// Records that the file holds what `delta` wrote over that host, and counts what it
885    /// superseded.
886    fn update(&mut self, delta: Delta) {
887        let held = match self.hosts.entry(delta.key) {
888            std::collections::hash_map::Entry::Occupied(slot) => {
889                let held = slot.into_mut();
890                self.superseded += held.rest.map_or(0, |was| u64::from(was.length));
891                held
892            }
893            std::collections::hash_map::Entry::Vacant(slot) => slot.insert(HeldHost::default()),
894        };
895        held.rest = Some(delta.rest);
896        for (key, mark) in delta.ports {
897            if let Some(was) = held.ports.insert(key, mark) {
898                self.superseded += u64::from(was.length);
899            }
900        }
901    }
902}
903
904/// A digest and length of `record` as serialised, the same bytes the file gets, fed
905/// to a hasher.
906fn mark(record: &impl serde::Serialize) -> Result<Mark, JournalError> {
907    use std::hash::Hasher;
908
909    struct Hashing {
910        hasher: std::collections::hash_map::DefaultHasher,
911        length: u64,
912    }
913    impl std::io::Write for Hashing {
914        fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
915            self.hasher.write(bytes);
916            self.length += bytes.len() as u64;
917            Ok(bytes.len())
918        }
919        fn flush(&mut self) -> std::io::Result<()> {
920            Ok(())
921        }
922    }
923
924    let mut hashing = Hashing {
925        hasher: std::collections::hash_map::DefaultHasher::new(),
926        length: 0,
927    };
928    serde_json::to_writer(&mut hashing, record).map_err(JournalError::json)?;
929    Ok(Mark {
930        digest: hashing.hasher.finish(),
931        // Plus a separator or newline; saturated past four gigabytes, beyond what a
932        // journal line is read in.
933        length: u32::try_from(hashing.length + 1).unwrap_or(u32::MAX),
934    })
935}
936
937/// A journal as it appears to a caller choosing between them. Read without taking
938/// the lock, so listing never disturbs a running scan.
939#[non_exhaustive]
940#[derive(Debug, Clone)]
941pub struct Entry {
942    /// Where it lives.
943    pub directory: PathBuf,
944    /// What it is a journal of.
945    pub manifest: JournalManifest,
946    /// How far it got, or `None` where that could not be read.
947    ///
948    /// A journal that never checkpointed carries a fresh cursor. `None` means the file
949    /// is there and this process cannot read it, usually because the scan ran under `sudo`; it
950    /// must not read as no progress, or a listing would offer to continue finished work.
951    pub checkpoint: Option<Checkpoint>,
952    /// Whether anything is writing it.
953    pub lock: LockState,
954}
955
956impl Entry {
957    /// A listing entry from its parts, for a caller rendering one it did not read off
958    /// disk.
959    pub fn new(
960        directory: PathBuf,
961        manifest: JournalManifest,
962        checkpoint: Option<Checkpoint>,
963        lock: LockState,
964    ) -> Self {
965        Self {
966            directory,
967            manifest,
968            checkpoint,
969            lock,
970        }
971    }
972
973    /// Whether this journal has anything left to do.
974    ///
975    /// A journal whose cursor covers the whole plan is finished; one that never
976    /// checkpointed has everything left. One whose cursor cannot be read is not finished, which
977    /// keeps a retention sweep from deleting it.
978    pub fn is_complete(&self) -> bool {
979        // A watch is never finished: its total is zero, so by the arithmetic below it would
980        // be complete when created. Another sitting can always be appended.
981        if self.kind() == ScanKind::Listen {
982            return false;
983        }
984
985        self.settled()
986            .is_some_and(|settled| settled >= self.manifest.total_targets)
987    }
988
989    /// Which phase this journal records.
990    ///
991    /// A sweep and a port scan count in different units, so a caller reporting progress
992    /// needs to know which this is.
993    pub fn kind(&self) -> ScanKind {
994        self.manifest.kind()
995    }
996
997    /// How many targets are settled, or `None` where the cursor could not be
998    /// read.
999    ///
1000    /// [`Checkpoint::settled_count`], which counts a listed position a watermark has
1001    /// passed once, as [`Cursor::from_checkpoint`](super::cursor::Cursor::from_checkpoint)
1002    /// does. `Checkpoint::read` keeps such entries, so without that a damaged list would be
1003    /// double-counted here, and [`is_complete`](Self::is_complete) could tip to `true`, which is
1004    /// what a retention sweep deletes on.
1005    pub fn settled(&self) -> Option<u128> {
1006        self.checkpoint
1007            .as_ref()
1008            .map(|checkpoint| u128::from(checkpoint.settled_count()))
1009    }
1010}
1011
1012/// Creates the directory journals are kept in, and gives it to the user who
1013/// invoked an elevated run.
1014///
1015/// Call this before [`Journal::create`].
1016///
1017/// What it creates is `0700` on Unix, as the XDG base directory specification asks of a
1018/// directory it creates, so nobody else can list which scans there are, or add one to be
1019/// resumed. One already there keeps its mode, which may be somebody's choice.
1020///
1021/// Raw strategies need root, so the first run on a machine is usually under `sudo`, and
1022/// [`paths::root`](super::paths::root) resolves the invoking user's home. The two
1023/// directories above each scan's directory are then created by root, and claiming a scan's
1024/// directory does not give them away. Left to root, every later unprivileged run, such as a
1025/// listening phase, finds a directory it cannot write to and silently records nothing.
1026///
1027/// The two directories are claimed whether or not this call created them, so an existing
1028/// installation left to root is repaired too. Claiming an already correct directory is a
1029/// `chown` to its current owner.
1030///
1031/// Above those two, what this call created is given too: a first run with no
1032/// `~/.local/state` creates it and `~/.local`. One already there belongs to whoever made it,
1033/// unless root owns it, the sign of an elevated run that gave nothing back; see the
1034/// `ownership` module.
1035///
1036/// Nothing outside the invoking user's home is given: a state root kept through `sudo`
1037/// that points elsewhere stays root's.
1038///
1039/// Best effort: a directory that cannot be given away is not worth failing a scan over, and
1040/// an unprivileged run has nobody to give it to.
1041pub fn prepare_root(root: &Path) -> std::io::Result<()> {
1042    let own = super::paths::root().as_deref() == Some(root);
1043    #[cfg(unix)]
1044    let home = super::ownership::invoking().map(|user| user.home.as_path());
1045    #[cfg(not(unix))]
1046    let home = None;
1047    prepare_root_with(root, own, home, |path, hand| match hand {
1048        Hand::Give => claim_directory_for_invoking_user(path),
1049        Hand::Reclaim => super::ownership::reclaim(path),
1050    })
1051}
1052
1053/// What [`prepare_root`] does with one directory on the way to a root.
1054#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1055enum Hand {
1056    /// Gives it to the invoking user, whoever owns it now.
1057    Give,
1058    /// Gives it back only if root owns it.
1059    Reclaim,
1060}
1061
1062/// [`prepare_root`] with the claim passed in, so a test can see what would be given
1063/// without an elevated process.
1064///
1065/// `own` says `root` is this crate's own location, the one case where what lies above it
1066/// may have been created by this engine on the user's behalf. A location a caller named
1067/// says where to write, not what lies above it.
1068///
1069/// `home` is the invoking user's, `None` for a run on nobody else's behalf; the directories
1070/// between it and the root are the ones a repair looks at.
1071fn prepare_root_with(
1072    root: &Path,
1073    own: bool,
1074    home: Option<&Path>,
1075    mut claim: impl FnMut(&Path, Hand),
1076) -> std::io::Result<()> {
1077    let created = super::ownership::create_missing(root, Some(0o700))?;
1078
1079    if own && let Some(above) = root.parent() {
1080        // Outermost first, as they were created.
1081        let mut between: Vec<&Path> = above
1082            .ancestors()
1083            .skip(1)
1084            .take_while(|directory| {
1085                home.is_some_and(|home| *directory != home && directory.starts_with(home))
1086            })
1087            .collect();
1088        between.reverse();
1089        for directory in between {
1090            let hand = if created.iter().any(|made| made == directory) {
1091                Hand::Give
1092            } else {
1093                Hand::Reclaim
1094            };
1095            claim(directory, hand);
1096        }
1097        claim(above, Hand::Give);
1098    }
1099    claim(root, Hand::Give);
1100
1101    Ok(())
1102}
1103
1104/// Every journal under a root, and everything standing there that could not be
1105/// listed.
1106///
1107/// What was passed over is returned as data so the caller decides how to report it: a
1108/// listing a person reads wants one line for ninety journals a newer build wrote, and a
1109/// prune wants their directories.
1110#[non_exhaustive]
1111#[derive(Debug, Clone, Default)]
1112pub struct Listing {
1113    /// The journals this build can read, newest first.
1114    pub entries: Vec<Entry>,
1115    /// What stands in the root as a journal would but could not be read as one.
1116    pub passed_over: Vec<PassedOver>,
1117}
1118
1119/// Something in a root of journals that [`list`] could not list, and why.
1120#[non_exhaustive]
1121#[derive(Debug, Clone, PartialEq, Eq)]
1122pub struct PassedOver {
1123    /// Its name in the root, which for a journal is its id.
1124    pub name: String,
1125    /// Where it stands.
1126    pub directory: PathBuf,
1127    /// Why it could not be listed.
1128    pub why: Unlisted,
1129}
1130
1131impl PassedOver {
1132    /// A passed-over entry from its parts, for a caller rendering one it did not read
1133    /// off disk.
1134    pub fn new(name: impl Into<String>, directory: PathBuf, why: Unlisted) -> Self {
1135        Self {
1136            name: name.into(),
1137            directory,
1138            why,
1139        }
1140    }
1141}
1142
1143/// Why [`list`] passed something over.
1144///
1145/// A kind, so a caller can report it once for a whole root: a machine that ran a
1146/// newer build has every journal it wrote passed over for the same reason.
1147/// [`fmt::Display`](std::fmt::Display) gives the sentence for one.
1148#[non_exhaustive]
1149#[derive(Debug, Clone, PartialEq, Eq)]
1150pub enum Unlisted {
1151    /// A link, not a directory.
1152    ///
1153    /// Every journal is a directory this crate made, so a link is none, and [`prune`] never
1154    /// removes one: under `sudo` a removal that followed it would be root's.
1155    Link,
1156    /// A journal whose manifest names a format newer than this build's.
1157    NewerFormat {
1158        /// The format the manifest names.
1159        found: u32,
1160    },
1161    /// A manifest that could not be read or parsed, in the failure's words: a missing
1162    /// permission, a file cut short.
1163    Unreadable(String),
1164}
1165
1166impl std::fmt::Display for Unlisted {
1167    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1168        match self {
1169            Self::Link => f.write_str("a link, not a journal (not followed)"),
1170            Self::NewerFormat { found } => write!(
1171                f,
1172                "journal format {found} is newer than this build's ({})",
1173                super::JOURNAL_VERSION
1174            ),
1175            Self::Unreadable(reason) => f.write_str(reason),
1176        }
1177    }
1178}
1179
1180/// Every journal under `root`, newest first, and what was passed over.
1181///
1182/// One unreadable journal does not hide the rest: it is returned in
1183/// [`Listing::passed_over`] with the reason (a newer format, a link where a journal should
1184/// be, a read failure), which its owner has to act on. A directory holding no manifest is
1185/// skipped silently, as one a scan starting now has yet to write.
1186///
1187/// An empty listing for a root that does not exist yet.
1188pub fn list(root: &Path) -> Result<Listing, JournalError> {
1189    let held = match kinds(root) {
1190        Ok(held) => held,
1191        Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Listing::default()),
1192        Err(e) => return Err(e.into()),
1193    };
1194
1195    let mut listing = Listing::default();
1196    for (name, kind) in held {
1197        let directory = root.join(&name);
1198        let name = name.to_string_lossy().into_owned();
1199        match kind {
1200            Kind::Directory => {}
1201            Kind::Link => {
1202                listing.passed_over.push(PassedOver {
1203                    name,
1204                    directory,
1205                    why: Unlisted::Link,
1206                });
1207                continue;
1208            }
1209            Kind::Other => continue,
1210        }
1211        let manifest = match read_manifest(&directory) {
1212            Ok(manifest) => manifest,
1213            Err(JournalError::Io(e)) if e.kind() == std::io::ErrorKind::NotFound => continue,
1214            Err(e) => {
1215                let why = match e {
1216                    JournalError::VersionTooNew { found, .. } => Unlisted::NewerFormat { found },
1217                    // The system's words alone; whoever renders this says whose they are.
1218                    JournalError::Io(e) => Unlisted::Unreadable(e.to_string()),
1219                    other => Unlisted::Unreadable(other.to_string()),
1220                };
1221                listing.passed_over.push(PassedOver {
1222                    name,
1223                    directory,
1224                    why,
1225                });
1226                continue;
1227            }
1228        };
1229        listing.entries.push(Entry {
1230            // `Err` is a cursor that exists and could not be read, recorded as `None`. A
1231            // journal that never checkpointed gets a fresh cursor from `read_checkpoint`.
1232            checkpoint: read_checkpoint(&directory).ok(),
1233            lock: super::lock::inspect(&directory.join(LOCK)),
1234            manifest,
1235            directory,
1236        });
1237    }
1238
1239    listing
1240        .entries
1241        .sort_by_key(|entry| std::cmp::Reverse(entry.manifest.created_at));
1242    listing.passed_over.sort_by(|a, b| a.name.cmp(&b.name));
1243    Ok(listing)
1244}
1245
1246/// The scan at `directory`, as the report it would have produced.
1247///
1248/// A journal holds everything a report is made of, so a finished scan can be read
1249/// back as it was when it ended: the hosts, the phases in the order they ran, and the engine
1250/// version from the manifest, so a scan run by an older engine still says so.
1251///
1252/// Read without the lock, like [`list`], so it is safe on a running scan; the result is then
1253/// a checkpoint behind the live one.
1254///
1255/// A journal missing its findings or phases reads as a scan that recorded none, since a
1256/// sitting can end before its first checkpoint.
1257pub fn report(directory: &Path) -> Result<ScanReport, JournalError> {
1258    let manifest = read_manifest(directory)?;
1259
1260    Ok(ScanReport::recorded(
1261        manifest.engine_version,
1262        read_phases(directory)?,
1263        read_findings(directory)?,
1264    ))
1265}
1266
1267/// How a phase is named in a refusal, in prose, since this reaches a person.
1268fn phase_name(kind: ScanKind) -> &'static str {
1269    match kind {
1270        ScanKind::Discovery => "host-discovery sweep",
1271        ScanKind::PortScan => "port scan",
1272        // A watch's journal is resumed by appending a sitting, but it is still one this
1273        // build writes.
1274        ScanKind::Listen => "listening phase",
1275    }
1276}
1277
1278/// Deletes the journal at `directory`.
1279///
1280/// Refuses one a live scan is writing. A caller pruning by age should read
1281/// [`Entry::lock`] and skip what is held, so the sweep can report what it left alone.
1282///
1283/// The refusal is a check, not an exclusion: a scan that takes the lock between the
1284/// inspection and the removal has its journal deleted under it and keeps writing to
1285/// unlinked files. Both parties are the same user's processes and the window is one lock
1286/// against one removal; closing it would need the lock held across the removal.
1287pub fn remove(directory: &Path) -> Result<(), OpenError> {
1288    let state = super::lock::inspect(&directory.join(LOCK));
1289    if !state.is_resumable() {
1290        return Err(OpenError::Locked(LockRefused::Held(state)));
1291    }
1292
1293    remove_directory(directory).map_err(JournalError::from)?;
1294    Ok(())
1295}
1296
1297/// One line of the record [`Journal::record_finished`] keeps: the hosts one
1298/// sitting finished every pass over.
1299#[derive(Debug, serde::Serialize, serde::Deserialize)]
1300struct FinishedRecord {
1301    hosts: Vec<String>,
1302}
1303
1304/// Reads back what a journal's earlier sittings found.
1305///
1306/// Records are folded together, each as a later account of the host, so a host
1307/// written once when it answered and again when its ports were classified comes back whole;
1308/// see [`Host::merge_later_account`].
1309///
1310/// Two records are the same host when they share any address. Local discovery promotes a
1311/// host's primary address when a better one turns up (a link-local giving way to a global),
1312/// so the same machine can be written under two primaries.
1313///
1314/// An address is the one [`ScopedIp::scoped`] makes of it with the record's zone, so two
1315/// machines answering to `fe80::1` on two links stay two hosts. That is the identity the
1316/// report and a diff key hosts by. A record that names no zone joins only others that name
1317/// none.
1318///
1319/// A missing file is no findings, since a journal can be read before its first host is
1320/// written.
1321fn read_findings(directory: &Path) -> Result<Vec<Host>, JournalError> {
1322    let file = match open_to_read(&directory.join(HOSTS)) {
1323        Ok(file) => file,
1324        Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()),
1325        Err(e) => return Err(e.into()),
1326    };
1327
1328    let mut reader = match crate::journal::format::Reader::open(std::io::BufReader::new(file)) {
1329        Ok(reader) => reader,
1330        // No header means the file was never opened for writing: a journal that stopped
1331        // before it found anything.
1332        Err(JournalError::NotAJournal) => return Ok(Vec::new()),
1333        Err(e) => return Err(e),
1334    };
1335
1336    // Slots, not a map: a record can join two hosts that were separate until then, and
1337    // the one it merges into keeps its slot while the other empties.
1338    let mut hosts: Vec<Option<Host>> = Vec::new();
1339    let mut slot_of: std::collections::BTreeMap<ScopedIp, usize> =
1340        std::collections::BTreeMap::new();
1341
1342    while let Some(record) = reader.read::<HostRecord>()? {
1343        let host = Host::from(&record);
1344
1345        let mut matched: Vec<usize> = scoped_ips(&host)
1346            .filter_map(|ip| slot_of.get(&ip).copied())
1347            .collect();
1348        matched.sort_unstable();
1349        matched.dedup();
1350
1351        let slot = match matched.split_first() {
1352            Some((&keep, absorb)) => {
1353                // Every host sharing an address with this record is the same machine.
1354                for &other in absorb {
1355                    if let Some(other) = hosts[other].take() {
1356                        merge_into(&mut hosts, keep, other, Host::merge);
1357                    }
1358                }
1359                merge_into(&mut hosts, keep, host, Host::merge_later_account);
1360                keep
1361            }
1362            None => {
1363                hosts.push(Some(host));
1364                hosts.len() - 1
1365            }
1366        };
1367
1368        let Some(settled) = hosts[slot].as_ref() else {
1369            continue;
1370        };
1371        for ip in scoped_ips(settled) {
1372            slot_of.insert(ip, slot);
1373        }
1374    }
1375
1376    Ok(hosts.into_iter().flatten().collect())
1377}
1378
1379/// Every address of `host`, each carrying the host's zone where the address
1380/// needs one to name a machine.
1381fn scoped_ips(host: &Host) -> impl Iterator<Item = ScopedIp> + '_ {
1382    host.ips().iter().map(|ip| match host.zone() {
1383        Some(zone) => ScopedIp::scoped(*ip, zone.clone()),
1384        None => ScopedIp::unscoped(*ip),
1385    })
1386}
1387
1388/// Folds `host` into the one at `slot` by `fold`, or puts it there if the
1389/// slot is empty.
1390///
1391/// A record folds in as a later account of what the slot holds, since the file is
1392/// appended in write order; see [`Host::merge_later_account`]. Two slots a record shows to
1393/// be one machine fold as any two accounts of a host do.
1394fn merge_into(hosts: &mut [Option<Host>], slot: usize, host: Host, fold: fn(&mut Host, Host)) {
1395    match hosts[slot].as_mut() {
1396        Some(existing) => fold(existing, host),
1397        None => hosts[slot] = Some(host),
1398    }
1399}
1400
1401/// How long the findings file in `directory` is, measured on the opened file since
1402/// nothing here looks up a journal's names by path. A missing file has length 0.
1403fn findings_length(directory: &Path) -> Result<u64, JournalError> {
1404    match open_to_read(&directory.join(HOSTS)) {
1405        Ok(file) => Ok(file.metadata()?.len()),
1406        Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(0),
1407        Err(e) => Err(e.into()),
1408    }
1409}
1410
1411/// Reads back what a journal's earlier sittings did, oldest first.
1412fn read_phases(directory: &Path) -> Result<Vec<ScanPhase>, JournalError> {
1413    let file = match open_to_read(&directory.join(PHASES)) {
1414        Ok(file) => file,
1415        Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()),
1416        Err(e) => return Err(e.into()),
1417    };
1418
1419    let mut reader = match crate::journal::format::Reader::open(std::io::BufReader::new(file)) {
1420        Ok(reader) => reader,
1421        Err(JournalError::NotAJournal) => return Ok(Vec::new()),
1422        Err(e) => return Err(e),
1423    };
1424
1425    let mut phases = Vec::new();
1426    while let Some(record) = reader.read::<PhaseRecord>()? {
1427        phases.push(ScanPhase::from(&record));
1428    }
1429
1430    // Sittings that never ended. A phase already appended was left by a sitting
1431    // stopped between the append and the removal, and is read once.
1432    for standing in standing_phases(directory) {
1433        let recorded = phases.iter().any(|phase| {
1434            phase.kind() == standing.kind() && phase.started_at() == standing.started_at()
1435        });
1436        if !recorded {
1437            phases.push(standing);
1438        }
1439    }
1440    // Oldest first; a sitting that never ended ran before whatever was appended after
1441    // it.
1442    phases.sort_by_key(ScanPhase::started_at);
1443    Ok(phases)
1444}
1445
1446/// Where the sitting holding `lock` writes its phases as they stand, named by start
1447/// time so name order is run order.
1448fn sitting_file(directory: &Path, lock: &Lock) -> PathBuf {
1449    let started = lock
1450        .record()
1451        .started_at
1452        .duration_since(SystemTime::UNIX_EPOCH)
1453        .unwrap_or_default()
1454        .as_nanos();
1455    directory.join(format!("{SITTING}{started:020}.jsonl"))
1456}
1457
1458/// The phases of every sitting in `directory` that never ended, oldest first.
1459///
1460/// A standing record is written whole by rename, so one that cannot be read is
1461/// something else at the name, such as a link, and is skipped.
1462fn standing_phases(directory: &Path) -> Vec<ScanPhase> {
1463    let Ok(held) = names(directory) else {
1464        return Vec::new();
1465    };
1466    let mut files: Vec<PathBuf> = held
1467        .into_iter()
1468        .filter(|name| {
1469            name.to_str()
1470                .is_some_and(|name| name.starts_with(SITTING) && name.ends_with(".jsonl"))
1471        })
1472        .map(|name| directory.join(name))
1473        .collect();
1474    files.sort();
1475
1476    let mut phases = Vec::new();
1477    for path in files {
1478        let Ok(file) = open_to_read(&path) else {
1479            continue;
1480        };
1481        let Ok(mut reader) = crate::journal::format::Reader::open(std::io::BufReader::new(file))
1482        else {
1483            continue;
1484        };
1485        while let Ok(Some(record)) = reader.read::<PhaseRecord>() {
1486            phases.push(ScanPhase::from(&record));
1487        }
1488    }
1489    phases
1490}
1491
1492/// Reads back the detection-run tapes a journal holds, for offline replay.
1493///
1494/// A missing file is no runs, as for a journal read before any detection ran.
1495pub fn read_detections(directory: &Path) -> Result<Vec<DetectionRunRecord>, JournalError> {
1496    let file = match open_to_read(&directory.join(DETECTIONS)) {
1497        Ok(file) => file,
1498        Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()),
1499        Err(e) => return Err(e.into()),
1500    };
1501
1502    let mut reader = match crate::journal::format::Reader::open(std::io::BufReader::new(file)) {
1503        Ok(reader) => reader,
1504        Err(JournalError::NotAJournal) => return Ok(Vec::new()),
1505        Err(e) => return Err(e),
1506    };
1507
1508    let mut runs = Vec::new();
1509    while let Some(line) = reader.read::<DetectionLine>()? {
1510        runs.extend(line.into_runs());
1511    }
1512    Ok(runs)
1513}
1514
1515/// Opens a findings file to add to it, making it whole first.
1516///
1517/// Establishes the precondition
1518/// [`format::Writer::append`](super::format::Writer::append) states, for all three append
1519/// sites: an `O_APPEND` descriptor cannot see what it appends to.
1520///
1521/// A torn tail stops being discardable once anything follows it.
1522/// [`format::Reader`](super::format::Reader) discards a torn line only while it is last. A
1523/// resumed sitting appending after the torn bytes would make the tear the prefix of a
1524/// newline-terminated line, which is corruption by the reader's rule, and the file could
1525/// then be neither read nor resumed. Truncating to the last newline discards exactly what
1526/// the reader would have.
1527///
1528/// A file with no header is mended or refused. [`Journal::open_findings`] creates the file
1529/// and writes its header in two steps, so a process killed between them leaves a
1530/// zero-length file; appending into it would write records under no header, which every
1531/// later read treats as no findings while the cursor advances. An empty file is mended by
1532/// writing the header. A non-empty file without one is somebody else's file at this name
1533/// and is refused, since prepending a header would claim its lines as this engine's
1534/// records.
1535fn open_for_append(path: &Path) -> Result<fs::File, JournalError> {
1536    mend(path)?;
1537    Ok(append_existing(path)?)
1538}
1539
1540/// Gives a findings file a header if it has none and no torn tail if it has one.
1541///
1542/// The header is checked through [`format::Reader::open`](super::format::Reader),
1543/// so the rule is the reader's own, version refusal included.
1544///
1545/// The header is validated before anything is truncated, so a stranger's file at this name
1546/// is refused intact.
1547fn mend(path: &Path) -> Result<(), JournalError> {
1548    use std::io::{Read, Seek, SeekFrom};
1549
1550    let mut file = open_existing(path)?;
1551    let length = file.metadata()?.len();
1552
1553    // The header write that did not finish, between `open_findings`' create and its
1554    // write.
1555    if length == 0 {
1556        super::format::Writer::create(&mut file)?.flush()?;
1557        return Ok(());
1558    }
1559
1560    super::format::Reader::open(std::io::BufReader::new(&file))?;
1561
1562    file.seek(SeekFrom::End(-1))?;
1563    let mut last = [0u8; 1];
1564    file.read_exact(&mut last)?;
1565    if last[0] == b'\n' {
1566        return Ok(());
1567    }
1568
1569    let keep = last_whole_line(&mut file, length)?;
1570    file.set_len(keep)?;
1571
1572    // No whole line anywhere, so the header line is the torn one. It parsed, so the
1573    // file is this engine's with nothing in it to keep; rewrite the header as for an empty
1574    // file.
1575    if keep == 0 {
1576        file.seek(SeekFrom::Start(0))?;
1577        super::format::Writer::create(&mut file)?.flush()?;
1578    }
1579
1580    Ok(())
1581}
1582
1583/// The offset just past the file's last newline: where a whole record last ended.
1584///
1585/// Read backwards a window at a time. This runs on every append, and a findings file
1586/// grows with the scan's duration, so reading it forwards would cost more every
1587/// checkpoint.
1588fn last_whole_line(file: &mut fs::File, length: u64) -> Result<u64, JournalError> {
1589    use std::io::{Read, Seek, SeekFrom};
1590
1591    /// Comfortably more than a record, so the answer is almost always one read.
1592    ///
1593    /// A `u64` to match the file length's width. As a `usize` on a 32-bit target, a length that
1594    /// is an exact multiple of 4 GiB would truncate to a zero window and the loop would hang
1595    /// holding the journal's lock.
1596    const WINDOW: u64 = 8 * 1024;
1597
1598    let mut end = length;
1599    while end > 0 {
1600        let size = WINDOW.min(end);
1601        let start = end - size;
1602
1603        file.seek(SeekFrom::Start(start))?;
1604        // Lossless: `size` is at most `WINDOW`.
1605        let mut window = vec![0u8; size as usize];
1606        file.read_exact(&mut window)?;
1607
1608        if let Some(at) = window.iter().rposition(|byte| *byte == b'\n') {
1609            return Ok(start + at as u64 + 1);
1610        }
1611
1612        end = start;
1613    }
1614
1615    Ok(0)
1616}
1617
1618/// The options a journal's job runs under, or `None` where none were recorded.
1619fn read_options(directory: &Path) -> Result<Option<JobOptions>, JournalError> {
1620    let text = match read_bounded(&directory.join(OPTIONS), "a journal's options") {
1621        Ok(text) => text,
1622        Err(JournalError::Io(e)) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None),
1623        Err(e) => return Err(e),
1624    };
1625    Ok(Some(
1626        serde_json::from_str(&text).map_err(JournalError::json)?,
1627    ))
1628}
1629
1630/// Reads a journal's manifest, refusing one written in a newer format than this
1631/// build understands.
1632fn read_manifest(directory: &Path) -> Result<JournalManifest, JournalError> {
1633    let text = read_bounded(&directory.join(MANIFEST), "a journal manifest")?;
1634    let manifest: JournalManifest = serde_json::from_str(&text).map_err(JournalError::json)?;
1635
1636    if manifest.journal_version > super::JOURNAL_VERSION {
1637        return Err(JournalError::VersionTooNew {
1638            found: manifest.journal_version,
1639            understood: super::JOURNAL_VERSION,
1640        });
1641    }
1642
1643    Ok(manifest)
1644}
1645
1646/// Reads how far a scan got, which is where a resume starts.
1647fn read_checkpoint(directory: &Path) -> Result<Checkpoint, JournalError> {
1648    match Checkpoint::read(&directory.join(CURSOR)) {
1649        Ok(checkpoint) => Ok(checkpoint),
1650        // A journal that stopped before its first checkpoint has settled nothing: a fresh
1651        // cursor.
1652        Err(JournalError::Io(e)) if e.kind() == std::io::ErrorKind::NotFound => {
1653            Ok(Checkpoint::default())
1654        }
1655        Err(e) => Err(e),
1656    }
1657}
1658
1659/// Milliseconds, as a ULID counts them, reaching the year 10 889.
1660const ID_TIME_BITS: u32 = 48;
1661
1662/// A sortable id: the millisecond the scan started, then randomness, in
1663/// Crockford base32.
1664///
1665/// Sorts by creation time as text, so a listing can be ordered without reading every
1666/// manifest.
1667fn mint_id() -> String {
1668    let millis = SystemTime::now()
1669        .duration_since(SystemTime::UNIX_EPOCH)
1670        .map_or(0, |since| since.as_millis())
1671        & ((1u128 << ID_TIME_BITS) - 1);
1672
1673    let random_bits = ID_CHARS as u32 * 5 - ID_TIME_BITS;
1674    let entropy = u128::from(rand::random::<u64>()) & ((1u128 << random_bits) - 1);
1675    let mut value = (millis << random_bits) | entropy;
1676
1677    let mut out = [b'0'; ID_CHARS];
1678    for slot in out.iter_mut().rev() {
1679        *slot = ID_ALPHABET[(value & 0x1F) as usize];
1680        value >>= 5;
1681    }
1682
1683    // Every byte came from `ID_ALPHABET`, which is ASCII.
1684    String::from_utf8(out.to_vec()).expect("base32 alphabet is ASCII")
1685}
1686
1687/// Takes a directory under `root` that nothing else holds, and its id.
1688///
1689/// `create_dir`, which fails on an existing directory, so two scans that minted the
1690/// same id never share one. A collision mints another id, since ids carry enough randomness
1691/// that this should almost never repeat.
1692fn claim_directory(root: &Path) -> Result<(String, PathBuf), JournalError> {
1693    /// Exhausting these means something other than chance: a root that is not a
1694    /// directory, or one nothing may write to.
1695    const ATTEMPTS: usize = 8;
1696
1697    for _ in 0..ATTEMPTS {
1698        let id = mint_id();
1699        let directory = root.join(&id);
1700
1701        match create_private_directory(&directory) {
1702            Ok(()) => {
1703                claim_directory_for_invoking_user(&directory);
1704                return Ok((id, directory));
1705            }
1706            Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => continue,
1707            Err(e) => return Err(e.into()),
1708        }
1709    }
1710
1711    Err(std::io::Error::new(
1712        std::io::ErrorKind::AlreadyExists,
1713        "could not find an unused id for this scan",
1714    )
1715    .into())
1716}
1717
1718/// Reads a whole journal file, refusing one past
1719/// [`MAX_READ_BYTES`](super::format::MAX_READ_BYTES).
1720///
1721/// The ceiling is applied through `take` before the read. The manifest and cursor
1722/// live in a directory belonging to a user while the reader is often root; see
1723/// [`MAX_READ_BYTES`](super::format::MAX_READ_BYTES). A link at the name is refused; see
1724/// [`open_to_read`].
1725pub(super) fn read_bounded(path: &Path, what: &str) -> Result<String, JournalError> {
1726    use std::io::Read;
1727
1728    let mut text = String::new();
1729    let read = open_to_read(path)?
1730        .take(super::format::MAX_READ_BYTES + 1)
1731        .read_to_string(&mut text)?;
1732
1733    if read as u64 > super::format::MAX_READ_BYTES {
1734        return Err(super::format::too_large(what));
1735    }
1736
1737    Ok(text)
1738}
1739
1740/// Writes a whole file at a journal's own mode and ownership, for files written once.
1741///
1742/// Staged and renamed, like [`Checkpoint::write_atomically`], so the name holds the whole
1743/// file or nothing. Used for the manifest, which every other read starts from: a torn
1744/// manifest would fail to parse and [`list`] would treat the journal as absent.
1745fn write_private(path: &Path, bytes: &[u8]) -> Result<(), JournalError> {
1746    use std::io::Write;
1747
1748    replace(path, &path.with_extension("tmp"), |mut file| {
1749        file.write_all(bytes)
1750    })?;
1751    Ok(())
1752}
1753
1754/// How long journals are kept.
1755///
1756/// A journal holds the addresses an engagement was pointed at, so it should not pile
1757/// up unseen; it is also evidence, so it should not vanish unasked. The defaults favour
1758/// keeping.
1759#[non_exhaustive]
1760#[derive(Debug, Clone, PartialEq, Eq)]
1761pub struct Retention {
1762    /// How long a finished journal is kept, or `None` to keep it indefinitely.
1763    pub completed_for: Option<Duration>,
1764    /// How long an unfinished one is kept, or `None` to keep it indefinitely.
1765    pub incomplete_for: Option<Duration>,
1766    /// The most journals to keep, or `None` for no cap.
1767    ///
1768    /// Applied after the ages above, removing finished journals before unfinished
1769    /// ones, since an unfinished journal may still be wanted.
1770    pub keep_at_most: Option<usize>,
1771    /// Whether journals this build cannot read go too.
1772    ///
1773    /// Off by default. A journal passed over is usually a newer build's, which that build
1774    /// still reads. Links are never removed either way; see [`Unlisted::Link`].
1775    pub unreadable: bool,
1776}
1777
1778impl Default for Retention {
1779    /// A month for finished journals, indefinitely for unfinished ones, and a cap of
1780    /// two hundred. A finished scan has a report, so its journal is a duplicate; an unfinished
1781    /// one is the only copy of work somebody may mean to continue.
1782    fn default() -> Self {
1783        Self {
1784            completed_for: Some(Duration::from_secs(30 * 24 * 60 * 60)),
1785            incomplete_for: None,
1786            keep_at_most: Some(200),
1787            unreadable: false,
1788        }
1789    }
1790}
1791
1792impl Retention {
1793    /// Removes nothing. For a caller who prunes on their own terms.
1794    pub fn keep_everything() -> Self {
1795        Self {
1796            completed_for: None,
1797            incomplete_for: None,
1798            keep_at_most: None,
1799            unreadable: false,
1800        }
1801    }
1802
1803    /// Which of `entries` this policy would remove, newest-first as [`list`]
1804    /// yields them.
1805    ///
1806    /// Pure, and separate from [`prune`] as
1807    /// [`lock::classify`](crate::journal::lock::classify) is, so the policy can be tested
1808    /// without building a real directory of journals.
1809    ///
1810    /// A journal something is writing is never selected, whatever its age.
1811    pub fn expired(&self, entries: &[Entry], now: SystemTime) -> Vec<usize> {
1812        let age_of = |entry: &Entry| {
1813            now.duration_since(entry.manifest.created_at)
1814                .unwrap_or_default()
1815        };
1816
1817        let mut removing: Vec<usize> = entries
1818            .iter()
1819            .enumerate()
1820            .filter(|(_, entry)| entry.lock.is_resumable())
1821            .filter(|(_, entry)| {
1822                let limit = if entry.is_complete() {
1823                    self.completed_for
1824                } else {
1825                    self.incomplete_for
1826                };
1827                limit.is_some_and(|limit| age_of(entry) > limit)
1828            })
1829            .map(|(index, _)| index)
1830            .collect();
1831
1832        let Some(cap) = self.keep_at_most else {
1833            return removing;
1834        };
1835
1836        // What the ages left, oldest last, since `list` is newest first.
1837        let mut surviving: Vec<usize> = (0..entries.len())
1838            .filter(|index| !removing.contains(index))
1839            .collect();
1840
1841        // Most worth keeping first: unfinished before finished, newer before older. The cap
1842        // drops from the end, so the oldest finished journal goes first.
1843        surviving.sort_by_key(|&index| {
1844            let entry = &entries[index];
1845            (
1846                entry.is_complete(),
1847                std::cmp::Reverse(entry.manifest.created_at),
1848            )
1849        });
1850
1851        while surviving.len() > cap {
1852            let Some(index) = surviving.pop() else { break };
1853            if entries[index].lock.is_resumable() {
1854                removing.push(index);
1855            }
1856        }
1857
1858        removing.sort_unstable();
1859        removing
1860    }
1861}
1862
1863/// What a prune did, and what it left alone.
1864#[non_exhaustive]
1865#[derive(Debug, Clone, Default, PartialEq, Eq)]
1866pub struct Pruned {
1867    /// The journals removed, by id.
1868    pub removed: Vec<String>,
1869    /// The journals a policy selected that could not be removed, each with why, so a
1870    /// sweep that stopped working is visible.
1871    pub held: Vec<Held>,
1872}
1873
1874/// A journal a prune could not remove.
1875#[non_exhaustive]
1876#[derive(Debug, Clone, PartialEq, Eq)]
1877pub struct Held {
1878    /// Which journal.
1879    pub id: String,
1880    /// Why it is still there.
1881    pub reason: String,
1882}
1883
1884impl Held {
1885    /// The journal `id`, kept for `reason`.
1886    pub fn new(id: impl Into<String>, reason: impl Into<String>) -> Self {
1887        Self {
1888            id: id.into(),
1889            reason: reason.into(),
1890        }
1891    }
1892}
1893
1894/// Removes the journals under `root` that `retention` no longer keeps.
1895///
1896/// Journals a scan is writing are never selected, so they are neither removed nor
1897/// reported as held. [`Pruned::held`] lists journals the policy chose and the filesystem
1898/// refused.
1899///
1900/// A journal this build cannot read is removed only under [`Retention::unreadable`], by its
1901/// directory name, after the usual lock check (a newer build may be writing it). A link is
1902/// never selected; [`list`] reports it.
1903pub fn prune(root: &Path, retention: &Retention) -> Result<Pruned, JournalError> {
1904    let listing = list(root)?;
1905    let mut pruned = Pruned::default();
1906
1907    for index in retention.expired(&listing.entries, SystemTime::now()) {
1908        let entry = &listing.entries[index];
1909        match remove(&entry.directory) {
1910            Ok(()) => pruned.removed.push(entry.manifest.id.clone()),
1911            Err(error) => pruned.held.push(Held {
1912                id: entry.manifest.id.clone(),
1913                reason: error.to_string(),
1914            }),
1915        }
1916    }
1917
1918    if retention.unreadable {
1919        let journals = listing
1920            .passed_over
1921            .iter()
1922            .filter(|passed| passed.why != Unlisted::Link);
1923        for passed in journals {
1924            match remove(&passed.directory) {
1925                Ok(()) => pruned.removed.push(passed.name.clone()),
1926                Err(error) => pruned
1927                    .held
1928                    .push(Held::new(passed.name.clone(), error.to_string())),
1929            }
1930        }
1931    }
1932
1933    Ok(pruned)
1934}
1935
1936// ╔════════════════════════════════════════════╗
1937// ║ ████████╗███████╗███████╗████████╗███████╗ ║
1938// ║ ╚══██╔══╝██╔════╝██╔════╝╚══██╔══╝██╔════╝ ║
1939// ║    ██║   █████╗  ███████╗   ██║   ███████╗ ║
1940// ║    ██║   ██╔══╝  ╚════██║   ██║   ╚════██║ ║
1941// ║    ██║   ███████╗███████║   ██║   ███████║ ║
1942// ║    ╚═╝   ╚══════╝╚══════╝   ╚═╝   ╚══════╝ ║
1943// ╚════════════════════════════════════════════╝
1944
1945#[cfg(test)]
1946mod tests {
1947    use super::*;
1948    use crate::journal::settle::Outcome;
1949    // Two tests here use the checkpoint ticker from `scanner`, since what they assert
1950    // is what a checkpointed scan leaves in the journal.
1951    use crate::model::exclusion::Exclusions;
1952    use crate::model::ip::set::IpSet;
1953    use crate::model::port::PortSet;
1954    use crate::model::target::{TargetMap, TargetSet};
1955    use crate::model::technique::TcpScanTechnique;
1956    use crate::scanner::checkpoint::spawn_checkpoints;
1957
1958    fn scratch(name: &str) -> PathBuf {
1959        let dir = std::env::temp_dir().join(format!("zond-store-{}-{name}", std::process::id()));
1960        let _ = fs::remove_dir_all(&dir);
1961        fs::create_dir_all(&dir).expect("scratch root");
1962        dir
1963    }
1964
1965    fn plan(range: &str, ports: &str) -> TargetMap {
1966        let mut map = TargetMap::new();
1967        map.add_unit(TargetSet::new(
1968            range.parse::<IpSet>().expect("a range"),
1969            ports.parse::<PortSet>().expect("ports"),
1970        ));
1971        map
1972    }
1973
1974    fn ports(map: &TargetMap) -> Plan {
1975        Plan::port_scan(map, &Exclusions::none(), TcpScanTechnique::Syn)
1976    }
1977
1978    fn begin(root: &Path, map: &TargetMap) -> Journal {
1979        Journal::create(root, &ports(map), Privilege::Raw, "test").expect("creates")
1980    }
1981
1982    /// A port phase that stood in for a liveness pass, naming `silent` the addresses
1983    /// it asked on every port and heard nothing from.
1984    fn standing_in(silent: &str) -> ScanPhase {
1985        use crate::config::ZondConfig;
1986        use crate::model::ip::range::IpRange;
1987        use crate::report::{LivenessSkip, PhaseParts, ScanSettings, TargetScope};
1988
1989        let silent: IpSet = silent.parse().expect("a range");
1990        ScanPhase::from_parts(PhaseParts {
1991            open: false,
1992            attachments: Vec::new(),
1993            kind: ScanKind::PortScan,
1994            started_at: SystemTime::UNIX_EPOCH,
1995            elapsed: Duration::from_secs(1),
1996            privilege: Some(Privilege::Raw),
1997            targets: TargetScope::from_ip_set(&mut IpSet::new(), &Exclusions::none()),
1998            settings: ScanSettings::from(&ZondConfig::default()),
1999            failures: Vec::new(),
2000            refusals: Vec::new(),
2001            unroutable: Vec::new(),
2002            refused_by_route: Vec::new(),
2003            timed_out: Vec::new(),
2004            icmp_rate_limited: Vec::new(),
2005            reached_by_connect: Vec::new(),
2006            undecided: Vec::new(),
2007            liveness_skipped: Some(LivenessSkip::PortsNoDearer),
2008            silent: silent.v4().iter().copied().map(IpRange::V4).collect(),
2009            stopped: None,
2010            passes_cut: Vec::new(),
2011            unreached: 0,
2012            unheard_probes: 0,
2013            probes: Vec::new(),
2014            origin: None,
2015        })
2016    }
2017
2018    /// A record a port scanner files at an address before hearing from it, as a
2019    /// checkpoint writes it mid-sitting.
2020    fn unheard(address: &str) -> Host {
2021        use crate::model::port::{Port, PortState, Protocol};
2022
2023        let mut host = Host::new(address.parse().expect("an address"));
2024        host.add_port(Port::new(80, Protocol::Tcp, PortState::NoReply));
2025        host
2026    }
2027
2028    /// A host that answered.
2029    fn heard(address: &str) -> Host {
2030        let mut host = Host::new(address.parse().expect("an address"));
2031        host.set_status(crate::model::host::HostStatus::Up);
2032        host
2033    }
2034
2035    /// A record an earlier sitting heard nothing from is not restored. The findings
2036    /// file can hold one written before the phase decided the address was silent; restored, it
2037    /// would sit among the resumed sitting's hosts though the report does not list it.
2038    #[test]
2039    fn a_record_an_earlier_sitting_heard_nothing_from_is_not_restored() {
2040        let root = scratch("unheard-restore");
2041        let map = plan("192.0.2.1-192.0.2.8", "80");
2042
2043        let directory = {
2044            let mut journal = begin(&root, &map);
2045            journal
2046                .record_hosts(&[heard("192.0.2.1"), unheard("192.0.2.5")])
2047                .expect("records");
2048            journal
2049                .record_phases(&[standing_in("192.0.2.5")])
2050                .expect("records");
2051            let directory = journal.directory().to_path_buf();
2052            journal.close().expect("closes");
2053            directory
2054        };
2055
2056        let (journal, _) =
2057            Journal::resume(&directory, &ports(&map), Privilege::Raw).expect("resumes");
2058        let restored: Vec<_> = journal.restored().iter().map(Host::primary_ip).collect();
2059
2060        assert_eq!(
2061            restored,
2062            ["192.0.2.1".parse::<std::net::IpAddr>().expect("an address")]
2063        );
2064        journal.close().expect("closes");
2065        std::fs::remove_dir_all(&root).ok();
2066    }
2067
2068    /// A sitting that heard nothing from an address leaves no record of it on disk. A
2069    /// checkpoint writes the record before the phase decides the address is silent, and the
2070    /// phase forgets it only in memory, so the close must rewrite the file.
2071    #[tokio::test]
2072    async fn a_sitting_that_heard_nothing_from_an_address_leaves_no_record_of_it() {
2073        let root = scratch("unheard-close");
2074        let map = plan("192.0.2.1-192.0.2.8", "80");
2075        let mut journal = begin(&root, &map);
2076        let directory = journal.directory().to_path_buf();
2077
2078        // Written mid-sitting and since forgotten by the phase: the live store holds only
2079        // the host that answered.
2080        journal
2081            .record_hosts(&[heard("192.0.2.1"), unheard("192.0.2.5")])
2082            .expect("records");
2083        let (_session, ctx) = crate::scanner::session::ScanSession::new();
2084        ctx.restore_hosts(&[heard("192.0.2.1")]);
2085
2086        spawn_checkpoints(journal, ctx.progress())
2087            .finish(&[standing_in("192.0.2.5")])
2088            .await;
2089
2090        let file = fs::read_to_string(directory.join(HOSTS)).expect("reads");
2091        assert!(
2092            !file.contains("192.0.2.5"),
2093            "a record of the silent address is still on disk:\n{file}"
2094        );
2095        let kept: Vec<_> = read_findings(&directory)
2096            .expect("reads")
2097            .iter()
2098            .map(Host::primary_ip)
2099            .collect();
2100        assert_eq!(
2101            kept,
2102            ["192.0.2.1".parse::<std::net::IpAddr>().expect("an address")]
2103        );
2104
2105        std::fs::remove_dir_all(&root).ok();
2106    }
2107
2108    /// A sitting killed before its port phase decides what it heard nothing from leaves
2109    /// each undecided record to the next sitting, and out of the job's report.
2110    ///
2111    /// A phase standing in for a liveness pass decides its unanswered records only at its end,
2112    /// while a target is settled as its answer or silence is stored. If the record stayed off
2113    /// disk until a verdict the killed sitting never reaches, its ports would be settled with
2114    /// nothing on file. Its probes at an address silent on every target are counted, being
2115    /// decided.
2116    #[tokio::test]
2117    async fn a_sitting_killed_before_its_verdicts_leaves_its_undecided_records_to_the_next() {
2118        use crate::config::ZondConfig;
2119        use crate::journal::settle::Outcome;
2120        use crate::model::port::{Port, PortState, Protocol};
2121        use crate::model::target::TargetIndex;
2122        use crate::report::{LivenessSkip, TargetScope};
2123        use crate::scanner::recorder::PhaseRecorder;
2124
2125        let root = scratch("unheard-killed");
2126        let map = plan("192.0.2.1-192.0.2.8", "80,443");
2127        let journal = begin(&root, &map);
2128        let directory = journal.directory().to_path_buf();
2129        let address = |address: &str| address.parse::<std::net::IpAddr>().expect("an address");
2130        let position = |at: &str, port: u16| {
2131            map.iter()
2132                .position(|target| target.ip == address(at) && target.port == port)
2133                .expect("a planned target") as u64
2134        };
2135
2136        let (_session, ctx) = crate::scanner::session::ScanSession::new();
2137        ctx.number_targets(TargetIndex::of(&map));
2138        let _phase = PhaseRecorder::start(
2139            ScanKind::PortScan,
2140            Privilege::Raw,
2141            TargetScope::from_ip_set(
2142                &mut "192.0.2.1-192.0.2.8".parse().expect("a range"),
2143                &Exclusions::none(),
2144            ),
2145            &ZondConfig::default(),
2146        )
2147        .skipping_liveness(LivenessSkip::PortsNoDearer)
2148        .opening_in(&ctx);
2149        ctx.await_verdicts();
2150        // A host; an address asked on one of its ports so far; and one asked on both and
2151        // heard from on neither.
2152        ctx.update_host(address("192.0.2.1"), |host| {
2153            host.set_status(crate::model::host::HostStatus::Up);
2154        });
2155        ctx.write_host(address("192.0.2.5"), |host| {
2156            *host = unheard("192.0.2.5");
2157            true
2158        });
2159        ctx.record_outcome(Outcome::Exhausted {
2160            position: position("192.0.2.5", 80),
2161        });
2162        ctx.write_host(address("192.0.2.6"), |host| {
2163            *host = unheard("192.0.2.6");
2164            host.add_port(Port::new(443, Protocol::Tcp, PortState::NoReply));
2165            true
2166        });
2167        for port in [80, 443] {
2168            ctx.record_outcome(Outcome::Exhausted {
2169                position: position("192.0.2.6", port),
2170            });
2171        }
2172
2173        // One checkpoint lands, and the process dies before the phase ends.
2174        let mut ticker = spawn_checkpoints(journal, ctx.progress());
2175        ticker.checkpointed().await;
2176        ticker.kill().await;
2177
2178        let hosts = report(&directory)
2179            .expect("reads")
2180            .hosts()
2181            .map(Host::primary_ip)
2182            .collect::<Vec<_>>();
2183        assert_eq!(
2184            hosts,
2185            [address("192.0.2.1")],
2186            "the report lists an undecided address"
2187        );
2188
2189        let (journal, _) =
2190            Journal::resume(&directory, &ports(&map), Privilege::Raw).expect("resumes");
2191        let restored: Vec<_> = journal
2192            .restored()
2193            .iter()
2194            .map(|host| (host.primary_ip(), host.ports().count()))
2195            .collect();
2196        assert_eq!(
2197            restored,
2198            [(address("192.0.2.1"), 0), (address("192.0.2.5"), 1)],
2199            "the undecided record, and only it beside the host, is the next sitting's"
2200        );
2201        let unheard: u128 = journal
2202            .earlier_phases()
2203            .iter()
2204            .map(ScanPhase::unheard_probes)
2205            .sum();
2206        assert_eq!(unheard, 2, "the silent address's probes went uncounted");
2207        journal.close().expect("closes");
2208        std::fs::remove_dir_all(&root).ok();
2209    }
2210
2211    /// A sitting killed before its verdicts still names the addresses it heard nothing
2212    /// from on every target. Their targets are settled on disk, so a resume asks nothing more
2213    /// of them; named only at the phase's end, they would be in no list of the job.
2214    #[tokio::test]
2215    async fn a_sitting_killed_before_its_verdicts_still_names_what_it_heard_nothing_from() {
2216        use crate::config::ZondConfig;
2217        use crate::journal::settle::Outcome;
2218        use crate::model::target::TargetIndex;
2219        use crate::report::TargetScope;
2220        use crate::scanner::recorder::PhaseRecorder;
2221
2222        let root = scratch("unheard-named");
2223        let map = plan("192.0.2.1-192.0.2.8", "80");
2224        let journal = begin(&root, &map);
2225        let directory = journal.directory().to_path_buf();
2226
2227        let (_session, ctx) = crate::scanner::session::ScanSession::new();
2228        ctx.number_targets(TargetIndex::of(&map));
2229        let _phase = PhaseRecorder::start(
2230            ScanKind::PortScan,
2231            Privilege::Raw,
2232            TargetScope::from_ip_set(&mut IpSet::new(), &Exclusions::none()),
2233            &ZondConfig::default(),
2234        )
2235        .opening_in(&ctx);
2236        ctx.await_verdicts();
2237        // Asked and settled silent, and asked with its answer still owed.
2238        for address in ["192.0.2.5", "192.0.2.6"] {
2239            ctx.write_host(
2240                address.parse::<std::net::IpAddr>().expect("an address"),
2241                |host| {
2242                    *host = unheard(address);
2243                    true
2244                },
2245            );
2246        }
2247        ctx.record_outcome(Outcome::Exhausted { position: 4 });
2248
2249        let mut ticker = spawn_checkpoints(journal, ctx.progress());
2250        ticker.checkpointed().await;
2251        ticker.kill().await;
2252
2253        let (journal, _) =
2254            Journal::resume(&directory, &ports(&map), Privilege::Raw).expect("resumes");
2255        let silent: Vec<_> = journal
2256            .earlier_phases()
2257            .iter()
2258            .flat_map(|phase| phase.silent().iter().copied())
2259            .collect();
2260        let named: IpSet = "192.0.2.5".parse().expect("an address");
2261        assert_eq!(
2262            silent,
2263            named
2264                .v4()
2265                .iter()
2266                .copied()
2267                .map(crate::model::ip::range::IpRange::V4)
2268                .collect::<Vec<_>>(),
2269            "the settled silent address is named, and only it"
2270        );
2271        journal.close().expect("closes");
2272        std::fs::remove_dir_all(&root).ok();
2273    }
2274
2275    /// A record awaiting its verdict is in the findings once the phase keeps it: an
2276    /// address the phase neither forgot nor heard from, one no route led to.
2277    #[tokio::test]
2278    async fn a_record_held_for_its_verdict_is_written_once_the_phase_keeps_it() {
2279        let root = scratch("unheard-kept");
2280        let map = plan("192.0.2.1-192.0.2.8", "80");
2281        let journal = begin(&root, &map);
2282        let directory = journal.directory().to_path_buf();
2283
2284        let (_session, ctx) = crate::scanner::session::ScanSession::new();
2285        ctx.await_verdicts();
2286        ctx.write_host(
2287            "192.0.2.5".parse::<std::net::IpAddr>().expect("an address"),
2288            |host| {
2289                *host = unheard("192.0.2.5");
2290                true
2291            },
2292        );
2293        let mut ticker = spawn_checkpoints(journal, ctx.progress());
2294        ticker.checkpointed().await;
2295        ctx.verdicts_reached();
2296        ticker.finish(&[]).await;
2297
2298        let kept: Vec<_> = read_findings(&directory)
2299            .expect("reads")
2300            .iter()
2301            .map(Host::primary_ip)
2302            .collect();
2303        assert_eq!(
2304            kept,
2305            ["192.0.2.5".parse::<std::net::IpAddr>().expect("an address")]
2306        );
2307        std::fs::remove_dir_all(&root).ok();
2308    }
2309
2310    /// Begin a scan, settle part of it, and continue from exactly where it stopped.
2311    #[test]
2312    fn a_scan_resumes_from_where_it_stopped() {
2313        let root = scratch("resume");
2314        let map = plan("192.0.2.1-192.0.2.4", "80,443");
2315
2316        let directory = {
2317            let mut journal = begin(&root, &map);
2318            let settlements = Settlements::default();
2319            for position in [0, 1, 2, 5] {
2320                settlements.record(Outcome::Answered { position });
2321            }
2322            journal.checkpoint(&settlements).expect("checkpoints");
2323
2324            let directory = journal.directory().to_path_buf();
2325            journal.close().expect("closes");
2326            directory
2327        };
2328
2329        let (_journal, checkpoint) =
2330            Journal::resume(&directory, &ports(&map), Privilege::Raw).expect("resumes");
2331
2332        assert_eq!(checkpoint.watermark, 3);
2333        let remaining: Vec<_> = checkpoint.remaining(map.iter()).collect();
2334        assert_eq!(remaining.len(), 4, "eight targets, four settled");
2335    }
2336
2337    /// A job's options are written by its first sitting and read back by every later
2338    /// one, and nothing a later sitting runs under replaces them. Otherwise a sitting that
2339    /// changed its pace would become the record of what the job asked.
2340    #[test]
2341    fn a_jobs_options_are_written_by_its_first_sitting_and_kept() {
2342        use crate::config::ZondConfig;
2343
2344        let root = scratch("options");
2345        let map = plan("192.0.2.1-192.0.2.4", "80,443");
2346        let first = ZondConfig {
2347            assume_up: true,
2348            traceroute: true,
2349            ..ZondConfig::default()
2350        };
2351
2352        let directory = {
2353            let mut journal = begin(&root, &map);
2354            assert!(journal.options().is_none(), "no sitting has started");
2355            journal
2356                .record_options(JobOptions::of(&first))
2357                .expect("records");
2358            let directory = journal.directory().to_path_buf();
2359            journal.close().expect("closes");
2360            directory
2361        };
2362
2363        let (mut journal, _) =
2364            Journal::resume(&directory, &ports(&map), Privilege::Raw).expect("resumes");
2365        assert_eq!(journal.options(), Some(&JobOptions::of(&first)));
2366
2367        journal
2368            .record_options(JobOptions::of(&ZondConfig::default()))
2369            .expect("a second record is a no-op");
2370        journal.close().expect("closes");
2371        let (journal, _) =
2372            Journal::resume(&directory, &ports(&map), Privilege::Raw).expect("resumes");
2373        assert_eq!(journal.options(), Some(&JobOptions::of(&first)));
2374    }
2375
2376    /// A journal an earlier sitting ran against without recording options is continued
2377    /// without any, since this sitting's are not what that one ran under.
2378    #[test]
2379    fn a_journal_older_than_its_options_is_left_without_them() {
2380        use crate::config::ZondConfig;
2381
2382        let root = scratch("options-older");
2383        let map = plan("192.0.2.1-192.0.2.4", "80,443");
2384
2385        let directory = {
2386            let mut journal = begin(&root, &map);
2387            let settlements = Settlements::default();
2388            settlements.record(Outcome::Answered { position: 0 });
2389            journal.checkpoint(&settlements).expect("checkpoints");
2390            let directory = journal.directory().to_path_buf();
2391            journal.close().expect("closes");
2392            directory
2393        };
2394
2395        let (mut journal, checkpoint) = Journal::resume(&directory, &ports(&map), Privilege::Raw)
2396            .expect("an older journal resumes");
2397        assert_eq!(checkpoint.watermark, 1);
2398        journal
2399            .record_options(JobOptions::of(&ZondConfig::default()))
2400            .expect("records nothing");
2401        assert!(journal.options().is_none());
2402        assert!(!directory.join(OPTIONS).exists());
2403    }
2404
2405    /// A plan edited between sittings renumbers positions past the edit, so the resume is
2406    /// refused.
2407    #[test]
2408    fn a_changed_plan_is_refused() {
2409        let root = scratch("changed-plan");
2410        let map = plan("192.0.2.1-192.0.2.4", "80,443");
2411
2412        let journal = begin(&root, &map);
2413        let directory = journal.directory().to_path_buf();
2414        journal.close().expect("closes");
2415
2416        let widened = plan("192.0.2.1-192.0.2.9", "80,443");
2417        let refused = Journal::resume(&directory, &ports(&widened), Privilege::Raw)
2418            .expect_err("the plan moved");
2419
2420        assert!(matches!(refused, OpenError::PlanChanged(_)), "{refused:?}");
2421    }
2422
2423    /// A journal being written is not resumed underneath its writer. A lock naming a live
2424    /// process that stopped checkpointing is refused as it stands and can be taken over when
2425    /// asked. This process stands in for a reissued pid.
2426    #[test]
2427    fn a_journal_whose_lock_went_stale_can_be_taken_over() {
2428        let root = scratch("stale-lock");
2429        let map = plan("192.0.2.1", "80");
2430        let directory = {
2431            let journal = begin(&root, &map);
2432            journal.directory().to_path_buf()
2433        };
2434        let stale = super::super::lock::LockRecord {
2435            pid: std::process::id(),
2436            boot: super::super::lock::boot_identity(),
2437            started_at: SystemTime::now() - Duration::from_secs(3_600),
2438            heartbeat: SystemTime::now() - Duration::from_secs(600),
2439        };
2440        fs::write(
2441            directory.join(LOCK),
2442            serde_json::to_string(&stale).expect("json"),
2443        )
2444        .expect("writes");
2445
2446        assert!(matches!(
2447            Journal::reopen(&directory, Privilege::Raw),
2448            Err(OpenError::Locked(LockRefused::Held(
2449                LockState::Stale { .. }
2450            )))
2451        ));
2452        let (journal, _, _) =
2453            Journal::take_over(&directory, Privilege::Raw).expect("takes the journal over");
2454        journal.close().expect("closes");
2455
2456        std::fs::remove_dir_all(&root).ok();
2457    }
2458
2459    #[test]
2460    fn a_live_journal_is_not_resumable() {
2461        let root = scratch("live");
2462        let map = plan("192.0.2.1", "80");
2463
2464        let journal = begin(&root, &map);
2465        let directory = journal.directory().to_path_buf();
2466
2467        let refused =
2468            Journal::resume(&directory, &ports(&map), Privilege::Raw).expect_err("it is held");
2469        assert!(matches!(refused, OpenError::Locked(_)), "{refused:?}");
2470
2471        // Once released, it opens.
2472        journal.close().expect("closes");
2473        Journal::resume(&directory, &ports(&map), Privilege::Raw).expect("now free");
2474    }
2475
2476    /// Listing reports progress and liveness without taking the lock.
2477    #[test]
2478    fn listing_describes_journals_without_locking_them() {
2479        let root = scratch("list");
2480        let map = plan("192.0.2.1-192.0.2.4", "80,443");
2481
2482        let mut journal = begin(&root, &map);
2483        let settlements = Settlements::default();
2484        for position in 0..4 {
2485            settlements.record(Outcome::Answered { position });
2486        }
2487        journal.checkpoint(&settlements).expect("checkpoints");
2488
2489        let listed = list(&root).expect("lists").entries;
2490        assert_eq!(listed.len(), 1);
2491
2492        let entry = &listed[0];
2493        assert_eq!(entry.manifest.total_targets, 8);
2494        assert_eq!(entry.settled(), Some(4));
2495        assert!(!entry.is_complete());
2496        assert!(
2497            matches!(entry.lock, LockState::Held { .. }),
2498            "the scan is still running: {:?}",
2499            entry.lock
2500        );
2501
2502        // Still writable, so the listing took nothing.
2503        journal
2504            .checkpoint(&settlements)
2505            .expect("still holds its lock");
2506    }
2507
2508    /// A journal that covered its plan reads as complete.
2509    #[test]
2510    fn a_finished_journal_reads_as_complete() {
2511        let root = scratch("complete");
2512        let map = plan("192.0.2.1-192.0.2.4", "80,443");
2513
2514        let mut journal = begin(&root, &map);
2515        let settlements = Settlements::default();
2516        for position in 0..8 {
2517            settlements.record(Outcome::Answered { position });
2518        }
2519        journal.checkpoint(&settlements).expect("checkpoints");
2520        journal.close().expect("closes");
2521
2522        let listed = list(&root).expect("lists").entries;
2523        assert!(listed[0].is_complete());
2524        assert_eq!(listed[0].settled(), Some(8));
2525    }
2526
2527    /// A journal that stopped before its first checkpoint reads as a fresh cursor, not an
2528    /// unreadable journal.
2529    #[test]
2530    fn a_journal_with_no_checkpoint_resumes_from_the_beginning() {
2531        let root = scratch("no-checkpoint");
2532        let map = plan("192.0.2.1-192.0.2.4", "80,443");
2533
2534        let journal = begin(&root, &map);
2535        let directory = journal.directory().to_path_buf();
2536        journal.close().expect("closes");
2537
2538        let (_journal, checkpoint) =
2539            Journal::resume(&directory, &ports(&map), Privilege::Raw).expect("resumes");
2540
2541        assert_eq!(checkpoint, Checkpoint::default());
2542        assert_eq!(checkpoint.remaining(map.iter()).count(), 8);
2543    }
2544
2545    /// Ids sort by creation time as text, including ids minted one after another.
2546    #[test]
2547    fn ids_are_sortable_and_distinct() {
2548        let mut ids: Vec<String> = (0..64).map(|_| mint_id()).collect();
2549
2550        assert!(ids.iter().all(|id| id.len() == ID_CHARS));
2551        let distinct: std::collections::BTreeSet<&String> = ids.iter().collect();
2552        assert_eq!(distinct.len(), ids.len(), "ids collided");
2553
2554        let sorted = {
2555            ids.sort();
2556            ids.clone()
2557        };
2558        assert_eq!(ids, sorted, "ids minted in one burst lost their order");
2559
2560        // And across milliseconds, where the clock decides.
2561        let first = mint_id();
2562        std::thread::sleep(std::time::Duration::from_millis(3));
2563        let second = mint_id();
2564        assert!(first < second, "{first} should sort before {second}");
2565    }
2566
2567    /// A listing skips what it cannot read, so one damaged journal does not hide the
2568    /// rest.
2569    #[test]
2570    fn an_unreadable_journal_does_not_hide_the_others() {
2571        let root = scratch("damaged");
2572        let map = plan("192.0.2.1", "80");
2573        begin(&root, &map).close().expect("closes");
2574
2575        let damaged = root.join("01JUNREADABLEJUNREADABLE00");
2576        fs::create_dir_all(&damaged).expect("directory");
2577        fs::write(damaged.join(MANIFEST), "{not json").expect("writes");
2578
2579        let listed = list(&root).expect("lists").entries;
2580        assert_eq!(listed.len(), 1, "the readable one is still there");
2581    }
2582
2583    /// Pruning removes a journal nobody is writing and refuses one somebody is.
2584    #[test]
2585    fn pruning_refuses_a_journal_that_is_being_written() {
2586        let root = scratch("prune");
2587        let map = plan("192.0.2.1", "80");
2588
2589        let journal = begin(&root, &map);
2590        let directory = journal.directory().to_path_buf();
2591
2592        assert!(matches!(remove(&directory), Err(OpenError::Locked(_))));
2593
2594        journal.close().expect("closes");
2595        remove(&directory).expect("removes a free journal");
2596        assert!(list(&root).expect("lists").entries.is_empty());
2597    }
2598
2599    /// The ticker writes a final checkpoint and releases the lock, so a scan that
2600    /// finishes between two ticks still records what it did.
2601    #[tokio::test]
2602    async fn the_checkpoint_task_writes_a_final_cursor_and_releases() {
2603        let root = scratch("ticker");
2604        let map = plan("192.0.2.1-192.0.2.4", "80,443");
2605        let journal = begin(&root, &map);
2606        let directory = journal.directory().to_path_buf();
2607
2608        let (_session, ctx) = crate::scanner::session::ScanSession::new();
2609        for position in 0..5 {
2610            ctx.record_outcome(Outcome::Answered { position });
2611        }
2612
2613        // Finishes well inside one tick.
2614        spawn_checkpoints(journal, ctx.progress()).finish(&[]).await;
2615
2616        let checkpoint = Checkpoint::read(&directory.join(CURSOR)).expect("a cursor was written");
2617        assert_eq!(checkpoint.watermark, 5);
2618        assert_eq!(
2619            super::super::lock::inspect(&directory.join(LOCK)),
2620            LockState::Free,
2621            "the lock outlived the scan and was then released"
2622        );
2623    }
2624
2625    /// A resumed sitting that settles nothing still writes a cursor covering the first
2626    /// sitting's work. Without seeding the live cursor from the resume point, the second
2627    /// sitting's checkpoint would erase the first's progress.
2628    #[test]
2629    fn a_resumed_cursor_carries_the_earlier_sittings_progress() {
2630        let root = scratch("carry-forward");
2631        let map = plan("192.0.2.1-192.0.2.4", "80,443");
2632
2633        let directory = {
2634            let mut journal = begin(&root, &map);
2635            let settlements = Settlements::default();
2636            for position in 0..3 {
2637                settlements.record(Outcome::Answered { position });
2638            }
2639            journal.checkpoint(&settlements).expect("checkpoints");
2640            let directory = journal.directory().to_path_buf();
2641            journal.close().expect("closes");
2642            directory
2643        };
2644
2645        let (mut journal, checkpoint) =
2646            Journal::resume(&directory, &ports(&map), Privilege::Raw).expect("resumes");
2647
2648        // Seeded from the resume point, as a scan does.
2649        let settlements = Settlements::resuming(&checkpoint);
2650        journal
2651            .checkpoint(&settlements)
2652            .expect("checkpoints having settled nothing new");
2653        journal.close().expect("closes");
2654
2655        let carried = Checkpoint::read(&directory.join(CURSOR)).expect("reads");
2656        assert_eq!(
2657            carried.watermark, 3,
2658            "the second sitting must not roll the cursor back"
2659        );
2660    }
2661
2662    /// Findings survive the journal.
2663    #[test]
2664    fn what_a_sitting_found_comes_back() {
2665        use crate::model::host::HostStatus;
2666        use crate::model::port::{Port, PortState, Protocol};
2667
2668        let root = scratch("findings");
2669        let map = plan("192.0.2.1-192.0.2.4", "80,443");
2670
2671        let directory = {
2672            let mut journal = begin(&root, &map);
2673
2674            let mut host = Host::new("192.0.2.1".parse().expect("an address"));
2675            host.set_status(HostStatus::Up);
2676            host.set_hostname(Some("router.example".to_string()));
2677            host.add_port(Port::new(80, Protocol::Tcp, PortState::Open));
2678
2679            journal.record_hosts(&[host]).expect("records");
2680
2681            let directory = journal.directory().to_path_buf();
2682            journal.close().expect("closes");
2683            directory
2684        };
2685
2686        let (journal, _) =
2687            Journal::resume(&directory, &ports(&map), Privilege::Raw).expect("resumes");
2688
2689        let restored = journal.restored();
2690        assert_eq!(restored.len(), 1);
2691        assert_eq!(restored[0].hostname(), Some("router.example"));
2692        assert_eq!(restored[0].port_count(), 1);
2693        assert!(restored[0].is_alive());
2694    }
2695
2696    /// A detection run's tape survives the round trip, so a recorded scan can be
2697    /// replayed offline. Its own file, so it never disturbs the hosts a resume reads.
2698    #[test]
2699    fn detection_run_tapes_survive_a_journal_round_trip() {
2700        use crate::detect::compute::{CapTape, CapTapeRecord, DetectionRunRecord, SpeakExchange};
2701        use crate::model::finding::{DetectionId, Version};
2702        use crate::record::DetectionIdRecord;
2703
2704        let root = scratch("detections");
2705        let map = plan("192.0.2.1", "80");
2706
2707        let tape = CapTape {
2708            speaks: vec![SpeakExchange {
2709                sent: b"PING\r\n".to_vec(),
2710                reply: Ok(b"+PONG\r\n".to_vec()),
2711            }],
2712            nows: vec![7],
2713            ..CapTape::default()
2714        };
2715        let detection = DetectionId::new("redis-unauth", Version::new(1, 0, 0), "abc").unwrap();
2716        let run = DetectionRunRecord {
2717            host: "192.0.2.1".to_string(),
2718            host_name: None,
2719            port: 80,
2720            protocol: "tcp".to_string(),
2721            detection: DetectionIdRecord::from(&detection),
2722            responses: vec!["+PONG\r\n".to_string()],
2723            tape: CapTapeRecord::from(&tape),
2724        };
2725
2726        let directory = {
2727            let mut journal = begin(&root, &map);
2728            journal
2729                .record_detections(std::slice::from_ref(&run))
2730                .expect("records the run");
2731            let directory = journal.directory().to_path_buf();
2732            journal.close().expect("closes");
2733            directory
2734        };
2735
2736        let read = read_detections(&directory).expect("reads the runs back");
2737        assert_eq!(
2738            read,
2739            vec![run],
2740            "a detection run did not survive the journal"
2741        );
2742        assert_eq!(
2743            read[0].tape.rebuild(),
2744            tape,
2745            "the tape did not rebuild to what was recorded"
2746        );
2747    }
2748
2749    /// A run over `port` of 192.0.2.1 by the passive detection `id`, which read
2750    /// `responses` and sent nothing.
2751    fn passive_run(
2752        id: &str,
2753        port: u16,
2754        responses: &[&str],
2755    ) -> crate::detect::compute::DetectionRunRecord {
2756        use crate::detect::compute::{CapTape, CapTapeRecord, DetectionRunRecord};
2757        use crate::model::finding::{DetectionId, Version};
2758        use crate::record::DetectionIdRecord;
2759
2760        let detection = DetectionId::new(id, Version::new(1, 0, 0), "abc").expect("an id");
2761        DetectionRunRecord {
2762            host: "192.0.2.1".to_string(),
2763            host_name: None,
2764            port,
2765            protocol: "tcp".to_string(),
2766            detection: DetectionIdRecord::from(&detection),
2767            responses: responses
2768                .iter()
2769                .map(|response| response.to_string())
2770                .collect(),
2771            tape: CapTapeRecord::from(&CapTape::default()),
2772        }
2773    }
2774
2775    /// Every passive detection matching a service reads the same responses gathered at
2776    /// its port, a dozen for any HTTP port, so a port's responses are written once and every
2777    /// run over it reads them back.
2778    #[test]
2779    fn a_ports_responses_are_written_once_however_many_runs_read_them() {
2780        let root = scratch("tapes-once");
2781        let map = plan("192.0.2.1", "80");
2782        let page = "HTTP/1.1 200 OK\r\nServer: a-distinctive-server\r\n\r\n";
2783
2784        let runs: Vec<_> = (0..13)
2785            .map(|n| passive_run(&format!("http-{n}"), 80, &[page]))
2786            .chain([passive_run("ssh-banner", 22, &["SSH-2.0-x\r\n"])])
2787            .collect();
2788
2789        let directory = {
2790            let mut journal = begin(&root, &map);
2791            journal.record_detections(&runs).expect("records the runs");
2792            let directory = journal.directory().to_path_buf();
2793            journal.close().expect("closes");
2794            directory
2795        };
2796
2797        let written = fs::read_to_string(directory.join(DETECTIONS)).expect("reads the file");
2798        assert_eq!(written.matches("a-distinctive-server").count(), 1);
2799        assert_eq!(read_detections(&directory).expect("reads back"), runs);
2800    }
2801
2802    /// A journal written one run per line, each with its own copy of what it read, is
2803    /// still replayed beside runs written one port per line.
2804    #[test]
2805    fn tapes_written_one_run_to_a_line_still_read_back() {
2806        use std::io::Write;
2807
2808        let root = scratch("tapes-legacy");
2809        let map = plan("192.0.2.1", "80");
2810        let grouped = passive_run("http-title", 80, &["HTTP/1.1 200 OK\r\n\r\n"]);
2811        let alone = passive_run("ssh-banner", 22, &["SSH-2.0-x\r\n"]);
2812
2813        let directory = {
2814            let mut journal = begin(&root, &map);
2815            journal
2816                .record_detections(std::slice::from_ref(&grouped))
2817                .expect("records the run");
2818            let directory = journal.directory().to_path_buf();
2819            journal.close().expect("closes");
2820            directory
2821        };
2822        let mut file = fs::OpenOptions::new()
2823            .append(true)
2824            .open(directory.join(DETECTIONS))
2825            .expect("opens the file");
2826        let line = serde_json::to_string(&alone).expect("serialises");
2827        writeln!(file, "{line}").expect("appends a run on its own");
2828
2829        assert_eq!(
2830            read_detections(&directory).expect("reads back"),
2831            [grouped, alone]
2832        );
2833    }
2834
2835    /// A host written more than once comes back whole: the scan that found it and the
2836    /// scan that classified its ports both wrote.
2837    #[test]
2838    fn repeated_records_for_one_host_are_folded_together() {
2839        use crate::model::host::HostStatus;
2840        use crate::model::port::{Port, PortState, Protocol};
2841
2842        let root = scratch("folded");
2843        let map = plan("192.0.2.1", "80");
2844        let ip: std::net::IpAddr = "192.0.2.1".parse().expect("an address");
2845
2846        let directory = {
2847            let mut journal = begin(&root, &map);
2848
2849            let mut found = Host::new(ip);
2850            found.set_status(HostStatus::Up);
2851            journal.record_hosts(&[found]).expect("the host answered");
2852
2853            let mut classified = Host::new(ip);
2854            classified.add_port(Port::new(80, Protocol::Tcp, PortState::Open));
2855            journal
2856                .record_hosts(&[classified])
2857                .expect("and then its ports");
2858
2859            let directory = journal.directory().to_path_buf();
2860            journal.close().expect("closes");
2861            directory
2862        };
2863
2864        let (journal, _) =
2865            Journal::resume(&directory, &ports(&map), Privilege::Raw).expect("resumes");
2866
2867        let restored = journal.restored();
2868        assert_eq!(restored.len(), 1, "one address is one host");
2869        assert!(restored[0].is_alive(), "the first record's status survived");
2870        assert_eq!(restored[0].port_count(), 1, "the second record's port too");
2871    }
2872
2873    /// `fe80::1` is a different machine on every link, so two gateways found under that
2874    /// number on two interfaces stay two hosts, as in the live report and a diff.
2875    #[test]
2876    fn link_locals_on_two_interfaces_come_back_as_two_hosts() {
2877        use crate::model::host::HostStatus;
2878        use crate::model::ip::scoped::Zone;
2879
2880        let root = scratch("zones");
2881        let map = plan("192.0.2.1", "80");
2882        let ip: std::net::IpAddr = "fe80::1".parse().expect("an address");
2883
2884        let directory = {
2885            let mut journal = begin(&root, &map);
2886            let hosts: Vec<Host> = [Zone::new(4, "en0"), Zone::new(9, "en7")]
2887                .into_iter()
2888                .map(|zone| {
2889                    let mut host = Host::new(ip);
2890                    host.set_status(HostStatus::Up);
2891                    host.set_zone(zone);
2892                    host
2893                })
2894                .collect();
2895            journal.record_hosts(&hosts).expect("records both");
2896
2897            let directory = journal.directory().to_path_buf();
2898            journal.close().expect("closes");
2899            directory
2900        };
2901
2902        assert_eq!(report(&directory).expect("reads").host_count(), 2);
2903
2904        let (journal, _) =
2905            Journal::resume(&directory, &ports(&map), Privilege::Raw).expect("resumes");
2906        let mut zones: Vec<&str> = journal
2907            .restored()
2908            .iter()
2909            .filter_map(|host| host.zone().map(Zone::name))
2910            .collect();
2911        zones.sort_unstable();
2912        assert_eq!(zones, ["en0", "en7"], "each restored on its own link");
2913    }
2914
2915    /// A journal that stopped before it found anything reads as no findings.
2916    #[test]
2917    fn a_journal_with_no_findings_restores_nothing() {
2918        let root = scratch("no-findings");
2919        let map = plan("192.0.2.1", "80");
2920
2921        let journal = begin(&root, &map);
2922        let directory = journal.directory().to_path_buf();
2923        journal.close().expect("closes");
2924
2925        let (journal, _) =
2926            Journal::resume(&directory, &ports(&map), Privilege::Raw).expect("resumes");
2927        assert!(journal.restored().is_empty());
2928    }
2929
2930    /// A finished scan comes back out of its journal as the report it produced:
2931    /// everything the end of a run prints is drawn from the hosts and the phases.
2932    #[test]
2933    fn a_journal_reads_back_as_the_report_its_scan_produced() {
2934        let root = scratch("replay");
2935        let map = plan("192.0.2.1-192.0.2.4", "80,443");
2936        let original = crate::export::fixture::report();
2937
2938        let directory = {
2939            let mut journal = begin(&root, &map);
2940            let hosts: Vec<Host> = original.hosts().cloned().collect();
2941            journal.record_hosts(&hosts).expect("records hosts");
2942            journal
2943                .record_phases(original.phases())
2944                .expect("records phases");
2945
2946            let directory = journal.directory().to_path_buf();
2947            journal.close().expect("closes");
2948            directory
2949        };
2950
2951        let replayed = report(&directory).expect("reads");
2952
2953        assert_eq!(replayed.host_count(), original.host_count());
2954        assert_eq!(replayed.phases().len(), original.phases().len());
2955        assert_eq!(
2956            replayed.summary().hosts_alive,
2957            original.summary().hosts_alive
2958        );
2959        assert_eq!(replayed.summary().ports_open, original.summary().ports_open);
2960
2961        // A phase carries what the run covered and how it went.
2962        let (before, after) = (&original.phases()[0], &replayed.phases()[0]);
2963        assert_eq!(after.kind(), before.kind());
2964        assert_eq!(after.privilege(), before.privilege());
2965        assert_eq!(after.failures().len(), before.failures().len());
2966        assert_eq!(
2967            replayed.is_partial(),
2968            original.is_partial(),
2969            "a scan that left ground uncovered has to still say so"
2970        );
2971    }
2972
2973    /// A report read back names the engine that ran the scan, not the one reading it.
2974    #[test]
2975    fn a_replayed_report_names_the_engine_that_ran_the_scan() {
2976        let root = scratch("replay-version");
2977        let map = plan("192.0.2.1", "80");
2978
2979        let directory = {
2980            let journal = begin(&root, &map);
2981            let directory = journal.directory().to_path_buf();
2982            journal.close().expect("closes");
2983            directory
2984        };
2985
2986        // The manifest as an earlier build would have left it.
2987        let path = directory.join(MANIFEST);
2988        let mut manifest: JournalManifest =
2989            serde_json::from_str(&fs::read_to_string(&path).expect("reads")).expect("parses");
2990        manifest.engine_version = "0.1.0".to_string();
2991        fs::write(&path, serde_json::to_vec(&manifest).expect("encodes")).expect("writes");
2992
2993        let replayed = report(&directory).expect("reads");
2994
2995        assert_eq!(replayed.engine_version(), "0.1.0");
2996        assert_ne!(
2997            replayed.engine_version(),
2998            crate::report::ENGINE_VERSION,
2999            "this build's version must not have been stamped over the record's"
3000        );
3001    }
3002
3003    /// A cursor that exists and cannot be read is not a scan that settled nothing.
3004    ///
3005    /// A `sudo` scan whose cursor was not handed over leaves exactly this. Reported as zero,
3006    /// every finished scan would list as untouched; reported as unknown, a reader is told to go
3007    /// and look.
3008    #[cfg(unix)]
3009    #[test]
3010    fn a_cursor_that_cannot_be_read_is_not_a_scan_that_settled_nothing() {
3011        use std::os::unix::fs::PermissionsExt;
3012
3013        let root = scratch("unreadable-cursor");
3014        let map = plan("192.0.2.1-192.0.2.4", "80");
3015
3016        let directory = {
3017            let mut journal = begin(&root, &map);
3018            let settlements = Settlements::default();
3019            settlements.record(Outcome::Answered { position: 0 });
3020            journal.checkpoint(&settlements).expect("checkpoints");
3021
3022            let directory = journal.directory().to_path_buf();
3023            journal.close().expect("closes");
3024            directory
3025        };
3026
3027        // Readable first, so the difference below is the permission, not the contents.
3028        let listed = list(&root).expect("lists").entries;
3029        assert_eq!(listed[0].settled(), Some(1));
3030
3031        fs::set_permissions(directory.join(CURSOR), fs::Permissions::from_mode(0o000))
3032            .expect("removes every permission");
3033
3034        let listed = list(&root)
3035            .expect("a journal it cannot read must not hide the rest")
3036            .entries;
3037        assert_eq!(listed.len(), 1, "the journal still lists");
3038        assert_eq!(
3039            listed[0].settled(),
3040            None,
3041            "unreadable is not zero: zero is a claim about the scan"
3042        );
3043        assert!(
3044            !listed[0].is_complete(),
3045            "and nothing that cannot be read may be called finished"
3046        );
3047
3048        // Left readable, so a failure here leaves a file the next run can clean up.
3049        let _ = fs::set_permissions(directory.join(CURSOR), fs::Permissions::from_mode(0o600));
3050    }
3051
3052    /// What a scan learns after its last checkpoint reaches the file.
3053    ///
3054    /// The enrichment passes (OS identification, the echo probe, traceroute) run at the end,
3055    /// often after the last timer checkpoint. If the closing write missed them, a replayed
3056    /// report would silently lack them.
3057    #[tokio::test]
3058    async fn what_a_scan_learns_after_a_checkpoint_still_reaches_the_file() {
3059        use crate::model::host::{HostStatus, StatusProtocol, StatusReason};
3060
3061        let root = scratch("late-findings");
3062        let map = plan("192.0.2.1", "80");
3063        let journal = begin(&root, &map);
3064        let directory = journal.directory().to_path_buf();
3065
3066        let (_session, ctx) = crate::scanner::session::ScanSession::new();
3067        let mut ticker = spawn_checkpoints(journal, ctx.progress());
3068
3069        // What the liveness pass found.
3070        let ip: std::net::IpAddr = "192.0.2.1".parse().expect("an address");
3071        ctx.update_host(ip, |host| {
3072            host.set_status(HostStatus::Up);
3073            host.record_evidence(
3074                HostStatus::Up,
3075                StatusReason::new(StatusProtocol::Arp, "answered"),
3076            );
3077        });
3078
3079        // A checkpoint lands, taking that and leaving nothing behind.
3080        ticker.checkpointed().await;
3081
3082        // Then the enrichment finds something else.
3083        ctx.update_host(ip, |host| {
3084            host.record_evidence(
3085                HostStatus::Up,
3086                StatusReason::new(StatusProtocol::IcmpEcho, "echo answered"),
3087            );
3088        });
3089
3090        ticker.finish(&[]).await;
3091
3092        let restored = read_findings(&directory).expect("reads");
3093        assert_eq!(restored.len(), 1, "one host was scanned");
3094
3095        let protocols: Vec<_> = restored[0]
3096            .reasons()
3097            .iter()
3098            .map(|reason| reason.protocol.clone())
3099            .collect();
3100        assert!(
3101            protocols.contains(&StatusProtocol::IcmpEcho),
3102            "what the scan learned last was lost: {protocols:?}"
3103        );
3104    }
3105
3106    fn aged(id: &str, created_at: SystemTime, settled: u64, total: u128) -> Entry {
3107        Entry {
3108            directory: PathBuf::from(id),
3109            manifest: JournalManifest {
3110                links: Vec::new(),
3111                journal_version: crate::journal::JOURNAL_VERSION,
3112                id: id.to_string(),
3113                engine_version: "0.0.0".to_string(),
3114                created_at,
3115                kind: crate::record::wire::scan_kind_name(ScanKind::PortScan).to_owned(),
3116                plan: crate::journal::manifest::PlanFingerprint::of(
3117                    &ports(&plan("192.0.2.1", "80")),
3118                    Privilege::Raw,
3119                ),
3120                targets: crate::record::PlanRecord::from(&plan("192.0.2.1", "80")),
3121                technique: TcpScanTechnique::Syn.name().to_owned(),
3122                sweep: false,
3123                privilege: Privilege::Raw,
3124                total_targets: total,
3125                order_seed: None,
3126                summary: String::new(),
3127            },
3128            checkpoint: Some(Checkpoint {
3129                watermark: settled,
3130                settled_above: Vec::new(),
3131                walked: None,
3132            }),
3133            lock: LockState::Free,
3134        }
3135    }
3136
3137    fn ago(seconds: u64) -> SystemTime {
3138        now() - Duration::from_secs(seconds)
3139    }
3140
3141    fn now() -> SystemTime {
3142        SystemTime::UNIX_EPOCH + Duration::from_secs(1_000_000)
3143    }
3144
3145    /// Finished journals age out; unfinished ones do not.
3146    #[test]
3147    fn age_removes_finished_journals_and_keeps_unfinished_ones() {
3148        let entries = vec![
3149            aged("finished-old", ago(100), 1, 1),
3150            aged("unfinished-old", ago(100), 0, 1),
3151            aged("finished-new", ago(1), 1, 1),
3152        ];
3153
3154        let retention = Retention {
3155            completed_for: Some(Duration::from_secs(50)),
3156            incomplete_for: None,
3157            keep_at_most: None,
3158            unreadable: false,
3159        };
3160
3161        assert_eq!(retention.expired(&entries, now()), vec![0]);
3162    }
3163
3164    /// A journal something is writing is never selected, however old.
3165    #[test]
3166    fn a_live_journal_is_never_pruned() {
3167        let mut entries = vec![aged("held", ago(10_000), 1, 1)];
3168        entries[0].lock = LockState::Held {
3169            pid: 1,
3170            last_beat: Duration::from_secs(1),
3171        };
3172
3173        let retention = Retention {
3174            completed_for: Some(Duration::from_secs(1)),
3175            incomplete_for: Some(Duration::from_secs(1)),
3176            keep_at_most: Some(0),
3177            unreadable: false,
3178        };
3179
3180        assert!(retention.expired(&entries, now()).is_empty());
3181    }
3182
3183    /// The cap takes finished journals before unfinished ones, and the oldest first within
3184    /// each.
3185    #[test]
3186    fn the_cap_removes_duplicates_before_unfinished_work() {
3187        // `list` yields newest first.
3188        let entries = vec![
3189            aged("finished-new", ago(1), 1, 1),
3190            aged("unfinished-mid", ago(2), 0, 1),
3191            aged("finished-old", ago(3), 1, 1),
3192        ];
3193
3194        let retention = Retention {
3195            completed_for: None,
3196            incomplete_for: None,
3197            keep_at_most: Some(2),
3198            unreadable: false,
3199        };
3200
3201        let removed = retention.expired(&entries, now());
3202        assert_eq!(removed, vec![2], "the older of the two finished ones");
3203
3204        // Tighter still, and the unfinished one is the last to go.
3205        let retention = Retention {
3206            keep_at_most: Some(1),
3207            ..retention
3208        };
3209        let removed = retention.expired(&entries, now());
3210        assert_eq!(
3211            removed,
3212            vec![0, 2],
3213            "both finished ones, the unfinished kept"
3214        );
3215    }
3216
3217    /// Keeping everything keeps everything.
3218    #[test]
3219    fn keeping_everything_removes_nothing() {
3220        let entries = vec![
3221            aged("ancient", ago(900_000), 1, 1),
3222            aged("also-ancient", ago(900_000), 0, 1),
3223        ];
3224
3225        assert!(
3226            Retention::keep_everything()
3227                .expired(&entries, now())
3228                .is_empty()
3229        );
3230    }
3231
3232    /// The default keeps an unfinished scan whatever its age, and lets a finished one go
3233    /// after a month.
3234    #[test]
3235    fn the_default_favours_unfinished_work() {
3236        let two_months = Duration::from_secs(60 * 24 * 60 * 60);
3237        let created = now() - two_months;
3238
3239        let entries = vec![
3240            aged("finished", created, 1, 1),
3241            aged("unfinished", created, 0, 1),
3242        ];
3243
3244        assert_eq!(Retention::default().expired(&entries, now()), vec![0]);
3245    }
3246
3247    /// End to end: a prune removes what the policy chose and names it.
3248    #[test]
3249    fn pruning_removes_what_the_policy_selected() {
3250        let root = scratch("retention");
3251        let map = plan("192.0.2.1", "80");
3252
3253        // Finished, so the default lets it go once it is old enough.
3254        let mut journal = begin(&root, &map);
3255        let settlements = Settlements::default();
3256        settlements.record(Outcome::Answered { position: 0 });
3257        journal.checkpoint(&settlements).expect("checkpoints");
3258        let id = journal.manifest().id.clone();
3259        journal.close().expect("closes");
3260
3261        // Nothing is old enough for the default yet.
3262        let untouched = prune(&root, &Retention::default()).expect("prunes");
3263        assert!(untouched.removed.is_empty());
3264        assert_eq!(list(&root).expect("lists").entries.len(), 1);
3265
3266        // A policy that keeps nothing finished takes it, and names it.
3267        let swept = prune(
3268            &root,
3269            &Retention {
3270                completed_for: Some(Duration::ZERO),
3271                ..Retention::default()
3272            },
3273        )
3274        .expect("prunes");
3275
3276        assert_eq!(swept.removed, vec![id]);
3277        assert!(swept.held.is_empty());
3278        assert!(list(&root).expect("lists").entries.is_empty());
3279
3280        std::fs::remove_dir_all(&root).ok();
3281    }
3282
3283    /// A host whose primary address is promoted mid-scan is still one host.
3284    ///
3285    /// Local discovery calls `Host::consider_primary_ip` when a better address turns up, so the
3286    /// same machine can be written under a link-local address and then a global one.
3287    #[test]
3288    fn a_host_written_under_two_addresses_comes_back_as_one() {
3289        use crate::model::host::HostStatus;
3290
3291        let root = scratch("promoted");
3292        let map = plan("192.0.2.1", "80");
3293        let link_local: std::net::IpAddr = "fe80::1".parse().expect("an address");
3294        let global: std::net::IpAddr = "2001:db8::1".parse().expect("an address");
3295
3296        let directory = {
3297            let mut journal = begin(&root, &map);
3298
3299            // Found over its link-local address first.
3300            let mut early = Host::new(link_local);
3301            early.set_status(HostStatus::Up);
3302            journal.record_hosts(&[early]).expect("records");
3303
3304            // Then a global address takes the primary slot.
3305            let mut promoted = Host::new(link_local);
3306            promoted.add_ip(global);
3307            assert!(
3308                promoted.consider_primary_ip(global),
3309                "the global address should lead"
3310            );
3311            promoted.set_hostname(Some("router.example".to_string()));
3312            journal.record_hosts(&[promoted]).expect("records again");
3313
3314            let directory = journal.directory().to_path_buf();
3315            journal.close().expect("closes");
3316            directory
3317        };
3318
3319        let (journal, _) =
3320            Journal::resume(&directory, &ports(&map), Privilege::Raw).expect("resumes");
3321
3322        let restored = journal.restored();
3323        assert_eq!(
3324            restored.len(),
3325            1,
3326            "one machine, written twice: {:?}",
3327            restored.iter().map(Host::primary_ip).collect::<Vec<_>>()
3328        );
3329        assert!(restored[0].is_alive(), "the first record's status survived");
3330        assert_eq!(restored[0].hostname(), Some("router.example"));
3331    }
3332
3333    /// Journal files are private to their owner.
3334    #[cfg(unix)]
3335    #[test]
3336    fn every_file_a_journal_writes_is_private() {
3337        use std::os::unix::fs::PermissionsExt;
3338
3339        let root = scratch("modes");
3340        let map = plan("192.0.2.1", "80");
3341
3342        let mut journal = begin(&root, &map);
3343        let settlements = Settlements::default();
3344        settlements.record(Outcome::Answered { position: 0 });
3345        journal
3346            .record(
3347                &[Host::new("192.0.2.1".parse().expect("an address"))],
3348                &settlements,
3349            )
3350            .expect("records");
3351        let directory = journal.directory().to_path_buf();
3352
3353        let mode_of = |path: &Path| {
3354            fs::metadata(path)
3355                .unwrap_or_else(|e| panic!("{path:?}: {e}"))
3356                .permissions()
3357                .mode()
3358                & 0o777
3359        };
3360
3361        assert_eq!(mode_of(&directory), 0o700, "the directory");
3362        for name in [MANIFEST, CURSOR, HOSTS, PHASES, LOCK] {
3363            assert_eq!(mode_of(&directory.join(name)), 0o600, "{name}");
3364        }
3365
3366        // And again after a heartbeat, which replaces the lock file.
3367        journal.checkpoint(&settlements).expect("beats");
3368        assert_eq!(mode_of(&directory.join(LOCK)), 0o600, "after a heartbeat");
3369
3370        journal.close().expect("closes");
3371        std::fs::remove_dir_all(&root).ok();
3372    }
3373
3374    /// The two append-only files refuse a link where the file should be, like every other
3375    /// journal file. The directory belongs to the invoking user and the writer is usually root,
3376    /// so a planted link would have root append an engagement's addresses wherever it
3377    /// points.
3378    #[cfg(unix)]
3379    #[test]
3380    fn appending_refuses_a_link_where_a_journal_file_should_be() {
3381        let root = scratch("nofollow");
3382        let map = plan("192.0.2.1", "80");
3383        let mut journal = begin(&root, &map);
3384        let directory = journal.directory().to_path_buf();
3385
3386        let elsewhere = root.join("not-the-journals-to-touch");
3387        fs::write(&elsewhere, b"untouched").expect("writes");
3388
3389        for name in [HOSTS, PHASES] {
3390            fs::remove_file(directory.join(name)).expect("removes");
3391            std::os::unix::fs::symlink(&elsewhere, directory.join(name)).expect("links");
3392        }
3393
3394        let phases = crate::export::fixture::report().phases().to_vec();
3395        assert!(
3396            journal
3397                .record_hosts(&[Host::new("192.0.2.1".parse().expect("an address"))])
3398                .is_err(),
3399            "a link was appended to as the findings file"
3400        );
3401        assert!(
3402            journal.record_phases(&phases).is_err(),
3403            "a link was appended to as the phases file"
3404        );
3405        assert_eq!(
3406            fs::read(&elsewhere).expect("reads"),
3407            b"untouched",
3408            "the file behind the link was written through"
3409        );
3410
3411        journal.close().expect("closes");
3412        std::fs::remove_dir_all(&root).ok();
3413    }
3414
3415    /// A host recorded more than once reads back with the round trips its last record
3416    /// holds, not every record's added together. Each record carries the whole window as it
3417    /// stood, so folding them as new samples would count early round trips twice and skew the
3418    /// average.
3419    #[test]
3420    fn a_host_recorded_twice_keeps_the_round_trips_its_last_record_holds() {
3421        use std::time::Duration;
3422
3423        let root = scratch("latency");
3424        let mut journal = begin(&root, &plan("192.0.2.1", "80"));
3425        let mut host = Host::new("192.0.2.1".parse().expect("an address"));
3426        host.add_rtts([Duration::from_millis(1), Duration::from_millis(2)]);
3427        journal.record_hosts(&[host.clone()]).expect("records");
3428        host.add_rtts([Duration::from_millis(6), Duration::from_millis(7)]);
3429        journal
3430            .record_hosts(&[host.clone()])
3431            .expect("records again");
3432        let directory = journal.directory().to_path_buf();
3433        journal.close().expect("closes");
3434
3435        let read = report(&directory).expect("reads");
3436        std::fs::remove_dir_all(&root).ok();
3437        let [restored] = read.hosts().collect::<Vec<_>>()[..] else {
3438            panic!("one host on record");
3439        };
3440        let figures = |host: &Host| (host.min_rtt(), host.average_rtt(), host.max_rtt());
3441        assert_eq!(figures(restored), figures(&host), "min, average, max");
3442        assert_eq!(
3443            restored.telemetry().history().len(),
3444            host.telemetry().history().len(),
3445            "round trips held"
3446        );
3447    }
3448
3449    /// A host written at every checkpoint reads back as the host it is, every field of
3450    /// it. Each record repeats its host's fields, so a field folded as though each record were
3451    /// new would count the repeats. Checked over everything a record keeps, for the richest
3452    /// hosts the schema describes, grown between checkpoints with a port changing at each.
3453    #[test]
3454    fn a_host_recorded_at_every_checkpoint_reads_back_as_the_host_it_is() {
3455        use crate::model::confidence::Confidence;
3456        use crate::model::finding::{DetectionClass, DetectionId, Finding, Severity, Version};
3457        use crate::model::host::{OsEvidence, OsSource};
3458
3459        let root = scratch("repeated");
3460        let mut journal = begin(&root, &plan("203.0.113.1-203.0.113.9", "22,80,443,445"));
3461        let mut hosts: Vec<Host> = crate::export::fixture::report().hosts().cloned().collect();
3462        for checkpoint in 0..3u64 {
3463            let finding = Finding::new(
3464                DetectionId::new(format!("check-{checkpoint}"), Version::new(1, 0, 0), "")
3465                    .expect("an id"),
3466                "A check",
3467                Severity::Low,
3468                Confidence::Certain,
3469                DetectionClass::Passive,
3470            )
3471            .expect("a finding");
3472            for host in &mut hosts {
3473                host.add_rtt(Duration::from_micros(700 + 100 * checkpoint));
3474                host.record_os_evidence(OsEvidence {
3475                    source: OsSource::ServiceBanner,
3476                    family: Some("Linux".to_string()),
3477                    device: None,
3478                    vendor: None,
3479                    product: None,
3480                    version: None,
3481                    kernel: None,
3482                    arch: None,
3483                    cpe: None,
3484                    confidence: 0.5,
3485                    evidence: format!("banner read at checkpoint {checkpoint}"),
3486                });
3487                host.add_finding(finding.clone());
3488                let first = host
3489                    .ports()
3490                    .next()
3491                    .map(|port| (port.number(), port.protocol()));
3492                if let Some((number, protocol)) = first {
3493                    host.add_port_finding(number, protocol, finding.clone());
3494                }
3495            }
3496            journal.record_hosts(&hosts).expect("records");
3497        }
3498        let directory = journal.directory().to_path_buf();
3499        journal.close().expect("closes");
3500
3501        let mut read = read_findings(&directory).expect("reads");
3502        std::fs::remove_dir_all(&root).ok();
3503        read.sort_by_key(Host::primary_ip);
3504        hosts.sort_by_key(Host::primary_ip);
3505        let as_recorded = |hosts: &[Host]| {
3506            hosts
3507                .iter()
3508                .map(|host| serde_json::to_value(HostRecord::from(host)).expect("serialises"))
3509                .collect::<Vec<_>>()
3510        };
3511        assert_eq!(as_recorded(&read), as_recorded(&hosts));
3512    }
3513
3514    /// A journal from a newer format is passed over by name and with why: the listing
3515    /// still holds every journal it can read.
3516    #[test]
3517    fn a_journal_from_a_newer_format_is_passed_over_by_name() {
3518        let root = scratch("passed-over");
3519        let map = plan("192.0.2.1", "80");
3520        let readable = begin(&root, &map);
3521        readable.close().expect("closes");
3522        let newer = begin(&root, &map);
3523        let newer_dir = newer.directory().to_path_buf();
3524        newer.close().expect("closes");
3525        age_forward(&newer_dir);
3526
3527        let listing = list(&root).expect("lists");
3528        std::fs::remove_dir_all(&root).ok();
3529
3530        assert_eq!(listing.entries.len(), 1, "the readable journal lists");
3531        let [passed] = listing.passed_over.as_slice() else {
3532            panic!("one passed over: {:?}", listing.passed_over);
3533        };
3534        assert_eq!(passed.directory, newer_dir);
3535        assert_eq!(
3536            Some(passed.name.as_str()),
3537            newer_dir.file_name().and_then(|name| name.to_str())
3538        );
3539        assert_eq!(
3540            passed.why,
3541            Unlisted::NewerFormat {
3542                found: super::super::JOURNAL_VERSION + 1
3543            }
3544        );
3545    }
3546
3547    /// A prune takes an unreadable journal only when asked to, by directory name, and
3548    /// never a link. Such a journal is usually a newer build's, which still reads it.
3549    #[cfg(unix)]
3550    #[test]
3551    fn a_prune_takes_unreadable_journals_only_when_asked_and_never_a_link() {
3552        let root = scratch("prune-unreadable");
3553        let elsewhere = scratch("prune-unreadable-elsewhere");
3554        let map = plan("192.0.2.1", "80");
3555        let newer = begin(&root, &map);
3556        let newer_dir = newer.directory().to_path_buf();
3557        newer.close().expect("closes");
3558        age_forward(&newer_dir);
3559        let target = begin(&elsewhere, &map);
3560        let target_dir = target.directory().to_path_buf();
3561        target.close().expect("closes");
3562        std::os::unix::fs::symlink(&target_dir, root.join("06LINKED00000000")).expect("links");
3563
3564        let ordinary = prune(&root, &Retention::default()).expect("prunes");
3565        assert!(ordinary.removed.is_empty() && ordinary.held.is_empty());
3566        assert!(newer_dir.exists(), "an ordinary sweep leaves it");
3567
3568        let mut asked = Retention::keep_everything();
3569        asked.unreadable = true;
3570        let pruned = prune(&root, &asked).expect("prunes");
3571        let newer_gone = !newer_dir.exists();
3572        let target_kept = target_dir.exists();
3573        let link_kept = root.join("06LINKED00000000").symlink_metadata().is_ok();
3574        std::fs::remove_dir_all(&root).ok();
3575        std::fs::remove_dir_all(&elsewhere).ok();
3576
3577        assert!(newer_gone, "asked for, it goes");
3578        assert_eq!(pruned.removed.len(), 1, "{pruned:?}");
3579        assert!(pruned.held.is_empty(), "a link is not selected: {pruned:?}");
3580        assert!(
3581            link_kept && target_kept,
3582            "neither the link nor what it names goes"
3583        );
3584    }
3585
3586    /// Rewrites a journal's manifest to claim the next format, as a newer build would.
3587    fn age_forward(directory: &Path) {
3588        let path = directory.join(MANIFEST);
3589        let mut manifest: serde_json::Value =
3590            serde_json::from_str(&fs::read_to_string(&path).expect("reads")).expect("parses");
3591        manifest["journal_version"] = serde_json::json!(super::super::JOURNAL_VERSION + 1);
3592        fs::write(&path, serde_json::to_vec(&manifest).expect("encodes")).expect("writes");
3593    }
3594
3595    /// A link in a root of journals is not listed as a journal. Followed, it would list
3596    /// a record from somewhere else, and under `sudo` have root look wherever it leads.
3597    #[cfg(unix)]
3598    #[test]
3599    fn a_link_in_a_root_of_journals_is_not_listed() {
3600        let root = scratch("listed-link");
3601        let elsewhere = scratch("listed-link-elsewhere");
3602        let journal = begin(&elsewhere, &plan("192.0.2.1", "80"));
3603        let directory = journal.directory().to_path_buf();
3604        journal.close().expect("closes");
3605        std::os::unix::fs::symlink(&directory, root.join("06LINKED00000000")).expect("links");
3606
3607        let listed = list(&root).expect("lists").entries.len();
3608        let where_it_is = list(&elsewhere).expect("lists").entries.len();
3609        std::fs::remove_dir_all(&root).ok();
3610        std::fs::remove_dir_all(&elsewhere).ok();
3611        assert_eq!(listed, 0, "the link was listed as a journal");
3612        assert_eq!(where_it_is, 1, "the journal itself lists");
3613    }
3614
3615    /// Reading a journal refuses a link where one of its files should be, and says so.
3616    ///
3617    /// The directory is the invoking user's and the reader is usually root, so a planted link
3618    /// would have root read whatever the user chose, such as another job's plan or a root-only
3619    /// file.
3620    #[cfg(unix)]
3621    #[test]
3622    fn reading_refuses_a_link_where_a_journal_file_should_be() {
3623        let root = scratch("read-nofollow");
3624        let mut ours = begin(&root, &plan("192.0.2.1", "80"));
3625        ours.record_hosts(&[Host::new("192.0.2.1".parse().expect("an address"))])
3626            .expect("records");
3627        ours.write_cursor(&Checkpoint::default())
3628            .expect("checkpoints");
3629        let directory = ours.directory().to_path_buf();
3630        ours.close().expect("closes");
3631
3632        let mut theirs = begin(&root, &plan("198.51.100.7", "80"));
3633        theirs
3634            .record_hosts(&[Host::new("198.51.100.7".parse().expect("an address"))])
3635            .expect("records");
3636        theirs
3637            .write_cursor(&Checkpoint::new(1, []))
3638            .expect("checkpoints");
3639        let elsewhere = theirs.directory().to_path_buf();
3640        theirs.close().expect("closes");
3641        let text = root.join("not-a-journal-file");
3642        fs::write(&text, b"somebody else's bytes").expect("writes");
3643
3644        let refused = |name: &str, error: String| {
3645            assert!(
3646                error.contains("is a link"),
3647                "{name}: refused, but not as a link: {error}"
3648            );
3649        };
3650        for name in [MANIFEST, CURSOR, HOSTS, PHASES] {
3651            let kept = directory.join(format!("{name}.kept"));
3652            fs::rename(directory.join(name), &kept).expect("moves aside");
3653            std::os::unix::fs::symlink(elsewhere.join(name), directory.join(name)).expect("links");
3654
3655            match Journal::reopen(&directory, Privilege::Raw) {
3656                Ok(_) => panic!("{name}: resumed through a link"),
3657                Err(error) => refused(name, error.to_string()),
3658            }
3659            if matches!(name, MANIFEST | HOSTS | PHASES) {
3660                match report(&directory) {
3661                    Ok(_) => panic!("{name}: reported through a link"),
3662                    Err(error) => refused(name, error.to_string()),
3663                }
3664            }
3665
3666            fs::remove_file(directory.join(name)).expect("removes the link");
3667            fs::rename(&kept, directory.join(name)).expect("restores");
3668        }
3669
3670        // Files read after a job's start are refused the same way.
3671        let journal = begin(&root, &plan("192.0.2.2", "80"));
3672        for name in [DETECTIONS, FINISHED] {
3673            std::os::unix::fs::symlink(&text, journal.directory().join(name)).expect("links");
3674        }
3675        match read_detections(journal.directory()) {
3676            Ok(runs) => panic!("detections read through a link: {runs:?}"),
3677            Err(error) => refused(DETECTIONS, error.to_string()),
3678        }
3679        match journal.finished_hosts() {
3680            Ok(hosts) => panic!("finished hosts read through a link: {hosts:?}"),
3681            Err(error) => refused(FINISHED, error.to_string()),
3682        }
3683        journal.close().expect("closes");
3684
3685        std::fs::remove_dir_all(&root).ok();
3686    }
3687
3688    /// A cursor naming positions the watermark already passed does not make the scan look
3689    /// more finished. `Checkpoint::read` keeps such entries and `Cursor::from_checkpoint`
3690    /// filters them, so the listing must count the same way: `is_complete` is what a retention
3691    /// sweep deletes on.
3692    #[test]
3693    fn a_cursor_repeating_settled_positions_does_not_inflate_the_count() {
3694        let root = scratch("inflated-cursor");
3695        let map = plan("192.0.2.1-192.0.2.4", "80,443");
3696
3697        let directory = {
3698            let journal = begin(&root, &map);
3699            let directory = journal.directory().to_path_buf();
3700            journal.close().expect("closes");
3701            directory
3702        };
3703
3704        // Four of the eight targets settled, and a list naming three of them again, as only
3705        // a damaged file would.
3706        let checkpoint = Checkpoint {
3707            watermark: 4,
3708            settled_above: vec![0, 1, 2, 6],
3709            walked: None,
3710        };
3711        checkpoint
3712            .write_atomically(&directory.join(CURSOR))
3713            .expect("writes");
3714
3715        let entry = list(&root)
3716            .expect("lists")
3717            .entries
3718            .into_iter()
3719            .next()
3720            .expect("one journal");
3721
3722        assert_eq!(
3723            entry.settled(),
3724            Some(5),
3725            "the watermark plus the one position genuinely above it"
3726        );
3727        assert!(
3728            !entry.is_complete(),
3729            "eight targets, five settled: a retention sweep must not take this"
3730        );
3731    }
3732
3733    /// A torn tail followed by another append. The reader discards a torn line only while
3734    /// it is last; under `O_APPEND` the tear would become the prefix of a newline-terminated
3735    /// line, and every later read would fail with `Malformed`. `format`'s
3736    /// `a_torn_final_line_ends_the_journal_without_an_error` covers reading a tear; this covers
3737    /// appending past one.
3738    #[test]
3739    fn an_append_after_a_torn_tail_leaves_the_journal_readable() {
3740        let root = scratch("torn-tail-append");
3741        let map = plan("192.0.2.1-192.0.2.4", "80");
3742        let first: std::net::IpAddr = "192.0.2.1".parse().expect("an address");
3743        let second: std::net::IpAddr = "192.0.2.2".parse().expect("an address");
3744
3745        let directory = {
3746            let mut journal = begin(&root, &map);
3747            journal.record_hosts(&[Host::new(first)]).expect("records");
3748            let directory = journal.directory().to_path_buf();
3749            journal.close().expect("closes");
3750            directory
3751        };
3752
3753        // The crash: a record that reached the file without its newline.
3754        {
3755            use std::io::Write;
3756            let mut file = fs::OpenOptions::new()
3757                .append(true)
3758                .open(directory.join(HOSTS))
3759                .expect("opens");
3760            file.write_all(br#"{"ip":"192.0.2.9","stat"#)
3761                .expect("tears");
3762        }
3763
3764        let (mut journal, _) = Journal::resume(&directory, &ports(&map), Privilege::Raw)
3765            .expect("resumes past the tear");
3766        journal.record_hosts(&[Host::new(second)]).expect("records");
3767        journal.close().expect("closes");
3768
3769        let restored = read_findings(&directory).expect("the journal still reads");
3770        let addresses: Vec<_> = restored.iter().map(Host::primary_ip).collect();
3771        assert_eq!(
3772            addresses,
3773            vec![first, second],
3774            "the whole record before the tear and the one after it both survive"
3775        );
3776    }
3777
3778    /// A header write that did not land. `open_findings` creates the file and writes its
3779    /// header in two steps, so a kill between them leaves a zero-length file. Appending into it
3780    /// would write records under no header, which every later read treats as no findings while
3781    /// the cursor advances.
3782    #[test]
3783    fn an_append_to_a_findings_file_whose_header_never_landed_mends_it() {
3784        let root = scratch("headerless-append");
3785        let map = plan("192.0.2.1", "80");
3786        let address: std::net::IpAddr = "192.0.2.1".parse().expect("an address");
3787
3788        let mut journal = begin(&root, &map);
3789        let directory = journal.directory().to_path_buf();
3790
3791        fs::write(directory.join(HOSTS), b"").expect("empties");
3792
3793        journal
3794            .record_hosts(&[Host::new(address)])
3795            .expect("records");
3796        journal.close().expect("closes");
3797
3798        let restored = read_findings(&directory).expect("reads");
3799        assert_eq!(
3800            restored.len(),
3801            1,
3802            "a record appended under no header is a record nothing reads back"
3803        );
3804    }
3805
3806    /// A file this engine did not write is refused, not given a header, and left exactly
3807    /// as it was, which is why the header is checked before anything is truncated.
3808    #[test]
3809    fn an_append_to_a_file_this_engine_did_not_write_is_refused() {
3810        let root = scratch("foreign-append");
3811        let map = plan("192.0.2.1", "80");
3812
3813        let mut journal = begin(&root, &map);
3814        let directory = journal.directory().to_path_buf();
3815
3816        // Complete lines, so nothing here is a torn tail, just not a journal.
3817        let foreign: &[u8] = b"{\"not\":\"a journal\"}\n{\"still\":\"not\"}";
3818        fs::write(directory.join(HOSTS), foreign).expect("plants");
3819
3820        assert!(
3821            journal
3822                .record_hosts(&[Host::new("192.0.2.1".parse().expect("an address"))])
3823                .is_err(),
3824            "a file with no header was appended to"
3825        );
3826        assert_eq!(
3827            fs::read(directory.join(HOSTS)).expect("reads"),
3828            foreign,
3829            "the file was edited before it was refused"
3830        );
3831
3832        journal.close().expect("closes");
3833    }
3834
3835    /// A journal that could not take its lock leaves nothing behind.
3836    #[test]
3837    fn a_journal_that_cannot_be_locked_leaves_no_directory() {
3838        let root = scratch("unlockable");
3839        let map = plan("192.0.2.1", "80");
3840
3841        // A file where the scan directory would go: the create cannot proceed, and what it
3842        // did get to is undone.
3843        let before = list(&root).expect("lists").entries.len();
3844        assert_eq!(before, 0);
3845
3846        let journal = begin(&root, &map);
3847        let directory = journal.directory().to_path_buf();
3848
3849        // Held by this process, so a second journal over the same directory is refused the
3850        // way real contention is.
3851        assert!(matches!(
3852            Journal::resume(&directory, &ports(&map), Privilege::Raw),
3853            Err(OpenError::Locked(_))
3854        ));
3855        assert!(directory.exists(), "the held journal is untouched");
3856
3857        journal.close().expect("closes");
3858        std::fs::remove_dir_all(&root).ok();
3859    }
3860
3861    /// A host written again carries only the ports that changed, and reads back whole.
3862    #[test]
3863    fn a_host_written_again_carries_only_what_changed_in_it() {
3864        use crate::model::port::{Port, PortState, Protocol};
3865
3866        let root = scratch("delta");
3867        let map = plan("192.0.2.1", "1-400");
3868        let mut journal = begin(&root, &map);
3869        let directory = journal.directory().to_path_buf();
3870        let length = || fs::metadata(directory.join(HOSTS)).expect("stats").len();
3871
3872        let ip: std::net::IpAddr = "192.0.2.1".parse().expect("an address");
3873        let mut host = Host::new(ip);
3874        for number in 1..=400 {
3875            host.add_port(Port::new(number, Protocol::Tcp, PortState::Closed));
3876        }
3877        let before = length();
3878        journal
3879            .record_hosts(std::slice::from_ref(&host))
3880            .expect("records");
3881        let first = length() - before;
3882
3883        host.add_port(Port::new(22, Protocol::Tcp, PortState::Open));
3884        let before = length();
3885        journal
3886            .record_hosts(std::slice::from_ref(&host))
3887            .expect("records");
3888        let second = length() - before;
3889
3890        assert!(
3891            second * 20 < first,
3892            "one port opening appended {second} bytes against the host's {first}"
3893        );
3894        journal.close().expect("closes");
3895
3896        let restored = read_findings(&directory).expect("reads");
3897        assert_eq!(restored.len(), 1);
3898        assert_eq!(restored[0].port_count(), 400, "every port read back");
3899        let open: Vec<u16> = restored[0]
3900            .ports()
3901            .filter(|port| port.state() == PortState::Open)
3902            .map(|port| port.number())
3903            .collect();
3904        assert_eq!(open, [22], "the change read back over what it changed");
3905        std::fs::remove_dir_all(&root).ok();
3906    }
3907
3908    /// What a record supersedes is counted: the previous record's host fields and the
3909    /// earlier record of each port it rewrites, and nothing for a port's first record. The count
3910    /// decides compaction, so missing superseded bytes would let the file grow without bound
3911    /// and counting new findings would rewrite a file with nothing to drop.
3912    #[test]
3913    fn a_record_counts_what_it_supersedes_and_not_what_it_adds() {
3914        use crate::model::port::{Port, PortState, Protocol};
3915
3916        let ip: std::net::IpAddr = "192.0.2.1".parse().expect("an address");
3917        let mut host = Host::new(ip);
3918        host.add_port(Port::new(22, Protocol::Tcp, PortState::Closed));
3919        let mut written = Written::default();
3920
3921        let (_, first) = written.delta(&host).expect("measures").expect("new");
3922        let (rest, port) = (
3923            u64::from(first.rest.length),
3924            u64::from(first.ports[0].1.length),
3925        );
3926        written.update(first);
3927        assert_eq!(written.superseded, 0, "a first record supersedes nothing");
3928
3929        host.add_port(Port::new(80, Protocol::Tcp, PortState::Closed));
3930        let (_, second) = written.delta(&host).expect("measures").expect("a new port");
3931        assert_eq!(second.ports.len(), 1, "only the new port is written");
3932        written.update(second);
3933        assert_eq!(written.superseded, rest, "the host's own fields, again");
3934
3935        host.add_port(Port::new(22, Protocol::Tcp, PortState::Open));
3936        let (_, third) = written
3937            .delta(&host)
3938            .expect("measures")
3939            .expect("a moved port");
3940        let rest_again = written.hosts[&host.scoped_ip()]
3941            .rest
3942            .map_or(0, |mark| u64::from(mark.length));
3943        written.update(third);
3944        assert_eq!(
3945            written.superseded,
3946            rest + rest_again + port,
3947            "and port 22's closed record under its open one"
3948        );
3949    }
3950
3951    /// A failed compaction leaves no partial copy and is not retried at the next
3952    /// checkpoint, since what stopped it (usually a full disk) would stop the next. Here the
3953    /// rename is refused by a directory standing where the findings file goes.
3954    #[test]
3955    fn a_failed_compaction_leaves_nothing_behind_and_waits_before_trying_again() {
3956        let root = scratch("failed-compaction");
3957        let map = plan("192.0.2.1", "80");
3958        let mut journal = begin(&root, &map);
3959        let directory = journal.directory().to_path_buf();
3960        let host = Host::new("192.0.2.1".parse().expect("an address"));
3961        journal
3962            .record_hosts(std::slice::from_ref(&host))
3963            .expect("records");
3964
3965        // As if the file had grown mostly superseded.
3966        journal.written.superseded = 3 * COMPACT_FLOOR;
3967        journal.length = 4 * COMPACT_FLOOR;
3968        assert!(journal.should_compact());
3969
3970        fs::remove_file(directory.join(HOSTS)).expect("removes the findings");
3971        fs::create_dir_all(directory.join(HOSTS).join("occupied")).expect("blocks the name");
3972        assert!(journal.compact(std::slice::from_ref(&host)).is_err());
3973
3974        assert!(
3975            !directory.join(HOSTS).with_extension("jsonl-tmp").exists(),
3976            "the partial copy is left holding the disk's room"
3977        );
3978        assert!(
3979            !journal.should_compact(),
3980            "a compaction that just failed is due again at once"
3981        );
3982        journal.written.superseded += 2 * COMPACT_FLOOR;
3983        assert!(
3984            journal.should_compact(),
3985            "and again once as much is superseded"
3986        );
3987
3988        drop(journal);
3989        std::fs::remove_dir_all(&root).ok();
3990    }
3991
3992    /// A findings file is rewritten once its superseded bytes outgrow the rest, and not
3993    /// while it is small or growing only by new findings.
3994    #[test]
3995    fn a_findings_file_is_compacted_by_what_it_superseded() {
3996        const MIB: u64 = 1024 * 1024;
3997
3998        assert!(
3999            !outgrown(60 * MIB, 0),
4000            "a file that grew by what the scan found holds nothing to drop"
4001        );
4002        assert!(!outgrown(6 * MIB, 3 * MIB), "a small file is left alone");
4003        assert!(outgrown(9 * MIB, 5 * MIB), "past the floor it is rewritten");
4004        assert!(
4005            !outgrown(50 * MIB, 20 * MIB),
4006            "twenty superseded beside thirty live is not yet worth it"
4007        );
4008        assert!(
4009            outgrown(50 * MIB, 26 * MIB),
4010            "twenty-six beside twenty-four is"
4011        );
4012    }
4013
4014    /// A long scan's findings file grows with its duration. Compaction bounds it and loses
4015    /// nothing.
4016    #[test]
4017    fn compaction_bounds_the_findings_file_without_losing_a_host() {
4018        use crate::model::host::HostStatus;
4019        use crate::model::port::{Port, PortState, Protocol};
4020
4021        let root = scratch("compaction");
4022        let map = plan("192.0.2.1-192.0.2.4", "80");
4023        let addresses: Vec<std::net::IpAddr> = (1..=4)
4024            .map(|last| format!("192.0.2.{last}").parse().expect("an address"))
4025            .collect();
4026
4027        let directory = {
4028            let mut journal = begin(&root, &map);
4029
4030            // Four hosts, rewritten many times over, as a long scan does.
4031            let mut live: Vec<Host> = addresses.iter().map(|ip| Host::new(*ip)).collect();
4032            for round in 0..70 {
4033                for host in &mut live {
4034                    host.set_status(HostStatus::Up);
4035                    host.add_port(Port::new(1000 + round, Protocol::Tcp, PortState::Open));
4036                }
4037                journal.record_hosts(&live).expect("records");
4038            }
4039
4040            journal.compact(&live).expect("compacts");
4041            assert!(!journal.should_compact(), "a file just written whole");
4042
4043            let directory = journal.directory().to_path_buf();
4044            journal.close().expect("closes");
4045            directory
4046        };
4047
4048        // The file now holds one record per host.
4049        let lines = std::fs::read_to_string(directory.join(HOSTS))
4050            .expect("reads")
4051            .lines()
4052            .count();
4053        assert_eq!(lines, 1 + 4, "a header and one record each");
4054
4055        // And nothing was lost.
4056        let (journal, _) =
4057            Journal::resume(&directory, &ports(&map), Privilege::Raw).expect("resumes");
4058
4059        let restored = journal.restored();
4060        assert_eq!(restored.len(), 4);
4061        for host in restored {
4062            assert!(host.is_alive());
4063            assert_eq!(host.port_count(), 70, "every round's port survived");
4064        }
4065
4066        std::fs::remove_dir_all(&root).ok();
4067    }
4068
4069    /// A scan can be continued knowing only its id: the plan comes back as recorded, so
4070    /// nothing has to be retyped differently from what ran first.
4071    #[test]
4072    fn a_journal_gives_back_the_plan_it_recorded() {
4073        let root = scratch("reopen");
4074        let map = plan("192.0.2.1-192.0.2.4", "80,443,u:53");
4075
4076        let directory = {
4077            let journal = Journal::create(
4078                &root,
4079                &Plan::port_scan(&map, &Exclusions::none(), TcpScanTechnique::Fin),
4080                Privilege::Raw,
4081                "test",
4082            )
4083            .expect("creates");
4084            let directory = journal.directory().to_path_buf();
4085            journal.close().expect("closes");
4086            directory
4087        };
4088
4089        let (journal, _checkpoint, recovered) =
4090            Journal::reopen(&directory, Privilege::Raw).expect("reopens");
4091
4092        let recovered = recovered.targets().expect("a port scan's plan");
4093        assert_eq!(
4094            recovered.iter().collect::<Vec<_>>(),
4095            map.iter().collect::<Vec<_>>(),
4096            "the same targets, in the same order, so positions still mean what they did"
4097        );
4098        assert_eq!(journal.manifest().technique(), TcpScanTechnique::Fin);
4099    }
4100
4101    /// Continuing a scan under different privileges is refused.
4102    #[test]
4103    fn a_journal_will_not_be_continued_under_different_privileges() {
4104        let root = scratch("reopen-privilege");
4105        let map = plan("192.0.2.1", "80");
4106
4107        let journal =
4108            Journal::create(&root, &ports(&map), Privilege::Raw, "test").expect("creates");
4109        let directory = journal.directory().to_path_buf();
4110        journal.close().expect("closes");
4111
4112        let refused =
4113            Journal::reopen(&directory, Privilege::Connect).expect_err("privileges differ");
4114        assert!(matches!(refused, OpenError::PlanChanged(_)), "{refused:?}");
4115    }
4116
4117    /// The root is created. The chown needs a real elevated process and is a no-op
4118    /// without one; this pins that the call produces a directory `Journal::create` can claim
4119    /// inside.
4120    #[test]
4121    fn preparing_a_root_creates_the_whole_path_to_it() {
4122        let root = scratch("prepare-root")
4123            .join("state")
4124            .join("zond")
4125            .join("journals");
4126        assert!(!root.exists());
4127
4128        prepare_root(&root).expect("the path is created");
4129        assert!(root.is_dir());
4130
4131        // And again on an existing root, the repair path, which must not fail.
4132        prepare_root(&root).expect("an existing root is not an error");
4133
4134        let plan = Plan::listen(vec![crate::model::ip::scoped::Zone::new(3, "en0")]);
4135        Journal::create(&root, &plan, Privilege::Connect, "listening")
4136            .expect("a journal is created inside it")
4137            .close()
4138            .expect("it closes");
4139    }
4140
4141    /// Every directory made on the way to the journals is the user's alone, so nobody
4142    /// else can list which scans there are or, under a lax umask, plant one to be resumed.
4143    #[cfg(unix)]
4144    #[test]
4145    fn preparing_a_root_creates_private_directories() {
4146        use std::os::unix::fs::PermissionsExt;
4147
4148        let home = scratch("prepare-root-private");
4149        prepare_root_with(
4150            &home.join(".local/state/zond/journals"),
4151            true,
4152            None,
4153            |_, _| {},
4154        )
4155        .expect("the path is created");
4156
4157        for below in [
4158            ".local",
4159            ".local/state",
4160            ".local/state/zond",
4161            ".local/state/zond/journals",
4162        ] {
4163            let mode = fs::metadata(home.join(below))
4164                .expect("made")
4165                .permissions()
4166                .mode();
4167            assert_eq!(
4168                format!("{:o}", mode & 0o777),
4169                "700",
4170                "{below} was created readable by others"
4171            );
4172        }
4173        let _ = fs::remove_dir_all(&home);
4174    }
4175
4176    /// Everything a first run under `sudo` creates on the way to its journals is given to
4177    /// the user who ran it, not only the last two directories, so `~/.local` and
4178    /// `~/.local/state` are not left to root.
4179    ///
4180    /// A directory on the way that already existed is given back only if root owns it. The
4181    /// home itself, and anything above it, is never given.
4182    #[test]
4183    fn preparing_a_root_gives_away_what_it_created_and_repairs_what_root_left() {
4184        use Hand::{Give, Reclaim};
4185
4186        let home = scratch("prepare-root-home");
4187        let root = home.join(".local/state/zond/journals");
4188        let prepared = |root: &Path, own: bool, home: Option<&Path>| {
4189            let mut claimed = Vec::new();
4190            prepare_root_with(root, own, home, |path, hand| {
4191                claimed.push((path.to_path_buf(), hand));
4192            })
4193            .expect("the path is created");
4194            claimed
4195        };
4196        let under = |pairs: &[(&str, Hand)]| -> Vec<(PathBuf, Hand)> {
4197            pairs
4198                .iter()
4199                .map(|(below, hand)| (home.join(below), *hand))
4200                .collect()
4201        };
4202
4203        assert_eq!(
4204            prepared(&root, true, Some(&home)),
4205            under(&[
4206                (".local", Give),
4207                (".local/state", Give),
4208                (".local/state/zond", Give),
4209                (".local/state/zond/journals", Give),
4210            ]),
4211            "a directory created for the journal was left to root"
4212        );
4213
4214        // With `~/.local/state` already there, it and `~/.local` are looked at for repair,
4215        // and the two directories this crate owns are given regardless.
4216        fs::remove_dir_all(home.join(".local/state/zond")).expect("removes");
4217        assert_eq!(
4218            prepared(&root, true, Some(&home)),
4219            under(&[
4220                (".local", Reclaim),
4221                (".local/state", Reclaim),
4222                (".local/state/zond", Give),
4223                (".local/state/zond/journals", Give),
4224            ])
4225        );
4226
4227        // A run on nobody else's behalf has no home to repair up to.
4228        assert_eq!(
4229            prepared(&root, true, None),
4230            under(&[
4231                (".local/state/zond", Give),
4232                (".local/state/zond/journals", Give)
4233            ])
4234        );
4235
4236        // A location the caller named is where to write: only the root itself is claimed.
4237        let named = home.join("elsewhere/journals");
4238        assert_eq!(
4239            prepared(&named, false, Some(&home)),
4240            [(named.clone(), Give)]
4241        );
4242
4243        let _ = fs::remove_dir_all(&home);
4244    }
4245
4246    /// A watch is never finished, so its journal always offers a resume and a retention
4247    /// sweep never takes it as done, although its total is zero.
4248    #[test]
4249    fn a_watch_is_never_complete_however_long_it_ran() {
4250        use crate::model::ip::scoped::Zone;
4251
4252        let root = scratch("watch-never-complete");
4253        let plan = Plan::listen(vec![Zone::new(3, "en0")]);
4254        let journal = Journal::create(&root, &plan, Privilege::Raw, "listening on en0")
4255            .expect("a journal is created");
4256        journal.close().expect("it closes");
4257
4258        let entries = list(&root).expect("the root lists").entries;
4259        let entry = entries.first().expect("the watch is there");
4260
4261        assert_eq!(entry.kind(), ScanKind::Listen);
4262        assert!(
4263            !entry.is_complete(),
4264            "another sitting can always be appended to a watch"
4265        );
4266    }
4267}