Skip to main content

zond_engine/journal/
format.rs

1// Copyright (c) 2026 Erik Lening (hollowpointer) and Contributors
2//
3// This file is part of Zond Engine, licensed under the GNU Affero General
4// Public License, version 3 or later. See the LICENSE file for details, or
5// <https://www.gnu.org/licenses/agpl-3.0.html>.
6//
7// SPDX-License-Identifier: AGPL-3.0-or-later
8
9//! # The journal's on-disk format
10//!
11//! One JSON record per line, the first of which describes the file: the framing, the
12//! versioning rules, and the vocabulary shared with the export path.
13//!
14//! ## Why this format is promised and the export DTOs are not
15//!
16//! [`export::schema`](crate::export::schema) is write-only: it borrows every field and
17//! names every enum with a `&'static str`, so the export path allocates nothing per port,
18//! and `import::json` reads back only the four fields a rescan needs. A journal has to
19//! reconstruct what the first sitting found in full, or the merged report loses it, so this
20//! side owns its data, reads what it wrote, and carries a version.
21//!
22//! ## Compatibility rules (the export document's)
23//!
24//! - **Unknown fields are ignored.** A journal from a newer build stays readable for what
25//!   it has in common with this one.
26//! - **An unknown enum string reads downward.** It is one field of one host in a file that
27//!   may hold hours of scanning, so it is read as the weakest value the type has: a port
28//!   state as `Unasked`, a sitting as a discovery sweep, a severity as `Info`.
29//!   [`record`](crate::record) makes that choice field by field and documents each; the
30//!   parsers return [`None`].
31//! - **[`JOURNAL_VERSION`] is required and checked.** A journal from a newer version is
32//!   refused, since its positions may not mean what this build thinks.
33//! - **A torn final line is discarded.** The writer appends without an fsync per record
34//!   (see [`journal`](super)), so a process killed mid-append leaves a partial last line.
35//!   That record falls inside the replay interval anyway.
36//!
37//! ## Versioned apart from the export schema
38//!
39//! [`SCHEMA_VERSION`](crate::format::SCHEMA_VERSION) versions the report document and this
40//! versions the journal, so an additive change to an export field does not invalidate every
41//! scan in flight on disk.
42//!
43//! ## One vocabulary
44//!
45//! The wire names are [`record::wire`](crate::record::wire)'s in both directions, so a port
46//! state is spelled the same in a report and in a journal.
47
48use std::io::{BufRead, Write};
49
50use serde::{Deserialize, Serialize};
51
52use crate::format::ENGINE_NAME;
53
54#[doc(inline)]
55pub use super::JOURNAL_VERSION;
56
57/// What went wrong reading a journal.
58#[non_exhaustive]
59#[derive(Debug, thiserror::Error)]
60pub enum JournalError {
61    /// The file did not begin with a header record: it is empty or not a journal this
62    /// engine wrote.
63    #[error("no journal header: this is not a file {ENGINE_NAME} wrote, or it is empty")]
64    NotAJournal,
65
66    /// The journal was written by a build whose format this one predates.
67    #[error(
68        "journal version {found} is newer than this build understands ({understood}); \
69         resume it with the engine that wrote it"
70    )]
71    VersionTooNew {
72        /// The version the file claims.
73        found: u32,
74        /// The newest version this build can read.
75        understood: u32,
76    },
77
78    /// A record could not be parsed, and it was not the torn last line.
79    #[error("journal line {line}: {message}")]
80    Malformed {
81        /// The 1-based line the failure is on.
82        line: u64,
83        /// What `serde` said about it.
84        message: String,
85    },
86
87    /// The file could not be read or written.
88    #[error("journal i/o: {0}")]
89    Io(#[from] std::io::Error),
90}
91
92/// The most one journal file, or one record inside it, may be read into memory.
93///
94/// Journal files are read out of a directory that belongs to a user (see
95/// [`file`](super::file)), often by a root process, so a planted file must not make that
96/// process allocate without limit. The import side applies the same standard.
97///
98/// 256 MiB is far past anything a journal holds. The largest record is one host with every
99/// port on both transports and the service detail behind each.
100pub(super) const MAX_READ_BYTES: u64 = 256 * 1024 * 1024;
101
102/// What a journal file being past [`MAX_READ_BYTES`] is reported as.
103pub(super) fn too_large(what: &str) -> JournalError {
104    JournalError::Malformed {
105        line: 0,
106        message: format!(
107            "{what} is larger than the {MAX_READ_BYTES} bytes a journal is read in; \
108             this is not a file this engine wrote"
109        ),
110    }
111}
112
113/// The first line of every journal file, so a file read on its own says what it
114/// is without a manifest beside it.
115///
116/// `engine` is recorded as well as the version because a journal may be found in a
117/// state directory months later, and should name what produced it.
118#[non_exhaustive]
119#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
120pub struct Header {
121    /// The format version, checked against [`JOURNAL_VERSION`] on open.
122    pub journal_version: u32,
123    /// Always [`ENGINE_NAME`].
124    pub engine: String,
125    /// The engine build that opened this journal, for diagnostics. A resume is gated by
126    /// the plan hash in the manifest, not by this.
127    pub engine_version: String,
128}
129
130impl Header {
131    /// The header this build writes.
132    pub fn current() -> Self {
133        Self {
134            journal_version: JOURNAL_VERSION,
135            engine: ENGINE_NAME.to_string(),
136            engine_version: env!("CARGO_PKG_VERSION").to_string(),
137        }
138    }
139}
140
141/// Writes records one per line, header first.
142///
143/// Does not flush per record beyond what the underlying writer does. Durability is
144/// the caller's policy; [`journal`](super) documents which failures it survives.
145#[derive(Debug)]
146pub struct Writer<W: Write> {
147    inner: W,
148}
149
150impl<W: Write> Writer<W> {
151    /// Begins a journal by writing its header.
152    pub fn create(mut inner: W) -> Result<Self, JournalError> {
153        writeln!(
154            inner,
155            "{}",
156            serde_json::to_string(&Header::current()).map_err(JournalError::json)?
157        )?;
158        Ok(Self { inner })
159    }
160
161    /// Continues a journal that already carries a header, appending to it.
162    ///
163    /// No header is written and none is checked; checking would need a seekable file.
164    /// The caller must have mended a missing header and a torn tail first, as `store`'s
165    /// `open_for_append` does. Appending after a torn tail makes the tear stop being the last
166    /// line, the only place [`Reader`] discards it, and appending to a file with no header
167    /// writes records nothing will read back.
168    pub fn append(inner: W) -> Self {
169        Self { inner }
170    }
171
172    /// Appends one record.
173    ///
174    /// The newline is written after the record, so a process killed part way through
175    /// leaves a line the reader discards. See [`Reader`].
176    pub fn write<T: Serialize>(&mut self, record: &T) -> Result<(), JournalError> {
177        writeln!(
178            self.inner,
179            "{}",
180            serde_json::to_string(record).map_err(JournalError::json)?
181        )?;
182        Ok(())
183    }
184
185    /// Flushes the underlying writer.
186    pub fn flush(&mut self) -> Result<(), JournalError> {
187        self.inner.flush()?;
188        Ok(())
189    }
190}
191
192impl JournalError {
193    /// A record `serde_json` would not write or read, at no line in
194    /// particular.
195    ///
196    /// Not a `From` impl, because that would make `serde_json`'s error type part of
197    /// this crate's public API.
198    pub(crate) fn json(error: serde_json::Error) -> Self {
199        JournalError::Malformed {
200            line: 0,
201            message: error.to_string(),
202        }
203    }
204}
205
206/// Reads one line, refusing one past [`MAX_READ_BYTES`].
207///
208/// The ceiling is applied through `take` before the read, so a file with no newline is
209/// refused after reading the ceiling, not the whole file.
210fn bounded_line<R: BufRead>(
211    inner: &mut R,
212    buffer: &mut String,
213    what: &str,
214) -> Result<usize, JournalError> {
215    let mut limited = std::io::Read::take(&mut *inner, MAX_READ_BYTES + 1);
216    let read = limited.read_line(buffer)?;
217    if read as u64 > MAX_READ_BYTES {
218        return Err(too_large(what));
219    }
220    Ok(read)
221}
222
223/// Reads a journal written by [`Writer`], validating its header on open.
224#[derive(Debug)]
225pub struct Reader<R: BufRead> {
226    inner: R,
227    line: u64,
228    /// Set once a line arrived without a trailing newline. Everything after it is a torn
229    /// write, so the reader stops there.
230    truncated: bool,
231}
232
233impl<R: BufRead> Reader<R> {
234    /// Opens a journal, reading and checking its header.
235    ///
236    /// Fails with [`JournalError::NotAJournal`] on an empty file or one whose
237    /// first line is not a header, and with [`JournalError::VersionTooNew`] on a
238    /// journal this build cannot promise to read.
239    pub fn open(mut inner: R) -> Result<Self, JournalError> {
240        let mut first = String::new();
241        let read = bounded_line(&mut inner, &mut first, "a journal header")?;
242
243        if read == 0 {
244            return Err(JournalError::NotAJournal);
245        }
246
247        let header: Header =
248            serde_json::from_str(first.trim_end()).map_err(|_| JournalError::NotAJournal)?;
249
250        if header.engine != ENGINE_NAME {
251            return Err(JournalError::NotAJournal);
252        }
253
254        if header.journal_version > JOURNAL_VERSION {
255            return Err(JournalError::VersionTooNew {
256                found: header.journal_version,
257                understood: JOURNAL_VERSION,
258            });
259        }
260
261        Ok(Self {
262            inner,
263            line: 1,
264            truncated: false,
265        })
266    }
267
268    /// The 1-based line most recently consumed, for a caller attaching a position to
269    /// an error about a record's contents.
270    pub fn line(&self) -> u64 {
271        self.line
272    }
273
274    /// Reads the next record, or `None` at the end of the journal.
275    ///
276    /// A final line with no trailing newline that does not parse is the torn tail of a
277    /// process that died mid-append, and ends the journal. A newline-terminated line that does
278    /// not parse was written whole and is reported as corruption.
279    pub fn read<T: for<'de> Deserialize<'de>>(&mut self) -> Result<Option<T>, JournalError> {
280        loop {
281            if self.truncated {
282                return Ok(None);
283            }
284
285            let mut buffer = String::new();
286            if bounded_line(&mut self.inner, &mut buffer, "a journal record")? == 0 {
287                return Ok(None);
288            }
289
290            self.line += 1;
291            let complete = buffer.ends_with('\n');
292            self.truncated = !complete;
293
294            let text = buffer.trim_end();
295            // Skipping blank lines lets a journal survive being concatenated.
296            if text.is_empty() {
297                continue;
298            }
299
300            return match serde_json::from_str(text) {
301                Ok(record) => Ok(Some(record)),
302                // Torn tail: the writer died between the record and its newline.
303                Err(_) if !complete => Ok(None),
304                Err(error) => Err(JournalError::Malformed {
305                    line: self.line,
306                    message: error.to_string(),
307                }),
308            };
309        }
310    }
311}
312
313// ╔════════════════════════════════════════════╗
314// ║ ████████╗███████╗███████╗████████╗███████╗ ║
315// ║ ╚══██╔══╝██╔════╝██╔════╝╚══██╔══╝██╔════╝ ║
316// ║    ██║   █████╗  ███████╗   ██║   ███████╗ ║
317// ║    ██║   ██╔══╝  ╚════██║   ██║   ╚════██║ ║
318// ║    ██║   ███████╗███████║   ██║   ███████║ ║
319// ║    ╚═╝   ╚══════╝╚══════╝   ╚═╝   ╚══════╝ ║
320// ╚════════════════════════════════════════════╝
321
322#[cfg(test)]
323mod tests {
324    use super::*;
325
326    #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
327    struct Sample {
328        name: String,
329        count: u32,
330    }
331
332    fn sample(name: &str) -> Sample {
333        Sample {
334            name: name.to_string(),
335            count: 7,
336        }
337    }
338
339    fn journal(records: &[Sample]) -> Vec<u8> {
340        let mut out = Vec::new();
341        let mut writer = Writer::create(&mut out).expect("header");
342        for record in records {
343            writer.write(record).expect("record");
344        }
345        out
346    }
347
348    /// What is written comes back identical.
349    #[test]
350    fn records_round_trip_through_the_framing() {
351        let written = vec![sample("first"), sample("second"), sample("third")];
352        let bytes = journal(&written);
353
354        let mut reader = Reader::open(bytes.as_slice()).expect("opens");
355        let mut read = Vec::new();
356        while let Some(record) = reader.read::<Sample>().expect("reads") {
357            read.push(record);
358        }
359
360        assert_eq!(read, written);
361    }
362
363    /// An empty file is not a journal.
364    #[test]
365    fn an_empty_file_is_not_a_journal() {
366        assert!(matches!(
367            Reader::open(&[][..]),
368            Err(JournalError::NotAJournal)
369        ));
370    }
371
372    /// Somebody else's JSONL does not open as a journal, however well-formed.
373    #[test]
374    fn a_file_without_a_header_is_refused() {
375        let bytes = b"{\"name\":\"first\",\"count\":7}\n";
376        assert!(matches!(
377            Reader::open(&bytes[..]),
378            Err(JournalError::NotAJournal)
379        ));
380    }
381
382    /// A journal from a future build is refused by name: its records may not mean what
383    /// this build would take them to mean.
384    #[test]
385    fn a_newer_journal_version_is_refused() {
386        let header = format!(
387            "{{\"journal_version\":{},\"engine\":\"{ENGINE_NAME}\",\"engine_version\":\"9.9.9\"}}\n",
388            JOURNAL_VERSION + 1
389        );
390
391        match Reader::open(header.as_bytes()) {
392            Err(JournalError::VersionTooNew { found, understood }) => {
393                assert_eq!(found, JOURNAL_VERSION + 1);
394                assert_eq!(understood, JOURNAL_VERSION);
395            }
396            other => panic!("expected a version refusal, got {other:?}"),
397        }
398    }
399
400    /// A process killed mid-append: every whole record before the tear is readable,
401    /// and the torn one is not an error.
402    #[test]
403    fn a_torn_final_line_ends_the_journal_without_an_error() {
404        let mut bytes = journal(&[sample("first"), sample("second")]);
405        bytes.extend_from_slice(b"{\"name\":\"third\",\"cou");
406
407        let mut reader = Reader::open(bytes.as_slice()).expect("opens");
408        let mut read = Vec::new();
409        while let Some(record) = reader.read::<Sample>().expect("reads past the tear") {
410            read.push(record);
411        }
412
413        assert_eq!(read, vec![sample("first"), sample("second")]);
414    }
415
416    /// A line written whole that does not parse is corruption, reported with its line
417    /// number.
418    #[test]
419    fn a_complete_line_that_does_not_parse_is_an_error() {
420        let mut bytes = journal(&[sample("first")]);
421        bytes.extend_from_slice(b"{\"name\":\"second\",\"cou\n");
422
423        let mut reader = Reader::open(bytes.as_slice()).expect("opens");
424        assert_eq!(
425            reader.read::<Sample>().expect("first"),
426            Some(sample("first"))
427        );
428
429        match reader.read::<Sample>() {
430            Err(JournalError::Malformed { line, .. }) => assert_eq!(line, 3),
431            other => panic!("expected a malformed line, got {other:?}"),
432        }
433    }
434
435    /// A journal from a newer build carrying fields this one does not know stays
436    /// readable for what the two have in common.
437    #[test]
438    fn unknown_fields_are_ignored() {
439        let mut bytes = journal(&[]);
440        bytes.extend_from_slice(b"{\"name\":\"first\",\"count\":7,\"arrived\":\"later\"}\n");
441
442        let mut reader = Reader::open(bytes.as_slice()).expect("opens");
443        assert_eq!(
444            reader.read::<Sample>().expect("reads"),
445            Some(sample("first"))
446        );
447    }
448}