zond_engine/journal/format.rs
1// Copyright (c) 2026 Erik Lening (hollowpointer) and Contributors
2//
3// This file is part of Zond Engine, licensed under the GNU Affero General
4// Public License, version 3 or later. See the LICENSE file for details, or
5// <https://www.gnu.org/licenses/agpl-3.0.html>.
6//
7// SPDX-License-Identifier: AGPL-3.0-or-later
8
9//! # The journal's on-disk format
10//!
11//! One JSON record per line, the first of which describes the file: the framing, the
12//! versioning rules, and the vocabulary shared with the export path.
13//!
14//! ## Why this format is promised and the export DTOs are not
15//!
16//! [`export::schema`](crate::export::schema) is write-only: it borrows every field and
17//! names every enum with a `&'static str`, so the export path allocates nothing per port,
18//! and `import::json` reads back only the four fields a rescan needs. A journal has to
19//! reconstruct what the first sitting found in full, or the merged report loses it, so this
20//! side owns its data, reads what it wrote, and carries a version.
21//!
22//! ## Compatibility rules (the export document's)
23//!
24//! - **Unknown fields are ignored.** A journal from a newer build stays readable for what
25//! it has in common with this one.
26//! - **An unknown enum string reads downward.** It is one field of one host in a file that
27//! may hold hours of scanning, so it is read as the weakest value the type has: a port
28//! state as `Unasked`, a sitting as a discovery sweep, a severity as `Info`.
29//! [`record`](crate::record) makes that choice field by field and documents each; the
30//! parsers return [`None`].
31//! - **[`JOURNAL_VERSION`] is required and checked.** A journal from a newer version is
32//! refused, since its positions may not mean what this build thinks.
33//! - **A torn final line is discarded.** The writer appends without an fsync per record
34//! (see [`journal`](super)), so a process killed mid-append leaves a partial last line.
35//! That record falls inside the replay interval anyway.
36//!
37//! ## Versioned apart from the export schema
38//!
39//! [`SCHEMA_VERSION`](crate::format::SCHEMA_VERSION) versions the report document and this
40//! versions the journal, so an additive change to an export field does not invalidate every
41//! scan in flight on disk.
42//!
43//! ## One vocabulary
44//!
45//! The wire names are [`record::wire`](crate::record::wire)'s in both directions, so a port
46//! state is spelled the same in a report and in a journal.
47
48use std::io::{BufRead, Write};
49
50use serde::{Deserialize, Serialize};
51
52use crate::format::ENGINE_NAME;
53
54#[doc(inline)]
55pub use super::JOURNAL_VERSION;
56
57/// What went wrong reading a journal.
58#[non_exhaustive]
59#[derive(Debug, thiserror::Error)]
60pub enum JournalError {
61 /// The file did not begin with a header record: it is empty or not a journal this
62 /// engine wrote.
63 #[error("no journal header: this is not a file {ENGINE_NAME} wrote, or it is empty")]
64 NotAJournal,
65
66 /// The journal was written by a build whose format this one predates.
67 #[error(
68 "journal version {found} is newer than this build understands ({understood}); \
69 resume it with the engine that wrote it"
70 )]
71 VersionTooNew {
72 /// The version the file claims.
73 found: u32,
74 /// The newest version this build can read.
75 understood: u32,
76 },
77
78 /// A record could not be parsed, and it was not the torn last line.
79 #[error("journal line {line}: {message}")]
80 Malformed {
81 /// The 1-based line the failure is on.
82 line: u64,
83 /// What `serde` said about it.
84 message: String,
85 },
86
87 /// The file could not be read or written.
88 #[error("journal i/o: {0}")]
89 Io(#[from] std::io::Error),
90}
91
92/// The most one journal file, or one record inside it, may be read into memory.
93///
94/// Journal files are read out of a directory that belongs to a user (see
95/// [`file`](super::file)), often by a root process, so a planted file must not make that
96/// process allocate without limit. The import side applies the same standard.
97///
98/// 256 MiB is far past anything a journal holds. The largest record is one host with every
99/// port on both transports and the service detail behind each.
100pub(super) const MAX_READ_BYTES: u64 = 256 * 1024 * 1024;
101
102/// What a journal file being past [`MAX_READ_BYTES`] is reported as.
103pub(super) fn too_large(what: &str) -> JournalError {
104 JournalError::Malformed {
105 line: 0,
106 message: format!(
107 "{what} is larger than the {MAX_READ_BYTES} bytes a journal is read in; \
108 this is not a file this engine wrote"
109 ),
110 }
111}
112
113/// The first line of every journal file, so a file read on its own says what it
114/// is without a manifest beside it.
115///
116/// `engine` is recorded as well as the version because a journal may be found in a
117/// state directory months later, and should name what produced it.
118#[non_exhaustive]
119#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
120pub struct Header {
121 /// The format version, checked against [`JOURNAL_VERSION`] on open.
122 pub journal_version: u32,
123 /// Always [`ENGINE_NAME`].
124 pub engine: String,
125 /// The engine build that opened this journal, for diagnostics. A resume is gated by
126 /// the plan hash in the manifest, not by this.
127 pub engine_version: String,
128}
129
130impl Header {
131 /// The header this build writes.
132 pub fn current() -> Self {
133 Self {
134 journal_version: JOURNAL_VERSION,
135 engine: ENGINE_NAME.to_string(),
136 engine_version: env!("CARGO_PKG_VERSION").to_string(),
137 }
138 }
139}
140
141/// Writes records one per line, header first.
142///
143/// Does not flush per record beyond what the underlying writer does. Durability is
144/// the caller's policy; [`journal`](super) documents which failures it survives.
145#[derive(Debug)]
146pub struct Writer<W: Write> {
147 inner: W,
148}
149
150impl<W: Write> Writer<W> {
151 /// Begins a journal by writing its header.
152 pub fn create(mut inner: W) -> Result<Self, JournalError> {
153 writeln!(
154 inner,
155 "{}",
156 serde_json::to_string(&Header::current()).map_err(JournalError::json)?
157 )?;
158 Ok(Self { inner })
159 }
160
161 /// Continues a journal that already carries a header, appending to it.
162 ///
163 /// No header is written and none is checked; checking would need a seekable file.
164 /// The caller must have mended a missing header and a torn tail first, as `store`'s
165 /// `open_for_append` does. Appending after a torn tail makes the tear stop being the last
166 /// line, the only place [`Reader`] discards it, and appending to a file with no header
167 /// writes records nothing will read back.
168 pub fn append(inner: W) -> Self {
169 Self { inner }
170 }
171
172 /// Appends one record.
173 ///
174 /// The newline is written after the record, so a process killed part way through
175 /// leaves a line the reader discards. See [`Reader`].
176 pub fn write<T: Serialize>(&mut self, record: &T) -> Result<(), JournalError> {
177 writeln!(
178 self.inner,
179 "{}",
180 serde_json::to_string(record).map_err(JournalError::json)?
181 )?;
182 Ok(())
183 }
184
185 /// Flushes the underlying writer.
186 pub fn flush(&mut self) -> Result<(), JournalError> {
187 self.inner.flush()?;
188 Ok(())
189 }
190}
191
192impl JournalError {
193 /// A record `serde_json` would not write or read, at no line in
194 /// particular.
195 ///
196 /// Not a `From` impl, because that would make `serde_json`'s error type part of
197 /// this crate's public API.
198 pub(crate) fn json(error: serde_json::Error) -> Self {
199 JournalError::Malformed {
200 line: 0,
201 message: error.to_string(),
202 }
203 }
204}
205
206/// Reads one line, refusing one past [`MAX_READ_BYTES`].
207///
208/// The ceiling is applied through `take` before the read, so a file with no newline is
209/// refused after reading the ceiling, not the whole file.
210fn bounded_line<R: BufRead>(
211 inner: &mut R,
212 buffer: &mut String,
213 what: &str,
214) -> Result<usize, JournalError> {
215 let mut limited = std::io::Read::take(&mut *inner, MAX_READ_BYTES + 1);
216 let read = limited.read_line(buffer)?;
217 if read as u64 > MAX_READ_BYTES {
218 return Err(too_large(what));
219 }
220 Ok(read)
221}
222
223/// Reads a journal written by [`Writer`], validating its header on open.
224#[derive(Debug)]
225pub struct Reader<R: BufRead> {
226 inner: R,
227 line: u64,
228 /// Set once a line arrived without a trailing newline. Everything after it is a torn
229 /// write, so the reader stops there.
230 truncated: bool,
231}
232
233impl<R: BufRead> Reader<R> {
234 /// Opens a journal, reading and checking its header.
235 ///
236 /// Fails with [`JournalError::NotAJournal`] on an empty file or one whose
237 /// first line is not a header, and with [`JournalError::VersionTooNew`] on a
238 /// journal this build cannot promise to read.
239 pub fn open(mut inner: R) -> Result<Self, JournalError> {
240 let mut first = String::new();
241 let read = bounded_line(&mut inner, &mut first, "a journal header")?;
242
243 if read == 0 {
244 return Err(JournalError::NotAJournal);
245 }
246
247 let header: Header =
248 serde_json::from_str(first.trim_end()).map_err(|_| JournalError::NotAJournal)?;
249
250 if header.engine != ENGINE_NAME {
251 return Err(JournalError::NotAJournal);
252 }
253
254 if header.journal_version > JOURNAL_VERSION {
255 return Err(JournalError::VersionTooNew {
256 found: header.journal_version,
257 understood: JOURNAL_VERSION,
258 });
259 }
260
261 Ok(Self {
262 inner,
263 line: 1,
264 truncated: false,
265 })
266 }
267
268 /// The 1-based line most recently consumed, for a caller attaching a position to
269 /// an error about a record's contents.
270 pub fn line(&self) -> u64 {
271 self.line
272 }
273
274 /// Reads the next record, or `None` at the end of the journal.
275 ///
276 /// A final line with no trailing newline that does not parse is the torn tail of a
277 /// process that died mid-append, and ends the journal. A newline-terminated line that does
278 /// not parse was written whole and is reported as corruption.
279 pub fn read<T: for<'de> Deserialize<'de>>(&mut self) -> Result<Option<T>, JournalError> {
280 loop {
281 if self.truncated {
282 return Ok(None);
283 }
284
285 let mut buffer = String::new();
286 if bounded_line(&mut self.inner, &mut buffer, "a journal record")? == 0 {
287 return Ok(None);
288 }
289
290 self.line += 1;
291 let complete = buffer.ends_with('\n');
292 self.truncated = !complete;
293
294 let text = buffer.trim_end();
295 // Skipping blank lines lets a journal survive being concatenated.
296 if text.is_empty() {
297 continue;
298 }
299
300 return match serde_json::from_str(text) {
301 Ok(record) => Ok(Some(record)),
302 // Torn tail: the writer died between the record and its newline.
303 Err(_) if !complete => Ok(None),
304 Err(error) => Err(JournalError::Malformed {
305 line: self.line,
306 message: error.to_string(),
307 }),
308 };
309 }
310 }
311}
312
313// ╔════════════════════════════════════════════╗
314// ║ ████████╗███████╗███████╗████████╗███████╗ ║
315// ║ ╚══██╔══╝██╔════╝██╔════╝╚══██╔══╝██╔════╝ ║
316// ║ ██║ █████╗ ███████╗ ██║ ███████╗ ║
317// ║ ██║ ██╔══╝ ╚════██║ ██║ ╚════██║ ║
318// ║ ██║ ███████╗███████║ ██║ ███████║ ║
319// ║ ╚═╝ ╚══════╝╚══════╝ ╚═╝ ╚══════╝ ║
320// ╚════════════════════════════════════════════╝
321
322#[cfg(test)]
323mod tests {
324 use super::*;
325
326 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
327 struct Sample {
328 name: String,
329 count: u32,
330 }
331
332 fn sample(name: &str) -> Sample {
333 Sample {
334 name: name.to_string(),
335 count: 7,
336 }
337 }
338
339 fn journal(records: &[Sample]) -> Vec<u8> {
340 let mut out = Vec::new();
341 let mut writer = Writer::create(&mut out).expect("header");
342 for record in records {
343 writer.write(record).expect("record");
344 }
345 out
346 }
347
348 /// What is written comes back identical.
349 #[test]
350 fn records_round_trip_through_the_framing() {
351 let written = vec![sample("first"), sample("second"), sample("third")];
352 let bytes = journal(&written);
353
354 let mut reader = Reader::open(bytes.as_slice()).expect("opens");
355 let mut read = Vec::new();
356 while let Some(record) = reader.read::<Sample>().expect("reads") {
357 read.push(record);
358 }
359
360 assert_eq!(read, written);
361 }
362
363 /// An empty file is not a journal.
364 #[test]
365 fn an_empty_file_is_not_a_journal() {
366 assert!(matches!(
367 Reader::open(&[][..]),
368 Err(JournalError::NotAJournal)
369 ));
370 }
371
372 /// Somebody else's JSONL does not open as a journal, however well-formed.
373 #[test]
374 fn a_file_without_a_header_is_refused() {
375 let bytes = b"{\"name\":\"first\",\"count\":7}\n";
376 assert!(matches!(
377 Reader::open(&bytes[..]),
378 Err(JournalError::NotAJournal)
379 ));
380 }
381
382 /// A journal from a future build is refused by name: its records may not mean what
383 /// this build would take them to mean.
384 #[test]
385 fn a_newer_journal_version_is_refused() {
386 let header = format!(
387 "{{\"journal_version\":{},\"engine\":\"{ENGINE_NAME}\",\"engine_version\":\"9.9.9\"}}\n",
388 JOURNAL_VERSION + 1
389 );
390
391 match Reader::open(header.as_bytes()) {
392 Err(JournalError::VersionTooNew { found, understood }) => {
393 assert_eq!(found, JOURNAL_VERSION + 1);
394 assert_eq!(understood, JOURNAL_VERSION);
395 }
396 other => panic!("expected a version refusal, got {other:?}"),
397 }
398 }
399
400 /// A process killed mid-append: every whole record before the tear is readable,
401 /// and the torn one is not an error.
402 #[test]
403 fn a_torn_final_line_ends_the_journal_without_an_error() {
404 let mut bytes = journal(&[sample("first"), sample("second")]);
405 bytes.extend_from_slice(b"{\"name\":\"third\",\"cou");
406
407 let mut reader = Reader::open(bytes.as_slice()).expect("opens");
408 let mut read = Vec::new();
409 while let Some(record) = reader.read::<Sample>().expect("reads past the tear") {
410 read.push(record);
411 }
412
413 assert_eq!(read, vec![sample("first"), sample("second")]);
414 }
415
416 /// A line written whole that does not parse is corruption, reported with its line
417 /// number.
418 #[test]
419 fn a_complete_line_that_does_not_parse_is_an_error() {
420 let mut bytes = journal(&[sample("first")]);
421 bytes.extend_from_slice(b"{\"name\":\"second\",\"cou\n");
422
423 let mut reader = Reader::open(bytes.as_slice()).expect("opens");
424 assert_eq!(
425 reader.read::<Sample>().expect("first"),
426 Some(sample("first"))
427 );
428
429 match reader.read::<Sample>() {
430 Err(JournalError::Malformed { line, .. }) => assert_eq!(line, 3),
431 other => panic!("expected a malformed line, got {other:?}"),
432 }
433 }
434
435 /// A journal from a newer build carrying fields this one does not know stays
436 /// readable for what the two have in common.
437 #[test]
438 fn unknown_fields_are_ignored() {
439 let mut bytes = journal(&[]);
440 bytes.extend_from_slice(b"{\"name\":\"first\",\"count\":7,\"arrived\":\"later\"}\n");
441
442 let mut reader = Reader::open(bytes.as_slice()).expect("opens");
443 assert_eq!(
444 reader.read::<Sample>().expect("reads"),
445 Some(sample("first"))
446 );
447 }
448}