Skip to main content

kcode_checksummed_frame_log/
lib.rs

1#![forbid(unsafe_code)]
2
3//! Durable checksummed binary frame logs.
4//!
5//! Files consist of caller-defined magic bytes followed by ordered frames.
6//! Each frame contains a kind byte, a little-endian payload length, the
7//! SHA-256 checksum of the payload, and the payload itself.
8
9use std::{
10    fs::{File, OpenOptions},
11    io::{Read, Write},
12    path::Path,
13};
14
15use anyhow::{Context as _, bail, ensure};
16use sha2::{Digest, Sha256};
17
18const FRAME_HEADER_BYTES: u64 = 1 + 8 + 32;
19const FRAME_HEADER_LEN: usize = 1 + 8 + 32;
20
21#[derive(Clone, Copy, Debug, Eq, PartialEq)]
22pub enum IncompleteTailPolicy {
23    Reject,
24    TruncateAndSync,
25}
26
27#[derive(Clone, Copy, Debug, Eq, PartialEq)]
28pub enum AppendDurability {
29    FileOnly,
30    FileAndParentDirectory,
31}
32
33pub fn create(
34    path: &Path,
35    magic: &[u8],
36    initial_kind: u8,
37    initial_payload: &[u8],
38) -> anyhow::Result<()> {
39    let parent = parent_directory(path);
40    ensure!(
41        parent.is_dir(),
42        "parent directory {} does not exist",
43        parent.display()
44    );
45
46    let mut file = OpenOptions::new()
47        .create_new(true)
48        .write(true)
49        .open(path)
50        .with_context(|| format!("creating {}", path.display()))?;
51    file.write_all(magic)?;
52    write_frame(&mut file, initial_kind, initial_payload)?;
53    file.sync_all()
54        .with_context(|| format!("synchronizing {}", path.display()))?;
55    sync_directory(parent)
56}
57
58pub fn visit_frames<F>(
59    path: &Path,
60    expected_magic: &[u8],
61    tail_policy: IncompleteTailPolicy,
62    mut visitor: F,
63) -> anyhow::Result<()>
64where
65    F: FnMut(u8, &[u8]) -> anyhow::Result<()>,
66{
67    let mut options = OpenOptions::new();
68    options.read(true);
69    if tail_policy == IncompleteTailPolicy::TruncateAndSync {
70        options.write(true);
71    }
72    let mut file = options
73        .open(path)
74        .with_context(|| format!("opening {}", path.display()))?;
75
76    let mut actual_magic = vec![0_u8; expected_magic.len()];
77    file.read_exact(&mut actual_magic)
78        .with_context(|| format!("reading magic from {}", path.display()))?;
79    ensure!(
80        actual_magic == expected_magic,
81        "{} has unexpected magic",
82        path.display()
83    );
84
85    let file_len = file.metadata()?.len();
86    let mut cursor =
87        u64::try_from(expected_magic.len()).context("magic length does not fit file offsets")?;
88
89    while cursor < file_len {
90        if file_len - cursor < FRAME_HEADER_BYTES {
91            return handle_incomplete_tail(&file, path, cursor, tail_policy, "frame header");
92        }
93
94        let mut header = [0_u8; FRAME_HEADER_LEN];
95        file.read_exact(&mut header)?;
96        let kind = header[0];
97        let payload_len = u64::from_le_bytes(
98            header[1..9]
99                .try_into()
100                .expect("frame length field has eight bytes"),
101        );
102        let frame_end = cursor
103            .checked_add(FRAME_HEADER_BYTES)
104            .and_then(|value| value.checked_add(payload_len))
105            .context("frame length overflow")?;
106
107        if frame_end > file_len {
108            return handle_incomplete_tail(&file, path, cursor, tail_policy, "frame payload");
109        }
110
111        let payload_len =
112            usize::try_from(payload_len).context("frame payload does not fit memory")?;
113        let mut payload = vec![0_u8; payload_len];
114        file.read_exact(&mut payload)?;
115
116        let checksum = Sha256::digest(payload.as_slice());
117        ensure!(
118            checksum[..] == header[9..],
119            "{} has a checksum-invalid complete frame",
120            path.display()
121        );
122
123        visitor(kind, &payload)?;
124        cursor = frame_end;
125    }
126
127    Ok(())
128}
129
130pub fn append(
131    path: &Path,
132    kind: u8,
133    payload: &[u8],
134    durability: AppendDurability,
135) -> anyhow::Result<()> {
136    let mut file = OpenOptions::new()
137        .append(true)
138        .open(path)
139        .with_context(|| format!("opening {} for append", path.display()))?;
140    write_frame(&mut file, kind, payload)?;
141    file.sync_all()
142        .with_context(|| format!("synchronizing {}", path.display()))?;
143
144    if durability == AppendDurability::FileAndParentDirectory {
145        sync_directory(parent_directory(path))?;
146    }
147    Ok(())
148}
149
150fn write_frame(file: &mut File, kind: u8, payload: &[u8]) -> anyhow::Result<()> {
151    let payload_len = u64::try_from(payload.len()).context("payload length does not fit u64")?;
152    let checksum = Sha256::digest(payload);
153
154    file.write_all(&[kind])?;
155    file.write_all(&payload_len.to_le_bytes())?;
156    file.write_all(&checksum)?;
157    file.write_all(payload)?;
158    Ok(())
159}
160
161fn handle_incomplete_tail(
162    file: &File,
163    path: &Path,
164    boundary: u64,
165    policy: IncompleteTailPolicy,
166    part: &str,
167) -> anyhow::Result<()> {
168    match policy {
169        IncompleteTailPolicy::Reject => {
170            bail!("{} has an incomplete trailing {part}", path.display())
171        }
172        IncompleteTailPolicy::TruncateAndSync => {
173            file.set_len(boundary)
174                .with_context(|| format!("truncating {}", path.display()))?;
175            file.sync_all()
176                .with_context(|| format!("synchronizing {}", path.display()))
177        }
178    }
179}
180
181fn parent_directory(path: &Path) -> &Path {
182    path.parent()
183        .filter(|parent| !parent.as_os_str().is_empty())
184        .unwrap_or_else(|| Path::new("."))
185}
186
187fn sync_directory(directory: &Path) -> anyhow::Result<()> {
188    File::open(directory)
189        .with_context(|| format!("opening directory {}", directory.display()))?
190        .sync_all()
191        .with_context(|| format!("synchronizing directory {}", directory.display()))
192}
193
194#[cfg(test)]
195mod tests {
196    use std::{
197        fs::OpenOptions,
198        io::Write as _,
199        path::PathBuf,
200        sync::atomic::{AtomicU64, Ordering},
201        time::{SystemTime, UNIX_EPOCH},
202    };
203
204    use super::*;
205
206    static NEXT_DIRECTORY: AtomicU64 = AtomicU64::new(0);
207
208    struct TestDirectory(PathBuf);
209
210    impl TestDirectory {
211        fn new(label: &str) -> Self {
212            let sequence = NEXT_DIRECTORY.fetch_add(1, Ordering::Relaxed);
213            let nanos = SystemTime::now()
214                .duration_since(UNIX_EPOCH)
215                .expect("system time precedes Unix epoch")
216                .as_nanos();
217            let path = std::env::temp_dir().join(format!(
218                "checksummed-frame-log-{label}-{}-{nanos}-{sequence}",
219                std::process::id()
220            ));
221            std::fs::create_dir(&path).unwrap();
222            Self(path)
223        }
224
225        fn path(&self, name: &str) -> PathBuf {
226            self.0.join(name)
227        }
228    }
229
230    impl Drop for TestDirectory {
231        fn drop(&mut self) {
232            let _ = std::fs::remove_dir_all(&self.0);
233        }
234    }
235
236    fn encoded_frame(kind: u8, payload: &[u8]) -> Vec<u8> {
237        let mut frame = Vec::new();
238        frame.push(kind);
239        frame.extend_from_slice(&(payload.len() as u64).to_le_bytes());
240        frame.extend_from_slice(&Sha256::digest(payload));
241        frame.extend_from_slice(payload);
242        frame
243    }
244
245    #[test]
246    fn create_and_append_use_the_specified_encoding_and_visit_in_order() {
247        let directory = TestDirectory::new("round-trip");
248        let path = directory.path("frames.bin");
249
250        create(&path, b"MAGIC\n", 3, b"first").unwrap();
251        append(
252            &path,
253            9,
254            b"second",
255            AppendDurability::FileAndParentDirectory,
256        )
257        .unwrap();
258
259        let mut expected = b"MAGIC\n".to_vec();
260        expected.extend_from_slice(&encoded_frame(3, b"first"));
261        expected.extend_from_slice(&encoded_frame(9, b"second"));
262        assert_eq!(std::fs::read(&path).unwrap(), expected);
263
264        let mut visited = Vec::new();
265        visit_frames(
266            &path,
267            b"MAGIC\n",
268            IncompleteTailPolicy::Reject,
269            |kind, payload| {
270                visited.push((kind, payload.to_vec()));
271                Ok(())
272            },
273        )
274        .unwrap();
275        assert_eq!(
276            visited,
277            vec![(3, b"first".to_vec()), (9, b"second".to_vec())]
278        );
279    }
280
281    #[test]
282    fn create_requires_an_existing_parent_and_uses_create_new() {
283        let directory = TestDirectory::new("create-new");
284        let missing_path = directory.path("missing/frames.bin");
285        assert!(create(&missing_path, b"M", 1, b"value").is_err());
286        assert!(!directory.path("missing").exists());
287
288        let path = directory.path("frames.bin");
289        create(&path, b"M", 1, b"value").unwrap();
290        let original = std::fs::read(&path).unwrap();
291        assert!(create(&path, b"M", 2, b"replacement").is_err());
292        assert_eq!(std::fs::read(&path).unwrap(), original);
293    }
294
295    #[test]
296    fn reject_preserves_an_incomplete_header_and_truncate_repairs_it() {
297        let directory = TestDirectory::new("header-tail");
298        let path = directory.path("frames.bin");
299        create(&path, b"M", 1, b"complete").unwrap();
300        let boundary = std::fs::metadata(&path).unwrap().len();
301
302        OpenOptions::new()
303            .append(true)
304            .open(&path)
305            .unwrap()
306            .write_all(&[7, 4, 0])
307            .unwrap();
308        let incomplete = std::fs::read(&path).unwrap();
309
310        assert!(visit_frames(&path, b"M", IncompleteTailPolicy::Reject, |_, _| Ok(())).is_err());
311        assert_eq!(std::fs::read(&path).unwrap(), incomplete);
312
313        visit_frames(
314            &path,
315            b"M",
316            IncompleteTailPolicy::TruncateAndSync,
317            |_, _| Ok(()),
318        )
319        .unwrap();
320        assert_eq!(std::fs::metadata(&path).unwrap().len(), boundary);
321    }
322
323    #[test]
324    fn truncate_repairs_an_incomplete_payload() {
325        let directory = TestDirectory::new("payload-tail");
326        let path = directory.path("frames.bin");
327        create(&path, b"M", 1, b"complete").unwrap();
328        let boundary = std::fs::metadata(&path).unwrap().len();
329
330        let mut incomplete = encoded_frame(2, b"missing bytes");
331        incomplete.truncate(FRAME_HEADER_LEN + 3);
332        OpenOptions::new()
333            .append(true)
334            .open(&path)
335            .unwrap()
336            .write_all(&incomplete)
337            .unwrap();
338
339        visit_frames(
340            &path,
341            b"M",
342            IncompleteTailPolicy::TruncateAndSync,
343            |_, _| Ok(()),
344        )
345        .unwrap();
346        assert_eq!(std::fs::metadata(&path).unwrap().len(), boundary);
347    }
348
349    #[test]
350    fn complete_checksum_failure_is_never_repaired() {
351        let directory = TestDirectory::new("checksum");
352        let path = directory.path("frames.bin");
353        create(&path, b"M", 1, b"payload").unwrap();
354
355        let mut bytes = std::fs::read(&path).unwrap();
356        let payload_offset = 1 + FRAME_HEADER_LEN;
357        bytes[payload_offset] ^= 0xff;
358        std::fs::write(&path, &bytes).unwrap();
359
360        assert!(
361            visit_frames(
362                &path,
363                b"M",
364                IncompleteTailPolicy::TruncateAndSync,
365                |_, _| Ok(())
366            )
367            .is_err()
368        );
369        assert_eq!(std::fs::read(&path).unwrap(), bytes);
370    }
371
372    #[test]
373    fn visitor_error_prevents_later_tail_repair() {
374        let directory = TestDirectory::new("visitor-error");
375        let path = directory.path("frames.bin");
376        create(&path, b"M", 1, b"complete").unwrap();
377        OpenOptions::new()
378            .append(true)
379            .open(&path)
380            .unwrap()
381            .write_all(&[2, 1])
382            .unwrap();
383        let original = std::fs::read(&path).unwrap();
384
385        let error = visit_frames(
386            &path,
387            b"M",
388            IncompleteTailPolicy::TruncateAndSync,
389            |_, _| anyhow::bail!("visitor stopped"),
390        )
391        .unwrap_err();
392        assert!(error.to_string().contains("visitor stopped"));
393        assert_eq!(std::fs::read(&path).unwrap(), original);
394    }
395
396    #[test]
397    fn magic_mismatch_fails_before_visiting() {
398        let directory = TestDirectory::new("magic");
399        let path = directory.path("frames.bin");
400        create(&path, b"RIGHT", 1, b"payload").unwrap();
401
402        let mut called = false;
403        assert!(
404            visit_frames(&path, b"WRONG", IncompleteTailPolicy::Reject, |_, _| {
405                called = true;
406                Ok(())
407            })
408            .is_err()
409        );
410        assert!(!called);
411    }
412
413    #[test]
414    fn append_does_not_validate_existing_bytes() {
415        let directory = TestDirectory::new("unvalidated-append");
416        let path = directory.path("frames.bin");
417        std::fs::write(&path, b"not a frame log").unwrap();
418
419        append(&path, 4, b"payload", AppendDurability::FileOnly).unwrap();
420
421        let mut expected = b"not a frame log".to_vec();
422        expected.extend_from_slice(&encoded_frame(4, b"payload"));
423        assert_eq!(std::fs::read(path).unwrap(), expected);
424    }
425}