1#![forbid(unsafe_code)]
2
3use std::{
10 fs::{File, OpenOptions},
11 io::{Read, Write},
12 path::Path,
13};
14
15use anyhow::{Context as _, bail, ensure};
16use sha2::{Digest, Sha256};
17
18const FRAME_HEADER_BYTES: u64 = 1 + 8 + 32;
19const FRAME_HEADER_LEN: usize = 1 + 8 + 32;
20
21#[derive(Clone, Copy, Debug, Eq, PartialEq)]
22pub enum IncompleteTailPolicy {
23 Reject,
24 TruncateAndSync,
25}
26
27#[derive(Clone, Copy, Debug, Eq, PartialEq)]
28pub enum AppendDurability {
29 FileOnly,
30 FileAndParentDirectory,
31}
32
33pub fn create(
34 path: &Path,
35 magic: &[u8],
36 initial_kind: u8,
37 initial_payload: &[u8],
38) -> anyhow::Result<()> {
39 let parent = parent_directory(path);
40 ensure!(
41 parent.is_dir(),
42 "parent directory {} does not exist",
43 parent.display()
44 );
45
46 let mut file = OpenOptions::new()
47 .create_new(true)
48 .write(true)
49 .open(path)
50 .with_context(|| format!("creating {}", path.display()))?;
51 file.write_all(magic)?;
52 write_frame(&mut file, initial_kind, initial_payload)?;
53 file.sync_all()
54 .with_context(|| format!("synchronizing {}", path.display()))?;
55 sync_directory(parent)
56}
57
58pub fn visit_frames<F>(
59 path: &Path,
60 expected_magic: &[u8],
61 tail_policy: IncompleteTailPolicy,
62 mut visitor: F,
63) -> anyhow::Result<()>
64where
65 F: FnMut(u8, &[u8]) -> anyhow::Result<()>,
66{
67 let mut options = OpenOptions::new();
68 options.read(true);
69 if tail_policy == IncompleteTailPolicy::TruncateAndSync {
70 options.write(true);
71 }
72 let mut file = options
73 .open(path)
74 .with_context(|| format!("opening {}", path.display()))?;
75
76 let mut actual_magic = vec![0_u8; expected_magic.len()];
77 file.read_exact(&mut actual_magic)
78 .with_context(|| format!("reading magic from {}", path.display()))?;
79 ensure!(
80 actual_magic == expected_magic,
81 "{} has unexpected magic",
82 path.display()
83 );
84
85 let file_len = file.metadata()?.len();
86 let mut cursor =
87 u64::try_from(expected_magic.len()).context("magic length does not fit file offsets")?;
88
89 while cursor < file_len {
90 if file_len - cursor < FRAME_HEADER_BYTES {
91 return handle_incomplete_tail(&file, path, cursor, tail_policy, "frame header");
92 }
93
94 let mut header = [0_u8; FRAME_HEADER_LEN];
95 file.read_exact(&mut header)?;
96 let kind = header[0];
97 let payload_len = u64::from_le_bytes(
98 header[1..9]
99 .try_into()
100 .expect("frame length field has eight bytes"),
101 );
102 let frame_end = cursor
103 .checked_add(FRAME_HEADER_BYTES)
104 .and_then(|value| value.checked_add(payload_len))
105 .context("frame length overflow")?;
106
107 if frame_end > file_len {
108 return handle_incomplete_tail(&file, path, cursor, tail_policy, "frame payload");
109 }
110
111 let payload_len =
112 usize::try_from(payload_len).context("frame payload does not fit memory")?;
113 let mut payload = vec![0_u8; payload_len];
114 file.read_exact(&mut payload)?;
115
116 let checksum = Sha256::digest(payload.as_slice());
117 ensure!(
118 checksum[..] == header[9..],
119 "{} has a checksum-invalid complete frame",
120 path.display()
121 );
122
123 visitor(kind, &payload)?;
124 cursor = frame_end;
125 }
126
127 Ok(())
128}
129
130pub fn append(
131 path: &Path,
132 kind: u8,
133 payload: &[u8],
134 durability: AppendDurability,
135) -> anyhow::Result<()> {
136 let mut file = OpenOptions::new()
137 .append(true)
138 .open(path)
139 .with_context(|| format!("opening {} for append", path.display()))?;
140 write_frame(&mut file, kind, payload)?;
141 file.sync_all()
142 .with_context(|| format!("synchronizing {}", path.display()))?;
143
144 if durability == AppendDurability::FileAndParentDirectory {
145 sync_directory(parent_directory(path))?;
146 }
147 Ok(())
148}
149
150fn write_frame(file: &mut File, kind: u8, payload: &[u8]) -> anyhow::Result<()> {
151 let payload_len = u64::try_from(payload.len()).context("payload length does not fit u64")?;
152 let checksum = Sha256::digest(payload);
153
154 file.write_all(&[kind])?;
155 file.write_all(&payload_len.to_le_bytes())?;
156 file.write_all(&checksum)?;
157 file.write_all(payload)?;
158 Ok(())
159}
160
161fn handle_incomplete_tail(
162 file: &File,
163 path: &Path,
164 boundary: u64,
165 policy: IncompleteTailPolicy,
166 part: &str,
167) -> anyhow::Result<()> {
168 match policy {
169 IncompleteTailPolicy::Reject => {
170 bail!("{} has an incomplete trailing {part}", path.display())
171 }
172 IncompleteTailPolicy::TruncateAndSync => {
173 file.set_len(boundary)
174 .with_context(|| format!("truncating {}", path.display()))?;
175 file.sync_all()
176 .with_context(|| format!("synchronizing {}", path.display()))
177 }
178 }
179}
180
181fn parent_directory(path: &Path) -> &Path {
182 path.parent()
183 .filter(|parent| !parent.as_os_str().is_empty())
184 .unwrap_or_else(|| Path::new("."))
185}
186
187fn sync_directory(directory: &Path) -> anyhow::Result<()> {
188 File::open(directory)
189 .with_context(|| format!("opening directory {}", directory.display()))?
190 .sync_all()
191 .with_context(|| format!("synchronizing directory {}", directory.display()))
192}
193
194#[cfg(test)]
195mod tests {
196 use std::{
197 fs::OpenOptions,
198 io::Write as _,
199 path::PathBuf,
200 sync::atomic::{AtomicU64, Ordering},
201 time::{SystemTime, UNIX_EPOCH},
202 };
203
204 use super::*;
205
206 static NEXT_DIRECTORY: AtomicU64 = AtomicU64::new(0);
207
208 struct TestDirectory(PathBuf);
209
210 impl TestDirectory {
211 fn new(label: &str) -> Self {
212 let sequence = NEXT_DIRECTORY.fetch_add(1, Ordering::Relaxed);
213 let nanos = SystemTime::now()
214 .duration_since(UNIX_EPOCH)
215 .expect("system time precedes Unix epoch")
216 .as_nanos();
217 let path = std::env::temp_dir().join(format!(
218 "checksummed-frame-log-{label}-{}-{nanos}-{sequence}",
219 std::process::id()
220 ));
221 std::fs::create_dir(&path).unwrap();
222 Self(path)
223 }
224
225 fn path(&self, name: &str) -> PathBuf {
226 self.0.join(name)
227 }
228 }
229
230 impl Drop for TestDirectory {
231 fn drop(&mut self) {
232 let _ = std::fs::remove_dir_all(&self.0);
233 }
234 }
235
236 fn encoded_frame(kind: u8, payload: &[u8]) -> Vec<u8> {
237 let mut frame = Vec::new();
238 frame.push(kind);
239 frame.extend_from_slice(&(payload.len() as u64).to_le_bytes());
240 frame.extend_from_slice(&Sha256::digest(payload));
241 frame.extend_from_slice(payload);
242 frame
243 }
244
245 #[test]
246 fn create_and_append_use_the_specified_encoding_and_visit_in_order() {
247 let directory = TestDirectory::new("round-trip");
248 let path = directory.path("frames.bin");
249
250 create(&path, b"MAGIC\n", 3, b"first").unwrap();
251 append(
252 &path,
253 9,
254 b"second",
255 AppendDurability::FileAndParentDirectory,
256 )
257 .unwrap();
258
259 let mut expected = b"MAGIC\n".to_vec();
260 expected.extend_from_slice(&encoded_frame(3, b"first"));
261 expected.extend_from_slice(&encoded_frame(9, b"second"));
262 assert_eq!(std::fs::read(&path).unwrap(), expected);
263
264 let mut visited = Vec::new();
265 visit_frames(
266 &path,
267 b"MAGIC\n",
268 IncompleteTailPolicy::Reject,
269 |kind, payload| {
270 visited.push((kind, payload.to_vec()));
271 Ok(())
272 },
273 )
274 .unwrap();
275 assert_eq!(
276 visited,
277 vec![(3, b"first".to_vec()), (9, b"second".to_vec())]
278 );
279 }
280
281 #[test]
282 fn create_requires_an_existing_parent_and_uses_create_new() {
283 let directory = TestDirectory::new("create-new");
284 let missing_path = directory.path("missing/frames.bin");
285 assert!(create(&missing_path, b"M", 1, b"value").is_err());
286 assert!(!directory.path("missing").exists());
287
288 let path = directory.path("frames.bin");
289 create(&path, b"M", 1, b"value").unwrap();
290 let original = std::fs::read(&path).unwrap();
291 assert!(create(&path, b"M", 2, b"replacement").is_err());
292 assert_eq!(std::fs::read(&path).unwrap(), original);
293 }
294
295 #[test]
296 fn reject_preserves_an_incomplete_header_and_truncate_repairs_it() {
297 let directory = TestDirectory::new("header-tail");
298 let path = directory.path("frames.bin");
299 create(&path, b"M", 1, b"complete").unwrap();
300 let boundary = std::fs::metadata(&path).unwrap().len();
301
302 OpenOptions::new()
303 .append(true)
304 .open(&path)
305 .unwrap()
306 .write_all(&[7, 4, 0])
307 .unwrap();
308 let incomplete = std::fs::read(&path).unwrap();
309
310 assert!(visit_frames(&path, b"M", IncompleteTailPolicy::Reject, |_, _| Ok(())).is_err());
311 assert_eq!(std::fs::read(&path).unwrap(), incomplete);
312
313 visit_frames(
314 &path,
315 b"M",
316 IncompleteTailPolicy::TruncateAndSync,
317 |_, _| Ok(()),
318 )
319 .unwrap();
320 assert_eq!(std::fs::metadata(&path).unwrap().len(), boundary);
321 }
322
323 #[test]
324 fn truncate_repairs_an_incomplete_payload() {
325 let directory = TestDirectory::new("payload-tail");
326 let path = directory.path("frames.bin");
327 create(&path, b"M", 1, b"complete").unwrap();
328 let boundary = std::fs::metadata(&path).unwrap().len();
329
330 let mut incomplete = encoded_frame(2, b"missing bytes");
331 incomplete.truncate(FRAME_HEADER_LEN + 3);
332 OpenOptions::new()
333 .append(true)
334 .open(&path)
335 .unwrap()
336 .write_all(&incomplete)
337 .unwrap();
338
339 visit_frames(
340 &path,
341 b"M",
342 IncompleteTailPolicy::TruncateAndSync,
343 |_, _| Ok(()),
344 )
345 .unwrap();
346 assert_eq!(std::fs::metadata(&path).unwrap().len(), boundary);
347 }
348
349 #[test]
350 fn complete_checksum_failure_is_never_repaired() {
351 let directory = TestDirectory::new("checksum");
352 let path = directory.path("frames.bin");
353 create(&path, b"M", 1, b"payload").unwrap();
354
355 let mut bytes = std::fs::read(&path).unwrap();
356 let payload_offset = 1 + FRAME_HEADER_LEN;
357 bytes[payload_offset] ^= 0xff;
358 std::fs::write(&path, &bytes).unwrap();
359
360 assert!(
361 visit_frames(
362 &path,
363 b"M",
364 IncompleteTailPolicy::TruncateAndSync,
365 |_, _| Ok(())
366 )
367 .is_err()
368 );
369 assert_eq!(std::fs::read(&path).unwrap(), bytes);
370 }
371
372 #[test]
373 fn visitor_error_prevents_later_tail_repair() {
374 let directory = TestDirectory::new("visitor-error");
375 let path = directory.path("frames.bin");
376 create(&path, b"M", 1, b"complete").unwrap();
377 OpenOptions::new()
378 .append(true)
379 .open(&path)
380 .unwrap()
381 .write_all(&[2, 1])
382 .unwrap();
383 let original = std::fs::read(&path).unwrap();
384
385 let error = visit_frames(
386 &path,
387 b"M",
388 IncompleteTailPolicy::TruncateAndSync,
389 |_, _| anyhow::bail!("visitor stopped"),
390 )
391 .unwrap_err();
392 assert!(error.to_string().contains("visitor stopped"));
393 assert_eq!(std::fs::read(&path).unwrap(), original);
394 }
395
396 #[test]
397 fn magic_mismatch_fails_before_visiting() {
398 let directory = TestDirectory::new("magic");
399 let path = directory.path("frames.bin");
400 create(&path, b"RIGHT", 1, b"payload").unwrap();
401
402 let mut called = false;
403 assert!(
404 visit_frames(&path, b"WRONG", IncompleteTailPolicy::Reject, |_, _| {
405 called = true;
406 Ok(())
407 })
408 .is_err()
409 );
410 assert!(!called);
411 }
412
413 #[test]
414 fn append_does_not_validate_existing_bytes() {
415 let directory = TestDirectory::new("unvalidated-append");
416 let path = directory.path("frames.bin");
417 std::fs::write(&path, b"not a frame log").unwrap();
418
419 append(&path, 4, b"payload", AppendDurability::FileOnly).unwrap();
420
421 let mut expected = b"not a frame log".to_vec();
422 expected.extend_from_slice(&encoded_frame(4, b"payload"));
423 assert_eq!(std::fs::read(path).unwrap(), expected);
424 }
425}