1use std::collections::HashMap;
2use std::fs::{self, OpenOptions};
3use std::io::{ErrorKind, Write};
4use std::path::{Path, PathBuf};
5use std::sync::{Mutex, MutexGuard, PoisonError};
6
7use anyhow::{Context, Result, anyhow, bail};
8use base64::Engine;
9use base64::prelude::BASE64_STANDARD;
10use chrono::{DateTime, Utc};
11use serde::{Deserialize, Serialize};
12use sha2::{Digest, Sha256};
13
14use crate::redact::redact;
15use crate::sealed::{self, SealError, SealKeys, seal_error};
16
17pub const MANIFEST_FILE: &str = "manifest.jsonl";
18pub const SEALED_MANIFEST_FILE: &str = "manifest.sealed";
21
22const JOURNAL_FILE: &str = "removal.journal";
23const RESEAL_MARKER: &str = "reseal.pending";
24const BLOB_SUFFIX: &str = ".jsonl.zst";
25const LEGACY_TMP_SUFFIX: &str = ".tmp";
26const COMPACTION_SLACK: usize = 64;
27
28#[derive(Debug, Clone, Copy, PartialEq, Eq)]
29pub enum Archived {
30 Stored,
31 Refreshed,
32 Unchanged,
33 Copy,
34}
35
36#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
37pub struct ArchiveEntry {
38 pub source: String,
39 pub file: String,
40 pub sha256: String,
41 pub archived_at: DateTime<Utc>,
42}
43
44pub struct TranscriptArchive {
53 dir: PathBuf,
54 state: Mutex<State>,
55}
56
57struct State {
58 entries: HashMap<String, ArchiveEntry>,
59 manifest_lines: usize,
60 plaintext_lines: usize,
61 manifest_damaged: bool,
62 unsealed: bool,
63 keys: Option<SealKeys>,
64}
65
66impl State {
67 fn needs_compaction(&self) -> bool {
68 self.manifest_damaged || self.manifest_lines > 2 * self.entries.len() + COMPACTION_SLACK
69 }
70
71 fn plaintext_pending(&self) -> bool {
72 self.keys.is_some() && self.plaintext_lines > 0
73 }
74
75 fn holder_of(&self, sha256: &str) -> Option<&ArchiveEntry> {
76 self.entries.values().find(|e| e.sha256 == sha256)
77 }
78
79 fn references(&self, file: &str) -> bool {
80 self.entries.values().any(|e| e.file == file)
81 }
82}
83
84impl TranscriptArchive {
85 pub fn open(dir: impl Into<PathBuf>) -> Result<Self> {
86 Self::open_with(dir.into(), None)
87 }
88
89 pub fn open_sealed(dir: impl Into<PathBuf>, keys: SealKeys) -> Result<Self> {
92 Self::open_with(dir.into(), Some(keys))
93 }
94
95 fn open_with(dir: PathBuf, keys: Option<SealKeys>) -> Result<Self> {
96 fs::create_dir_all(&dir)
97 .with_context(|| format!("creating the transcript archive at {}", dir.display()))?;
98 let manifest = dir.join(MANIFEST_FILE);
99 if !manifest.exists() {
100 sealed::write_file(&manifest, None, MANIFEST_FILE, b"")
101 .with_context(|| format!("creating {}", manifest.display()))?;
102 }
103 let (state, torn, superseded) = load_manifest(&dir, keys)?;
104 let archive = Self {
105 dir,
106 state: Mutex::new(state),
107 };
108 {
109 let mut state = archive.state();
110 if torn || state.needs_compaction() || state.plaintext_pending() {
111 archive.compact(&mut state)?;
112 }
113 for file in superseded.iter().filter(|f| !state.references(f)) {
114 archive.remove_blob(file);
115 }
116 archive.finish_interrupted_removal(&state)?;
117 }
118 Ok(archive)
119 }
120
121 pub fn holds_unsealed(&self) -> bool {
125 self.state().unsealed
126 }
127
128 pub fn seal_keys(&self) -> Option<SealKeys> {
130 self.state().keys.clone()
131 }
132
133 pub fn set_seal_keys(&self, keys: SealKeys) {
136 self.state().keys = Some(keys);
137 }
138
139 pub fn reseal(&self) -> Result<usize> {
145 if self.state().keys.is_none() {
146 return Err(no_keys());
147 }
148 let mut resealed = 0;
149 for file in self.blob_names()? {
150 let state = self.state();
151 let keys = state.keys.clone().ok_or_else(no_keys)?;
152 let path = self.dir.join(&file);
153 if path.exists() && sealed::reseal_file(&path, &keys, &file)? {
154 resealed += 1;
155 }
156 }
157 let mut state = self.state();
158 let keys = state.keys.clone().ok_or_else(no_keys)?;
159 let journal = self.dir.join(JOURNAL_FILE);
160 if journal.exists() && sealed::reseal_file(&journal, &keys, JOURNAL_FILE)? {
161 resealed += 1;
162 }
163 if !self.manifest_sealed_under(&keys)? {
164 self.compact(&mut state)?;
165 resealed += 1;
166 }
167 self.remove_tmp_files()?;
168 let marker = self.dir.join(RESEAL_MARKER);
169 match fs::remove_file(&marker) {
170 Ok(()) => {}
171 Err(e) if e.kind() == ErrorKind::NotFound => {}
172 Err(e) => return Err(e).with_context(|| format!("removing {}", marker.display())),
173 }
174 state.unsealed = false;
175 Ok(resealed)
176 }
177
178 pub fn remove_where(
182 &self,
183 doomed: impl Fn(&ArchiveEntry) -> bool,
184 ) -> Result<Vec<ArchiveEntry>> {
185 let mut state = self.state();
186 if state.plaintext_pending() {
187 self.compact(&mut state)?;
188 }
189 let removed: Vec<ArchiveEntry> = state
190 .entries
191 .values()
192 .filter(|e| doomed(e))
193 .cloned()
194 .collect();
195 if removed.is_empty() {
196 return Ok(removed);
197 }
198 for entry in &removed {
199 state.entries.remove(&entry.source);
200 }
201 let mut doomed_blobs: Vec<&str> = removed
202 .iter()
203 .map(|e| e.file.as_str())
204 .filter(|file| !state.references(file))
205 .collect();
206 doomed_blobs.sort_unstable();
207 doomed_blobs.dedup();
208 if !doomed_blobs.is_empty() {
209 let journal = self.dir.join(JOURNAL_FILE);
210 sealed::write_file(
211 &journal,
212 state.keys.as_ref(),
213 JOURNAL_FILE,
214 doomed_blobs.join("\n").as_bytes(),
215 )
216 .with_context(|| format!("writing {}", journal.display()))?;
217 }
218 self.compact(&mut state)?;
219 for file in &doomed_blobs {
220 self.remove_blob(file);
221 }
222 self.remove_journal();
223 Ok(removed)
224 }
225
226 pub fn holder(&self, sha256: &str) -> Option<ArchiveEntry> {
228 self.state().holder_of(sha256).cloned()
229 }
230
231 pub fn dir(&self) -> &Path {
232 &self.dir
233 }
234
235 pub fn archive(&self, source: &Path) -> Result<Archived> {
236 let mut state = self.state();
237 let bytes = fs::read(source).with_context(|| format!("reading {}", source.display()))?;
238 let sha256 = hex(&Sha256::digest(&bytes));
239 let key = source_key(source);
240
241 let previous = state.entries.get(&key).cloned();
242 if let Some(prev) = &previous
243 && prev.sha256 == sha256
244 && self.dir.join(&prev.file).exists()
245 {
246 return Ok(Archived::Unchanged);
247 }
248
249 let file = format!("{sha256}{BLOB_SUFFIX}");
250 let shared = state
251 .holder_of(&sha256)
252 .is_some_and(|holder| self.dir.join(&holder.file).exists());
253 if !shared {
254 self.write_blob(state.keys.as_ref(), &file, &bytes)?;
255 }
256 let entry = ArchiveEntry {
257 source: key.clone(),
258 file,
259 sha256,
260 archived_at: Utc::now(),
261 };
262 state.entries.insert(key, entry.clone());
263 if state.needs_compaction() {
264 self.compact(&mut state)?;
265 } else if let Err(e) = self.append(&mut state, &entry) {
266 state.manifest_damaged = true;
267 return Err(e);
268 }
269
270 if let Some(prev) = &previous
271 && !state.references(&prev.file)
272 {
273 self.remove_blob(&prev.file);
274 }
275
276 Ok(match (shared, previous) {
277 (true, _) => Archived::Copy,
278 (false, None) => Archived::Stored,
279 (false, Some(_)) => Archived::Refreshed,
280 })
281 }
282
283 pub fn entry(&self, source: &Path) -> Option<ArchiveEntry> {
284 self.state().entries.get(&source_key(source)).cloned()
285 }
286
287 pub fn read(&self, source: &Path) -> Result<Option<String>> {
288 let (entry, keys) = {
289 let state = self.state();
290 (
291 state.entries.get(&source_key(source)).cloned(),
292 state.keys.clone(),
293 )
294 };
295 let Some(entry) = entry else {
296 return Ok(None);
297 };
298 let path = self.dir.join(&entry.file);
299 let compressed = sealed::read_file(&path, keys.as_ref(), &entry.file)?;
300 let bytes = zstd::decode_all(compressed.as_slice())
301 .with_context(|| format!("decompressing {}", path.display()))?;
302 let text =
303 String::from_utf8(bytes).with_context(|| format!("decoding {}", path.display()))?;
304 Ok(Some(text))
305 }
306
307 pub fn entries(&self) -> Vec<ArchiveEntry> {
309 self.state().entries.values().cloned().collect()
310 }
311
312 pub fn len(&self) -> usize {
313 self.state().entries.len()
314 }
315
316 pub fn is_empty(&self) -> bool {
317 self.len() == 0
318 }
319
320 fn state(&self) -> MutexGuard<'_, State> {
321 self.state.lock().unwrap_or_else(PoisonError::into_inner)
322 }
323
324 fn write_blob(&self, keys: Option<&SealKeys>, file: &str, source_bytes: &[u8]) -> Result<()> {
325 let text = redact(&String::from_utf8_lossy(source_bytes)).text;
326 let compressed =
327 zstd::encode_all(text.as_bytes(), 0).context("compressing a transcript")?;
328 let target = self.dir.join(file);
329 sealed::write_file(&target, keys, file, &compressed)
330 .with_context(|| format!("writing {}", target.display()))
331 }
332
333 fn blob_names(&self) -> Result<Vec<String>> {
334 let mut names = Vec::new();
335 for entry in
336 fs::read_dir(&self.dir).with_context(|| format!("listing {}", self.dir.display()))?
337 {
338 let name = entry
339 .with_context(|| format!("listing {}", self.dir.display()))?
340 .file_name()
341 .to_string_lossy()
342 .into_owned();
343 if name.ends_with(BLOB_SUFFIX) {
344 names.push(name);
345 }
346 }
347 Ok(names)
348 }
349
350 fn manifest_sealed_under(&self, keys: &SealKeys) -> Result<bool> {
351 let plaintext = read_manifest_text(&self.dir.join(MANIFEST_FILE))?;
352 if plaintext.lines().any(|l| !l.trim().is_empty()) {
353 return Ok(false);
354 }
355 let text = read_manifest_text(&self.dir.join(SEALED_MANIFEST_FILE))?;
356 Ok(text.lines().filter(|l| !l.trim().is_empty()).all(|line| {
357 BASE64_STANDARD
358 .decode(line.trim())
359 .is_ok_and(|bytes| sealed::key_id_of(&bytes) == Some(keys.current_id()))
360 }))
361 }
362
363 fn remove_tmp_files(&self) -> Result<()> {
366 for entry in
367 fs::read_dir(&self.dir).with_context(|| format!("listing {}", self.dir.display()))?
368 {
369 let entry = entry.with_context(|| format!("listing {}", self.dir.display()))?;
370 let name = entry.file_name().to_string_lossy().into_owned();
371 if sealed::is_tmp_name(&name) || name.ends_with(LEGACY_TMP_SUFFIX) {
372 fs::remove_file(entry.path())
373 .or_else(|e| match e.kind() {
374 ErrorKind::NotFound => Ok(()),
375 _ => Err(e),
376 })
377 .with_context(|| format!("removing {}", entry.path().display()))?;
378 }
379 }
380 Ok(())
381 }
382
383 fn finish_interrupted_removal(&self, state: &State) -> Result<()> {
387 let journal = self.dir.join(JOURNAL_FILE);
388 if !journal.exists() {
389 return Ok(());
390 }
391 let bytes = sealed::read_file(&journal, state.keys.as_ref(), JOURNAL_FILE)?;
392 let text =
393 String::from_utf8(bytes).with_context(|| format!("decoding {}", journal.display()))?;
394 for name in text.lines().map(str::trim) {
395 let is_blob_name = name.ends_with(BLOB_SUFFIX) && !name.contains(['/', '\\']);
396 if is_blob_name && !state.references(name) {
397 self.remove_blob(name);
398 }
399 }
400 self.remove_journal();
401 Ok(())
402 }
403
404 fn remove_journal(&self) {
405 let path = self.dir.join(JOURNAL_FILE);
406 match fs::remove_file(&path) {
407 Ok(()) => {}
408 Err(e) if e.kind() == ErrorKind::NotFound => {}
409 Err(e) => {
410 tracing::warn!(path = %path.display(), error = %e, "removing the archive removal journal")
411 }
412 }
413 }
414
415 fn remove_blob(&self, file: &str) {
416 let path = self.dir.join(file);
417 match fs::remove_file(&path) {
418 Ok(()) => {}
419 Err(e) if e.kind() == ErrorKind::NotFound => {}
420 Err(e) => {
421 tracing::warn!(path = %path.display(), error = %e, "removing an unused archive blob")
422 }
423 }
424 }
425
426 fn append(&self, state: &mut State, entry: &ArchiveEntry) -> Result<()> {
427 let path = self.dir.join(manifest_file(state.keys.as_ref()));
428 let mut line = manifest_line(state.keys.as_ref(), entry)?;
429 line.push('\n');
430 let mut file = OpenOptions::new()
431 .create(true)
432 .append(true)
433 .open(&path)
434 .with_context(|| format!("opening {}", path.display()))?;
435 file.write_all(line.as_bytes())
436 .and_then(|()| file.sync_data())
437 .with_context(|| format!("appending to {}", path.display()))?;
438 state.manifest_lines += 1;
439 if state.keys.is_none() {
440 state.plaintext_lines += 1;
441 state.unsealed = true;
442 }
443 Ok(())
444 }
445
446 fn compact(&self, state: &mut State) -> Result<()> {
447 let mut entries: Vec<&ArchiveEntry> = state.entries.values().collect();
448 entries.sort_by(|a, b| a.source.cmp(&b.source));
449 let mut body = String::new();
450 for entry in &entries {
451 body.push_str(&manifest_line(state.keys.as_ref(), entry)?);
452 body.push('\n');
453 }
454 let count = entries.len();
455 let name = manifest_file(state.keys.as_ref());
456 let path = self.dir.join(name);
457 sealed::write_file(&path, None, name, body.as_bytes())
458 .with_context(|| format!("compacting {}", path.display()))?;
459 if state.keys.is_some() {
460 let plaintext = self.dir.join(MANIFEST_FILE);
461 if fs::metadata(&plaintext).is_ok_and(|m| m.len() > 0) {
462 let marker = self.dir.join(RESEAL_MARKER);
463 sealed::write_file(&marker, None, RESEAL_MARKER, b"")
464 .with_context(|| format!("writing {}", marker.display()))?;
465 sealed::write_file(&plaintext, None, MANIFEST_FILE, b"")
466 .with_context(|| format!("emptying {}", plaintext.display()))?;
467 }
468 state.plaintext_lines = 0;
469 } else {
470 state.plaintext_lines = count;
471 state.unsealed |= count > 0;
472 }
473 state.manifest_lines = count;
474 state.manifest_damaged = false;
475 Ok(())
476 }
477}
478
479fn no_keys() -> anyhow::Error {
480 anyhow!("the transcript archive has no sealing keys to reseal under")
481}
482
483fn manifest_file(keys: Option<&SealKeys>) -> &'static str {
484 match keys {
485 Some(_) => SEALED_MANIFEST_FILE,
486 None => MANIFEST_FILE,
487 }
488}
489
490fn manifest_line(keys: Option<&SealKeys>, entry: &ArchiveEntry) -> Result<String> {
491 let json = serde_json::to_string(entry)?;
492 match keys {
493 Some(keys) => {
494 Ok(BASE64_STANDARD.encode(sealed::seal(keys, SEALED_MANIFEST_FILE, json.as_bytes())?))
495 }
496 None => Ok(json),
497 }
498}
499
500fn parse_plaintext_line(line: &str) -> Result<ArchiveEntry> {
501 Ok(serde_json::from_str(line)?)
502}
503
504fn parse_sealed_line(line: &str, keys: Option<&SealKeys>) -> Result<ArchiveEntry> {
505 let sealed_bytes = BASE64_STANDARD.decode(line.trim())?;
506 if !sealed::looks_sealed(&sealed_bytes) {
507 bail!("the line is not a sealed record");
508 }
509 let keys = keys.ok_or(SealError::KeyRequired)?;
510 Ok(serde_json::from_slice(&sealed::open(
511 keys,
512 SEALED_MANIFEST_FILE,
513 &sealed_bytes,
514 )?)?)
515}
516
517fn read_manifest_text(path: &Path) -> Result<String> {
518 match fs::read_to_string(path) {
519 Ok(text) => Ok(text),
520 Err(e) if e.kind() == ErrorKind::NotFound => Ok(String::new()),
521 Err(e) => Err(e).with_context(|| format!("reading {}", path.display())),
522 }
523}
524
525fn read_manifest(
530 path: &Path,
531 parse: impl Fn(&str) -> Result<ArchiveEntry>,
532) -> Result<(HashMap<String, ArchiveEntry>, usize, bool)> {
533 let text = read_manifest_text(path)?;
534 let mut entries = HashMap::new();
535 let mut lines = 0;
536 let mut torn = !text.is_empty() && !text.ends_with('\n');
537 for line in text.lines().filter(|l| !l.trim().is_empty()) {
538 lines += 1;
539 match parse(line) {
540 Ok(entry) => {
541 entries.insert(entry.source.clone(), entry);
542 }
543 Err(e) if is_missing_key(&e) => {
544 return Err(e).with_context(|| format!("reading {}", path.display()));
545 }
546 Err(e) => {
547 torn = true;
548 tracing::warn!(path = %path.display(), error = %e, "ignoring an unreadable archive manifest line");
549 }
550 }
551 }
552 Ok((entries, lines, torn))
553}
554
555fn load_manifest(dir: &Path, keys: Option<SealKeys>) -> Result<(State, bool, Vec<String>)> {
559 let (mut entries, sealed_lines, sealed_torn) =
560 read_manifest(&dir.join(SEALED_MANIFEST_FILE), |line| {
561 parse_sealed_line(line, keys.as_ref())
562 })?;
563 let (plaintext, plaintext_lines, plaintext_torn) =
564 read_manifest(&dir.join(MANIFEST_FILE), parse_plaintext_line)?;
565 let unsealed = !plaintext.is_empty() || dir.join(RESEAL_MARKER).exists();
566 let mut superseded = Vec::new();
567 for (source, entry) in plaintext {
568 match entries.get(&source) {
569 Some(held) if held.archived_at > entry.archived_at => superseded.push(entry.file),
570 Some(held) => {
571 superseded.push(held.file.clone());
572 entries.insert(source, entry);
573 }
574 None => {
575 entries.insert(source, entry);
576 }
577 }
578 }
579 let state = State {
580 entries,
581 manifest_lines: sealed_lines + plaintext_lines,
582 plaintext_lines,
583 manifest_damaged: false,
584 unsealed,
585 keys,
586 };
587 Ok((state, sealed_torn || plaintext_torn, superseded))
588}
589
590fn is_missing_key(error: &anyhow::Error) -> bool {
591 matches!(
592 seal_error(error),
593 Some(SealError::KeyRequired | SealError::UnknownKey(_))
594 )
595}
596
597fn source_key(source: &Path) -> String {
598 source.to_string_lossy().into_owned()
599}
600
601fn hex(bytes: &[u8]) -> String {
602 bytes.iter().map(|b| format!("{b:02x}")).collect()
603}
604
605#[cfg(test)]
606mod tests {
607 use std::fs;
608 use std::io::Write;
609
610 use base64::Engine;
611 use base64::prelude::BASE64_STANDARD;
612
613 use super::*;
614 use crate::keys::KeyGuard;
615 use crate::sealed::{self, SealError, SealKey, seal_error};
616
617 fn blobs(archive: &TranscriptArchive) -> Vec<String> {
618 let mut names: Vec<String> = fs::read_dir(archive.dir())
619 .unwrap()
620 .map(|e| e.unwrap().file_name().to_string_lossy().into_owned())
621 .filter(|n| n.ends_with(".jsonl.zst"))
622 .collect();
623 names.sort();
624 names
625 }
626
627 fn manifest_lines(archive: &TranscriptArchive) -> Vec<String> {
628 fs::read_to_string(archive.dir().join(MANIFEST_FILE))
629 .unwrap()
630 .lines()
631 .map(str::to_string)
632 .collect()
633 }
634
635 struct Fixture {
636 _tmp: tempfile::TempDir,
637 projects: PathBuf,
638 archive_dir: PathBuf,
639 }
640
641 impl Fixture {
642 fn new() -> Self {
643 let tmp = tempfile::tempdir().unwrap();
644 let projects = tmp.path().join("projects");
645 fs::create_dir_all(&projects).unwrap();
646 let archive_dir = tmp.path().join("data").join("transcripts");
647 Self {
648 _tmp: tmp,
649 projects,
650 archive_dir,
651 }
652 }
653
654 fn write(&self, name: &str, text: &str) -> PathBuf {
655 let path = self.projects.join(name);
656 fs::write(&path, text).unwrap();
657 path
658 }
659
660 fn open(&self) -> TranscriptArchive {
661 TranscriptArchive::open(&self.archive_dir).unwrap()
662 }
663 }
664
665 #[test]
666 fn open_creates_the_directory() {
667 let fx = Fixture::new();
668 let archive = fx.open();
669 assert!(archive.dir().is_dir());
670 assert!(archive.is_empty());
671 }
672
673 #[test]
674 fn a_new_source_is_stored_then_unchanged() {
675 let fx = Fixture::new();
676 let source = fx.write("a.jsonl", "{\"line\":1}\n");
677 let archive = fx.open();
678
679 assert_eq!(archive.archive(&source).unwrap(), Archived::Stored);
680 assert_eq!(archive.archive(&source).unwrap(), Archived::Unchanged);
681
682 assert_eq!(archive.len(), 1);
683 assert_eq!(blobs(&archive).len(), 1);
684 assert_eq!(manifest_lines(&archive).len(), 1);
685 let entry = archive.entry(&source).unwrap();
686 assert_eq!(entry.source, source.to_string_lossy());
687 assert_eq!(entry.file, format!("{}.jsonl.zst", entry.sha256));
688 assert_eq!(entry.sha256.len(), 64);
689 assert_eq!(
690 archive.read(&source).unwrap().as_deref(),
691 Some("{\"line\":1}\n")
692 );
693 }
694
695 #[test]
696 fn a_changed_source_is_refreshed_and_its_old_blob_removed() {
697 let fx = Fixture::new();
698 let source = fx.write("a.jsonl", "{\"line\":1}\n");
699 let archive = fx.open();
700 archive.archive(&source).unwrap();
701 let old = archive.entry(&source).unwrap();
702
703 fs::write(&source, "{\"line\":1}\n{\"line\":2}\n").unwrap();
704 assert_eq!(archive.archive(&source).unwrap(), Archived::Refreshed);
705
706 let new = archive.entry(&source).unwrap();
707 assert_ne!(new.sha256, old.sha256);
708 assert_eq!(
709 archive.read(&source).unwrap().as_deref(),
710 Some("{\"line\":1}\n{\"line\":2}\n")
711 );
712 assert_eq!(blobs(&archive), vec![new.file]);
713 assert!(!archive.dir().join(&old.file).exists());
714 }
715
716 #[test]
717 fn a_copy_at_another_path_shares_the_blob() {
718 let fx = Fixture::new();
719 let first = fx.write("a.jsonl", "{\"same\":true}\n");
720 let second = fx.write("b.jsonl", "{\"same\":true}\n");
721 let archive = fx.open();
722
723 assert_eq!(archive.archive(&first).unwrap(), Archived::Stored);
724 assert_eq!(archive.archive(&second).unwrap(), Archived::Copy);
725
726 assert_eq!(archive.len(), 2);
727 assert_eq!(blobs(&archive).len(), 1);
728 assert_eq!(
729 archive.entry(&first).unwrap().file,
730 archive.entry(&second).unwrap().file
731 );
732 assert_eq!(
733 archive.read(&second).unwrap().as_deref(),
734 Some("{\"same\":true}\n")
735 );
736 }
737
738 #[test]
739 fn refreshing_one_copy_keeps_the_blob_the_other_still_uses() {
740 let fx = Fixture::new();
741 let first = fx.write("a.jsonl", "{\"same\":true}\n");
742 let second = fx.write("b.jsonl", "{\"same\":true}\n");
743 let archive = fx.open();
744 archive.archive(&first).unwrap();
745 archive.archive(&second).unwrap();
746
747 fs::write(&first, "{\"same\":false}\n").unwrap();
748 assert_eq!(archive.archive(&first).unwrap(), Archived::Refreshed);
749
750 assert_eq!(blobs(&archive).len(), 2);
751 assert_eq!(
752 archive.read(&second).unwrap().as_deref(),
753 Some("{\"same\":true}\n")
754 );
755 }
756
757 #[test]
758 fn the_archive_outlives_a_deleted_source() {
759 let fx = Fixture::new();
760 let source = fx.write("a.jsonl", "{\"kept\":true}\n");
761 let archive = fx.open();
762 archive.archive(&source).unwrap();
763
764 fs::remove_file(&source).unwrap();
765
766 assert!(archive.archive(&source).is_err());
767 assert_eq!(
768 archive.read(&source).unwrap().as_deref(),
769 Some("{\"kept\":true}\n")
770 );
771 assert_eq!(blobs(&archive).len(), 1);
772 let reopened = fx.open();
773 assert_eq!(
774 reopened.read(&source).unwrap().as_deref(),
775 Some("{\"kept\":true}\n")
776 );
777 }
778
779 #[test]
780 fn an_unknown_source_reads_as_none() {
781 let fx = Fixture::new();
782 let archive = fx.open();
783 let missing = fx.projects.join("never.jsonl");
784 assert!(archive.entry(&missing).is_none());
785 assert_eq!(archive.read(&missing).unwrap(), None);
786 }
787
788 #[test]
789 fn archived_text_is_redacted() {
790 let fx = Fixture::new();
791 let secret = format!("gh{}_{}", "p", "aB3dE6".repeat(6));
792 let source = fx.write(
793 "a.jsonl",
794 &format!("{{\"text\":\"token {secret} and the build passed\"}}\n"),
795 );
796 let archive = fx.open();
797 archive.archive(&source).unwrap();
798
799 let text = archive.read(&source).unwrap().unwrap();
800 assert!(!text.contains(&secret));
801 assert!(text.contains("the build passed"));
802 assert!(text.contains("[REDACTED:github_token]"));
803
804 let entry = archive.entry(&source).unwrap();
805 let raw = fs::read(archive.dir().join(&entry.file)).unwrap();
806 let decoded = String::from_utf8(zstd::decode_all(raw.as_slice()).unwrap()).unwrap();
807 assert!(!decoded.contains(&secret));
808 assert!(decoded.contains("the build passed"));
809 }
810
811 #[test]
812 fn the_blob_is_named_by_the_source_bytes() {
813 let fx = Fixture::new();
814 let secret = format!("gh{}_{}", "p", "aB3dE6".repeat(6));
815 let source = fx.write("a.jsonl", &format!("{secret}\n"));
816 let archive = fx.open();
817 archive.archive(&source).unwrap();
818
819 let changed = format!("gh{}_{}", "p", "zY9xW8".repeat(6));
820 fs::write(&source, format!("{changed}\n")).unwrap();
821 assert_eq!(archive.archive(&source).unwrap(), Archived::Refreshed);
822 }
823
824 #[test]
825 fn reopening_loads_the_manifest() {
826 let fx = Fixture::new();
827 let first = fx.write("a.jsonl", "{\"a\":1}\n");
828 let second = fx.write("b.jsonl", "{\"b\":2}\n");
829 let archive = fx.open();
830 archive.archive(&first).unwrap();
831 archive.archive(&second).unwrap();
832 let entry = archive.entry(&first).unwrap();
833 drop(archive);
834
835 let reopened = fx.open();
836 assert_eq!(reopened.len(), 2);
837 assert_eq!(reopened.entry(&first), Some(entry));
838 assert_eq!(reopened.archive(&first).unwrap(), Archived::Unchanged);
839 assert_eq!(
840 reopened.read(&second).unwrap().as_deref(),
841 Some("{\"b\":2}\n")
842 );
843 }
844
845 #[test]
846 fn a_torn_last_manifest_line_is_ignored() {
847 let fx = Fixture::new();
848 let first = fx.write("a.jsonl", "{\"a\":1}\n");
849 let second = fx.write("b.jsonl", "{\"b\":2}\n");
850 let archive = fx.open();
851 archive.archive(&first).unwrap();
852 archive.archive(&second).unwrap();
853 drop(archive);
854 let mut manifest = fs::OpenOptions::new()
855 .append(true)
856 .open(fx.archive_dir.join(MANIFEST_FILE))
857 .unwrap();
858 manifest.write_all(b"{\"source\":\"/x/c.js").unwrap();
859 drop(manifest);
860
861 let reopened = fx.open();
862 assert_eq!(reopened.len(), 2);
863
864 let third = fx.write("c.jsonl", "{\"c\":3}\n");
865 assert_eq!(reopened.archive(&third).unwrap(), Archived::Stored);
866 drop(reopened);
867 let again = fx.open();
868 assert_eq!(again.len(), 3);
869 assert_eq!(again.read(&third).unwrap().as_deref(), Some("{\"c\":3}\n"));
870 }
871
872 #[test]
873 fn a_failed_manifest_append_is_repaired_by_the_next_write() {
874 use std::os::unix::fs::PermissionsExt;
875
876 let fx = Fixture::new();
877 let first = fx.write("a.jsonl", "{\"a\":1}\n");
878 let second = fx.write("b.jsonl", "{\"b\":2}\n");
879 let third = fx.write("c.jsonl", "{\"c\":3}\n");
880 let archive = fx.open();
881 archive.archive(&first).unwrap();
882 let manifest = fx.archive_dir.join(MANIFEST_FILE);
883 fs::set_permissions(&manifest, fs::Permissions::from_mode(0o400)).unwrap();
884
885 assert!(archive.archive(&second).is_err());
886 assert_eq!(archive.archive(&third).unwrap(), Archived::Stored);
887 drop(archive);
888
889 let reopened = fx.open();
890 assert_eq!(reopened.len(), 3);
891 assert_eq!(
892 reopened.read(&second).unwrap().as_deref(),
893 Some("{\"b\":2}\n")
894 );
895 }
896
897 #[test]
898 fn compaction_keeps_the_latest_entry_per_source() {
899 let fx = Fixture::new();
900 let first = fx.write("a.jsonl", "{\"a\":1}\n");
901 let second = fx.write("b.jsonl", "{\"b\":1}\n");
902 let archive = fx.open();
903 archive.archive(&first).unwrap();
904 archive.archive(&second).unwrap();
905 let latest_second = archive.entry(&second).unwrap();
906 drop(archive);
907
908 let manifest = fx.archive_dir.join(MANIFEST_FILE);
909 let stale = ArchiveEntry {
910 source: first.to_string_lossy().into_owned(),
911 file: "0".repeat(64) + ".jsonl.zst",
912 sha256: "0".repeat(64),
913 archived_at: Utc::now(),
914 };
915 let mut lines = fs::read_to_string(&manifest).unwrap();
916 let mut stale_lines = String::new();
917 for _ in 0..80 {
918 stale_lines.push_str(&serde_json::to_string(&stale).unwrap());
919 stale_lines.push('\n');
920 }
921 lines.insert_str(0, &stale_lines);
922 fs::write(&manifest, &lines).unwrap();
923
924 let reopened = fx.open();
925 let kept = manifest_lines(&reopened);
926 assert_eq!(kept.len(), 2);
927 assert_eq!(reopened.len(), 2);
928 assert_ne!(reopened.entry(&first).unwrap().sha256, stale.sha256);
929 assert_eq!(reopened.entry(&second), Some(latest_second));
930 assert_eq!(
931 reopened.read(&first).unwrap().as_deref(),
932 Some("{\"a\":1}\n")
933 );
934 }
935
936 #[test]
937 fn a_long_running_archive_keeps_its_manifest_compact() {
938 let fx = Fixture::new();
939 let source = fx.write("a.jsonl", "");
940 let archive = fx.open();
941 let mut text = String::new();
942 for i in 0..100 {
943 text.push_str(&format!("{{\"line\":{i}}}\n"));
944 fs::write(&source, &text).unwrap();
945 archive.archive(&source).unwrap();
946 }
947
948 assert!(manifest_lines(&archive).len() <= 2 + 64);
949 assert_eq!(blobs(&archive).len(), 1);
950 drop(archive);
951 let reopened = fx.open();
952 assert_eq!(reopened.read(&source).unwrap(), Some(text));
953 }
954
955 #[test]
956 fn removed_entries_leave_the_manifest_and_their_blobs_and_stay_gone_on_reopen() {
957 let fx = Fixture::new();
958 let gone = fx.write("gone.jsonl", "{\"line\":\"secret\"}\n");
959 let copy = fx.write("copy.jsonl", "{\"line\":\"secret\"}\n");
960 let kept = fx.write("kept.jsonl", "{\"line\":\"kept\"}\n");
961 let archive = fx.open();
962 for source in [&gone, ©, &kept] {
963 archive.archive(source).unwrap();
964 }
965 let sha = archive.entry(&gone).unwrap().sha256;
966 assert!(archive.holder(&sha).is_some());
967
968 let removed = archive.remove_where(|e| e.source.contains("gone")).unwrap();
969 assert_eq!(removed.len(), 1);
970 assert!(
971 archive.holder(&sha).is_some(),
972 "the copy still uses the blob"
973 );
974 assert_eq!(blobs(&archive).len(), 2);
975
976 archive.remove_where(|e| e.source.contains("copy")).unwrap();
977 assert!(archive.holder(&sha).is_none());
978 assert_eq!(blobs(&archive).len(), 1);
979 assert!(archive.entry(&gone).is_none());
980 assert!(manifest_lines(&archive).iter().all(|l| !l.contains("gone")));
981
982 let reopened = fx.open();
983 assert_eq!(reopened.len(), 1);
984 assert!(reopened.entry(&kept).is_some());
985 }
986
987 fn stray_blob(archive: &TranscriptArchive, fill: char) -> PathBuf {
988 let path = archive
989 .dir()
990 .join(format!("{}{BLOB_SUFFIX}", fill.to_string().repeat(64)));
991 fs::write(&path, b"unreferenced").unwrap();
992 path
993 }
994
995 #[test]
996 fn a_blob_no_entry_or_journal_names_survives_open() {
997 let fx = Fixture::new();
998 let source = fx.write("a.jsonl", "{\"line\":1}\n");
999 let archive = fx.open();
1000 archive.archive(&source).unwrap();
1001 let stray = stray_blob(&archive, 'f');
1002 drop(archive);
1003
1004 let reopened = fx.open();
1005 assert!(stray.exists());
1006 assert_eq!(blobs(&reopened).len(), 2);
1007 }
1008
1009 #[test]
1010 fn a_journal_left_by_a_crash_after_compaction_has_its_blobs_deleted_at_open() {
1011 let fx = Fixture::new();
1012 let kept = fx.write("kept.jsonl", "{\"line\":\"kept\"}\n");
1013 let archive = fx.open();
1014 archive.archive(&kept).unwrap();
1015 let doomed = stray_blob(&archive, 'a');
1016 let bystander = stray_blob(&archive, 'b');
1017 let kept_blob = archive.entry(&kept).unwrap().file;
1018 let names = format!(
1019 "{}\n{kept_blob}\n",
1020 doomed.file_name().unwrap().to_string_lossy()
1021 );
1022 fs::write(archive.dir().join(JOURNAL_FILE), names).unwrap();
1023 drop(archive);
1024
1025 let reopened = fx.open();
1026 assert!(!doomed.exists());
1027 assert!(bystander.exists());
1028 assert!(reopened.dir().join(&kept_blob).exists());
1029 assert!(!reopened.dir().join(JOURNAL_FILE).exists());
1030 assert_eq!(reopened.len(), 1);
1031 }
1032
1033 #[test]
1034 fn remove_where_leaves_no_journal_behind() {
1035 let fx = Fixture::new();
1036 let source = fx.write("a.jsonl", "{\"line\":1}\n");
1037 let archive = fx.open();
1038 archive.archive(&source).unwrap();
1039 archive.remove_where(|_| true).unwrap();
1040 assert!(!archive.dir().join(JOURNAL_FILE).exists());
1041 assert!(blobs(&archive).is_empty());
1042 }
1043
1044 #[test]
1045 fn a_missing_manifest_never_deletes_blobs() {
1046 let fx = Fixture::new();
1047 let source = fx.write("a.jsonl", "{\"line\":1}\n");
1048 let archive = fx.open();
1049 archive.archive(&source).unwrap();
1050 let dir = archive.dir().to_path_buf();
1051 drop(archive);
1052 fs::remove_file(dir.join(MANIFEST_FILE)).unwrap();
1053
1054 let reopened = fx.open();
1055 assert!(reopened.is_empty());
1056 assert_eq!(blobs(&reopened).len(), 1);
1057 }
1058
1059 fn key(byte: u8) -> SealKey {
1060 SealKey::from_bytes(KeyGuard::new([byte; 32])).unwrap()
1061 }
1062
1063 fn keys(byte: u8) -> SealKeys {
1064 SealKeys::single(key(byte))
1065 }
1066
1067 fn files(dir: &Path) -> Vec<PathBuf> {
1068 let mut paths: Vec<PathBuf> = fs::read_dir(dir)
1069 .unwrap()
1070 .map(|e| e.unwrap().path())
1071 .collect();
1072 paths.sort();
1073 paths
1074 }
1075
1076 fn contains(haystack: &[u8], needle: &[u8]) -> bool {
1077 haystack.windows(needle.len()).any(|w| w == needle)
1078 }
1079
1080 fn assert_all_sealed_under(dir: &Path, keys: &SealKeys) {
1081 for path in files(dir) {
1082 let name = path.file_name().unwrap().to_string_lossy().into_owned();
1083 if name == MANIFEST_FILE {
1084 assert_eq!(fs::read_to_string(&path).unwrap(), "");
1085 } else if name == SEALED_MANIFEST_FILE {
1086 let text = fs::read_to_string(&path).unwrap();
1087 for line in text.lines() {
1088 let sealed = BASE64_STANDARD.decode(line).unwrap();
1089 assert_eq!(sealed::key_id_of(&sealed), Some(keys.current_id()));
1090 }
1091 } else {
1092 assert_eq!(
1093 sealed::sealed_key_id(&path).unwrap(),
1094 Some(keys.current_id()),
1095 "{name}"
1096 );
1097 }
1098 }
1099 }
1100
1101 impl Fixture {
1102 fn open_sealed(&self, keys: SealKeys) -> TranscriptArchive {
1103 TranscriptArchive::open_sealed(&self.archive_dir, keys).unwrap()
1104 }
1105 }
1106
1107 #[test]
1108 fn a_sealed_archive_holds_neither_the_transcript_text_nor_its_source_path() {
1109 let fx = Fixture::new();
1110 let text = "{\"text\":\"the quarterly migration plan\"}\n".repeat(4);
1111 let source = fx.write("orchard-ledger.jsonl", &text);
1112 let archive = fx.open_sealed(keys(1));
1113 archive.archive(&source).unwrap();
1114
1115 let path_bytes = source.to_string_lossy().into_owned();
1116 for path in files(archive.dir()) {
1117 let raw = fs::read(&path).unwrap();
1118 assert!(!contains(&raw, b"quarterly migration"), "{path:?}");
1119 assert!(!contains(&raw, b"orchard-ledger"), "{path:?}");
1120 assert!(!contains(&raw, path_bytes.as_bytes()), "{path:?}");
1121 assert!(zstd::decode_all(raw.as_slice()).is_err(), "{path:?}");
1122 }
1123 assert_all_sealed_under(archive.dir(), &keys(1));
1124 assert_eq!(archive.read(&source).unwrap(), Some(text.clone()));
1125 drop(archive);
1126 let reopened = fx.open_sealed(keys(1));
1127 assert_eq!(reopened.read(&source).unwrap(), Some(text));
1128 }
1129
1130 #[test]
1131 fn a_sealed_archive_does_not_open_without_its_key() {
1132 let fx = Fixture::new();
1133 let source = fx.write("a.jsonl", "{\"line\":1}\n");
1134 let archive = fx.open_sealed(keys(2));
1135 archive.archive(&source).unwrap();
1136 drop(archive);
1137
1138 let err = TranscriptArchive::open(&fx.archive_dir)
1139 .err()
1140 .expect("an unkeyed open fails");
1141 assert_eq!(seal_error(&err), Some(&SealError::KeyRequired));
1142 let err = TranscriptArchive::open_sealed(&fx.archive_dir, keys(3))
1143 .err()
1144 .expect("another key fails");
1145 assert!(matches!(seal_error(&err), Some(SealError::UnknownKey(_))));
1146 assert_eq!(blobs(&fx.open_sealed(keys(2))).len(), 1);
1147 }
1148
1149 #[test]
1150 fn a_plaintext_archive_opens_sealed_and_reseal_seals_every_file() {
1151 let fx = Fixture::new();
1152 let first = fx.write("a.jsonl", "{\"a\":1}\n");
1153 let copy = fx.write("b.jsonl", "{\"a\":1}\n");
1154 let second = fx.write("c.jsonl", "{\"c\":3}\n");
1155 let plain = fx.open();
1156 for source in [&first, ©, &second] {
1157 plain.archive(source).unwrap();
1158 }
1159 drop(plain);
1160 fs::write(fx.archive_dir.join("manifest.jsonl.tmp"), "{\"source\":").unwrap();
1161
1162 let archive = fx.open_sealed(keys(4));
1163 assert_eq!(archive.read(©).unwrap().as_deref(), Some("{\"a\":1}\n"));
1164 assert_eq!(
1165 archive.read(&second).unwrap().as_deref(),
1166 Some("{\"c\":3}\n")
1167 );
1168 assert!(manifest_lines(&archive).is_empty());
1169
1170 assert_eq!(archive.reseal().unwrap(), 2);
1171 assert_all_sealed_under(archive.dir(), &keys(4));
1172 assert_eq!(archive.reseal().unwrap(), 0);
1173 assert_eq!(
1174 archive.read(&first).unwrap().as_deref(),
1175 Some("{\"a\":1}\n")
1176 );
1177 drop(archive);
1178
1179 let reopened = fx.open_sealed(keys(4));
1180 assert_eq!(reopened.len(), 3);
1181 assert_eq!(
1182 reopened.read(&second).unwrap().as_deref(),
1183 Some("{\"c\":3}\n")
1184 );
1185 }
1186
1187 #[test]
1188 fn a_rotation_then_reseal_leaves_only_the_new_key() {
1189 let fx = Fixture::new();
1190 let first = fx.write("a.jsonl", "{\"a\":1}\n");
1191 let second = fx.write("b.jsonl", "{\"b\":2}\n");
1192 let archive = fx.open_sealed(keys(5));
1193 archive.archive(&first).unwrap();
1194
1195 archive.set_seal_keys(SealKeys::new(key(6), vec![key(5)]));
1196 archive.archive(&second).unwrap();
1197 assert_eq!(
1198 archive.read(&first).unwrap().as_deref(),
1199 Some("{\"a\":1}\n")
1200 );
1201 assert_eq!(archive.reseal().unwrap(), 2);
1202 assert_all_sealed_under(archive.dir(), &keys(6));
1203 drop(archive);
1204
1205 let reopened = fx.open_sealed(keys(6));
1206 assert_eq!(reopened.len(), 2);
1207 assert_eq!(
1208 reopened.read(&first).unwrap().as_deref(),
1209 Some("{\"a\":1}\n")
1210 );
1211 assert_eq!(
1212 reopened.read(&second).unwrap().as_deref(),
1213 Some("{\"b\":2}\n")
1214 );
1215 }
1216
1217 #[test]
1218 fn a_torn_sealed_manifest_line_is_ignored() {
1219 let fx = Fixture::new();
1220 let first = fx.write("a.jsonl", "{\"a\":1}\n");
1221 let second = fx.write("b.jsonl", "{\"b\":2}\n");
1222 let archive = fx.open_sealed(keys(7));
1223 archive.archive(&first).unwrap();
1224 archive.archive(&second).unwrap();
1225 let line = sealed_manifest_lines(archive.dir())[0].clone();
1226 drop(archive);
1227 for cut in [line.len() / 2, line.len() - 3] {
1228 let mut manifest = fs::OpenOptions::new()
1229 .append(true)
1230 .open(fx.archive_dir.join(SEALED_MANIFEST_FILE))
1231 .unwrap();
1232 manifest.write_all(&line.as_bytes()[..cut]).unwrap();
1233 drop(manifest);
1234
1235 let reopened = fx.open_sealed(keys(7));
1236 assert_eq!(reopened.len(), 2);
1237 assert_eq!(sealed_manifest_lines(reopened.dir()).len(), 2);
1238 }
1239
1240 let reopened = fx.open_sealed(keys(7));
1241 let third = fx.write("c.jsonl", "{\"c\":3}\n");
1242 assert_eq!(reopened.archive(&third).unwrap(), Archived::Stored);
1243 drop(reopened);
1244 let again = fx.open_sealed(keys(7));
1245 assert_eq!(again.len(), 3);
1246 assert_eq!(again.read(&third).unwrap().as_deref(), Some("{\"c\":3}\n"));
1247 assert_all_sealed_under(again.dir(), &keys(7));
1248 }
1249
1250 #[test]
1251 fn removal_works_sealed_and_an_interrupted_removal_is_finished_from_a_sealed_journal() {
1252 let fx = Fixture::new();
1253 let gone = fx.write("gone.jsonl", "{\"line\":\"gone\"}\n");
1254 let kept = fx.write("kept.jsonl", "{\"line\":\"kept\"}\n");
1255 let archive = fx.open_sealed(keys(8));
1256 archive.archive(&gone).unwrap();
1257 archive.archive(&kept).unwrap();
1258
1259 let removed = archive.remove_where(|e| e.source.contains("gone")).unwrap();
1260 assert_eq!(removed.len(), 1);
1261 assert_eq!(blobs(&archive).len(), 1);
1262 assert!(!archive.dir().join(JOURNAL_FILE).exists());
1263
1264 let doomed = stray_blob(&archive, 'a');
1265 let bystander = stray_blob(&archive, 'b');
1266 let names = format!("{}\n", doomed.file_name().unwrap().to_string_lossy());
1267 sealed::write_file(
1268 &archive.dir().join(JOURNAL_FILE),
1269 Some(&keys(8)),
1270 JOURNAL_FILE,
1271 names.as_bytes(),
1272 )
1273 .unwrap();
1274 drop(archive);
1275
1276 let reopened = fx.open_sealed(keys(8));
1277 assert!(!doomed.exists());
1278 assert!(bystander.exists());
1279 assert!(!reopened.dir().join(JOURNAL_FILE).exists());
1280 assert_eq!(reopened.len(), 1);
1281 assert_eq!(
1282 reopened.read(&kept).unwrap().as_deref(),
1283 Some("{\"line\":\"kept\"}\n")
1284 );
1285 assert!(manifest_lines(&reopened).is_empty());
1286 assert_eq!(sealed_manifest_lines(reopened.dir()).len(), 1);
1287 }
1288
1289 #[test]
1290 fn a_sealed_journal_is_not_read_without_the_key() {
1291 let fx = Fixture::new();
1292 let archive = fx.open();
1293 let doomed = stray_blob(&archive, 'c');
1294 let names = format!("{}\n", doomed.file_name().unwrap().to_string_lossy());
1295 sealed::write_file(
1296 &archive.dir().join(JOURNAL_FILE),
1297 Some(&keys(9)),
1298 JOURNAL_FILE,
1299 names.as_bytes(),
1300 )
1301 .unwrap();
1302 drop(archive);
1303
1304 let err = TranscriptArchive::open(&fx.archive_dir)
1305 .err()
1306 .expect("an unkeyed open fails");
1307 assert_eq!(seal_error(&err), Some(&SealError::KeyRequired));
1308 assert!(doomed.exists());
1309 }
1310
1311 fn sealed_manifest_lines(dir: &Path) -> Vec<String> {
1312 match fs::read_to_string(dir.join(SEALED_MANIFEST_FILE)) {
1313 Ok(text) => text.lines().map(str::to_string).collect(),
1314 Err(e) if e.kind() == ErrorKind::NotFound => Vec::new(),
1315 Err(e) => panic!("{e}"),
1316 }
1317 }
1318
1319 fn line_from_a_binary_before_sealing(
1320 fx: &Fixture,
1321 source: &Path,
1322 archived_at: DateTime<Utc>,
1323 ) -> String {
1324 let bytes = fs::read(source).unwrap();
1325 let sha256 = hex(&Sha256::digest(&bytes));
1326 let file = format!("{sha256}{BLOB_SUFFIX}");
1327 fs::write(
1328 fx.archive_dir.join(&file),
1329 zstd::encode_all(bytes.as_slice(), 0).unwrap(),
1330 )
1331 .unwrap();
1332 serde_json::to_string(&ArchiveEntry {
1333 source: source_key(source),
1334 file,
1335 sha256,
1336 archived_at,
1337 })
1338 .unwrap()
1339 }
1340
1341 #[test]
1342 fn a_sealed_archive_leaves_the_plaintext_manifest_readable_by_a_binary_from_before_sealing() {
1343 let fx = Fixture::new();
1344 let first = fx.write("a.jsonl", "{\"a\":1}\n");
1345 let plain = fx.open();
1346 plain.archive(&first).unwrap();
1347 drop(plain);
1348
1349 let archive = fx.open_sealed(keys(10));
1350 let second = fx.write("b.jsonl", "{\"b\":2}\n");
1351 archive.archive(&second).unwrap();
1352 archive.reseal().unwrap();
1353 let third = fx.write("c.jsonl", "{\"c\":3}\n");
1354 archive.archive(&third).unwrap();
1355
1356 for line in manifest_lines(&archive) {
1357 serde_json::from_str::<ArchiveEntry>(&line).unwrap();
1358 }
1359 assert_eq!(sealed_manifest_lines(archive.dir()).len(), 3);
1360 }
1361
1362 #[test]
1363 fn entries_a_binary_from_before_sealing_appended_are_merged_and_then_sealed() {
1364 let fx = Fixture::new();
1365 let kept = fx.write("kept.jsonl", "{\"k\":1}\n");
1366 let changed = fx.write("changed.jsonl", "{\"v\":1}\n");
1367 let archive = fx.open_sealed(keys(11));
1368 archive.archive(&kept).unwrap();
1369 archive.archive(&changed).unwrap();
1370 let old_blob = archive.entry(&changed).unwrap().file;
1371 drop(archive);
1372
1373 let added = fx.write("added.jsonl", "{\"n\":1}\n");
1374 fs::write(&changed, "{\"v\":2}\n").unwrap();
1375 let later = Utc::now() + chrono::Duration::seconds(1);
1376 let lines = [
1377 line_from_a_binary_before_sealing(&fx, &added, later),
1378 line_from_a_binary_before_sealing(&fx, &changed, later),
1379 ];
1380 let mut manifest = fs::OpenOptions::new()
1381 .append(true)
1382 .open(fx.archive_dir.join(MANIFEST_FILE))
1383 .unwrap();
1384 for line in &lines {
1385 writeln!(manifest, "{line}").unwrap();
1386 }
1387 drop(manifest);
1388
1389 let reopened = fx.open_sealed(keys(11));
1390 assert_eq!(reopened.len(), 3);
1391 assert_eq!(
1392 reopened.read(&added).unwrap().as_deref(),
1393 Some("{\"n\":1}\n")
1394 );
1395 assert_eq!(
1396 reopened.read(&changed).unwrap().as_deref(),
1397 Some("{\"v\":2}\n")
1398 );
1399 assert_eq!(
1400 reopened.read(&kept).unwrap().as_deref(),
1401 Some("{\"k\":1}\n")
1402 );
1403 assert!(!reopened.dir().join(&old_blob).exists());
1404 assert!(manifest_lines(&reopened).is_empty());
1405 assert!(reopened.holds_unsealed());
1406
1407 reopened.reseal().unwrap();
1408 assert!(!reopened.holds_unsealed());
1409 assert_all_sealed_under(reopened.dir(), &keys(11));
1410 drop(reopened);
1411 let again = fx.open_sealed(keys(11));
1412 assert_eq!(again.len(), 3);
1413 assert!(!again.holds_unsealed());
1414 }
1415
1416 #[test]
1417 fn a_removal_after_keys_arrive_leaves_no_plaintext_entry_behind() {
1418 let fx = Fixture::new();
1419 let gone = fx.write("gone.jsonl", "{\"line\":\"gone\"}\n");
1420 let kept = fx.write("kept.jsonl", "{\"line\":\"kept\"}\n");
1421 let archive = fx.open();
1422 archive.archive(&gone).unwrap();
1423 archive.archive(&kept).unwrap();
1424
1425 archive.set_seal_keys(keys(12));
1426 archive.remove_where(|e| e.source.contains("gone")).unwrap();
1427 assert!(manifest_lines(&archive).is_empty());
1428 drop(archive);
1429
1430 let reopened = fx.open_sealed(keys(12));
1431 assert_eq!(reopened.len(), 1);
1432 assert!(reopened.entry(&gone).is_none());
1433 }
1434
1435 #[test]
1436 fn a_fold_cut_short_before_its_reseal_still_reports_unsealed_files_after_a_reopen() {
1437 let fx = Fixture::new();
1438 let first = fx.write("a.jsonl", "{\"a\":1}\n");
1439 let plain = fx.open();
1440 plain.archive(&first).unwrap();
1441 drop(plain);
1442
1443 let folded = fx.open_sealed(keys(13));
1444 assert!(folded.holds_unsealed());
1445 assert!(manifest_lines(&folded).is_empty());
1446 drop(folded);
1447
1448 let reopened = fx.open_sealed(keys(13));
1449 assert!(reopened.holds_unsealed());
1450 reopened.reseal().unwrap();
1451 assert_all_sealed_under(reopened.dir(), &keys(13));
1452 drop(reopened);
1453 assert!(!fx.open_sealed(keys(13)).holds_unsealed());
1454 }
1455}