1use std::collections::HashMap;
2use std::fs::{self, OpenOptions};
3use std::io::{ErrorKind, Write};
4use std::path::{Path, PathBuf};
5use std::sync::{Mutex, MutexGuard, PoisonError};
6
7use anyhow::{Context, Result, anyhow, bail};
8use base64::Engine;
9use base64::prelude::BASE64_STANDARD;
10use chrono::{DateTime, Utc};
11use serde::{Deserialize, Serialize};
12use sha2::{Digest, Sha256};
13
14use crate::redact::redact;
15use crate::sealed::{self, SealError, SealKeys, seal_error};
16
17pub const MANIFEST_FILE: &str = "manifest.jsonl";
18pub const SEALED_MANIFEST_FILE: &str = "manifest.sealed";
21
22const JOURNAL_FILE: &str = "removal.journal";
23const RESEAL_MARKER: &str = "reseal.pending";
24const BLOB_SUFFIX: &str = ".jsonl.zst";
25const LEGACY_TMP_SUFFIX: &str = ".tmp";
26const COMPACTION_SLACK: usize = 64;
27
28#[derive(Debug, Clone, Copy, PartialEq, Eq)]
29pub enum Archived {
30 Stored,
31 Refreshed,
32 Unchanged,
33 Copy,
34}
35
36#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
37pub struct ArchiveEntry {
38 pub source: String,
39 pub file: String,
40 pub sha256: String,
41 pub archived_at: DateTime<Utc>,
42}
43
44pub struct TranscriptArchive {
53 dir: PathBuf,
54 state: Mutex<State>,
55}
56
57struct State {
58 entries: HashMap<String, ArchiveEntry>,
59 manifest_lines: usize,
60 plaintext_lines: usize,
61 manifest_damaged: bool,
62 unsealed: bool,
63 keys: Option<SealKeys>,
64}
65
66impl State {
67 fn needs_compaction(&self) -> bool {
68 self.manifest_damaged || self.manifest_lines > 2 * self.entries.len() + COMPACTION_SLACK
69 }
70
71 fn plaintext_pending(&self) -> bool {
72 self.keys.is_some() && self.plaintext_lines > 0
73 }
74
75 fn holder_of(&self, sha256: &str) -> Option<&ArchiveEntry> {
76 self.entries.values().find(|e| e.sha256 == sha256)
77 }
78
79 fn references(&self, file: &str) -> bool {
80 self.entries.values().any(|e| e.file == file)
81 }
82}
83
84impl TranscriptArchive {
85 pub fn open(dir: impl Into<PathBuf>) -> Result<Self> {
86 Self::open_with(dir.into(), None)
87 }
88
89 pub fn open_sealed(dir: impl Into<PathBuf>, keys: SealKeys) -> Result<Self> {
92 Self::open_with(dir.into(), Some(keys))
93 }
94
95 fn open_with(dir: PathBuf, keys: Option<SealKeys>) -> Result<Self> {
96 fs::create_dir_all(&dir)
97 .with_context(|| format!("creating the transcript archive at {}", dir.display()))?;
98 let manifest = dir.join(MANIFEST_FILE);
99 if !manifest.exists() {
100 sealed::write_file(&manifest, None, MANIFEST_FILE, b"")
101 .with_context(|| format!("creating {}", manifest.display()))?;
102 }
103 let (state, torn, superseded) = load_manifest(&dir, keys)?;
104 let archive = Self {
105 dir,
106 state: Mutex::new(state),
107 };
108 {
109 let mut state = archive.state();
110 if torn || state.needs_compaction() || state.plaintext_pending() {
111 archive.compact(&mut state)?;
112 }
113 for file in superseded.iter().filter(|f| !state.references(f)) {
114 archive.remove_blob(file);
115 }
116 archive.finish_interrupted_removal(&state)?;
117 }
118 Ok(archive)
119 }
120
121 pub fn holds_unsealed(&self) -> bool {
125 self.state().unsealed
126 }
127
128 pub fn seal_keys(&self) -> Option<SealKeys> {
130 self.state().keys.clone()
131 }
132
133 pub fn set_seal_keys(&self, keys: SealKeys) {
136 self.state().keys = Some(keys);
137 }
138
139 pub fn reseal(&self) -> Result<usize> {
145 if self.state().keys.is_none() {
146 return Err(no_keys());
147 }
148 let mut resealed = 0;
149 for file in self.blob_names()? {
150 let state = self.state();
151 let keys = state.keys.clone().ok_or_else(no_keys)?;
152 let path = self.dir.join(&file);
153 if path.exists() && sealed::reseal_file(&path, &keys, &file)? {
154 resealed += 1;
155 }
156 }
157 let mut state = self.state();
158 let keys = state.keys.clone().ok_or_else(no_keys)?;
159 let journal = self.dir.join(JOURNAL_FILE);
160 if journal.exists() && sealed::reseal_file(&journal, &keys, JOURNAL_FILE)? {
161 resealed += 1;
162 }
163 if !self.manifest_sealed_under(&keys)? {
164 self.compact(&mut state)?;
165 resealed += 1;
166 }
167 self.remove_tmp_files()?;
168 let marker = self.dir.join(RESEAL_MARKER);
169 match fs::remove_file(&marker) {
170 Ok(()) => {}
171 Err(e) if e.kind() == ErrorKind::NotFound => {}
172 Err(e) => return Err(e).with_context(|| format!("removing {}", marker.display())),
173 }
174 state.unsealed = false;
175 Ok(resealed)
176 }
177
178 pub fn remove_where(
182 &self,
183 doomed: impl Fn(&ArchiveEntry) -> bool,
184 ) -> Result<Vec<ArchiveEntry>> {
185 let mut state = self.state();
186 if state.plaintext_pending() {
187 self.compact(&mut state)?;
188 }
189 let removed: Vec<ArchiveEntry> = state
190 .entries
191 .values()
192 .filter(|e| doomed(e))
193 .cloned()
194 .collect();
195 if removed.is_empty() {
196 return Ok(removed);
197 }
198 for entry in &removed {
199 state.entries.remove(&entry.source);
200 }
201 let mut doomed_blobs: Vec<&str> = removed
202 .iter()
203 .map(|e| e.file.as_str())
204 .filter(|file| !state.references(file))
205 .collect();
206 doomed_blobs.sort_unstable();
207 doomed_blobs.dedup();
208 if !doomed_blobs.is_empty() {
209 let journal = self.dir.join(JOURNAL_FILE);
210 sealed::write_file(
211 &journal,
212 state.keys.as_ref(),
213 JOURNAL_FILE,
214 doomed_blobs.join("\n").as_bytes(),
215 )
216 .with_context(|| format!("writing {}", journal.display()))?;
217 }
218 self.compact(&mut state)?;
219 for file in &doomed_blobs {
220 self.remove_blob(file);
221 }
222 self.remove_journal();
223 Ok(removed)
224 }
225
226 pub fn copy_into(&self, dest: &Path) -> Result<()> {
232 self.copy_into_with(dest, || {})
233 }
234
235 fn copy_into_with(&self, dest: &Path, between_phases: impl FnOnce()) -> Result<()> {
236 crate::private_files::create_private_dir(dest)?;
237 for file in self.blob_names()? {
238 copy_if_present(&self.dir.join(&file), &dest.join(&file))?;
239 }
240 between_phases();
241 let state = self.state();
242 for name in [
243 MANIFEST_FILE,
244 SEALED_MANIFEST_FILE,
245 JOURNAL_FILE,
246 RESEAL_MARKER,
247 ] {
248 copy_if_present(&self.dir.join(name), &dest.join(name))?;
249 }
250 for entry in state.entries.values() {
251 let target = dest.join(&entry.file);
252 if !target.exists() {
253 crate::dir_copy::copy_file_synced(&self.dir.join(&entry.file), &target)?;
254 }
255 }
256 for entry in fs::read_dir(dest).with_context(|| format!("listing {}", dest.display()))? {
257 let name = entry?.file_name().to_string_lossy().into_owned();
258 if name.ends_with(BLOB_SUFFIX) && !state.references(&name) {
259 let path = dest.join(&name);
260 fs::remove_file(&path).with_context(|| format!("removing {}", path.display()))?;
261 }
262 }
263 drop(state);
264 crate::dir_copy::sync_dir(dest)
265 }
266
267 pub fn holder(&self, sha256: &str) -> Option<ArchiveEntry> {
269 self.state().holder_of(sha256).cloned()
270 }
271
272 pub fn dir(&self) -> &Path {
273 &self.dir
274 }
275
276 pub fn archive(&self, source: &Path) -> Result<Archived> {
277 let mut state = self.state();
278 let bytes = fs::read(source).with_context(|| format!("reading {}", source.display()))?;
279 let sha256 = hex(&Sha256::digest(&bytes));
280 let key = source_key(source);
281
282 let previous = state.entries.get(&key).cloned();
283 if let Some(prev) = &previous
284 && prev.sha256 == sha256
285 && self.dir.join(&prev.file).exists()
286 {
287 return Ok(Archived::Unchanged);
288 }
289
290 let file = format!("{sha256}{BLOB_SUFFIX}");
291 let shared = state
292 .holder_of(&sha256)
293 .is_some_and(|holder| self.dir.join(&holder.file).exists());
294 if !shared {
295 self.write_blob(state.keys.as_ref(), &file, &bytes)?;
296 }
297 let entry = ArchiveEntry {
298 source: key.clone(),
299 file,
300 sha256,
301 archived_at: Utc::now(),
302 };
303 state.entries.insert(key, entry.clone());
304 if state.needs_compaction() {
305 self.compact(&mut state)?;
306 } else if let Err(e) = self.append(&mut state, &entry) {
307 state.manifest_damaged = true;
308 return Err(e);
309 }
310
311 if let Some(prev) = &previous
312 && !state.references(&prev.file)
313 {
314 self.remove_blob(&prev.file);
315 }
316
317 Ok(match (shared, previous) {
318 (true, _) => Archived::Copy,
319 (false, None) => Archived::Stored,
320 (false, Some(_)) => Archived::Refreshed,
321 })
322 }
323
324 pub fn entry(&self, source: &Path) -> Option<ArchiveEntry> {
325 self.state().entries.get(&source_key(source)).cloned()
326 }
327
328 pub fn read(&self, source: &Path) -> Result<Option<String>> {
329 let (entry, keys) = {
330 let state = self.state();
331 (
332 state.entries.get(&source_key(source)).cloned(),
333 state.keys.clone(),
334 )
335 };
336 let Some(entry) = entry else {
337 return Ok(None);
338 };
339 let path = self.dir.join(&entry.file);
340 let compressed = sealed::read_file(&path, keys.as_ref(), &entry.file)?;
341 let bytes = zstd::decode_all(compressed.as_slice())
342 .with_context(|| format!("decompressing {}", path.display()))?;
343 let text =
344 String::from_utf8(bytes).with_context(|| format!("decoding {}", path.display()))?;
345 Ok(Some(text))
346 }
347
348 pub fn entries(&self) -> Vec<ArchiveEntry> {
350 self.state().entries.values().cloned().collect()
351 }
352
353 pub fn len(&self) -> usize {
354 self.state().entries.len()
355 }
356
357 pub fn is_empty(&self) -> bool {
358 self.len() == 0
359 }
360
361 fn state(&self) -> MutexGuard<'_, State> {
362 self.state.lock().unwrap_or_else(PoisonError::into_inner)
363 }
364
365 fn write_blob(&self, keys: Option<&SealKeys>, file: &str, source_bytes: &[u8]) -> Result<()> {
366 let text = redact(&String::from_utf8_lossy(source_bytes)).text;
367 let compressed =
368 zstd::encode_all(text.as_bytes(), 0).context("compressing a transcript")?;
369 let target = self.dir.join(file);
370 sealed::write_file(&target, keys, file, &compressed)
371 .with_context(|| format!("writing {}", target.display()))
372 }
373
374 fn blob_names(&self) -> Result<Vec<String>> {
375 let mut names = Vec::new();
376 for entry in
377 fs::read_dir(&self.dir).with_context(|| format!("listing {}", self.dir.display()))?
378 {
379 let name = entry
380 .with_context(|| format!("listing {}", self.dir.display()))?
381 .file_name()
382 .to_string_lossy()
383 .into_owned();
384 if name.ends_with(BLOB_SUFFIX) {
385 names.push(name);
386 }
387 }
388 Ok(names)
389 }
390
391 fn manifest_sealed_under(&self, keys: &SealKeys) -> Result<bool> {
392 let plaintext = read_manifest_text(&self.dir.join(MANIFEST_FILE))?;
393 if plaintext.lines().any(|l| !l.trim().is_empty()) {
394 return Ok(false);
395 }
396 let text = read_manifest_text(&self.dir.join(SEALED_MANIFEST_FILE))?;
397 Ok(text.lines().filter(|l| !l.trim().is_empty()).all(|line| {
398 BASE64_STANDARD
399 .decode(line.trim())
400 .is_ok_and(|bytes| sealed::key_id_of(&bytes) == Some(keys.current_id()))
401 }))
402 }
403
404 fn remove_tmp_files(&self) -> Result<()> {
407 for entry in
408 fs::read_dir(&self.dir).with_context(|| format!("listing {}", self.dir.display()))?
409 {
410 let entry = entry.with_context(|| format!("listing {}", self.dir.display()))?;
411 let name = entry.file_name().to_string_lossy().into_owned();
412 if sealed::is_tmp_name(&name) || name.ends_with(LEGACY_TMP_SUFFIX) {
413 fs::remove_file(entry.path())
414 .or_else(|e| match e.kind() {
415 ErrorKind::NotFound => Ok(()),
416 _ => Err(e),
417 })
418 .with_context(|| format!("removing {}", entry.path().display()))?;
419 }
420 }
421 Ok(())
422 }
423
424 fn finish_interrupted_removal(&self, state: &State) -> Result<()> {
428 let journal = self.dir.join(JOURNAL_FILE);
429 if !journal.exists() {
430 return Ok(());
431 }
432 let bytes = sealed::read_file(&journal, state.keys.as_ref(), JOURNAL_FILE)?;
433 let text =
434 String::from_utf8(bytes).with_context(|| format!("decoding {}", journal.display()))?;
435 for name in text.lines().map(str::trim) {
436 let is_blob_name = name.ends_with(BLOB_SUFFIX) && !name.contains(['/', '\\']);
437 if is_blob_name && !state.references(name) {
438 self.remove_blob(name);
439 }
440 }
441 self.remove_journal();
442 Ok(())
443 }
444
445 fn remove_journal(&self) {
446 let path = self.dir.join(JOURNAL_FILE);
447 match fs::remove_file(&path) {
448 Ok(()) => {}
449 Err(e) if e.kind() == ErrorKind::NotFound => {}
450 Err(e) => {
451 tracing::warn!(path = %path.display(), error = %e, "removing the archive removal journal")
452 }
453 }
454 }
455
456 fn remove_blob(&self, file: &str) {
457 let path = self.dir.join(file);
458 match fs::remove_file(&path) {
459 Ok(()) => {}
460 Err(e) if e.kind() == ErrorKind::NotFound => {}
461 Err(e) => {
462 tracing::warn!(path = %path.display(), error = %e, "removing an unused archive blob")
463 }
464 }
465 }
466
467 fn append(&self, state: &mut State, entry: &ArchiveEntry) -> Result<()> {
468 let path = self.dir.join(manifest_file(state.keys.as_ref()));
469 let mut line = manifest_line(state.keys.as_ref(), entry)?;
470 line.push('\n');
471 let mut file = OpenOptions::new()
472 .create(true)
473 .append(true)
474 .open(&path)
475 .with_context(|| format!("opening {}", path.display()))?;
476 file.write_all(line.as_bytes())
477 .and_then(|()| file.sync_data())
478 .with_context(|| format!("appending to {}", path.display()))?;
479 state.manifest_lines += 1;
480 if state.keys.is_none() {
481 state.plaintext_lines += 1;
482 state.unsealed = true;
483 }
484 Ok(())
485 }
486
487 fn compact(&self, state: &mut State) -> Result<()> {
488 let mut entries: Vec<&ArchiveEntry> = state.entries.values().collect();
489 entries.sort_by(|a, b| a.source.cmp(&b.source));
490 let mut body = String::new();
491 for entry in &entries {
492 body.push_str(&manifest_line(state.keys.as_ref(), entry)?);
493 body.push('\n');
494 }
495 let count = entries.len();
496 let name = manifest_file(state.keys.as_ref());
497 let path = self.dir.join(name);
498 sealed::write_file(&path, None, name, body.as_bytes())
499 .with_context(|| format!("compacting {}", path.display()))?;
500 if state.keys.is_some() {
501 let plaintext = self.dir.join(MANIFEST_FILE);
502 if fs::metadata(&plaintext).is_ok_and(|m| m.len() > 0) {
503 let marker = self.dir.join(RESEAL_MARKER);
504 sealed::write_file(&marker, None, RESEAL_MARKER, b"")
505 .with_context(|| format!("writing {}", marker.display()))?;
506 sealed::write_file(&plaintext, None, MANIFEST_FILE, b"")
507 .with_context(|| format!("emptying {}", plaintext.display()))?;
508 }
509 state.plaintext_lines = 0;
510 } else {
511 state.plaintext_lines = count;
512 state.unsealed |= count > 0;
513 }
514 state.manifest_lines = count;
515 state.manifest_damaged = false;
516 Ok(())
517 }
518}
519
520fn copy_if_present(from: &Path, to: &Path) -> Result<()> {
521 match crate::dir_copy::copy_file_synced(from, to) {
522 Err(e) if crate::dir_copy::is_not_found(&e) => Ok(()),
523 other => other,
524 }
525}
526
527fn no_keys() -> anyhow::Error {
528 anyhow!("the transcript archive has no sealing keys to reseal under")
529}
530
531fn manifest_file(keys: Option<&SealKeys>) -> &'static str {
532 match keys {
533 Some(_) => SEALED_MANIFEST_FILE,
534 None => MANIFEST_FILE,
535 }
536}
537
538fn manifest_line(keys: Option<&SealKeys>, entry: &ArchiveEntry) -> Result<String> {
539 let json = serde_json::to_string(entry)?;
540 match keys {
541 Some(keys) => {
542 Ok(BASE64_STANDARD.encode(sealed::seal(keys, SEALED_MANIFEST_FILE, json.as_bytes())?))
543 }
544 None => Ok(json),
545 }
546}
547
548fn parse_plaintext_line(line: &str) -> Result<ArchiveEntry> {
549 Ok(serde_json::from_str(line)?)
550}
551
552fn parse_sealed_line(line: &str, keys: Option<&SealKeys>) -> Result<ArchiveEntry> {
553 let sealed_bytes = BASE64_STANDARD.decode(line.trim())?;
554 if !sealed::looks_sealed(&sealed_bytes) {
555 bail!("the line is not a sealed record");
556 }
557 let keys = keys.ok_or(SealError::KeyRequired)?;
558 Ok(serde_json::from_slice(&sealed::open(
559 keys,
560 SEALED_MANIFEST_FILE,
561 &sealed_bytes,
562 )?)?)
563}
564
565fn read_manifest_text(path: &Path) -> Result<String> {
566 match fs::read_to_string(path) {
567 Ok(text) => Ok(text),
568 Err(e) if e.kind() == ErrorKind::NotFound => Ok(String::new()),
569 Err(e) => Err(e).with_context(|| format!("reading {}", path.display())),
570 }
571}
572
573fn read_manifest(
578 path: &Path,
579 parse: impl Fn(&str) -> Result<ArchiveEntry>,
580) -> Result<(HashMap<String, ArchiveEntry>, usize, bool)> {
581 let text = read_manifest_text(path)?;
582 let mut entries = HashMap::new();
583 let mut lines = 0;
584 let mut torn = !text.is_empty() && !text.ends_with('\n');
585 for line in text.lines().filter(|l| !l.trim().is_empty()) {
586 lines += 1;
587 match parse(line) {
588 Ok(entry) => {
589 entries.insert(entry.source.clone(), entry);
590 }
591 Err(e) if is_missing_key(&e) => {
592 return Err(e).with_context(|| format!("reading {}", path.display()));
593 }
594 Err(e) => {
595 torn = true;
596 tracing::warn!(path = %path.display(), error = %e, "ignoring an unreadable archive manifest line");
597 }
598 }
599 }
600 Ok((entries, lines, torn))
601}
602
603fn load_manifest(dir: &Path, keys: Option<SealKeys>) -> Result<(State, bool, Vec<String>)> {
607 let (mut entries, sealed_lines, sealed_torn) =
608 read_manifest(&dir.join(SEALED_MANIFEST_FILE), |line| {
609 parse_sealed_line(line, keys.as_ref())
610 })?;
611 let (plaintext, plaintext_lines, plaintext_torn) =
612 read_manifest(&dir.join(MANIFEST_FILE), parse_plaintext_line)?;
613 let unsealed = !plaintext.is_empty() || dir.join(RESEAL_MARKER).exists();
614 let mut superseded = Vec::new();
615 for (source, entry) in plaintext {
616 match entries.get(&source) {
617 Some(held) if held.archived_at > entry.archived_at => superseded.push(entry.file),
618 Some(held) => {
619 superseded.push(held.file.clone());
620 entries.insert(source, entry);
621 }
622 None => {
623 entries.insert(source, entry);
624 }
625 }
626 }
627 let state = State {
628 entries,
629 manifest_lines: sealed_lines + plaintext_lines,
630 plaintext_lines,
631 manifest_damaged: false,
632 unsealed,
633 keys,
634 };
635 Ok((state, sealed_torn || plaintext_torn, superseded))
636}
637
638fn is_missing_key(error: &anyhow::Error) -> bool {
639 matches!(
640 seal_error(error),
641 Some(SealError::KeyRequired | SealError::UnknownKey(_))
642 )
643}
644
645fn source_key(source: &Path) -> String {
646 source.to_string_lossy().into_owned()
647}
648
649fn hex(bytes: &[u8]) -> String {
650 bytes.iter().map(|b| format!("{b:02x}")).collect()
651}
652
653#[cfg(test)]
654mod tests {
655 use std::fs;
656 use std::io::Write;
657
658 use base64::Engine;
659 use base64::prelude::BASE64_STANDARD;
660
661 use super::*;
662 use crate::keys::KeyGuard;
663 use crate::sealed::{self, SealError, SealKey, seal_error};
664
665 fn blobs(archive: &TranscriptArchive) -> Vec<String> {
666 let mut names: Vec<String> = fs::read_dir(archive.dir())
667 .unwrap()
668 .map(|e| e.unwrap().file_name().to_string_lossy().into_owned())
669 .filter(|n| n.ends_with(".jsonl.zst"))
670 .collect();
671 names.sort();
672 names
673 }
674
675 fn manifest_lines(archive: &TranscriptArchive) -> Vec<String> {
676 fs::read_to_string(archive.dir().join(MANIFEST_FILE))
677 .unwrap()
678 .lines()
679 .map(str::to_string)
680 .collect()
681 }
682
683 struct Fixture {
684 _tmp: tempfile::TempDir,
685 projects: PathBuf,
686 archive_dir: PathBuf,
687 }
688
689 impl Fixture {
690 fn new() -> Self {
691 let tmp = tempfile::tempdir().unwrap();
692 let projects = tmp.path().join("projects");
693 fs::create_dir_all(&projects).unwrap();
694 let archive_dir = tmp.path().join("data").join("transcripts");
695 Self {
696 _tmp: tmp,
697 projects,
698 archive_dir,
699 }
700 }
701
702 fn write(&self, name: &str, text: &str) -> PathBuf {
703 let path = self.projects.join(name);
704 fs::write(&path, text).unwrap();
705 path
706 }
707
708 fn open(&self) -> TranscriptArchive {
709 TranscriptArchive::open(&self.archive_dir).unwrap()
710 }
711 }
712
713 #[test]
714 fn open_creates_the_directory() {
715 let fx = Fixture::new();
716 let archive = fx.open();
717 assert!(archive.dir().is_dir());
718 assert!(archive.is_empty());
719 }
720
721 #[test]
722 fn a_new_source_is_stored_then_unchanged() {
723 let fx = Fixture::new();
724 let source = fx.write("a.jsonl", "{\"line\":1}\n");
725 let archive = fx.open();
726
727 assert_eq!(archive.archive(&source).unwrap(), Archived::Stored);
728 assert_eq!(archive.archive(&source).unwrap(), Archived::Unchanged);
729
730 assert_eq!(archive.len(), 1);
731 assert_eq!(blobs(&archive).len(), 1);
732 assert_eq!(manifest_lines(&archive).len(), 1);
733 let entry = archive.entry(&source).unwrap();
734 assert_eq!(entry.source, source.to_string_lossy());
735 assert_eq!(entry.file, format!("{}.jsonl.zst", entry.sha256));
736 assert_eq!(entry.sha256.len(), 64);
737 assert_eq!(
738 archive.read(&source).unwrap().as_deref(),
739 Some("{\"line\":1}\n")
740 );
741 }
742
743 #[test]
744 fn a_changed_source_is_refreshed_and_its_old_blob_removed() {
745 let fx = Fixture::new();
746 let source = fx.write("a.jsonl", "{\"line\":1}\n");
747 let archive = fx.open();
748 archive.archive(&source).unwrap();
749 let old = archive.entry(&source).unwrap();
750
751 fs::write(&source, "{\"line\":1}\n{\"line\":2}\n").unwrap();
752 assert_eq!(archive.archive(&source).unwrap(), Archived::Refreshed);
753
754 let new = archive.entry(&source).unwrap();
755 assert_ne!(new.sha256, old.sha256);
756 assert_eq!(
757 archive.read(&source).unwrap().as_deref(),
758 Some("{\"line\":1}\n{\"line\":2}\n")
759 );
760 assert_eq!(blobs(&archive), vec![new.file]);
761 assert!(!archive.dir().join(&old.file).exists());
762 }
763
764 #[test]
765 fn a_copy_at_another_path_shares_the_blob() {
766 let fx = Fixture::new();
767 let first = fx.write("a.jsonl", "{\"same\":true}\n");
768 let second = fx.write("b.jsonl", "{\"same\":true}\n");
769 let archive = fx.open();
770
771 assert_eq!(archive.archive(&first).unwrap(), Archived::Stored);
772 assert_eq!(archive.archive(&second).unwrap(), Archived::Copy);
773
774 assert_eq!(archive.len(), 2);
775 assert_eq!(blobs(&archive).len(), 1);
776 assert_eq!(
777 archive.entry(&first).unwrap().file,
778 archive.entry(&second).unwrap().file
779 );
780 assert_eq!(
781 archive.read(&second).unwrap().as_deref(),
782 Some("{\"same\":true}\n")
783 );
784 }
785
786 #[test]
787 fn refreshing_one_copy_keeps_the_blob_the_other_still_uses() {
788 let fx = Fixture::new();
789 let first = fx.write("a.jsonl", "{\"same\":true}\n");
790 let second = fx.write("b.jsonl", "{\"same\":true}\n");
791 let archive = fx.open();
792 archive.archive(&first).unwrap();
793 archive.archive(&second).unwrap();
794
795 fs::write(&first, "{\"same\":false}\n").unwrap();
796 assert_eq!(archive.archive(&first).unwrap(), Archived::Refreshed);
797
798 assert_eq!(blobs(&archive).len(), 2);
799 assert_eq!(
800 archive.read(&second).unwrap().as_deref(),
801 Some("{\"same\":true}\n")
802 );
803 }
804
805 #[test]
806 fn the_archive_outlives_a_deleted_source() {
807 let fx = Fixture::new();
808 let source = fx.write("a.jsonl", "{\"kept\":true}\n");
809 let archive = fx.open();
810 archive.archive(&source).unwrap();
811
812 fs::remove_file(&source).unwrap();
813
814 assert!(archive.archive(&source).is_err());
815 assert_eq!(
816 archive.read(&source).unwrap().as_deref(),
817 Some("{\"kept\":true}\n")
818 );
819 assert_eq!(blobs(&archive).len(), 1);
820 let reopened = fx.open();
821 assert_eq!(
822 reopened.read(&source).unwrap().as_deref(),
823 Some("{\"kept\":true}\n")
824 );
825 }
826
827 #[test]
828 fn an_unknown_source_reads_as_none() {
829 let fx = Fixture::new();
830 let archive = fx.open();
831 let missing = fx.projects.join("never.jsonl");
832 assert!(archive.entry(&missing).is_none());
833 assert_eq!(archive.read(&missing).unwrap(), None);
834 }
835
836 #[test]
837 fn archived_text_is_redacted() {
838 let fx = Fixture::new();
839 let secret = format!("gh{}_{}", "p", "aB3dE6".repeat(6));
840 let source = fx.write(
841 "a.jsonl",
842 &format!("{{\"text\":\"token {secret} and the build passed\"}}\n"),
843 );
844 let archive = fx.open();
845 archive.archive(&source).unwrap();
846
847 let text = archive.read(&source).unwrap().unwrap();
848 assert!(!text.contains(&secret));
849 assert!(text.contains("the build passed"));
850 assert!(text.contains("[REDACTED:github_token]"));
851
852 let entry = archive.entry(&source).unwrap();
853 let raw = fs::read(archive.dir().join(&entry.file)).unwrap();
854 let decoded = String::from_utf8(zstd::decode_all(raw.as_slice()).unwrap()).unwrap();
855 assert!(!decoded.contains(&secret));
856 assert!(decoded.contains("the build passed"));
857 }
858
859 #[test]
860 fn the_blob_is_named_by_the_source_bytes() {
861 let fx = Fixture::new();
862 let secret = format!("gh{}_{}", "p", "aB3dE6".repeat(6));
863 let source = fx.write("a.jsonl", &format!("{secret}\n"));
864 let archive = fx.open();
865 archive.archive(&source).unwrap();
866
867 let changed = format!("gh{}_{}", "p", "zY9xW8".repeat(6));
868 fs::write(&source, format!("{changed}\n")).unwrap();
869 assert_eq!(archive.archive(&source).unwrap(), Archived::Refreshed);
870 }
871
872 #[test]
873 fn reopening_loads_the_manifest() {
874 let fx = Fixture::new();
875 let first = fx.write("a.jsonl", "{\"a\":1}\n");
876 let second = fx.write("b.jsonl", "{\"b\":2}\n");
877 let archive = fx.open();
878 archive.archive(&first).unwrap();
879 archive.archive(&second).unwrap();
880 let entry = archive.entry(&first).unwrap();
881 drop(archive);
882
883 let reopened = fx.open();
884 assert_eq!(reopened.len(), 2);
885 assert_eq!(reopened.entry(&first), Some(entry));
886 assert_eq!(reopened.archive(&first).unwrap(), Archived::Unchanged);
887 assert_eq!(
888 reopened.read(&second).unwrap().as_deref(),
889 Some("{\"b\":2}\n")
890 );
891 }
892
893 #[test]
894 fn a_torn_last_manifest_line_is_ignored() {
895 let fx = Fixture::new();
896 let first = fx.write("a.jsonl", "{\"a\":1}\n");
897 let second = fx.write("b.jsonl", "{\"b\":2}\n");
898 let archive = fx.open();
899 archive.archive(&first).unwrap();
900 archive.archive(&second).unwrap();
901 drop(archive);
902 let mut manifest = fs::OpenOptions::new()
903 .append(true)
904 .open(fx.archive_dir.join(MANIFEST_FILE))
905 .unwrap();
906 manifest.write_all(b"{\"source\":\"/x/c.js").unwrap();
907 drop(manifest);
908
909 let reopened = fx.open();
910 assert_eq!(reopened.len(), 2);
911
912 let third = fx.write("c.jsonl", "{\"c\":3}\n");
913 assert_eq!(reopened.archive(&third).unwrap(), Archived::Stored);
914 drop(reopened);
915 let again = fx.open();
916 assert_eq!(again.len(), 3);
917 assert_eq!(again.read(&third).unwrap().as_deref(), Some("{\"c\":3}\n"));
918 }
919
920 #[test]
921 fn a_failed_manifest_append_is_repaired_by_the_next_write() {
922 use std::os::unix::fs::PermissionsExt;
923
924 let fx = Fixture::new();
925 let first = fx.write("a.jsonl", "{\"a\":1}\n");
926 let second = fx.write("b.jsonl", "{\"b\":2}\n");
927 let third = fx.write("c.jsonl", "{\"c\":3}\n");
928 let archive = fx.open();
929 archive.archive(&first).unwrap();
930 let manifest = fx.archive_dir.join(MANIFEST_FILE);
931 fs::set_permissions(&manifest, fs::Permissions::from_mode(0o400)).unwrap();
932
933 assert!(archive.archive(&second).is_err());
934 assert_eq!(archive.archive(&third).unwrap(), Archived::Stored);
935 drop(archive);
936
937 let reopened = fx.open();
938 assert_eq!(reopened.len(), 3);
939 assert_eq!(
940 reopened.read(&second).unwrap().as_deref(),
941 Some("{\"b\":2}\n")
942 );
943 }
944
945 #[test]
946 fn compaction_keeps_the_latest_entry_per_source() {
947 let fx = Fixture::new();
948 let first = fx.write("a.jsonl", "{\"a\":1}\n");
949 let second = fx.write("b.jsonl", "{\"b\":1}\n");
950 let archive = fx.open();
951 archive.archive(&first).unwrap();
952 archive.archive(&second).unwrap();
953 let latest_second = archive.entry(&second).unwrap();
954 drop(archive);
955
956 let manifest = fx.archive_dir.join(MANIFEST_FILE);
957 let stale = ArchiveEntry {
958 source: first.to_string_lossy().into_owned(),
959 file: "0".repeat(64) + ".jsonl.zst",
960 sha256: "0".repeat(64),
961 archived_at: Utc::now(),
962 };
963 let mut lines = fs::read_to_string(&manifest).unwrap();
964 let mut stale_lines = String::new();
965 for _ in 0..80 {
966 stale_lines.push_str(&serde_json::to_string(&stale).unwrap());
967 stale_lines.push('\n');
968 }
969 lines.insert_str(0, &stale_lines);
970 fs::write(&manifest, &lines).unwrap();
971
972 let reopened = fx.open();
973 let kept = manifest_lines(&reopened);
974 assert_eq!(kept.len(), 2);
975 assert_eq!(reopened.len(), 2);
976 assert_ne!(reopened.entry(&first).unwrap().sha256, stale.sha256);
977 assert_eq!(reopened.entry(&second), Some(latest_second));
978 assert_eq!(
979 reopened.read(&first).unwrap().as_deref(),
980 Some("{\"a\":1}\n")
981 );
982 }
983
984 #[test]
985 fn a_long_running_archive_keeps_its_manifest_compact() {
986 let fx = Fixture::new();
987 let source = fx.write("a.jsonl", "");
988 let archive = fx.open();
989 let mut text = String::new();
990 for i in 0..100 {
991 text.push_str(&format!("{{\"line\":{i}}}\n"));
992 fs::write(&source, &text).unwrap();
993 archive.archive(&source).unwrap();
994 }
995
996 assert!(manifest_lines(&archive).len() <= 2 + 64);
997 assert_eq!(blobs(&archive).len(), 1);
998 drop(archive);
999 let reopened = fx.open();
1000 assert_eq!(reopened.read(&source).unwrap(), Some(text));
1001 }
1002
1003 #[test]
1004 fn removed_entries_leave_the_manifest_and_their_blobs_and_stay_gone_on_reopen() {
1005 let fx = Fixture::new();
1006 let gone = fx.write("gone.jsonl", "{\"line\":\"secret\"}\n");
1007 let copy = fx.write("copy.jsonl", "{\"line\":\"secret\"}\n");
1008 let kept = fx.write("kept.jsonl", "{\"line\":\"kept\"}\n");
1009 let archive = fx.open();
1010 for source in [&gone, ©, &kept] {
1011 archive.archive(source).unwrap();
1012 }
1013 let sha = archive.entry(&gone).unwrap().sha256;
1014 assert!(archive.holder(&sha).is_some());
1015
1016 let removed = archive.remove_where(|e| e.source.contains("gone")).unwrap();
1017 assert_eq!(removed.len(), 1);
1018 assert!(
1019 archive.holder(&sha).is_some(),
1020 "the copy still uses the blob"
1021 );
1022 assert_eq!(blobs(&archive).len(), 2);
1023
1024 archive.remove_where(|e| e.source.contains("copy")).unwrap();
1025 assert!(archive.holder(&sha).is_none());
1026 assert_eq!(blobs(&archive).len(), 1);
1027 assert!(archive.entry(&gone).is_none());
1028 assert!(manifest_lines(&archive).iter().all(|l| !l.contains("gone")));
1029
1030 let reopened = fx.open();
1031 assert_eq!(reopened.len(), 1);
1032 assert!(reopened.entry(&kept).is_some());
1033 }
1034
1035 fn stray_blob(archive: &TranscriptArchive, fill: char) -> PathBuf {
1036 let path = archive
1037 .dir()
1038 .join(format!("{}{BLOB_SUFFIX}", fill.to_string().repeat(64)));
1039 fs::write(&path, b"unreferenced").unwrap();
1040 path
1041 }
1042
1043 #[test]
1044 fn a_blob_no_entry_or_journal_names_survives_open() {
1045 let fx = Fixture::new();
1046 let source = fx.write("a.jsonl", "{\"line\":1}\n");
1047 let archive = fx.open();
1048 archive.archive(&source).unwrap();
1049 let stray = stray_blob(&archive, 'f');
1050 drop(archive);
1051
1052 let reopened = fx.open();
1053 assert!(stray.exists());
1054 assert_eq!(blobs(&reopened).len(), 2);
1055 }
1056
1057 #[test]
1058 fn a_journal_left_by_a_crash_after_compaction_has_its_blobs_deleted_at_open() {
1059 let fx = Fixture::new();
1060 let kept = fx.write("kept.jsonl", "{\"line\":\"kept\"}\n");
1061 let archive = fx.open();
1062 archive.archive(&kept).unwrap();
1063 let doomed = stray_blob(&archive, 'a');
1064 let bystander = stray_blob(&archive, 'b');
1065 let kept_blob = archive.entry(&kept).unwrap().file;
1066 let names = format!(
1067 "{}\n{kept_blob}\n",
1068 doomed.file_name().unwrap().to_string_lossy()
1069 );
1070 fs::write(archive.dir().join(JOURNAL_FILE), names).unwrap();
1071 drop(archive);
1072
1073 let reopened = fx.open();
1074 assert!(!doomed.exists());
1075 assert!(bystander.exists());
1076 assert!(reopened.dir().join(&kept_blob).exists());
1077 assert!(!reopened.dir().join(JOURNAL_FILE).exists());
1078 assert_eq!(reopened.len(), 1);
1079 }
1080
1081 #[test]
1082 fn remove_where_leaves_no_journal_behind() {
1083 let fx = Fixture::new();
1084 let source = fx.write("a.jsonl", "{\"line\":1}\n");
1085 let archive = fx.open();
1086 archive.archive(&source).unwrap();
1087 archive.remove_where(|_| true).unwrap();
1088 assert!(!archive.dir().join(JOURNAL_FILE).exists());
1089 assert!(blobs(&archive).is_empty());
1090 }
1091
1092 #[test]
1093 fn a_missing_manifest_never_deletes_blobs() {
1094 let fx = Fixture::new();
1095 let source = fx.write("a.jsonl", "{\"line\":1}\n");
1096 let archive = fx.open();
1097 archive.archive(&source).unwrap();
1098 let dir = archive.dir().to_path_buf();
1099 drop(archive);
1100 fs::remove_file(dir.join(MANIFEST_FILE)).unwrap();
1101
1102 let reopened = fx.open();
1103 assert!(reopened.is_empty());
1104 assert_eq!(blobs(&reopened).len(), 1);
1105 }
1106
1107 fn key(byte: u8) -> SealKey {
1108 SealKey::from_bytes(KeyGuard::new([byte; 32])).unwrap()
1109 }
1110
1111 fn keys(byte: u8) -> SealKeys {
1112 SealKeys::single(key(byte))
1113 }
1114
1115 fn files(dir: &Path) -> Vec<PathBuf> {
1116 let mut paths: Vec<PathBuf> = fs::read_dir(dir)
1117 .unwrap()
1118 .map(|e| e.unwrap().path())
1119 .collect();
1120 paths.sort();
1121 paths
1122 }
1123
1124 fn contains(haystack: &[u8], needle: &[u8]) -> bool {
1125 haystack.windows(needle.len()).any(|w| w == needle)
1126 }
1127
1128 fn assert_all_sealed_under(dir: &Path, keys: &SealKeys) {
1129 for path in files(dir) {
1130 let name = path.file_name().unwrap().to_string_lossy().into_owned();
1131 if name == MANIFEST_FILE {
1132 assert_eq!(fs::read_to_string(&path).unwrap(), "");
1133 } else if name == SEALED_MANIFEST_FILE {
1134 let text = fs::read_to_string(&path).unwrap();
1135 for line in text.lines() {
1136 let sealed = BASE64_STANDARD.decode(line).unwrap();
1137 assert_eq!(sealed::key_id_of(&sealed), Some(keys.current_id()));
1138 }
1139 } else {
1140 assert_eq!(
1141 sealed::sealed_key_id(&path).unwrap(),
1142 Some(keys.current_id()),
1143 "{name}"
1144 );
1145 }
1146 }
1147 }
1148
1149 impl Fixture {
1150 fn open_sealed(&self, keys: SealKeys) -> TranscriptArchive {
1151 TranscriptArchive::open_sealed(&self.archive_dir, keys).unwrap()
1152 }
1153 }
1154
1155 #[test]
1156 fn a_sealed_archive_holds_neither_the_transcript_text_nor_its_source_path() {
1157 let fx = Fixture::new();
1158 let text = "{\"text\":\"the quarterly migration plan\"}\n".repeat(4);
1159 let source = fx.write("orchard-ledger.jsonl", &text);
1160 let archive = fx.open_sealed(keys(1));
1161 archive.archive(&source).unwrap();
1162
1163 let path_bytes = source.to_string_lossy().into_owned();
1164 for path in files(archive.dir()) {
1165 let raw = fs::read(&path).unwrap();
1166 assert!(!contains(&raw, b"quarterly migration"), "{path:?}");
1167 assert!(!contains(&raw, b"orchard-ledger"), "{path:?}");
1168 assert!(!contains(&raw, path_bytes.as_bytes()), "{path:?}");
1169 assert!(zstd::decode_all(raw.as_slice()).is_err(), "{path:?}");
1170 }
1171 assert_all_sealed_under(archive.dir(), &keys(1));
1172 assert_eq!(archive.read(&source).unwrap(), Some(text.clone()));
1173 drop(archive);
1174 let reopened = fx.open_sealed(keys(1));
1175 assert_eq!(reopened.read(&source).unwrap(), Some(text));
1176 }
1177
1178 #[test]
1179 fn a_copy_opens_with_every_entry_under_the_same_keys() {
1180 let fx = Fixture::new();
1181 let a = fx.write("a.jsonl", "{\"line\":\"alpha\"}\n");
1182 let b = fx.write("b.jsonl", "{\"line\":\"beta\"}\n");
1183 let archive = fx.open_sealed(keys(5));
1184 archive.archive(&a).unwrap();
1185 archive.archive(&b).unwrap();
1186
1187 let dest = fx._tmp.path().join("backup-transcripts");
1188 archive.copy_into(&dest).unwrap();
1189 let copy = TranscriptArchive::open_sealed(&dest, keys(5)).unwrap();
1190 assert_eq!(copy.len(), 2);
1191 assert_eq!(copy.read(&a).unwrap(), archive.read(&a).unwrap());
1192 assert_eq!(copy.read(&b).unwrap(), archive.read(&b).unwrap());
1193 }
1194
1195 #[test]
1196 fn a_blob_removed_while_the_copy_runs_leaves_nothing_in_it() {
1197 let fx = Fixture::new();
1198 let a = fx.write("a.jsonl", "{\"line\":\"the deleted session\"}\n");
1199 let b = fx.write("b.jsonl", "{\"line\":\"kept\"}\n");
1200 let archive = fx.open_sealed(keys(6));
1201 archive.archive(&a).unwrap();
1202 archive.archive(&b).unwrap();
1203 let doomed = archive.entry(&a).unwrap();
1204
1205 let dest = fx._tmp.path().join("backup-transcripts");
1206 archive
1207 .copy_into_with(&dest, || {
1208 archive.remove_where(|e| e.source == doomed.source).unwrap();
1209 })
1210 .unwrap();
1211 assert!(!dest.join(&doomed.file).exists());
1212 let copy = TranscriptArchive::open_sealed(&dest, keys(6)).unwrap();
1213 assert_eq!(copy.len(), 1);
1214 assert_eq!(copy.read(&a).unwrap(), None);
1215 assert!(copy.read(&b).unwrap().is_some());
1216 }
1217
1218 #[test]
1219 fn a_copy_of_an_archive_awaiting_reseal_still_awaits_it() {
1220 let fx = Fixture::new();
1221 let a = fx.write("a.jsonl", "{\"line\":1}\n");
1222 fx.open().archive(&a).unwrap();
1223 let archive = fx.open_sealed(keys(7));
1224 assert!(archive.holds_unsealed());
1225
1226 let dest = fx._tmp.path().join("backup-transcripts");
1227 archive.copy_into(&dest).unwrap();
1228 let copy = TranscriptArchive::open_sealed(&dest, keys(7)).unwrap();
1229 assert!(copy.holds_unsealed());
1230 assert!(copy.read(&a).unwrap().is_some());
1231 }
1232
1233 #[test]
1234 fn a_sealed_archive_does_not_open_without_its_key() {
1235 let fx = Fixture::new();
1236 let source = fx.write("a.jsonl", "{\"line\":1}\n");
1237 let archive = fx.open_sealed(keys(2));
1238 archive.archive(&source).unwrap();
1239 drop(archive);
1240
1241 let err = TranscriptArchive::open(&fx.archive_dir)
1242 .err()
1243 .expect("an unkeyed open fails");
1244 assert_eq!(seal_error(&err), Some(&SealError::KeyRequired));
1245 let err = TranscriptArchive::open_sealed(&fx.archive_dir, keys(3))
1246 .err()
1247 .expect("another key fails");
1248 assert!(matches!(seal_error(&err), Some(SealError::UnknownKey(_))));
1249 assert_eq!(blobs(&fx.open_sealed(keys(2))).len(), 1);
1250 }
1251
1252 #[test]
1253 fn a_plaintext_archive_opens_sealed_and_reseal_seals_every_file() {
1254 let fx = Fixture::new();
1255 let first = fx.write("a.jsonl", "{\"a\":1}\n");
1256 let copy = fx.write("b.jsonl", "{\"a\":1}\n");
1257 let second = fx.write("c.jsonl", "{\"c\":3}\n");
1258 let plain = fx.open();
1259 for source in [&first, ©, &second] {
1260 plain.archive(source).unwrap();
1261 }
1262 drop(plain);
1263 fs::write(fx.archive_dir.join("manifest.jsonl.tmp"), "{\"source\":").unwrap();
1264
1265 let archive = fx.open_sealed(keys(4));
1266 assert_eq!(archive.read(©).unwrap().as_deref(), Some("{\"a\":1}\n"));
1267 assert_eq!(
1268 archive.read(&second).unwrap().as_deref(),
1269 Some("{\"c\":3}\n")
1270 );
1271 assert!(manifest_lines(&archive).is_empty());
1272
1273 assert_eq!(archive.reseal().unwrap(), 2);
1274 assert_all_sealed_under(archive.dir(), &keys(4));
1275 assert_eq!(archive.reseal().unwrap(), 0);
1276 assert_eq!(
1277 archive.read(&first).unwrap().as_deref(),
1278 Some("{\"a\":1}\n")
1279 );
1280 drop(archive);
1281
1282 let reopened = fx.open_sealed(keys(4));
1283 assert_eq!(reopened.len(), 3);
1284 assert_eq!(
1285 reopened.read(&second).unwrap().as_deref(),
1286 Some("{\"c\":3}\n")
1287 );
1288 }
1289
1290 #[test]
1291 fn a_rotation_then_reseal_leaves_only_the_new_key() {
1292 let fx = Fixture::new();
1293 let first = fx.write("a.jsonl", "{\"a\":1}\n");
1294 let second = fx.write("b.jsonl", "{\"b\":2}\n");
1295 let archive = fx.open_sealed(keys(5));
1296 archive.archive(&first).unwrap();
1297
1298 archive.set_seal_keys(SealKeys::new(key(6), vec![key(5)]));
1299 archive.archive(&second).unwrap();
1300 assert_eq!(
1301 archive.read(&first).unwrap().as_deref(),
1302 Some("{\"a\":1}\n")
1303 );
1304 assert_eq!(archive.reseal().unwrap(), 2);
1305 assert_all_sealed_under(archive.dir(), &keys(6));
1306 drop(archive);
1307
1308 let reopened = fx.open_sealed(keys(6));
1309 assert_eq!(reopened.len(), 2);
1310 assert_eq!(
1311 reopened.read(&first).unwrap().as_deref(),
1312 Some("{\"a\":1}\n")
1313 );
1314 assert_eq!(
1315 reopened.read(&second).unwrap().as_deref(),
1316 Some("{\"b\":2}\n")
1317 );
1318 }
1319
1320 #[test]
1321 fn a_torn_sealed_manifest_line_is_ignored() {
1322 let fx = Fixture::new();
1323 let first = fx.write("a.jsonl", "{\"a\":1}\n");
1324 let second = fx.write("b.jsonl", "{\"b\":2}\n");
1325 let archive = fx.open_sealed(keys(7));
1326 archive.archive(&first).unwrap();
1327 archive.archive(&second).unwrap();
1328 let line = sealed_manifest_lines(archive.dir())[0].clone();
1329 drop(archive);
1330 for cut in [line.len() / 2, line.len() - 3] {
1331 let mut manifest = fs::OpenOptions::new()
1332 .append(true)
1333 .open(fx.archive_dir.join(SEALED_MANIFEST_FILE))
1334 .unwrap();
1335 manifest.write_all(&line.as_bytes()[..cut]).unwrap();
1336 drop(manifest);
1337
1338 let reopened = fx.open_sealed(keys(7));
1339 assert_eq!(reopened.len(), 2);
1340 assert_eq!(sealed_manifest_lines(reopened.dir()).len(), 2);
1341 }
1342
1343 let reopened = fx.open_sealed(keys(7));
1344 let third = fx.write("c.jsonl", "{\"c\":3}\n");
1345 assert_eq!(reopened.archive(&third).unwrap(), Archived::Stored);
1346 drop(reopened);
1347 let again = fx.open_sealed(keys(7));
1348 assert_eq!(again.len(), 3);
1349 assert_eq!(again.read(&third).unwrap().as_deref(), Some("{\"c\":3}\n"));
1350 assert_all_sealed_under(again.dir(), &keys(7));
1351 }
1352
1353 #[test]
1354 fn removal_works_sealed_and_an_interrupted_removal_is_finished_from_a_sealed_journal() {
1355 let fx = Fixture::new();
1356 let gone = fx.write("gone.jsonl", "{\"line\":\"gone\"}\n");
1357 let kept = fx.write("kept.jsonl", "{\"line\":\"kept\"}\n");
1358 let archive = fx.open_sealed(keys(8));
1359 archive.archive(&gone).unwrap();
1360 archive.archive(&kept).unwrap();
1361
1362 let removed = archive.remove_where(|e| e.source.contains("gone")).unwrap();
1363 assert_eq!(removed.len(), 1);
1364 assert_eq!(blobs(&archive).len(), 1);
1365 assert!(!archive.dir().join(JOURNAL_FILE).exists());
1366
1367 let doomed = stray_blob(&archive, 'a');
1368 let bystander = stray_blob(&archive, 'b');
1369 let names = format!("{}\n", doomed.file_name().unwrap().to_string_lossy());
1370 sealed::write_file(
1371 &archive.dir().join(JOURNAL_FILE),
1372 Some(&keys(8)),
1373 JOURNAL_FILE,
1374 names.as_bytes(),
1375 )
1376 .unwrap();
1377 drop(archive);
1378
1379 let reopened = fx.open_sealed(keys(8));
1380 assert!(!doomed.exists());
1381 assert!(bystander.exists());
1382 assert!(!reopened.dir().join(JOURNAL_FILE).exists());
1383 assert_eq!(reopened.len(), 1);
1384 assert_eq!(
1385 reopened.read(&kept).unwrap().as_deref(),
1386 Some("{\"line\":\"kept\"}\n")
1387 );
1388 assert!(manifest_lines(&reopened).is_empty());
1389 assert_eq!(sealed_manifest_lines(reopened.dir()).len(), 1);
1390 }
1391
1392 #[test]
1393 fn a_sealed_journal_is_not_read_without_the_key() {
1394 let fx = Fixture::new();
1395 let archive = fx.open();
1396 let doomed = stray_blob(&archive, 'c');
1397 let names = format!("{}\n", doomed.file_name().unwrap().to_string_lossy());
1398 sealed::write_file(
1399 &archive.dir().join(JOURNAL_FILE),
1400 Some(&keys(9)),
1401 JOURNAL_FILE,
1402 names.as_bytes(),
1403 )
1404 .unwrap();
1405 drop(archive);
1406
1407 let err = TranscriptArchive::open(&fx.archive_dir)
1408 .err()
1409 .expect("an unkeyed open fails");
1410 assert_eq!(seal_error(&err), Some(&SealError::KeyRequired));
1411 assert!(doomed.exists());
1412 }
1413
1414 fn sealed_manifest_lines(dir: &Path) -> Vec<String> {
1415 match fs::read_to_string(dir.join(SEALED_MANIFEST_FILE)) {
1416 Ok(text) => text.lines().map(str::to_string).collect(),
1417 Err(e) if e.kind() == ErrorKind::NotFound => Vec::new(),
1418 Err(e) => panic!("{e}"),
1419 }
1420 }
1421
1422 fn line_from_a_binary_before_sealing(
1423 fx: &Fixture,
1424 source: &Path,
1425 archived_at: DateTime<Utc>,
1426 ) -> String {
1427 let bytes = fs::read(source).unwrap();
1428 let sha256 = hex(&Sha256::digest(&bytes));
1429 let file = format!("{sha256}{BLOB_SUFFIX}");
1430 fs::write(
1431 fx.archive_dir.join(&file),
1432 zstd::encode_all(bytes.as_slice(), 0).unwrap(),
1433 )
1434 .unwrap();
1435 serde_json::to_string(&ArchiveEntry {
1436 source: source_key(source),
1437 file,
1438 sha256,
1439 archived_at,
1440 })
1441 .unwrap()
1442 }
1443
1444 #[test]
1445 fn a_sealed_archive_leaves_the_plaintext_manifest_readable_by_a_binary_from_before_sealing() {
1446 let fx = Fixture::new();
1447 let first = fx.write("a.jsonl", "{\"a\":1}\n");
1448 let plain = fx.open();
1449 plain.archive(&first).unwrap();
1450 drop(plain);
1451
1452 let archive = fx.open_sealed(keys(10));
1453 let second = fx.write("b.jsonl", "{\"b\":2}\n");
1454 archive.archive(&second).unwrap();
1455 archive.reseal().unwrap();
1456 let third = fx.write("c.jsonl", "{\"c\":3}\n");
1457 archive.archive(&third).unwrap();
1458
1459 for line in manifest_lines(&archive) {
1460 serde_json::from_str::<ArchiveEntry>(&line).unwrap();
1461 }
1462 assert_eq!(sealed_manifest_lines(archive.dir()).len(), 3);
1463 }
1464
1465 #[test]
1466 fn entries_a_binary_from_before_sealing_appended_are_merged_and_then_sealed() {
1467 let fx = Fixture::new();
1468 let kept = fx.write("kept.jsonl", "{\"k\":1}\n");
1469 let changed = fx.write("changed.jsonl", "{\"v\":1}\n");
1470 let archive = fx.open_sealed(keys(11));
1471 archive.archive(&kept).unwrap();
1472 archive.archive(&changed).unwrap();
1473 let old_blob = archive.entry(&changed).unwrap().file;
1474 drop(archive);
1475
1476 let added = fx.write("added.jsonl", "{\"n\":1}\n");
1477 fs::write(&changed, "{\"v\":2}\n").unwrap();
1478 let later = Utc::now() + chrono::Duration::seconds(1);
1479 let lines = [
1480 line_from_a_binary_before_sealing(&fx, &added, later),
1481 line_from_a_binary_before_sealing(&fx, &changed, later),
1482 ];
1483 let mut manifest = fs::OpenOptions::new()
1484 .append(true)
1485 .open(fx.archive_dir.join(MANIFEST_FILE))
1486 .unwrap();
1487 for line in &lines {
1488 writeln!(manifest, "{line}").unwrap();
1489 }
1490 drop(manifest);
1491
1492 let reopened = fx.open_sealed(keys(11));
1493 assert_eq!(reopened.len(), 3);
1494 assert_eq!(
1495 reopened.read(&added).unwrap().as_deref(),
1496 Some("{\"n\":1}\n")
1497 );
1498 assert_eq!(
1499 reopened.read(&changed).unwrap().as_deref(),
1500 Some("{\"v\":2}\n")
1501 );
1502 assert_eq!(
1503 reopened.read(&kept).unwrap().as_deref(),
1504 Some("{\"k\":1}\n")
1505 );
1506 assert!(!reopened.dir().join(&old_blob).exists());
1507 assert!(manifest_lines(&reopened).is_empty());
1508 assert!(reopened.holds_unsealed());
1509
1510 reopened.reseal().unwrap();
1511 assert!(!reopened.holds_unsealed());
1512 assert_all_sealed_under(reopened.dir(), &keys(11));
1513 drop(reopened);
1514 let again = fx.open_sealed(keys(11));
1515 assert_eq!(again.len(), 3);
1516 assert!(!again.holds_unsealed());
1517 }
1518
1519 #[test]
1520 fn a_removal_after_keys_arrive_leaves_no_plaintext_entry_behind() {
1521 let fx = Fixture::new();
1522 let gone = fx.write("gone.jsonl", "{\"line\":\"gone\"}\n");
1523 let kept = fx.write("kept.jsonl", "{\"line\":\"kept\"}\n");
1524 let archive = fx.open();
1525 archive.archive(&gone).unwrap();
1526 archive.archive(&kept).unwrap();
1527
1528 archive.set_seal_keys(keys(12));
1529 archive.remove_where(|e| e.source.contains("gone")).unwrap();
1530 assert!(manifest_lines(&archive).is_empty());
1531 drop(archive);
1532
1533 let reopened = fx.open_sealed(keys(12));
1534 assert_eq!(reopened.len(), 1);
1535 assert!(reopened.entry(&gone).is_none());
1536 }
1537
1538 #[test]
1539 fn a_fold_cut_short_before_its_reseal_still_reports_unsealed_files_after_a_reopen() {
1540 let fx = Fixture::new();
1541 let first = fx.write("a.jsonl", "{\"a\":1}\n");
1542 let plain = fx.open();
1543 plain.archive(&first).unwrap();
1544 drop(plain);
1545
1546 let folded = fx.open_sealed(keys(13));
1547 assert!(folded.holds_unsealed());
1548 assert!(manifest_lines(&folded).is_empty());
1549 drop(folded);
1550
1551 let reopened = fx.open_sealed(keys(13));
1552 assert!(reopened.holds_unsealed());
1553 reopened.reseal().unwrap();
1554 assert_all_sealed_under(reopened.dir(), &keys(13));
1555 drop(reopened);
1556 assert!(!fx.open_sealed(keys(13)).holds_unsealed());
1557 }
1558}