Skip to main content

second_brain_core/
transcript_archive.rs

1use std::collections::HashMap;
2use std::fs::{self, OpenOptions};
3use std::io::{ErrorKind, Write};
4use std::path::{Path, PathBuf};
5use std::sync::{Mutex, MutexGuard, PoisonError};
6
7use anyhow::{Context, Result, anyhow, bail};
8use base64::Engine;
9use base64::prelude::BASE64_STANDARD;
10use chrono::{DateTime, Utc};
11use serde::{Deserialize, Serialize};
12use sha2::{Digest, Sha256};
13
14use crate::redact::redact;
15use crate::sealed::{self, SealError, SealKeys, seal_error};
16
17pub const MANIFEST_FILE: &str = "manifest.jsonl";
18/// Sealed manifest records, one base64 line each. A binary from before
19/// sealing reads only `MANIFEST_FILE`, so it never drops these as torn.
20pub const SEALED_MANIFEST_FILE: &str = "manifest.sealed";
21
22const JOURNAL_FILE: &str = "removal.journal";
23const RESEAL_MARKER: &str = "reseal.pending";
24const BLOB_SUFFIX: &str = ".jsonl.zst";
25const LEGACY_TMP_SUFFIX: &str = ".tmp";
26const COMPACTION_SLACK: usize = 64;
27
28#[derive(Debug, Clone, Copy, PartialEq, Eq)]
29pub enum Archived {
30    Stored,
31    Refreshed,
32    Unchanged,
33    Copy,
34}
35
36#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
37pub struct ArchiveEntry {
38    pub source: String,
39    pub file: String,
40    pub sha256: String,
41    pub archived_at: DateTime<Utc>,
42}
43
44/// Redacted zstd copies of transcripts, keyed by source path; an entry is
45/// never dropped because its source was deleted, only by `remove_where`.
46///
47/// Opened sealed, every blob and the removal journal is written under the
48/// current key and manifest records go to `SEALED_MANIFEST_FILE`. Plaintext
49/// blobs are read as-is until `reseal`, and plaintext manifest entries (from
50/// before sealing, or appended by an older binary) are folded into the sealed
51/// manifest at open, the later of two entries for one source winning.
52pub struct TranscriptArchive {
53    dir: PathBuf,
54    state: Mutex<State>,
55}
56
57struct State {
58    entries: HashMap<String, ArchiveEntry>,
59    manifest_lines: usize,
60    plaintext_lines: usize,
61    manifest_damaged: bool,
62    unsealed: bool,
63    keys: Option<SealKeys>,
64}
65
66impl State {
67    fn needs_compaction(&self) -> bool {
68        self.manifest_damaged || self.manifest_lines > 2 * self.entries.len() + COMPACTION_SLACK
69    }
70
71    fn plaintext_pending(&self) -> bool {
72        self.keys.is_some() && self.plaintext_lines > 0
73    }
74
75    fn holder_of(&self, sha256: &str) -> Option<&ArchiveEntry> {
76        self.entries.values().find(|e| e.sha256 == sha256)
77    }
78
79    fn references(&self, file: &str) -> bool {
80        self.entries.values().any(|e| e.file == file)
81    }
82}
83
84impl TranscriptArchive {
85    pub fn open(dir: impl Into<PathBuf>) -> Result<Self> {
86        Self::open_with(dir.into(), None)
87    }
88
89    /// Opens the archive writing everything sealed under `keys`. An archive
90    /// written before sealing opens as it is; `reseal` seals what it holds.
91    pub fn open_sealed(dir: impl Into<PathBuf>, keys: SealKeys) -> Result<Self> {
92        Self::open_with(dir.into(), Some(keys))
93    }
94
95    fn open_with(dir: PathBuf, keys: Option<SealKeys>) -> Result<Self> {
96        fs::create_dir_all(&dir)
97            .with_context(|| format!("creating the transcript archive at {}", dir.display()))?;
98        let manifest = dir.join(MANIFEST_FILE);
99        if !manifest.exists() {
100            sealed::write_file(&manifest, None, MANIFEST_FILE, b"")
101                .with_context(|| format!("creating {}", manifest.display()))?;
102        }
103        let (state, torn, superseded) = load_manifest(&dir, keys)?;
104        let archive = Self {
105            dir,
106            state: Mutex::new(state),
107        };
108        {
109            let mut state = archive.state();
110            if torn || state.needs_compaction() || state.plaintext_pending() {
111                archive.compact(&mut state)?;
112            }
113            for file in superseded.iter().filter(|f| !state.references(f)) {
114                archive.remove_blob(file);
115            }
116            archive.finish_interrupted_removal(&state)?;
117        }
118        Ok(archive)
119    }
120
121    /// True while files written in plaintext may remain: plaintext manifest
122    /// entries were loaded or written, by this process or by one that died
123    /// before its `reseal` finished, and no `reseal` has run since.
124    pub fn holds_unsealed(&self) -> bool {
125        self.state().unsealed
126    }
127
128    /// None when the archive writes plaintext.
129    pub fn seal_keys(&self) -> Option<SealKeys> {
130        self.state().keys.clone()
131    }
132
133    /// Seals what is written from now on under the current key of `keys`,
134    /// which must also hold every key the archive's files are sealed under.
135    pub fn set_seal_keys(&self, keys: SealKeys) {
136        self.state().keys = Some(keys);
137    }
138
139    /// Rewrites every blob, the manifest and any removal journal that is
140    /// plaintext or sealed under a retired key under the current key, and
141    /// deletes temporary files a crashed write left. Returns how many files
142    /// were rewritten. Each blob is resealed under the state lock, so a
143    /// concurrent removal never sees its blob renamed back into place.
144    pub fn reseal(&self) -> Result<usize> {
145        if self.state().keys.is_none() {
146            return Err(no_keys());
147        }
148        let mut resealed = 0;
149        for file in self.blob_names()? {
150            let state = self.state();
151            let keys = state.keys.clone().ok_or_else(no_keys)?;
152            let path = self.dir.join(&file);
153            if path.exists() && sealed::reseal_file(&path, &keys, &file)? {
154                resealed += 1;
155            }
156        }
157        let mut state = self.state();
158        let keys = state.keys.clone().ok_or_else(no_keys)?;
159        let journal = self.dir.join(JOURNAL_FILE);
160        if journal.exists() && sealed::reseal_file(&journal, &keys, JOURNAL_FILE)? {
161            resealed += 1;
162        }
163        if !self.manifest_sealed_under(&keys)? {
164            self.compact(&mut state)?;
165            resealed += 1;
166        }
167        self.remove_tmp_files()?;
168        let marker = self.dir.join(RESEAL_MARKER);
169        match fs::remove_file(&marker) {
170            Ok(()) => {}
171            Err(e) if e.kind() == ErrorKind::NotFound => {}
172            Err(e) => return Err(e).with_context(|| format!("removing {}", marker.display())),
173        }
174        state.unsealed = false;
175        Ok(resealed)
176    }
177
178    /// Drops every entry `doomed` picks: the blobs to delete are journaled,
179    /// then the manifest is rewritten, then the blobs go and the journal with
180    /// them. A crash in between is finished at the next open.
181    pub fn remove_where(
182        &self,
183        doomed: impl Fn(&ArchiveEntry) -> bool,
184    ) -> Result<Vec<ArchiveEntry>> {
185        let mut state = self.state();
186        if state.plaintext_pending() {
187            self.compact(&mut state)?;
188        }
189        let removed: Vec<ArchiveEntry> = state
190            .entries
191            .values()
192            .filter(|e| doomed(e))
193            .cloned()
194            .collect();
195        if removed.is_empty() {
196            return Ok(removed);
197        }
198        for entry in &removed {
199            state.entries.remove(&entry.source);
200        }
201        let mut doomed_blobs: Vec<&str> = removed
202            .iter()
203            .map(|e| e.file.as_str())
204            .filter(|file| !state.references(file))
205            .collect();
206        doomed_blobs.sort_unstable();
207        doomed_blobs.dedup();
208        if !doomed_blobs.is_empty() {
209            let journal = self.dir.join(JOURNAL_FILE);
210            sealed::write_file(
211                &journal,
212                state.keys.as_ref(),
213                JOURNAL_FILE,
214                doomed_blobs.join("\n").as_bytes(),
215            )
216            .with_context(|| format!("writing {}", journal.display()))?;
217        }
218        self.compact(&mut state)?;
219        for file in &doomed_blobs {
220            self.remove_blob(file);
221        }
222        self.remove_journal();
223        Ok(removed)
224    }
225
226    /// An entry whose blob has this sha256, if any does.
227    pub fn holder(&self, sha256: &str) -> Option<ArchiveEntry> {
228        self.state().holder_of(sha256).cloned()
229    }
230
231    pub fn dir(&self) -> &Path {
232        &self.dir
233    }
234
235    pub fn archive(&self, source: &Path) -> Result<Archived> {
236        let mut state = self.state();
237        let bytes = fs::read(source).with_context(|| format!("reading {}", source.display()))?;
238        let sha256 = hex(&Sha256::digest(&bytes));
239        let key = source_key(source);
240
241        let previous = state.entries.get(&key).cloned();
242        if let Some(prev) = &previous
243            && prev.sha256 == sha256
244            && self.dir.join(&prev.file).exists()
245        {
246            return Ok(Archived::Unchanged);
247        }
248
249        let file = format!("{sha256}{BLOB_SUFFIX}");
250        let shared = state
251            .holder_of(&sha256)
252            .is_some_and(|holder| self.dir.join(&holder.file).exists());
253        if !shared {
254            self.write_blob(state.keys.as_ref(), &file, &bytes)?;
255        }
256        let entry = ArchiveEntry {
257            source: key.clone(),
258            file,
259            sha256,
260            archived_at: Utc::now(),
261        };
262        state.entries.insert(key, entry.clone());
263        if state.needs_compaction() {
264            self.compact(&mut state)?;
265        } else if let Err(e) = self.append(&mut state, &entry) {
266            state.manifest_damaged = true;
267            return Err(e);
268        }
269
270        if let Some(prev) = &previous
271            && !state.references(&prev.file)
272        {
273            self.remove_blob(&prev.file);
274        }
275
276        Ok(match (shared, previous) {
277            (true, _) => Archived::Copy,
278            (false, None) => Archived::Stored,
279            (false, Some(_)) => Archived::Refreshed,
280        })
281    }
282
283    pub fn entry(&self, source: &Path) -> Option<ArchiveEntry> {
284        self.state().entries.get(&source_key(source)).cloned()
285    }
286
287    pub fn read(&self, source: &Path) -> Result<Option<String>> {
288        let (entry, keys) = {
289            let state = self.state();
290            (
291                state.entries.get(&source_key(source)).cloned(),
292                state.keys.clone(),
293            )
294        };
295        let Some(entry) = entry else {
296            return Ok(None);
297        };
298        let path = self.dir.join(&entry.file);
299        let compressed = sealed::read_file(&path, keys.as_ref(), &entry.file)?;
300        let bytes = zstd::decode_all(compressed.as_slice())
301            .with_context(|| format!("decompressing {}", path.display()))?;
302        let text =
303            String::from_utf8(bytes).with_context(|| format!("decoding {}", path.display()))?;
304        Ok(Some(text))
305    }
306
307    /// Every archived transcript, in no particular order.
308    pub fn entries(&self) -> Vec<ArchiveEntry> {
309        self.state().entries.values().cloned().collect()
310    }
311
312    pub fn len(&self) -> usize {
313        self.state().entries.len()
314    }
315
316    pub fn is_empty(&self) -> bool {
317        self.len() == 0
318    }
319
320    fn state(&self) -> MutexGuard<'_, State> {
321        self.state.lock().unwrap_or_else(PoisonError::into_inner)
322    }
323
324    fn write_blob(&self, keys: Option<&SealKeys>, file: &str, source_bytes: &[u8]) -> Result<()> {
325        let text = redact(&String::from_utf8_lossy(source_bytes)).text;
326        let compressed =
327            zstd::encode_all(text.as_bytes(), 0).context("compressing a transcript")?;
328        let target = self.dir.join(file);
329        sealed::write_file(&target, keys, file, &compressed)
330            .with_context(|| format!("writing {}", target.display()))
331    }
332
333    fn blob_names(&self) -> Result<Vec<String>> {
334        let mut names = Vec::new();
335        for entry in
336            fs::read_dir(&self.dir).with_context(|| format!("listing {}", self.dir.display()))?
337        {
338            let name = entry
339                .with_context(|| format!("listing {}", self.dir.display()))?
340                .file_name()
341                .to_string_lossy()
342                .into_owned();
343            if name.ends_with(BLOB_SUFFIX) {
344                names.push(name);
345            }
346        }
347        Ok(names)
348    }
349
350    fn manifest_sealed_under(&self, keys: &SealKeys) -> Result<bool> {
351        let plaintext = read_manifest_text(&self.dir.join(MANIFEST_FILE))?;
352        if plaintext.lines().any(|l| !l.trim().is_empty()) {
353            return Ok(false);
354        }
355        let text = read_manifest_text(&self.dir.join(SEALED_MANIFEST_FILE))?;
356        Ok(text.lines().filter(|l| !l.trim().is_empty()).all(|line| {
357            BASE64_STANDARD
358                .decode(line.trim())
359                .is_ok_and(|bytes| sealed::key_id_of(&bytes) == Some(keys.current_id()))
360        }))
361    }
362
363    /// Only safe under the state lock: every write this archive makes runs
364    /// under it, so no temporary file is in use.
365    fn remove_tmp_files(&self) -> Result<()> {
366        for entry in
367            fs::read_dir(&self.dir).with_context(|| format!("listing {}", self.dir.display()))?
368        {
369            let entry = entry.with_context(|| format!("listing {}", self.dir.display()))?;
370            let name = entry.file_name().to_string_lossy().into_owned();
371            if sealed::is_tmp_name(&name) || name.ends_with(LEGACY_TMP_SUFFIX) {
372                fs::remove_file(entry.path())
373                    .or_else(|e| match e.kind() {
374                        ErrorKind::NotFound => Ok(()),
375                        _ => Err(e),
376                    })
377                    .with_context(|| format!("removing {}", entry.path().display()))?;
378            }
379        }
380        Ok(())
381    }
382
383    /// Deletes the blobs a removal journal names that no entry uses, then the
384    /// journal. Blobs the journal does not name are never touched, so a lost
385    /// manifest cannot delete an archive.
386    fn finish_interrupted_removal(&self, state: &State) -> Result<()> {
387        let journal = self.dir.join(JOURNAL_FILE);
388        if !journal.exists() {
389            return Ok(());
390        }
391        let bytes = sealed::read_file(&journal, state.keys.as_ref(), JOURNAL_FILE)?;
392        let text =
393            String::from_utf8(bytes).with_context(|| format!("decoding {}", journal.display()))?;
394        for name in text.lines().map(str::trim) {
395            let is_blob_name = name.ends_with(BLOB_SUFFIX) && !name.contains(['/', '\\']);
396            if is_blob_name && !state.references(name) {
397                self.remove_blob(name);
398            }
399        }
400        self.remove_journal();
401        Ok(())
402    }
403
404    fn remove_journal(&self) {
405        let path = self.dir.join(JOURNAL_FILE);
406        match fs::remove_file(&path) {
407            Ok(()) => {}
408            Err(e) if e.kind() == ErrorKind::NotFound => {}
409            Err(e) => {
410                tracing::warn!(path = %path.display(), error = %e, "removing the archive removal journal")
411            }
412        }
413    }
414
415    fn remove_blob(&self, file: &str) {
416        let path = self.dir.join(file);
417        match fs::remove_file(&path) {
418            Ok(()) => {}
419            Err(e) if e.kind() == ErrorKind::NotFound => {}
420            Err(e) => {
421                tracing::warn!(path = %path.display(), error = %e, "removing an unused archive blob")
422            }
423        }
424    }
425
426    fn append(&self, state: &mut State, entry: &ArchiveEntry) -> Result<()> {
427        let path = self.dir.join(manifest_file(state.keys.as_ref()));
428        let mut line = manifest_line(state.keys.as_ref(), entry)?;
429        line.push('\n');
430        let mut file = OpenOptions::new()
431            .create(true)
432            .append(true)
433            .open(&path)
434            .with_context(|| format!("opening {}", path.display()))?;
435        file.write_all(line.as_bytes())
436            .and_then(|()| file.sync_data())
437            .with_context(|| format!("appending to {}", path.display()))?;
438        state.manifest_lines += 1;
439        if state.keys.is_none() {
440            state.plaintext_lines += 1;
441            state.unsealed = true;
442        }
443        Ok(())
444    }
445
446    fn compact(&self, state: &mut State) -> Result<()> {
447        let mut entries: Vec<&ArchiveEntry> = state.entries.values().collect();
448        entries.sort_by(|a, b| a.source.cmp(&b.source));
449        let mut body = String::new();
450        for entry in &entries {
451            body.push_str(&manifest_line(state.keys.as_ref(), entry)?);
452            body.push('\n');
453        }
454        let count = entries.len();
455        let name = manifest_file(state.keys.as_ref());
456        let path = self.dir.join(name);
457        sealed::write_file(&path, None, name, body.as_bytes())
458            .with_context(|| format!("compacting {}", path.display()))?;
459        if state.keys.is_some() {
460            let plaintext = self.dir.join(MANIFEST_FILE);
461            if fs::metadata(&plaintext).is_ok_and(|m| m.len() > 0) {
462                let marker = self.dir.join(RESEAL_MARKER);
463                sealed::write_file(&marker, None, RESEAL_MARKER, b"")
464                    .with_context(|| format!("writing {}", marker.display()))?;
465                sealed::write_file(&plaintext, None, MANIFEST_FILE, b"")
466                    .with_context(|| format!("emptying {}", plaintext.display()))?;
467            }
468            state.plaintext_lines = 0;
469        } else {
470            state.plaintext_lines = count;
471            state.unsealed |= count > 0;
472        }
473        state.manifest_lines = count;
474        state.manifest_damaged = false;
475        Ok(())
476    }
477}
478
479fn no_keys() -> anyhow::Error {
480    anyhow!("the transcript archive has no sealing keys to reseal under")
481}
482
483fn manifest_file(keys: Option<&SealKeys>) -> &'static str {
484    match keys {
485        Some(_) => SEALED_MANIFEST_FILE,
486        None => MANIFEST_FILE,
487    }
488}
489
490fn manifest_line(keys: Option<&SealKeys>, entry: &ArchiveEntry) -> Result<String> {
491    let json = serde_json::to_string(entry)?;
492    match keys {
493        Some(keys) => {
494            Ok(BASE64_STANDARD.encode(sealed::seal(keys, SEALED_MANIFEST_FILE, json.as_bytes())?))
495        }
496        None => Ok(json),
497    }
498}
499
500fn parse_plaintext_line(line: &str) -> Result<ArchiveEntry> {
501    Ok(serde_json::from_str(line)?)
502}
503
504fn parse_sealed_line(line: &str, keys: Option<&SealKeys>) -> Result<ArchiveEntry> {
505    let sealed_bytes = BASE64_STANDARD.decode(line.trim())?;
506    if !sealed::looks_sealed(&sealed_bytes) {
507        bail!("the line is not a sealed record");
508    }
509    let keys = keys.ok_or(SealError::KeyRequired)?;
510    Ok(serde_json::from_slice(&sealed::open(
511        keys,
512        SEALED_MANIFEST_FILE,
513        &sealed_bytes,
514    )?)?)
515}
516
517fn read_manifest_text(path: &Path) -> Result<String> {
518    match fs::read_to_string(path) {
519        Ok(text) => Ok(text),
520        Err(e) if e.kind() == ErrorKind::NotFound => Ok(String::new()),
521        Err(e) => Err(e).with_context(|| format!("reading {}", path.display())),
522    }
523}
524
525/// Also returns the line count and a flag that is true when a line failed to
526/// parse: a torn tail would swallow the next appended line, so the caller
527/// rewrites the manifest. A line sealed under a key not held is an error
528/// rather than a torn line, so a missing key never compacts entries away.
529fn read_manifest(
530    path: &Path,
531    parse: impl Fn(&str) -> Result<ArchiveEntry>,
532) -> Result<(HashMap<String, ArchiveEntry>, usize, bool)> {
533    let text = read_manifest_text(path)?;
534    let mut entries = HashMap::new();
535    let mut lines = 0;
536    let mut torn = !text.is_empty() && !text.ends_with('\n');
537    for line in text.lines().filter(|l| !l.trim().is_empty()) {
538        lines += 1;
539        match parse(line) {
540            Ok(entry) => {
541                entries.insert(entry.source.clone(), entry);
542            }
543            Err(e) if is_missing_key(&e) => {
544                return Err(e).with_context(|| format!("reading {}", path.display()));
545            }
546            Err(e) => {
547                torn = true;
548                tracing::warn!(path = %path.display(), error = %e, "ignoring an unreadable archive manifest line");
549            }
550        }
551    }
552    Ok((entries, lines, torn))
553}
554
555/// Merges the plaintext manifest over the sealed one, the later entry for a
556/// source winning; also returns the torn flag and the blobs of the entries
557/// that lost, which the caller removes once the merge is written.
558fn load_manifest(dir: &Path, keys: Option<SealKeys>) -> Result<(State, bool, Vec<String>)> {
559    let (mut entries, sealed_lines, sealed_torn) =
560        read_manifest(&dir.join(SEALED_MANIFEST_FILE), |line| {
561            parse_sealed_line(line, keys.as_ref())
562        })?;
563    let (plaintext, plaintext_lines, plaintext_torn) =
564        read_manifest(&dir.join(MANIFEST_FILE), parse_plaintext_line)?;
565    let unsealed = !plaintext.is_empty() || dir.join(RESEAL_MARKER).exists();
566    let mut superseded = Vec::new();
567    for (source, entry) in plaintext {
568        match entries.get(&source) {
569            Some(held) if held.archived_at > entry.archived_at => superseded.push(entry.file),
570            Some(held) => {
571                superseded.push(held.file.clone());
572                entries.insert(source, entry);
573            }
574            None => {
575                entries.insert(source, entry);
576            }
577        }
578    }
579    let state = State {
580        entries,
581        manifest_lines: sealed_lines + plaintext_lines,
582        plaintext_lines,
583        manifest_damaged: false,
584        unsealed,
585        keys,
586    };
587    Ok((state, sealed_torn || plaintext_torn, superseded))
588}
589
590fn is_missing_key(error: &anyhow::Error) -> bool {
591    matches!(
592        seal_error(error),
593        Some(SealError::KeyRequired | SealError::UnknownKey(_))
594    )
595}
596
597fn source_key(source: &Path) -> String {
598    source.to_string_lossy().into_owned()
599}
600
601fn hex(bytes: &[u8]) -> String {
602    bytes.iter().map(|b| format!("{b:02x}")).collect()
603}
604
605#[cfg(test)]
606mod tests {
607    use std::fs;
608    use std::io::Write;
609
610    use base64::Engine;
611    use base64::prelude::BASE64_STANDARD;
612
613    use super::*;
614    use crate::keys::KeyGuard;
615    use crate::sealed::{self, SealError, SealKey, seal_error};
616
617    fn blobs(archive: &TranscriptArchive) -> Vec<String> {
618        let mut names: Vec<String> = fs::read_dir(archive.dir())
619            .unwrap()
620            .map(|e| e.unwrap().file_name().to_string_lossy().into_owned())
621            .filter(|n| n.ends_with(".jsonl.zst"))
622            .collect();
623        names.sort();
624        names
625    }
626
627    fn manifest_lines(archive: &TranscriptArchive) -> Vec<String> {
628        fs::read_to_string(archive.dir().join(MANIFEST_FILE))
629            .unwrap()
630            .lines()
631            .map(str::to_string)
632            .collect()
633    }
634
635    struct Fixture {
636        _tmp: tempfile::TempDir,
637        projects: PathBuf,
638        archive_dir: PathBuf,
639    }
640
641    impl Fixture {
642        fn new() -> Self {
643            let tmp = tempfile::tempdir().unwrap();
644            let projects = tmp.path().join("projects");
645            fs::create_dir_all(&projects).unwrap();
646            let archive_dir = tmp.path().join("data").join("transcripts");
647            Self {
648                _tmp: tmp,
649                projects,
650                archive_dir,
651            }
652        }
653
654        fn write(&self, name: &str, text: &str) -> PathBuf {
655            let path = self.projects.join(name);
656            fs::write(&path, text).unwrap();
657            path
658        }
659
660        fn open(&self) -> TranscriptArchive {
661            TranscriptArchive::open(&self.archive_dir).unwrap()
662        }
663    }
664
665    #[test]
666    fn open_creates_the_directory() {
667        let fx = Fixture::new();
668        let archive = fx.open();
669        assert!(archive.dir().is_dir());
670        assert!(archive.is_empty());
671    }
672
673    #[test]
674    fn a_new_source_is_stored_then_unchanged() {
675        let fx = Fixture::new();
676        let source = fx.write("a.jsonl", "{\"line\":1}\n");
677        let archive = fx.open();
678
679        assert_eq!(archive.archive(&source).unwrap(), Archived::Stored);
680        assert_eq!(archive.archive(&source).unwrap(), Archived::Unchanged);
681
682        assert_eq!(archive.len(), 1);
683        assert_eq!(blobs(&archive).len(), 1);
684        assert_eq!(manifest_lines(&archive).len(), 1);
685        let entry = archive.entry(&source).unwrap();
686        assert_eq!(entry.source, source.to_string_lossy());
687        assert_eq!(entry.file, format!("{}.jsonl.zst", entry.sha256));
688        assert_eq!(entry.sha256.len(), 64);
689        assert_eq!(
690            archive.read(&source).unwrap().as_deref(),
691            Some("{\"line\":1}\n")
692        );
693    }
694
695    #[test]
696    fn a_changed_source_is_refreshed_and_its_old_blob_removed() {
697        let fx = Fixture::new();
698        let source = fx.write("a.jsonl", "{\"line\":1}\n");
699        let archive = fx.open();
700        archive.archive(&source).unwrap();
701        let old = archive.entry(&source).unwrap();
702
703        fs::write(&source, "{\"line\":1}\n{\"line\":2}\n").unwrap();
704        assert_eq!(archive.archive(&source).unwrap(), Archived::Refreshed);
705
706        let new = archive.entry(&source).unwrap();
707        assert_ne!(new.sha256, old.sha256);
708        assert_eq!(
709            archive.read(&source).unwrap().as_deref(),
710            Some("{\"line\":1}\n{\"line\":2}\n")
711        );
712        assert_eq!(blobs(&archive), vec![new.file]);
713        assert!(!archive.dir().join(&old.file).exists());
714    }
715
716    #[test]
717    fn a_copy_at_another_path_shares_the_blob() {
718        let fx = Fixture::new();
719        let first = fx.write("a.jsonl", "{\"same\":true}\n");
720        let second = fx.write("b.jsonl", "{\"same\":true}\n");
721        let archive = fx.open();
722
723        assert_eq!(archive.archive(&first).unwrap(), Archived::Stored);
724        assert_eq!(archive.archive(&second).unwrap(), Archived::Copy);
725
726        assert_eq!(archive.len(), 2);
727        assert_eq!(blobs(&archive).len(), 1);
728        assert_eq!(
729            archive.entry(&first).unwrap().file,
730            archive.entry(&second).unwrap().file
731        );
732        assert_eq!(
733            archive.read(&second).unwrap().as_deref(),
734            Some("{\"same\":true}\n")
735        );
736    }
737
738    #[test]
739    fn refreshing_one_copy_keeps_the_blob_the_other_still_uses() {
740        let fx = Fixture::new();
741        let first = fx.write("a.jsonl", "{\"same\":true}\n");
742        let second = fx.write("b.jsonl", "{\"same\":true}\n");
743        let archive = fx.open();
744        archive.archive(&first).unwrap();
745        archive.archive(&second).unwrap();
746
747        fs::write(&first, "{\"same\":false}\n").unwrap();
748        assert_eq!(archive.archive(&first).unwrap(), Archived::Refreshed);
749
750        assert_eq!(blobs(&archive).len(), 2);
751        assert_eq!(
752            archive.read(&second).unwrap().as_deref(),
753            Some("{\"same\":true}\n")
754        );
755    }
756
757    #[test]
758    fn the_archive_outlives_a_deleted_source() {
759        let fx = Fixture::new();
760        let source = fx.write("a.jsonl", "{\"kept\":true}\n");
761        let archive = fx.open();
762        archive.archive(&source).unwrap();
763
764        fs::remove_file(&source).unwrap();
765
766        assert!(archive.archive(&source).is_err());
767        assert_eq!(
768            archive.read(&source).unwrap().as_deref(),
769            Some("{\"kept\":true}\n")
770        );
771        assert_eq!(blobs(&archive).len(), 1);
772        let reopened = fx.open();
773        assert_eq!(
774            reopened.read(&source).unwrap().as_deref(),
775            Some("{\"kept\":true}\n")
776        );
777    }
778
779    #[test]
780    fn an_unknown_source_reads_as_none() {
781        let fx = Fixture::new();
782        let archive = fx.open();
783        let missing = fx.projects.join("never.jsonl");
784        assert!(archive.entry(&missing).is_none());
785        assert_eq!(archive.read(&missing).unwrap(), None);
786    }
787
788    #[test]
789    fn archived_text_is_redacted() {
790        let fx = Fixture::new();
791        let secret = format!("gh{}_{}", "p", "aB3dE6".repeat(6));
792        let source = fx.write(
793            "a.jsonl",
794            &format!("{{\"text\":\"token {secret} and the build passed\"}}\n"),
795        );
796        let archive = fx.open();
797        archive.archive(&source).unwrap();
798
799        let text = archive.read(&source).unwrap().unwrap();
800        assert!(!text.contains(&secret));
801        assert!(text.contains("the build passed"));
802        assert!(text.contains("[REDACTED:github_token]"));
803
804        let entry = archive.entry(&source).unwrap();
805        let raw = fs::read(archive.dir().join(&entry.file)).unwrap();
806        let decoded = String::from_utf8(zstd::decode_all(raw.as_slice()).unwrap()).unwrap();
807        assert!(!decoded.contains(&secret));
808        assert!(decoded.contains("the build passed"));
809    }
810
811    #[test]
812    fn the_blob_is_named_by_the_source_bytes() {
813        let fx = Fixture::new();
814        let secret = format!("gh{}_{}", "p", "aB3dE6".repeat(6));
815        let source = fx.write("a.jsonl", &format!("{secret}\n"));
816        let archive = fx.open();
817        archive.archive(&source).unwrap();
818
819        let changed = format!("gh{}_{}", "p", "zY9xW8".repeat(6));
820        fs::write(&source, format!("{changed}\n")).unwrap();
821        assert_eq!(archive.archive(&source).unwrap(), Archived::Refreshed);
822    }
823
824    #[test]
825    fn reopening_loads_the_manifest() {
826        let fx = Fixture::new();
827        let first = fx.write("a.jsonl", "{\"a\":1}\n");
828        let second = fx.write("b.jsonl", "{\"b\":2}\n");
829        let archive = fx.open();
830        archive.archive(&first).unwrap();
831        archive.archive(&second).unwrap();
832        let entry = archive.entry(&first).unwrap();
833        drop(archive);
834
835        let reopened = fx.open();
836        assert_eq!(reopened.len(), 2);
837        assert_eq!(reopened.entry(&first), Some(entry));
838        assert_eq!(reopened.archive(&first).unwrap(), Archived::Unchanged);
839        assert_eq!(
840            reopened.read(&second).unwrap().as_deref(),
841            Some("{\"b\":2}\n")
842        );
843    }
844
845    #[test]
846    fn a_torn_last_manifest_line_is_ignored() {
847        let fx = Fixture::new();
848        let first = fx.write("a.jsonl", "{\"a\":1}\n");
849        let second = fx.write("b.jsonl", "{\"b\":2}\n");
850        let archive = fx.open();
851        archive.archive(&first).unwrap();
852        archive.archive(&second).unwrap();
853        drop(archive);
854        let mut manifest = fs::OpenOptions::new()
855            .append(true)
856            .open(fx.archive_dir.join(MANIFEST_FILE))
857            .unwrap();
858        manifest.write_all(b"{\"source\":\"/x/c.js").unwrap();
859        drop(manifest);
860
861        let reopened = fx.open();
862        assert_eq!(reopened.len(), 2);
863
864        let third = fx.write("c.jsonl", "{\"c\":3}\n");
865        assert_eq!(reopened.archive(&third).unwrap(), Archived::Stored);
866        drop(reopened);
867        let again = fx.open();
868        assert_eq!(again.len(), 3);
869        assert_eq!(again.read(&third).unwrap().as_deref(), Some("{\"c\":3}\n"));
870    }
871
872    #[test]
873    fn a_failed_manifest_append_is_repaired_by_the_next_write() {
874        use std::os::unix::fs::PermissionsExt;
875
876        let fx = Fixture::new();
877        let first = fx.write("a.jsonl", "{\"a\":1}\n");
878        let second = fx.write("b.jsonl", "{\"b\":2}\n");
879        let third = fx.write("c.jsonl", "{\"c\":3}\n");
880        let archive = fx.open();
881        archive.archive(&first).unwrap();
882        let manifest = fx.archive_dir.join(MANIFEST_FILE);
883        fs::set_permissions(&manifest, fs::Permissions::from_mode(0o400)).unwrap();
884
885        assert!(archive.archive(&second).is_err());
886        assert_eq!(archive.archive(&third).unwrap(), Archived::Stored);
887        drop(archive);
888
889        let reopened = fx.open();
890        assert_eq!(reopened.len(), 3);
891        assert_eq!(
892            reopened.read(&second).unwrap().as_deref(),
893            Some("{\"b\":2}\n")
894        );
895    }
896
897    #[test]
898    fn compaction_keeps_the_latest_entry_per_source() {
899        let fx = Fixture::new();
900        let first = fx.write("a.jsonl", "{\"a\":1}\n");
901        let second = fx.write("b.jsonl", "{\"b\":1}\n");
902        let archive = fx.open();
903        archive.archive(&first).unwrap();
904        archive.archive(&second).unwrap();
905        let latest_second = archive.entry(&second).unwrap();
906        drop(archive);
907
908        let manifest = fx.archive_dir.join(MANIFEST_FILE);
909        let stale = ArchiveEntry {
910            source: first.to_string_lossy().into_owned(),
911            file: "0".repeat(64) + ".jsonl.zst",
912            sha256: "0".repeat(64),
913            archived_at: Utc::now(),
914        };
915        let mut lines = fs::read_to_string(&manifest).unwrap();
916        let mut stale_lines = String::new();
917        for _ in 0..80 {
918            stale_lines.push_str(&serde_json::to_string(&stale).unwrap());
919            stale_lines.push('\n');
920        }
921        lines.insert_str(0, &stale_lines);
922        fs::write(&manifest, &lines).unwrap();
923
924        let reopened = fx.open();
925        let kept = manifest_lines(&reopened);
926        assert_eq!(kept.len(), 2);
927        assert_eq!(reopened.len(), 2);
928        assert_ne!(reopened.entry(&first).unwrap().sha256, stale.sha256);
929        assert_eq!(reopened.entry(&second), Some(latest_second));
930        assert_eq!(
931            reopened.read(&first).unwrap().as_deref(),
932            Some("{\"a\":1}\n")
933        );
934    }
935
936    #[test]
937    fn a_long_running_archive_keeps_its_manifest_compact() {
938        let fx = Fixture::new();
939        let source = fx.write("a.jsonl", "");
940        let archive = fx.open();
941        let mut text = String::new();
942        for i in 0..100 {
943            text.push_str(&format!("{{\"line\":{i}}}\n"));
944            fs::write(&source, &text).unwrap();
945            archive.archive(&source).unwrap();
946        }
947
948        assert!(manifest_lines(&archive).len() <= 2 + 64);
949        assert_eq!(blobs(&archive).len(), 1);
950        drop(archive);
951        let reopened = fx.open();
952        assert_eq!(reopened.read(&source).unwrap(), Some(text));
953    }
954
955    #[test]
956    fn removed_entries_leave_the_manifest_and_their_blobs_and_stay_gone_on_reopen() {
957        let fx = Fixture::new();
958        let gone = fx.write("gone.jsonl", "{\"line\":\"secret\"}\n");
959        let copy = fx.write("copy.jsonl", "{\"line\":\"secret\"}\n");
960        let kept = fx.write("kept.jsonl", "{\"line\":\"kept\"}\n");
961        let archive = fx.open();
962        for source in [&gone, &copy, &kept] {
963            archive.archive(source).unwrap();
964        }
965        let sha = archive.entry(&gone).unwrap().sha256;
966        assert!(archive.holder(&sha).is_some());
967
968        let removed = archive.remove_where(|e| e.source.contains("gone")).unwrap();
969        assert_eq!(removed.len(), 1);
970        assert!(
971            archive.holder(&sha).is_some(),
972            "the copy still uses the blob"
973        );
974        assert_eq!(blobs(&archive).len(), 2);
975
976        archive.remove_where(|e| e.source.contains("copy")).unwrap();
977        assert!(archive.holder(&sha).is_none());
978        assert_eq!(blobs(&archive).len(), 1);
979        assert!(archive.entry(&gone).is_none());
980        assert!(manifest_lines(&archive).iter().all(|l| !l.contains("gone")));
981
982        let reopened = fx.open();
983        assert_eq!(reopened.len(), 1);
984        assert!(reopened.entry(&kept).is_some());
985    }
986
987    fn stray_blob(archive: &TranscriptArchive, fill: char) -> PathBuf {
988        let path = archive
989            .dir()
990            .join(format!("{}{BLOB_SUFFIX}", fill.to_string().repeat(64)));
991        fs::write(&path, b"unreferenced").unwrap();
992        path
993    }
994
995    #[test]
996    fn a_blob_no_entry_or_journal_names_survives_open() {
997        let fx = Fixture::new();
998        let source = fx.write("a.jsonl", "{\"line\":1}\n");
999        let archive = fx.open();
1000        archive.archive(&source).unwrap();
1001        let stray = stray_blob(&archive, 'f');
1002        drop(archive);
1003
1004        let reopened = fx.open();
1005        assert!(stray.exists());
1006        assert_eq!(blobs(&reopened).len(), 2);
1007    }
1008
1009    #[test]
1010    fn a_journal_left_by_a_crash_after_compaction_has_its_blobs_deleted_at_open() {
1011        let fx = Fixture::new();
1012        let kept = fx.write("kept.jsonl", "{\"line\":\"kept\"}\n");
1013        let archive = fx.open();
1014        archive.archive(&kept).unwrap();
1015        let doomed = stray_blob(&archive, 'a');
1016        let bystander = stray_blob(&archive, 'b');
1017        let kept_blob = archive.entry(&kept).unwrap().file;
1018        let names = format!(
1019            "{}\n{kept_blob}\n",
1020            doomed.file_name().unwrap().to_string_lossy()
1021        );
1022        fs::write(archive.dir().join(JOURNAL_FILE), names).unwrap();
1023        drop(archive);
1024
1025        let reopened = fx.open();
1026        assert!(!doomed.exists());
1027        assert!(bystander.exists());
1028        assert!(reopened.dir().join(&kept_blob).exists());
1029        assert!(!reopened.dir().join(JOURNAL_FILE).exists());
1030        assert_eq!(reopened.len(), 1);
1031    }
1032
1033    #[test]
1034    fn remove_where_leaves_no_journal_behind() {
1035        let fx = Fixture::new();
1036        let source = fx.write("a.jsonl", "{\"line\":1}\n");
1037        let archive = fx.open();
1038        archive.archive(&source).unwrap();
1039        archive.remove_where(|_| true).unwrap();
1040        assert!(!archive.dir().join(JOURNAL_FILE).exists());
1041        assert!(blobs(&archive).is_empty());
1042    }
1043
1044    #[test]
1045    fn a_missing_manifest_never_deletes_blobs() {
1046        let fx = Fixture::new();
1047        let source = fx.write("a.jsonl", "{\"line\":1}\n");
1048        let archive = fx.open();
1049        archive.archive(&source).unwrap();
1050        let dir = archive.dir().to_path_buf();
1051        drop(archive);
1052        fs::remove_file(dir.join(MANIFEST_FILE)).unwrap();
1053
1054        let reopened = fx.open();
1055        assert!(reopened.is_empty());
1056        assert_eq!(blobs(&reopened).len(), 1);
1057    }
1058
1059    fn key(byte: u8) -> SealKey {
1060        SealKey::from_bytes(KeyGuard::new([byte; 32])).unwrap()
1061    }
1062
1063    fn keys(byte: u8) -> SealKeys {
1064        SealKeys::single(key(byte))
1065    }
1066
1067    fn files(dir: &Path) -> Vec<PathBuf> {
1068        let mut paths: Vec<PathBuf> = fs::read_dir(dir)
1069            .unwrap()
1070            .map(|e| e.unwrap().path())
1071            .collect();
1072        paths.sort();
1073        paths
1074    }
1075
1076    fn contains(haystack: &[u8], needle: &[u8]) -> bool {
1077        haystack.windows(needle.len()).any(|w| w == needle)
1078    }
1079
1080    fn assert_all_sealed_under(dir: &Path, keys: &SealKeys) {
1081        for path in files(dir) {
1082            let name = path.file_name().unwrap().to_string_lossy().into_owned();
1083            if name == MANIFEST_FILE {
1084                assert_eq!(fs::read_to_string(&path).unwrap(), "");
1085            } else if name == SEALED_MANIFEST_FILE {
1086                let text = fs::read_to_string(&path).unwrap();
1087                for line in text.lines() {
1088                    let sealed = BASE64_STANDARD.decode(line).unwrap();
1089                    assert_eq!(sealed::key_id_of(&sealed), Some(keys.current_id()));
1090                }
1091            } else {
1092                assert_eq!(
1093                    sealed::sealed_key_id(&path).unwrap(),
1094                    Some(keys.current_id()),
1095                    "{name}"
1096                );
1097            }
1098        }
1099    }
1100
1101    impl Fixture {
1102        fn open_sealed(&self, keys: SealKeys) -> TranscriptArchive {
1103            TranscriptArchive::open_sealed(&self.archive_dir, keys).unwrap()
1104        }
1105    }
1106
1107    #[test]
1108    fn a_sealed_archive_holds_neither_the_transcript_text_nor_its_source_path() {
1109        let fx = Fixture::new();
1110        let text = "{\"text\":\"the quarterly migration plan\"}\n".repeat(4);
1111        let source = fx.write("orchard-ledger.jsonl", &text);
1112        let archive = fx.open_sealed(keys(1));
1113        archive.archive(&source).unwrap();
1114
1115        let path_bytes = source.to_string_lossy().into_owned();
1116        for path in files(archive.dir()) {
1117            let raw = fs::read(&path).unwrap();
1118            assert!(!contains(&raw, b"quarterly migration"), "{path:?}");
1119            assert!(!contains(&raw, b"orchard-ledger"), "{path:?}");
1120            assert!(!contains(&raw, path_bytes.as_bytes()), "{path:?}");
1121            assert!(zstd::decode_all(raw.as_slice()).is_err(), "{path:?}");
1122        }
1123        assert_all_sealed_under(archive.dir(), &keys(1));
1124        assert_eq!(archive.read(&source).unwrap(), Some(text.clone()));
1125        drop(archive);
1126        let reopened = fx.open_sealed(keys(1));
1127        assert_eq!(reopened.read(&source).unwrap(), Some(text));
1128    }
1129
1130    #[test]
1131    fn a_sealed_archive_does_not_open_without_its_key() {
1132        let fx = Fixture::new();
1133        let source = fx.write("a.jsonl", "{\"line\":1}\n");
1134        let archive = fx.open_sealed(keys(2));
1135        archive.archive(&source).unwrap();
1136        drop(archive);
1137
1138        let err = TranscriptArchive::open(&fx.archive_dir)
1139            .err()
1140            .expect("an unkeyed open fails");
1141        assert_eq!(seal_error(&err), Some(&SealError::KeyRequired));
1142        let err = TranscriptArchive::open_sealed(&fx.archive_dir, keys(3))
1143            .err()
1144            .expect("another key fails");
1145        assert!(matches!(seal_error(&err), Some(SealError::UnknownKey(_))));
1146        assert_eq!(blobs(&fx.open_sealed(keys(2))).len(), 1);
1147    }
1148
1149    #[test]
1150    fn a_plaintext_archive_opens_sealed_and_reseal_seals_every_file() {
1151        let fx = Fixture::new();
1152        let first = fx.write("a.jsonl", "{\"a\":1}\n");
1153        let copy = fx.write("b.jsonl", "{\"a\":1}\n");
1154        let second = fx.write("c.jsonl", "{\"c\":3}\n");
1155        let plain = fx.open();
1156        for source in [&first, &copy, &second] {
1157            plain.archive(source).unwrap();
1158        }
1159        drop(plain);
1160        fs::write(fx.archive_dir.join("manifest.jsonl.tmp"), "{\"source\":").unwrap();
1161
1162        let archive = fx.open_sealed(keys(4));
1163        assert_eq!(archive.read(&copy).unwrap().as_deref(), Some("{\"a\":1}\n"));
1164        assert_eq!(
1165            archive.read(&second).unwrap().as_deref(),
1166            Some("{\"c\":3}\n")
1167        );
1168        assert!(manifest_lines(&archive).is_empty());
1169
1170        assert_eq!(archive.reseal().unwrap(), 2);
1171        assert_all_sealed_under(archive.dir(), &keys(4));
1172        assert_eq!(archive.reseal().unwrap(), 0);
1173        assert_eq!(
1174            archive.read(&first).unwrap().as_deref(),
1175            Some("{\"a\":1}\n")
1176        );
1177        drop(archive);
1178
1179        let reopened = fx.open_sealed(keys(4));
1180        assert_eq!(reopened.len(), 3);
1181        assert_eq!(
1182            reopened.read(&second).unwrap().as_deref(),
1183            Some("{\"c\":3}\n")
1184        );
1185    }
1186
1187    #[test]
1188    fn a_rotation_then_reseal_leaves_only_the_new_key() {
1189        let fx = Fixture::new();
1190        let first = fx.write("a.jsonl", "{\"a\":1}\n");
1191        let second = fx.write("b.jsonl", "{\"b\":2}\n");
1192        let archive = fx.open_sealed(keys(5));
1193        archive.archive(&first).unwrap();
1194
1195        archive.set_seal_keys(SealKeys::new(key(6), vec![key(5)]));
1196        archive.archive(&second).unwrap();
1197        assert_eq!(
1198            archive.read(&first).unwrap().as_deref(),
1199            Some("{\"a\":1}\n")
1200        );
1201        assert_eq!(archive.reseal().unwrap(), 2);
1202        assert_all_sealed_under(archive.dir(), &keys(6));
1203        drop(archive);
1204
1205        let reopened = fx.open_sealed(keys(6));
1206        assert_eq!(reopened.len(), 2);
1207        assert_eq!(
1208            reopened.read(&first).unwrap().as_deref(),
1209            Some("{\"a\":1}\n")
1210        );
1211        assert_eq!(
1212            reopened.read(&second).unwrap().as_deref(),
1213            Some("{\"b\":2}\n")
1214        );
1215    }
1216
1217    #[test]
1218    fn a_torn_sealed_manifest_line_is_ignored() {
1219        let fx = Fixture::new();
1220        let first = fx.write("a.jsonl", "{\"a\":1}\n");
1221        let second = fx.write("b.jsonl", "{\"b\":2}\n");
1222        let archive = fx.open_sealed(keys(7));
1223        archive.archive(&first).unwrap();
1224        archive.archive(&second).unwrap();
1225        let line = sealed_manifest_lines(archive.dir())[0].clone();
1226        drop(archive);
1227        for cut in [line.len() / 2, line.len() - 3] {
1228            let mut manifest = fs::OpenOptions::new()
1229                .append(true)
1230                .open(fx.archive_dir.join(SEALED_MANIFEST_FILE))
1231                .unwrap();
1232            manifest.write_all(&line.as_bytes()[..cut]).unwrap();
1233            drop(manifest);
1234
1235            let reopened = fx.open_sealed(keys(7));
1236            assert_eq!(reopened.len(), 2);
1237            assert_eq!(sealed_manifest_lines(reopened.dir()).len(), 2);
1238        }
1239
1240        let reopened = fx.open_sealed(keys(7));
1241        let third = fx.write("c.jsonl", "{\"c\":3}\n");
1242        assert_eq!(reopened.archive(&third).unwrap(), Archived::Stored);
1243        drop(reopened);
1244        let again = fx.open_sealed(keys(7));
1245        assert_eq!(again.len(), 3);
1246        assert_eq!(again.read(&third).unwrap().as_deref(), Some("{\"c\":3}\n"));
1247        assert_all_sealed_under(again.dir(), &keys(7));
1248    }
1249
1250    #[test]
1251    fn removal_works_sealed_and_an_interrupted_removal_is_finished_from_a_sealed_journal() {
1252        let fx = Fixture::new();
1253        let gone = fx.write("gone.jsonl", "{\"line\":\"gone\"}\n");
1254        let kept = fx.write("kept.jsonl", "{\"line\":\"kept\"}\n");
1255        let archive = fx.open_sealed(keys(8));
1256        archive.archive(&gone).unwrap();
1257        archive.archive(&kept).unwrap();
1258
1259        let removed = archive.remove_where(|e| e.source.contains("gone")).unwrap();
1260        assert_eq!(removed.len(), 1);
1261        assert_eq!(blobs(&archive).len(), 1);
1262        assert!(!archive.dir().join(JOURNAL_FILE).exists());
1263
1264        let doomed = stray_blob(&archive, 'a');
1265        let bystander = stray_blob(&archive, 'b');
1266        let names = format!("{}\n", doomed.file_name().unwrap().to_string_lossy());
1267        sealed::write_file(
1268            &archive.dir().join(JOURNAL_FILE),
1269            Some(&keys(8)),
1270            JOURNAL_FILE,
1271            names.as_bytes(),
1272        )
1273        .unwrap();
1274        drop(archive);
1275
1276        let reopened = fx.open_sealed(keys(8));
1277        assert!(!doomed.exists());
1278        assert!(bystander.exists());
1279        assert!(!reopened.dir().join(JOURNAL_FILE).exists());
1280        assert_eq!(reopened.len(), 1);
1281        assert_eq!(
1282            reopened.read(&kept).unwrap().as_deref(),
1283            Some("{\"line\":\"kept\"}\n")
1284        );
1285        assert!(manifest_lines(&reopened).is_empty());
1286        assert_eq!(sealed_manifest_lines(reopened.dir()).len(), 1);
1287    }
1288
1289    #[test]
1290    fn a_sealed_journal_is_not_read_without_the_key() {
1291        let fx = Fixture::new();
1292        let archive = fx.open();
1293        let doomed = stray_blob(&archive, 'c');
1294        let names = format!("{}\n", doomed.file_name().unwrap().to_string_lossy());
1295        sealed::write_file(
1296            &archive.dir().join(JOURNAL_FILE),
1297            Some(&keys(9)),
1298            JOURNAL_FILE,
1299            names.as_bytes(),
1300        )
1301        .unwrap();
1302        drop(archive);
1303
1304        let err = TranscriptArchive::open(&fx.archive_dir)
1305            .err()
1306            .expect("an unkeyed open fails");
1307        assert_eq!(seal_error(&err), Some(&SealError::KeyRequired));
1308        assert!(doomed.exists());
1309    }
1310
1311    fn sealed_manifest_lines(dir: &Path) -> Vec<String> {
1312        match fs::read_to_string(dir.join(SEALED_MANIFEST_FILE)) {
1313            Ok(text) => text.lines().map(str::to_string).collect(),
1314            Err(e) if e.kind() == ErrorKind::NotFound => Vec::new(),
1315            Err(e) => panic!("{e}"),
1316        }
1317    }
1318
1319    fn line_from_a_binary_before_sealing(
1320        fx: &Fixture,
1321        source: &Path,
1322        archived_at: DateTime<Utc>,
1323    ) -> String {
1324        let bytes = fs::read(source).unwrap();
1325        let sha256 = hex(&Sha256::digest(&bytes));
1326        let file = format!("{sha256}{BLOB_SUFFIX}");
1327        fs::write(
1328            fx.archive_dir.join(&file),
1329            zstd::encode_all(bytes.as_slice(), 0).unwrap(),
1330        )
1331        .unwrap();
1332        serde_json::to_string(&ArchiveEntry {
1333            source: source_key(source),
1334            file,
1335            sha256,
1336            archived_at,
1337        })
1338        .unwrap()
1339    }
1340
1341    #[test]
1342    fn a_sealed_archive_leaves_the_plaintext_manifest_readable_by_a_binary_from_before_sealing() {
1343        let fx = Fixture::new();
1344        let first = fx.write("a.jsonl", "{\"a\":1}\n");
1345        let plain = fx.open();
1346        plain.archive(&first).unwrap();
1347        drop(plain);
1348
1349        let archive = fx.open_sealed(keys(10));
1350        let second = fx.write("b.jsonl", "{\"b\":2}\n");
1351        archive.archive(&second).unwrap();
1352        archive.reseal().unwrap();
1353        let third = fx.write("c.jsonl", "{\"c\":3}\n");
1354        archive.archive(&third).unwrap();
1355
1356        for line in manifest_lines(&archive) {
1357            serde_json::from_str::<ArchiveEntry>(&line).unwrap();
1358        }
1359        assert_eq!(sealed_manifest_lines(archive.dir()).len(), 3);
1360    }
1361
1362    #[test]
1363    fn entries_a_binary_from_before_sealing_appended_are_merged_and_then_sealed() {
1364        let fx = Fixture::new();
1365        let kept = fx.write("kept.jsonl", "{\"k\":1}\n");
1366        let changed = fx.write("changed.jsonl", "{\"v\":1}\n");
1367        let archive = fx.open_sealed(keys(11));
1368        archive.archive(&kept).unwrap();
1369        archive.archive(&changed).unwrap();
1370        let old_blob = archive.entry(&changed).unwrap().file;
1371        drop(archive);
1372
1373        let added = fx.write("added.jsonl", "{\"n\":1}\n");
1374        fs::write(&changed, "{\"v\":2}\n").unwrap();
1375        let later = Utc::now() + chrono::Duration::seconds(1);
1376        let lines = [
1377            line_from_a_binary_before_sealing(&fx, &added, later),
1378            line_from_a_binary_before_sealing(&fx, &changed, later),
1379        ];
1380        let mut manifest = fs::OpenOptions::new()
1381            .append(true)
1382            .open(fx.archive_dir.join(MANIFEST_FILE))
1383            .unwrap();
1384        for line in &lines {
1385            writeln!(manifest, "{line}").unwrap();
1386        }
1387        drop(manifest);
1388
1389        let reopened = fx.open_sealed(keys(11));
1390        assert_eq!(reopened.len(), 3);
1391        assert_eq!(
1392            reopened.read(&added).unwrap().as_deref(),
1393            Some("{\"n\":1}\n")
1394        );
1395        assert_eq!(
1396            reopened.read(&changed).unwrap().as_deref(),
1397            Some("{\"v\":2}\n")
1398        );
1399        assert_eq!(
1400            reopened.read(&kept).unwrap().as_deref(),
1401            Some("{\"k\":1}\n")
1402        );
1403        assert!(!reopened.dir().join(&old_blob).exists());
1404        assert!(manifest_lines(&reopened).is_empty());
1405        assert!(reopened.holds_unsealed());
1406
1407        reopened.reseal().unwrap();
1408        assert!(!reopened.holds_unsealed());
1409        assert_all_sealed_under(reopened.dir(), &keys(11));
1410        drop(reopened);
1411        let again = fx.open_sealed(keys(11));
1412        assert_eq!(again.len(), 3);
1413        assert!(!again.holds_unsealed());
1414    }
1415
1416    #[test]
1417    fn a_removal_after_keys_arrive_leaves_no_plaintext_entry_behind() {
1418        let fx = Fixture::new();
1419        let gone = fx.write("gone.jsonl", "{\"line\":\"gone\"}\n");
1420        let kept = fx.write("kept.jsonl", "{\"line\":\"kept\"}\n");
1421        let archive = fx.open();
1422        archive.archive(&gone).unwrap();
1423        archive.archive(&kept).unwrap();
1424
1425        archive.set_seal_keys(keys(12));
1426        archive.remove_where(|e| e.source.contains("gone")).unwrap();
1427        assert!(manifest_lines(&archive).is_empty());
1428        drop(archive);
1429
1430        let reopened = fx.open_sealed(keys(12));
1431        assert_eq!(reopened.len(), 1);
1432        assert!(reopened.entry(&gone).is_none());
1433    }
1434
1435    #[test]
1436    fn a_fold_cut_short_before_its_reseal_still_reports_unsealed_files_after_a_reopen() {
1437        let fx = Fixture::new();
1438        let first = fx.write("a.jsonl", "{\"a\":1}\n");
1439        let plain = fx.open();
1440        plain.archive(&first).unwrap();
1441        drop(plain);
1442
1443        let folded = fx.open_sealed(keys(13));
1444        assert!(folded.holds_unsealed());
1445        assert!(manifest_lines(&folded).is_empty());
1446        drop(folded);
1447
1448        let reopened = fx.open_sealed(keys(13));
1449        assert!(reopened.holds_unsealed());
1450        reopened.reseal().unwrap();
1451        assert_all_sealed_under(reopened.dir(), &keys(13));
1452        drop(reopened);
1453        assert!(!fx.open_sealed(keys(13)).holds_unsealed());
1454    }
1455}