Skip to main content

second_brain_core/
transcript_archive.rs

1use std::collections::HashMap;
2use std::fs::{self, OpenOptions};
3use std::io::{ErrorKind, Write};
4use std::path::{Path, PathBuf};
5use std::sync::{Mutex, MutexGuard, PoisonError};
6
7use anyhow::{Context, Result, anyhow, bail};
8use base64::Engine;
9use base64::prelude::BASE64_STANDARD;
10use chrono::{DateTime, Utc};
11use serde::{Deserialize, Serialize};
12use sha2::{Digest, Sha256};
13
14use crate::redact::redact;
15use crate::sealed::{self, SealError, SealKeys, seal_error};
16
17pub const MANIFEST_FILE: &str = "manifest.jsonl";
18/// Sealed manifest records, one base64 line each. A binary from before
19/// sealing reads only `MANIFEST_FILE`, so it never drops these as torn.
20pub const SEALED_MANIFEST_FILE: &str = "manifest.sealed";
21
22const JOURNAL_FILE: &str = "removal.journal";
23const RESEAL_MARKER: &str = "reseal.pending";
24const BLOB_SUFFIX: &str = ".jsonl.zst";
25const LEGACY_TMP_SUFFIX: &str = ".tmp";
26const COMPACTION_SLACK: usize = 64;
27
28#[derive(Debug, Clone, Copy, PartialEq, Eq)]
29pub enum Archived {
30    Stored,
31    Refreshed,
32    Unchanged,
33    Copy,
34}
35
36#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
37pub struct ArchiveEntry {
38    pub source: String,
39    pub file: String,
40    pub sha256: String,
41    pub archived_at: DateTime<Utc>,
42}
43
44/// Redacted zstd copies of transcripts, keyed by source path; an entry is
45/// never dropped because its source was deleted, only by `remove_where`.
46///
47/// Opened sealed, every blob and the removal journal is written under the
48/// current key and manifest records go to `SEALED_MANIFEST_FILE`. Plaintext
49/// blobs are read as-is until `reseal`, and plaintext manifest entries (from
50/// before sealing, or appended by an older binary) are folded into the sealed
51/// manifest at open, the later of two entries for one source winning.
52pub struct TranscriptArchive {
53    dir: PathBuf,
54    state: Mutex<State>,
55}
56
57struct State {
58    entries: HashMap<String, ArchiveEntry>,
59    manifest_lines: usize,
60    plaintext_lines: usize,
61    manifest_damaged: bool,
62    unsealed: bool,
63    keys: Option<SealKeys>,
64}
65
66impl State {
67    fn needs_compaction(&self) -> bool {
68        self.manifest_damaged || self.manifest_lines > 2 * self.entries.len() + COMPACTION_SLACK
69    }
70
71    fn plaintext_pending(&self) -> bool {
72        self.keys.is_some() && self.plaintext_lines > 0
73    }
74
75    fn holder_of(&self, sha256: &str) -> Option<&ArchiveEntry> {
76        self.entries.values().find(|e| e.sha256 == sha256)
77    }
78
79    fn references(&self, file: &str) -> bool {
80        self.entries.values().any(|e| e.file == file)
81    }
82}
83
84impl TranscriptArchive {
85    pub fn open(dir: impl Into<PathBuf>) -> Result<Self> {
86        Self::open_with(dir.into(), None)
87    }
88
89    /// Opens the archive writing everything sealed under `keys`. An archive
90    /// written before sealing opens as it is; `reseal` seals what it holds.
91    pub fn open_sealed(dir: impl Into<PathBuf>, keys: SealKeys) -> Result<Self> {
92        Self::open_with(dir.into(), Some(keys))
93    }
94
95    fn open_with(dir: PathBuf, keys: Option<SealKeys>) -> Result<Self> {
96        fs::create_dir_all(&dir)
97            .with_context(|| format!("creating the transcript archive at {}", dir.display()))?;
98        let manifest = dir.join(MANIFEST_FILE);
99        if !manifest.exists() {
100            sealed::write_file(&manifest, None, MANIFEST_FILE, b"")
101                .with_context(|| format!("creating {}", manifest.display()))?;
102        }
103        let (state, torn, superseded) = load_manifest(&dir, keys)?;
104        let archive = Self {
105            dir,
106            state: Mutex::new(state),
107        };
108        {
109            let mut state = archive.state();
110            if torn || state.needs_compaction() || state.plaintext_pending() {
111                archive.compact(&mut state)?;
112            }
113            for file in superseded.iter().filter(|f| !state.references(f)) {
114                archive.remove_blob(file);
115            }
116            archive.finish_interrupted_removal(&state)?;
117        }
118        Ok(archive)
119    }
120
121    /// True while files written in plaintext may remain: plaintext manifest
122    /// entries were loaded or written, by this process or by one that died
123    /// before its `reseal` finished, and no `reseal` has run since.
124    pub fn holds_unsealed(&self) -> bool {
125        self.state().unsealed
126    }
127
128    /// None when the archive writes plaintext.
129    pub fn seal_keys(&self) -> Option<SealKeys> {
130        self.state().keys.clone()
131    }
132
133    /// Seals what is written from now on under the current key of `keys`,
134    /// which must also hold every key the archive's files are sealed under.
135    pub fn set_seal_keys(&self, keys: SealKeys) {
136        self.state().keys = Some(keys);
137    }
138
139    /// Rewrites every blob, the manifest and any removal journal that is
140    /// plaintext or sealed under a retired key under the current key, and
141    /// deletes temporary files a crashed write left. Returns how many files
142    /// were rewritten. Each blob is resealed under the state lock, so a
143    /// concurrent removal never sees its blob renamed back into place.
144    pub fn reseal(&self) -> Result<usize> {
145        if self.state().keys.is_none() {
146            return Err(no_keys());
147        }
148        let mut resealed = 0;
149        for file in self.blob_names()? {
150            let state = self.state();
151            let keys = state.keys.clone().ok_or_else(no_keys)?;
152            let path = self.dir.join(&file);
153            if path.exists() && sealed::reseal_file(&path, &keys, &file)? {
154                resealed += 1;
155            }
156        }
157        let mut state = self.state();
158        let keys = state.keys.clone().ok_or_else(no_keys)?;
159        let journal = self.dir.join(JOURNAL_FILE);
160        if journal.exists() && sealed::reseal_file(&journal, &keys, JOURNAL_FILE)? {
161            resealed += 1;
162        }
163        if !self.manifest_sealed_under(&keys)? {
164            self.compact(&mut state)?;
165            resealed += 1;
166        }
167        self.remove_tmp_files()?;
168        let marker = self.dir.join(RESEAL_MARKER);
169        match fs::remove_file(&marker) {
170            Ok(()) => {}
171            Err(e) if e.kind() == ErrorKind::NotFound => {}
172            Err(e) => return Err(e).with_context(|| format!("removing {}", marker.display())),
173        }
174        state.unsealed = false;
175        Ok(resealed)
176    }
177
178    /// Drops every entry `doomed` picks: the blobs to delete are journaled,
179    /// then the manifest is rewritten, then the blobs go and the journal with
180    /// them. A crash in between is finished at the next open.
181    pub fn remove_where(
182        &self,
183        doomed: impl Fn(&ArchiveEntry) -> bool,
184    ) -> Result<Vec<ArchiveEntry>> {
185        let mut state = self.state();
186        if state.plaintext_pending() {
187            self.compact(&mut state)?;
188        }
189        let removed: Vec<ArchiveEntry> = state
190            .entries
191            .values()
192            .filter(|e| doomed(e))
193            .cloned()
194            .collect();
195        if removed.is_empty() {
196            return Ok(removed);
197        }
198        for entry in &removed {
199            state.entries.remove(&entry.source);
200        }
201        let mut doomed_blobs: Vec<&str> = removed
202            .iter()
203            .map(|e| e.file.as_str())
204            .filter(|file| !state.references(file))
205            .collect();
206        doomed_blobs.sort_unstable();
207        doomed_blobs.dedup();
208        if !doomed_blobs.is_empty() {
209            let journal = self.dir.join(JOURNAL_FILE);
210            sealed::write_file(
211                &journal,
212                state.keys.as_ref(),
213                JOURNAL_FILE,
214                doomed_blobs.join("\n").as_bytes(),
215            )
216            .with_context(|| format!("writing {}", journal.display()))?;
217        }
218        self.compact(&mut state)?;
219        for file in &doomed_blobs {
220            self.remove_blob(file);
221        }
222        self.remove_journal();
223        Ok(removed)
224    }
225
226    /// Copies the archive into `dest`, which must not exist, while archiving
227    /// goes on. The manifest is appended in place, so it is copied under the
228    /// state lock; blobs are content-addressed and removed only after the
229    /// manifest stops naming them, so most are copied without the lock and
230    /// any the copied manifest no longer names are dropped from the copy.
231    pub fn copy_into(&self, dest: &Path) -> Result<()> {
232        self.copy_into_with(dest, || {})
233    }
234
235    fn copy_into_with(&self, dest: &Path, between_phases: impl FnOnce()) -> Result<()> {
236        crate::private_files::create_private_dir(dest)?;
237        for file in self.blob_names()? {
238            copy_if_present(&self.dir.join(&file), &dest.join(&file))?;
239        }
240        between_phases();
241        let state = self.state();
242        for name in [
243            MANIFEST_FILE,
244            SEALED_MANIFEST_FILE,
245            JOURNAL_FILE,
246            RESEAL_MARKER,
247        ] {
248            copy_if_present(&self.dir.join(name), &dest.join(name))?;
249        }
250        for entry in state.entries.values() {
251            let target = dest.join(&entry.file);
252            if !target.exists() {
253                crate::dir_copy::copy_file_synced(&self.dir.join(&entry.file), &target)?;
254            }
255        }
256        for entry in fs::read_dir(dest).with_context(|| format!("listing {}", dest.display()))? {
257            let name = entry?.file_name().to_string_lossy().into_owned();
258            if name.ends_with(BLOB_SUFFIX) && !state.references(&name) {
259                let path = dest.join(&name);
260                fs::remove_file(&path).with_context(|| format!("removing {}", path.display()))?;
261            }
262        }
263        drop(state);
264        crate::dir_copy::sync_dir(dest)
265    }
266
267    /// An entry whose blob has this sha256, if any does.
268    pub fn holder(&self, sha256: &str) -> Option<ArchiveEntry> {
269        self.state().holder_of(sha256).cloned()
270    }
271
272    pub fn dir(&self) -> &Path {
273        &self.dir
274    }
275
276    pub fn archive(&self, source: &Path) -> Result<Archived> {
277        let mut state = self.state();
278        let bytes = fs::read(source).with_context(|| format!("reading {}", source.display()))?;
279        let sha256 = hex(&Sha256::digest(&bytes));
280        let key = source_key(source);
281
282        let previous = state.entries.get(&key).cloned();
283        if let Some(prev) = &previous
284            && prev.sha256 == sha256
285            && self.dir.join(&prev.file).exists()
286        {
287            return Ok(Archived::Unchanged);
288        }
289
290        let file = format!("{sha256}{BLOB_SUFFIX}");
291        let shared = state
292            .holder_of(&sha256)
293            .is_some_and(|holder| self.dir.join(&holder.file).exists());
294        if !shared {
295            self.write_blob(state.keys.as_ref(), &file, &bytes)?;
296        }
297        let entry = ArchiveEntry {
298            source: key.clone(),
299            file,
300            sha256,
301            archived_at: Utc::now(),
302        };
303        state.entries.insert(key, entry.clone());
304        if state.needs_compaction() {
305            self.compact(&mut state)?;
306        } else if let Err(e) = self.append(&mut state, &entry) {
307            state.manifest_damaged = true;
308            return Err(e);
309        }
310
311        if let Some(prev) = &previous
312            && !state.references(&prev.file)
313        {
314            self.remove_blob(&prev.file);
315        }
316
317        Ok(match (shared, previous) {
318            (true, _) => Archived::Copy,
319            (false, None) => Archived::Stored,
320            (false, Some(_)) => Archived::Refreshed,
321        })
322    }
323
324    pub fn entry(&self, source: &Path) -> Option<ArchiveEntry> {
325        self.state().entries.get(&source_key(source)).cloned()
326    }
327
328    pub fn read(&self, source: &Path) -> Result<Option<String>> {
329        let (entry, keys) = {
330            let state = self.state();
331            (
332                state.entries.get(&source_key(source)).cloned(),
333                state.keys.clone(),
334            )
335        };
336        let Some(entry) = entry else {
337            return Ok(None);
338        };
339        let path = self.dir.join(&entry.file);
340        let compressed = sealed::read_file(&path, keys.as_ref(), &entry.file)?;
341        let bytes = zstd::decode_all(compressed.as_slice())
342            .with_context(|| format!("decompressing {}", path.display()))?;
343        let text =
344            String::from_utf8(bytes).with_context(|| format!("decoding {}", path.display()))?;
345        Ok(Some(text))
346    }
347
348    /// Every archived transcript, in no particular order.
349    pub fn entries(&self) -> Vec<ArchiveEntry> {
350        self.state().entries.values().cloned().collect()
351    }
352
353    pub fn len(&self) -> usize {
354        self.state().entries.len()
355    }
356
357    pub fn is_empty(&self) -> bool {
358        self.len() == 0
359    }
360
361    fn state(&self) -> MutexGuard<'_, State> {
362        self.state.lock().unwrap_or_else(PoisonError::into_inner)
363    }
364
365    fn write_blob(&self, keys: Option<&SealKeys>, file: &str, source_bytes: &[u8]) -> Result<()> {
366        let text = redact(&String::from_utf8_lossy(source_bytes)).text;
367        let compressed =
368            zstd::encode_all(text.as_bytes(), 0).context("compressing a transcript")?;
369        let target = self.dir.join(file);
370        sealed::write_file(&target, keys, file, &compressed)
371            .with_context(|| format!("writing {}", target.display()))
372    }
373
374    fn blob_names(&self) -> Result<Vec<String>> {
375        let mut names = Vec::new();
376        for entry in
377            fs::read_dir(&self.dir).with_context(|| format!("listing {}", self.dir.display()))?
378        {
379            let name = entry
380                .with_context(|| format!("listing {}", self.dir.display()))?
381                .file_name()
382                .to_string_lossy()
383                .into_owned();
384            if name.ends_with(BLOB_SUFFIX) {
385                names.push(name);
386            }
387        }
388        Ok(names)
389    }
390
391    fn manifest_sealed_under(&self, keys: &SealKeys) -> Result<bool> {
392        let plaintext = read_manifest_text(&self.dir.join(MANIFEST_FILE))?;
393        if plaintext.lines().any(|l| !l.trim().is_empty()) {
394            return Ok(false);
395        }
396        let text = read_manifest_text(&self.dir.join(SEALED_MANIFEST_FILE))?;
397        Ok(text.lines().filter(|l| !l.trim().is_empty()).all(|line| {
398            BASE64_STANDARD
399                .decode(line.trim())
400                .is_ok_and(|bytes| sealed::key_id_of(&bytes) == Some(keys.current_id()))
401        }))
402    }
403
404    /// Only safe under the state lock: every write this archive makes runs
405    /// under it, so no temporary file is in use.
406    fn remove_tmp_files(&self) -> Result<()> {
407        for entry in
408            fs::read_dir(&self.dir).with_context(|| format!("listing {}", self.dir.display()))?
409        {
410            let entry = entry.with_context(|| format!("listing {}", self.dir.display()))?;
411            let name = entry.file_name().to_string_lossy().into_owned();
412            if sealed::is_tmp_name(&name) || name.ends_with(LEGACY_TMP_SUFFIX) {
413                fs::remove_file(entry.path())
414                    .or_else(|e| match e.kind() {
415                        ErrorKind::NotFound => Ok(()),
416                        _ => Err(e),
417                    })
418                    .with_context(|| format!("removing {}", entry.path().display()))?;
419            }
420        }
421        Ok(())
422    }
423
424    /// Deletes the blobs a removal journal names that no entry uses, then the
425    /// journal. Blobs the journal does not name are never touched, so a lost
426    /// manifest cannot delete an archive.
427    fn finish_interrupted_removal(&self, state: &State) -> Result<()> {
428        let journal = self.dir.join(JOURNAL_FILE);
429        if !journal.exists() {
430            return Ok(());
431        }
432        let bytes = sealed::read_file(&journal, state.keys.as_ref(), JOURNAL_FILE)?;
433        let text =
434            String::from_utf8(bytes).with_context(|| format!("decoding {}", journal.display()))?;
435        for name in text.lines().map(str::trim) {
436            let is_blob_name = name.ends_with(BLOB_SUFFIX) && !name.contains(['/', '\\']);
437            if is_blob_name && !state.references(name) {
438                self.remove_blob(name);
439            }
440        }
441        self.remove_journal();
442        Ok(())
443    }
444
445    fn remove_journal(&self) {
446        let path = self.dir.join(JOURNAL_FILE);
447        match fs::remove_file(&path) {
448            Ok(()) => {}
449            Err(e) if e.kind() == ErrorKind::NotFound => {}
450            Err(e) => {
451                tracing::warn!(path = %path.display(), error = %e, "removing the archive removal journal")
452            }
453        }
454    }
455
456    fn remove_blob(&self, file: &str) {
457        let path = self.dir.join(file);
458        match fs::remove_file(&path) {
459            Ok(()) => {}
460            Err(e) if e.kind() == ErrorKind::NotFound => {}
461            Err(e) => {
462                tracing::warn!(path = %path.display(), error = %e, "removing an unused archive blob")
463            }
464        }
465    }
466
467    fn append(&self, state: &mut State, entry: &ArchiveEntry) -> Result<()> {
468        let path = self.dir.join(manifest_file(state.keys.as_ref()));
469        let mut line = manifest_line(state.keys.as_ref(), entry)?;
470        line.push('\n');
471        let mut file = OpenOptions::new()
472            .create(true)
473            .append(true)
474            .open(&path)
475            .with_context(|| format!("opening {}", path.display()))?;
476        file.write_all(line.as_bytes())
477            .and_then(|()| file.sync_data())
478            .with_context(|| format!("appending to {}", path.display()))?;
479        state.manifest_lines += 1;
480        if state.keys.is_none() {
481            state.plaintext_lines += 1;
482            state.unsealed = true;
483        }
484        Ok(())
485    }
486
487    fn compact(&self, state: &mut State) -> Result<()> {
488        let mut entries: Vec<&ArchiveEntry> = state.entries.values().collect();
489        entries.sort_by(|a, b| a.source.cmp(&b.source));
490        let mut body = String::new();
491        for entry in &entries {
492            body.push_str(&manifest_line(state.keys.as_ref(), entry)?);
493            body.push('\n');
494        }
495        let count = entries.len();
496        let name = manifest_file(state.keys.as_ref());
497        let path = self.dir.join(name);
498        sealed::write_file(&path, None, name, body.as_bytes())
499            .with_context(|| format!("compacting {}", path.display()))?;
500        if state.keys.is_some() {
501            let plaintext = self.dir.join(MANIFEST_FILE);
502            if fs::metadata(&plaintext).is_ok_and(|m| m.len() > 0) {
503                let marker = self.dir.join(RESEAL_MARKER);
504                sealed::write_file(&marker, None, RESEAL_MARKER, b"")
505                    .with_context(|| format!("writing {}", marker.display()))?;
506                sealed::write_file(&plaintext, None, MANIFEST_FILE, b"")
507                    .with_context(|| format!("emptying {}", plaintext.display()))?;
508            }
509            state.plaintext_lines = 0;
510        } else {
511            state.plaintext_lines = count;
512            state.unsealed |= count > 0;
513        }
514        state.manifest_lines = count;
515        state.manifest_damaged = false;
516        Ok(())
517    }
518}
519
520fn copy_if_present(from: &Path, to: &Path) -> Result<()> {
521    match crate::dir_copy::copy_file_synced(from, to) {
522        Err(e) if crate::dir_copy::is_not_found(&e) => Ok(()),
523        other => other,
524    }
525}
526
527fn no_keys() -> anyhow::Error {
528    anyhow!("the transcript archive has no sealing keys to reseal under")
529}
530
531fn manifest_file(keys: Option<&SealKeys>) -> &'static str {
532    match keys {
533        Some(_) => SEALED_MANIFEST_FILE,
534        None => MANIFEST_FILE,
535    }
536}
537
538fn manifest_line(keys: Option<&SealKeys>, entry: &ArchiveEntry) -> Result<String> {
539    let json = serde_json::to_string(entry)?;
540    match keys {
541        Some(keys) => {
542            Ok(BASE64_STANDARD.encode(sealed::seal(keys, SEALED_MANIFEST_FILE, json.as_bytes())?))
543        }
544        None => Ok(json),
545    }
546}
547
548fn parse_plaintext_line(line: &str) -> Result<ArchiveEntry> {
549    Ok(serde_json::from_str(line)?)
550}
551
552fn parse_sealed_line(line: &str, keys: Option<&SealKeys>) -> Result<ArchiveEntry> {
553    let sealed_bytes = BASE64_STANDARD.decode(line.trim())?;
554    if !sealed::looks_sealed(&sealed_bytes) {
555        bail!("the line is not a sealed record");
556    }
557    let keys = keys.ok_or(SealError::KeyRequired)?;
558    Ok(serde_json::from_slice(&sealed::open(
559        keys,
560        SEALED_MANIFEST_FILE,
561        &sealed_bytes,
562    )?)?)
563}
564
565fn read_manifest_text(path: &Path) -> Result<String> {
566    match fs::read_to_string(path) {
567        Ok(text) => Ok(text),
568        Err(e) if e.kind() == ErrorKind::NotFound => Ok(String::new()),
569        Err(e) => Err(e).with_context(|| format!("reading {}", path.display())),
570    }
571}
572
573/// Also returns the line count and a flag that is true when a line failed to
574/// parse: a torn tail would swallow the next appended line, so the caller
575/// rewrites the manifest. A line sealed under a key not held is an error
576/// rather than a torn line, so a missing key never compacts entries away.
577fn read_manifest(
578    path: &Path,
579    parse: impl Fn(&str) -> Result<ArchiveEntry>,
580) -> Result<(HashMap<String, ArchiveEntry>, usize, bool)> {
581    let text = read_manifest_text(path)?;
582    let mut entries = HashMap::new();
583    let mut lines = 0;
584    let mut torn = !text.is_empty() && !text.ends_with('\n');
585    for line in text.lines().filter(|l| !l.trim().is_empty()) {
586        lines += 1;
587        match parse(line) {
588            Ok(entry) => {
589                entries.insert(entry.source.clone(), entry);
590            }
591            Err(e) if is_missing_key(&e) => {
592                return Err(e).with_context(|| format!("reading {}", path.display()));
593            }
594            Err(e) => {
595                torn = true;
596                tracing::warn!(path = %path.display(), error = %e, "ignoring an unreadable archive manifest line");
597            }
598        }
599    }
600    Ok((entries, lines, torn))
601}
602
603/// Merges the plaintext manifest over the sealed one, the later entry for a
604/// source winning; also returns the torn flag and the blobs of the entries
605/// that lost, which the caller removes once the merge is written.
606fn load_manifest(dir: &Path, keys: Option<SealKeys>) -> Result<(State, bool, Vec<String>)> {
607    let (mut entries, sealed_lines, sealed_torn) =
608        read_manifest(&dir.join(SEALED_MANIFEST_FILE), |line| {
609            parse_sealed_line(line, keys.as_ref())
610        })?;
611    let (plaintext, plaintext_lines, plaintext_torn) =
612        read_manifest(&dir.join(MANIFEST_FILE), parse_plaintext_line)?;
613    let unsealed = !plaintext.is_empty() || dir.join(RESEAL_MARKER).exists();
614    let mut superseded = Vec::new();
615    for (source, entry) in plaintext {
616        match entries.get(&source) {
617            Some(held) if held.archived_at > entry.archived_at => superseded.push(entry.file),
618            Some(held) => {
619                superseded.push(held.file.clone());
620                entries.insert(source, entry);
621            }
622            None => {
623                entries.insert(source, entry);
624            }
625        }
626    }
627    let state = State {
628        entries,
629        manifest_lines: sealed_lines + plaintext_lines,
630        plaintext_lines,
631        manifest_damaged: false,
632        unsealed,
633        keys,
634    };
635    Ok((state, sealed_torn || plaintext_torn, superseded))
636}
637
638fn is_missing_key(error: &anyhow::Error) -> bool {
639    matches!(
640        seal_error(error),
641        Some(SealError::KeyRequired | SealError::UnknownKey(_))
642    )
643}
644
645fn source_key(source: &Path) -> String {
646    source.to_string_lossy().into_owned()
647}
648
649fn hex(bytes: &[u8]) -> String {
650    bytes.iter().map(|b| format!("{b:02x}")).collect()
651}
652
653#[cfg(test)]
654mod tests {
655    use std::fs;
656    use std::io::Write;
657
658    use base64::Engine;
659    use base64::prelude::BASE64_STANDARD;
660
661    use super::*;
662    use crate::keys::KeyGuard;
663    use crate::sealed::{self, SealError, SealKey, seal_error};
664
665    fn blobs(archive: &TranscriptArchive) -> Vec<String> {
666        let mut names: Vec<String> = fs::read_dir(archive.dir())
667            .unwrap()
668            .map(|e| e.unwrap().file_name().to_string_lossy().into_owned())
669            .filter(|n| n.ends_with(".jsonl.zst"))
670            .collect();
671        names.sort();
672        names
673    }
674
675    fn manifest_lines(archive: &TranscriptArchive) -> Vec<String> {
676        fs::read_to_string(archive.dir().join(MANIFEST_FILE))
677            .unwrap()
678            .lines()
679            .map(str::to_string)
680            .collect()
681    }
682
683    struct Fixture {
684        _tmp: tempfile::TempDir,
685        projects: PathBuf,
686        archive_dir: PathBuf,
687    }
688
689    impl Fixture {
690        fn new() -> Self {
691            let tmp = tempfile::tempdir().unwrap();
692            let projects = tmp.path().join("projects");
693            fs::create_dir_all(&projects).unwrap();
694            let archive_dir = tmp.path().join("data").join("transcripts");
695            Self {
696                _tmp: tmp,
697                projects,
698                archive_dir,
699            }
700        }
701
702        fn write(&self, name: &str, text: &str) -> PathBuf {
703            let path = self.projects.join(name);
704            fs::write(&path, text).unwrap();
705            path
706        }
707
708        fn open(&self) -> TranscriptArchive {
709            TranscriptArchive::open(&self.archive_dir).unwrap()
710        }
711    }
712
713    #[test]
714    fn open_creates_the_directory() {
715        let fx = Fixture::new();
716        let archive = fx.open();
717        assert!(archive.dir().is_dir());
718        assert!(archive.is_empty());
719    }
720
721    #[test]
722    fn a_new_source_is_stored_then_unchanged() {
723        let fx = Fixture::new();
724        let source = fx.write("a.jsonl", "{\"line\":1}\n");
725        let archive = fx.open();
726
727        assert_eq!(archive.archive(&source).unwrap(), Archived::Stored);
728        assert_eq!(archive.archive(&source).unwrap(), Archived::Unchanged);
729
730        assert_eq!(archive.len(), 1);
731        assert_eq!(blobs(&archive).len(), 1);
732        assert_eq!(manifest_lines(&archive).len(), 1);
733        let entry = archive.entry(&source).unwrap();
734        assert_eq!(entry.source, source.to_string_lossy());
735        assert_eq!(entry.file, format!("{}.jsonl.zst", entry.sha256));
736        assert_eq!(entry.sha256.len(), 64);
737        assert_eq!(
738            archive.read(&source).unwrap().as_deref(),
739            Some("{\"line\":1}\n")
740        );
741    }
742
743    #[test]
744    fn a_changed_source_is_refreshed_and_its_old_blob_removed() {
745        let fx = Fixture::new();
746        let source = fx.write("a.jsonl", "{\"line\":1}\n");
747        let archive = fx.open();
748        archive.archive(&source).unwrap();
749        let old = archive.entry(&source).unwrap();
750
751        fs::write(&source, "{\"line\":1}\n{\"line\":2}\n").unwrap();
752        assert_eq!(archive.archive(&source).unwrap(), Archived::Refreshed);
753
754        let new = archive.entry(&source).unwrap();
755        assert_ne!(new.sha256, old.sha256);
756        assert_eq!(
757            archive.read(&source).unwrap().as_deref(),
758            Some("{\"line\":1}\n{\"line\":2}\n")
759        );
760        assert_eq!(blobs(&archive), vec![new.file]);
761        assert!(!archive.dir().join(&old.file).exists());
762    }
763
764    #[test]
765    fn a_copy_at_another_path_shares_the_blob() {
766        let fx = Fixture::new();
767        let first = fx.write("a.jsonl", "{\"same\":true}\n");
768        let second = fx.write("b.jsonl", "{\"same\":true}\n");
769        let archive = fx.open();
770
771        assert_eq!(archive.archive(&first).unwrap(), Archived::Stored);
772        assert_eq!(archive.archive(&second).unwrap(), Archived::Copy);
773
774        assert_eq!(archive.len(), 2);
775        assert_eq!(blobs(&archive).len(), 1);
776        assert_eq!(
777            archive.entry(&first).unwrap().file,
778            archive.entry(&second).unwrap().file
779        );
780        assert_eq!(
781            archive.read(&second).unwrap().as_deref(),
782            Some("{\"same\":true}\n")
783        );
784    }
785
786    #[test]
787    fn refreshing_one_copy_keeps_the_blob_the_other_still_uses() {
788        let fx = Fixture::new();
789        let first = fx.write("a.jsonl", "{\"same\":true}\n");
790        let second = fx.write("b.jsonl", "{\"same\":true}\n");
791        let archive = fx.open();
792        archive.archive(&first).unwrap();
793        archive.archive(&second).unwrap();
794
795        fs::write(&first, "{\"same\":false}\n").unwrap();
796        assert_eq!(archive.archive(&first).unwrap(), Archived::Refreshed);
797
798        assert_eq!(blobs(&archive).len(), 2);
799        assert_eq!(
800            archive.read(&second).unwrap().as_deref(),
801            Some("{\"same\":true}\n")
802        );
803    }
804
805    #[test]
806    fn the_archive_outlives_a_deleted_source() {
807        let fx = Fixture::new();
808        let source = fx.write("a.jsonl", "{\"kept\":true}\n");
809        let archive = fx.open();
810        archive.archive(&source).unwrap();
811
812        fs::remove_file(&source).unwrap();
813
814        assert!(archive.archive(&source).is_err());
815        assert_eq!(
816            archive.read(&source).unwrap().as_deref(),
817            Some("{\"kept\":true}\n")
818        );
819        assert_eq!(blobs(&archive).len(), 1);
820        let reopened = fx.open();
821        assert_eq!(
822            reopened.read(&source).unwrap().as_deref(),
823            Some("{\"kept\":true}\n")
824        );
825    }
826
827    #[test]
828    fn an_unknown_source_reads_as_none() {
829        let fx = Fixture::new();
830        let archive = fx.open();
831        let missing = fx.projects.join("never.jsonl");
832        assert!(archive.entry(&missing).is_none());
833        assert_eq!(archive.read(&missing).unwrap(), None);
834    }
835
836    #[test]
837    fn archived_text_is_redacted() {
838        let fx = Fixture::new();
839        let secret = format!("gh{}_{}", "p", "aB3dE6".repeat(6));
840        let source = fx.write(
841            "a.jsonl",
842            &format!("{{\"text\":\"token {secret} and the build passed\"}}\n"),
843        );
844        let archive = fx.open();
845        archive.archive(&source).unwrap();
846
847        let text = archive.read(&source).unwrap().unwrap();
848        assert!(!text.contains(&secret));
849        assert!(text.contains("the build passed"));
850        assert!(text.contains("[REDACTED:github_token]"));
851
852        let entry = archive.entry(&source).unwrap();
853        let raw = fs::read(archive.dir().join(&entry.file)).unwrap();
854        let decoded = String::from_utf8(zstd::decode_all(raw.as_slice()).unwrap()).unwrap();
855        assert!(!decoded.contains(&secret));
856        assert!(decoded.contains("the build passed"));
857    }
858
859    #[test]
860    fn the_blob_is_named_by_the_source_bytes() {
861        let fx = Fixture::new();
862        let secret = format!("gh{}_{}", "p", "aB3dE6".repeat(6));
863        let source = fx.write("a.jsonl", &format!("{secret}\n"));
864        let archive = fx.open();
865        archive.archive(&source).unwrap();
866
867        let changed = format!("gh{}_{}", "p", "zY9xW8".repeat(6));
868        fs::write(&source, format!("{changed}\n")).unwrap();
869        assert_eq!(archive.archive(&source).unwrap(), Archived::Refreshed);
870    }
871
872    #[test]
873    fn reopening_loads_the_manifest() {
874        let fx = Fixture::new();
875        let first = fx.write("a.jsonl", "{\"a\":1}\n");
876        let second = fx.write("b.jsonl", "{\"b\":2}\n");
877        let archive = fx.open();
878        archive.archive(&first).unwrap();
879        archive.archive(&second).unwrap();
880        let entry = archive.entry(&first).unwrap();
881        drop(archive);
882
883        let reopened = fx.open();
884        assert_eq!(reopened.len(), 2);
885        assert_eq!(reopened.entry(&first), Some(entry));
886        assert_eq!(reopened.archive(&first).unwrap(), Archived::Unchanged);
887        assert_eq!(
888            reopened.read(&second).unwrap().as_deref(),
889            Some("{\"b\":2}\n")
890        );
891    }
892
893    #[test]
894    fn a_torn_last_manifest_line_is_ignored() {
895        let fx = Fixture::new();
896        let first = fx.write("a.jsonl", "{\"a\":1}\n");
897        let second = fx.write("b.jsonl", "{\"b\":2}\n");
898        let archive = fx.open();
899        archive.archive(&first).unwrap();
900        archive.archive(&second).unwrap();
901        drop(archive);
902        let mut manifest = fs::OpenOptions::new()
903            .append(true)
904            .open(fx.archive_dir.join(MANIFEST_FILE))
905            .unwrap();
906        manifest.write_all(b"{\"source\":\"/x/c.js").unwrap();
907        drop(manifest);
908
909        let reopened = fx.open();
910        assert_eq!(reopened.len(), 2);
911
912        let third = fx.write("c.jsonl", "{\"c\":3}\n");
913        assert_eq!(reopened.archive(&third).unwrap(), Archived::Stored);
914        drop(reopened);
915        let again = fx.open();
916        assert_eq!(again.len(), 3);
917        assert_eq!(again.read(&third).unwrap().as_deref(), Some("{\"c\":3}\n"));
918    }
919
920    #[test]
921    fn a_failed_manifest_append_is_repaired_by_the_next_write() {
922        use std::os::unix::fs::PermissionsExt;
923
924        let fx = Fixture::new();
925        let first = fx.write("a.jsonl", "{\"a\":1}\n");
926        let second = fx.write("b.jsonl", "{\"b\":2}\n");
927        let third = fx.write("c.jsonl", "{\"c\":3}\n");
928        let archive = fx.open();
929        archive.archive(&first).unwrap();
930        let manifest = fx.archive_dir.join(MANIFEST_FILE);
931        fs::set_permissions(&manifest, fs::Permissions::from_mode(0o400)).unwrap();
932
933        assert!(archive.archive(&second).is_err());
934        assert_eq!(archive.archive(&third).unwrap(), Archived::Stored);
935        drop(archive);
936
937        let reopened = fx.open();
938        assert_eq!(reopened.len(), 3);
939        assert_eq!(
940            reopened.read(&second).unwrap().as_deref(),
941            Some("{\"b\":2}\n")
942        );
943    }
944
945    #[test]
946    fn compaction_keeps_the_latest_entry_per_source() {
947        let fx = Fixture::new();
948        let first = fx.write("a.jsonl", "{\"a\":1}\n");
949        let second = fx.write("b.jsonl", "{\"b\":1}\n");
950        let archive = fx.open();
951        archive.archive(&first).unwrap();
952        archive.archive(&second).unwrap();
953        let latest_second = archive.entry(&second).unwrap();
954        drop(archive);
955
956        let manifest = fx.archive_dir.join(MANIFEST_FILE);
957        let stale = ArchiveEntry {
958            source: first.to_string_lossy().into_owned(),
959            file: "0".repeat(64) + ".jsonl.zst",
960            sha256: "0".repeat(64),
961            archived_at: Utc::now(),
962        };
963        let mut lines = fs::read_to_string(&manifest).unwrap();
964        let mut stale_lines = String::new();
965        for _ in 0..80 {
966            stale_lines.push_str(&serde_json::to_string(&stale).unwrap());
967            stale_lines.push('\n');
968        }
969        lines.insert_str(0, &stale_lines);
970        fs::write(&manifest, &lines).unwrap();
971
972        let reopened = fx.open();
973        let kept = manifest_lines(&reopened);
974        assert_eq!(kept.len(), 2);
975        assert_eq!(reopened.len(), 2);
976        assert_ne!(reopened.entry(&first).unwrap().sha256, stale.sha256);
977        assert_eq!(reopened.entry(&second), Some(latest_second));
978        assert_eq!(
979            reopened.read(&first).unwrap().as_deref(),
980            Some("{\"a\":1}\n")
981        );
982    }
983
984    #[test]
985    fn a_long_running_archive_keeps_its_manifest_compact() {
986        let fx = Fixture::new();
987        let source = fx.write("a.jsonl", "");
988        let archive = fx.open();
989        let mut text = String::new();
990        for i in 0..100 {
991            text.push_str(&format!("{{\"line\":{i}}}\n"));
992            fs::write(&source, &text).unwrap();
993            archive.archive(&source).unwrap();
994        }
995
996        assert!(manifest_lines(&archive).len() <= 2 + 64);
997        assert_eq!(blobs(&archive).len(), 1);
998        drop(archive);
999        let reopened = fx.open();
1000        assert_eq!(reopened.read(&source).unwrap(), Some(text));
1001    }
1002
1003    #[test]
1004    fn removed_entries_leave_the_manifest_and_their_blobs_and_stay_gone_on_reopen() {
1005        let fx = Fixture::new();
1006        let gone = fx.write("gone.jsonl", "{\"line\":\"secret\"}\n");
1007        let copy = fx.write("copy.jsonl", "{\"line\":\"secret\"}\n");
1008        let kept = fx.write("kept.jsonl", "{\"line\":\"kept\"}\n");
1009        let archive = fx.open();
1010        for source in [&gone, &copy, &kept] {
1011            archive.archive(source).unwrap();
1012        }
1013        let sha = archive.entry(&gone).unwrap().sha256;
1014        assert!(archive.holder(&sha).is_some());
1015
1016        let removed = archive.remove_where(|e| e.source.contains("gone")).unwrap();
1017        assert_eq!(removed.len(), 1);
1018        assert!(
1019            archive.holder(&sha).is_some(),
1020            "the copy still uses the blob"
1021        );
1022        assert_eq!(blobs(&archive).len(), 2);
1023
1024        archive.remove_where(|e| e.source.contains("copy")).unwrap();
1025        assert!(archive.holder(&sha).is_none());
1026        assert_eq!(blobs(&archive).len(), 1);
1027        assert!(archive.entry(&gone).is_none());
1028        assert!(manifest_lines(&archive).iter().all(|l| !l.contains("gone")));
1029
1030        let reopened = fx.open();
1031        assert_eq!(reopened.len(), 1);
1032        assert!(reopened.entry(&kept).is_some());
1033    }
1034
1035    fn stray_blob(archive: &TranscriptArchive, fill: char) -> PathBuf {
1036        let path = archive
1037            .dir()
1038            .join(format!("{}{BLOB_SUFFIX}", fill.to_string().repeat(64)));
1039        fs::write(&path, b"unreferenced").unwrap();
1040        path
1041    }
1042
1043    #[test]
1044    fn a_blob_no_entry_or_journal_names_survives_open() {
1045        let fx = Fixture::new();
1046        let source = fx.write("a.jsonl", "{\"line\":1}\n");
1047        let archive = fx.open();
1048        archive.archive(&source).unwrap();
1049        let stray = stray_blob(&archive, 'f');
1050        drop(archive);
1051
1052        let reopened = fx.open();
1053        assert!(stray.exists());
1054        assert_eq!(blobs(&reopened).len(), 2);
1055    }
1056
1057    #[test]
1058    fn a_journal_left_by_a_crash_after_compaction_has_its_blobs_deleted_at_open() {
1059        let fx = Fixture::new();
1060        let kept = fx.write("kept.jsonl", "{\"line\":\"kept\"}\n");
1061        let archive = fx.open();
1062        archive.archive(&kept).unwrap();
1063        let doomed = stray_blob(&archive, 'a');
1064        let bystander = stray_blob(&archive, 'b');
1065        let kept_blob = archive.entry(&kept).unwrap().file;
1066        let names = format!(
1067            "{}\n{kept_blob}\n",
1068            doomed.file_name().unwrap().to_string_lossy()
1069        );
1070        fs::write(archive.dir().join(JOURNAL_FILE), names).unwrap();
1071        drop(archive);
1072
1073        let reopened = fx.open();
1074        assert!(!doomed.exists());
1075        assert!(bystander.exists());
1076        assert!(reopened.dir().join(&kept_blob).exists());
1077        assert!(!reopened.dir().join(JOURNAL_FILE).exists());
1078        assert_eq!(reopened.len(), 1);
1079    }
1080
1081    #[test]
1082    fn remove_where_leaves_no_journal_behind() {
1083        let fx = Fixture::new();
1084        let source = fx.write("a.jsonl", "{\"line\":1}\n");
1085        let archive = fx.open();
1086        archive.archive(&source).unwrap();
1087        archive.remove_where(|_| true).unwrap();
1088        assert!(!archive.dir().join(JOURNAL_FILE).exists());
1089        assert!(blobs(&archive).is_empty());
1090    }
1091
1092    #[test]
1093    fn a_missing_manifest_never_deletes_blobs() {
1094        let fx = Fixture::new();
1095        let source = fx.write("a.jsonl", "{\"line\":1}\n");
1096        let archive = fx.open();
1097        archive.archive(&source).unwrap();
1098        let dir = archive.dir().to_path_buf();
1099        drop(archive);
1100        fs::remove_file(dir.join(MANIFEST_FILE)).unwrap();
1101
1102        let reopened = fx.open();
1103        assert!(reopened.is_empty());
1104        assert_eq!(blobs(&reopened).len(), 1);
1105    }
1106
1107    fn key(byte: u8) -> SealKey {
1108        SealKey::from_bytes(KeyGuard::new([byte; 32])).unwrap()
1109    }
1110
1111    fn keys(byte: u8) -> SealKeys {
1112        SealKeys::single(key(byte))
1113    }
1114
1115    fn files(dir: &Path) -> Vec<PathBuf> {
1116        let mut paths: Vec<PathBuf> = fs::read_dir(dir)
1117            .unwrap()
1118            .map(|e| e.unwrap().path())
1119            .collect();
1120        paths.sort();
1121        paths
1122    }
1123
1124    fn contains(haystack: &[u8], needle: &[u8]) -> bool {
1125        haystack.windows(needle.len()).any(|w| w == needle)
1126    }
1127
1128    fn assert_all_sealed_under(dir: &Path, keys: &SealKeys) {
1129        for path in files(dir) {
1130            let name = path.file_name().unwrap().to_string_lossy().into_owned();
1131            if name == MANIFEST_FILE {
1132                assert_eq!(fs::read_to_string(&path).unwrap(), "");
1133            } else if name == SEALED_MANIFEST_FILE {
1134                let text = fs::read_to_string(&path).unwrap();
1135                for line in text.lines() {
1136                    let sealed = BASE64_STANDARD.decode(line).unwrap();
1137                    assert_eq!(sealed::key_id_of(&sealed), Some(keys.current_id()));
1138                }
1139            } else {
1140                assert_eq!(
1141                    sealed::sealed_key_id(&path).unwrap(),
1142                    Some(keys.current_id()),
1143                    "{name}"
1144                );
1145            }
1146        }
1147    }
1148
1149    impl Fixture {
1150        fn open_sealed(&self, keys: SealKeys) -> TranscriptArchive {
1151            TranscriptArchive::open_sealed(&self.archive_dir, keys).unwrap()
1152        }
1153    }
1154
1155    #[test]
1156    fn a_sealed_archive_holds_neither_the_transcript_text_nor_its_source_path() {
1157        let fx = Fixture::new();
1158        let text = "{\"text\":\"the quarterly migration plan\"}\n".repeat(4);
1159        let source = fx.write("orchard-ledger.jsonl", &text);
1160        let archive = fx.open_sealed(keys(1));
1161        archive.archive(&source).unwrap();
1162
1163        let path_bytes = source.to_string_lossy().into_owned();
1164        for path in files(archive.dir()) {
1165            let raw = fs::read(&path).unwrap();
1166            assert!(!contains(&raw, b"quarterly migration"), "{path:?}");
1167            assert!(!contains(&raw, b"orchard-ledger"), "{path:?}");
1168            assert!(!contains(&raw, path_bytes.as_bytes()), "{path:?}");
1169            assert!(zstd::decode_all(raw.as_slice()).is_err(), "{path:?}");
1170        }
1171        assert_all_sealed_under(archive.dir(), &keys(1));
1172        assert_eq!(archive.read(&source).unwrap(), Some(text.clone()));
1173        drop(archive);
1174        let reopened = fx.open_sealed(keys(1));
1175        assert_eq!(reopened.read(&source).unwrap(), Some(text));
1176    }
1177
1178    #[test]
1179    fn a_copy_opens_with_every_entry_under_the_same_keys() {
1180        let fx = Fixture::new();
1181        let a = fx.write("a.jsonl", "{\"line\":\"alpha\"}\n");
1182        let b = fx.write("b.jsonl", "{\"line\":\"beta\"}\n");
1183        let archive = fx.open_sealed(keys(5));
1184        archive.archive(&a).unwrap();
1185        archive.archive(&b).unwrap();
1186
1187        let dest = fx._tmp.path().join("backup-transcripts");
1188        archive.copy_into(&dest).unwrap();
1189        let copy = TranscriptArchive::open_sealed(&dest, keys(5)).unwrap();
1190        assert_eq!(copy.len(), 2);
1191        assert_eq!(copy.read(&a).unwrap(), archive.read(&a).unwrap());
1192        assert_eq!(copy.read(&b).unwrap(), archive.read(&b).unwrap());
1193    }
1194
1195    #[test]
1196    fn a_blob_removed_while_the_copy_runs_leaves_nothing_in_it() {
1197        let fx = Fixture::new();
1198        let a = fx.write("a.jsonl", "{\"line\":\"the deleted session\"}\n");
1199        let b = fx.write("b.jsonl", "{\"line\":\"kept\"}\n");
1200        let archive = fx.open_sealed(keys(6));
1201        archive.archive(&a).unwrap();
1202        archive.archive(&b).unwrap();
1203        let doomed = archive.entry(&a).unwrap();
1204
1205        let dest = fx._tmp.path().join("backup-transcripts");
1206        archive
1207            .copy_into_with(&dest, || {
1208                archive.remove_where(|e| e.source == doomed.source).unwrap();
1209            })
1210            .unwrap();
1211        assert!(!dest.join(&doomed.file).exists());
1212        let copy = TranscriptArchive::open_sealed(&dest, keys(6)).unwrap();
1213        assert_eq!(copy.len(), 1);
1214        assert_eq!(copy.read(&a).unwrap(), None);
1215        assert!(copy.read(&b).unwrap().is_some());
1216    }
1217
1218    #[test]
1219    fn a_copy_of_an_archive_awaiting_reseal_still_awaits_it() {
1220        let fx = Fixture::new();
1221        let a = fx.write("a.jsonl", "{\"line\":1}\n");
1222        fx.open().archive(&a).unwrap();
1223        let archive = fx.open_sealed(keys(7));
1224        assert!(archive.holds_unsealed());
1225
1226        let dest = fx._tmp.path().join("backup-transcripts");
1227        archive.copy_into(&dest).unwrap();
1228        let copy = TranscriptArchive::open_sealed(&dest, keys(7)).unwrap();
1229        assert!(copy.holds_unsealed());
1230        assert!(copy.read(&a).unwrap().is_some());
1231    }
1232
1233    #[test]
1234    fn a_sealed_archive_does_not_open_without_its_key() {
1235        let fx = Fixture::new();
1236        let source = fx.write("a.jsonl", "{\"line\":1}\n");
1237        let archive = fx.open_sealed(keys(2));
1238        archive.archive(&source).unwrap();
1239        drop(archive);
1240
1241        let err = TranscriptArchive::open(&fx.archive_dir)
1242            .err()
1243            .expect("an unkeyed open fails");
1244        assert_eq!(seal_error(&err), Some(&SealError::KeyRequired));
1245        let err = TranscriptArchive::open_sealed(&fx.archive_dir, keys(3))
1246            .err()
1247            .expect("another key fails");
1248        assert!(matches!(seal_error(&err), Some(SealError::UnknownKey(_))));
1249        assert_eq!(blobs(&fx.open_sealed(keys(2))).len(), 1);
1250    }
1251
1252    #[test]
1253    fn a_plaintext_archive_opens_sealed_and_reseal_seals_every_file() {
1254        let fx = Fixture::new();
1255        let first = fx.write("a.jsonl", "{\"a\":1}\n");
1256        let copy = fx.write("b.jsonl", "{\"a\":1}\n");
1257        let second = fx.write("c.jsonl", "{\"c\":3}\n");
1258        let plain = fx.open();
1259        for source in [&first, &copy, &second] {
1260            plain.archive(source).unwrap();
1261        }
1262        drop(plain);
1263        fs::write(fx.archive_dir.join("manifest.jsonl.tmp"), "{\"source\":").unwrap();
1264
1265        let archive = fx.open_sealed(keys(4));
1266        assert_eq!(archive.read(&copy).unwrap().as_deref(), Some("{\"a\":1}\n"));
1267        assert_eq!(
1268            archive.read(&second).unwrap().as_deref(),
1269            Some("{\"c\":3}\n")
1270        );
1271        assert!(manifest_lines(&archive).is_empty());
1272
1273        assert_eq!(archive.reseal().unwrap(), 2);
1274        assert_all_sealed_under(archive.dir(), &keys(4));
1275        assert_eq!(archive.reseal().unwrap(), 0);
1276        assert_eq!(
1277            archive.read(&first).unwrap().as_deref(),
1278            Some("{\"a\":1}\n")
1279        );
1280        drop(archive);
1281
1282        let reopened = fx.open_sealed(keys(4));
1283        assert_eq!(reopened.len(), 3);
1284        assert_eq!(
1285            reopened.read(&second).unwrap().as_deref(),
1286            Some("{\"c\":3}\n")
1287        );
1288    }
1289
1290    #[test]
1291    fn a_rotation_then_reseal_leaves_only_the_new_key() {
1292        let fx = Fixture::new();
1293        let first = fx.write("a.jsonl", "{\"a\":1}\n");
1294        let second = fx.write("b.jsonl", "{\"b\":2}\n");
1295        let archive = fx.open_sealed(keys(5));
1296        archive.archive(&first).unwrap();
1297
1298        archive.set_seal_keys(SealKeys::new(key(6), vec![key(5)]));
1299        archive.archive(&second).unwrap();
1300        assert_eq!(
1301            archive.read(&first).unwrap().as_deref(),
1302            Some("{\"a\":1}\n")
1303        );
1304        assert_eq!(archive.reseal().unwrap(), 2);
1305        assert_all_sealed_under(archive.dir(), &keys(6));
1306        drop(archive);
1307
1308        let reopened = fx.open_sealed(keys(6));
1309        assert_eq!(reopened.len(), 2);
1310        assert_eq!(
1311            reopened.read(&first).unwrap().as_deref(),
1312            Some("{\"a\":1}\n")
1313        );
1314        assert_eq!(
1315            reopened.read(&second).unwrap().as_deref(),
1316            Some("{\"b\":2}\n")
1317        );
1318    }
1319
1320    #[test]
1321    fn a_torn_sealed_manifest_line_is_ignored() {
1322        let fx = Fixture::new();
1323        let first = fx.write("a.jsonl", "{\"a\":1}\n");
1324        let second = fx.write("b.jsonl", "{\"b\":2}\n");
1325        let archive = fx.open_sealed(keys(7));
1326        archive.archive(&first).unwrap();
1327        archive.archive(&second).unwrap();
1328        let line = sealed_manifest_lines(archive.dir())[0].clone();
1329        drop(archive);
1330        for cut in [line.len() / 2, line.len() - 3] {
1331            let mut manifest = fs::OpenOptions::new()
1332                .append(true)
1333                .open(fx.archive_dir.join(SEALED_MANIFEST_FILE))
1334                .unwrap();
1335            manifest.write_all(&line.as_bytes()[..cut]).unwrap();
1336            drop(manifest);
1337
1338            let reopened = fx.open_sealed(keys(7));
1339            assert_eq!(reopened.len(), 2);
1340            assert_eq!(sealed_manifest_lines(reopened.dir()).len(), 2);
1341        }
1342
1343        let reopened = fx.open_sealed(keys(7));
1344        let third = fx.write("c.jsonl", "{\"c\":3}\n");
1345        assert_eq!(reopened.archive(&third).unwrap(), Archived::Stored);
1346        drop(reopened);
1347        let again = fx.open_sealed(keys(7));
1348        assert_eq!(again.len(), 3);
1349        assert_eq!(again.read(&third).unwrap().as_deref(), Some("{\"c\":3}\n"));
1350        assert_all_sealed_under(again.dir(), &keys(7));
1351    }
1352
1353    #[test]
1354    fn removal_works_sealed_and_an_interrupted_removal_is_finished_from_a_sealed_journal() {
1355        let fx = Fixture::new();
1356        let gone = fx.write("gone.jsonl", "{\"line\":\"gone\"}\n");
1357        let kept = fx.write("kept.jsonl", "{\"line\":\"kept\"}\n");
1358        let archive = fx.open_sealed(keys(8));
1359        archive.archive(&gone).unwrap();
1360        archive.archive(&kept).unwrap();
1361
1362        let removed = archive.remove_where(|e| e.source.contains("gone")).unwrap();
1363        assert_eq!(removed.len(), 1);
1364        assert_eq!(blobs(&archive).len(), 1);
1365        assert!(!archive.dir().join(JOURNAL_FILE).exists());
1366
1367        let doomed = stray_blob(&archive, 'a');
1368        let bystander = stray_blob(&archive, 'b');
1369        let names = format!("{}\n", doomed.file_name().unwrap().to_string_lossy());
1370        sealed::write_file(
1371            &archive.dir().join(JOURNAL_FILE),
1372            Some(&keys(8)),
1373            JOURNAL_FILE,
1374            names.as_bytes(),
1375        )
1376        .unwrap();
1377        drop(archive);
1378
1379        let reopened = fx.open_sealed(keys(8));
1380        assert!(!doomed.exists());
1381        assert!(bystander.exists());
1382        assert!(!reopened.dir().join(JOURNAL_FILE).exists());
1383        assert_eq!(reopened.len(), 1);
1384        assert_eq!(
1385            reopened.read(&kept).unwrap().as_deref(),
1386            Some("{\"line\":\"kept\"}\n")
1387        );
1388        assert!(manifest_lines(&reopened).is_empty());
1389        assert_eq!(sealed_manifest_lines(reopened.dir()).len(), 1);
1390    }
1391
1392    #[test]
1393    fn a_sealed_journal_is_not_read_without_the_key() {
1394        let fx = Fixture::new();
1395        let archive = fx.open();
1396        let doomed = stray_blob(&archive, 'c');
1397        let names = format!("{}\n", doomed.file_name().unwrap().to_string_lossy());
1398        sealed::write_file(
1399            &archive.dir().join(JOURNAL_FILE),
1400            Some(&keys(9)),
1401            JOURNAL_FILE,
1402            names.as_bytes(),
1403        )
1404        .unwrap();
1405        drop(archive);
1406
1407        let err = TranscriptArchive::open(&fx.archive_dir)
1408            .err()
1409            .expect("an unkeyed open fails");
1410        assert_eq!(seal_error(&err), Some(&SealError::KeyRequired));
1411        assert!(doomed.exists());
1412    }
1413
1414    fn sealed_manifest_lines(dir: &Path) -> Vec<String> {
1415        match fs::read_to_string(dir.join(SEALED_MANIFEST_FILE)) {
1416            Ok(text) => text.lines().map(str::to_string).collect(),
1417            Err(e) if e.kind() == ErrorKind::NotFound => Vec::new(),
1418            Err(e) => panic!("{e}"),
1419        }
1420    }
1421
1422    fn line_from_a_binary_before_sealing(
1423        fx: &Fixture,
1424        source: &Path,
1425        archived_at: DateTime<Utc>,
1426    ) -> String {
1427        let bytes = fs::read(source).unwrap();
1428        let sha256 = hex(&Sha256::digest(&bytes));
1429        let file = format!("{sha256}{BLOB_SUFFIX}");
1430        fs::write(
1431            fx.archive_dir.join(&file),
1432            zstd::encode_all(bytes.as_slice(), 0).unwrap(),
1433        )
1434        .unwrap();
1435        serde_json::to_string(&ArchiveEntry {
1436            source: source_key(source),
1437            file,
1438            sha256,
1439            archived_at,
1440        })
1441        .unwrap()
1442    }
1443
1444    #[test]
1445    fn a_sealed_archive_leaves_the_plaintext_manifest_readable_by_a_binary_from_before_sealing() {
1446        let fx = Fixture::new();
1447        let first = fx.write("a.jsonl", "{\"a\":1}\n");
1448        let plain = fx.open();
1449        plain.archive(&first).unwrap();
1450        drop(plain);
1451
1452        let archive = fx.open_sealed(keys(10));
1453        let second = fx.write("b.jsonl", "{\"b\":2}\n");
1454        archive.archive(&second).unwrap();
1455        archive.reseal().unwrap();
1456        let third = fx.write("c.jsonl", "{\"c\":3}\n");
1457        archive.archive(&third).unwrap();
1458
1459        for line in manifest_lines(&archive) {
1460            serde_json::from_str::<ArchiveEntry>(&line).unwrap();
1461        }
1462        assert_eq!(sealed_manifest_lines(archive.dir()).len(), 3);
1463    }
1464
1465    #[test]
1466    fn entries_a_binary_from_before_sealing_appended_are_merged_and_then_sealed() {
1467        let fx = Fixture::new();
1468        let kept = fx.write("kept.jsonl", "{\"k\":1}\n");
1469        let changed = fx.write("changed.jsonl", "{\"v\":1}\n");
1470        let archive = fx.open_sealed(keys(11));
1471        archive.archive(&kept).unwrap();
1472        archive.archive(&changed).unwrap();
1473        let old_blob = archive.entry(&changed).unwrap().file;
1474        drop(archive);
1475
1476        let added = fx.write("added.jsonl", "{\"n\":1}\n");
1477        fs::write(&changed, "{\"v\":2}\n").unwrap();
1478        let later = Utc::now() + chrono::Duration::seconds(1);
1479        let lines = [
1480            line_from_a_binary_before_sealing(&fx, &added, later),
1481            line_from_a_binary_before_sealing(&fx, &changed, later),
1482        ];
1483        let mut manifest = fs::OpenOptions::new()
1484            .append(true)
1485            .open(fx.archive_dir.join(MANIFEST_FILE))
1486            .unwrap();
1487        for line in &lines {
1488            writeln!(manifest, "{line}").unwrap();
1489        }
1490        drop(manifest);
1491
1492        let reopened = fx.open_sealed(keys(11));
1493        assert_eq!(reopened.len(), 3);
1494        assert_eq!(
1495            reopened.read(&added).unwrap().as_deref(),
1496            Some("{\"n\":1}\n")
1497        );
1498        assert_eq!(
1499            reopened.read(&changed).unwrap().as_deref(),
1500            Some("{\"v\":2}\n")
1501        );
1502        assert_eq!(
1503            reopened.read(&kept).unwrap().as_deref(),
1504            Some("{\"k\":1}\n")
1505        );
1506        assert!(!reopened.dir().join(&old_blob).exists());
1507        assert!(manifest_lines(&reopened).is_empty());
1508        assert!(reopened.holds_unsealed());
1509
1510        reopened.reseal().unwrap();
1511        assert!(!reopened.holds_unsealed());
1512        assert_all_sealed_under(reopened.dir(), &keys(11));
1513        drop(reopened);
1514        let again = fx.open_sealed(keys(11));
1515        assert_eq!(again.len(), 3);
1516        assert!(!again.holds_unsealed());
1517    }
1518
1519    #[test]
1520    fn a_removal_after_keys_arrive_leaves_no_plaintext_entry_behind() {
1521        let fx = Fixture::new();
1522        let gone = fx.write("gone.jsonl", "{\"line\":\"gone\"}\n");
1523        let kept = fx.write("kept.jsonl", "{\"line\":\"kept\"}\n");
1524        let archive = fx.open();
1525        archive.archive(&gone).unwrap();
1526        archive.archive(&kept).unwrap();
1527
1528        archive.set_seal_keys(keys(12));
1529        archive.remove_where(|e| e.source.contains("gone")).unwrap();
1530        assert!(manifest_lines(&archive).is_empty());
1531        drop(archive);
1532
1533        let reopened = fx.open_sealed(keys(12));
1534        assert_eq!(reopened.len(), 1);
1535        assert!(reopened.entry(&gone).is_none());
1536    }
1537
1538    #[test]
1539    fn a_fold_cut_short_before_its_reseal_still_reports_unsealed_files_after_a_reopen() {
1540        let fx = Fixture::new();
1541        let first = fx.write("a.jsonl", "{\"a\":1}\n");
1542        let plain = fx.open();
1543        plain.archive(&first).unwrap();
1544        drop(plain);
1545
1546        let folded = fx.open_sealed(keys(13));
1547        assert!(folded.holds_unsealed());
1548        assert!(manifest_lines(&folded).is_empty());
1549        drop(folded);
1550
1551        let reopened = fx.open_sealed(keys(13));
1552        assert!(reopened.holds_unsealed());
1553        reopened.reseal().unwrap();
1554        assert_all_sealed_under(reopened.dir(), &keys(13));
1555        drop(reopened);
1556        assert!(!fx.open_sealed(keys(13)).holds_unsealed());
1557    }
1558}