Skip to main content

second_brain_core/
transcript_archive.rs

1use std::collections::HashMap;
2use std::fs::{self, File, OpenOptions};
3use std::io::{ErrorKind, Write};
4use std::path::{Path, PathBuf};
5use std::sync::{Mutex, MutexGuard, PoisonError};
6
7use anyhow::{Context, Result};
8use chrono::{DateTime, Utc};
9use serde::{Deserialize, Serialize};
10use sha2::{Digest, Sha256};
11
12use crate::redact::redact;
13
14pub const MANIFEST_FILE: &str = "manifest.jsonl";
15
16const BLOB_SUFFIX: &str = ".jsonl.zst";
17const COMPACTION_SLACK: usize = 64;
18
19#[derive(Debug, Clone, Copy, PartialEq, Eq)]
20pub enum Archived {
21    Stored,
22    Refreshed,
23    Unchanged,
24    Copy,
25}
26
27#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
28pub struct ArchiveEntry {
29    pub source: String,
30    pub file: String,
31    pub sha256: String,
32    pub archived_at: DateTime<Utc>,
33}
34
35/// Redacted zstd copies of transcripts, keyed by source path; an entry is
36/// never dropped because its source was deleted.
37pub struct TranscriptArchive {
38    dir: PathBuf,
39    state: Mutex<State>,
40}
41
42struct State {
43    entries: HashMap<String, ArchiveEntry>,
44    manifest_lines: usize,
45    manifest_damaged: bool,
46}
47
48impl State {
49    fn needs_compaction(&self) -> bool {
50        self.manifest_damaged || self.manifest_lines > 2 * self.entries.len() + COMPACTION_SLACK
51    }
52
53    fn holder_of(&self, sha256: &str) -> Option<&ArchiveEntry> {
54        self.entries.values().find(|e| e.sha256 == sha256)
55    }
56
57    fn references(&self, file: &str) -> bool {
58        self.entries.values().any(|e| e.file == file)
59    }
60}
61
62impl TranscriptArchive {
63    pub fn open(dir: impl Into<PathBuf>) -> Result<Self> {
64        let dir = dir.into();
65        fs::create_dir_all(&dir)
66            .with_context(|| format!("creating the transcript archive at {}", dir.display()))?;
67        let manifest = dir.join(MANIFEST_FILE);
68        if !manifest.exists() {
69            write_atomic(&manifest, b"")
70                .with_context(|| format!("creating {}", manifest.display()))?;
71        }
72        let (state, torn) = load_manifest(&manifest)?;
73        let archive = Self {
74            dir,
75            state: Mutex::new(state),
76        };
77        {
78            let mut state = archive.state();
79            if torn || state.needs_compaction() {
80                archive.compact(&mut state)?;
81            }
82        }
83        Ok(archive)
84    }
85
86    pub fn dir(&self) -> &Path {
87        &self.dir
88    }
89
90    pub fn archive(&self, source: &Path) -> Result<Archived> {
91        let mut state = self.state();
92        let bytes = fs::read(source).with_context(|| format!("reading {}", source.display()))?;
93        let sha256 = hex(&Sha256::digest(&bytes));
94        let key = source_key(source);
95
96        let previous = state.entries.get(&key).cloned();
97        if let Some(prev) = &previous
98            && prev.sha256 == sha256
99            && self.dir.join(&prev.file).exists()
100        {
101            return Ok(Archived::Unchanged);
102        }
103
104        let file = format!("{sha256}{BLOB_SUFFIX}");
105        let shared = state
106            .holder_of(&sha256)
107            .is_some_and(|holder| self.dir.join(&holder.file).exists());
108        if !shared {
109            self.write_blob(&file, &bytes)?;
110        }
111        let entry = ArchiveEntry {
112            source: key.clone(),
113            file,
114            sha256,
115            archived_at: Utc::now(),
116        };
117        state.entries.insert(key, entry.clone());
118        if state.needs_compaction() {
119            self.compact(&mut state)?;
120        } else if let Err(e) = self.append(&mut state, &entry) {
121            state.manifest_damaged = true;
122            return Err(e);
123        }
124
125        if let Some(prev) = &previous
126            && !state.references(&prev.file)
127        {
128            self.remove_blob(&prev.file);
129        }
130
131        Ok(match (shared, previous) {
132            (true, _) => Archived::Copy,
133            (false, None) => Archived::Stored,
134            (false, Some(_)) => Archived::Refreshed,
135        })
136    }
137
138    pub fn entry(&self, source: &Path) -> Option<ArchiveEntry> {
139        self.state().entries.get(&source_key(source)).cloned()
140    }
141
142    pub fn read(&self, source: &Path) -> Result<Option<String>> {
143        let Some(entry) = self.entry(source) else {
144            return Ok(None);
145        };
146        let path = self.dir.join(&entry.file);
147        let compressed = fs::read(&path).with_context(|| format!("reading {}", path.display()))?;
148        let bytes = zstd::decode_all(compressed.as_slice())
149            .with_context(|| format!("decompressing {}", path.display()))?;
150        let text =
151            String::from_utf8(bytes).with_context(|| format!("decoding {}", path.display()))?;
152        Ok(Some(text))
153    }
154
155    /// Every archived transcript, in no particular order.
156    pub fn entries(&self) -> Vec<ArchiveEntry> {
157        self.state().entries.values().cloned().collect()
158    }
159
160    pub fn len(&self) -> usize {
161        self.state().entries.len()
162    }
163
164    pub fn is_empty(&self) -> bool {
165        self.len() == 0
166    }
167
168    fn state(&self) -> MutexGuard<'_, State> {
169        self.state.lock().unwrap_or_else(PoisonError::into_inner)
170    }
171
172    fn write_blob(&self, file: &str, source_bytes: &[u8]) -> Result<()> {
173        let text = redact(&String::from_utf8_lossy(source_bytes)).text;
174        let compressed =
175            zstd::encode_all(text.as_bytes(), 0).context("compressing a transcript")?;
176        let target = self.dir.join(file);
177        write_atomic(&target, &compressed).with_context(|| format!("writing {}", target.display()))
178    }
179
180    fn remove_blob(&self, file: &str) {
181        let path = self.dir.join(file);
182        match fs::remove_file(&path) {
183            Ok(()) => {}
184            Err(e) if e.kind() == ErrorKind::NotFound => {}
185            Err(e) => {
186                tracing::warn!(path = %path.display(), error = %e, "removing an unused archive blob")
187            }
188        }
189    }
190
191    fn append(&self, state: &mut State, entry: &ArchiveEntry) -> Result<()> {
192        let path = self.dir.join(MANIFEST_FILE);
193        let mut line = serde_json::to_string(entry)?;
194        line.push('\n');
195        let mut file = OpenOptions::new()
196            .create(true)
197            .append(true)
198            .open(&path)
199            .with_context(|| format!("opening {}", path.display()))?;
200        file.write_all(line.as_bytes())
201            .and_then(|()| file.sync_data())
202            .with_context(|| format!("appending to {}", path.display()))?;
203        state.manifest_lines += 1;
204        Ok(())
205    }
206
207    fn compact(&self, state: &mut State) -> Result<()> {
208        let mut entries: Vec<&ArchiveEntry> = state.entries.values().collect();
209        entries.sort_by(|a, b| a.source.cmp(&b.source));
210        let mut body = String::new();
211        for entry in &entries {
212            body.push_str(&serde_json::to_string(entry)?);
213            body.push('\n');
214        }
215        let path = self.dir.join(MANIFEST_FILE);
216        write_atomic(&path, body.as_bytes())
217            .with_context(|| format!("compacting {}", path.display()))?;
218        state.manifest_lines = entries.len();
219        state.manifest_damaged = false;
220        Ok(())
221    }
222}
223
224/// The flag is true when a line failed to parse: a torn tail would swallow
225/// the next appended line, so the caller rewrites the manifest.
226fn load_manifest(path: &Path) -> Result<(State, bool)> {
227    let text = match fs::read_to_string(path) {
228        Ok(text) => text,
229        Err(e) if e.kind() == ErrorKind::NotFound => String::new(),
230        Err(e) => return Err(e).with_context(|| format!("reading {}", path.display())),
231    };
232    let mut state = State {
233        entries: HashMap::new(),
234        manifest_lines: 0,
235        manifest_damaged: false,
236    };
237    let mut torn = !text.is_empty() && !text.ends_with('\n');
238    for line in text.lines().filter(|l| !l.trim().is_empty()) {
239        state.manifest_lines += 1;
240        match serde_json::from_str::<ArchiveEntry>(line) {
241            Ok(entry) => {
242                state.entries.insert(entry.source.clone(), entry);
243            }
244            Err(e) => {
245                torn = true;
246                tracing::warn!(path = %path.display(), error = %e, "ignoring an unreadable archive manifest line");
247            }
248        }
249    }
250    Ok((state, torn))
251}
252
253fn write_atomic(target: &Path, bytes: &[u8]) -> std::io::Result<()> {
254    let mut tmp_name = target.file_name().unwrap_or_default().to_os_string();
255    tmp_name.push(".tmp");
256    let tmp = target.with_file_name(tmp_name);
257    let mut file = File::create(&tmp)?;
258    file.write_all(bytes)?;
259    file.sync_all()?;
260    fs::rename(&tmp, target)?;
261    match target.parent() {
262        Some(dir) => File::open(dir)?.sync_all(),
263        None => Ok(()),
264    }
265}
266
267fn source_key(source: &Path) -> String {
268    source.to_string_lossy().into_owned()
269}
270
271fn hex(bytes: &[u8]) -> String {
272    bytes.iter().map(|b| format!("{b:02x}")).collect()
273}
274
275#[cfg(test)]
276mod tests {
277    use std::fs;
278    use std::io::Write;
279
280    use super::*;
281
282    fn blobs(archive: &TranscriptArchive) -> Vec<String> {
283        let mut names: Vec<String> = fs::read_dir(archive.dir())
284            .unwrap()
285            .map(|e| e.unwrap().file_name().to_string_lossy().into_owned())
286            .filter(|n| n.ends_with(".jsonl.zst"))
287            .collect();
288        names.sort();
289        names
290    }
291
292    fn manifest_lines(archive: &TranscriptArchive) -> Vec<String> {
293        fs::read_to_string(archive.dir().join(MANIFEST_FILE))
294            .unwrap()
295            .lines()
296            .map(str::to_string)
297            .collect()
298    }
299
300    struct Fixture {
301        _tmp: tempfile::TempDir,
302        projects: PathBuf,
303        archive_dir: PathBuf,
304    }
305
306    impl Fixture {
307        fn new() -> Self {
308            let tmp = tempfile::tempdir().unwrap();
309            let projects = tmp.path().join("projects");
310            fs::create_dir_all(&projects).unwrap();
311            let archive_dir = tmp.path().join("data").join("transcripts");
312            Self {
313                _tmp: tmp,
314                projects,
315                archive_dir,
316            }
317        }
318
319        fn write(&self, name: &str, text: &str) -> PathBuf {
320            let path = self.projects.join(name);
321            fs::write(&path, text).unwrap();
322            path
323        }
324
325        fn open(&self) -> TranscriptArchive {
326            TranscriptArchive::open(&self.archive_dir).unwrap()
327        }
328    }
329
330    #[test]
331    fn open_creates_the_directory() {
332        let fx = Fixture::new();
333        let archive = fx.open();
334        assert!(archive.dir().is_dir());
335        assert!(archive.is_empty());
336    }
337
338    #[test]
339    fn a_new_source_is_stored_then_unchanged() {
340        let fx = Fixture::new();
341        let source = fx.write("a.jsonl", "{\"line\":1}\n");
342        let archive = fx.open();
343
344        assert_eq!(archive.archive(&source).unwrap(), Archived::Stored);
345        assert_eq!(archive.archive(&source).unwrap(), Archived::Unchanged);
346
347        assert_eq!(archive.len(), 1);
348        assert_eq!(blobs(&archive).len(), 1);
349        assert_eq!(manifest_lines(&archive).len(), 1);
350        let entry = archive.entry(&source).unwrap();
351        assert_eq!(entry.source, source.to_string_lossy());
352        assert_eq!(entry.file, format!("{}.jsonl.zst", entry.sha256));
353        assert_eq!(entry.sha256.len(), 64);
354        assert_eq!(
355            archive.read(&source).unwrap().as_deref(),
356            Some("{\"line\":1}\n")
357        );
358    }
359
360    #[test]
361    fn a_changed_source_is_refreshed_and_its_old_blob_removed() {
362        let fx = Fixture::new();
363        let source = fx.write("a.jsonl", "{\"line\":1}\n");
364        let archive = fx.open();
365        archive.archive(&source).unwrap();
366        let old = archive.entry(&source).unwrap();
367
368        fs::write(&source, "{\"line\":1}\n{\"line\":2}\n").unwrap();
369        assert_eq!(archive.archive(&source).unwrap(), Archived::Refreshed);
370
371        let new = archive.entry(&source).unwrap();
372        assert_ne!(new.sha256, old.sha256);
373        assert_eq!(
374            archive.read(&source).unwrap().as_deref(),
375            Some("{\"line\":1}\n{\"line\":2}\n")
376        );
377        assert_eq!(blobs(&archive), vec![new.file]);
378        assert!(!archive.dir().join(&old.file).exists());
379    }
380
381    #[test]
382    fn a_copy_at_another_path_shares_the_blob() {
383        let fx = Fixture::new();
384        let first = fx.write("a.jsonl", "{\"same\":true}\n");
385        let second = fx.write("b.jsonl", "{\"same\":true}\n");
386        let archive = fx.open();
387
388        assert_eq!(archive.archive(&first).unwrap(), Archived::Stored);
389        assert_eq!(archive.archive(&second).unwrap(), Archived::Copy);
390
391        assert_eq!(archive.len(), 2);
392        assert_eq!(blobs(&archive).len(), 1);
393        assert_eq!(
394            archive.entry(&first).unwrap().file,
395            archive.entry(&second).unwrap().file
396        );
397        assert_eq!(
398            archive.read(&second).unwrap().as_deref(),
399            Some("{\"same\":true}\n")
400        );
401    }
402
403    #[test]
404    fn refreshing_one_copy_keeps_the_blob_the_other_still_uses() {
405        let fx = Fixture::new();
406        let first = fx.write("a.jsonl", "{\"same\":true}\n");
407        let second = fx.write("b.jsonl", "{\"same\":true}\n");
408        let archive = fx.open();
409        archive.archive(&first).unwrap();
410        archive.archive(&second).unwrap();
411
412        fs::write(&first, "{\"same\":false}\n").unwrap();
413        assert_eq!(archive.archive(&first).unwrap(), Archived::Refreshed);
414
415        assert_eq!(blobs(&archive).len(), 2);
416        assert_eq!(
417            archive.read(&second).unwrap().as_deref(),
418            Some("{\"same\":true}\n")
419        );
420    }
421
422    #[test]
423    fn the_archive_outlives_a_deleted_source() {
424        let fx = Fixture::new();
425        let source = fx.write("a.jsonl", "{\"kept\":true}\n");
426        let archive = fx.open();
427        archive.archive(&source).unwrap();
428
429        fs::remove_file(&source).unwrap();
430
431        assert!(archive.archive(&source).is_err());
432        assert_eq!(
433            archive.read(&source).unwrap().as_deref(),
434            Some("{\"kept\":true}\n")
435        );
436        assert_eq!(blobs(&archive).len(), 1);
437        let reopened = fx.open();
438        assert_eq!(
439            reopened.read(&source).unwrap().as_deref(),
440            Some("{\"kept\":true}\n")
441        );
442    }
443
444    #[test]
445    fn an_unknown_source_reads_as_none() {
446        let fx = Fixture::new();
447        let archive = fx.open();
448        let missing = fx.projects.join("never.jsonl");
449        assert!(archive.entry(&missing).is_none());
450        assert_eq!(archive.read(&missing).unwrap(), None);
451    }
452
453    #[test]
454    fn archived_text_is_redacted() {
455        let fx = Fixture::new();
456        let secret = format!("gh{}_{}", "p", "aB3dE6".repeat(6));
457        let source = fx.write(
458            "a.jsonl",
459            &format!("{{\"text\":\"token {secret} and the build passed\"}}\n"),
460        );
461        let archive = fx.open();
462        archive.archive(&source).unwrap();
463
464        let text = archive.read(&source).unwrap().unwrap();
465        assert!(!text.contains(&secret));
466        assert!(text.contains("the build passed"));
467        assert!(text.contains("[REDACTED:github_token]"));
468
469        let entry = archive.entry(&source).unwrap();
470        let raw = fs::read(archive.dir().join(&entry.file)).unwrap();
471        let decoded = String::from_utf8(zstd::decode_all(raw.as_slice()).unwrap()).unwrap();
472        assert!(!decoded.contains(&secret));
473        assert!(decoded.contains("the build passed"));
474    }
475
476    #[test]
477    fn the_blob_is_named_by_the_source_bytes() {
478        let fx = Fixture::new();
479        let secret = format!("gh{}_{}", "p", "aB3dE6".repeat(6));
480        let source = fx.write("a.jsonl", &format!("{secret}\n"));
481        let archive = fx.open();
482        archive.archive(&source).unwrap();
483
484        let changed = format!("gh{}_{}", "p", "zY9xW8".repeat(6));
485        fs::write(&source, format!("{changed}\n")).unwrap();
486        assert_eq!(archive.archive(&source).unwrap(), Archived::Refreshed);
487    }
488
489    #[test]
490    fn reopening_loads_the_manifest() {
491        let fx = Fixture::new();
492        let first = fx.write("a.jsonl", "{\"a\":1}\n");
493        let second = fx.write("b.jsonl", "{\"b\":2}\n");
494        let archive = fx.open();
495        archive.archive(&first).unwrap();
496        archive.archive(&second).unwrap();
497        let entry = archive.entry(&first).unwrap();
498        drop(archive);
499
500        let reopened = fx.open();
501        assert_eq!(reopened.len(), 2);
502        assert_eq!(reopened.entry(&first), Some(entry));
503        assert_eq!(reopened.archive(&first).unwrap(), Archived::Unchanged);
504        assert_eq!(
505            reopened.read(&second).unwrap().as_deref(),
506            Some("{\"b\":2}\n")
507        );
508    }
509
510    #[test]
511    fn a_torn_last_manifest_line_is_ignored() {
512        let fx = Fixture::new();
513        let first = fx.write("a.jsonl", "{\"a\":1}\n");
514        let second = fx.write("b.jsonl", "{\"b\":2}\n");
515        let archive = fx.open();
516        archive.archive(&first).unwrap();
517        archive.archive(&second).unwrap();
518        drop(archive);
519        let mut manifest = fs::OpenOptions::new()
520            .append(true)
521            .open(fx.archive_dir.join(MANIFEST_FILE))
522            .unwrap();
523        manifest.write_all(b"{\"source\":\"/x/c.js").unwrap();
524        drop(manifest);
525
526        let reopened = fx.open();
527        assert_eq!(reopened.len(), 2);
528
529        let third = fx.write("c.jsonl", "{\"c\":3}\n");
530        assert_eq!(reopened.archive(&third).unwrap(), Archived::Stored);
531        drop(reopened);
532        let again = fx.open();
533        assert_eq!(again.len(), 3);
534        assert_eq!(again.read(&third).unwrap().as_deref(), Some("{\"c\":3}\n"));
535    }
536
537    #[test]
538    fn a_failed_manifest_append_is_repaired_by_the_next_write() {
539        use std::os::unix::fs::PermissionsExt;
540
541        let fx = Fixture::new();
542        let first = fx.write("a.jsonl", "{\"a\":1}\n");
543        let second = fx.write("b.jsonl", "{\"b\":2}\n");
544        let third = fx.write("c.jsonl", "{\"c\":3}\n");
545        let archive = fx.open();
546        archive.archive(&first).unwrap();
547        let manifest = fx.archive_dir.join(MANIFEST_FILE);
548        fs::set_permissions(&manifest, fs::Permissions::from_mode(0o400)).unwrap();
549
550        assert!(archive.archive(&second).is_err());
551        assert_eq!(archive.archive(&third).unwrap(), Archived::Stored);
552        drop(archive);
553
554        let reopened = fx.open();
555        assert_eq!(reopened.len(), 3);
556        assert_eq!(
557            reopened.read(&second).unwrap().as_deref(),
558            Some("{\"b\":2}\n")
559        );
560    }
561
562    #[test]
563    fn compaction_keeps_the_latest_entry_per_source() {
564        let fx = Fixture::new();
565        let first = fx.write("a.jsonl", "{\"a\":1}\n");
566        let second = fx.write("b.jsonl", "{\"b\":1}\n");
567        let archive = fx.open();
568        archive.archive(&first).unwrap();
569        archive.archive(&second).unwrap();
570        let latest_second = archive.entry(&second).unwrap();
571        drop(archive);
572
573        let manifest = fx.archive_dir.join(MANIFEST_FILE);
574        let stale = ArchiveEntry {
575            source: first.to_string_lossy().into_owned(),
576            file: "0".repeat(64) + ".jsonl.zst",
577            sha256: "0".repeat(64),
578            archived_at: Utc::now(),
579        };
580        let mut lines = fs::read_to_string(&manifest).unwrap();
581        let mut stale_lines = String::new();
582        for _ in 0..80 {
583            stale_lines.push_str(&serde_json::to_string(&stale).unwrap());
584            stale_lines.push('\n');
585        }
586        lines.insert_str(0, &stale_lines);
587        fs::write(&manifest, &lines).unwrap();
588
589        let reopened = fx.open();
590        let kept = manifest_lines(&reopened);
591        assert_eq!(kept.len(), 2);
592        assert_eq!(reopened.len(), 2);
593        assert_ne!(reopened.entry(&first).unwrap().sha256, stale.sha256);
594        assert_eq!(reopened.entry(&second), Some(latest_second));
595        assert_eq!(
596            reopened.read(&first).unwrap().as_deref(),
597            Some("{\"a\":1}\n")
598        );
599    }
600
601    #[test]
602    fn a_long_running_archive_keeps_its_manifest_compact() {
603        let fx = Fixture::new();
604        let source = fx.write("a.jsonl", "");
605        let archive = fx.open();
606        let mut text = String::new();
607        for i in 0..100 {
608            text.push_str(&format!("{{\"line\":{i}}}\n"));
609            fs::write(&source, &text).unwrap();
610            archive.archive(&source).unwrap();
611        }
612
613        assert!(manifest_lines(&archive).len() <= 2 + 64);
614        assert_eq!(blobs(&archive).len(), 1);
615        drop(archive);
616        let reopened = fx.open();
617        assert_eq!(reopened.read(&source).unwrap(), Some(text));
618    }
619}