1use std::collections::HashMap;
2use std::fs::{self, File, OpenOptions};
3use std::io::{ErrorKind, Write};
4use std::path::{Path, PathBuf};
5use std::sync::{Mutex, MutexGuard, PoisonError};
6
7use anyhow::{Context, Result};
8use chrono::{DateTime, Utc};
9use serde::{Deserialize, Serialize};
10use sha2::{Digest, Sha256};
11
12use crate::redact::redact;
13
14pub const MANIFEST_FILE: &str = "manifest.jsonl";
15
16const BLOB_SUFFIX: &str = ".jsonl.zst";
17const COMPACTION_SLACK: usize = 64;
18
19#[derive(Debug, Clone, Copy, PartialEq, Eq)]
20pub enum Archived {
21 Stored,
22 Refreshed,
23 Unchanged,
24 Copy,
25}
26
27#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
28pub struct ArchiveEntry {
29 pub source: String,
30 pub file: String,
31 pub sha256: String,
32 pub archived_at: DateTime<Utc>,
33}
34
35pub struct TranscriptArchive {
38 dir: PathBuf,
39 state: Mutex<State>,
40}
41
42struct State {
43 entries: HashMap<String, ArchiveEntry>,
44 manifest_lines: usize,
45 manifest_damaged: bool,
46}
47
48impl State {
49 fn needs_compaction(&self) -> bool {
50 self.manifest_damaged || self.manifest_lines > 2 * self.entries.len() + COMPACTION_SLACK
51 }
52
53 fn holder_of(&self, sha256: &str) -> Option<&ArchiveEntry> {
54 self.entries.values().find(|e| e.sha256 == sha256)
55 }
56
57 fn references(&self, file: &str) -> bool {
58 self.entries.values().any(|e| e.file == file)
59 }
60}
61
62impl TranscriptArchive {
63 pub fn open(dir: impl Into<PathBuf>) -> Result<Self> {
64 let dir = dir.into();
65 fs::create_dir_all(&dir)
66 .with_context(|| format!("creating the transcript archive at {}", dir.display()))?;
67 let manifest = dir.join(MANIFEST_FILE);
68 if !manifest.exists() {
69 write_atomic(&manifest, b"")
70 .with_context(|| format!("creating {}", manifest.display()))?;
71 }
72 let (state, torn) = load_manifest(&manifest)?;
73 let archive = Self {
74 dir,
75 state: Mutex::new(state),
76 };
77 {
78 let mut state = archive.state();
79 if torn || state.needs_compaction() {
80 archive.compact(&mut state)?;
81 }
82 }
83 Ok(archive)
84 }
85
86 pub fn dir(&self) -> &Path {
87 &self.dir
88 }
89
90 pub fn archive(&self, source: &Path) -> Result<Archived> {
91 let mut state = self.state();
92 let bytes = fs::read(source).with_context(|| format!("reading {}", source.display()))?;
93 let sha256 = hex(&Sha256::digest(&bytes));
94 let key = source_key(source);
95
96 let previous = state.entries.get(&key).cloned();
97 if let Some(prev) = &previous
98 && prev.sha256 == sha256
99 && self.dir.join(&prev.file).exists()
100 {
101 return Ok(Archived::Unchanged);
102 }
103
104 let file = format!("{sha256}{BLOB_SUFFIX}");
105 let shared = state
106 .holder_of(&sha256)
107 .is_some_and(|holder| self.dir.join(&holder.file).exists());
108 if !shared {
109 self.write_blob(&file, &bytes)?;
110 }
111 let entry = ArchiveEntry {
112 source: key.clone(),
113 file,
114 sha256,
115 archived_at: Utc::now(),
116 };
117 state.entries.insert(key, entry.clone());
118 if state.needs_compaction() {
119 self.compact(&mut state)?;
120 } else if let Err(e) = self.append(&mut state, &entry) {
121 state.manifest_damaged = true;
122 return Err(e);
123 }
124
125 if let Some(prev) = &previous
126 && !state.references(&prev.file)
127 {
128 self.remove_blob(&prev.file);
129 }
130
131 Ok(match (shared, previous) {
132 (true, _) => Archived::Copy,
133 (false, None) => Archived::Stored,
134 (false, Some(_)) => Archived::Refreshed,
135 })
136 }
137
138 pub fn entry(&self, source: &Path) -> Option<ArchiveEntry> {
139 self.state().entries.get(&source_key(source)).cloned()
140 }
141
142 pub fn read(&self, source: &Path) -> Result<Option<String>> {
143 let Some(entry) = self.entry(source) else {
144 return Ok(None);
145 };
146 let path = self.dir.join(&entry.file);
147 let compressed = fs::read(&path).with_context(|| format!("reading {}", path.display()))?;
148 let bytes = zstd::decode_all(compressed.as_slice())
149 .with_context(|| format!("decompressing {}", path.display()))?;
150 let text =
151 String::from_utf8(bytes).with_context(|| format!("decoding {}", path.display()))?;
152 Ok(Some(text))
153 }
154
155 pub fn entries(&self) -> Vec<ArchiveEntry> {
157 self.state().entries.values().cloned().collect()
158 }
159
160 pub fn len(&self) -> usize {
161 self.state().entries.len()
162 }
163
164 pub fn is_empty(&self) -> bool {
165 self.len() == 0
166 }
167
168 fn state(&self) -> MutexGuard<'_, State> {
169 self.state.lock().unwrap_or_else(PoisonError::into_inner)
170 }
171
172 fn write_blob(&self, file: &str, source_bytes: &[u8]) -> Result<()> {
173 let text = redact(&String::from_utf8_lossy(source_bytes)).text;
174 let compressed =
175 zstd::encode_all(text.as_bytes(), 0).context("compressing a transcript")?;
176 let target = self.dir.join(file);
177 write_atomic(&target, &compressed).with_context(|| format!("writing {}", target.display()))
178 }
179
180 fn remove_blob(&self, file: &str) {
181 let path = self.dir.join(file);
182 match fs::remove_file(&path) {
183 Ok(()) => {}
184 Err(e) if e.kind() == ErrorKind::NotFound => {}
185 Err(e) => {
186 tracing::warn!(path = %path.display(), error = %e, "removing an unused archive blob")
187 }
188 }
189 }
190
191 fn append(&self, state: &mut State, entry: &ArchiveEntry) -> Result<()> {
192 let path = self.dir.join(MANIFEST_FILE);
193 let mut line = serde_json::to_string(entry)?;
194 line.push('\n');
195 let mut file = OpenOptions::new()
196 .create(true)
197 .append(true)
198 .open(&path)
199 .with_context(|| format!("opening {}", path.display()))?;
200 file.write_all(line.as_bytes())
201 .and_then(|()| file.sync_data())
202 .with_context(|| format!("appending to {}", path.display()))?;
203 state.manifest_lines += 1;
204 Ok(())
205 }
206
207 fn compact(&self, state: &mut State) -> Result<()> {
208 let mut entries: Vec<&ArchiveEntry> = state.entries.values().collect();
209 entries.sort_by(|a, b| a.source.cmp(&b.source));
210 let mut body = String::new();
211 for entry in &entries {
212 body.push_str(&serde_json::to_string(entry)?);
213 body.push('\n');
214 }
215 let path = self.dir.join(MANIFEST_FILE);
216 write_atomic(&path, body.as_bytes())
217 .with_context(|| format!("compacting {}", path.display()))?;
218 state.manifest_lines = entries.len();
219 state.manifest_damaged = false;
220 Ok(())
221 }
222}
223
224fn load_manifest(path: &Path) -> Result<(State, bool)> {
227 let text = match fs::read_to_string(path) {
228 Ok(text) => text,
229 Err(e) if e.kind() == ErrorKind::NotFound => String::new(),
230 Err(e) => return Err(e).with_context(|| format!("reading {}", path.display())),
231 };
232 let mut state = State {
233 entries: HashMap::new(),
234 manifest_lines: 0,
235 manifest_damaged: false,
236 };
237 let mut torn = !text.is_empty() && !text.ends_with('\n');
238 for line in text.lines().filter(|l| !l.trim().is_empty()) {
239 state.manifest_lines += 1;
240 match serde_json::from_str::<ArchiveEntry>(line) {
241 Ok(entry) => {
242 state.entries.insert(entry.source.clone(), entry);
243 }
244 Err(e) => {
245 torn = true;
246 tracing::warn!(path = %path.display(), error = %e, "ignoring an unreadable archive manifest line");
247 }
248 }
249 }
250 Ok((state, torn))
251}
252
253fn write_atomic(target: &Path, bytes: &[u8]) -> std::io::Result<()> {
254 let mut tmp_name = target.file_name().unwrap_or_default().to_os_string();
255 tmp_name.push(".tmp");
256 let tmp = target.with_file_name(tmp_name);
257 let mut file = File::create(&tmp)?;
258 file.write_all(bytes)?;
259 file.sync_all()?;
260 fs::rename(&tmp, target)?;
261 match target.parent() {
262 Some(dir) => File::open(dir)?.sync_all(),
263 None => Ok(()),
264 }
265}
266
267fn source_key(source: &Path) -> String {
268 source.to_string_lossy().into_owned()
269}
270
271fn hex(bytes: &[u8]) -> String {
272 bytes.iter().map(|b| format!("{b:02x}")).collect()
273}
274
275#[cfg(test)]
276mod tests {
277 use std::fs;
278 use std::io::Write;
279
280 use super::*;
281
282 fn blobs(archive: &TranscriptArchive) -> Vec<String> {
283 let mut names: Vec<String> = fs::read_dir(archive.dir())
284 .unwrap()
285 .map(|e| e.unwrap().file_name().to_string_lossy().into_owned())
286 .filter(|n| n.ends_with(".jsonl.zst"))
287 .collect();
288 names.sort();
289 names
290 }
291
292 fn manifest_lines(archive: &TranscriptArchive) -> Vec<String> {
293 fs::read_to_string(archive.dir().join(MANIFEST_FILE))
294 .unwrap()
295 .lines()
296 .map(str::to_string)
297 .collect()
298 }
299
300 struct Fixture {
301 _tmp: tempfile::TempDir,
302 projects: PathBuf,
303 archive_dir: PathBuf,
304 }
305
306 impl Fixture {
307 fn new() -> Self {
308 let tmp = tempfile::tempdir().unwrap();
309 let projects = tmp.path().join("projects");
310 fs::create_dir_all(&projects).unwrap();
311 let archive_dir = tmp.path().join("data").join("transcripts");
312 Self {
313 _tmp: tmp,
314 projects,
315 archive_dir,
316 }
317 }
318
319 fn write(&self, name: &str, text: &str) -> PathBuf {
320 let path = self.projects.join(name);
321 fs::write(&path, text).unwrap();
322 path
323 }
324
325 fn open(&self) -> TranscriptArchive {
326 TranscriptArchive::open(&self.archive_dir).unwrap()
327 }
328 }
329
330 #[test]
331 fn open_creates_the_directory() {
332 let fx = Fixture::new();
333 let archive = fx.open();
334 assert!(archive.dir().is_dir());
335 assert!(archive.is_empty());
336 }
337
338 #[test]
339 fn a_new_source_is_stored_then_unchanged() {
340 let fx = Fixture::new();
341 let source = fx.write("a.jsonl", "{\"line\":1}\n");
342 let archive = fx.open();
343
344 assert_eq!(archive.archive(&source).unwrap(), Archived::Stored);
345 assert_eq!(archive.archive(&source).unwrap(), Archived::Unchanged);
346
347 assert_eq!(archive.len(), 1);
348 assert_eq!(blobs(&archive).len(), 1);
349 assert_eq!(manifest_lines(&archive).len(), 1);
350 let entry = archive.entry(&source).unwrap();
351 assert_eq!(entry.source, source.to_string_lossy());
352 assert_eq!(entry.file, format!("{}.jsonl.zst", entry.sha256));
353 assert_eq!(entry.sha256.len(), 64);
354 assert_eq!(
355 archive.read(&source).unwrap().as_deref(),
356 Some("{\"line\":1}\n")
357 );
358 }
359
360 #[test]
361 fn a_changed_source_is_refreshed_and_its_old_blob_removed() {
362 let fx = Fixture::new();
363 let source = fx.write("a.jsonl", "{\"line\":1}\n");
364 let archive = fx.open();
365 archive.archive(&source).unwrap();
366 let old = archive.entry(&source).unwrap();
367
368 fs::write(&source, "{\"line\":1}\n{\"line\":2}\n").unwrap();
369 assert_eq!(archive.archive(&source).unwrap(), Archived::Refreshed);
370
371 let new = archive.entry(&source).unwrap();
372 assert_ne!(new.sha256, old.sha256);
373 assert_eq!(
374 archive.read(&source).unwrap().as_deref(),
375 Some("{\"line\":1}\n{\"line\":2}\n")
376 );
377 assert_eq!(blobs(&archive), vec![new.file]);
378 assert!(!archive.dir().join(&old.file).exists());
379 }
380
381 #[test]
382 fn a_copy_at_another_path_shares_the_blob() {
383 let fx = Fixture::new();
384 let first = fx.write("a.jsonl", "{\"same\":true}\n");
385 let second = fx.write("b.jsonl", "{\"same\":true}\n");
386 let archive = fx.open();
387
388 assert_eq!(archive.archive(&first).unwrap(), Archived::Stored);
389 assert_eq!(archive.archive(&second).unwrap(), Archived::Copy);
390
391 assert_eq!(archive.len(), 2);
392 assert_eq!(blobs(&archive).len(), 1);
393 assert_eq!(
394 archive.entry(&first).unwrap().file,
395 archive.entry(&second).unwrap().file
396 );
397 assert_eq!(
398 archive.read(&second).unwrap().as_deref(),
399 Some("{\"same\":true}\n")
400 );
401 }
402
403 #[test]
404 fn refreshing_one_copy_keeps_the_blob_the_other_still_uses() {
405 let fx = Fixture::new();
406 let first = fx.write("a.jsonl", "{\"same\":true}\n");
407 let second = fx.write("b.jsonl", "{\"same\":true}\n");
408 let archive = fx.open();
409 archive.archive(&first).unwrap();
410 archive.archive(&second).unwrap();
411
412 fs::write(&first, "{\"same\":false}\n").unwrap();
413 assert_eq!(archive.archive(&first).unwrap(), Archived::Refreshed);
414
415 assert_eq!(blobs(&archive).len(), 2);
416 assert_eq!(
417 archive.read(&second).unwrap().as_deref(),
418 Some("{\"same\":true}\n")
419 );
420 }
421
422 #[test]
423 fn the_archive_outlives_a_deleted_source() {
424 let fx = Fixture::new();
425 let source = fx.write("a.jsonl", "{\"kept\":true}\n");
426 let archive = fx.open();
427 archive.archive(&source).unwrap();
428
429 fs::remove_file(&source).unwrap();
430
431 assert!(archive.archive(&source).is_err());
432 assert_eq!(
433 archive.read(&source).unwrap().as_deref(),
434 Some("{\"kept\":true}\n")
435 );
436 assert_eq!(blobs(&archive).len(), 1);
437 let reopened = fx.open();
438 assert_eq!(
439 reopened.read(&source).unwrap().as_deref(),
440 Some("{\"kept\":true}\n")
441 );
442 }
443
444 #[test]
445 fn an_unknown_source_reads_as_none() {
446 let fx = Fixture::new();
447 let archive = fx.open();
448 let missing = fx.projects.join("never.jsonl");
449 assert!(archive.entry(&missing).is_none());
450 assert_eq!(archive.read(&missing).unwrap(), None);
451 }
452
453 #[test]
454 fn archived_text_is_redacted() {
455 let fx = Fixture::new();
456 let secret = format!("gh{}_{}", "p", "aB3dE6".repeat(6));
457 let source = fx.write(
458 "a.jsonl",
459 &format!("{{\"text\":\"token {secret} and the build passed\"}}\n"),
460 );
461 let archive = fx.open();
462 archive.archive(&source).unwrap();
463
464 let text = archive.read(&source).unwrap().unwrap();
465 assert!(!text.contains(&secret));
466 assert!(text.contains("the build passed"));
467 assert!(text.contains("[REDACTED:github_token]"));
468
469 let entry = archive.entry(&source).unwrap();
470 let raw = fs::read(archive.dir().join(&entry.file)).unwrap();
471 let decoded = String::from_utf8(zstd::decode_all(raw.as_slice()).unwrap()).unwrap();
472 assert!(!decoded.contains(&secret));
473 assert!(decoded.contains("the build passed"));
474 }
475
476 #[test]
477 fn the_blob_is_named_by_the_source_bytes() {
478 let fx = Fixture::new();
479 let secret = format!("gh{}_{}", "p", "aB3dE6".repeat(6));
480 let source = fx.write("a.jsonl", &format!("{secret}\n"));
481 let archive = fx.open();
482 archive.archive(&source).unwrap();
483
484 let changed = format!("gh{}_{}", "p", "zY9xW8".repeat(6));
485 fs::write(&source, format!("{changed}\n")).unwrap();
486 assert_eq!(archive.archive(&source).unwrap(), Archived::Refreshed);
487 }
488
489 #[test]
490 fn reopening_loads_the_manifest() {
491 let fx = Fixture::new();
492 let first = fx.write("a.jsonl", "{\"a\":1}\n");
493 let second = fx.write("b.jsonl", "{\"b\":2}\n");
494 let archive = fx.open();
495 archive.archive(&first).unwrap();
496 archive.archive(&second).unwrap();
497 let entry = archive.entry(&first).unwrap();
498 drop(archive);
499
500 let reopened = fx.open();
501 assert_eq!(reopened.len(), 2);
502 assert_eq!(reopened.entry(&first), Some(entry));
503 assert_eq!(reopened.archive(&first).unwrap(), Archived::Unchanged);
504 assert_eq!(
505 reopened.read(&second).unwrap().as_deref(),
506 Some("{\"b\":2}\n")
507 );
508 }
509
510 #[test]
511 fn a_torn_last_manifest_line_is_ignored() {
512 let fx = Fixture::new();
513 let first = fx.write("a.jsonl", "{\"a\":1}\n");
514 let second = fx.write("b.jsonl", "{\"b\":2}\n");
515 let archive = fx.open();
516 archive.archive(&first).unwrap();
517 archive.archive(&second).unwrap();
518 drop(archive);
519 let mut manifest = fs::OpenOptions::new()
520 .append(true)
521 .open(fx.archive_dir.join(MANIFEST_FILE))
522 .unwrap();
523 manifest.write_all(b"{\"source\":\"/x/c.js").unwrap();
524 drop(manifest);
525
526 let reopened = fx.open();
527 assert_eq!(reopened.len(), 2);
528
529 let third = fx.write("c.jsonl", "{\"c\":3}\n");
530 assert_eq!(reopened.archive(&third).unwrap(), Archived::Stored);
531 drop(reopened);
532 let again = fx.open();
533 assert_eq!(again.len(), 3);
534 assert_eq!(again.read(&third).unwrap().as_deref(), Some("{\"c\":3}\n"));
535 }
536
537 #[test]
538 fn a_failed_manifest_append_is_repaired_by_the_next_write() {
539 use std::os::unix::fs::PermissionsExt;
540
541 let fx = Fixture::new();
542 let first = fx.write("a.jsonl", "{\"a\":1}\n");
543 let second = fx.write("b.jsonl", "{\"b\":2}\n");
544 let third = fx.write("c.jsonl", "{\"c\":3}\n");
545 let archive = fx.open();
546 archive.archive(&first).unwrap();
547 let manifest = fx.archive_dir.join(MANIFEST_FILE);
548 fs::set_permissions(&manifest, fs::Permissions::from_mode(0o400)).unwrap();
549
550 assert!(archive.archive(&second).is_err());
551 assert_eq!(archive.archive(&third).unwrap(), Archived::Stored);
552 drop(archive);
553
554 let reopened = fx.open();
555 assert_eq!(reopened.len(), 3);
556 assert_eq!(
557 reopened.read(&second).unwrap().as_deref(),
558 Some("{\"b\":2}\n")
559 );
560 }
561
562 #[test]
563 fn compaction_keeps_the_latest_entry_per_source() {
564 let fx = Fixture::new();
565 let first = fx.write("a.jsonl", "{\"a\":1}\n");
566 let second = fx.write("b.jsonl", "{\"b\":1}\n");
567 let archive = fx.open();
568 archive.archive(&first).unwrap();
569 archive.archive(&second).unwrap();
570 let latest_second = archive.entry(&second).unwrap();
571 drop(archive);
572
573 let manifest = fx.archive_dir.join(MANIFEST_FILE);
574 let stale = ArchiveEntry {
575 source: first.to_string_lossy().into_owned(),
576 file: "0".repeat(64) + ".jsonl.zst",
577 sha256: "0".repeat(64),
578 archived_at: Utc::now(),
579 };
580 let mut lines = fs::read_to_string(&manifest).unwrap();
581 let mut stale_lines = String::new();
582 for _ in 0..80 {
583 stale_lines.push_str(&serde_json::to_string(&stale).unwrap());
584 stale_lines.push('\n');
585 }
586 lines.insert_str(0, &stale_lines);
587 fs::write(&manifest, &lines).unwrap();
588
589 let reopened = fx.open();
590 let kept = manifest_lines(&reopened);
591 assert_eq!(kept.len(), 2);
592 assert_eq!(reopened.len(), 2);
593 assert_ne!(reopened.entry(&first).unwrap().sha256, stale.sha256);
594 assert_eq!(reopened.entry(&second), Some(latest_second));
595 assert_eq!(
596 reopened.read(&first).unwrap().as_deref(),
597 Some("{\"a\":1}\n")
598 );
599 }
600
601 #[test]
602 fn a_long_running_archive_keeps_its_manifest_compact() {
603 let fx = Fixture::new();
604 let source = fx.write("a.jsonl", "");
605 let archive = fx.open();
606 let mut text = String::new();
607 for i in 0..100 {
608 text.push_str(&format!("{{\"line\":{i}}}\n"));
609 fs::write(&source, &text).unwrap();
610 archive.archive(&source).unwrap();
611 }
612
613 assert!(manifest_lines(&archive).len() <= 2 + 64);
614 assert_eq!(blobs(&archive).len(), 1);
615 drop(archive);
616 let reopened = fx.open();
617 assert_eq!(reopened.read(&source).unwrap(), Some(text));
618 }
619}