use std::collections::HashMap;
use std::fs::{self, File, OpenOptions};
use std::io::{ErrorKind, Write};
use std::path::{Path, PathBuf};
use std::sync::{Mutex, MutexGuard, PoisonError};
use anyhow::{Context, Result};
use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use crate::redact::redact;
pub const MANIFEST_FILE: &str = "manifest.jsonl";
const BLOB_SUFFIX: &str = ".jsonl.zst";
const COMPACTION_SLACK: usize = 64;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Archived {
Stored,
Refreshed,
Unchanged,
Copy,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ArchiveEntry {
pub source: String,
pub file: String,
pub sha256: String,
pub archived_at: DateTime<Utc>,
}
pub struct TranscriptArchive {
dir: PathBuf,
state: Mutex<State>,
}
struct State {
entries: HashMap<String, ArchiveEntry>,
manifest_lines: usize,
manifest_damaged: bool,
}
impl State {
fn needs_compaction(&self) -> bool {
self.manifest_damaged || self.manifest_lines > 2 * self.entries.len() + COMPACTION_SLACK
}
fn holder_of(&self, sha256: &str) -> Option<&ArchiveEntry> {
self.entries.values().find(|e| e.sha256 == sha256)
}
fn references(&self, file: &str) -> bool {
self.entries.values().any(|e| e.file == file)
}
}
impl TranscriptArchive {
pub fn open(dir: impl Into<PathBuf>) -> Result<Self> {
let dir = dir.into();
fs::create_dir_all(&dir)
.with_context(|| format!("creating the transcript archive at {}", dir.display()))?;
let manifest = dir.join(MANIFEST_FILE);
if !manifest.exists() {
write_atomic(&manifest, b"")
.with_context(|| format!("creating {}", manifest.display()))?;
}
let (state, torn) = load_manifest(&manifest)?;
let archive = Self {
dir,
state: Mutex::new(state),
};
{
let mut state = archive.state();
if torn || state.needs_compaction() {
archive.compact(&mut state)?;
}
}
Ok(archive)
}
pub fn dir(&self) -> &Path {
&self.dir
}
pub fn archive(&self, source: &Path) -> Result<Archived> {
let mut state = self.state();
let bytes = fs::read(source).with_context(|| format!("reading {}", source.display()))?;
let sha256 = hex(&Sha256::digest(&bytes));
let key = source_key(source);
let previous = state.entries.get(&key).cloned();
if let Some(prev) = &previous
&& prev.sha256 == sha256
&& self.dir.join(&prev.file).exists()
{
return Ok(Archived::Unchanged);
}
let file = format!("{sha256}{BLOB_SUFFIX}");
let shared = state
.holder_of(&sha256)
.is_some_and(|holder| self.dir.join(&holder.file).exists());
if !shared {
self.write_blob(&file, &bytes)?;
}
let entry = ArchiveEntry {
source: key.clone(),
file,
sha256,
archived_at: Utc::now(),
};
state.entries.insert(key, entry.clone());
if state.needs_compaction() {
self.compact(&mut state)?;
} else if let Err(e) = self.append(&mut state, &entry) {
state.manifest_damaged = true;
return Err(e);
}
if let Some(prev) = &previous
&& !state.references(&prev.file)
{
self.remove_blob(&prev.file);
}
Ok(match (shared, previous) {
(true, _) => Archived::Copy,
(false, None) => Archived::Stored,
(false, Some(_)) => Archived::Refreshed,
})
}
pub fn entry(&self, source: &Path) -> Option<ArchiveEntry> {
self.state().entries.get(&source_key(source)).cloned()
}
pub fn read(&self, source: &Path) -> Result<Option<String>> {
let Some(entry) = self.entry(source) else {
return Ok(None);
};
let path = self.dir.join(&entry.file);
let compressed = fs::read(&path).with_context(|| format!("reading {}", path.display()))?;
let bytes = zstd::decode_all(compressed.as_slice())
.with_context(|| format!("decompressing {}", path.display()))?;
let text =
String::from_utf8(bytes).with_context(|| format!("decoding {}", path.display()))?;
Ok(Some(text))
}
pub fn entries(&self) -> Vec<ArchiveEntry> {
self.state().entries.values().cloned().collect()
}
pub fn len(&self) -> usize {
self.state().entries.len()
}
pub fn is_empty(&self) -> bool {
self.len() == 0
}
fn state(&self) -> MutexGuard<'_, State> {
self.state.lock().unwrap_or_else(PoisonError::into_inner)
}
fn write_blob(&self, file: &str, source_bytes: &[u8]) -> Result<()> {
let text = redact(&String::from_utf8_lossy(source_bytes)).text;
let compressed =
zstd::encode_all(text.as_bytes(), 0).context("compressing a transcript")?;
let target = self.dir.join(file);
write_atomic(&target, &compressed).with_context(|| format!("writing {}", target.display()))
}
fn remove_blob(&self, file: &str) {
let path = self.dir.join(file);
match fs::remove_file(&path) {
Ok(()) => {}
Err(e) if e.kind() == ErrorKind::NotFound => {}
Err(e) => {
tracing::warn!(path = %path.display(), error = %e, "removing an unused archive blob")
}
}
}
fn append(&self, state: &mut State, entry: &ArchiveEntry) -> Result<()> {
let path = self.dir.join(MANIFEST_FILE);
let mut line = serde_json::to_string(entry)?;
line.push('\n');
let mut file = OpenOptions::new()
.create(true)
.append(true)
.open(&path)
.with_context(|| format!("opening {}", path.display()))?;
file.write_all(line.as_bytes())
.and_then(|()| file.sync_data())
.with_context(|| format!("appending to {}", path.display()))?;
state.manifest_lines += 1;
Ok(())
}
fn compact(&self, state: &mut State) -> Result<()> {
let mut entries: Vec<&ArchiveEntry> = state.entries.values().collect();
entries.sort_by(|a, b| a.source.cmp(&b.source));
let mut body = String::new();
for entry in &entries {
body.push_str(&serde_json::to_string(entry)?);
body.push('\n');
}
let path = self.dir.join(MANIFEST_FILE);
write_atomic(&path, body.as_bytes())
.with_context(|| format!("compacting {}", path.display()))?;
state.manifest_lines = entries.len();
state.manifest_damaged = false;
Ok(())
}
}
fn load_manifest(path: &Path) -> Result<(State, bool)> {
let text = match fs::read_to_string(path) {
Ok(text) => text,
Err(e) if e.kind() == ErrorKind::NotFound => String::new(),
Err(e) => return Err(e).with_context(|| format!("reading {}", path.display())),
};
let mut state = State {
entries: HashMap::new(),
manifest_lines: 0,
manifest_damaged: false,
};
let mut torn = !text.is_empty() && !text.ends_with('\n');
for line in text.lines().filter(|l| !l.trim().is_empty()) {
state.manifest_lines += 1;
match serde_json::from_str::<ArchiveEntry>(line) {
Ok(entry) => {
state.entries.insert(entry.source.clone(), entry);
}
Err(e) => {
torn = true;
tracing::warn!(path = %path.display(), error = %e, "ignoring an unreadable archive manifest line");
}
}
}
Ok((state, torn))
}
fn write_atomic(target: &Path, bytes: &[u8]) -> std::io::Result<()> {
let mut tmp_name = target.file_name().unwrap_or_default().to_os_string();
tmp_name.push(".tmp");
let tmp = target.with_file_name(tmp_name);
let mut file = File::create(&tmp)?;
file.write_all(bytes)?;
file.sync_all()?;
fs::rename(&tmp, target)?;
match target.parent() {
Some(dir) => File::open(dir)?.sync_all(),
None => Ok(()),
}
}
fn source_key(source: &Path) -> String {
source.to_string_lossy().into_owned()
}
fn hex(bytes: &[u8]) -> String {
bytes.iter().map(|b| format!("{b:02x}")).collect()
}
#[cfg(test)]
mod tests {
use std::fs;
use std::io::Write;
use super::*;
fn blobs(archive: &TranscriptArchive) -> Vec<String> {
let mut names: Vec<String> = fs::read_dir(archive.dir())
.unwrap()
.map(|e| e.unwrap().file_name().to_string_lossy().into_owned())
.filter(|n| n.ends_with(".jsonl.zst"))
.collect();
names.sort();
names
}
fn manifest_lines(archive: &TranscriptArchive) -> Vec<String> {
fs::read_to_string(archive.dir().join(MANIFEST_FILE))
.unwrap()
.lines()
.map(str::to_string)
.collect()
}
struct Fixture {
_tmp: tempfile::TempDir,
projects: PathBuf,
archive_dir: PathBuf,
}
impl Fixture {
fn new() -> Self {
let tmp = tempfile::tempdir().unwrap();
let projects = tmp.path().join("projects");
fs::create_dir_all(&projects).unwrap();
let archive_dir = tmp.path().join("data").join("transcripts");
Self {
_tmp: tmp,
projects,
archive_dir,
}
}
fn write(&self, name: &str, text: &str) -> PathBuf {
let path = self.projects.join(name);
fs::write(&path, text).unwrap();
path
}
fn open(&self) -> TranscriptArchive {
TranscriptArchive::open(&self.archive_dir).unwrap()
}
}
#[test]
fn open_creates_the_directory() {
let fx = Fixture::new();
let archive = fx.open();
assert!(archive.dir().is_dir());
assert!(archive.is_empty());
}
#[test]
fn a_new_source_is_stored_then_unchanged() {
let fx = Fixture::new();
let source = fx.write("a.jsonl", "{\"line\":1}\n");
let archive = fx.open();
assert_eq!(archive.archive(&source).unwrap(), Archived::Stored);
assert_eq!(archive.archive(&source).unwrap(), Archived::Unchanged);
assert_eq!(archive.len(), 1);
assert_eq!(blobs(&archive).len(), 1);
assert_eq!(manifest_lines(&archive).len(), 1);
let entry = archive.entry(&source).unwrap();
assert_eq!(entry.source, source.to_string_lossy());
assert_eq!(entry.file, format!("{}.jsonl.zst", entry.sha256));
assert_eq!(entry.sha256.len(), 64);
assert_eq!(
archive.read(&source).unwrap().as_deref(),
Some("{\"line\":1}\n")
);
}
#[test]
fn a_changed_source_is_refreshed_and_its_old_blob_removed() {
let fx = Fixture::new();
let source = fx.write("a.jsonl", "{\"line\":1}\n");
let archive = fx.open();
archive.archive(&source).unwrap();
let old = archive.entry(&source).unwrap();
fs::write(&source, "{\"line\":1}\n{\"line\":2}\n").unwrap();
assert_eq!(archive.archive(&source).unwrap(), Archived::Refreshed);
let new = archive.entry(&source).unwrap();
assert_ne!(new.sha256, old.sha256);
assert_eq!(
archive.read(&source).unwrap().as_deref(),
Some("{\"line\":1}\n{\"line\":2}\n")
);
assert_eq!(blobs(&archive), vec![new.file]);
assert!(!archive.dir().join(&old.file).exists());
}
#[test]
fn a_copy_at_another_path_shares_the_blob() {
let fx = Fixture::new();
let first = fx.write("a.jsonl", "{\"same\":true}\n");
let second = fx.write("b.jsonl", "{\"same\":true}\n");
let archive = fx.open();
assert_eq!(archive.archive(&first).unwrap(), Archived::Stored);
assert_eq!(archive.archive(&second).unwrap(), Archived::Copy);
assert_eq!(archive.len(), 2);
assert_eq!(blobs(&archive).len(), 1);
assert_eq!(
archive.entry(&first).unwrap().file,
archive.entry(&second).unwrap().file
);
assert_eq!(
archive.read(&second).unwrap().as_deref(),
Some("{\"same\":true}\n")
);
}
#[test]
fn refreshing_one_copy_keeps_the_blob_the_other_still_uses() {
let fx = Fixture::new();
let first = fx.write("a.jsonl", "{\"same\":true}\n");
let second = fx.write("b.jsonl", "{\"same\":true}\n");
let archive = fx.open();
archive.archive(&first).unwrap();
archive.archive(&second).unwrap();
fs::write(&first, "{\"same\":false}\n").unwrap();
assert_eq!(archive.archive(&first).unwrap(), Archived::Refreshed);
assert_eq!(blobs(&archive).len(), 2);
assert_eq!(
archive.read(&second).unwrap().as_deref(),
Some("{\"same\":true}\n")
);
}
#[test]
fn the_archive_outlives_a_deleted_source() {
let fx = Fixture::new();
let source = fx.write("a.jsonl", "{\"kept\":true}\n");
let archive = fx.open();
archive.archive(&source).unwrap();
fs::remove_file(&source).unwrap();
assert!(archive.archive(&source).is_err());
assert_eq!(
archive.read(&source).unwrap().as_deref(),
Some("{\"kept\":true}\n")
);
assert_eq!(blobs(&archive).len(), 1);
let reopened = fx.open();
assert_eq!(
reopened.read(&source).unwrap().as_deref(),
Some("{\"kept\":true}\n")
);
}
#[test]
fn an_unknown_source_reads_as_none() {
let fx = Fixture::new();
let archive = fx.open();
let missing = fx.projects.join("never.jsonl");
assert!(archive.entry(&missing).is_none());
assert_eq!(archive.read(&missing).unwrap(), None);
}
#[test]
fn archived_text_is_redacted() {
let fx = Fixture::new();
let secret = format!("gh{}_{}", "p", "aB3dE6".repeat(6));
let source = fx.write(
"a.jsonl",
&format!("{{\"text\":\"token {secret} and the build passed\"}}\n"),
);
let archive = fx.open();
archive.archive(&source).unwrap();
let text = archive.read(&source).unwrap().unwrap();
assert!(!text.contains(&secret));
assert!(text.contains("the build passed"));
assert!(text.contains("[REDACTED:github_token]"));
let entry = archive.entry(&source).unwrap();
let raw = fs::read(archive.dir().join(&entry.file)).unwrap();
let decoded = String::from_utf8(zstd::decode_all(raw.as_slice()).unwrap()).unwrap();
assert!(!decoded.contains(&secret));
assert!(decoded.contains("the build passed"));
}
#[test]
fn the_blob_is_named_by_the_source_bytes() {
let fx = Fixture::new();
let secret = format!("gh{}_{}", "p", "aB3dE6".repeat(6));
let source = fx.write("a.jsonl", &format!("{secret}\n"));
let archive = fx.open();
archive.archive(&source).unwrap();
let changed = format!("gh{}_{}", "p", "zY9xW8".repeat(6));
fs::write(&source, format!("{changed}\n")).unwrap();
assert_eq!(archive.archive(&source).unwrap(), Archived::Refreshed);
}
#[test]
fn reopening_loads_the_manifest() {
let fx = Fixture::new();
let first = fx.write("a.jsonl", "{\"a\":1}\n");
let second = fx.write("b.jsonl", "{\"b\":2}\n");
let archive = fx.open();
archive.archive(&first).unwrap();
archive.archive(&second).unwrap();
let entry = archive.entry(&first).unwrap();
drop(archive);
let reopened = fx.open();
assert_eq!(reopened.len(), 2);
assert_eq!(reopened.entry(&first), Some(entry));
assert_eq!(reopened.archive(&first).unwrap(), Archived::Unchanged);
assert_eq!(
reopened.read(&second).unwrap().as_deref(),
Some("{\"b\":2}\n")
);
}
#[test]
fn a_torn_last_manifest_line_is_ignored() {
let fx = Fixture::new();
let first = fx.write("a.jsonl", "{\"a\":1}\n");
let second = fx.write("b.jsonl", "{\"b\":2}\n");
let archive = fx.open();
archive.archive(&first).unwrap();
archive.archive(&second).unwrap();
drop(archive);
let mut manifest = fs::OpenOptions::new()
.append(true)
.open(fx.archive_dir.join(MANIFEST_FILE))
.unwrap();
manifest.write_all(b"{\"source\":\"/x/c.js").unwrap();
drop(manifest);
let reopened = fx.open();
assert_eq!(reopened.len(), 2);
let third = fx.write("c.jsonl", "{\"c\":3}\n");
assert_eq!(reopened.archive(&third).unwrap(), Archived::Stored);
drop(reopened);
let again = fx.open();
assert_eq!(again.len(), 3);
assert_eq!(again.read(&third).unwrap().as_deref(), Some("{\"c\":3}\n"));
}
#[test]
fn a_failed_manifest_append_is_repaired_by_the_next_write() {
use std::os::unix::fs::PermissionsExt;
let fx = Fixture::new();
let first = fx.write("a.jsonl", "{\"a\":1}\n");
let second = fx.write("b.jsonl", "{\"b\":2}\n");
let third = fx.write("c.jsonl", "{\"c\":3}\n");
let archive = fx.open();
archive.archive(&first).unwrap();
let manifest = fx.archive_dir.join(MANIFEST_FILE);
fs::set_permissions(&manifest, fs::Permissions::from_mode(0o400)).unwrap();
assert!(archive.archive(&second).is_err());
assert_eq!(archive.archive(&third).unwrap(), Archived::Stored);
drop(archive);
let reopened = fx.open();
assert_eq!(reopened.len(), 3);
assert_eq!(
reopened.read(&second).unwrap().as_deref(),
Some("{\"b\":2}\n")
);
}
#[test]
fn compaction_keeps_the_latest_entry_per_source() {
let fx = Fixture::new();
let first = fx.write("a.jsonl", "{\"a\":1}\n");
let second = fx.write("b.jsonl", "{\"b\":1}\n");
let archive = fx.open();
archive.archive(&first).unwrap();
archive.archive(&second).unwrap();
let latest_second = archive.entry(&second).unwrap();
drop(archive);
let manifest = fx.archive_dir.join(MANIFEST_FILE);
let stale = ArchiveEntry {
source: first.to_string_lossy().into_owned(),
file: "0".repeat(64) + ".jsonl.zst",
sha256: "0".repeat(64),
archived_at: Utc::now(),
};
let mut lines = fs::read_to_string(&manifest).unwrap();
let mut stale_lines = String::new();
for _ in 0..80 {
stale_lines.push_str(&serde_json::to_string(&stale).unwrap());
stale_lines.push('\n');
}
lines.insert_str(0, &stale_lines);
fs::write(&manifest, &lines).unwrap();
let reopened = fx.open();
let kept = manifest_lines(&reopened);
assert_eq!(kept.len(), 2);
assert_eq!(reopened.len(), 2);
assert_ne!(reopened.entry(&first).unwrap().sha256, stale.sha256);
assert_eq!(reopened.entry(&second), Some(latest_second));
assert_eq!(
reopened.read(&first).unwrap().as_deref(),
Some("{\"a\":1}\n")
);
}
#[test]
fn a_long_running_archive_keeps_its_manifest_compact() {
let fx = Fixture::new();
let source = fx.write("a.jsonl", "");
let archive = fx.open();
let mut text = String::new();
for i in 0..100 {
text.push_str(&format!("{{\"line\":{i}}}\n"));
fs::write(&source, &text).unwrap();
archive.archive(&source).unwrap();
}
assert!(manifest_lines(&archive).len() <= 2 + 64);
assert_eq!(blobs(&archive).len(), 1);
drop(archive);
let reopened = fx.open();
assert_eq!(reopened.read(&source).unwrap(), Some(text));
}
}