use std::collections::BTreeSet;
use std::fs::{File, OpenOptions};
use std::io::Write;
use std::path::{Path, PathBuf};
use serde::{Deserialize, Serialize};
use crate::error::{Error, Result};
use crate::types::{Symbol, VenueId};
fn default_book_level() -> crate::types::BookLevel {
crate::types::BookLevel::L2
}
pub const MANIFEST_FILE: &str = "_manifest.jsonl";
pub const QUARANTINE_DIR: &str = "_quarantine";
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct FileRecord {
pub path: String,
pub venue: VenueId,
pub symbol: Symbol,
pub date: String,
#[serde(default = "default_book_level")]
pub book_level: crate::types::BookLevel,
#[serde(default)]
pub feed_depth: Option<usize>,
pub rows: u64,
pub bytes: u64,
pub first_recv_wall: i64,
pub last_recv_wall: i64,
pub closed_wall: i64,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct TruncationRecord {
pub venue: Option<VenueId>,
pub symbol: Option<Symbol>,
pub detected_wall: i64,
pub reason: String,
pub quarantined: Option<String>,
pub lost_from_wall: Option<i64>,
pub lost_to_wall: i64,
pub unrecoverable_rows: Option<u64>,
}
impl TruncationRecord {
pub fn lost_nanos(&self) -> Option<i64> {
Some(self.lost_to_wall - self.lost_from_wall?)
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct CompactionRecord {
pub produced: FileRecord,
pub retired: Vec<String>,
pub at_wall: i64,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct RetentionRecord {
pub removed: Vec<String>,
pub reason: String,
pub policy: String,
pub rows_removed: u64,
pub bytes_removed: u64,
pub at_wall: i64,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(tag = "kind", rename_all = "lowercase")]
pub enum ManifestEntry {
File(FileRecord),
Truncation(TruncationRecord),
Compaction(CompactionRecord),
Retention(RetentionRecord),
}
#[derive(Debug)]
pub struct Manifest {
path: PathBuf,
files: Vec<FileRecord>,
truncations: Vec<TruncationRecord>,
retired: BTreeSet<String>,
retentions: Vec<RetentionRecord>,
}
impl Manifest {
pub fn open(root: impl AsRef<Path>) -> Result<Self> {
let root = root.as_ref();
std::fs::create_dir_all(root)?;
let path = root.join(MANIFEST_FILE);
let mut files = Vec::new();
let mut truncations = Vec::new();
let mut retentions: Vec<RetentionRecord> = Vec::new();
let mut retired: BTreeSet<String> = BTreeSet::new();
if path.exists() {
let text = std::fs::read_to_string(&path)?;
let lines: Vec<&str> = text.lines().collect();
for (i, line) in lines.iter().enumerate() {
if line.trim().is_empty() {
continue;
}
match serde_json::from_str::<ManifestEntry>(line) {
Ok(ManifestEntry::File(f)) => files.push(f),
Ok(ManifestEntry::Truncation(t)) => truncations.push(t),
Ok(ManifestEntry::Compaction(c)) => {
retired.extend(c.retired.iter().cloned());
files.push(c.produced);
}
Ok(ManifestEntry::Retention(r)) => {
retired.extend(r.removed.iter().cloned());
retentions.push(r);
}
Err(e) if i + 1 == lines.len() => {
tracing::warn!(
path = %path.display(),
error = %e,
"discarding a torn final manifest line"
);
}
Err(e) => {
return Err(Error::Other(format!(
"{}:{}: corrupt manifest entry: {e}",
path.display(),
i + 1
)));
}
}
}
}
files.retain(|f| !retired.contains(&f.path));
Ok(Manifest {
path,
files,
truncations,
retired,
retentions,
})
}
pub fn path(&self) -> &Path {
&self.path
}
pub fn files(&self) -> &[FileRecord] {
&self.files
}
pub fn truncations(&self) -> &[TruncationRecord] {
&self.truncations
}
pub fn known_paths(&self) -> BTreeSet<String> {
let mut paths: BTreeSet<String> = self.files.iter().map(|f| f.path.clone()).collect();
paths.extend(self.retired.iter().cloned());
paths
}
pub fn retentions(&self) -> &[RetentionRecord] {
&self.retentions
}
pub fn retired_paths(&self) -> &BTreeSet<String> {
&self.retired
}
pub fn last_covered(&self, venue: VenueId, symbol: &Symbol) -> Option<i64> {
self.files
.iter()
.filter(|f| f.venue == venue && f.symbol == *symbol)
.map(|f| f.last_recv_wall)
.max()
}
pub fn total_rows(&self) -> u64 {
self.files.iter().map(|f| f.rows).sum()
}
fn append(&self, entry: &ManifestEntry) -> Result<()> {
let mut line = serde_json::to_string(entry)?;
line.push('\n');
let mut file = OpenOptions::new()
.create(true)
.append(true)
.open(&self.path)?;
file.write_all(line.as_bytes())?;
file.sync_data()?;
Ok(())
}
pub fn record_file(&mut self, record: FileRecord) -> Result<()> {
self.append(&ManifestEntry::File(record.clone()))?;
self.files.push(record);
Ok(())
}
pub fn record_retention(&mut self, record: RetentionRecord) -> Result<()> {
self.append(&ManifestEntry::Retention(record.clone()))?;
self.retired.extend(record.removed.iter().cloned());
self.files.retain(|f| !self.retired.contains(&f.path));
self.retentions.push(record);
Ok(())
}
pub fn record_compaction(&mut self, record: CompactionRecord) -> Result<()> {
self.append(&ManifestEntry::Compaction(record.clone()))?;
self.retired.extend(record.retired.iter().cloned());
self.files.retain(|f| !self.retired.contains(&f.path));
self.files.push(record.produced);
Ok(())
}
pub fn record_truncation(&mut self, record: TruncationRecord) -> Result<()> {
self.append(&ManifestEntry::Truncation(record.clone()))?;
self.truncations.push(record);
Ok(())
}
pub fn sync_dir(&self) -> Result<()> {
if let Some(dir) = self.path.parent() {
File::open(dir)?.sync_all()?;
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
fn record(path: &str, rows: u64, first: i64, last: i64) -> FileRecord {
FileRecord {
path: path.to_string(),
venue: VenueId::Kraken,
symbol: Symbol::new("BTC", "USD"),
date: "2026-08-24".to_string(),
book_level: crate::types::BookLevel::L2,
feed_depth: None,
rows,
bytes: rows * 40,
first_recv_wall: first,
last_recv_wall: last,
closed_wall: last + 1,
}
}
#[test]
fn entries_survive_a_reopen() {
let dir = tempfile::tempdir().unwrap();
{
let mut m = Manifest::open(dir.path()).unwrap();
m.record_file(record("a.parquet", 10, 100, 200)).unwrap();
m.record_file(record("b.parquet", 5, 200, 300)).unwrap();
}
let m = Manifest::open(dir.path()).unwrap();
assert_eq!(m.files().len(), 2);
assert_eq!(m.total_rows(), 15);
assert_eq!(
m.last_covered(VenueId::Kraken, &Symbol::new("BTC", "USD")),
Some(300)
);
assert_eq!(
m.last_covered(VenueId::Coinbase, &Symbol::new("BTC", "USD")),
None
);
}
#[test]
fn a_torn_final_line_is_dropped_rather_than_failing_the_open() {
let dir = tempfile::tempdir().unwrap();
{
let mut m = Manifest::open(dir.path()).unwrap();
m.record_file(record("a.parquet", 10, 100, 200)).unwrap();
}
let path = dir.path().join(MANIFEST_FILE);
let mut text = std::fs::read_to_string(&path).unwrap();
text.push_str("{\"kind\":\"file\",\"path\":\"b.par");
std::fs::write(&path, text).unwrap();
let m = Manifest::open(dir.path()).unwrap();
assert_eq!(m.files().len(), 1, "the complete entry must survive");
assert!(!m.known_paths().contains("b.parquet"));
}
#[test]
fn corruption_anywhere_but_the_last_line_is_an_error() {
let dir = tempfile::tempdir().unwrap();
std::fs::write(
dir.path().join(MANIFEST_FILE),
"not json\n{\"kind\":\"file\",\"path\":\"a\"}\n",
)
.unwrap();
let err = Manifest::open(dir.path()).unwrap_err().to_string();
assert!(err.contains(":1:"), "the error must name the line: {err}");
}
#[test]
fn a_truncation_records_the_window_it_could_not_cover() {
let dir = tempfile::tempdir().unwrap();
let mut m = Manifest::open(dir.path()).unwrap();
m.record_truncation(TruncationRecord {
venue: Some(VenueId::Kraken),
symbol: Some(Symbol::new("BTC", "USD")),
detected_wall: 1_000,
reason: "process died mid-write".to_string(),
quarantined: Some("_quarantine/part-7.parquet".to_string()),
lost_from_wall: Some(400),
lost_to_wall: 1_000,
unrecoverable_rows: None,
})
.unwrap();
let m = Manifest::open(dir.path()).unwrap();
assert_eq!(m.truncations().len(), 1);
assert_eq!(m.truncations()[0].lost_nanos(), Some(600));
assert_eq!(
m.truncations()[0].unrecoverable_rows,
None,
"a footerless file cannot be counted, and None says so"
);
}
#[test]
fn a_truncation_with_no_prior_coverage_reports_an_open_start() {
let t = TruncationRecord {
venue: None,
symbol: None,
detected_wall: 10,
reason: "first run died".to_string(),
quarantined: None,
lost_from_wall: None,
lost_to_wall: 10,
unrecoverable_rows: None,
};
assert_eq!(t.lost_nanos(), None);
}
#[test]
fn compaction_swaps_files_in_one_atomic_entry() {
let dir = tempfile::tempdir().unwrap();
{
let mut m = Manifest::open(dir.path()).unwrap();
m.record_file(record("a.parquet", 10, 100, 200)).unwrap();
m.record_file(record("b.parquet", 5, 200, 300)).unwrap();
m.record_compaction(CompactionRecord {
produced: record("daily.parquet", 15, 100, 300),
retired: vec!["a.parquet".to_string(), "b.parquet".to_string()],
at_wall: 400,
})
.unwrap();
}
let m = Manifest::open(dir.path()).unwrap();
assert_eq!(m.files().len(), 1, "only the compacted file remains live");
assert_eq!(m.files()[0].path, "daily.parquet");
assert_eq!(m.total_rows(), 15, "rows must not be double counted");
assert!(m.known_paths().contains("a.parquet"));
assert_eq!(m.retired_paths().len(), 2);
}
#[test]
fn an_empty_archive_opens_clean() {
let dir = tempfile::tempdir().unwrap();
let m = Manifest::open(dir.path().join("nested")).unwrap();
assert!(m.files().is_empty());
assert!(m.truncations().is_empty());
assert_eq!(m.total_rows(), 0);
m.sync_dir().unwrap();
}
}