use std::{
fs,
path::{Path, PathBuf},
};
use arrow::array::RecordBatch;
use parquet::{
arrow::{ArrowWriter, arrow_reader::ParquetRecordBatchReaderBuilder},
basic::Compression,
file::properties::WriterProperties,
};
use thiserror::Error;
use v_utils::trades::{ExchangeName, Symbol};
use crate::schema::UnixNanos;
#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
pub enum Lane {
Snapshots,
Deltas,
Trades,
Closes,
Custom,
}
impl Lane {
pub fn dir_name(self) -> &'static str {
match self {
Lane::Snapshots => "snapshots",
Lane::Deltas => "deltas",
Lane::Trades => "trades",
Lane::Closes => "closes",
Lane::Custom => "custom",
}
}
pub fn compression(self) -> Compression {
match self {
Lane::Snapshots | Lane::Deltas | Lane::Trades => Compression::ZSTD(parquet::basic::ZstdLevel::default()),
Lane::Closes | Lane::Custom => Compression::SNAPPY,
}
}
}
#[derive(Clone, Debug)]
pub struct Catalog {
root: PathBuf,
}
impl Catalog {
pub fn new(root: impl Into<PathBuf>) -> Self {
Self { root: root.into() }
}
pub fn root(&self) -> &Path {
&self.root
}
pub fn lane_dir(&self, key: &LaneKey) -> PathBuf {
match key {
LaneKey::Book { lane, exchange, symbol } => self
.root
.join("data")
.join(lane.dir_name())
.join(symbol.pair.base().to_string())
.join(symbol.pair.quote().to_string())
.join(format!("{exchange}{}", symbol.instrument)),
LaneKey::Custom { type_name } => self.root.join("data").join("custom").join(type_name),
}
}
pub fn write(&self, key: &LaneKey, batch: &RecordBatch, ts_min: UnixNanos, ts_max: UnixNanos) -> Result<PathBuf, CatalogError> {
assert!(ts_min <= ts_max, "ts_min must be <= ts_max");
let dir = self.lane_dir(key);
fs::create_dir_all(&dir)?;
let existing = self.list(key)?;
for e in &existing {
if intervals_overlap((e.ts_min, e.ts_max), (ts_min, ts_max)) {
return Err(CatalogError::OverlappingInterval {
existing: (e.ts_min, e.ts_max),
new: (ts_min, ts_max),
});
}
}
let path = dir.join(format!("{ts_min}_{ts_max}.parquet"));
let file = fs::File::create(&path)?;
let props = WriterProperties::builder().set_compression(key.lane().compression()).build();
let mut writer = ArrowWriter::try_new(file, batch.schema(), Some(props))?;
writer.write(batch)?;
writer.close()?;
Ok(path)
}
pub fn list(&self, key: &LaneKey) -> Result<Vec<FileEntry>, CatalogError> {
let dir = self.lane_dir(key);
if !dir.exists() {
return Ok(Vec::new());
}
let mut entries = Vec::new();
for ent in fs::read_dir(&dir)? {
let ent = ent?;
let path = ent.path();
if path.extension().and_then(|s| s.to_str()) != Some("parquet") {
continue;
}
let stem = path.file_stem().and_then(|s| s.to_str()).ok_or_else(|| CatalogError::BadFilename(path.display().to_string()))?;
let (lo, hi) = stem.split_once('_').ok_or_else(|| CatalogError::BadFilename(path.display().to_string()))?;
let ts_min: i64 = lo.parse().map_err(|_| CatalogError::BadFilename(path.display().to_string()))?;
let ts_max: i64 = hi.parse().map_err(|_| CatalogError::BadFilename(path.display().to_string()))?;
entries.push(FileEntry { path, ts_min, ts_max });
}
entries.sort_by_key(|e| e.ts_min);
Ok(entries)
}
pub fn list_range(&self, key: &LaneKey, start: UnixNanos, end: UnixNanos) -> Result<Vec<FileEntry>, CatalogError> {
Ok(self.list(key)?.into_iter().filter(|e| e.ts_max >= start && e.ts_min <= end).collect())
}
pub fn read(&self, path: &Path) -> Result<Vec<RecordBatch>, CatalogError> {
let file = fs::File::open(path)?;
let builder = ParquetRecordBatchReaderBuilder::try_new(file)?;
let reader = builder.build()?;
let mut out = Vec::new();
for batch in reader {
out.push(batch?);
}
Ok(out)
}
}
#[derive(Debug, Error)]
pub enum CatalogError {
#[error("io: {0}")]
Io(#[from] std::io::Error),
#[error("arrow: {0}")]
Arrow(#[from] arrow::error::ArrowError),
#[error("parquet: {0}")]
Parquet(#[from] parquet::errors::ParquetError),
#[error("write would create overlapping interval: existing {existing:?}, new {new:?}")]
OverlappingInterval { existing: (UnixNanos, UnixNanos), new: (UnixNanos, UnixNanos) },
#[error("malformed filename: {0}")]
BadFilename(String),
}
#[derive(Clone, Debug, Eq, Hash, PartialEq)]
pub enum LaneKey {
Book { lane: Lane, exchange: ExchangeName, symbol: Symbol },
Custom { type_name: String },
}
impl LaneKey {
pub fn book(lane: Lane, exchange: ExchangeName, symbol: Symbol) -> Self {
debug_assert!(!matches!(lane, Lane::Custom), "Custom lane uses LaneKey::custom, not book");
Self::Book { lane, exchange, symbol }
}
pub fn custom(type_name: impl Into<String>) -> Self {
Self::Custom { type_name: type_name.into() }
}
pub fn lane(&self) -> Lane {
match self {
Self::Book { lane, .. } => *lane,
Self::Custom { .. } => Lane::Custom,
}
}
}
#[derive(Clone, Debug)]
pub struct FileEntry {
pub path: PathBuf,
pub ts_min: UnixNanos,
pub ts_max: UnixNanos,
}
fn intervals_overlap(a: (UnixNanos, UnixNanos), b: (UnixNanos, UnixNanos)) -> bool {
a.0 <= b.1 && b.0 <= a.1
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
use arrow::array::{BooleanArray, Int32Array, Int64Array, RecordBatch, UInt8Array, UInt32Array, UInt64Array};
use tempfile::tempdir;
use v_utils::trades::Instrument;
use super::*;
use crate::schema::{FileMetadata, lane_schema, with_metadata};
fn test_symbol() -> Symbol {
Symbol::new("BTC-USDT".try_into().unwrap(), Instrument::Spot)
}
fn meta() -> FileMetadata {
FileMetadata {
exchange: "binance".into(),
pair: "BTC-USDT".into(),
price_precision: 2,
qty_precision: 5,
}
}
fn one_delta_batch() -> RecordBatch {
let schema = with_metadata(lane_schema(Lane::Deltas), meta());
RecordBatch::try_new(
schema,
vec![
Arc::new(Int64Array::from(vec![1_i64])),
Arc::new(Int64Array::from(vec![1_i64])),
Arc::new(UInt64Array::from(vec![1_u64])),
Arc::new(BooleanArray::from(vec![false])),
Arc::new(UInt8Array::from(vec![0_u8])),
Arc::new(Int32Array::from(vec![1_i32])),
Arc::new(UInt32Array::from(vec![1_u32])),
],
)
.unwrap()
}
#[test]
fn write_list_read_round_trip() {
let dir = tempdir().unwrap();
let cat = Catalog::new(dir.path());
let key = LaneKey::book(Lane::Deltas, ExchangeName::Binance, test_symbol());
let batch = one_delta_batch();
let path = cat.write(&key, &batch, 110, 210).unwrap();
assert!(path.exists());
let listed = cat.list(&key).unwrap();
assert_eq!(listed.len(), 1);
assert_eq!(listed[0].ts_min, 110);
assert_eq!(listed[0].ts_max, 210);
let read = cat.read(&listed[0].path).unwrap();
assert_eq!(read.len(), 1);
assert_eq!(read[0].num_rows(), 1);
}
#[test]
fn refuses_overlapping_write() {
let dir = tempdir().unwrap();
let cat = Catalog::new(dir.path());
let key = LaneKey::book(Lane::Deltas, ExchangeName::Binance, test_symbol());
let batch = one_delta_batch();
cat.write(&key, &batch, 100, 200).unwrap();
let err = cat.write(&key, &batch, 150, 250).unwrap_err();
assert!(matches!(err, CatalogError::OverlappingInterval { .. }));
}
#[test]
fn list_range_prunes() {
let dir = tempdir().unwrap();
let cat = Catalog::new(dir.path());
let key = LaneKey::book(Lane::Deltas, ExchangeName::Binance, test_symbol());
let batch = one_delta_batch();
cat.write(&key, &batch, 100, 200).unwrap();
cat.write(&key, &batch, 300, 400).unwrap();
cat.write(&key, &batch, 500, 600).unwrap();
let pruned = cat.list_range(&key, 250, 450).unwrap();
assert_eq!(pruned.len(), 1);
assert_eq!(pruned[0].ts_min, 300);
}
}