use std::collections::VecDeque;
use std::path::{Path, PathBuf};
use arrow::array::{
Array, BooleanArray, Int64Array, RecordBatch, StringArray, TimestampNanosecondArray,
UInt8Array, UInt32Array, UInt64Array,
};
use parquet::arrow::arrow_reader::{ParquetRecordBatchReader, ParquetRecordBatchReaderBuilder};
use crate::error::{Error, Result};
use crate::fixed::Fixed;
use crate::store::manifest::FileRecord;
use crate::store::schema::EventKind;
use crate::types::{BookLevel, Side};
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Row {
pub event: EventKind,
pub msg_index: u64,
pub recv_wall: i64,
pub venue_ts: Option<i64>,
pub side: Side,
pub price: Fixed,
pub qty: Fixed,
pub suspect: bool,
pub book_level: BookLevel,
pub order_id: Option<String>,
pub action: Option<String>,
pub queue_index: Option<u32>,
pub queue_certainty: Option<String>,
pub traded_qty: Option<Fixed>,
}
fn column<'a, T: 'static>(batch: &'a RecordBatch, name: &str) -> Result<&'a T> {
batch
.column_by_name(name)
.and_then(|c| c.as_any().downcast_ref::<T>())
.ok_or_else(|| Error::Other(format!("archive row is missing column {name}")))
}
pub fn decode(batch: &RecordBatch) -> Result<Vec<Row>> {
let event = column::<UInt8Array>(batch, "event")?;
let msg_index = column::<UInt64Array>(batch, "msg_index")?;
let recv_wall = column::<TimestampNanosecondArray>(batch, "recv_wall")?;
let venue_ts = column::<TimestampNanosecondArray>(batch, "venue_ts")?;
let side = column::<UInt8Array>(batch, "side")?;
let price = column::<Int64Array>(batch, "price")?;
let qty = column::<Int64Array>(batch, "qty")?;
let suspect = column::<BooleanArray>(batch, "suspect")?;
let level = batch
.column_by_name("book_level")
.and_then(|c| c.as_any().downcast_ref::<UInt8Array>());
let order_id = batch
.column_by_name("order_id")
.and_then(|c| c.as_any().downcast_ref::<StringArray>());
let action = batch
.column_by_name("action")
.and_then(|c| c.as_any().downcast_ref::<StringArray>());
let queue_index = batch
.column_by_name("queue_index")
.and_then(|c| c.as_any().downcast_ref::<UInt32Array>());
let queue_certainty = batch
.column_by_name("queue_certainty")
.and_then(|c| c.as_any().downcast_ref::<StringArray>());
let traded_qty = batch
.column_by_name("traded_qty")
.and_then(|c| c.as_any().downcast_ref::<Int64Array>());
let mut rows = Vec::with_capacity(batch.num_rows());
for i in 0..batch.num_rows() {
rows.push(Row {
event: EventKind::from_u8(event.value(i))
.ok_or_else(|| Error::Other(format!("unknown event kind {}", event.value(i))))?,
msg_index: msg_index.value(i),
recv_wall: recv_wall.value(i),
venue_ts: (!venue_ts.is_null(i)).then(|| venue_ts.value(i)),
side: if side.value(i) == 0 {
Side::Bid
} else {
Side::Ask
},
price: Fixed::from_mantissa(price.value(i)),
qty: Fixed::from_mantissa(qty.value(i)),
suspect: suspect.value(i),
book_level: match level.map(|l| l.value(i)) {
Some(3) => BookLevel::L3,
_ => BookLevel::L2,
},
order_id: order_id
.filter(|c| !c.is_null(i))
.map(|c| c.value(i).to_string()),
action: action
.filter(|c| !c.is_null(i))
.map(|c| c.value(i).to_string()),
queue_index: queue_index.filter(|c| !c.is_null(i)).map(|c| c.value(i)),
queue_certainty: queue_certainty
.filter(|c| !c.is_null(i))
.map(|c| c.value(i).to_string()),
traded_qty: traded_qty
.filter(|c| !c.is_null(i))
.map(|c| Fixed::from_mantissa(c.value(i))),
});
}
Ok(rows)
}
pub struct RowStream {
root: PathBuf,
files: VecDeque<FileRecord>,
current: Option<ParquetRecordBatchReader>,
buffer: VecDeque<Row>,
until_wall: i64,
after_wall: Option<i64>,
finished: bool,
files_opened: usize,
rows_yielded: u64,
}
impl RowStream {
pub fn new(root: impl Into<PathBuf>, files: Vec<FileRecord>, until_wall: i64) -> Self {
RowStream {
root: root.into(),
files: files.into(),
current: None,
buffer: VecDeque::new(),
until_wall,
after_wall: None,
finished: false,
files_opened: 0,
rows_yielded: 0,
}
}
pub fn after(mut self, wall: i64) -> Self {
self.after_wall = Some(wall);
self
}
pub fn files_opened(&self) -> usize {
self.files_opened
}
pub fn rows_yielded(&self) -> u64 {
self.rows_yielded
}
pub fn buffered_rows(&self) -> usize {
self.buffer.len()
}
fn open_next(&mut self) -> Result<bool> {
let Some(record) = self.files.pop_front() else {
return Ok(false);
};
let path = self.root.join(&record.path);
let file = std::fs::File::open(&path)?;
let reader = ParquetRecordBatchReaderBuilder::try_new(file)
.map_err(|e| Error::Other(format!("{}: {e}", path.display())))?
.build()
.map_err(|e| Error::Other(format!("{}: {e}", path.display())))?;
self.current = Some(reader);
self.files_opened += 1;
Ok(true)
}
fn fill(&mut self) -> Result<bool> {
loop {
if let Some(reader) = self.current.as_mut() {
match reader.next() {
Some(Ok(batch)) => {
let rows = decode(&batch)?;
if rows.is_empty() {
continue;
}
self.buffer.extend(rows);
return Ok(true);
}
Some(Err(e)) => return Err(Error::Other(e.to_string())),
None => self.current = None,
}
}
if self.current.is_none() && !self.open_next()? {
return Ok(false);
}
}
}
}
impl Iterator for RowStream {
type Item = Result<Row>;
fn next(&mut self) -> Option<Self::Item> {
loop {
if self.finished {
return None;
}
let Some(row) = self.buffer.pop_front() else {
match self.fill() {
Ok(true) => continue,
Ok(false) => {
self.finished = true;
return None;
}
Err(e) => {
self.finished = true;
return Some(Err(e));
}
}
};
if row.recv_wall > self.until_wall {
self.finished = true;
return None;
}
if self.after_wall.is_some_and(|w| row.recv_wall <= w) {
continue;
}
self.rows_yielded += 1;
return Some(Ok(row));
}
}
}
pub fn partition_files(
reader: &crate::store::reader::ArchiveReader,
venue: crate::types::VenueId,
symbol: &crate::types::Symbol,
date: &str,
) -> Vec<FileRecord> {
let mut files: Vec<FileRecord> = reader
.files()
.iter()
.filter(|f| f.venue == venue && f.symbol == *symbol && f.date == date)
.cloned()
.collect();
files.sort_by_key(|f| (f.first_recv_wall, f.path.clone()));
files
}
pub fn relative(root: &Path, record: &FileRecord) -> PathBuf {
root.join(&record.path)
}
pub struct MessageStream {
rows: RowStream,
held: Option<Row>,
finished: bool,
}
impl MessageStream {
pub fn new(rows: RowStream) -> Self {
MessageStream {
rows,
held: None,
finished: false,
}
}
pub fn files_opened(&self) -> usize {
self.rows.files_opened()
}
pub fn rows_yielded(&self) -> u64 {
self.rows.rows_yielded()
}
pub fn buffered_rows(&self) -> usize {
self.rows.buffered_rows() + usize::from(self.held.is_some())
}
}
fn same_message(a: &Row, b: &Row) -> bool {
a.msg_index == b.msg_index && a.event == b.event && a.recv_wall == b.recv_wall
}
pub fn split_messages(rows: &[Row]) -> Messages<'_> {
Messages { rows }
}
pub struct Messages<'a> {
rows: &'a [Row],
}
impl<'a> Iterator for Messages<'a> {
type Item = &'a [Row];
fn next(&mut self) -> Option<&'a [Row]> {
let first = self.rows.first()?;
let mut n = 1;
while n < self.rows.len() && same_message(first, &self.rows[n]) {
n += 1;
}
let (message, rest) = self.rows.split_at(n);
self.rows = rest;
Some(message)
}
}
impl Iterator for MessageStream {
type Item = Result<Vec<Row>>;
fn next(&mut self) -> Option<Self::Item> {
if self.finished {
return None;
}
let mut message: Vec<Row> = Vec::new();
if let Some(row) = self.held.take() {
message.push(row);
}
loop {
match self.rows.next() {
Some(Ok(row)) => {
if message
.first()
.is_some_and(|first| !same_message(first, &row))
{
self.held = Some(row);
return Some(Ok(message));
}
message.push(row);
}
Some(Err(e)) => {
self.finished = true;
return Some(Err(e));
}
None => {
self.finished = true;
return (!message.is_empty()).then_some(Ok(message));
}
}
}
}
}
#[cfg(test)]
mod message_split_tests {
use super::*;
use crate::types::{BookLevel, Side};
fn row(msg_index: u64, event: EventKind, recv_wall: i64, price: i64) -> Row {
Row {
event,
msg_index,
recv_wall,
venue_ts: None,
side: Side::Bid,
price: Fixed::from_mantissa(price),
qty: Fixed::from_mantissa(1),
suspect: false,
book_level: BookLevel::L2,
order_id: None,
action: None,
queue_index: None,
queue_certainty: None,
traded_qty: None,
}
}
#[test]
fn levels_of_one_message_stay_together() {
let rows = vec![
row(0, EventKind::Delta, 100, 1),
row(0, EventKind::Delta, 100, 2),
row(1, EventKind::Delta, 200, 3),
];
let split: Vec<usize> = split_messages(&rows).map(|m| m.len()).collect();
assert_eq!(split, vec![2, 1]);
}
#[test]
fn all_three_fields_end_a_message() {
let by_index = vec![
row(0, EventKind::Delta, 100, 1),
row(1, EventKind::Delta, 100, 2),
];
let by_kind = vec![
row(0, EventKind::Snapshot, 100, 1),
row(0, EventKind::Delta, 100, 2),
];
let by_time = vec![
row(0, EventKind::Delta, 100, 1),
row(0, EventKind::Delta, 101, 2),
];
for rows in [by_index, by_kind, by_time] {
assert_eq!(split_messages(&rows).count(), 2);
}
}
#[test]
fn every_row_lands_in_exactly_one_message() {
let rows = vec![
row(0, EventKind::Snapshot, 10, 1),
row(0, EventKind::Snapshot, 10, 2),
row(1, EventKind::Delta, 20, 3),
row(2, EventKind::Delta, 30, 4),
row(2, EventKind::Delta, 30, 5),
];
let regrouped: Vec<Row> = split_messages(&rows).flatten().cloned().collect();
assert_eq!(regrouped.len(), rows.len());
for (a, b) in regrouped.iter().zip(rows.iter()) {
assert_eq!(a.msg_index, b.msg_index);
assert_eq!(a.price, b.price);
}
}
#[test]
fn an_empty_run_yields_no_messages() {
assert_eq!(split_messages(&[]).count(), 0);
}
}