pub mod aggregate;
pub mod replay;
use crate::book::L2Book;
use crate::book::l3::L3Book;
use crate::book::replay::BookReplayer;
use crate::error::Result;
use crate::reconstruct::{Origin, Reconstructor, Request, Trust};
use crate::store::rows::{MessageStream, Row, RowStream};
use crate::store::schema::EventKind;
use crate::types::{BookLevel, Symbol, VenueId};
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Query {
pub venue: VenueId,
pub symbol: Symbol,
pub from_wall: i64,
pub to_wall: i64,
pub depth: Option<usize>,
pub use_checkpoints: bool,
}
impl Query {
pub fn new(venue: VenueId, symbol: &Symbol, from_wall: i64, to_wall: i64) -> Self {
Query {
venue,
symbol: symbol.clone(),
from_wall,
to_wall,
depth: None,
use_checkpoints: true,
}
}
pub fn with_depth(mut self, depth: usize) -> Self {
self.depth = Some(depth);
self
}
pub fn without_checkpoints(mut self) -> Self {
self.use_checkpoints = false;
self
}
pub fn duration_nanos(&self) -> i64 {
(self.to_wall - self.from_wall).max(0)
}
pub fn cursor(&self, reconstructor: &Reconstructor) -> Result<BookCursor> {
BookCursor::open(reconstructor, self)
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Tick {
pub at_wall: i64,
pub venue_ts: Option<i64>,
pub kind: EventKind,
pub changes: usize,
pub suspect: bool,
pub traded_qty: Option<crate::Fixed>,
}
pub struct BookCursor {
replayer: BookReplayer,
stream: MessageStream,
depth: Option<usize>,
at_wall: i64,
origin: Origin,
seeded_trust: Trust,
book_level: BookLevel,
messages: u64,
exhausted: bool,
}
impl BookCursor {
fn open(reconstructor: &Reconstructor, query: &Query) -> Result<Self> {
let mut request = Request::new(query.venue, &query.symbol, query.from_wall);
if !query.use_checkpoints {
request = request.without_checkpoints();
}
let start = reconstructor.at(&request)?;
let reader = reconstructor.reader();
let date = crate::clock::format_utc_date(query.from_wall);
let mut files =
crate::store::rows::partition_files(reader, query.venue, &query.symbol, &date);
let end_date = crate::clock::format_utc_date(query.to_wall);
if end_date != date {
files.extend(crate::store::rows::partition_files(
reader,
query.venue,
&query.symbol,
&end_date,
));
}
let feed_depth = files.first().and_then(|f| f.feed_depth);
files.retain(|f| f.last_recv_wall > query.from_wall && f.first_recv_wall <= query.to_wall);
let mut replayer = BookReplayer::new(query.symbol.clone(), start.book_level, feed_depth);
replayer.seed_l2(
start.book.top(crate::types::Side::Bid, usize::MAX),
start.book.top(crate::types::Side::Ask, usize::MAX),
query.from_wall,
);
if let Some(l3) = start.l3.as_ref() {
let mut orders = Vec::new();
for side in [crate::types::Side::Bid, crate::types::Side::Ask] {
for (price, _) in start.book.top(side, usize::MAX) {
for order in l3.queue_at(side, price) {
orders.push((order.id.clone(), side, order.price, order.qty));
}
}
}
replayer.seed_l3(orders, query.from_wall);
}
let stream = MessageStream::new(
RowStream::new(reader.root(), files, query.to_wall).after(query.from_wall),
);
Ok(BookCursor {
replayer,
stream,
depth: query.depth,
at_wall: query.from_wall,
origin: start.origin,
seeded_trust: start.trust,
book_level: start.book_level,
messages: 0,
exhausted: false,
})
}
pub fn advance(&mut self) -> Result<Option<Tick>> {
if self.exhausted {
return Ok(None);
}
let Some(message) = self.stream.next() else {
self.exhausted = true;
return Ok(None);
};
let message = message?;
let Some(first) = message.first() else {
self.exhausted = true;
return Ok(None);
};
let tick = Tick {
at_wall: first.recv_wall,
venue_ts: first.venue_ts,
kind: first.event,
changes: message.len(),
suspect: message.iter().any(|r| r.suspect),
traded_qty: traded_quantity(&message),
};
self.at_wall = tick.at_wall;
self.replayer.apply_message(&message);
self.messages += 1;
Ok(Some(tick))
}
pub fn book(&mut self) -> &L2Book {
self.replayer.book_ref()
}
pub fn top(&mut self, side: crate::types::Side) -> Vec<(crate::Fixed, crate::Fixed)> {
let depth = self.depth.unwrap_or(usize::MAX);
self.replayer.book_ref().top(side, depth)
}
pub fn l3(&self) -> Option<&L3Book> {
self.replayer.l3()
}
pub fn at_wall(&self) -> i64 {
self.at_wall
}
pub fn messages(&self) -> u64 {
self.messages
}
pub fn rows(&self) -> u64 {
self.replayer.rows()
}
pub fn book_level(&self) -> BookLevel {
self.book_level
}
pub fn origin(&self) -> Origin {
self.origin
}
pub fn trust(&self) -> Trust {
Trust {
suspect_rows: self.seeded_trust.suspect_rows + self.replayer.suspect_rows(),
suspect_span: match (self.seeded_trust.suspect_span, self.replayer.suspect_span()) {
(Some((a1, b1)), Some((a2, b2))) => Some((a1.min(a2), b1.max(b2))),
(only, None) | (None, only) => only,
},
truncations: self.seeded_trust.truncations.clone(),
}
}
pub fn files_opened(&self) -> usize {
self.stream.files_opened()
}
pub fn buffered_rows(&self) -> usize {
self.stream.buffered_rows()
}
pub fn for_each<F>(&mut self, mut f: F) -> Result<u64>
where
F: FnMut(&Tick, &mut BookCursor) -> Result<()>,
{
let mut seen = 0;
while let Some(tick) = self.advance()? {
f(&tick, self)?;
seen += 1;
}
Ok(seen)
}
}
fn traded_quantity(message: &[Row]) -> Option<crate::Fixed> {
if message.first()?.book_level != BookLevel::L3 {
return None;
}
let mut total = crate::Fixed::ZERO;
for row in message {
if let Some(traded) = row.traded_qty {
total = total.checked_add(traded).unwrap_or(total);
}
}
Some(total)
}