use std::collections::{BTreeMap, BTreeSet, VecDeque};
use std::sync::Arc;
use std::time::Duration;
use crate::book::{BookDelta, BookSnapshot, L2Book};
use crate::clock::{Clock, Stamp};
use crate::error::{Error, Result};
use crate::gap::{GapLog, GapReport, SuspectCause};
use crate::limits::VenueLimiter;
use crate::recorder::RawRecorder;
use crate::sequence::{SeqState, SeqVerdict, Unverifiable};
use crate::types::{Symbol, VenueId};
use crate::venue::{
ControlKind, FeedEvent, Keepalive, RawFrame, SnapshotSource, ValidationTiming, Venue,
VenueCapabilities,
};
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum SessionAction {
Resnapshot {
symbols: Vec<Symbol>,
cause: SuspectCause,
},
SubscriptionFailed(String),
ReconnectRequested(String),
}
pub struct BookSession {
venue: Arc<dyn Venue>,
caps: VenueCapabilities,
venue_id: VenueId,
symbols: Vec<Symbol>,
books: BTreeMap<Symbol, L2Book>,
ready: BTreeSet<Symbol>,
connection_seq: SeqState,
symbol_seq: BTreeMap<Symbol, SeqState>,
scratch: L2Book,
pending: BTreeMap<Symbol, VecDeque<BookDelta>>,
l3_books: BTreeMap<Symbol, crate::book::l3::L3Book>,
l3_snapshot_floor: BTreeMap<Symbol, i64>,
l3_crossed_since: BTreeMap<Symbol, i64>,
archive: Option<crate::pipeline::RowAccumulator>,
msg_index: u64,
gaps: GapLog,
}
const PENDING_CAPACITY: usize = 1024;
impl BookSession {
pub fn new(venue: Arc<dyn Venue>, symbols: Vec<Symbol>) -> Self {
let caps = venue.capabilities().clone();
let venue_id = caps.id;
let mut books = BTreeMap::new();
let mut symbol_seq = BTreeMap::new();
let mut gaps = GapLog::new();
for symbol in &symbols {
books.insert(symbol.clone(), caps.new_book(symbol));
symbol_seq.insert(symbol.clone(), caps.new_seq_state());
gaps.ensure_symbol(caps.id, symbol);
}
for limit in &caps.detection_limits {
gaps.declare_limit(limit.clone());
}
let l3_books: BTreeMap<Symbol, crate::book::l3::L3Book> =
if caps.book_level == crate::types::BookLevel::L3 {
symbols
.iter()
.map(|s| (s.clone(), crate::book::l3::L3Book::new(s.clone())))
.collect()
} else {
BTreeMap::new()
};
let connection_seq = caps.new_seq_state();
let scratch = caps.new_book(&Symbol::new("SCRATCH", "SCRATCH"));
BookSession {
venue,
caps,
venue_id,
symbols,
books,
ready: BTreeSet::new(),
connection_seq,
symbol_seq,
scratch,
pending: BTreeMap::new(),
l3_books,
l3_snapshot_floor: BTreeMap::new(),
l3_crossed_since: BTreeMap::new(),
archive: None,
msg_index: 0,
gaps,
}
}
pub fn enable_archive(&mut self, batch_rows: usize) {
let depth = match self.caps.feed_depth {
crate::venue::FeedDepth::Limited(n) => Some(n),
crate::venue::FeedDepth::Full => None,
};
self.archive =
Some(crate::pipeline::RowAccumulator::new(batch_rows).with_feed_depth(depth));
}
pub fn take_archive_batches(
&mut self,
) -> Vec<(
crate::store::writer::PartitionKey,
arrow::array::RecordBatch,
(i64, i64),
)> {
self.archive
.as_mut()
.map(|a| a.take_ready())
.unwrap_or_default()
}
pub fn flush_archive(
&mut self,
) -> Vec<(
crate::store::writer::PartitionKey,
arrow::array::RecordBatch,
(i64, i64),
)> {
self.archive
.as_mut()
.map(|a| a.take_all())
.unwrap_or_default()
}
pub fn archive_pending_rows(&self) -> usize {
self.archive.as_ref().map(|a| a.pending_rows()).unwrap_or(0)
}
pub fn record_backpressure_drop(&mut self, symbol: &Symbol, rows: u64, at: Stamp) {
self.gaps
.record_backpressure_drop(self.venue_id, symbol, rows, at);
}
fn buffers_deltas(&self) -> bool {
self.caps.snapshot_source == SnapshotSource::Rest
}
fn remember(&mut self, delta: &BookDelta) {
if !self.buffers_deltas() {
return;
}
let queue = self.pending.entry(delta.symbol.clone()).or_default();
if queue.len() >= PENDING_CAPACITY {
queue.pop_front();
}
queue.push_back(delta.clone());
}
pub fn venue_id(&self) -> VenueId {
self.venue_id
}
pub fn symbols(&self) -> &[Symbol] {
&self.symbols
}
pub fn book(&self, symbol: &Symbol) -> Option<&L2Book> {
self.books.get(symbol)
}
pub fn l3_book(&self, symbol: &Symbol) -> Option<&crate::book::l3::L3Book> {
self.l3_books.get(symbol)
}
pub fn apply_order_snapshot(
&mut self,
snapshot: crate::venue::OrderSnapshot,
) -> Vec<SessionAction> {
let symbol = snapshot.symbol.clone();
let at = snapshot.stamps.recv;
if let Some(nanos) = snapshot.stamps.venue_nanos {
self.l3_snapshot_floor.insert(symbol.clone(), nanos);
}
if let Some(book) = self.l3_books.get_mut(&symbol) {
book.seed(snapshot.orders, at);
let l2 = book.to_l2(snapshot.stamps);
self.books.insert(symbol.clone(), l2);
self.ready.insert(symbol.clone());
self.gaps.record_resnapshot(self.venue_id, &symbol, at);
if let Some(state) = self.symbol_seq.get_mut(&symbol) {
state.reset();
}
}
Vec::new()
}
fn apply_order(
&mut self,
venue: &Arc<dyn Venue>,
order: crate::book::l3::OrderEvent,
at: Stamp,
actions: &mut Vec<SessionAction>,
blocked: bool,
) {
let symbol = order.symbol.clone();
if !self.l3_books.contains_key(&symbol) {
return;
}
self.gaps.record_message(self.venue_id, &symbol, at);
let event = FeedEvent::Order(Box::new(order.clone()));
let verdict = {
let state = self
.symbol_seq
.get_mut(&symbol)
.expect("state exists for every subscribed symbol");
venue.validate_sequence(state, &event, &self.scratch)
};
self.gaps
.record_verdict(self.venue_id, &symbol, &verdict, at);
if verdict.requires_resnapshot() {
self.mark_suspect(&symbol, cause_for(&verdict), at, actions);
return;
}
if blocked || !verdict.should_apply() {
return;
}
if let (Some(floor), Some(event_nanos)) = (
self.l3_snapshot_floor.get(&symbol),
order.stamps.venue_nanos,
) && event_nanos <= *floor
{
self.gaps.record_verdict(
self.venue_id,
&symbol,
&SeqVerdict::Stale {
seq: event_nanos.max(0) as u128,
floor: (*floor).max(0) as u128,
},
at,
);
return;
}
let mut outcome = self
.l3_books
.get_mut(&symbol)
.expect("book exists")
.apply(&order);
if outcome
.anomalies
.iter()
.any(|a| matches!(a, crate::book::l3::L3Anomaly::Crossed { .. }))
{
let pruned = self
.l3_books
.get_mut(&symbol)
.expect("book exists")
.prune_crossed_seeded();
if pruned > 0 {
self.gaps
.record_stale_seed_orders(self.venue_id, &symbol, pruned as u64);
outcome
.anomalies
.retain(|a| !matches!(a, crate::book::l3::L3Anomaly::Crossed { .. }));
}
}
let seeded = self.ready.contains(&symbol);
let now = order.stamps.venue_nanos.unwrap_or(at.wall_nanos);
let crossed_persisted = self.track_l3_crossing(&symbol, now);
for anomaly in &outcome.anomalies {
let corrupting = match anomaly {
crate::book::l3::L3Anomaly::UnknownOrder { .. } => seeded,
crate::book::l3::L3Anomaly::PriceMoved { .. } => false,
crate::book::l3::L3Anomaly::Crossed { .. } => crossed_persisted,
_ => true,
};
self.gaps
.record_l3_anomaly(self.venue_id, &symbol, anomaly, at, corrupting);
if corrupting {
self.mark_suspect(&symbol, SuspectCause::AwaitingSnapshot, at, actions);
}
}
let l2 = self
.l3_books
.get(&symbol)
.expect("book exists")
.to_l2(order.stamps);
self.books.insert(symbol.clone(), l2);
self.gaps.record_applied(self.venue_id, &symbol);
self.gaps
.record_skew(self.venue_id, &symbol, order.stamps.skew_nanos());
self.archive_order(&order, outcome.traded_qty);
}
fn track_l3_crossing(&mut self, symbol: &Symbol, now_venue_nanos: i64) -> bool {
const TRANSIENT_LIMIT_NANOS: i64 = 1_000_000;
let crossed = self
.l3_books
.get(symbol)
.map(|b| b.is_crossed())
.unwrap_or(false);
if !crossed {
self.l3_crossed_since.remove(symbol);
return false;
}
match self.l3_crossed_since.get(symbol) {
Some(since) => now_venue_nanos - since > TRANSIENT_LIMIT_NANOS,
None => {
self.l3_crossed_since
.insert(symbol.clone(), now_venue_nanos);
false
}
}
}
fn archive_order(&mut self, order: &crate::book::l3::OrderEvent, traded: Option<crate::Fixed>) {
let suspect = self.gaps.is_suspect(self.venue_id, &order.symbol);
let position = self
.l3_books
.get(&order.symbol)
.and_then(|b| b.queue_position(&order.order_id));
let venue_id = self.venue_id;
let index = self.msg_index;
self.msg_index += 1;
if let Some(archive) = self.archive.as_mut() {
archive.push_order(venue_id, order, position, traded, index, suspect);
}
}
pub fn is_ready(&self, symbol: &Symbol) -> bool {
self.ready.contains(symbol)
}
pub fn gaps(&self) -> &GapLog {
&self.gaps
}
pub fn report(&self) -> GapReport {
self.gaps.report()
}
pub fn seal(&mut self, at: Stamp) {
self.gaps.seal(at);
}
pub fn ingest(&mut self, frame: &RawFrame) -> Result<Vec<SessionAction>> {
let venue = Arc::clone(&self.venue);
let events = venue.parse_delta(frame)?;
let mut actions = Vec::new();
let at = frame.stamp;
for event in &events {
if let FeedEvent::Control(control) = event
&& control.kind == ControlKind::SubscriptionError
{
for symbol in self.symbols.clone() {
self.gaps.open_window(
self.venue_id,
&symbol,
SuspectCause::AwaitingSnapshot,
at,
);
self.ready.remove(&symbol);
}
actions.push(SessionAction::SubscriptionFailed(control.detail.clone()));
}
}
let mut frame_blocked = false;
if self.caps.gap_is_connection_wide() {
if let Some(event) = events.first() {
let verdict =
venue.validate_sequence(&mut self.connection_seq, event, &self.scratch);
frame_blocked = !verdict.should_apply();
if verdict.requires_resnapshot() {
let cause = cause_for(&verdict);
for symbol in self.symbols.clone() {
self.gaps
.record_verdict(self.venue_id, &symbol, &verdict, at);
self.ready.remove(&symbol);
}
actions.push(SessionAction::Resnapshot {
symbols: self.symbols.clone(),
cause,
});
frame_blocked = true;
} else {
for symbol in self.symbols.clone() {
self.gaps
.record_verdict(self.venue_id, &symbol, &verdict, at);
}
}
}
}
for event in events {
match event {
FeedEvent::Control(_) => {}
FeedEvent::Snapshot(snapshot) => {
self.apply_snapshot(&venue, snapshot, at, &mut actions);
}
FeedEvent::Delta(delta) => {
self.apply_one_delta(&venue, delta, at, &mut actions, frame_blocked);
}
FeedEvent::Order(order) => {
self.apply_order(&venue, *order, at, &mut actions, frame_blocked);
}
}
}
Ok(actions)
}
fn replay_pending(
&mut self,
venue: &Arc<dyn Venue>,
symbol: &Symbol,
actions: &mut Vec<SessionAction>,
) {
if !self.buffers_deltas() {
return;
}
let Some(queue) = self.pending.get_mut(symbol) else {
return;
};
let buffered: Vec<BookDelta> = queue.drain(..).collect();
for delta in buffered {
if !self.ready.contains(symbol) {
break;
}
let at = delta.stamps.recv;
self.apply_one_delta_inner(venue, delta, at, actions, false, false);
}
}
fn apply_one_delta(
&mut self,
venue: &Arc<dyn Venue>,
delta: BookDelta,
at: Stamp,
actions: &mut Vec<SessionAction>,
blocked: bool,
) {
self.apply_one_delta_inner(venue, delta, at, actions, blocked, true)
}
fn apply_one_delta_inner(
&mut self,
venue: &Arc<dyn Venue>,
delta: BookDelta,
at: Stamp,
actions: &mut Vec<SessionAction>,
blocked: bool,
from_live: bool,
) {
let symbol = delta.symbol.clone();
if !self.books.contains_key(&symbol) {
return;
}
if from_live {
self.gaps.record_message(self.venue_id, &symbol, at);
}
if blocked || !self.ready.contains(&symbol) {
self.remember(&delta);
if from_live {
self.gaps.record_verdict(
self.venue_id,
&symbol,
&SeqVerdict::Unverifiable(Unverifiable::AwaitingSnapshot),
at,
);
if self.caps.snapshot_source == SnapshotSource::Rest {
actions.push(SessionAction::Resnapshot {
symbols: vec![symbol.clone()],
cause: SuspectCause::AwaitingSnapshot,
});
}
}
return;
}
match self.caps.timing {
ValidationTiming::BeforeApply => {
let verdict = if self.caps.gap_is_connection_wide() {
SeqVerdict::InOrder
} else {
let state = self
.symbol_seq
.get_mut(&symbol)
.expect("state exists for every subscribed symbol");
let v = venue.validate_sequence(
state,
&FeedEvent::Delta(delta.clone()),
&self.scratch,
);
if from_live {
self.gaps.record_verdict(self.venue_id, &symbol, &v, at);
}
v
};
if verdict.requires_resnapshot() {
self.remember(&delta);
self.mark_suspect(&symbol, cause_for(&verdict), at, actions);
return;
}
if !verdict.should_apply() {
return;
}
let outcome = self
.books
.get_mut(&symbol)
.expect("book exists")
.apply_delta(&delta);
self.remember(&delta);
if from_live {
self.gaps
.record_skew(self.venue_id, &symbol, delta.stamps.skew_nanos());
}
self.absorb_outcome(&symbol, outcome, at, actions);
self.archive_delta(&delta, from_live);
}
ValidationTiming::AfterApply => {
let outcome = self
.books
.get_mut(&symbol)
.expect("book exists")
.apply_delta(&delta);
let book = self.books.get(&symbol).expect("book exists");
let state = self
.symbol_seq
.get_mut(&symbol)
.expect("state exists for every subscribed symbol");
let verdict =
venue.validate_sequence(state, &FeedEvent::Delta(delta.clone()), book);
if from_live {
self.gaps
.record_verdict(self.venue_id, &symbol, &verdict, at);
self.remember(&delta);
}
if verdict.requires_resnapshot() {
self.mark_suspect(&symbol, cause_for(&verdict), at, actions);
} else if from_live {
self.gaps.record_applied(self.venue_id, &symbol);
self.gaps
.record_skew(self.venue_id, &symbol, delta.stamps.skew_nanos());
}
self.absorb_outcome(&symbol, outcome, at, actions);
self.archive_delta(&delta, from_live);
}
}
}
pub fn apply_rest_snapshot(&mut self, snapshot: BookSnapshot) -> Vec<SessionAction> {
let venue = Arc::clone(&self.venue);
let at = snapshot.stamps.recv;
let mut actions = Vec::new();
self.apply_snapshot(&venue, snapshot, at, &mut actions);
actions
}
fn apply_snapshot(
&mut self,
venue: &Arc<dyn Venue>,
snapshot: BookSnapshot,
at: Stamp,
actions: &mut Vec<SessionAction>,
) {
let symbol = snapshot.symbol.clone();
if !self.books.contains_key(&symbol) {
return;
}
self.gaps.record_message(self.venue_id, &symbol, at);
if let Some(state) = self.symbol_seq.get_mut(&symbol) {
state.reset();
}
let outcome = self
.books
.get_mut(&symbol)
.expect("book exists")
.reset_from_snapshot(&snapshot);
let verdict = match self.caps.timing {
ValidationTiming::BeforeApply => {
let state = if self.caps.gap_is_connection_wide() {
&mut self.connection_seq
} else {
self.symbol_seq.get_mut(&symbol).expect("state exists")
};
venue.validate_sequence(
state,
&FeedEvent::Snapshot(snapshot.clone()),
&self.scratch,
)
}
ValidationTiming::AfterApply => {
let book = self.books.get(&symbol).expect("book exists");
let state = self.symbol_seq.get_mut(&symbol).expect("state exists");
venue.validate_sequence(state, &FeedEvent::Snapshot(snapshot.clone()), book)
}
};
self.gaps
.record_verdict(self.venue_id, &symbol, &verdict, at);
if verdict.is_loss() {
self.ready.remove(&symbol);
self.mark_suspect(&symbol, cause_for(&verdict), at, actions);
} else {
self.ready.insert(symbol.clone());
self.gaps.record_resnapshot(self.venue_id, &symbol, at);
}
self.absorb_outcome(&symbol, outcome, at, actions);
let suspect = self.gaps.is_suspect(self.venue_id, &symbol);
let venue_id = self.venue_id;
let index = self.msg_index;
self.msg_index += 1;
if let Some(archive) = self.archive.as_mut() {
archive.push_snapshot(venue_id, &snapshot, index, suspect);
}
self.replay_pending(venue, &symbol, actions);
}
fn archive_delta(&mut self, delta: &BookDelta, from_live: bool) {
if !from_live {
return;
}
let suspect = self.gaps.is_suspect(self.venue_id, &delta.symbol);
let venue_id = self.venue_id;
let index = self.msg_index;
self.msg_index += 1;
if let Some(archive) = self.archive.as_mut() {
archive.push_delta(venue_id, delta, index, suspect);
}
}
fn absorb_outcome(
&mut self,
symbol: &Symbol,
outcome: crate::book::ApplyOutcome,
at: Stamp,
actions: &mut Vec<SessionAction>,
) {
for anomaly in &outcome.anomalies {
let corrupting = self.caps.corrupts_book(anomaly);
self.gaps
.record_anomaly(self.venue_id, symbol, *anomaly, at, corrupting);
}
if let Some(anomaly) = outcome
.anomalies
.iter()
.find(|a| self.caps.corrupts_book(a))
{
self.mark_suspect(symbol, SuspectCause::Anomaly(*anomaly), at, actions);
}
}
fn mark_suspect(
&mut self,
symbol: &Symbol,
cause: SuspectCause,
at: Stamp,
actions: &mut Vec<SessionAction>,
) {
self.ready.remove(symbol);
self.gaps.open_window(self.venue_id, symbol, cause, at);
let already_queued = actions.iter().any(|a| match a {
SessionAction::Resnapshot { symbols, .. } => symbols.contains(symbol),
_ => false,
});
if !already_queued {
actions.push(SessionAction::Resnapshot {
symbols: vec![symbol.clone()],
cause,
});
}
}
pub fn on_disconnect(&mut self, at: Stamp) {
for symbol in self.symbols.clone() {
self.ready.remove(&symbol);
self.gaps.record_disconnect(self.venue_id, &symbol, at);
}
self.connection_seq.reset();
for state in self.symbol_seq.values_mut() {
state.reset();
}
}
}
fn cause_for(verdict: &SeqVerdict) -> SuspectCause {
match verdict {
SeqVerdict::Gap(evidence) => SuspectCause::SequenceGap {
size: evidence.size,
},
SeqVerdict::Divergence { expected, computed } => SuspectCause::ChecksumDivergence {
expected: *expected,
computed: *computed,
},
SeqVerdict::Reset { .. } => SuspectCause::SequenceReset,
SeqVerdict::OutOfOrder { previous, observed } => SuspectCause::OutOfOrder {
previous: *previous,
observed: *observed,
},
_ => SuspectCause::AwaitingSnapshot,
}
}
#[derive(Debug, Clone, Copy)]
pub struct Backoff {
pub base: Duration,
pub max: Duration,
pub factor: u32,
attempt: u32,
}
impl Default for Backoff {
fn default() -> Self {
Backoff {
base: Duration::from_millis(250),
max: Duration::from_secs(30),
factor: 2,
attempt: 0,
}
}
}
impl Backoff {
pub fn reset(&mut self) {
self.attempt = 0;
}
pub fn attempt(&self) -> u32 {
self.attempt
}
pub fn next(&mut self, clock: &dyn Clock) -> Duration {
let scale = self.factor.saturating_pow(self.attempt.min(16));
let raw = self.base.saturating_mul(scale.max(1)).min(self.max);
self.attempt = self.attempt.saturating_add(1);
let jitter_bucket = (clock.stamp().mono_nanos % 256) as u32;
let jitter = raw / 4 * jitter_bucket / 256;
(raw.saturating_sub(raw / 8) + jitter).min(self.max)
}
}
#[derive(Default, Clone)]
pub struct RunSinks {
pub tape: Option<Arc<tokio::sync::Mutex<RawRecorder>>>,
pub archive: Option<Arc<crate::pipeline::Pipeline>>,
pub archive_batch_rows: usize,
pub progress: Option<Arc<dyn ProgressSink>>,
pub archive_flush_interval: Duration,
}
impl RunSinks {
pub fn tape(recorder: RawRecorder) -> Self {
RunSinks {
tape: Some(Arc::new(tokio::sync::Mutex::new(recorder))),
..Self::none()
}
}
pub fn none() -> Self {
RunSinks {
tape: None,
archive: None,
progress: None,
archive_batch_rows: 4_096,
archive_flush_interval: Duration::from_secs(1),
}
}
pub fn with_archive(mut self, pipeline: Arc<crate::pipeline::Pipeline>) -> Self {
self.archive = Some(pipeline);
self
}
pub fn with_progress(mut self, sink: Arc<dyn ProgressSink>) -> Self {
self.progress = Some(sink);
self
}
}
#[derive(Debug, Clone, Copy, Default)]
pub struct StopAfter {
pub frames: Option<u64>,
pub duration: Option<Duration>,
}
impl StopAfter {
pub fn frames(n: u64) -> Self {
StopAfter {
frames: Some(n),
..Default::default()
}
}
pub fn duration(d: Duration) -> Self {
StopAfter {
duration: Some(d),
..Default::default()
}
}
}
#[derive(Debug, Clone)]
pub struct RunSnapshot {
pub venue: VenueId,
pub frames: u64,
pub reconnects: u32,
pub last_frame_wall: Option<i64>,
pub report: GapReport,
}
pub trait ProgressSink: Send + Sync {
fn update(&self, snapshot: RunSnapshot);
}
#[derive(Debug)]
pub struct RunOutcome {
pub report: GapReport,
pub frames: u64,
pub reconnects: u32,
}
impl RunOutcome {
pub fn merge(self, other: RunOutcome) -> RunOutcome {
RunOutcome {
report: self.report.merge(other.report),
frames: self.frames + other.frames,
reconnects: self.reconnects + other.reconnects,
}
}
}
pub async fn run_connection(
venue: Arc<dyn Venue>,
symbols: Vec<Symbol>,
clock: Arc<dyn Clock>,
stop: StopAfter,
sinks: RunSinks,
) -> Result<RunOutcome> {
let recorder = sinks.tape.clone();
let mut session = BookSession::new(Arc::clone(&venue), symbols.clone());
if sinks.archive.is_some() {
session.enable_archive(sinks.archive_batch_rows);
}
let mut backoff = Backoff::default();
let mut frames: u64 = 0;
let mut reconnects: u32 = 0;
let mut last_frame_wall: Option<i64> = None;
let started = tokio::time::Instant::now();
let caps = venue.capabilities().clone();
let mut limiter = VenueLimiter::new(&caps.budget);
let deadline = stop.duration.map(|d| started + d);
enum Wake {
Frame(Result<Option<RawFrame>>),
Keepalive,
ArchiveFlush,
Idle,
Deadline,
}
const IDLE_TIMEOUT: Duration = Duration::from_secs(60);
'outer: loop {
if deadline.is_some_and(|d| tokio::time::Instant::now() >= d) {
break;
}
let wait = limiter.connects.acquire(clock.as_ref());
if !wait.is_zero() {
tokio::time::sleep(wait).await;
}
let mut transport = match venue.connect().await {
Ok(t) => t,
Err(e) if e.is_transient() => {
let delay = backoff.next(clock.as_ref());
tracing::warn!(venue = %venue.id(), error = %e, ?delay, "connect failed, backing off");
tokio::time::sleep(delay).await;
reconnects += 1;
continue;
}
Err(e) => return Err(e),
};
for frame in venue.subscribe(&symbols)? {
transport.send_text(&frame).await?;
}
if caps.snapshot_source == SnapshotSource::Rest {
for symbol in &symbols {
let wait = limiter.rest.acquire(clock.as_ref());
if !wait.is_zero() {
tokio::time::sleep(wait).await;
}
if let Some(orders) = venue.order_snapshot(symbol).await? {
session.apply_order_snapshot(orders);
continue;
}
let snapshot = venue.snapshot(symbol).await?;
for action in session.apply_rest_snapshot(snapshot) {
tracing::warn!(
venue = %venue.id(), %symbol, ?action,
"snapshot did not settle the book"
);
}
}
}
backoff.reset();
let mut last_frame_at = tokio::time::Instant::now();
let mut archive_flush = (sinks.archive.is_some() || sinks.progress.is_some()).then(|| {
let mut interval = tokio::time::interval(sinks.archive_flush_interval);
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
interval
});
let mut keepalive = match caps.keepalive {
Keepalive::Text { every, .. } => {
let mut interval = tokio::time::interval(every);
interval.reset();
Some(interval)
}
Keepalive::ProtocolPings => None,
};
loop {
if stop.frames.is_some_and(|limit| frames >= limit) {
break 'outer;
}
let wake = tokio::select! {
biased;
result = transport.recv() => Wake::Frame(result),
_ = tokio::time::sleep_until(last_frame_at + IDLE_TIMEOUT) => Wake::Idle,
_ = async {
match keepalive.as_mut() {
Some(ticker) => { ticker.tick().await; }
None => std::future::pending::<()>().await,
}
} => Wake::Keepalive,
_ = async {
match archive_flush.as_mut() {
Some(ticker) => { ticker.tick().await; }
None => std::future::pending::<()>().await,
}
} => Wake::ArchiveFlush,
_ = async {
match deadline {
Some(at) => tokio::time::sleep_until(at).await,
None => std::future::pending::<()>().await,
}
} => Wake::Deadline,
};
let next = match wake {
Wake::Deadline => break 'outer,
Wake::Idle => {
tracing::warn!(
venue = %venue.id(), ?IDLE_TIMEOUT,
"feed went silent on a live socket, reconnecting"
);
publish(
&sinks,
&venue,
&session,
frames,
reconnects,
last_frame_wall,
);
session.on_disconnect(clock.stamp());
reconnects += 1;
let delay = backoff.next(clock.as_ref());
tokio::time::sleep(delay).await;
continue 'outer;
}
Wake::Keepalive => {
if let Keepalive::Text { payload, .. } = caps.keepalive {
transport.send_text(payload).await?;
}
continue;
}
Wake::ArchiveFlush => {
publish(
&sinks,
&venue,
&session,
frames,
reconnects,
last_frame_wall,
);
let pending = session.flush_archive();
drain_to_archive(&mut session, &sinks, pending, clock.stamp()).await;
continue;
}
Wake::Frame(result) => result,
};
match next {
Ok(Some(frame)) => {
frames += 1;
last_frame_at = tokio::time::Instant::now();
last_frame_wall = Some(frame.stamp.wall_nanos);
if let Some(rec) = recorder.as_ref() {
rec.lock().await.record(&frame).await?;
}
let ingested = session.ingest(&frame);
if sinks.archive.is_some() {
let batches = session.take_archive_batches();
drain_to_archive(&mut session, &sinks, batches, frame.stamp).await;
}
match ingested {
Ok(actions) => {
for action in actions {
match action {
SessionAction::Resnapshot { symbols, cause } => {
tracing::warn!(
venue = %venue.id(), ?symbols, %cause,
"re-snapshotting"
);
match caps.snapshot_source {
SnapshotSource::InBand => {
session.on_disconnect(frame.stamp);
reconnects += 1;
let delay = backoff.next(clock.as_ref());
tokio::time::sleep(delay).await;
continue 'outer;
}
SnapshotSource::None => {}
SnapshotSource::Rest => {
for symbol in symbols {
let wait = limiter.rest.acquire(clock.as_ref());
if !wait.is_zero() {
tokio::time::sleep(wait).await;
}
let snapshot = venue.snapshot(&symbol).await?;
session.apply_rest_snapshot(snapshot);
}
}
}
}
SessionAction::ReconnectRequested(detail) => {
tracing::info!(
venue = %venue.id(), %detail,
"venue asked us to reconnect"
);
session.on_disconnect(frame.stamp);
reconnects += 1;
continue 'outer;
}
SessionAction::SubscriptionFailed(detail) => {
return Err(Error::Subscription {
venue: venue.id(),
reason: detail,
});
}
}
}
}
Err(e) => tracing::warn!(venue = %venue.id(), error = %e, "unparsed frame"),
}
}
Ok(None) => {
tracing::warn!(venue = %venue.id(), "feed closed");
session.on_disconnect(clock.stamp());
reconnects += 1;
let delay = backoff.next(clock.as_ref());
tokio::time::sleep(delay).await;
continue 'outer;
}
Err(e) if e.is_transient() => {
tracing::warn!(venue = %venue.id(), error = %e, "transport error");
session.on_disconnect(clock.stamp());
reconnects += 1;
let delay = backoff.next(clock.as_ref());
tokio::time::sleep(delay).await;
continue 'outer;
}
Err(e) => return Err(e),
}
}
}
if sinks.archive.is_some() {
let remaining = session.flush_archive();
drain_to_archive(&mut session, &sinks, remaining, clock.stamp()).await;
}
session.seal(clock.stamp());
if let Some(rec) = recorder.as_ref() {
rec.lock().await.flush().await?;
}
Ok(RunOutcome {
report: session.report(),
frames,
reconnects,
})
}
fn publish(
sinks: &RunSinks,
venue: &Arc<dyn Venue>,
session: &BookSession,
frames: u64,
reconnects: u32,
last_frame_wall: Option<i64>,
) {
let Some(sink) = sinks.progress.as_ref() else {
return;
};
sink.update(RunSnapshot {
venue: venue.id(),
frames,
reconnects,
last_frame_wall,
report: session.report(),
});
}
async fn drain_to_archive(
session: &mut BookSession,
sinks: &RunSinks,
batches: Vec<(
crate::store::writer::PartitionKey,
arrow::array::RecordBatch,
(i64, i64),
)>,
at: Stamp,
) {
let Some(pipeline) = sinks.archive.as_ref() else {
return;
};
for (key, batch, span) in batches {
match pipeline.submit(key.clone(), batch, span).await {
crate::pipeline::Submitted::Dropped { rows } => {
tracing::warn!(
venue = %key.venue, symbol = %key.symbol, rows,
"writer fell behind, dropping rows"
);
session.record_backpressure_drop(&key.symbol, rows as u64, at);
}
crate::pipeline::Submitted::Blocked(waited) => {
tracing::warn!(
venue = %key.venue, symbol = %key.symbol, ?waited,
"blocked waiting for the writer"
);
}
crate::pipeline::Submitted::WriterStopped => {
tracing::error!(venue = %key.venue, "the archive writer stopped");
}
crate::pipeline::Submitted::Accepted => {}
}
}
}
pub async fn run(
venue: Arc<dyn Venue>,
symbols: Vec<Symbol>,
clock: Arc<dyn Clock>,
stop: StopAfter,
sinks: RunSinks,
) -> Result<(RunOutcome, RunSinks)> {
let caps = venue.capabilities().clone();
let plan = caps
.plan_subscriptions(&symbols)
.map_err(|e| Error::Other(e.to_string()))?;
tracing::info!(
venue = %venue.id(),
connections = plan.connection_count(),
blast_radius = plan.worst_case_blast_radius(caps.gap_is_connection_wide()),
rationale = plan.rationale,
"subscription plan"
);
let mut tasks = Vec::with_capacity(plan.connection_count());
for group in plan.connections {
tasks.push(tokio::spawn(run_connection(
Arc::clone(&venue),
group,
Arc::clone(&clock),
stop,
sinks.clone(),
)));
}
let mut merged: Option<RunOutcome> = None;
let mut failure: Option<Error> = None;
for task in tasks {
match task.await {
Ok(Ok(outcome)) => {
merged = Some(match merged {
Some(acc) => acc.merge(outcome),
None => outcome,
});
}
Ok(Err(e)) => failure = failure.or(Some(e)),
Err(join) => failure = failure.or(Some(Error::Other(join.to_string()))),
}
}
match (merged, failure) {
(Some(outcome), _) => Ok((outcome, sinks)),
(None, Some(e)) => Err(e),
(None, None) => Err(Error::Other("the plan produced no connections".to_string())),
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::clock::ManualClock;
#[test]
fn backoff_grows_geometrically_and_is_capped() {
let clock = ManualClock::new(0, 0);
let mut b = Backoff {
base: Duration::from_millis(100),
max: Duration::from_secs(2),
factor: 2,
attempt: 0,
};
let delays: Vec<Duration> = (0..8).map(|_| b.next(&clock)).collect();
assert!(delays[0] < delays[1] && delays[1] < delays[2]);
assert!(delays.iter().all(|d| *d <= Duration::from_secs(2)));
assert_eq!(delays[6], delays[7], "growth must stop at the cap");
assert_eq!(delays[7], Duration::from_millis(1_750));
b.reset();
assert_eq!(b.attempt(), 0);
}
#[test]
fn merging_connection_outcomes_sums_their_work() {
let outcome = |frames, reconnects| RunOutcome {
report: GapReport {
rows: Vec::new(),
limits: Vec::new(),
},
frames,
reconnects,
};
let merged = outcome(100, 1).merge(outcome(50, 2));
assert_eq!(merged.frames, 150);
assert_eq!(merged.reconnects, 3);
}
#[test]
fn backoff_jitter_is_deterministic_for_a_given_clock() {
let a = ManualClock::new(0, 7);
let b = ManualClock::new(0, 7);
let mut x = Backoff::default();
let mut y = Backoff::default();
let xs: Vec<Duration> = (0..5).map(|_| x.next(&a)).collect();
let ys: Vec<Duration> = (0..5).map(|_| y.next(&b)).collect();
assert_eq!(xs, ys);
}
}