pub mod binance_us;
pub mod bitstamp;
pub mod bybit;
pub mod coinbase;
pub mod kraken;
pub mod okx;
pub mod registry;
use std::fmt;
use std::time::Duration;
use async_trait::async_trait;
use crate::book::l3::{OrderEvent, OrderId};
use crate::book::{BookDelta, BookSnapshot, L2Book};
use crate::clock::Stamp;
use crate::error::Result;
use crate::gap::DetectionLimit;
use crate::limits::{BudgetError, SubscriptionPlan, VenueBudget};
use crate::sequence::{SeqState, SeqVerdict};
use crate::types::{BookLevel, Side, Symbol, VenueId, VenueSymbol};
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct RawFrame {
pub venue: VenueId,
pub payload: Vec<u8>,
pub stamp: Stamp,
}
impl RawFrame {
pub fn new(venue: VenueId, payload: impl Into<Vec<u8>>, stamp: Stamp) -> Self {
RawFrame {
venue,
payload: payload.into(),
stamp,
}
}
pub fn text(&self) -> std::borrow::Cow<'_, str> {
String::from_utf8_lossy(&self.payload)
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct OrderSnapshot {
pub symbol: Symbol,
pub orders: Vec<(OrderId, Side, crate::Fixed, crate::Fixed)>,
pub stamps: crate::clock::Timestamps,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum FeedEvent {
Snapshot(BookSnapshot),
Delta(BookDelta),
Order(Box<OrderEvent>),
Control(ControlEvent),
}
impl FeedEvent {
pub fn symbol(&self) -> Option<&Symbol> {
match self {
FeedEvent::Snapshot(s) => Some(&s.symbol),
FeedEvent::Delta(d) => Some(&d.symbol),
FeedEvent::Order(o) => Some(&o.symbol),
FeedEvent::Control(_) => None,
}
}
pub fn stamp(&self) -> Stamp {
match self {
FeedEvent::Snapshot(s) => s.stamps.recv,
FeedEvent::Delta(d) => d.stamps.recv,
FeedEvent::Order(o) => o.stamps.recv,
FeedEvent::Control(c) => c.stamp,
}
}
pub fn is_book_event(&self) -> bool {
!matches!(self, FeedEvent::Control(_))
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ControlEvent {
pub kind: ControlKind,
pub detail: String,
pub stamp: Stamp,
pub seq: Option<u64>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ControlKind {
SubscriptionAck,
SubscriptionError,
Heartbeat,
Status,
ReconnectRequested,
Ignored,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ValidationScheme {
Counter { step: u64, restart_floor: u64 },
Checksum { depth: usize },
Chained,
UpdateIdRange,
MonotonicTimestamp,
None,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum SequenceScope {
PerConnection,
PerSymbol,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ValidationTiming {
BeforeApply,
AfterApply,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum SnapshotSource {
InBand,
Rest,
None,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum FeedDepth {
Full,
Limited(usize),
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum DecimalEncoding {
Strings,
JsonNumbers,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum RedundantDeletes {
Never,
Expected,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Keepalive {
ProtocolPings,
Text {
payload: &'static str,
every: Duration,
},
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum VenueTimestamps {
None,
PerMessage,
PerMessageAndPerLevel,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct VenueCapabilities {
pub id: VenueId,
pub book_level: BookLevel,
pub validation: ValidationScheme,
pub scope: SequenceScope,
pub timing: ValidationTiming,
pub snapshot_source: SnapshotSource,
pub feed_depth: FeedDepth,
pub decimals: DecimalEncoding,
pub redundant_deletes: RedundantDeletes,
pub timestamps: VenueTimestamps,
pub keepalive: Keepalive,
pub budget: VenueBudget,
pub requires_auth: bool,
pub detection_limits: Vec<DetectionLimit>,
}
impl VenueCapabilities {
pub fn new_seq_state(&self) -> SeqState {
match self.validation {
ValidationScheme::Counter {
step,
restart_floor,
} => SeqState::counter(step, restart_floor),
ValidationScheme::Checksum { .. } => SeqState::checksum(),
ValidationScheme::Chained => SeqState::chain(),
ValidationScheme::UpdateIdRange => SeqState::range(),
ValidationScheme::MonotonicTimestamp => SeqState::timestamp(),
ValidationScheme::None => SeqState::Unverifiable,
}
}
pub fn new_book(&self, symbol: &Symbol) -> L2Book {
match self.feed_depth {
FeedDepth::Full => L2Book::new(symbol.clone()),
FeedDepth::Limited(n) => L2Book::with_depth_limit(symbol.clone(), n),
}
}
pub fn gap_is_connection_wide(&self) -> bool {
self.scope == SequenceScope::PerConnection
}
pub fn plan_subscriptions(
&self,
symbols: &[crate::types::Symbol],
) -> std::result::Result<SubscriptionPlan, BudgetError> {
crate::limits::plan(symbols, &self.budget, self.gap_is_connection_wide())
}
pub fn can_detect_loss(&self) -> bool {
!matches!(
self.validation,
ValidationScheme::None | ValidationScheme::MonotonicTimestamp
)
}
pub fn corrupts_book(&self, anomaly: &crate::book::BookAnomaly) -> bool {
match anomaly {
crate::book::BookAnomaly::RemovedMissingLevel { .. } => {
self.redundant_deletes == RedundantDeletes::Never
}
other => other.corrupts_book(),
}
}
}
#[async_trait]
pub trait FeedTransport: Send {
async fn recv(&mut self) -> Result<Option<RawFrame>>;
async fn send_text(&mut self, text: &str) -> Result<()>;
async fn close(&mut self) -> Result<()>;
}
#[async_trait]
pub trait HttpFetch: Send + Sync + fmt::Debug {
async fn get(&self, url: &str) -> Result<Vec<u8>>;
}
#[async_trait]
pub trait Venue: Send + Sync {
fn capabilities(&self) -> &VenueCapabilities;
fn id(&self) -> VenueId {
self.capabilities().id
}
fn venue_symbol(&self, symbol: &Symbol) -> VenueSymbol;
fn canonical_symbol(&self, raw: &str) -> Result<Symbol>;
async fn connect(&self) -> Result<Box<dyn FeedTransport>>;
fn subscribe(&self, symbols: &[Symbol]) -> Result<Vec<String>>;
async fn order_snapshot(&self, _symbol: &Symbol) -> Result<Option<OrderSnapshot>> {
Ok(None)
}
async fn snapshot(&self, symbol: &Symbol) -> Result<BookSnapshot>;
fn parse_delta(&self, frame: &RawFrame) -> Result<Vec<FeedEvent>>;
fn validate_sequence(
&self,
state: &mut SeqState,
event: &FeedEvent,
book: &L2Book,
) -> SeqVerdict;
}
#[cfg(test)]
mod tests {
use super::*;
use crate::sequence::CounterState;
fn caps(scheme: ValidationScheme, depth: FeedDepth) -> VenueCapabilities {
VenueCapabilities {
id: VenueId::Coinbase,
book_level: BookLevel::L2,
validation: scheme,
scope: SequenceScope::PerConnection,
timing: ValidationTiming::BeforeApply,
snapshot_source: SnapshotSource::InBand,
feed_depth: depth,
decimals: DecimalEncoding::Strings,
redundant_deletes: RedundantDeletes::Never,
timestamps: VenueTimestamps::PerMessage,
keepalive: Keepalive::ProtocolPings,
budget: VenueBudget::conservative(),
requires_auth: false,
detection_limits: Vec::new(),
}
}
#[test]
fn capabilities_build_the_matching_validator() {
let counter = caps(
ValidationScheme::Counter {
step: 1,
restart_floor: 0,
},
FeedDepth::Full,
);
assert_eq!(
counter.new_seq_state(),
SeqState::Counter(CounterState::new(1, 0))
);
let checksum = caps(ValidationScheme::Checksum { depth: 10 }, FeedDepth::Full);
assert!(matches!(checksum.new_seq_state(), SeqState::Checksum(_)));
let none = caps(ValidationScheme::None, FeedDepth::Full);
assert_eq!(none.new_seq_state(), SeqState::Unverifiable);
}
#[test]
fn a_depth_limited_feed_produces_a_truncating_book() {
let symbol = Symbol::new("BTC", "USD");
let full = caps(ValidationScheme::None, FeedDepth::Full);
assert_eq!(full.new_book(&symbol).depth_limit(), None);
let limited = caps(ValidationScheme::None, FeedDepth::Limited(25));
assert_eq!(limited.new_book(&symbol).depth_limit(), Some(25));
}
#[test]
fn a_timestamp_only_venue_admits_it_cannot_detect_loss() {
let mut c = caps(ValidationScheme::MonotonicTimestamp, FeedDepth::Full);
assert!(!c.can_detect_loss());
c.validation = ValidationScheme::None;
assert!(!c.can_detect_loss());
c.validation = ValidationScheme::Chained;
assert!(c.can_detect_loss());
}
#[test]
fn each_new_scheme_builds_its_matching_validator() {
use crate::sequence::SeqState;
for (scheme, matches) in [
(
ValidationScheme::Chained,
matches!(
caps(ValidationScheme::Chained, FeedDepth::Full).new_seq_state(),
SeqState::Chain(_)
),
),
(
ValidationScheme::UpdateIdRange,
matches!(
caps(ValidationScheme::UpdateIdRange, FeedDepth::Full).new_seq_state(),
SeqState::Range(_)
),
),
(
ValidationScheme::MonotonicTimestamp,
matches!(
caps(ValidationScheme::MonotonicTimestamp, FeedDepth::Full).new_seq_state(),
SeqState::Timestamp(_)
),
),
] {
assert!(matches, "{scheme:?} built the wrong validator");
}
}
#[test]
fn the_subscription_plan_follows_the_sequence_scope() {
let symbols: Vec<crate::types::Symbol> = (0..4)
.map(|i| crate::types::Symbol::new(&format!("A{i}"), "USD"))
.collect();
let mut c = caps(ValidationScheme::None, FeedDepth::Full);
c.scope = SequenceScope::PerConnection;
let spread = c.plan_subscriptions(&symbols).unwrap();
assert_eq!(spread.worst_case_blast_radius(true), 1);
c.scope = SequenceScope::PerSymbol;
let packed = c.plan_subscriptions(&symbols).unwrap();
assert_eq!(packed.connection_count(), 1);
}
#[test]
fn connection_scope_is_readable_without_naming_a_venue() {
let mut c = caps(ValidationScheme::None, FeedDepth::Full);
assert!(c.gap_is_connection_wide());
c.scope = SequenceScope::PerSymbol;
assert!(!c.gap_is_connection_wide());
}
}