use std::any::Any;
use std::path::PathBuf;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
use crate::ws_health::WsHealth;
use dashmap::DashMap;
use digdigdig3::connector_manager::ExchangeHub;
use digdigdig3::core::types::{
AccountType, ConnectorCapabilities, ExchangeId, StreamEvent, StreamType, SubscriptionRequest,
Symbol, SymbolInfo,
};
use digdigdig3::core::websocket::KlineInterval;
use digdigdig3::core::utils::SymbolNormalizer;
use futures_util::StreamExt;
use tokio::sync::{broadcast, mpsc, oneshot, RwLock};
use crate::data::{
AggTradePoint, AuctionEventPoint, BalanceUpdatePoint, BarPoint, BasisPoint, BlockTradePoint,
CompositeIndexPoint,
FootprintPoint,
FundingRatePoint, FundingRateIndicatorsPoint, FundingRateFullPoint,
FundingSettlementPoint, HistoricalVolatilityPoint,
IndexPriceKlinePoint, IndexPricePoint, IndexPriceIndicatorsPoint,
InsuranceFundPoint,
LiquidationPoint, LiquidationIndicatorsPoint, LiquidationFullPoint,
LiquidationBucketPoint,
LongShortRatioPoint,
MarkPriceKlinePoint, MarkPricePoint, MarkPriceIndicatorsPoint, MarkPriceFullPoint,
MarketWarningPoint,
ObDeltaPoint, ObDeltaIndicatorsPoint,
ObSnapshotPoint, ObSnapshotIndicatorsPoint,
OpenInterestPoint, OpenInterestIndicatorsPoint,
OptionGreeksPoint,
OrderUpdatePoint, OrderbookL3Point, PositionUpdatePoint,
PredictedFundingPoint, PremiumIndexKlinePoint, RiskLimitPoint, SettlementEventPoint,
TakerVolumePoint,
TickerPoint, TickerIndicatorsPoint, TickerFullPoint,
TradePoint, VolatilityIndexPoint,
};
use crate::persistence::PersistDepth;
use crate::derived::{
BasisDerived, DerivedStream, FundingSettlementDerived, TradeToBarDerived,
TradeToRangeBarDerived, TradeToTickBarDerived, TradeToVolumeBarDerived,
TradeToFootprintDerived, interval_to_ms,
};
#[cfg(not(target_arch = "wasm32"))]
use crate::polling;
#[cfg(not(target_arch = "wasm32"))]
use crate::polling::PollSource;
use crate::series::DiskStore;
use crate::series::{DataPoint, Kind, Series, SeriesKey};
use crate::subscription::{Entry, Event, FailedStream, MultiplexRef, Stream};
use crate::{
PersistenceConfig, Result, StationBuilder, StationError, SubscribeReport, SubscriptionHandle,
SubscriptionSet,
};
pub struct Station {
pub(crate) inner: Arc<StationInner>,
}
pub(crate) struct StationInner {
pub(crate) hub: Arc<ExchangeHub>,
pub(crate) storage_root: PathBuf,
pub(crate) persistence: PersistenceConfig,
pub(crate) muxes: DashMap<SeriesKey, Multiplexer>,
pub(crate) series_handles: DashMap<SeriesKey, Arc<dyn Any + Send + Sync>>,
pub(crate) warm_start_capacity: usize,
pub(crate) gap_heal: crate::GapHealConfig,
pub(crate) unsubscribe_grace: std::time::Duration,
pub(crate) orderbook_rest_seed: bool,
pub(crate) orderbook_seed_depth: usize,
pub(crate) connector_tx: broadcast::Sender<crate::subscription::Event>,
pub(crate) exchange_info_cache: DashMap<(ExchangeId, AccountType), Vec<SymbolInfo>>,
}
pub(crate) struct Multiplexer {
pub(crate) tx: broadcast::Sender<Event>,
pub(crate) consumers: Arc<AtomicUsize>,
pub(crate) shutdown: Option<oneshot::Sender<()>>,
pub(crate) grace_cancel: Option<oneshot::Sender<()>>,
}
impl std::fmt::Debug for Station {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("Station")
.field("storage_root", &self.inner.storage_root)
.field("persistence", &self.inner.persistence)
.field("muxes", &self.inner.muxes.len())
.finish()
}
}
impl Station {
pub fn builder() -> StationBuilder { StationBuilder::new() }
pub fn storage_root(&self) -> &std::path::Path { &self.inner.storage_root }
pub fn active_streams(&self) -> usize { self.inner.muxes.len() }
pub fn hub(&self) -> Arc<ExchangeHub> {
self.inner.hub.clone()
}
pub fn series<T: DataPoint + 'static>(&self, key: &SeriesKey)
-> Option<Arc<RwLock<Series<T>>>>
{
let erased = self.inner.series_handles.get(key)?;
erased
.downcast_ref::<Arc<RwLock<Series<T>>>>()
.map(Arc::clone)
}
pub fn register_consumer(
&self,
quota: crate::quota::ConsumerQuota,
) -> crate::quota::ConsumerHandle {
let rest_bucket = quota.max_rest_per_window.map(|cap| {
crate::quota::TokenBucket::new(cap, quota.rest_window)
});
crate::quota::ConsumerHandle {
station: Arc::clone(&self.inner),
quota,
rest_bucket: Arc::new(tokio::sync::Mutex::new(rest_bucket)),
refs: tokio::sync::Mutex::new((0, Vec::new())),
}
}
pub(crate) async fn from_builder(b: StationBuilder) -> Result<Self> {
let _ = digdigdig3::core::install_default_crypto_provider();
#[cfg(not(target_arch = "wasm32"))]
if b.persistence.enabled {
std::fs::create_dir_all(&b.storage_root).map_err(StationError::Io)?;
}
let (connector_tx, _) = broadcast::channel(256);
Ok(Self {
inner: Arc::new(StationInner {
hub: Arc::new(ExchangeHub::new()),
storage_root: b.storage_root,
persistence: b.persistence,
muxes: DashMap::new(),
series_handles: DashMap::new(),
warm_start_capacity: b.warm_start.max(1),
gap_heal: b.gap_heal,
unsubscribe_grace: b.unsubscribe_grace,
orderbook_rest_seed: b.orderbook_rest_seed,
orderbook_seed_depth: b.orderbook_seed_depth,
connector_tx,
exchange_info_cache: DashMap::new(),
}),
})
}
pub fn connector_events(&self) -> broadcast::Receiver<crate::subscription::Event> {
self.inner.connector_tx.subscribe()
}
pub fn symbols(&self, exchange: ExchangeId) -> Vec<SymbolInfo> {
let mut out = Vec::new();
for entry in self.inner.exchange_info_cache.iter() {
if entry.key().0 == exchange {
out.extend_from_slice(entry.value());
}
}
out
}
pub fn ws_health(&self, key: &SeriesKey) -> Option<WsHealth> {
self.inner.muxes.get(key)?;
Some(WsHealth {
connected: true,
rtt_ms: None,
last_message_ms: None,
})
}
pub fn ws_health_for_exchange(
&self,
exchange: ExchangeId,
) -> Option<WsHealth> {
let mut any_connected = false;
let mut rtts: Vec<u64> = Vec::new();
let mut last_msgs: Vec<i64> = Vec::new();
for entry in self.inner.muxes.iter() {
if entry.key().exchange != exchange {
continue;
}
let h = WsHealth {
connected: true,
rtt_ms: None,
last_message_ms: None,
};
any_connected = true;
if let Some(rtt) = h.rtt_ms {
rtts.push(rtt);
}
if let Some(ts) = h.last_message_ms {
last_msgs.push(ts);
}
}
if !any_connected {
return None;
}
let rtt_ms = if rtts.is_empty() {
None
} else {
rtts.sort_unstable();
Some(rtts[rtts.len() / 2])
};
Some(WsHealth {
connected: true,
rtt_ms,
last_message_ms: last_msgs.into_iter().max(),
})
}
pub async fn warmup(&self, exchanges: &[ExchangeId]) -> crate::subscription::WarmupReport {
use crate::subscription::{Event, WarmupReport};
let mut ok = Vec::new();
let mut failed: Vec<(ExchangeId, String)> = Vec::new();
#[cfg(not(target_arch = "wasm32"))]
{
let mut join_set: tokio::task::JoinSet<(ExchangeId, Result<()>)> =
tokio::task::JoinSet::new();
for &eid in exchanges {
if self.inner.hub.is_connected(eid) {
let _ = self.inner.connector_tx.send(Event::ConnectorReady { exchange: eid });
ok.push(eid);
} else {
let hub = Arc::clone(&self.inner.hub);
join_set.spawn(async move {
(eid, hub.connect_public(eid, false).await.map_err(|e| {
crate::StationError::Core(e.to_string())
}))
});
}
}
while let Some(res) = join_set.join_next().await {
match res {
Ok((eid, Ok(()))) => {
let _ = self.inner.connector_tx.send(Event::ConnectorReady { exchange: eid });
ok.push(eid);
}
Ok((eid, Err(e))) => {
tracing::warn!(?eid, ?e, "warmup: connect_public failed");
failed.push((eid, e.to_string()));
}
Err(join_err) => {
tracing::warn!(?join_err, "warmup: task panicked");
}
}
}
}
#[cfg(target_arch = "wasm32")]
{
for &eid in exchanges {
if self.inner.hub.is_connected(eid) {
let _ = self.inner.connector_tx.send(Event::ConnectorReady { exchange: eid });
ok.push(eid);
} else {
match self.inner.hub.connect_public(eid, false).await {
Ok(()) => {
let _ = self.inner.connector_tx.send(Event::ConnectorReady { exchange: eid });
ok.push(eid);
}
Err(e) => {
tracing::warn!(?eid, ?e, "warmup: connect_public failed");
failed.push((eid, e.to_string()));
}
}
}
}
}
const ACCOUNT_TYPES: &[AccountType] = &[AccountType::Spot, AccountType::FuturesCross];
for eid in ok.iter().copied() {
let Some(connector) = self.inner.hub.rest(eid) else {
continue;
};
for &at in ACCOUNT_TYPES {
if let Some(cached) = self.inner.exchange_info_cache.get(&(eid, at)) {
let symbols = cached.value().clone();
let _ = self.inner.connector_tx.send(Event::SymbolsLoaded {
exchange: eid,
account_type: at,
symbols,
});
continue;
}
match connector.get_exchange_info(at).await {
Ok(symbols) if !symbols.is_empty() => {
self.inner.exchange_info_cache.insert((eid, at), symbols.clone());
let _ = self.inner.connector_tx.send(Event::SymbolsLoaded {
exchange: eid,
account_type: at,
symbols,
});
}
Ok(_empty) => {
}
Err(e) => {
use digdigdig3::core::types::ExchangeError;
match &e {
ExchangeError::WireAbsent(_)
| ExchangeError::NotImplemented(_) => {
}
other => {
tracing::warn!(?eid, ?at, ?other, "warmup: get_exchange_info failed");
failed.push((eid, other.to_string()));
}
}
}
}
}
}
WarmupReport { ok, failed }
}
pub async fn subscribe(&self, set: SubscriptionSet) -> Result<SubscribeReport> {
if set.is_empty() {
return Err(StationError::Subscribe("empty SubscriptionSet".into()));
}
let (tx, rx) = mpsc::unbounded_channel::<Event>();
let mut refs: Vec<MultiplexRef> = Vec::new();
let mut ok: Vec<SeriesKey> = Vec::new();
let mut failed: Vec<FailedStream> = Vec::new();
for entry in set.entries {
if let Err(e) = self
.inner
.hub
.connect_public(entry.exchange, false)
.await
{
tracing::debug!(?e, ?entry.exchange, "connect_public failed; warm-start REST backfill will be skipped");
}
let needs_ws = entry.streams.iter().any(|s| {
s.to_kind().is_poll_only().is_none()
});
if needs_ws {
let ws_connect_result = if let Some(ref creds) = entry.credentials {
#[cfg(not(target_arch = "wasm32"))]
{
self.inner
.hub
.connect_websocket_with_credentials(
entry.exchange,
entry.account_type,
creds.clone(),
)
.await
}
#[cfg(target_arch = "wasm32")]
{
let _ = creds;
Err(digdigdig3::core::types::ExchangeError::NotImplemented(
"private WS streams not supported on wasm32".into(),
))
}
} else {
self.inner
.hub
.connect_websocket(entry.exchange, entry.account_type, false)
.await
};
if let Err(e) = ws_connect_result
{
let err_msg = format!("connect_websocket: {e}");
for s in &entry.streams {
if s.to_kind().is_poll_only().is_some() {
continue;
}
failed.push(FailedStream {
exchange: entry.exchange,
account_type: entry.account_type,
symbol: entry.symbol.clone(),
stream: s.clone(),
error: StationError::Core(err_msg.clone()),
});
}
let has_non_ws = entry.streams.iter().any(|s| {
s.to_kind().is_poll_only().is_some()
});
if !has_non_ws {
continue;
}
}
}
let (canonical, raw) = if entry.is_raw {
(
Symbol::with_raw("", "", entry.symbol.clone()),
entry.symbol.clone(),
)
} else {
let canonical = parse_symbol(&entry.symbol);
match SymbolNormalizer::to_exchange(
entry.exchange,
&canonical,
entry.account_type,
) {
Ok(r) => (canonical, r),
Err(e) => {
let err_msg = format!("symbol normalize: {e}");
for s in &entry.streams {
failed.push(FailedStream {
exchange: entry.exchange,
account_type: entry.account_type,
symbol: entry.symbol.clone(),
stream: s.clone(),
error: StationError::Subscribe(err_msg.clone()),
});
}
continue;
}
}
};
let raw = if let Some(rest) = self.inner.hub.rest(entry.exchange) {
rest.resolve_market_symbol(&raw, entry.account_type).await
} else {
raw
};
for s in &entry.streams {
let kind = s.to_kind();
let key = SeriesKey {
exchange: entry.exchange,
account_type: entry.account_type,
symbol: raw.clone(),
kind: kind.clone(),
};
if let Some(caps) = self.inner.hub.capabilities(entry.exchange) {
if caps_explicitly_unsupported(&caps, &kind) {
let reason = format!(
"{:?} capability not declared on {:?} — neither WS nor REST available",
kind, entry.exchange,
);
tracing::debug!(
?key, reason,
"capability fail-fast: skipping acquire_or_spawn"
);
failed.push(FailedStream {
exchange: entry.exchange,
account_type: entry.account_type,
symbol: entry.symbol.clone(),
stream: s.clone(),
error: StationError::StreamNotSupported(reason),
});
continue;
}
}
let (bcast_tx, pending_seed) = match self
.acquire_or_spawn(&key, &entry, &canonical, &raw, s)
.await
{
Ok(pair) => pair,
Err(e) => {
if e.is_not_supported() {
tracing::debug!(?key, ?e, "stream not supported; skipping");
} else {
tracing::info!(?key, ?e, "subscribe failed; skipping");
}
failed.push(FailedStream {
exchange: entry.exchange,
account_type: entry.account_type,
symbol: entry.symbol.clone(),
stream: s.clone(),
error: e,
});
continue;
}
};
let mut bcast_rx = bcast_tx.subscribe();
let tx_clone = tx.clone();
let label = entry.symbol.clone();
{
let relay_fut = Box::pin(async move {
while let Ok(mut ev) = bcast_rx.recv().await {
ev.set_symbol(label.clone());
if tx_clone.send(ev).is_err() {
break;
}
}
});
#[cfg(not(target_arch = "wasm32"))]
tokio::spawn(relay_fut);
#[cfg(target_arch = "wasm32")]
wasm_bindgen_futures::spawn_local(relay_fut);
}
if let Some(seed_ev) = pending_seed {
if bcast_tx.send(seed_ev).is_err() {
tracing::debug!(
target: "dig3::ob_seed",
?key,
"ob delta seed: send failed after relay wired (unexpected)"
);
}
}
refs.push(MultiplexRef {
station: Arc::downgrade(&self.inner),
key: key.clone(),
});
ok.push(key);
}
}
Ok(SubscribeReport {
handle: SubscriptionHandle { rx, _refs: refs },
ok,
failed,
})
}
async fn acquire_or_spawn(
&self,
key: &SeriesKey,
entry: &Entry,
canonical: &Symbol,
raw_symbol: &str,
stream: &Stream,
) -> Result<(broadcast::Sender<Event>, Option<Event>)> {
if let Some(mut mux) = self.inner.muxes.get_mut(key) {
if let Some(cancel) = mux.grace_cancel.take() {
let _ = cancel.send(());
}
mux.consumers.fetch_add(1, Ordering::SeqCst);
return Ok((mux.tx.clone(), None));
}
if key.kind.is_derived() {
#[cfg(not(target_arch = "wasm32"))]
{
if let Kind::Basis = &key.kind {
let hub = &self.inner.hub;
if let Some(source) = polling::basis_poll_source(hub, key.exchange) {
tracing::info!(
target: "dig3::station::native_source",
exchange = ?key.exchange,
symbol = %key.symbol,
"Kind::Basis: native REST basis-history available — using poll path"
);
let poll_spec = crate::series::PollSpec {
cadence: source.cadence(),
jitter_pct: 10,
};
return self
.acquire_or_spawn_polled_with_source::<BasisPoint, _>(
key, poll_spec, raw_symbol, source,
)
.await
.map(|tx| (tx, None));
} else {
tracing::info!(
target: "dig3::station::native_source",
exchange = ?key.exchange,
symbol = %key.symbol,
"Kind::Basis: no native source — deriving from mark+index in RAM"
);
}
}
if let Kind::FundingSettlement = &key.kind {
let hub = &self.inner.hub;
if let Some(source) = polling::funding_poll_source(hub, key.exchange) {
tracing::info!(
target: "dig3::station::native_source",
exchange = ?key.exchange,
symbol = %key.symbol,
"Kind::FundingSettlement: native REST funding-rate history available — using poll path"
);
let poll_spec = crate::series::PollSpec {
cadence: source.cadence(),
jitter_pct: 10,
};
return self
.acquire_or_spawn_polled_with_source::<FundingSettlementPoint, _>(
key, poll_spec, raw_symbol, source,
)
.await
.map(|tx| (tx, None));
} else {
tracing::info!(
target: "dig3::station::native_source",
exchange = ?key.exchange,
symbol = %key.symbol,
"Kind::FundingSettlement: no native source — deriving from funding-rate flips in RAM"
);
}
}
}
return match &key.kind {
Kind::Basis => {
self.acquire_or_spawn_derived::<BasisDerived>(key, entry, canonical, raw_symbol).await.map(|tx| (tx, None))
}
Kind::FundingSettlement => {
self.acquire_or_spawn_derived::<FundingSettlementDerived>(key, entry, canonical, raw_symbol).await.map(|tx| (tx, None))
}
Kind::RangeBar(_) => {
self.acquire_or_spawn_derived::<TradeToRangeBarDerived>(key, entry, canonical, raw_symbol).await.map(|tx| (tx, None))
}
Kind::TickBar(_) => {
self.acquire_or_spawn_derived::<TradeToTickBarDerived>(key, entry, canonical, raw_symbol).await.map(|tx| (tx, None))
}
Kind::VolumeBar(_) => {
self.acquire_or_spawn_derived::<TradeToVolumeBarDerived>(key, entry, canonical, raw_symbol).await.map(|tx| (tx, None))
}
Kind::Footprint(_) => {
self.acquire_or_spawn_derived::<TradeToFootprintDerived>(key, entry, canonical, raw_symbol).await.map(|tx| (tx, None))
}
_ => unreachable!("is_derived() returned true for unhandled kind — update acquire_or_spawn dispatch"),
};
}
#[cfg(not(target_arch = "wasm32"))]
if let Some(poll_spec) = key.kind.is_poll_only() {
return self.acquire_or_spawn_polled(key, entry, poll_spec, raw_symbol).await.map(|tx| (tx, None));
}
#[cfg(target_arch = "wasm32")]
if key.kind.is_poll_only().is_some() {
return Err(StationError::StreamNotSupported(format!(
"poll-only streams not supported on wasm32 ({:?})",
key.kind
)));
}
let sym = Symbol::with_raw(&canonical.base, &canonical.quote, raw_symbol.to_string());
let req = ws_request_for(&key.kind, sym, entry.account_type);
let ws = self
.inner
.hub
.ws(entry.exchange, entry.account_type)
.ok_or_else(|| StationError::Core("ws handle missing post-connect".into()))?;
if let Err(e) = ws.subscribe(req.clone()).await {
use digdigdig3::core::types::WebSocketError;
let is_not_supported = matches!(
e,
WebSocketError::WireAbsent(_) | WebSocketError::NotImplemented(_)
);
if is_not_supported {
if let Kind::Kline(iv) = &key.kind {
if interval_to_ms(iv.as_str()).is_none() {
return Err(StationError::StreamNotSupported(format!(
"Kline interval {:?} is unknown — cannot aggregate from trades",
iv.as_str()
)));
}
tracing::debug!(
target: "dig3::station::derived",
exchange = ?key.exchange,
symbol = %key.symbol,
interval = %iv,
"native Kline WS not supported — falling back to TradeToBarDerived"
);
return self
.acquire_or_spawn_derived::<TradeToBarDerived>(key, entry, canonical, raw_symbol)
.await
.map(|tx| (tx, None));
}
}
return Err(match e {
WebSocketError::WireAbsent(msg)
| WebSocketError::NotImplemented(msg) => {
StationError::StreamNotSupported(msg)
}
other => StationError::Subscribe(format!("ws.subscribe: {other}")),
});
}
let (bcast_tx, _) = broadcast::channel::<Event>(1024);
let consumers = Arc::new(AtomicUsize::new(1));
let (shutdown_tx, shutdown_rx) = oneshot::channel::<()>();
let _ = stream;
let warm_n = self.inner.warm_start_capacity;
let hub = self.inner.hub.clone();
let acct = entry.account_type;
let raw_s = raw_symbol.to_string();
let mut pending_seed: Option<Event> = None;
match &key.kind {
Kind::Trade => {
let seed = if warm_n > 0 {
crate::backfill::trades_recent(&hub, key.exchange, acct, &raw_s, warm_n).await
} else { Vec::new() };
spawn_forwarder::<TradePoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), seed, req.clone());
}
Kind::Kline(interval) => {
let seed = if warm_n > 0 {
crate::backfill::klines_recent(&hub, key.exchange, acct, &raw_s, interval.as_str(), warm_n).await
} else { Vec::new() };
spawn_forwarder::<BarPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), seed, req.clone());
}
Kind::AggTrade => {
let seed = if warm_n > 0 {
crate::backfill::agg_trades_recent(&hub, key.exchange, acct, &raw_s, warm_n).await
} else { Vec::new() };
spawn_forwarder::<AggTradePoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), seed, req.clone());
}
Kind::Ticker => match self.inner.persistence.depth_for(&key.kind) {
Some(PersistDepth::Indicators) => {
let seed = if warm_n > 0 {
crate::backfill::tickers_indicators_recent(&hub, key.exchange, acct, &raw_s, warm_n).await
} else { Vec::new() };
spawn_forwarder::<TickerIndicatorsPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), seed, req.clone());
}
Some(PersistDepth::Full) => {
let seed = if warm_n > 0 {
crate::backfill::tickers_full_recent(&hub, key.exchange, acct, &raw_s, warm_n).await
} else { Vec::new() };
spawn_forwarder::<TickerFullPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), seed, req.clone());
}
_ => spawn_forwarder::<TickerPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
},
Kind::Orderbook => {
let ob_seed = if self.inner.orderbook_rest_seed {
ob_rest_seed(&hub, key.exchange, acct, &raw_s, self.inner.orderbook_seed_depth).await
} else {
Vec::new()
};
match self.inner.persistence.depth_for(&key.kind) {
Some(PersistDepth::Indicators) | Some(PersistDepth::Full) => {
spawn_forwarder::<ObSnapshotIndicatorsPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone());
}
_ => spawn_forwarder::<ObSnapshotPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), ob_seed, req.clone()),
}
}
Kind::OrderbookDelta => {
pending_seed = if self.inner.orderbook_rest_seed {
#[cfg(not(target_arch = "wasm32"))]
{
let snapshots = ob_rest_seed(&hub, key.exchange, acct, &raw_s, self.inner.orderbook_seed_depth).await;
snapshots.into_iter().next().map(|point| Event::OrderbookSnapshot {
exchange: key.exchange,
symbol: raw_s.clone(),
point,
})
}
#[cfg(target_arch = "wasm32")]
{
tracing::warn!(
target: "dig3::ob_seed",
exchange = ?key.exchange, symbol = raw_s.as_str(),
"orderbook REST seed for delta stream skipped on wasm32 (possible CORS) — continuing WS-only"
);
None
}
} else {
None
};
match self.inner.persistence.depth_for(&key.kind) {
Some(PersistDepth::Indicators) | Some(PersistDepth::Full) =>
spawn_forwarder::<ObDeltaIndicatorsPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
_ => spawn_forwarder::<ObDeltaPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
}
}
Kind::MarkPrice => match self.inner.persistence.depth_for(&key.kind) {
Some(PersistDepth::Indicators) => {
let seed = if warm_n > 0 {
crate::backfill::mark_price_indicators_recent(&hub, key.exchange, acct, &raw_s, warm_n).await
} else { Vec::new() };
spawn_forwarder::<MarkPriceIndicatorsPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), seed, req.clone());
}
Some(PersistDepth::Full) => {
let seed = if warm_n > 0 {
crate::backfill::mark_price_full_recent(&hub, key.exchange, acct, &raw_s, warm_n).await
} else { Vec::new() };
spawn_forwarder::<MarkPriceFullPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), seed, req.clone());
}
_ => {
let seed = if warm_n > 0 {
crate::backfill::mark_price_recent(&hub, key.exchange, acct, &raw_s, warm_n).await
} else { Vec::new() };
spawn_forwarder::<MarkPricePoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), seed, req.clone());
}
},
Kind::FundingRate => match self.inner.persistence.depth_for(&key.kind) {
Some(PersistDepth::Indicators) => {
let seed = if warm_n > 0 {
crate::backfill::funding_rate_indicators_recent(&hub, key.exchange, acct, &raw_s, warm_n).await
} else { Vec::new() };
spawn_forwarder::<FundingRateIndicatorsPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), seed, req.clone());
}
Some(PersistDepth::Full) => {
let seed = if warm_n > 0 {
crate::backfill::funding_rate_full_recent(&hub, key.exchange, acct, &raw_s, warm_n).await
} else { Vec::new() };
spawn_forwarder::<FundingRateFullPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), seed, req.clone());
}
_ => {
let seed = if warm_n > 0 {
crate::backfill::funding_rate_recent(&hub, key.exchange, acct, &raw_s, warm_n).await
} else { Vec::new() };
spawn_forwarder::<FundingRatePoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), seed, req.clone());
}
},
Kind::OpenInterest => match self.inner.persistence.depth_for(&key.kind) {
Some(PersistDepth::Indicators) | Some(PersistDepth::Full) => {
let seed = if warm_n > 0 {
crate::backfill::open_interest_indicators_recent(&hub, key.exchange, acct, &raw_s, warm_n).await
} else { Vec::new() };
spawn_forwarder::<OpenInterestIndicatorsPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), seed, req.clone());
}
_ => {
let seed = if warm_n > 0 {
crate::backfill::open_interest_recent(&hub, key.exchange, acct, &raw_s, warm_n).await
} else { Vec::new() };
spawn_forwarder::<OpenInterestPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), seed, req.clone());
}
},
Kind::Liquidation => match self.inner.persistence.depth_for(&key.kind) {
Some(PersistDepth::Indicators) => {
let seed = if warm_n > 0 {
crate::backfill::liquidation_indicators_recent(&hub, key.exchange, acct, &raw_s, warm_n).await
} else { Vec::new() };
spawn_forwarder::<LiquidationIndicatorsPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), seed, req.clone());
}
Some(PersistDepth::Full) => {
let seed = if warm_n > 0 {
crate::backfill::liquidation_full_recent(&hub, key.exchange, acct, &raw_s, warm_n).await
} else { Vec::new() };
spawn_forwarder::<LiquidationFullPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), seed, req.clone());
}
_ => {
let seed = if warm_n > 0 {
crate::backfill::liquidations_recent(&hub, key.exchange, acct, &raw_s, warm_n).await
} else { Vec::new() };
spawn_forwarder::<LiquidationPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), seed, req.clone());
}
},
Kind::BlockTrade => spawn_forwarder::<BlockTradePoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
Kind::AuctionEvent => spawn_forwarder::<AuctionEventPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
Kind::IndexPrice => match self.inner.persistence.depth_for(&key.kind) {
Some(PersistDepth::Indicators) | Some(PersistDepth::Full) => spawn_forwarder::<IndexPriceIndicatorsPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
_ => spawn_forwarder::<IndexPricePoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
},
Kind::CompositeIndex => spawn_forwarder::<CompositeIndexPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
Kind::OptionGreeks => spawn_forwarder::<OptionGreeksPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
Kind::VolatilityIndex => spawn_forwarder::<VolatilityIndexPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
Kind::HistoricalVolatility => spawn_forwarder::<HistoricalVolatilityPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
Kind::LongShortRatio => spawn_forwarder::<LongShortRatioPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
Kind::TakerVolume => spawn_forwarder::<TakerVolumePoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
Kind::LiquidationBucket => spawn_forwarder::<LiquidationBucketPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
Kind::Basis => spawn_forwarder::<BasisPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
Kind::InsuranceFund => {
let seed = if warm_n > 0 {
crate::backfill::insurance_fund_recent(&hub, key.exchange, acct, &raw_s, warm_n).await
} else { Vec::new() };
spawn_forwarder::<InsuranceFundPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), seed, req.clone());
}
Kind::OrderbookL3 => spawn_forwarder::<OrderbookL3Point>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
Kind::SettlementEvent => spawn_forwarder::<SettlementEventPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
Kind::MarketWarning => spawn_forwarder::<MarketWarningPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
Kind::RiskLimit => spawn_forwarder::<RiskLimitPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
Kind::PredictedFunding => spawn_forwarder::<PredictedFundingPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
Kind::FundingSettlement => spawn_forwarder::<FundingSettlementPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
Kind::MarkPriceKline(iv) => {
let seed = if warm_n > 0 {
crate::backfill::mark_price_klines_recent(&hub, key.exchange, acct, &raw_s, iv.as_str(), warm_n).await
} else { Vec::new() };
spawn_forwarder::<MarkPriceKlinePoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), seed, req.clone());
}
Kind::IndexPriceKline(iv) => {
let seed = if warm_n > 0 {
crate::backfill::index_price_klines_recent(&hub, key.exchange, acct, &raw_s, iv.as_str(), warm_n).await
} else { Vec::new() };
spawn_forwarder::<IndexPriceKlinePoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), seed, req.clone());
}
Kind::PremiumIndexKline(iv) => {
let seed = if warm_n > 0 {
crate::backfill::premium_index_klines_recent(&hub, key.exchange, acct, &raw_s, iv.as_str(), warm_n).await
} else { Vec::new() };
spawn_forwarder::<PremiumIndexKlinePoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), seed, req.clone());
}
Kind::OrderUpdate => spawn_forwarder::<OrderUpdatePoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
Kind::BalanceUpdate => spawn_forwarder::<BalanceUpdatePoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
Kind::PositionUpdate => spawn_forwarder::<PositionUpdatePoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req),
Kind::RangeBar(_) | Kind::TickBar(_) | Kind::VolumeBar(_) | Kind::Footprint(_) => {
unreachable!("derived kinds dispatched before forwarder match")
}
}
self.inner.muxes.insert(
key.clone(),
Multiplexer { tx: bcast_tx.clone(), consumers, shutdown: Some(shutdown_tx), grace_cancel: None },
);
Ok((bcast_tx, pending_seed))
}
}
impl Station {
#[cfg(not(target_arch = "wasm32"))]
fn acquire_or_spawn_derived<'a, D: DerivedStream>(
&'a self,
key: &'a SeriesKey,
entry: &'a Entry,
canonical: &'a digdigdig3::core::types::Symbol,
raw_symbol: &'a str,
) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<broadcast::Sender<Event>>> + Send + 'a>>
where
Event: EventFrom<D::Output>,
{
Box::pin(async move {
self.acquire_or_spawn_derived_body::<D>(key, entry, canonical, raw_symbol).await
})
}
#[cfg(target_arch = "wasm32")]
fn acquire_or_spawn_derived<'a, D: DerivedStream>(
&'a self,
key: &'a SeriesKey,
entry: &'a Entry,
canonical: &'a digdigdig3::core::types::Symbol,
raw_symbol: &'a str,
) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<broadcast::Sender<Event>>> + 'a>>
where
Event: EventFrom<D::Output>,
{
Box::pin(async move {
self.acquire_or_spawn_derived_body::<D>(key, entry, canonical, raw_symbol).await
})
}
async fn acquire_or_spawn_derived_body<D: DerivedStream>(
&self,
key: &SeriesKey,
entry: &Entry,
canonical: &digdigdig3::core::types::Symbol,
raw_symbol: &str,
) -> Result<broadcast::Sender<Event>>
where
Event: EventFrom<D::Output>,
{
let mut upstream_rxs: Vec<broadcast::Receiver<Event>> = Vec::new();
let mut upstream_keys: Vec<SeriesKey> = Vec::new();
for dep_stream in D::deps() {
let dep_kind = dep_stream.to_kind();
debug_assert!(
!dep_kind.is_derived(),
"DerivedStream::deps() must not list derived kinds (no derived-of-derived)"
);
let dep_key = SeriesKey {
exchange: key.exchange,
account_type: key.account_type,
symbol: raw_symbol.to_string(),
kind: dep_kind,
};
let (up_tx, _) = self
.acquire_or_spawn(&dep_key, entry, canonical, raw_symbol, dep_stream)
.await?;
upstream_rxs.push(up_tx.subscribe());
upstream_keys.push(dep_key);
}
let warm_n = self.inner.warm_start_capacity;
let mut agg_seed_per_dep: Vec<Vec<Event>> = Vec::with_capacity(D::deps().len());
for dep_stream in D::deps() {
let dep_kind = dep_stream.to_kind();
let mut seed_events: Vec<Event> = Vec::new();
if warm_n > 0 && matches!(dep_kind, Kind::Trade) {
let caps_opt = self.inner.hub.capabilities(key.exchange);
let use_agg = caps_opt.as_ref().map(|c| c.has_agg_trades).unwrap_or(false);
if use_agg {
let agg = crate::backfill::agg_trades_recent(
&self.inner.hub, key.exchange, entry.account_type, raw_symbol, warm_n,
).await;
for ap in agg {
seed_events.push(Event::Trade {
exchange: key.exchange,
symbol: raw_symbol.to_string(),
point: TradePoint {
ts_ms: ap.ts_ms,
price: ap.price,
quantity: ap.quantity,
side: ap.side,
trade_id_hash: 0,
},
});
}
} else {
let trades = crate::backfill::trades_recent(
&self.inner.hub, key.exchange, entry.account_type, raw_symbol, warm_n,
).await;
for pt in trades {
seed_events.push(Event::Trade {
exchange: key.exchange,
symbol: raw_symbol.to_string(),
point: pt,
});
}
}
} else if warm_n > 0 && matches!(dep_kind, Kind::MarkPrice) {
let pts = crate::backfill::mark_price_recent(
&self.inner.hub, key.exchange, entry.account_type, raw_symbol, warm_n,
).await;
for pt in pts {
seed_events.push(Event::MarkPrice {
exchange: key.exchange,
symbol: raw_symbol.to_string(),
point: pt,
});
}
} else if warm_n > 0 && matches!(dep_kind, Kind::IndexPrice) {
let pts = crate::backfill::mark_price_recent(
&self.inner.hub, key.exchange, entry.account_type, raw_symbol, warm_n,
).await;
for pt in pts {
if pt.index.is_finite() {
seed_events.push(Event::IndexPrice {
exchange: key.exchange,
symbol: raw_symbol.to_string(),
point: IndexPricePoint { ts_ms: pt.ts_ms, price: pt.index },
});
}
}
}
agg_seed_per_dep.push(seed_events);
}
let (bcast_tx, _) = broadcast::channel::<Event>(512);
let consumers = Arc::new(AtomicUsize::new(1));
let (shutdown_tx, shutdown_rx) = oneshot::channel::<()>();
spawn_derived_forwarder::<D>(
self,
key,
upstream_rxs,
upstream_keys,
bcast_tx.clone(),
shutdown_rx,
raw_symbol.to_string(),
agg_seed_per_dep,
);
self.inner.muxes.insert(
key.clone(),
Multiplexer { tx: bcast_tx.clone(), consumers, shutdown: Some(shutdown_tx), grace_cancel: None },
);
Ok(bcast_tx)
}
}
#[cfg(not(target_arch = "wasm32"))]
impl Station {
async fn acquire_or_spawn_polled(
&self,
key: &SeriesKey,
entry: &Entry,
poll_spec: crate::series::PollSpec,
raw_symbol: &str,
) -> Result<broadcast::Sender<Event>> {
use crate::station::Multiplexer;
let (bcast_tx, _) = broadcast::channel::<Event>(1024);
let consumers = Arc::new(AtomicUsize::new(1));
let (shutdown_tx, shutdown_rx) = oneshot::channel::<()>();
let label = raw_symbol.to_string();
match &key.kind {
Kind::LongShortRatio => {
let source = polling::lsr_poll_source(entry.exchange)
.ok_or_else(|| StationError::StreamNotSupported(format!(
"LongShortRatio REST polling not supported for {:?}",
entry.exchange
)))?;
polling::spawn_poller::<LongShortRatioPoint, _>(
self, key, source, poll_spec, bcast_tx.clone(), shutdown_rx, label,
);
}
Kind::HistoricalVolatility => {
let source = polling::hv_poll_source(entry.exchange)
.ok_or_else(|| StationError::StreamNotSupported(format!(
"HistoricalVolatility REST polling not supported for {:?} \
(Deribit only)",
entry.exchange
)))?;
polling::spawn_poller::<HistoricalVolatilityPoint, _>(
self, key, source, poll_spec, bcast_tx.clone(), shutdown_rx, label,
);
}
Kind::TakerVolume => {
let source = polling::taker_volume_poll_source(&self.inner.hub, entry.exchange)
.ok_or_else(|| StationError::StreamNotSupported(format!(
"TakerVolume REST polling not supported for {:?}",
entry.exchange
)))?;
polling::spawn_poller::<TakerVolumePoint, _>(
self, key, source, poll_spec, bcast_tx.clone(), shutdown_rx, label,
);
}
Kind::LiquidationBucket => {
let source = polling::liquidation_bucket_poll_source(&self.inner.hub, entry.exchange)
.ok_or_else(|| StationError::StreamNotSupported(format!(
"LiquidationBucket REST polling not supported for {:?}",
entry.exchange
)))?;
polling::spawn_poller::<LiquidationBucketPoint, _>(
self, key, source, poll_spec, bcast_tx.clone(), shutdown_rx, label,
);
}
other => {
return Err(StationError::StreamNotSupported(format!(
"acquire_or_spawn_polled: no poll source for {:?}",
other
)));
}
}
self.inner.muxes.insert(
key.clone(),
Multiplexer {
tx: bcast_tx.clone(),
consumers,
shutdown: Some(shutdown_tx),
grace_cancel: None,
},
);
Ok(bcast_tx)
}
async fn acquire_or_spawn_polled_with_source<T, S>(
&self,
key: &SeriesKey,
poll_spec: crate::series::PollSpec,
raw_symbol: &str,
source: S,
) -> Result<broadcast::Sender<Event>>
where
T: crate::series::DataPoint + 'static,
S: crate::polling::PollSource<T>,
Event: EventFrom<T>,
{
use crate::station::Multiplexer;
let (bcast_tx, _) = broadcast::channel::<Event>(1024);
let consumers = Arc::new(AtomicUsize::new(1));
let (shutdown_tx, shutdown_rx) = oneshot::channel::<()>();
let label = raw_symbol.to_string();
polling::spawn_poller::<T, S>(
self, key, source, poll_spec, bcast_tx.clone(), shutdown_rx, label,
);
self.inner.muxes.insert(
key.clone(),
Multiplexer {
tx: bcast_tx.clone(),
consumers,
shutdown: Some(shutdown_tx),
grace_cancel: None,
},
);
Ok(bcast_tx)
}
}
impl StationInner {
pub(crate) fn release_consumer(self: &Arc<Self>, key: &SeriesKey) {
let (became_zero, grace) = {
let Some(mux) = self.muxes.get(key) else { return; };
let prev = mux.consumers.fetch_sub(1, Ordering::SeqCst);
(prev <= 1, self.unsubscribe_grace)
};
if !became_zero {
return;
}
if grace.is_zero() {
if let Some((_, mut mux)) = self.muxes.remove(key) {
if let Some(tx) = mux.shutdown.take() {
let _ = tx.send(());
}
}
return;
}
let (cancel_tx, cancel_rx) = oneshot::channel::<()>();
{
let Some(mut mux) = self.muxes.get_mut(key) else { return; };
mux.grace_cancel = Some(cancel_tx);
}
let inner = Arc::clone(self);
let key = key.clone();
let grace_fut = Box::pin(async move {
#[cfg(not(target_arch = "wasm32"))]
let timed_out = tokio::select! {
_ = cancel_rx => false,
_ = tokio::time::sleep(grace) => true,
};
#[cfg(target_arch = "wasm32")]
let timed_out = tokio::select! {
_ = cancel_rx => false,
_ = gloo_timers::future::sleep(grace) => true,
};
if timed_out {
let still_zero = inner
.muxes
.get(&key)
.map(|m| m.consumers.load(Ordering::SeqCst) == 0)
.unwrap_or(false);
if still_zero {
if let Some((_, mut mux)) = inner.muxes.remove(&key) {
if let Some(tx) = mux.shutdown.take() {
let _ = tx.send(());
}
}
inner.series_handles.remove(&key);
}
}
});
#[cfg(not(target_arch = "wasm32"))]
tokio::spawn(grace_fut);
#[cfg(target_arch = "wasm32")]
wasm_bindgen_futures::spawn_local(grace_fut);
}
}
fn spawn_derived_forwarder<D: DerivedStream + 'static>(
station: &Station,
key: &SeriesKey,
upstream_rxs: Vec<broadcast::Receiver<Event>>,
upstream_keys: Vec<SeriesKey>,
bcast_tx: broadcast::Sender<Event>,
mut shutdown_rx: oneshot::Receiver<()>,
symbol_label: String,
agg_seed_per_dep: Vec<Vec<Event>>,
) where
Event: EventFrom<D::Output>,
{
let inner = station.inner.clone();
let key = key.clone();
let storage_root = inner.storage_root.clone();
let persistence = inner.persistence.clone();
let warm = inner.warm_start_capacity;
let exchange = key.exchange;
{
let derived_fut = Box::pin(async move {
#[cfg(not(target_arch = "wasm32"))]
let mut disk: Option<DiskStore<D::Output>> = None;
#[cfg(not(target_arch = "wasm32"))]
if persistence.is_enabled_for(&key.kind) {
match DiskStore::<D::Output>::new(&storage_root, key.clone()).await {
Ok(store) => disk = Some(store),
Err(e) => tracing::warn!(?e, ?key, "derived: disk store open failed"),
}
}
#[cfg(target_arch = "wasm32")]
let _ = (&storage_root, &persistence);
let mut series = Series::<D::Output>::new(warm);
#[cfg(not(target_arch = "wasm32"))]
if let Some(d) = disk.as_ref() {
match d.read_tail(warm).await {
Ok(tail) => {
for point in &tail {
series.upsert_by_ts(point.clone());
let _ = bcast_tx.send(Event::from_point(
exchange,
key.account_type,
&symbol_label,
&key.kind,
point.clone(),
));
}
}
Err(e) => tracing::debug!(?e, ?key, "derived: disk warm-seed read_tail failed"),
}
}
let mut state = D::new_for_key(&key);
for (dep_idx, seed_events) in agg_seed_per_dep.iter().enumerate() {
if seed_events.is_empty() {
continue;
}
let emitted = state.seed_from_events(seed_events, dep_idx);
for point in emitted {
#[cfg(not(target_arch = "wasm32"))]
if let Some(d) = disk.as_mut() {
if let Err(e) = d.append(&point) {
tracing::warn!(?e, ?key, "derived agg-seed: disk append failed");
}
}
series.upsert_by_ts(point.clone());
let _ = bcast_tx.send(Event::from_point(
exchange,
key.account_type,
&symbol_label,
&key.kind,
point,
));
}
}
if D::deps() == [Stream::Trade] {
if let Some(dep_key) = upstream_keys.first() {
let trade_handle = inner
.series_handles
.get(dep_key)
.and_then(|e| {
e.downcast_ref::<Arc<RwLock<Series<TradePoint>>>>().map(Arc::clone)
});
if let Some(handle) = trade_handle {
let buffered = handle.read().await.snapshot(); for pt in buffered {
let ev = Event::Trade {
exchange,
symbol: symbol_label.clone(),
point: pt,
};
if let Some(point) = state.on_upstream_event(&ev, 0) {
#[cfg(not(target_arch = "wasm32"))]
if let Some(d) = disk.as_mut() {
if let Err(e) = d.append(&point) {
tracing::warn!(?e, ?key, "derived seed: disk append failed");
}
}
series.upsert_by_ts(point.clone());
let _ = bcast_tx.send(Event::from_point(
exchange,
key.account_type,
&symbol_label,
&key.kind,
point,
));
}
}
}
}
}
let tagged: Vec<_> = upstream_rxs
.into_iter()
.enumerate()
.map(|(idx, rx)| {
tokio_stream::wrappers::BroadcastStream::new(rx)
.filter_map(move |res| async move {
match res {
Ok(ev) => Some((idx, ev)),
Err(tokio_stream::wrappers::errors::BroadcastStreamRecvError::Lagged(n)) => {
tracing::warn!(dep_idx = idx, lagged = n, "derived: upstream lagged — events dropped");
None
}
}
})
.boxed()
})
.collect();
let mut merged = futures_util::stream::select_all(tagged);
loop {
let item_opt = tokio::select! {
_ = &mut shutdown_rx => break,
item = merged.next() => item,
};
let Some((dep_idx, ev)) = item_opt else {
tracing::info!(?key, "derived: all upstreams closed — exiting");
break;
};
if let Some(point) = state.on_upstream_event(&ev, dep_idx) {
#[cfg(not(target_arch = "wasm32"))]
if let Some(d) = disk.as_mut() {
if let Err(e) = d.append(&point) {
tracing::warn!(?e, "derived: disk store append failed");
}
}
series.push(point.clone());
let _ = bcast_tx.send(Event::from_point(exchange, key.account_type, &symbol_label, &key.kind, point));
}
}
#[cfg(not(target_arch = "wasm32"))]
if let Some(mut d) = disk { let _ = d.flush().await; }
let _ = series;
for up_key in &upstream_keys {
inner.release_consumer(up_key);
}
let still_consumers = inner
.muxes
.get(&key)
.map(|m| m.consumers.load(Ordering::SeqCst))
.unwrap_or(0);
if still_consumers == 0 {
inner.muxes.remove(&key);
}
});
#[cfg(not(target_arch = "wasm32"))]
tokio::spawn(derived_fut);
#[cfg(target_arch = "wasm32")]
wasm_bindgen_futures::spawn_local(derived_fut);
}
}
fn spawn_forwarder<T: DataPoint + 'static>(
station: &Station,
key: &SeriesKey,
ws: Arc<dyn digdigdig3::core::traits::WebSocketConnector>,
bcast_tx: broadcast::Sender<Event>,
mut shutdown_rx: oneshot::Receiver<()>,
symbol_label: String,
rest_seed: Vec<T>,
sub_req: SubscriptionRequest,
) where
Event: EventFrom<T>,
{
let inner = station.inner.clone();
let key = key.clone();
let storage_root = inner.storage_root.clone();
let persistence = inner.persistence.clone();
let warm = inner.warm_start_capacity;
let exchange = key.exchange;
let gap_cfg = inner.gap_heal;
let hub_for_heal = inner.hub.clone();
let shared_series: Arc<RwLock<Series<T>>> = Arc::new(RwLock::new(Series::new(warm)));
let erased: Arc<dyn Any + Send + Sync> = Arc::new(Arc::clone(&shared_series));
inner.series_handles.insert(key.clone(), erased);
{
let forwarder_fut = Box::pin(async move {
#[cfg(not(target_arch = "wasm32"))]
let mut disk: Option<DiskStore<T>> = None;
#[cfg(not(target_arch = "wasm32"))]
if persistence.is_enabled_for(&key.kind) {
match DiskStore::<T>::new(&storage_root, key.clone()).await {
Ok(store) => disk = Some(store),
Err(e) => tracing::warn!(?e, ?key, "disk store open failed"),
}
}
#[cfg(target_arch = "wasm32")]
let mut disk: Option<DiskStore<T>> = None;
#[cfg(target_arch = "wasm32")]
if persistence.is_enabled_for(&key.kind) {
match DiskStore::<T>::new(key.clone()).await {
Ok(store) => disk = Some(store),
Err(e) => tracing::warn!(?e, ?key, "wasm OPFS disk store open failed"),
}
}
#[cfg(target_arch = "wasm32")]
let _ = &storage_root;
let mut last_emitted_ms: i64 = 0;
if warm > 0 {
let disk_tail: Vec<T> = if let Some(d) = disk.as_ref() {
d.read_tail(warm).await.unwrap_or_default()
} else {
Vec::new()
};
let seed_points: Vec<T> = if disk_tail.is_empty() && !rest_seed.is_empty() {
rest_seed
} else {
disk_tail
};
for p in &seed_points {
let _ = bcast_tx.send(Event::from_point(exchange, key.account_type, &symbol_label, &key.kind, p.clone()));
last_emitted_ms = last_emitted_ms.max(p.timestamp_ms());
}
shared_series.write().await.seed(seed_points);
}
let mut stream = ws.event_stream();
#[cfg(not(target_arch = "wasm32"))]
let silence_timeout = std::time::Duration::from_secs(
std::env::var("DIG3_WS_SILENCE_SECS").ok().and_then(|s| s.parse().ok()).unwrap_or(60),
);
#[cfg(target_arch = "wasm32")]
let silence_timeout_ms: u32 = 60_000;
#[cfg(not(target_arch = "wasm32"))]
let debug_slow_ms: u64 = std::env::var("DIG3_DEBUG_SLOW_CONSUMER_MS")
.ok()
.and_then(|s| s.parse().ok())
.unwrap_or(0);
#[cfg(target_arch = "wasm32")]
let mut wasm_flush_counter: u32 = 0;
#[cfg(target_arch = "wasm32")]
const WASM_FLUSH_EVERY: u32 = 64;
loop {
#[cfg(not(target_arch = "wasm32"))]
let item_opt = tokio::select! {
_ = &mut shutdown_rx => break,
res = tokio::time::timeout(silence_timeout, stream.next()) => res,
};
#[cfg(target_arch = "wasm32")]
let item_opt: std::result::Result<
Option<std::result::Result<_, digdigdig3::core::types::WebSocketError>>,
(),
> = tokio::select! {
_ = &mut shutdown_rx => break,
_ = gloo_timers::future::sleep(std::time::Duration::from_millis(silence_timeout_ms as u64)) => Err(()),
item = stream.next() => Ok(item),
};
let trigger_heal_reason: Option<&'static str> = match &item_opt {
Err(_) => Some("silence_timeout"),
Ok(None) => Some("stream_ended"),
Ok(Some(Err(_))) => Some("stream_err"),
Ok(Some(Ok(_))) => None,
};
if let Some(reason) = trigger_heal_reason {
let is_kline_family = matches!(
&key.kind,
Kind::Kline(_) | Kind::MarkPriceKline(_)
| Kind::IndexPriceKline(_) | Kind::PremiumIndexKline(_)
);
if !is_kline_family {
tracing::info!(
target: "dig3::gap_heal",
?key, reason,
"non-kline stream disconnect — forwarder exiting (no resub for non-kline kinds)"
);
break;
}
tracing::info!(target: "dig3::gap_heal", ?key, reason, "ws disconnect detected → heal + resub");
{
let mut series_guard = shared_series.write().await;
run_kline_heal::<T>(
&hub_for_heal, &key, &gap_cfg, &symbol_label,
last_emitted_ms, exchange,
&mut *series_guard, &mut disk, &bcast_tx,
).await;
last_emitted_ms = last_emitted_ms.max(
series_guard.last().map(|p| p.timestamp_ms()).unwrap_or(0)
);
}
let unsub_res = ws.unsubscribe(sub_req.clone()).await;
let sub_res = ws.subscribe(sub_req.clone()).await;
tracing::info!(
target: "dig3::gap_heal",
?key,
unsub_ok = unsub_res.is_ok(),
sub_ok = sub_res.is_ok(),
"resub cycle complete"
);
if let Err(e) = unsub_res {
tracing::debug!(target: "dig3::gap_heal", ?key, ?e, "unsubscribe failed (best-effort)");
}
if let Err(e) = sub_res {
tracing::warn!(target: "dig3::gap_heal", ?key, ?e, "resubscribe failed");
}
drop(stream);
stream = ws.event_stream();
continue;
}
let ev = match item_opt {
Ok(Some(Ok(ev))) => ev,
_ => unreachable!(),
};
if !event_matches_key(&ev, &key) {
continue;
}
let Some(point) = T::from_stream_event(&ev) else {
continue;
};
#[cfg(not(target_arch = "wasm32"))]
if let Some(d) = disk.as_mut() {
if let Err(e) = d.append(&point) {
tracing::warn!(?e, "disk store append failed");
}
}
#[cfg(target_arch = "wasm32")]
if let Some(d) = disk.as_mut() {
d.append(&point);
}
#[cfg(target_arch = "wasm32")]
{
wasm_flush_counter = wasm_flush_counter.wrapping_add(1);
if wasm_flush_counter % WASM_FLUSH_EVERY == 0 {
if let Some(d) = disk.as_mut() {
if let Err(e) = d.flush().await {
tracing::warn!(?e, "wasm OPFS periodic flush failed");
}
}
}
}
let pt_ts = point.timestamp_ms();
{
let mut series_guard = shared_series.write().await;
if matches!(&key.kind, Kind::Kline(_) | Kind::MarkPriceKline(_) | Kind::IndexPriceKline(_) | Kind::PremiumIndexKline(_)) {
series_guard.upsert_by_ts(point.clone());
} else {
series_guard.push(point.clone());
}
}
last_emitted_ms = last_emitted_ms.max(pt_ts);
let _ = bcast_tx.send(Event::from_point(exchange, key.account_type, &symbol_label, &key.kind, point));
#[cfg(not(target_arch = "wasm32"))]
if debug_slow_ms > 0 {
tokio::time::sleep(std::time::Duration::from_millis(debug_slow_ms)).await;
}
}
if let Some(mut d) = disk { let _ = d.flush().await; }
let still_consumers = inner
.muxes
.get(&key)
.map(|m| m.consumers.load(Ordering::SeqCst))
.unwrap_or(0);
if still_consumers == 0 {
inner.muxes.remove(&key);
}
inner.series_handles.remove(&key);
});
#[cfg(not(target_arch = "wasm32"))]
tokio::spawn(forwarder_fut);
#[cfg(target_arch = "wasm32")]
wasm_bindgen_futures::spawn_local(forwarder_fut);
}
}
async fn run_kline_heal<T: DataPoint + 'static>(
hub: &Arc<ExchangeHub>,
key: &SeriesKey,
cfg: &crate::GapHealConfig,
symbol_label: &str,
last_emitted_ms: i64,
exchange: digdigdig3::core::types::ExchangeId,
series: &mut Series<T>,
disk: &mut Option<DiskStore<T>>,
bcast_tx: &broadcast::Sender<Event>,
) where
Event: EventFrom<T>,
{
if !cfg.enabled {
return;
}
let Kind::Kline(iv) = &key.kind else { return; };
let now_ms = chrono::Utc::now().timestamp_millis();
let limit = crate::gap_heal::heal_limit(cfg, iv.as_str(), last_emitted_ms, now_ms);
tracing::info!(
target: "dig3::gap_heal",
?key,
last_emitted_ms,
limit,
"kline heal: pulling REST"
);
let pulled: Vec<T> = cast_vec(
crate::gap_heal::heal_klines(hub, key.exchange, key.account_type, &key.symbol, iv.as_str(), last_emitted_ms, limit).await
);
let pulled_count = pulled.len();
let new_to_emit = crate::gap_heal::select_heal_window(pulled.clone(), last_emitted_ms);
let emitted_count = new_to_emit.len();
for p in pulled {
if let Some(d) = disk.as_mut() {
let _ = d.append(&p);
}
series.upsert_by_ts(p);
}
for p in new_to_emit {
let _ = bcast_tx.send(Event::from_point(exchange, key.account_type, symbol_label, &key.kind, p));
}
if let Some(d) = disk.as_mut() { let _ = d.flush().await; }
tracing::info!(
target: "dig3::gap_heal",
?key,
pulled_count,
emitted_count,
"kline heal: applied"
);
}
fn cast_vec<A: 'static, B: 'static>(v: Vec<A>) -> Vec<B> {
if std::any::TypeId::of::<A>() == std::any::TypeId::of::<B>() {
let mut v = std::mem::ManuallyDrop::new(v);
let (ptr, len, cap) = (v.as_mut_ptr() as *mut B, v.len(), v.capacity());
unsafe { Vec::from_raw_parts(ptr, len, cap) }
} else {
Vec::new()
}
}
fn event_matches_key(ev: &StreamEvent, key: &SeriesKey) -> bool {
let want = key.symbol.as_str();
let got: Option<&str> = event_raw_symbol(ev);
match got {
Some("") => true,
Some(s) => s.eq_ignore_ascii_case(want),
None => true,
}
}
fn event_raw_symbol(ev: &StreamEvent) -> Option<&str> {
match ev {
StreamEvent::Trade { symbol, .. } => Some(symbol),
StreamEvent::AggTrade { symbol, .. } => Some(symbol),
StreamEvent::Ticker { symbol, .. } => Some(symbol),
StreamEvent::Kline { symbol, .. } => Some(symbol),
StreamEvent::OrderbookSnapshot { symbol, .. } => Some(symbol),
StreamEvent::OrderbookDelta { symbol, .. } => Some(symbol),
StreamEvent::MarkPrice { symbol, .. } => Some(symbol),
StreamEvent::FundingRate { symbol, .. } => Some(symbol),
StreamEvent::OpenInterestUpdate { symbol, .. } => Some(symbol),
StreamEvent::Liquidation { symbol, .. } => Some(symbol),
StreamEvent::LongShortRatio { symbol, .. } => Some(symbol),
StreamEvent::MarkPriceKline { symbol, .. } => Some(symbol),
StreamEvent::IndexPriceKline { symbol, .. } => Some(symbol),
StreamEvent::PremiumIndexKline { symbol, .. } => Some(symbol),
StreamEvent::IndexPrice { symbol, .. } => Some(symbol),
StreamEvent::HistoricalVolatility { symbol, .. } => Some(symbol),
StreamEvent::InsuranceFund { symbol, .. } => Some(symbol),
StreamEvent::Basis { symbol, .. } => Some(symbol),
StreamEvent::OptionGreeks { symbol, .. } => Some(symbol),
StreamEvent::VolatilityIndex { symbol, .. } => Some(symbol),
StreamEvent::BlockTrade { symbol, .. } => Some(symbol),
StreamEvent::AuctionEvent { symbol, .. } => Some(symbol),
StreamEvent::MarketWarning { symbol, .. } => symbol.as_deref(),
StreamEvent::OrderbookL3 { symbol, .. } => Some(symbol),
StreamEvent::SettlementEvent { symbol, .. } => Some(symbol),
StreamEvent::RiskLimit { symbol, .. } => Some(symbol),
StreamEvent::PredictedFunding { symbol, .. } => Some(symbol),
StreamEvent::FundingSettlement { symbol, .. } => Some(symbol),
StreamEvent::CompositeIndex { symbol, .. } => Some(symbol),
StreamEvent::OrderUpdate { symbol: _, event: _ }
| StreamEvent::BalanceUpdate(_)
| StreamEvent::PositionUpdate { symbol: _, event: _ } => None,
StreamEvent::Batch(_) => None,
}
}
pub(crate) trait EventFrom<T> {
fn from_point(
exchange: digdigdig3::core::types::ExchangeId,
account_type: digdigdig3::core::types::AccountType,
symbol: &str,
kind: &Kind,
p: T,
) -> Self;
}
impl EventFrom<TradePoint> for Event {
fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: TradePoint) -> Self {
Event::Trade { exchange, symbol: symbol.to_string(), point }
}
}
impl EventFrom<AggTradePoint> for Event {
fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: AggTradePoint) -> Self {
Event::AggTrade { exchange, symbol: symbol.to_string(), point }
}
}
impl EventFrom<BarPoint> for Event {
fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, kind: &Kind, point: BarPoint) -> Self {
let timeframe = match kind { Kind::Kline(iv) => iv.clone(), _ => KlineInterval::new("") };
Event::Bar { exchange, symbol: symbol.to_string(), timeframe, point }
}
}
impl EventFrom<TickerPoint> for Event {
fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: TickerPoint) -> Self {
Event::Ticker { exchange, symbol: symbol.to_string(), point }
}
}
impl EventFrom<ObSnapshotPoint> for Event {
fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: ObSnapshotPoint) -> Self {
Event::OrderbookSnapshot { exchange, symbol: symbol.to_string(), point }
}
}
impl EventFrom<ObDeltaPoint> for Event {
fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: ObDeltaPoint) -> Self {
Event::OrderbookDelta { exchange, symbol: symbol.to_string(), point }
}
}
impl EventFrom<MarkPricePoint> for Event {
fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: MarkPricePoint) -> Self {
Event::MarkPrice { exchange, symbol: symbol.to_string(), point }
}
}
impl EventFrom<FundingRatePoint> for Event {
fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: FundingRatePoint) -> Self {
Event::FundingRate { exchange, symbol: symbol.to_string(), point }
}
}
impl EventFrom<OpenInterestPoint> for Event {
fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: OpenInterestPoint) -> Self {
Event::OpenInterest { exchange, symbol: symbol.to_string(), point }
}
}
impl EventFrom<LiquidationPoint> for Event {
fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: LiquidationPoint) -> Self {
Event::Liquidation { exchange, symbol: symbol.to_string(), point }
}
}
impl EventFrom<BlockTradePoint> for Event {
fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: BlockTradePoint) -> Self {
Event::BlockTrade { exchange, symbol: symbol.to_string(), point }
}
}
impl EventFrom<AuctionEventPoint> for Event {
fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: AuctionEventPoint) -> Self {
Event::AuctionEvent { exchange, symbol: symbol.to_string(), point }
}
}
impl EventFrom<IndexPricePoint> for Event {
fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: IndexPricePoint) -> Self {
Event::IndexPrice { exchange, symbol: symbol.to_string(), point }
}
}
impl EventFrom<CompositeIndexPoint> for Event {
fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: CompositeIndexPoint) -> Self {
Event::CompositeIndex { exchange, symbol: symbol.to_string(), point }
}
}
impl EventFrom<OptionGreeksPoint> for Event {
fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: OptionGreeksPoint) -> Self {
Event::OptionGreeks { exchange, symbol: symbol.to_string(), point }
}
}
impl EventFrom<VolatilityIndexPoint> for Event {
fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: VolatilityIndexPoint) -> Self {
Event::VolatilityIndex { exchange, symbol: symbol.to_string(), point }
}
}
impl EventFrom<HistoricalVolatilityPoint> for Event {
fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: HistoricalVolatilityPoint) -> Self {
Event::HistoricalVolatility { exchange, symbol: symbol.to_string(), point }
}
}
impl EventFrom<LongShortRatioPoint> for Event {
fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: LongShortRatioPoint) -> Self {
Event::LongShortRatio { exchange, symbol: symbol.to_string(), point }
}
}
impl EventFrom<TakerVolumePoint> for Event {
fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: TakerVolumePoint) -> Self {
Event::TakerVolume { exchange, symbol: symbol.to_string(), point }
}
}
impl EventFrom<LiquidationBucketPoint> for Event {
fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: LiquidationBucketPoint) -> Self {
Event::LiquidationBucket { exchange, symbol: symbol.to_string(), point }
}
}
impl EventFrom<BasisPoint> for Event {
fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: BasisPoint) -> Self {
Event::Basis { exchange, symbol: symbol.to_string(), point }
}
}
impl EventFrom<InsuranceFundPoint> for Event {
fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: InsuranceFundPoint) -> Self {
Event::InsuranceFund { exchange, symbol: symbol.to_string(), point }
}
}
impl EventFrom<OrderbookL3Point> for Event {
fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: OrderbookL3Point) -> Self {
Event::OrderbookL3 { exchange, symbol: symbol.to_string(), point }
}
}
impl EventFrom<SettlementEventPoint> for Event {
fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: SettlementEventPoint) -> Self {
Event::SettlementEvent { exchange, symbol: symbol.to_string(), point }
}
}
impl EventFrom<MarketWarningPoint> for Event {
fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: MarketWarningPoint) -> Self {
Event::MarketWarning { exchange, symbol: symbol.to_string(), point }
}
}
impl EventFrom<RiskLimitPoint> for Event {
fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: RiskLimitPoint) -> Self {
Event::RiskLimit { exchange, symbol: symbol.to_string(), point }
}
}
impl EventFrom<PredictedFundingPoint> for Event {
fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: PredictedFundingPoint) -> Self {
Event::PredictedFunding { exchange, symbol: symbol.to_string(), point }
}
}
impl EventFrom<FundingSettlementPoint> for Event {
fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: FundingSettlementPoint) -> Self {
Event::FundingSettlement { exchange, symbol: symbol.to_string(), point }
}
}
impl EventFrom<MarkPriceKlinePoint> for Event {
fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, kind: &Kind, point: MarkPriceKlinePoint) -> Self {
let timeframe = match kind { Kind::MarkPriceKline(iv) => iv.clone(), _ => KlineInterval::new("") };
Event::MarkPriceKline { exchange, symbol: symbol.to_string(), timeframe, point }
}
}
impl EventFrom<IndexPriceKlinePoint> for Event {
fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, kind: &Kind, point: IndexPriceKlinePoint) -> Self {
let timeframe = match kind { Kind::IndexPriceKline(iv) => iv.clone(), _ => KlineInterval::new("") };
Event::IndexPriceKline { exchange, symbol: symbol.to_string(), timeframe, point }
}
}
impl EventFrom<PremiumIndexKlinePoint> for Event {
fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, kind: &Kind, point: PremiumIndexKlinePoint) -> Self {
let timeframe = match kind { Kind::PremiumIndexKline(iv) => iv.clone(), _ => KlineInterval::new("") };
Event::PremiumIndexKline { exchange, symbol: symbol.to_string(), timeframe, point }
}
}
impl EventFrom<OrderUpdatePoint> for Event {
fn from_point(exchange: digdigdig3::core::types::ExchangeId, account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: OrderUpdatePoint) -> Self {
Event::OrderUpdate { exchange, account_type, symbol: symbol.to_string(), point }
}
}
impl EventFrom<BalanceUpdatePoint> for Event {
fn from_point(exchange: digdigdig3::core::types::ExchangeId, account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: BalanceUpdatePoint) -> Self {
Event::BalanceUpdate { exchange, account_type, symbol: symbol.to_string(), point }
}
}
impl EventFrom<PositionUpdatePoint> for Event {
fn from_point(exchange: digdigdig3::core::types::ExchangeId, account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: PositionUpdatePoint) -> Self {
Event::PositionUpdate { exchange, account_type, symbol: symbol.to_string(), point }
}
}
impl EventFrom<FootprintPoint> for Event {
fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: FootprintPoint) -> Self {
Event::Footprint { exchange, symbol: symbol.to_string(), point }
}
}
impl EventFrom<TickerIndicatorsPoint> for Event {
fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, p: TickerIndicatorsPoint) -> Self {
Event::Ticker { exchange, symbol: symbol.to_string(), point: TickerPoint {
ts_ms: p.ts_ms, last: p.last, bid: p.bid, ask: p.ask,
high_24h: p.high_24h, low_24h: p.low_24h,
vol_24h: p.vol_24h, quote_vol_24h: p.quote_vol_24h,
change_pct_24h: p.change_pct_24h,
}}
}
}
impl EventFrom<TickerFullPoint> for Event {
fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, p: TickerFullPoint) -> Self {
Event::Ticker { exchange, symbol: symbol.to_string(), point: TickerPoint {
ts_ms: p.ts_ms, last: p.last, bid: p.bid, ask: p.ask,
high_24h: p.high_24h, low_24h: p.low_24h,
vol_24h: p.vol_24h, quote_vol_24h: p.quote_vol_24h,
change_pct_24h: p.price_change_pct_24h,
}}
}
}
impl EventFrom<MarkPriceIndicatorsPoint> for Event {
fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, p: MarkPriceIndicatorsPoint) -> Self {
Event::MarkPrice { exchange, symbol: symbol.to_string(), point: MarkPricePoint {
ts_ms: p.ts_ms, mark: p.mark, index: p.index,
}}
}
}
impl EventFrom<MarkPriceFullPoint> for Event {
fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, p: MarkPriceFullPoint) -> Self {
Event::MarkPrice { exchange, symbol: symbol.to_string(), point: MarkPricePoint {
ts_ms: p.ts_ms, mark: p.mark, index: p.index,
}}
}
}
impl EventFrom<FundingRateIndicatorsPoint> for Event {
fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, p: FundingRateIndicatorsPoint) -> Self {
Event::FundingRate { exchange, symbol: symbol.to_string(), point: FundingRatePoint {
ts_ms: p.ts_ms, rate: p.rate,
next_funding_time_ms: p.next_funding_time_ms.unwrap_or(0),
}}
}
}
impl EventFrom<FundingRateFullPoint> for Event {
fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, p: FundingRateFullPoint) -> Self {
Event::FundingRate { exchange, symbol: symbol.to_string(), point: FundingRatePoint {
ts_ms: p.ts_ms, rate: p.rate,
next_funding_time_ms: p.next_funding_time_ms.unwrap_or(0),
}}
}
}
impl EventFrom<OpenInterestIndicatorsPoint> for Event {
fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, p: OpenInterestIndicatorsPoint) -> Self {
Event::OpenInterest { exchange, symbol: symbol.to_string(), point: OpenInterestPoint {
ts_ms: p.ts_ms, open_interest: p.open_interest,
open_interest_value: p.open_interest_value,
}}
}
}
impl EventFrom<LiquidationIndicatorsPoint> for Event {
fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, p: LiquidationIndicatorsPoint) -> Self {
Event::Liquidation { exchange, symbol: symbol.to_string(), point: LiquidationPoint {
ts_ms: p.ts_ms, price: p.price, quantity: p.quantity,
value: p.value, side: p.side,
}}
}
}
impl EventFrom<LiquidationFullPoint> for Event {
fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, p: LiquidationFullPoint) -> Self {
Event::Liquidation { exchange, symbol: symbol.to_string(), point: LiquidationPoint {
ts_ms: p.ts_ms, price: p.price, quantity: p.quantity,
value: p.value, side: p.side,
}}
}
}
impl EventFrom<IndexPriceIndicatorsPoint> for Event {
fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, p: IndexPriceIndicatorsPoint) -> Self {
Event::IndexPrice { exchange, symbol: symbol.to_string(), point: IndexPricePoint {
ts_ms: p.ts_ms, price: p.price,
}}
}
}
impl EventFrom<ObSnapshotIndicatorsPoint> for Event {
fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, p: ObSnapshotIndicatorsPoint) -> Self {
Event::OrderbookSnapshot { exchange, symbol: symbol.to_string(), point: ObSnapshotPoint {
ts_ms: p.ts_ms, bids: p.bids, asks: p.asks,
}}
}
}
impl EventFrom<ObDeltaIndicatorsPoint> for Event {
fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, p: ObDeltaIndicatorsPoint) -> Self {
Event::OrderbookDelta { exchange, symbol: symbol.to_string(), point: ObDeltaPoint {
ts_ms: p.ts_ms, bid_changes: p.bid_changes, ask_changes: p.ask_changes,
}}
}
}
async fn ob_rest_seed(
hub: &Arc<digdigdig3::connector_manager::ExchangeHub>,
exchange: digdigdig3::core::types::ExchangeId,
account: digdigdig3::core::types::AccountType,
symbol: &str,
depth: usize,
) -> Vec<ObSnapshotPoint> {
let Some(rest) = hub.rest(exchange) else {
tracing::warn!(
target: "dig3::ob_seed",
?exchange, symbol,
"orderbook REST seed: connector not initialized — continuing WS-only"
);
return Vec::new();
};
let depth_u16 = depth.min(u16::MAX as usize) as u16;
match rest
.get_orderbook(
digdigdig3::core::types::SymbolInput::Raw(symbol),
Some(depth_u16),
account,
)
.await
{
Ok(ob) if ob.bids.is_empty() && ob.asks.is_empty() => {
tracing::warn!(
target: "dig3::ob_seed",
?exchange, symbol,
"orderbook REST seed returned empty snapshot — continuing WS-only"
);
Vec::new()
}
Ok(ob) => {
tracing::debug!(
target: "dig3::ob_seed",
?exchange, symbol,
bids = ob.bids.len(),
asks = ob.asks.len(),
"orderbook REST seed ok"
);
vec![ObSnapshotPoint::from_orderbook(&ob)]
}
Err(e) => {
tracing::warn!(
target: "dig3::ob_seed",
?exchange, symbol, ?e,
"orderbook REST seed failed — continuing WS-only"
);
Vec::new()
}
}
}
fn ws_request_for(
kind: &Kind,
sym: Symbol,
account: digdigdig3::core::types::AccountType,
) -> SubscriptionRequest {
let stream_type = match kind {
Kind::Trade => StreamType::Trade,
Kind::AggTrade => StreamType::AggTrade,
Kind::Kline(iv) => StreamType::Kline { interval: iv.as_str().to_string() },
Kind::Ticker => StreamType::Ticker,
Kind::Orderbook => StreamType::Orderbook,
Kind::OrderbookDelta => StreamType::OrderbookDelta,
Kind::MarkPrice => StreamType::MarkPrice,
Kind::FundingRate => StreamType::FundingRate,
Kind::OpenInterest => StreamType::OpenInterest,
Kind::Liquidation => StreamType::Liquidation,
Kind::BlockTrade => StreamType::BlockTrade,
Kind::AuctionEvent => StreamType::AuctionEvent,
Kind::IndexPrice => StreamType::IndexPrice,
Kind::CompositeIndex => StreamType::CompositeIndex,
Kind::OptionGreeks => StreamType::OptionGreeks,
Kind::VolatilityIndex => StreamType::VolatilityIndex,
Kind::HistoricalVolatility => StreamType::HistoricalVolatility,
Kind::LongShortRatio => StreamType::LongShortRatio,
Kind::TakerVolume | Kind::LiquidationBucket => {
unreachable!("TakerVolume/LiquidationBucket are poll-only — ws_request_for must not be called for them")
}
Kind::Basis => StreamType::Basis,
Kind::InsuranceFund => StreamType::InsuranceFund,
Kind::OrderbookL3 => StreamType::OrderbookL3,
Kind::SettlementEvent => StreamType::SettlementEvent,
Kind::MarketWarning => StreamType::MarketWarning,
Kind::RiskLimit => StreamType::RiskLimit,
Kind::PredictedFunding => StreamType::PredictedFunding,
Kind::FundingSettlement => StreamType::FundingSettlement,
Kind::MarkPriceKline(iv) => StreamType::MarkPriceKline { interval: iv.as_str().to_string() },
Kind::IndexPriceKline(iv) => StreamType::IndexPriceKline { interval: iv.as_str().to_string() },
Kind::PremiumIndexKline(iv) => StreamType::PremiumIndexKline { interval: iv.as_str().to_string() },
Kind::OrderUpdate => StreamType::OrderUpdate,
Kind::BalanceUpdate => StreamType::BalanceUpdate,
Kind::PositionUpdate => StreamType::PositionUpdate,
Kind::RangeBar(_) | Kind::TickBar(_) | Kind::VolumeBar(_) | Kind::Footprint(_) => {
unreachable!("derived kinds must not call ws_request_for")
}
};
SubscriptionRequest {
symbol: sym,
stream_type,
account_type: account,
depth: None,
update_speed_ms: None,
}
}
fn parse_symbol(s: &str) -> Symbol {
if let Some((b, q)) = s.split_once(['-', '/', '_']) {
return Symbol::new(b, q);
}
let upper = s.to_uppercase();
for q in ["USDT", "USDC", "USD", "BTC", "ETH", "BUSD", "EUR", "JPY"] {
if let Some(base) = upper.strip_suffix(q) {
if !base.is_empty() {
return Symbol::new(base, q);
}
}
}
Symbol::new(&upper, "")
}
pub(crate) fn caps_explicitly_unsupported(caps: &ConnectorCapabilities, kind: &Kind) -> bool {
match kind {
Kind::Trade => !caps.has_recent_trades && !caps.has_ws_trades,
Kind::AggTrade => !caps.has_agg_trades && !caps.has_ws_trades,
Kind::Kline(_) => !caps.has_klines && !caps.has_ws_klines,
Kind::Ticker => !caps.has_ticker && !caps.has_ws_ticker,
Kind::Orderbook => !caps.has_orderbook && !caps.has_ws_orderbook,
Kind::OrderbookDelta => !caps.has_orderbook && !caps.has_ws_orderbook,
Kind::MarkPrice => !caps.has_premium_index && !caps.has_ws_mark_price,
Kind::FundingRate => !caps.has_funding_rate_history && !caps.has_ws_funding_rate,
Kind::OpenInterest => !caps.has_open_interest_history,
Kind::Liquidation => !caps.has_liquidation_history,
Kind::MarkPriceKline(_) => !caps.has_mark_price_klines,
Kind::IndexPriceKline(_) => !caps.has_index_price_klines,
Kind::PremiumIndexKline(_) => !caps.has_premium_index_klines,
Kind::LongShortRatio => !caps.has_long_short_ratio_history,
Kind::TakerVolume => !caps.has_taker_volume_history,
Kind::LiquidationBucket => !caps.has_liquidation_bucket_history,
Kind::InsuranceFund => !caps.has_insurance_fund,
Kind::RangeBar(_)
| Kind::TickBar(_)
| Kind::VolumeBar(_)
| Kind::Footprint(_)
| Kind::Basis
| Kind::FundingSettlement => false,
Kind::BlockTrade
| Kind::AuctionEvent
| Kind::IndexPrice
| Kind::CompositeIndex
| Kind::OptionGreeks
| Kind::VolatilityIndex
| Kind::HistoricalVolatility
| Kind::OrderbookL3
| Kind::SettlementEvent
| Kind::MarketWarning
| Kind::RiskLimit
| Kind::PredictedFunding
| Kind::OrderUpdate
| Kind::BalanceUpdate
| Kind::PositionUpdate => false,
}
}