1use std::sync::Arc;
29use std::time::Duration;
30
31use digdigdig3::connector_manager::ExchangeHub;
32use digdigdig3::core::types::{AccountType, ExchangeId};
33
34use crate::series::DataPoint;
35use crate::Result;
36
37#[cfg(not(target_arch = "wasm32"))]
39use std::sync::atomic::Ordering;
40#[cfg(not(target_arch = "wasm32"))]
41use tokio::sync::{broadcast, mpsc, oneshot};
42#[cfg(not(target_arch = "wasm32"))]
43use crate::data::{
44 BasisPoint, FundingSettlementPoint, HistoricalVolatilityPoint, LiquidationBucketPoint,
45 LongShortRatioPoint, TakerVolumePoint,
46};
47#[cfg(not(target_arch = "wasm32"))]
48use crate::series::{DiskStore, PollSpec, SeriesKey};
49#[cfg(not(target_arch = "wasm32"))]
50use crate::subscription::Event;
51#[cfg(not(target_arch = "wasm32"))]
52use crate::StationError;
53#[cfg(not(target_arch = "wasm32"))]
54use crate::station::{
55 flush_disk_store, recv_flush_request, EventFrom, FlushAck, FlushHandle, Station,
56};
57
58pub trait PollSource<T: DataPoint>: Send + Sync + 'static {
78 fn poll(
87 &self,
88 hub: Arc<ExchangeHub>,
89 exchange: ExchangeId,
90 account_type: AccountType,
91 symbol: String,
92 ) -> impl std::future::Future<Output = Result<Vec<T>>> + Send;
93
94 fn cadence(&self) -> Duration;
96}
97
98#[cfg(not(target_arch = "wasm32"))]
116pub(crate) fn spawn_poller<T, S>(
117 station: &Station,
118 key: &SeriesKey,
119 source: S,
120 poll_spec: PollSpec,
121 bcast_tx: broadcast::Sender<Event>,
122 shutdown_rx: oneshot::Receiver<()>,
123 symbol_label: String,
124) where
125 T: DataPoint + 'static,
126 S: PollSource<T>,
127 Event: EventFrom<T>,
128{
129 let inner = station.inner.clone();
130 let key = key.clone();
131 let storage_root = inner.storage_root.clone();
132 let persistence = inner.persistence.clone();
133 let exchange = key.exchange;
134 let hub = inner.hub.clone();
135 let account_type = key.account_type;
136 let raw_symbol = key.symbol.clone();
137
138 let (exit_ack_tx, exit_ack_rx) = oneshot::channel::<()>();
143 inner.exit_acks.insert(key.clone(), exit_ack_rx);
144
145 tokio::spawn(async move {
146 let mut disk: Option<DiskStore<T>> = None;
148 if persistence.is_enabled_for(&key.kind) {
149 match DiskStore::<T>::with_idx_every_and_retention(
150 &storage_root, key.clone(), 1024, persistence.retention_days,
151 ).await {
152 Ok(store) => disk = Some(store),
153 Err(e) => tracing::warn!(?e, ?key, "poll: disk store open failed"),
154 }
155 }
156
157 let mut flush_rx: Option<mpsc::Receiver<FlushAck>> = if disk.is_some() {
162 let (handle, rx) = FlushHandle::channel();
163 inner.flush_handles.insert(key.clone(), handle);
164 Some(rx)
165 } else {
166 None
167 };
168
169 let mut last_emitted_ms: i64 = 0;
171
172 if let Some(d) = disk.as_ref() {
174 if let Ok(tail) = d.read_tail(500).await {
175 for p in &tail {
176 let _ = bcast_tx
177 .send(Event::from_point(exchange, key.account_type, &symbol_label, &key.kind, p.clone()));
178 last_emitted_ms = last_emitted_ms.max(p.timestamp_ms());
179 }
180 }
181 }
182
183 {
187 let jitter_max_ms = (poll_spec.cadence.as_millis() as u64)
188 .saturating_mul(poll_spec.jitter_pct as u64)
189 / 100;
190 if jitter_max_ms > 0 {
191 let seed = key
192 .symbol
193 .as_bytes()
194 .iter()
195 .fold(0u64, |acc, &b| acc.wrapping_mul(31).wrapping_add(b as u64));
196 let sleep_ms = seed % jitter_max_ms.max(1);
198 tokio::time::sleep(Duration::from_millis(sleep_ms)).await;
199 }
200 }
201
202 let mut interval = tokio::time::interval(source.cadence());
203 interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
204
205 let mut consecutive_errors: u32 = 0;
206 const DEGRADE_THRESHOLD: u32 = 10;
207
208 let mut shutdown_rx = shutdown_rx;
209
210 loop {
211 tokio::select! {
212 biased;
213 _ = &mut shutdown_rx => break,
214 ack = recv_flush_request(&mut flush_rx) => {
215 let result = flush_disk_store(&mut disk).await;
216 let _ = ack.send(result);
217 continue;
218 }
219 _ = interval.tick() => {}
220 }
221
222 let pts = match source.poll(hub.clone(), exchange, account_type, raw_symbol.clone()).await {
223 Ok(v) => {
224 consecutive_errors = 0;
225 v
226 }
227 Err(e) => {
228 consecutive_errors += 1;
229 if consecutive_errors == 1 || consecutive_errors == DEGRADE_THRESHOLD {
230 tracing::warn!(
231 target: "dig3::poll",
232 ?key,
233 consecutive_errors,
234 error = %e,
235 "poller REST error{}",
236 if consecutive_errors >= DEGRADE_THRESHOLD { " — poller degraded" } else { "" }
237 );
238 }
239 continue;
241 }
242 };
243
244 for pt in pts {
246 if pt.timestamp_ms() <= last_emitted_ms {
247 continue; }
249 if let Some(d) = disk.as_mut() {
250 if let Err(e) = d.append(&pt) {
251 tracing::warn!(?e, "poll: disk append failed");
252 }
253 }
254 last_emitted_ms = pt.timestamp_ms();
255 let _ =
256 bcast_tx.send(Event::from_point(exchange, key.account_type, &symbol_label, &key.kind, pt));
257 }
258 }
259
260 if flush_rx.is_some() {
263 inner.flush_handles.remove(&key);
264 }
265 if let Some(mut d) = disk {
267 let _ = d.flush().await;
268 }
269
270 let still_consumers = inner
272 .muxes
273 .get(&key)
274 .map(|m| m.consumers.load(Ordering::SeqCst))
275 .unwrap_or(0);
276 if still_consumers == 0 {
277 inner.muxes.remove(&key);
278 }
279 inner.exit_acks.remove(&key);
283 let _ = exit_ack_tx.send(());
284 });
285}
286
287#[cfg(not(target_arch = "wasm32"))]
304pub struct LongShortRatioPoll {
305 cadence: Duration,
306}
307
308#[cfg(not(target_arch = "wasm32"))]
309impl LongShortRatioPoll {
310 pub fn new() -> Self {
311 Self {
312 cadence: Duration::from_secs(5 * 60),
313 }
314 }
315
316 fn period_for(exchange: ExchangeId) -> &'static str {
318 match exchange {
319 ExchangeId::Bybit => "5min",
320 _ => "5m", }
322 }
323}
324
325#[cfg(not(target_arch = "wasm32"))]
326impl Default for LongShortRatioPoll {
327 fn default() -> Self {
328 Self::new()
329 }
330}
331
332#[cfg(not(target_arch = "wasm32"))]
333impl PollSource<LongShortRatioPoint> for LongShortRatioPoll {
334 fn poll(
335 &self,
336 hub: Arc<ExchangeHub>,
337 exchange: ExchangeId,
338 account_type: AccountType,
339 symbol: String,
340 ) -> impl std::future::Future<Output = Result<Vec<LongShortRatioPoint>>> + Send {
341 let period = Self::period_for(exchange);
342 async move {
343 let connector = hub
344 .rest(exchange)
345 .ok_or_else(|| StationError::Core("REST connector missing for LSR poll".into()))?;
346 let raw = connector
347 .get_long_short_ratio_history(
348 symbol.as_str().into(),
349 period,
350 None,
351 None,
352 Some(500),
353 account_type,
354 )
355 .await
356 .map_err(|e| StationError::Core(format!("poll LSR: {e}")))?;
357 Ok(raw
358 .into_iter()
359 .map(|r| LongShortRatioPoint {
360 ts_ms: r.timestamp,
361 ratio: r.ratio.unwrap_or_else(|| {
362 if r.short_ratio > 0.0 {
363 r.long_ratio / r.short_ratio
364 } else {
365 1.0
366 }
367 }),
368 long_pct: r.long_ratio,
369 short_pct: r.short_ratio,
370 })
371 .collect())
372 }
373 }
374
375 fn cadence(&self) -> Duration {
376 self.cadence
377 }
378}
379
380#[cfg(not(target_arch = "wasm32"))]
390pub struct DeribitHvPoll {
391 cadence: Duration,
392}
393
394#[cfg(not(target_arch = "wasm32"))]
395impl DeribitHvPoll {
396 pub fn new() -> Self {
397 Self {
398 cadence: Duration::from_secs(60 * 60),
399 }
400 }
401}
402
403#[cfg(not(target_arch = "wasm32"))]
404impl Default for DeribitHvPoll {
405 fn default() -> Self {
406 Self::new()
407 }
408}
409
410#[cfg(not(target_arch = "wasm32"))]
411impl PollSource<HistoricalVolatilityPoint> for DeribitHvPoll {
412 fn poll(
413 &self,
414 hub: Arc<ExchangeHub>,
415 _exchange: ExchangeId,
416 _account_type: AccountType,
417 symbol: String, ) -> impl std::future::Future<Output = Result<Vec<HistoricalVolatilityPoint>>> + Send {
419 async move {
420 let connector = hub
421 .rest(ExchangeId::Deribit)
422 .ok_or_else(|| StationError::Core("Deribit REST connector missing for HV poll".into()))?;
423 let raw = connector
424 .get_historical_volatility(&symbol)
425 .await
426 .map_err(|e| StationError::Core(format!("poll HV: {e}")))?;
427 Ok(raw
428 .into_iter()
429 .map(|h| HistoricalVolatilityPoint {
430 ts_ms: h.timestamp,
431 volatility: h.volatility,
432 })
433 .collect())
434 }
435 }
436
437 fn cadence(&self) -> Duration {
438 self.cadence
439 }
440}
441
442#[cfg(not(target_arch = "wasm32"))]
452pub(crate) fn lsr_poll_source(exchange: ExchangeId) -> Option<LongShortRatioPoll> {
453 match exchange {
454 ExchangeId::Binance | ExchangeId::Bybit | ExchangeId::OKX => {
455 Some(LongShortRatioPoll::new())
456 }
457 _ => None,
458 }
459}
460
461#[cfg(not(target_arch = "wasm32"))]
465pub(crate) fn hv_poll_source(exchange: ExchangeId) -> Option<DeribitHvPoll> {
466 match exchange {
467 ExchangeId::Deribit => Some(DeribitHvPoll::new()),
468 _ => None,
469 }
470}
471
472#[cfg(not(target_arch = "wasm32"))]
482pub struct BasisHistoryPoll {
483 pub period: String,
485}
486
487#[cfg(not(target_arch = "wasm32"))]
488impl BasisHistoryPoll {
489 pub fn new(period: impl Into<String>) -> Self {
490 Self { period: period.into() }
491 }
492}
493
494#[cfg(not(target_arch = "wasm32"))]
495impl PollSource<BasisPoint> for BasisHistoryPoll {
496 fn poll(
497 &self,
498 hub: Arc<ExchangeHub>,
499 exchange: ExchangeId,
500 account_type: AccountType,
501 symbol: String,
502 ) -> impl std::future::Future<Output = Result<Vec<BasisPoint>>> + Send {
503 let period = self.period.clone();
504 async move {
505 let connector = hub
506 .rest(exchange)
507 .ok_or_else(|| StationError::Core("REST connector missing for basis history poll".into()))?;
508 let raw = connector
509 .get_basis_history(
510 symbol.as_str().into(),
511 &period,
512 None,
513 None,
514 Some(500),
515 account_type,
516 )
517 .await
518 .map_err(|e| StationError::Core(format!("poll basis history: {e}")))?;
519 Ok(raw
520 .into_iter()
521 .map(|b| BasisPoint {
522 ts_ms: b.timestamp,
523 value: b.basis,
524 mark: b.futures_price.unwrap_or(f64::NAN),
525 index: b.index_price.unwrap_or(f64::NAN),
526 })
527 .collect())
528 }
529 }
530
531 fn cadence(&self) -> Duration {
532 Duration::from_secs(60)
535 }
536}
537
538#[cfg(not(target_arch = "wasm32"))]
548pub struct FundingHistoryPoll;
549
550#[cfg(not(target_arch = "wasm32"))]
551impl PollSource<FundingSettlementPoint> for FundingHistoryPoll {
552 fn poll(
553 &self,
554 hub: Arc<ExchangeHub>,
555 exchange: ExchangeId,
556 account_type: AccountType,
557 symbol: String,
558 ) -> impl std::future::Future<Output = Result<Vec<FundingSettlementPoint>>> + Send {
559 async move {
560 let connector = hub
561 .rest(exchange)
562 .ok_or_else(|| StationError::Core("REST connector missing for funding history poll".into()))?;
563 let raw = connector
564 .get_funding_rate_history(
565 symbol.as_str().into(),
566 None,
567 None,
568 Some(500),
569 account_type,
570 )
571 .await
572 .map_err(|e| StationError::Core(format!("poll funding history: {e}")))?;
573 Ok(raw
574 .into_iter()
575 .map(|f| FundingSettlementPoint {
576 ts_ms: f.timestamp,
577 settled_rate: f.rate,
578 settlement_time: f.next_funding_time.unwrap_or(f.timestamp),
579 })
580 .collect())
581 }
582 }
583
584 fn cadence(&self) -> Duration {
585 Duration::from_secs(5 * 60)
586 }
587}
588
589#[cfg(not(target_arch = "wasm32"))]
595pub(crate) fn basis_poll_source(hub: &ExchangeHub, exchange: ExchangeId) -> Option<BasisHistoryPoll> {
596 let caps = hub.capabilities(exchange)?;
597 if caps.has_basis_history {
598 Some(BasisHistoryPoll::new("1h"))
599 } else {
600 None
601 }
602}
603
604#[cfg(not(target_arch = "wasm32"))]
611pub(crate) fn funding_poll_source(hub: &ExchangeHub, exchange: ExchangeId) -> Option<FundingHistoryPoll> {
612 let caps = hub.capabilities(exchange)?;
613 if caps.has_funding_rate_history {
614 Some(FundingHistoryPoll)
615 } else {
616 None
617 }
618}
619
620#[cfg(not(target_arch = "wasm32"))]
630pub struct TakerVolumePoll {
631 cadence: Duration,
632 period: String,
633}
634
635#[cfg(not(target_arch = "wasm32"))]
636impl TakerVolumePoll {
637 pub fn new(period: impl Into<String>) -> Self {
638 Self {
639 cadence: Duration::from_secs(5 * 60),
640 period: period.into(),
641 }
642 }
643}
644
645#[cfg(not(target_arch = "wasm32"))]
646impl PollSource<TakerVolumePoint> for TakerVolumePoll {
647 fn poll(
648 &self,
649 hub: Arc<ExchangeHub>,
650 exchange: ExchangeId,
651 account_type: AccountType,
652 symbol: String,
653 ) -> impl std::future::Future<Output = Result<Vec<TakerVolumePoint>>> + Send {
654 let period = self.period.clone();
655 async move {
656 let connector = hub
657 .rest(exchange)
658 .ok_or_else(|| StationError::Core("REST connector missing for taker_volume poll".into()))?;
659 let raw = connector
660 .get_taker_volume_history(
661 symbol.as_str().into(),
662 &period,
663 None,
664 None,
665 Some(500),
666 account_type,
667 )
668 .await
669 .map_err(|e| StationError::Core(format!("poll taker_volume: {e}")))?;
670 Ok(raw
671 .into_iter()
672 .map(|t| TakerVolumePoint {
673 ts_ms: t.timestamp,
674 buy_volume: t.buy_volume,
675 sell_volume: t.sell_volume,
676 buy_sell_ratio: t.buy_sell_ratio.unwrap_or(f64::NAN),
677 long_taker_size: t.long_taker_size.unwrap_or(f64::NAN),
678 short_taker_size: t.short_taker_size.unwrap_or(f64::NAN),
679 })
680 .collect())
681 }
682 }
683
684 fn cadence(&self) -> Duration {
685 self.cadence
686 }
687}
688
689#[cfg(not(target_arch = "wasm32"))]
694pub(crate) fn taker_volume_poll_source(hub: &ExchangeHub, exchange: ExchangeId) -> Option<TakerVolumePoll> {
695 let caps = hub.capabilities(exchange)?;
696 if caps.has_taker_volume_history {
697 Some(TakerVolumePoll::new("5m"))
698 } else {
699 None
700 }
701}
702
703#[cfg(not(target_arch = "wasm32"))]
713pub struct LiquidationBucketPoll {
714 cadence: Duration,
715 period: String,
716}
717
718#[cfg(not(target_arch = "wasm32"))]
719impl LiquidationBucketPoll {
720 pub fn new(period: impl Into<String>) -> Self {
721 Self {
722 cadence: Duration::from_secs(5 * 60),
723 period: period.into(),
724 }
725 }
726}
727
728#[cfg(not(target_arch = "wasm32"))]
729impl PollSource<LiquidationBucketPoint> for LiquidationBucketPoll {
730 fn poll(
731 &self,
732 hub: Arc<ExchangeHub>,
733 exchange: ExchangeId,
734 account_type: AccountType,
735 symbol: String,
736 ) -> impl std::future::Future<Output = Result<Vec<LiquidationBucketPoint>>> + Send {
737 let period = self.period.clone();
738 async move {
739 let connector = hub
740 .rest(exchange)
741 .ok_or_else(|| StationError::Core("REST connector missing for liquidation_bucket poll".into()))?;
742 let raw = connector
743 .get_liquidation_bucket_history(
744 symbol.as_str().into(),
745 &period,
746 None,
747 None,
748 Some(500),
749 account_type,
750 )
751 .await
752 .map_err(|e| StationError::Core(format!("poll liquidation_bucket: {e}")))?;
753 Ok(raw
754 .into_iter()
755 .map(|b| LiquidationBucketPoint {
756 ts_ms: b.timestamp,
757 long_liq_size: b.long_liq_size.unwrap_or(f64::NAN),
758 short_liq_size: b.short_liq_size.unwrap_or(f64::NAN),
759 long_liq_amount: b.long_liq_amount.unwrap_or(f64::NAN),
760 short_liq_amount: b.short_liq_amount.unwrap_or(f64::NAN),
761 long_liq_usd: b.long_liq_usd.unwrap_or(f64::NAN),
762 short_liq_usd: b.short_liq_usd.unwrap_or(f64::NAN),
763 })
764 .collect())
765 }
766 }
767
768 fn cadence(&self) -> Duration {
769 self.cadence
770 }
771}
772
773#[cfg(not(target_arch = "wasm32"))]
778pub(crate) fn liquidation_bucket_poll_source(hub: &ExchangeHub, exchange: ExchangeId) -> Option<LiquidationBucketPoll> {
779 let caps = hub.capabilities(exchange)?;
780 if caps.has_liquidation_bucket_history {
781 Some(LiquidationBucketPoll::new("5m"))
782 } else {
783 None
784 }
785}
786
787#[cfg(test)]
792mod tests {
793 use crate::series::Kind;
794
795 #[test]
797 fn kind_lsr_poll_spec() {
798 let spec = Kind::LongShortRatio.is_poll_only().unwrap();
799 assert_eq!(spec.cadence, std::time::Duration::from_secs(300));
800 assert_eq!(spec.jitter_pct, 10);
801 }
802
803 #[test]
804 fn kind_hv_poll_spec() {
805 let spec = Kind::HistoricalVolatility.is_poll_only().unwrap();
806 assert_eq!(spec.cadence, std::time::Duration::from_secs(3600));
807 assert_eq!(spec.jitter_pct, 5);
808 }
809
810 #[cfg(not(target_arch = "wasm32"))]
812 mod native {
813 use super::super::*;
814
815 #[test]
816 fn lsr_poll_cadence() {
817 assert_eq!(LongShortRatioPoll::new().cadence(), Duration::from_secs(300));
818 }
819
820 #[test]
821 fn hv_poll_cadence() {
822 assert_eq!(DeribitHvPoll::new().cadence(), Duration::from_secs(3600));
823 }
824
825 #[test]
826 fn lsr_poll_source_allow_list() {
827 assert!(lsr_poll_source(ExchangeId::Binance).is_some());
828 assert!(lsr_poll_source(ExchangeId::Bybit).is_some());
829 assert!(lsr_poll_source(ExchangeId::OKX).is_some());
830 assert!(lsr_poll_source(ExchangeId::Deribit).is_none());
831 assert!(lsr_poll_source(ExchangeId::Kraken).is_none());
832 }
833
834 #[test]
835 fn hv_poll_source_allow_list() {
836 assert!(hv_poll_source(ExchangeId::Deribit).is_some());
837 assert!(hv_poll_source(ExchangeId::Binance).is_none());
838 assert!(hv_poll_source(ExchangeId::Bybit).is_none());
839 assert!(hv_poll_source(ExchangeId::OKX).is_none());
840 }
841
842 #[test]
843 fn basis_history_poll_cadence() {
844 assert_eq!(
845 BasisHistoryPoll::new("1h").cadence(),
846 Duration::from_secs(60)
847 );
848 }
849
850 #[test]
851 fn funding_history_poll_cadence() {
852 assert_eq!(
853 FundingHistoryPoll.cadence(),
854 Duration::from_secs(300)
855 );
856 }
857
858 #[test]
859 fn lsr_period_for_exchange() {
860 assert_eq!(LongShortRatioPoll::period_for(ExchangeId::Bybit), "5min");
861 assert_eq!(LongShortRatioPoll::period_for(ExchangeId::Binance), "5m");
862 assert_eq!(LongShortRatioPoll::period_for(ExchangeId::OKX), "5m");
863 }
864
865 #[test]
866 fn taker_volume_poll_cadence() {
867 assert_eq!(TakerVolumePoll::new("5m").cadence(), Duration::from_secs(300));
868 }
869
870 #[test]
871 fn liquidation_bucket_poll_cadence() {
872 assert_eq!(LiquidationBucketPoll::new("5m").cadence(), Duration::from_secs(300));
873 }
874
875 #[test]
876 fn taker_volume_poll_source_allow_list() {
877 let _: fn(&ExchangeHub, ExchangeId) -> Option<TakerVolumePoll> = taker_volume_poll_source;
880 }
881
882 #[test]
883 fn liquidation_bucket_poll_source_allow_list() {
884 let _: fn(&ExchangeHub, ExchangeId) -> Option<LiquidationBucketPoll> = liquidation_bucket_poll_source;
885 }
886 }
887}