1use std::any::Any;
2use std::path::PathBuf;
3use std::sync::atomic::{AtomicUsize, Ordering};
4use std::sync::Arc;
5
6use crate::ws_health::WsHealth;
7
8use dashmap::DashMap;
9use digdigdig3::connector_manager::ExchangeHub;
10use digdigdig3::core::types::{
11 AccountType, ConnectorCapabilities, ExchangeId, StreamEvent, StreamType, SubscriptionRequest,
12 Symbol, SymbolInfo,
13};
14use digdigdig3::core::websocket::KlineInterval;
15use digdigdig3::core::utils::SymbolNormalizer;
16use futures_util::StreamExt;
17use tokio::sync::{broadcast, mpsc, oneshot, RwLock};
18
19use crate::data::{
20 AggTradePoint, AuctionEventPoint, BalanceUpdatePoint, BarPoint, BasisPoint, BlockTradePoint,
21 CompositeIndexPoint,
22 FootprintPoint,
23 FundingRatePoint, FundingRateIndicatorsPoint, FundingRateFullPoint,
24 FundingSettlementPoint, HistoricalVolatilityPoint,
25 IndexPriceKlinePoint, IndexPricePoint, IndexPriceIndicatorsPoint,
26 InsuranceFundPoint,
27 LiquidationPoint, LiquidationIndicatorsPoint, LiquidationFullPoint,
28 LiquidationBucketPoint,
29 LongShortRatioPoint,
30 MarkPriceKlinePoint, MarkPricePoint, MarkPriceIndicatorsPoint, MarkPriceFullPoint,
31 MarketWarningPoint,
32 ObDeltaPoint, ObDeltaIndicatorsPoint,
33 ObSnapshotPoint, ObSnapshotIndicatorsPoint,
34 OpenInterestPoint, OpenInterestIndicatorsPoint,
35 OptionGreeksPoint,
36 OrderUpdatePoint, OrderbookL3Point, PositionUpdatePoint,
37 PredictedFundingPoint, PremiumIndexKlinePoint, RiskLimitPoint, SettlementEventPoint,
38 TakerVolumePoint,
39 TickerPoint, TickerIndicatorsPoint, TickerFullPoint,
40 TradePoint, VolatilityIndexPoint,
41 KagiSegmentPoint, PnfColumnPoint, RenkoBrickPoint, ScalarBarPoint,
42 ThreeLineBreakLinePoint, TpoSessionPoint,
43};
44use crate::persistence::PersistDepth;
45use crate::derived::{
46 BasisDerived, DerivedStream, FundingSettlementDerived, TradeToBarDerived,
47 TradeToRangeBarDerived, TradeToTickBarDerived, TradeToVolumeBarDerived,
48 TradeToFootprintDerived, TradeToRenkoBarDerived, TradeToPnfBarDerived,
49 TradeToKagiBarDerived, TradeToCvdLineDerived, TradeToThreeLineBreakDerived,
50 TpoFromKline1mDerived,
51 TpoFromTradeDerived, TradeToDollarBarDerived, TradeToTickImbalanceDerived,
52 TradeToVolumeImbalanceDerived, TradeToRunBarDerived, interval_to_ms,
53};
54use crate::series::TpoSource;
55#[cfg(not(target_arch = "wasm32"))]
56use crate::polling;
57#[cfg(not(target_arch = "wasm32"))]
58use crate::polling::PollSource;
59use crate::series::DiskStore;
60use crate::series::{DataPoint, Kind, Series, SeriesKey};
61use crate::subscription::{Entry, Event, FailedStream, MultiplexRef, Stream};
62use crate::{
63 PersistenceConfig, Result, StationBuilder, StationError, SubscribeReport, SubscriptionHandle,
64 SubscriptionSet,
65};
66
67
68pub struct Station {
72 pub(crate) inner: Arc<StationInner>,
73}
74
75pub(crate) struct StationInner {
76 pub(crate) hub: Arc<ExchangeHub>,
77 pub(crate) storage_root: PathBuf,
78 pub(crate) persistence: PersistenceConfig,
79 pub(crate) muxes: DashMap<SeriesKey, Multiplexer>,
80 pub(crate) series_handles: DashMap<SeriesKey, Arc<dyn Any + Send + Sync>>,
87 pub(crate) warm_start_capacity: usize,
88 pub(crate) gap_heal: crate::GapHealConfig,
89 pub(crate) unsubscribe_grace: std::time::Duration,
92 pub(crate) orderbook_rest_seed: bool,
95 pub(crate) orderbook_seed_depth: usize,
97 pub(crate) connector_tx: broadcast::Sender<crate::subscription::Event>,
101 pub(crate) exchange_info_cache: DashMap<(ExchangeId, AccountType), Vec<SymbolInfo>>,
104 pub(crate) flush_handles: DashMap<SeriesKey, FlushHandle>,
113 pub(crate) exit_acks: DashMap<SeriesKey, ExitAck>,
122 pub(crate) seed_outcomes: DashMap<SeriesKey, crate::SeedOutcome>,
132}
133
134pub(crate) type ExitAck = oneshot::Receiver<()>;
137
138pub(crate) type FlushAck = oneshot::Sender<std::io::Result<()>>;
143
144#[derive(Clone)]
152pub(crate) struct FlushHandle(mpsc::Sender<FlushAck>);
153
154impl FlushHandle {
155 pub(crate) fn channel() -> (Self, mpsc::Receiver<FlushAck>) {
159 let (tx, rx) = mpsc::channel(1);
160 (Self(tx), rx)
161 }
162}
163
164pub(crate) struct Multiplexer {
167 pub(crate) tx: broadcast::Sender<Event>,
168 pub(crate) consumers: Arc<AtomicUsize>,
169 pub(crate) shutdown: Option<oneshot::Sender<()>>,
170 pub(crate) grace_cancel: Option<oneshot::Sender<()>>,
177}
178
179impl std::fmt::Debug for Station {
180 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
181 f.debug_struct("Station")
182 .field("storage_root", &self.inner.storage_root)
183 .field("persistence", &self.inner.persistence)
184 .field("muxes", &self.inner.muxes.len())
185 .finish()
186 }
187}
188
189#[derive(Debug, Clone)]
194pub enum RewarmedPoints {
195 Renko(Vec<RenkoBrickPoint>),
196 Pnf(Vec<PnfColumnPoint>),
197 Kagi(Vec<KagiSegmentPoint>),
198 ThreeLineBreak(Vec<ThreeLineBreakLinePoint>),
199 Bar(Vec<BarPoint>),
201 Cvd(Vec<ScalarBarPoint>),
202 Tpo(Vec<TpoSessionPoint>),
203}
204
205impl RewarmedPoints {
206 pub fn len(&self) -> usize {
208 match self {
209 RewarmedPoints::Renko(v) => v.len(),
210 RewarmedPoints::Pnf(v) => v.len(),
211 RewarmedPoints::Kagi(v) => v.len(),
212 RewarmedPoints::ThreeLineBreak(v) => v.len(),
213 RewarmedPoints::Bar(v) => v.len(),
214 RewarmedPoints::Cvd(v) => v.len(),
215 RewarmedPoints::Tpo(v) => v.len(),
216 }
217 }
218
219 pub fn is_empty(&self) -> bool {
220 self.len() == 0
221 }
222}
223
224#[derive(Debug, Clone)]
235pub struct RewarmOutcome {
236 pub points: RewarmedPoints,
237 pub seed: crate::SeedOutcome,
238}
239
240impl RewarmOutcome {
241 pub fn len(&self) -> usize { self.points.len() }
243 pub fn is_empty(&self) -> bool { self.points.is_empty() }
244 pub fn is_capability_exhausted(&self) -> bool {
248 matches!(
249 self.seed.truncated_by,
250 Some(crate::TruncationReason::VenueRecentOnly | crate::TruncationReason::VenueWindowCap)
251 )
252 }
253}
254
255impl Station {
256 pub fn builder() -> StationBuilder { StationBuilder::new() }
257 pub fn storage_root(&self) -> &std::path::Path { &self.inner.storage_root }
258 pub fn active_streams(&self) -> usize { self.inner.muxes.len() }
259
260 pub fn hub(&self) -> Arc<ExchangeHub> {
270 self.inner.hub.clone()
271 }
272
273 pub fn series<T: DataPoint + 'static>(&self, key: &SeriesKey)
286 -> Option<Arc<RwLock<Series<T>>>>
287 {
288 let erased = self.inner.series_handles.get(key)?;
289 erased
293 .downcast_ref::<Arc<RwLock<Series<T>>>>()
294 .map(Arc::clone)
295 }
296
297 pub fn register_consumer(
306 &self,
307 quota: crate::quota::ConsumerQuota,
308 ) -> crate::quota::ConsumerHandle {
309 let rest_bucket = quota.max_rest_per_window.map(|cap| {
310 crate::quota::TokenBucket::new(cap, quota.rest_window)
311 });
312 crate::quota::ConsumerHandle {
313 station: Arc::clone(&self.inner),
314 quota,
315 rest_bucket: Arc::new(tokio::sync::Mutex::new(rest_bucket)),
316 refs: tokio::sync::Mutex::new((0, Vec::new())),
317 }
318 }
319
320 pub(crate) async fn from_builder(b: StationBuilder) -> Result<Self> {
321 let _ = digdigdig3::core::install_default_crypto_provider();
322 #[cfg(not(target_arch = "wasm32"))]
326 if b.persistence.enabled {
327 std::fs::create_dir_all(&b.storage_root).map_err(StationError::Io)?;
328 }
329 let (connector_tx, _) = broadcast::channel(256);
330 Ok(Self {
331 inner: Arc::new(StationInner {
332 hub: Arc::new(ExchangeHub::new()),
333 storage_root: b.storage_root,
334 persistence: b.persistence,
335 muxes: DashMap::new(),
336 series_handles: DashMap::new(),
337 warm_start_capacity: b.warm_start.max(1),
338 gap_heal: b.gap_heal,
339 unsubscribe_grace: b.unsubscribe_grace,
340 orderbook_rest_seed: b.orderbook_rest_seed,
341 orderbook_seed_depth: b.orderbook_seed_depth,
342 connector_tx,
343 exchange_info_cache: DashMap::new(),
344 flush_handles: DashMap::new(),
345 exit_acks: DashMap::new(),
346 seed_outcomes: DashMap::new(),
347 }),
348 })
349 }
350
351 pub fn connector_events(&self) -> broadcast::Receiver<crate::subscription::Event> {
359 self.inner.connector_tx.subscribe()
360 }
361
362 pub fn symbols(&self, exchange: ExchangeId) -> Vec<SymbolInfo> {
365 let mut out = Vec::new();
366 for entry in self.inner.exchange_info_cache.iter() {
367 if entry.key().0 == exchange {
368 out.extend_from_slice(entry.value());
369 }
370 }
371 out
372 }
373
374 pub fn ws_health(&self, key: &SeriesKey) -> Option<WsHealth> {
389 self.inner.muxes.get(key)?;
391 Some(WsHealth {
392 connected: true,
393 rtt_ms: None,
394 last_message_ms: None,
395 })
396 }
397
398 pub fn ws_health_for_exchange(
407 &self,
408 exchange: ExchangeId,
409 ) -> Option<WsHealth> {
410 let mut any_connected = false;
411 let mut rtts: Vec<u64> = Vec::new();
412 let mut last_msgs: Vec<i64> = Vec::new();
413
414 for entry in self.inner.muxes.iter() {
415 if entry.key().exchange != exchange {
416 continue;
417 }
418 let h = WsHealth {
420 connected: true,
421 rtt_ms: None,
422 last_message_ms: None,
423 };
424 any_connected = true;
425 if let Some(rtt) = h.rtt_ms {
426 rtts.push(rtt);
427 }
428 if let Some(ts) = h.last_message_ms {
429 last_msgs.push(ts);
430 }
431 }
432
433 if !any_connected {
434 return None;
435 }
436
437 let rtt_ms = if rtts.is_empty() {
438 None
439 } else {
440 rtts.sort_unstable();
441 Some(rtts[rtts.len() / 2])
442 };
443
444 Some(WsHealth {
445 connected: true,
446 rtt_ms,
447 last_message_ms: last_msgs.into_iter().max(),
448 })
449 }
450
451 pub async fn flush_persistence(&self) -> std::result::Result<(), Vec<(SeriesKey, std::io::Error)>> {
467 let handles: Vec<(SeriesKey, FlushHandle)> = self
468 .inner
469 .flush_handles
470 .iter()
471 .map(|entry| (entry.key().clone(), entry.value().clone()))
472 .collect();
473
474 let mut errors = Vec::new();
475
476 for (key, handle) in handles {
477 let (ack_tx, ack_rx) = oneshot::channel();
478 match handle.0.send(ack_tx).await {
479 Ok(()) => match ack_rx.await {
480 Ok(Ok(())) => {}
481 Ok(Err(io_err)) => errors.push((key, io_err)),
482 Err(_) => {}
485 },
486 Err(_) => {}
489 }
490 }
491
492 self.inner
499 .flush_handles
500 .retain(|_, handle| !handle.0.is_closed());
501
502 if errors.is_empty() {
503 Ok(())
504 } else {
505 Err(errors)
506 }
507 }
508
509 pub async fn force_unsubscribe_and_await(&self, key: &SeriesKey) -> Result<()> {
528 let removed = {
536 let Some(mut mux) = self.inner.muxes.get_mut(key) else {
537 return Ok(());
539 };
540 if let Some(cancel) = mux.grace_cancel.take() {
541 let _ = cancel.send(());
542 }
543 mux.shutdown.take()
544 };
545
546 self.inner.muxes.remove(key);
547 self.inner.series_handles.remove(key);
548
549 if let Some(shutdown_tx) = removed {
550 let _ = shutdown_tx.send(());
551 }
552
553 if let Some((_, ack_rx)) = self.inner.exit_acks.remove(key) {
561 let _ = ack_rx.await;
562 }
563
564 Ok(())
565 }
566
567 pub async fn rewarm_footprint(
591 &self,
592 key: &SeriesKey,
593 raw_symbol: &str,
594 warm_n: usize,
595 ) -> Vec<FootprintPoint> {
596 let Some(handle) = self.inner.series_handles.get(key).and_then(|e| {
599 e.downcast_ref::<Arc<RwLock<Series<FootprintPoint>>>>().map(Arc::clone)
600 }) else {
601 return Vec::new();
602 };
603 if warm_n == 0 {
604 return Vec::new();
605 }
606
607 let caps_opt = self.inner.hub.capabilities(key.exchange);
611 let use_agg = caps_opt.as_ref().map(|c| c.has_agg_trades).unwrap_or(false);
612 let mut seed_events: Vec<Event> = Vec::new();
613 if use_agg {
614 let n_pages = ((warm_n + 999) / 1000).max(1);
615 let (agg, _outcome) = crate::backfill::agg_trades_paginated(
616 &self.inner.hub, key.exchange, key.account_type, raw_symbol, 1000, n_pages,
617 )
618 .await;
619 for ap in agg {
620 seed_events.push(Event::Trade {
621 exchange: key.exchange,
622 symbol: raw_symbol.to_string(),
623 point: TradePoint {
624 ts_ms: ap.ts_ms,
625 price: ap.price,
626 quantity: ap.quantity,
627 side: ap.side,
628 trade_id_hash: 0,
629 },
630 });
631 }
632 } else {
633 let (trades, _outcome) = crate::backfill::trades_recent(
634 &self.inner.hub, key.exchange, key.account_type, raw_symbol, warm_n,
635 )
636 .await;
637 for pt in trades {
638 seed_events.push(Event::Trade {
639 exchange: key.exchange,
640 symbol: raw_symbol.to_string(),
641 point: pt,
642 });
643 }
644 }
645 if seed_events.is_empty() {
646 return Vec::new();
647 }
648 let mut state = TradeToFootprintDerived::new_for_key(key);
654 let mut emissions: Vec<FootprintPoint> = Vec::new();
655 for chunk in seed_events.chunks(5_000) {
656 emissions.extend(state.seed_from_events(chunk, 0));
657 #[cfg(target_arch = "wasm32")]
658 gloo_timers::future::sleep(std::time::Duration::from_millis(0)).await;
659 #[cfg(not(target_arch = "wasm32"))]
660 tokio::task::yield_now().await;
661 }
662 let mut rebuilt: Vec<FootprintPoint> = Vec::new();
667 for p in emissions {
668 match rebuilt.last_mut() {
669 Some(last) if last.timestamp_ms() == p.timestamp_ms() => *last = p,
670 _ => rebuilt.push(p),
671 }
672 }
673 if rebuilt.is_empty() {
674 return Vec::new();
675 }
676
677 let mut guard = handle.write().await;
683 let existing = guard.snapshot();
684 let head_ts = existing.first().map(|p| p.timestamp_ms());
685 let older: Vec<FootprintPoint> = rebuilt
686 .into_iter()
687 .filter(|p| head_ts.is_none_or(|h| p.timestamp_ms() <= h))
688 .collect();
689 if older.is_empty() {
690 return Vec::new();
691 }
692 let boundary_replaced = head_ts
693 .is_some_and(|h| older.last().is_some_and(|p| p.timestamp_ms() == h));
694 let keep_from = usize::from(boundary_replaced);
695 let merged_len = older.len() + existing.len().saturating_sub(keep_from);
696 let mut fresh = Series::new(merged_len.max(guard.capacity()));
697 fresh.extend(older.iter().cloned());
698 fresh.extend(existing.into_iter().skip(keep_from));
699 *guard = fresh;
700 older
701 }
702
703 pub async fn rewarm_derived(
724 &self,
725 key: &SeriesKey,
726 raw_symbol: &str,
727 warm_n: usize,
728 ) -> Result<RewarmOutcome> {
729 match &key.kind {
730 Kind::RenkoBar(_, _) => {
731 let (v, seed) = self.rewarm_renko(key, raw_symbol, warm_n).await;
732 Ok(RewarmOutcome { points: RewarmedPoints::Renko(v), seed })
733 }
734 Kind::PnfBar(_, _) => {
735 let (v, seed) = self.rewarm_pnf(key, raw_symbol, warm_n).await;
736 Ok(RewarmOutcome { points: RewarmedPoints::Pnf(v), seed })
737 }
738 Kind::KagiBar(_) => {
739 let (v, seed) = self.rewarm_derived_standalone::<TradeToKagiBarDerived>(key, raw_symbol, warm_n).await;
740 Ok(RewarmOutcome { points: RewarmedPoints::Kagi(v), seed })
741 }
742 Kind::ThreeLineBreak { .. } => {
743 let (v, seed) = self.rewarm_derived_standalone::<TradeToThreeLineBreakDerived>(key, raw_symbol, warm_n).await;
744 Ok(RewarmOutcome { points: RewarmedPoints::ThreeLineBreak(v), seed })
745 }
746 Kind::RangeBar(_) => {
747 let (v, seed) = self.rewarm_derived_standalone::<TradeToRangeBarDerived>(key, raw_symbol, warm_n).await;
748 Ok(RewarmOutcome { points: RewarmedPoints::Bar(v), seed })
749 }
750 Kind::VolumeBar(_) => {
751 let (v, seed) = self.rewarm_derived_standalone::<TradeToVolumeBarDerived>(key, raw_symbol, warm_n).await;
752 Ok(RewarmOutcome { points: RewarmedPoints::Bar(v), seed })
753 }
754 Kind::TickBar(_) => {
755 let (v, seed) = self.rewarm_derived_standalone::<TradeToTickBarDerived>(key, raw_symbol, warm_n).await;
756 Ok(RewarmOutcome { points: RewarmedPoints::Bar(v), seed })
757 }
758 Kind::DollarBar { .. } => {
759 let (v, seed) = self.rewarm_derived_standalone::<TradeToDollarBarDerived>(key, raw_symbol, warm_n).await;
760 Ok(RewarmOutcome { points: RewarmedPoints::Bar(v), seed })
761 }
762 Kind::CvdLine => {
763 let (v, seed) = self.rewarm_cvd(key, raw_symbol, warm_n).await;
764 Ok(RewarmOutcome { points: RewarmedPoints::Cvd(v), seed })
765 }
766 Kind::TpoProfile(_, _) => {
767 let (v, seed) = self.rewarm_tpo(key, raw_symbol, warm_n).await;
768 Ok(RewarmOutcome { points: RewarmedPoints::Tpo(v), seed })
769 }
770 Kind::TickImbalanceBar { .. } => Err(StationError::RewarmUnsupported(
771 "TickImbalanceBar carries EMA state (E[T]/E[|theta|]) over the entire \
772 history — not reconstructible from emitted output; use teardown+resubscribe."
773 .to_string(),
774 )),
775 Kind::VolumeImbalanceBar { .. } => Err(StationError::RewarmUnsupported(
776 "VolumeImbalanceBar carries EMA state over the entire history — not \
777 reconstructible from emitted output; use teardown+resubscribe."
778 .to_string(),
779 )),
780 Kind::RunBar { .. } => Err(StationError::RewarmUnsupported(
781 "RunBar carries EMA state over the entire history — not reconstructible \
782 from emitted output; use teardown+resubscribe."
783 .to_string(),
784 )),
785 Kind::Footprint(_) => Err(StationError::RewarmUnsupported(
786 "Footprint has its own dedicated rewarm_footprint — call that instead."
787 .to_string(),
788 )),
789 other => Err(StationError::RewarmUnsupported(format!(
790 "{other:?} is not a prepend-fold-capable derived kind"
791 ))),
792 }
793 }
794
795 async fn rewarm_fetch_trade_window(
805 &self,
806 key: &SeriesKey,
807 raw_symbol: &str,
808 warm_n: usize,
809 ) -> (Vec<Event>, crate::SeedOutcome) {
810 if warm_n == 0 {
811 return (Vec::new(), crate::SeedOutcome::full(0, 0, crate::SeedSource::AggTradesPaginated));
812 }
813 let caps_opt = self.inner.hub.capabilities(key.exchange);
814 let use_agg = caps_opt.as_ref().map(|c| c.has_agg_trades).unwrap_or(false);
815 let mut seed_events: Vec<Event> = Vec::new();
816 let outcome;
817 if use_agg {
818 let n_pages = ((warm_n + 999) / 1000).max(1);
819 let (agg, agg_outcome) = crate::backfill::agg_trades_paginated(
820 &self.inner.hub, key.exchange, key.account_type, raw_symbol, 1000, n_pages,
821 )
822 .await;
823 for ap in agg {
824 seed_events.push(Event::Trade {
825 exchange: key.exchange,
826 symbol: raw_symbol.to_string(),
827 point: TradePoint {
828 ts_ms: ap.ts_ms,
829 price: ap.price,
830 quantity: ap.quantity,
831 side: ap.side,
832 trade_id_hash: 0,
833 },
834 });
835 }
836 outcome = agg_outcome;
837 } else {
838 let (trades, trades_outcome) = crate::backfill::trades_recent(
839 &self.inner.hub, key.exchange, key.account_type, raw_symbol, warm_n,
840 )
841 .await;
842 for pt in trades {
843 seed_events.push(Event::Trade {
844 exchange: key.exchange,
845 symbol: raw_symbol.to_string(),
846 point: pt,
847 });
848 }
849 outcome = trades_outcome;
850 }
851 (seed_events, outcome)
852 }
853
854 async fn rewarm_fetch_kline_sufficient_window(
874 &self,
875 key: &SeriesKey,
876 raw_symbol: &str,
877 warm_n: usize,
878 ) -> (Vec<Event>, crate::SeedOutcome) {
879 if warm_n == 0 {
880 return (Vec::new(), crate::SeedOutcome::full(0, 0, crate::SeedSource::Klines));
881 }
882 let n_kline_pages = ((warm_n + 999) / 1000).clamp(1, 100);
883 let (kline_bars, outcome) = crate::backfill::klines_paginated(
884 &self.inner.hub, key.exchange, key.account_type, raw_symbol, "1m", 1000, n_kline_pages,
885 )
886 .await;
887 let kline_interval_ms = interval_to_ms("1m").unwrap_or(60_000);
888 let mut events = Vec::new();
889 if let Some((_last, closed)) = kline_bars.split_last() {
890 for bar in closed {
891 events.extend(crate::derived::kline_to_synthetic_trades(
892 bar, kline_interval_ms, key.exchange, raw_symbol,
893 ));
894 }
895 }
896 (events, outcome)
897 }
898
899 async fn rewarm_fold<D: DerivedStream>(
905 key: &SeriesKey,
906 events: &[Event],
907 seed: impl FnOnce(&mut D),
908 ) -> Vec<D::Output> {
909 let mut state = D::new_for_key(key);
910 seed(&mut state);
911 let mut emissions: Vec<D::Output> = Vec::new();
912 for chunk in events.chunks(5_000) {
913 emissions.extend(state.seed_from_events(chunk, 0));
914 #[cfg(target_arch = "wasm32")]
915 gloo_timers::future::sleep(std::time::Duration::from_millis(0)).await;
916 #[cfg(not(target_arch = "wasm32"))]
917 tokio::task::yield_now().await;
918 }
919 emissions
920 }
921
922 async fn rewarm_splice<T: DataPoint + Clone, Id: PartialEq>(
937 &self,
938 key: &SeriesKey,
939 rebuilt: Vec<T>,
940 ident: impl Fn(&T) -> Id,
941 ) -> Vec<T> {
942 let Some(handle) = self.inner.series_handles.get(key).and_then(|e| {
943 e.downcast_ref::<Arc<RwLock<Series<T>>>>().map(Arc::clone)
944 }) else {
945 return Vec::new();
946 };
947 if rebuilt.is_empty() {
948 return Vec::new();
949 }
950 let mut guard = handle.write().await;
951 let existing = guard.snapshot();
952 let head_ts = existing.first().map(|p| p.timestamp_ms());
953 let mut older: Vec<T> = rebuilt
954 .into_iter()
955 .filter(|p| head_ts.is_none_or(|h| p.timestamp_ms() <= h))
956 .collect();
957 if older.is_empty() {
958 return Vec::new();
959 }
960 let head_ident = existing.first().map(&ident);
965 if head_ident.as_ref().is_some_and(|h| older.last().is_some_and(|p| &ident(p) == h)) {
966 older.pop();
967 }
968 if older.is_empty() {
969 return Vec::new();
970 }
971 let merged_len = older.len() + existing.len();
972 let mut fresh = Series::new(merged_len.max(guard.capacity()));
973 fresh.extend(older.iter().cloned());
974 fresh.extend(existing);
975 *guard = fresh;
976 older
977 }
978
979 fn rewarm_dedup<T: Clone, Id: PartialEq>(emissions: Vec<T>, ident: impl Fn(&T) -> Id) -> Vec<T> {
986 let mut out: Vec<T> = Vec::new();
987 for p in emissions {
988 match out.last() {
989 Some(last) if ident(last) == ident(&p) => {
990 let idx = out.len() - 1;
991 out[idx] = p;
992 }
993 _ => out.push(p),
994 }
995 }
996 out
997 }
998
999 async fn rewarm_renko(
1011 &self,
1012 key: &SeriesKey,
1013 raw_symbol: &str,
1014 warm_n: usize,
1015 ) -> (Vec<RenkoBrickPoint>, crate::SeedOutcome) {
1016 let empty_outcome = crate::SeedOutcome::full(warm_n, 0, crate::SeedSource::AggTradesPaginated);
1017 let Some(handle) = self.inner.series_handles.get(key).and_then(|e| {
1018 e.downcast_ref::<Arc<RwLock<Series<RenkoBrickPoint>>>>().map(Arc::clone)
1019 }) else {
1020 return (Vec::new(), empty_outcome);
1021 };
1022 let grid_floor = {
1023 let guard = handle.read().await;
1024 match guard.snapshot().first() {
1025 Some(first) => first.bottom,
1026 None => return (Vec::new(), empty_outcome),
1027 }
1028 };
1029 let (events, outcome) = self.rewarm_fetch_kline_sufficient_window(key, raw_symbol, warm_n).await;
1030 if events.is_empty() {
1031 return (Vec::new(), outcome);
1032 }
1033 let emissions = Self::rewarm_fold::<TradeToRenkoBarDerived>(key, &events, |state| {
1034 state.preset_grid_anchor(grid_floor);
1035 })
1036 .await;
1037 let rebuilt = Self::rewarm_dedup(emissions, |p: &RenkoBrickPoint| p.open_time);
1038 if rebuilt.is_empty() {
1039 return (Vec::new(), outcome);
1040 }
1041 (self.rewarm_splice(key, rebuilt, |p: &RenkoBrickPoint| p.open_time).await, outcome)
1042 }
1043
1044 async fn rewarm_pnf(
1048 &self,
1049 key: &SeriesKey,
1050 raw_symbol: &str,
1051 warm_n: usize,
1052 ) -> (Vec<PnfColumnPoint>, crate::SeedOutcome) {
1053 let empty_outcome = crate::SeedOutcome::full(warm_n, 0, crate::SeedSource::AggTradesPaginated);
1054 let Some(handle) = self.inner.series_handles.get(key).and_then(|e| {
1055 e.downcast_ref::<Arc<RwLock<Series<PnfColumnPoint>>>>().map(Arc::clone)
1056 }) else {
1057 return (Vec::new(), empty_outcome);
1058 };
1059 let (bottom, top, is_x) = {
1060 let guard = handle.read().await;
1061 match guard.snapshot().first() {
1062 Some(first) => (first.bottom, first.top, first.is_x),
1063 None => return (Vec::new(), empty_outcome),
1064 }
1065 };
1066 let (events, outcome) = self.rewarm_fetch_kline_sufficient_window(key, raw_symbol, warm_n).await;
1067 if events.is_empty() {
1068 return (Vec::new(), outcome);
1069 }
1070 let emissions = Self::rewarm_fold::<TradeToPnfBarDerived>(key, &events, |state| {
1071 state.preset_grid(bottom, top, is_x);
1072 })
1073 .await;
1074 let rebuilt = Self::rewarm_dedup(emissions, |p: &PnfColumnPoint| p.column_id);
1075 if rebuilt.is_empty() {
1076 return (Vec::new(), outcome);
1077 }
1078 (self.rewarm_splice(key, rebuilt, |p: &PnfColumnPoint| p.column_id).await, outcome)
1079 }
1080
1081 async fn rewarm_derived_standalone<D>(
1097 &self,
1098 key: &SeriesKey,
1099 raw_symbol: &str,
1100 warm_n: usize,
1101 ) -> (Vec<D::Output>, crate::SeedOutcome)
1102 where
1103 D: DerivedStream,
1104 D::Output: Clone,
1105 {
1106 let empty_outcome = crate::SeedOutcome::full(warm_n, 0, crate::SeedSource::AggTradesPaginated);
1107 if self.inner.series_handles.get(key).is_none() {
1108 return (Vec::new(), empty_outcome);
1109 }
1110 let is_kline_sufficient_kind = matches!(
1111 key.kind,
1112 Kind::KagiBar(_) | Kind::ThreeLineBreak { .. }
1113 | Kind::RangeBar(_) | Kind::VolumeBar(_) | Kind::DollarBar { .. }
1114 );
1115 let (events, outcome) = if is_kline_sufficient_kind {
1116 self.rewarm_fetch_kline_sufficient_window(key, raw_symbol, warm_n).await
1117 } else {
1118 self.rewarm_fetch_trade_window(key, raw_symbol, warm_n).await
1119 };
1120 if events.is_empty() {
1121 return (Vec::new(), outcome);
1122 }
1123 let emissions = Self::rewarm_fold::<D>(key, &events, |_state| {}).await;
1124 let rebuilt = Self::rewarm_dedup(emissions, |p: &D::Output| p.timestamp_ms());
1125 if rebuilt.is_empty() {
1126 return (Vec::new(), outcome);
1127 }
1128 (self.rewarm_splice(key, rebuilt, |p: &D::Output| p.timestamp_ms()).await, outcome)
1129 }
1130
1131 async fn rewarm_cvd(
1138 &self,
1139 key: &SeriesKey,
1140 raw_symbol: &str,
1141 warm_n: usize,
1142 ) -> (Vec<ScalarBarPoint>, crate::SeedOutcome) {
1143 let empty_outcome = crate::SeedOutcome::full(warm_n, 0, crate::SeedSource::AggTradesPaginated);
1144 let Some(handle) = self.inner.series_handles.get(key).and_then(|e| {
1145 e.downcast_ref::<Arc<RwLock<Series<ScalarBarPoint>>>>().map(Arc::clone)
1146 }) else {
1147 return (Vec::new(), empty_outcome);
1148 };
1149 let existing_first_value = {
1150 let guard = handle.read().await;
1151 match guard.snapshot().first() {
1152 Some(first) => first.value,
1153 None => return (Vec::new(), empty_outcome),
1154 }
1155 };
1156 let (events, outcome) = self.rewarm_fetch_trade_window(key, raw_symbol, warm_n).await;
1157 if events.is_empty() {
1158 return (Vec::new(), outcome);
1159 }
1160 let emissions = Self::rewarm_fold::<TradeToCvdLineDerived>(key, &events, |_state| {}).await;
1161 let mut rebuilt = Self::rewarm_dedup(emissions, |p: &ScalarBarPoint| p.ts_ms);
1162 if rebuilt.is_empty() {
1163 return (Vec::new(), outcome);
1164 }
1165 let offset = existing_first_value - rebuilt.last().map(|p| p.value).unwrap_or(0.0);
1169 for p in &mut rebuilt {
1170 p.value += offset;
1171 }
1172 (self.rewarm_splice(key, rebuilt, |p: &ScalarBarPoint| p.ts_ms).await, outcome)
1173 }
1174
1175 async fn rewarm_tpo(
1183 &self,
1184 key: &SeriesKey,
1185 raw_symbol: &str,
1186 warm_n: usize,
1187 ) -> (Vec<TpoSessionPoint>, crate::SeedOutcome) {
1188 let empty_outcome = crate::SeedOutcome::full(warm_n, 0, crate::SeedSource::Klines);
1189 if self.inner.series_handles.get(key).is_none() {
1190 return (Vec::new(), empty_outcome);
1191 }
1192 let (rebuilt, outcome): (Vec<TpoSessionPoint>, crate::SeedOutcome) = match &key.kind {
1193 Kind::TpoProfile(_, TpoSource::Kline1m) => {
1194 let (kline_bars, kline_outcome) = crate::backfill::klines_paginated(
1195 &self.inner.hub, key.exchange, key.account_type, raw_symbol, "1m", 1000,
1196 ((warm_n + 999) / 1000).max(1),
1197 )
1198 .await;
1199 if kline_bars.is_empty() {
1200 return (Vec::new(), kline_outcome);
1201 }
1202 let interval = digdigdig3::core::websocket::KlineInterval::new("1m");
1203 let events: Vec<Event> = kline_bars
1204 .into_iter()
1205 .map(|point| Event::Bar {
1206 exchange: key.exchange,
1207 symbol: raw_symbol.to_string(),
1208 timeframe: interval.clone(),
1209 point,
1210 })
1211 .collect();
1212 let emissions = Self::rewarm_fold::<TpoFromKline1mDerived>(key, &events, |_state| {}).await;
1213 (Self::rewarm_dedup(emissions, |p: &TpoSessionPoint| p.open_time), kline_outcome)
1214 }
1215 Kind::TpoProfile(_, TpoSource::TradeBucket) => {
1216 let (events, trade_outcome) = self.rewarm_fetch_trade_window(key, raw_symbol, warm_n).await;
1217 if events.is_empty() {
1218 return (Vec::new(), trade_outcome);
1219 }
1220 let emissions = Self::rewarm_fold::<TpoFromTradeDerived>(key, &events, |_state| {}).await;
1221 (Self::rewarm_dedup(emissions, |p: &TpoSessionPoint| p.open_time), trade_outcome)
1222 }
1223 _ => return (Vec::new(), empty_outcome),
1224 };
1225 if rebuilt.is_empty() {
1226 return (Vec::new(), outcome);
1227 }
1228 (self.rewarm_splice(key, rebuilt, |p: &TpoSessionPoint| p.open_time).await, outcome)
1229 }
1230
1231 pub async fn warmup(&self, exchanges: &[ExchangeId]) -> crate::subscription::WarmupReport {
1244 use crate::subscription::{Event, WarmupReport};
1245
1246 let mut ok = Vec::new();
1247 let mut failed: Vec<(ExchangeId, String)> = Vec::new();
1248
1249 #[cfg(not(target_arch = "wasm32"))]
1251 {
1252 let mut join_set: tokio::task::JoinSet<(ExchangeId, Result<()>)> =
1253 tokio::task::JoinSet::new();
1254
1255 for &eid in exchanges {
1256 if self.inner.hub.is_connected(eid) {
1257 let _ = self.inner.connector_tx.send(Event::ConnectorReady { exchange: eid });
1258 ok.push(eid);
1259 } else {
1260 let hub = Arc::clone(&self.inner.hub);
1261 join_set.spawn(async move {
1262 (eid, hub.connect_public(eid, false).await.map_err(|e| {
1263 crate::StationError::Core(e.to_string())
1264 }))
1265 });
1266 }
1267 }
1268
1269 while let Some(res) = join_set.join_next().await {
1270 match res {
1271 Ok((eid, Ok(()))) => {
1272 let _ = self.inner.connector_tx.send(Event::ConnectorReady { exchange: eid });
1273 ok.push(eid);
1274 }
1275 Ok((eid, Err(e))) => {
1276 tracing::warn!(?eid, ?e, "warmup: connect_public failed");
1277 failed.push((eid, e.to_string()));
1278 }
1279 Err(join_err) => {
1280 tracing::warn!(?join_err, "warmup: task panicked");
1281 }
1282 }
1283 }
1284 }
1285 #[cfg(target_arch = "wasm32")]
1287 {
1288 for &eid in exchanges {
1289 if self.inner.hub.is_connected(eid) {
1290 let _ = self.inner.connector_tx.send(Event::ConnectorReady { exchange: eid });
1291 ok.push(eid);
1292 } else {
1293 match self.inner.hub.connect_public(eid, false).await {
1294 Ok(()) => {
1295 let _ = self.inner.connector_tx.send(Event::ConnectorReady { exchange: eid });
1296 ok.push(eid);
1297 }
1298 Err(e) => {
1299 tracing::warn!(?eid, ?e, "warmup: connect_public failed");
1300 failed.push((eid, e.to_string()));
1301 }
1302 }
1303 }
1304 }
1305 }
1306
1307 const ACCOUNT_TYPES: &[AccountType] = &[AccountType::Spot, AccountType::FuturesCross];
1309
1310 for eid in ok.iter().copied() {
1311 let Some(connector) = self.inner.hub.rest(eid) else {
1312 continue;
1314 };
1315 for &at in ACCOUNT_TYPES {
1316 if let Some(cached) = self.inner.exchange_info_cache.get(&(eid, at)) {
1318 let symbols = cached.value().clone();
1319 let _ = self.inner.connector_tx.send(Event::SymbolsLoaded {
1320 exchange: eid,
1321 account_type: at,
1322 symbols,
1323 });
1324 continue;
1325 }
1326 match connector.get_exchange_info(at).await {
1328 Ok(symbols) if !symbols.is_empty() => {
1329 self.inner.exchange_info_cache.insert((eid, at), symbols.clone());
1330 let _ = self.inner.connector_tx.send(Event::SymbolsLoaded {
1331 exchange: eid,
1332 account_type: at,
1333 symbols,
1334 });
1335 }
1336 Ok(_empty) => {
1337 }
1339 Err(e) => {
1340 use digdigdig3::core::types::ExchangeError;
1341 match &e {
1342 ExchangeError::WireAbsent(_)
1343 | ExchangeError::NotImplemented(_) => {
1344 }
1347 other => {
1348 tracing::warn!(?eid, ?at, ?other, "warmup: get_exchange_info failed");
1349 failed.push((eid, other.to_string()));
1350 }
1351 }
1352 }
1353 }
1354 }
1355 }
1356
1357 WarmupReport { ok, failed }
1358 }
1359
1360 pub async fn subscribe(&self, set: SubscriptionSet) -> Result<SubscribeReport> {
1372 if set.is_empty() {
1373 return Err(StationError::Subscribe("empty SubscriptionSet".into()));
1374 }
1375
1376 let (tx, rx) = mpsc::unbounded_channel::<Event>();
1377 let mut refs: Vec<MultiplexRef> = Vec::new();
1378 let mut ok: Vec<SeriesKey> = Vec::new();
1379 let mut failed: Vec<FailedStream> = Vec::new();
1380 let mut seed_outcomes: Vec<(SeriesKey, crate::SeedOutcome)> = Vec::new();
1381
1382 for entry in set.entries {
1383 if let Err(e) = self
1387 .inner
1388 .hub
1389 .connect_public(entry.exchange, false)
1390 .await
1391 {
1392 tracing::debug!(?e, ?entry.exchange, "connect_public failed; warm-start REST backfill will be skipped");
1393 }
1394
1395 let needs_ws = entry.streams.iter().any(|s| {
1405 s.to_kind().is_poll_only().is_none()
1406 });
1407
1408 if needs_ws {
1409 let ws_connect_result = if let Some(ref creds) = entry.credentials {
1413 #[cfg(not(target_arch = "wasm32"))]
1414 {
1415 self.inner
1416 .hub
1417 .connect_websocket_with_credentials(
1418 entry.exchange,
1419 entry.account_type,
1420 creds.clone(),
1421 )
1422 .await
1423 }
1424 #[cfg(target_arch = "wasm32")]
1425 {
1426 let _ = creds;
1427 Err(digdigdig3::core::types::ExchangeError::NotImplemented(
1428 "private WS streams not supported on wasm32".into(),
1429 ))
1430 }
1431 } else {
1432 self.inner
1433 .hub
1434 .connect_websocket(entry.exchange, entry.account_type, false)
1435 .await
1436 };
1437 if let Err(e) = ws_connect_result
1438 {
1439 let err_msg = format!("connect_websocket: {e}");
1440 for s in &entry.streams {
1441 if s.to_kind().is_poll_only().is_some() {
1445 continue;
1446 }
1447 failed.push(FailedStream {
1448 exchange: entry.exchange,
1449 account_type: entry.account_type,
1450 symbol: entry.symbol.clone(),
1451 stream: s.clone(),
1452 error: StationError::Core(err_msg.clone()),
1453 });
1454 }
1455 let has_non_ws = entry.streams.iter().any(|s| {
1458 s.to_kind().is_poll_only().is_some()
1459 });
1460 if !has_non_ws {
1461 continue;
1462 }
1463 }
1464 }
1465
1466 let (canonical, raw) = if entry.is_raw {
1477 (
1478 Symbol::with_raw("", "", entry.symbol.clone()),
1479 entry.symbol.clone(),
1480 )
1481 } else {
1482 let canonical = parse_symbol(&entry.symbol);
1483 match SymbolNormalizer::to_exchange(
1484 entry.exchange,
1485 &canonical,
1486 entry.account_type,
1487 ) {
1488 Ok(r) => (canonical, r),
1489 Err(e) => {
1490 let err_msg = format!("symbol normalize: {e}");
1491 for s in &entry.streams {
1492 failed.push(FailedStream {
1493 exchange: entry.exchange,
1494 account_type: entry.account_type,
1495 symbol: entry.symbol.clone(),
1496 stream: s.clone(),
1497 error: StationError::Subscribe(err_msg.clone()),
1498 });
1499 }
1500 continue;
1501 }
1502 }
1503 };
1504
1505 let raw = if let Some(rest) = self.inner.hub.rest(entry.exchange) {
1513 rest.resolve_market_symbol(&raw, entry.account_type).await
1514 } else {
1515 raw
1516 };
1517
1518 for s in &entry.streams {
1519 let kind = s.to_kind();
1520 let key = SeriesKey {
1521 exchange: entry.exchange,
1522 account_type: entry.account_type,
1523 symbol: raw.clone(),
1524 kind: kind.clone(),
1525 };
1526
1527 if let Some(caps) = self.inner.hub.capabilities(entry.exchange) {
1532 if caps_explicitly_unsupported(&caps, &kind) {
1533 let reason = format!(
1534 "{:?} capability not declared on {:?} — neither WS nor REST available",
1535 kind, entry.exchange,
1536 );
1537 tracing::debug!(
1538 ?key, reason,
1539 "capability fail-fast: skipping acquire_or_spawn"
1540 );
1541 failed.push(FailedStream {
1542 exchange: entry.exchange,
1543 account_type: entry.account_type,
1544 symbol: entry.symbol.clone(),
1545 stream: s.clone(),
1546 error: StationError::StreamNotSupported(reason),
1547 });
1548 continue;
1549 }
1550 }
1551
1552 let (bcast_tx, pending_seed) = match self
1553 .acquire_or_spawn(&key, &entry, &canonical, &raw, s)
1554 .await
1555 {
1556 Ok(pair) => pair,
1557 Err(e) => {
1558 if e.is_not_supported() {
1562 tracing::debug!(?key, ?e, "stream not supported; skipping");
1563 } else {
1564 tracing::info!(?key, ?e, "subscribe failed; skipping");
1565 }
1566 failed.push(FailedStream {
1567 exchange: entry.exchange,
1568 account_type: entry.account_type,
1569 symbol: entry.symbol.clone(),
1570 stream: s.clone(),
1571 error: e,
1572 });
1573 continue;
1574 }
1575 };
1576
1577 let mut bcast_rx = bcast_tx.subscribe();
1578 let tx_clone = tx.clone();
1579 let label = entry.symbol.clone();
1585 {
1586 let relay_fut = Box::pin(async move {
1587 loop {
1601 match bcast_rx.recv().await {
1602 Ok(mut ev) => {
1603 ev.set_symbol(label.clone());
1604 if tx_clone.send(ev).is_err() {
1605 break;
1606 }
1607 }
1608 Err(tokio::sync::broadcast::error::RecvError::Lagged(_n)) => {
1609 continue;
1611 }
1612 Err(tokio::sync::broadcast::error::RecvError::Closed) => {
1613 break;
1614 }
1615 }
1616 }
1617 });
1618 #[cfg(not(target_arch = "wasm32"))]
1619 tokio::spawn(relay_fut);
1620 #[cfg(target_arch = "wasm32")]
1621 wasm_bindgen_futures::spawn_local(relay_fut);
1622 }
1623
1624 if let Some(seed_ev) = pending_seed {
1630 if bcast_tx.send(seed_ev).is_err() {
1631 tracing::debug!(
1632 target: "dig3::ob_seed",
1633 ?key,
1634 "ob delta seed: send failed after relay wired (unexpected)"
1635 );
1636 }
1637 }
1638
1639 refs.push(MultiplexRef {
1640 station: Arc::downgrade(&self.inner),
1641 key: key.clone(),
1642 });
1643 if let Some((_, outcome)) = self.inner.seed_outcomes.remove(&key) {
1648 seed_outcomes.push((key.clone(), outcome));
1649 }
1650 ok.push(key);
1651 }
1652 }
1653
1654 Ok(SubscribeReport {
1655 handle: SubscriptionHandle { rx, _refs: refs },
1656 ok,
1657 failed,
1658 seed_outcomes,
1659 })
1660 }
1661
1662 async fn acquire_or_spawn(
1667 &self,
1668 key: &SeriesKey,
1669 entry: &Entry,
1670 canonical: &Symbol,
1671 raw_symbol: &str,
1672 stream: &Stream,
1673 ) -> Result<(broadcast::Sender<Event>, Option<Event>)> {
1674 if let Some(mut mux) = self.inner.muxes.get_mut(key) {
1675 if let Some(cancel) = mux.grace_cancel.take() {
1680 let _ = cancel.send(());
1681 }
1682 mux.consumers.fetch_add(1, Ordering::SeqCst);
1683 return Ok((mux.tx.clone(), None));
1684 }
1685
1686 if key.kind.is_derived() {
1697 #[cfg(not(target_arch = "wasm32"))]
1698 {
1699 if let Kind::Basis = &key.kind {
1700 let hub = &self.inner.hub;
1701 if let Some(source) = polling::basis_poll_source(hub, key.exchange) {
1702 tracing::info!(
1703 target: "dig3::station::native_source",
1704 exchange = ?key.exchange,
1705 symbol = %key.symbol,
1706 "Kind::Basis: native REST basis-history available — using poll path"
1707 );
1708 let poll_spec = crate::series::PollSpec {
1709 cadence: source.cadence(),
1710 jitter_pct: 10,
1711 };
1712 return self
1713 .acquire_or_spawn_polled_with_source::<BasisPoint, _>(
1714 key, poll_spec, raw_symbol, source,
1715 )
1716 .await
1717 .map(|tx| (tx, None));
1718 } else {
1719 tracing::info!(
1720 target: "dig3::station::native_source",
1721 exchange = ?key.exchange,
1722 symbol = %key.symbol,
1723 "Kind::Basis: no native source — deriving from mark+index in RAM"
1724 );
1725 }
1726 }
1727 if let Kind::FundingSettlement = &key.kind {
1728 let hub = &self.inner.hub;
1729 if let Some(source) = polling::funding_poll_source(hub, key.exchange) {
1730 tracing::info!(
1731 target: "dig3::station::native_source",
1732 exchange = ?key.exchange,
1733 symbol = %key.symbol,
1734 "Kind::FundingSettlement: native REST funding-rate history available — using poll path"
1735 );
1736 let poll_spec = crate::series::PollSpec {
1737 cadence: source.cadence(),
1738 jitter_pct: 10,
1739 };
1740 return self
1741 .acquire_or_spawn_polled_with_source::<FundingSettlementPoint, _>(
1742 key, poll_spec, raw_symbol, source,
1743 )
1744 .await
1745 .map(|tx| (tx, None));
1746 } else {
1747 tracing::info!(
1748 target: "dig3::station::native_source",
1749 exchange = ?key.exchange,
1750 symbol = %key.symbol,
1751 "Kind::FundingSettlement: no native source — deriving from funding-rate flips in RAM"
1752 );
1753 }
1754 }
1755 }
1756
1757 return match &key.kind {
1758 Kind::Basis => {
1759 self.acquire_or_spawn_derived::<BasisDerived>(key, entry, canonical, raw_symbol).await.map(|tx| (tx, None))
1760 }
1761 Kind::FundingSettlement => {
1762 self.acquire_or_spawn_derived::<FundingSettlementDerived>(key, entry, canonical, raw_symbol).await.map(|tx| (tx, None))
1763 }
1764 Kind::RangeBar(_) => {
1765 self.acquire_or_spawn_derived::<TradeToRangeBarDerived>(key, entry, canonical, raw_symbol).await.map(|tx| (tx, None))
1766 }
1767 Kind::TickBar(_) => {
1768 self.acquire_or_spawn_derived::<TradeToTickBarDerived>(key, entry, canonical, raw_symbol).await.map(|tx| (tx, None))
1769 }
1770 Kind::VolumeBar(_) => {
1771 self.acquire_or_spawn_derived::<TradeToVolumeBarDerived>(key, entry, canonical, raw_symbol).await.map(|tx| (tx, None))
1772 }
1773 Kind::Footprint(_) => {
1774 self.acquire_or_spawn_derived::<TradeToFootprintDerived>(key, entry, canonical, raw_symbol).await.map(|tx| (tx, None))
1775 }
1776 Kind::RenkoBar(_, _) => {
1777 self.acquire_or_spawn_derived::<TradeToRenkoBarDerived>(key, entry, canonical, raw_symbol).await.map(|tx| (tx, None))
1778 }
1779 Kind::PnfBar(_, _) => {
1780 self.acquire_or_spawn_derived::<TradeToPnfBarDerived>(key, entry, canonical, raw_symbol).await.map(|tx| (tx, None))
1781 }
1782 Kind::KagiBar(_) => {
1783 self.acquire_or_spawn_derived::<TradeToKagiBarDerived>(key, entry, canonical, raw_symbol).await.map(|tx| (tx, None))
1784 }
1785 Kind::CvdLine => {
1786 self.acquire_or_spawn_derived::<TradeToCvdLineDerived>(key, entry, canonical, raw_symbol).await.map(|tx| (tx, None))
1787 }
1788 Kind::ThreeLineBreak { .. } => {
1789 self.acquire_or_spawn_derived::<TradeToThreeLineBreakDerived>(key, entry, canonical, raw_symbol).await.map(|tx| (tx, None))
1790 }
1791 Kind::TpoProfile(_, TpoSource::Kline1m) => {
1792 self.acquire_or_spawn_derived::<TpoFromKline1mDerived>(key, entry, canonical, raw_symbol).await.map(|tx| (tx, None))
1793 }
1794 Kind::TpoProfile(_, TpoSource::TradeBucket) => {
1795 self.acquire_or_spawn_derived::<TpoFromTradeDerived>(key, entry, canonical, raw_symbol).await.map(|tx| (tx, None))
1796 }
1797 Kind::DollarBar { .. } => {
1798 self.acquire_or_spawn_derived::<TradeToDollarBarDerived>(key, entry, canonical, raw_symbol).await.map(|tx| (tx, None))
1799 }
1800 Kind::TickImbalanceBar { .. } => {
1801 self.acquire_or_spawn_derived::<TradeToTickImbalanceDerived>(key, entry, canonical, raw_symbol).await.map(|tx| (tx, None))
1802 }
1803 Kind::VolumeImbalanceBar { .. } => {
1804 self.acquire_or_spawn_derived::<TradeToVolumeImbalanceDerived>(key, entry, canonical, raw_symbol).await.map(|tx| (tx, None))
1805 }
1806 Kind::RunBar { .. } => {
1807 self.acquire_or_spawn_derived::<TradeToRunBarDerived>(key, entry, canonical, raw_symbol).await.map(|tx| (tx, None))
1808 }
1809 _ => unreachable!("is_derived() returned true for unhandled kind — update acquire_or_spawn dispatch"),
1810 };
1811 }
1812
1813 #[cfg(not(target_arch = "wasm32"))]
1817 if let Some(poll_spec) = key.kind.is_poll_only() {
1818 return self.acquire_or_spawn_polled(key, entry, poll_spec, raw_symbol).await.map(|tx| (tx, None));
1819 }
1820 #[cfg(target_arch = "wasm32")]
1822 if key.kind.is_poll_only().is_some() {
1823 return Err(StationError::StreamNotSupported(format!(
1824 "poll-only streams not supported on wasm32 ({:?})",
1825 key.kind
1826 )));
1827 }
1828
1829 let sym = Symbol::with_raw(&canonical.base, &canonical.quote, raw_symbol.to_string());
1830 let req = ws_request_for(&key.kind, sym, entry.account_type);
1831
1832 let ws = self
1833 .inner
1834 .hub
1835 .ws(entry.exchange, entry.account_type)
1836 .ok_or_else(|| StationError::Core("ws handle missing post-connect".into()))?;
1837 if let Err(e) = ws.subscribe(req.clone()).await {
1854 use digdigdig3::core::types::WebSocketError;
1855 let is_not_supported = matches!(
1856 e,
1857 WebSocketError::WireAbsent(_) | WebSocketError::NotImplemented(_)
1858 );
1859 if is_not_supported {
1860 if let Kind::Kline(iv) = &key.kind {
1861 if interval_to_ms(iv.as_str()).is_none() {
1863 return Err(StationError::StreamNotSupported(format!(
1864 "Kline interval {:?} is unknown — cannot aggregate from trades",
1865 iv.as_str()
1866 )));
1867 }
1868 tracing::debug!(
1869 target: "dig3::station::derived",
1870 exchange = ?key.exchange,
1871 symbol = %key.symbol,
1872 interval = %iv,
1873 "native Kline WS not supported — falling back to TradeToBarDerived"
1874 );
1875 return self
1876 .acquire_or_spawn_derived::<TradeToBarDerived>(key, entry, canonical, raw_symbol)
1877 .await
1878 .map(|tx| (tx, None));
1879 }
1880 }
1881 return Err(match e {
1882 WebSocketError::WireAbsent(msg)
1883 | WebSocketError::NotImplemented(msg) => {
1884 StationError::StreamNotSupported(msg)
1885 }
1886 other => StationError::Subscribe(format!("ws.subscribe: {other}")),
1887 });
1888 }
1889
1890 let (bcast_tx, _) = broadcast::channel::<Event>(1024);
1891 let consumers = Arc::new(AtomicUsize::new(1));
1892 let (shutdown_tx, shutdown_rx) = oneshot::channel::<()>();
1893
1894 let _ = stream; let warm_n = self.inner.warm_start_capacity;
1900 let hub = self.inner.hub.clone();
1901 let acct = entry.account_type;
1902 let raw_s = raw_symbol.to_string();
1903
1904 let mut pending_seed: Option<Event> = None;
1909
1910 match &key.kind {
1911 Kind::Trade => {
1912 let seed = if warm_n > 0 {
1913 crate::backfill::trades_recent(&hub, key.exchange, acct, &raw_s, warm_n).await.0
1914 } else { Vec::new() };
1915 spawn_forwarder::<TradePoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), seed, req.clone());
1916 }
1917 Kind::Kline(interval) => {
1918 let seed = if warm_n > 0 {
1919 crate::backfill::klines_recent(&hub, key.exchange, acct, &raw_s, interval.as_str(), warm_n).await
1920 } else { Vec::new() };
1921 spawn_forwarder::<BarPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), seed, req.clone());
1922 }
1923 Kind::AggTrade => {
1924 let seed = if warm_n > 0 {
1925 crate::backfill::agg_trades_recent(&hub, key.exchange, acct, &raw_s, warm_n).await.0
1926 } else { Vec::new() };
1927 spawn_forwarder::<AggTradePoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), seed, req.clone());
1928 }
1929 Kind::Ticker => match self.inner.persistence.depth_for(&key.kind) {
1930 Some(PersistDepth::Indicators) => {
1931 let seed = if warm_n > 0 {
1932 crate::backfill::tickers_indicators_recent(&hub, key.exchange, acct, &raw_s, warm_n).await
1933 } else { Vec::new() };
1934 spawn_forwarder::<TickerIndicatorsPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), seed, req.clone());
1935 }
1936 Some(PersistDepth::Full) => {
1937 let seed = if warm_n > 0 {
1938 crate::backfill::tickers_full_recent(&hub, key.exchange, acct, &raw_s, warm_n).await
1939 } else { Vec::new() };
1940 spawn_forwarder::<TickerFullPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), seed, req.clone());
1941 }
1942 _ => spawn_forwarder::<TickerPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
1943 },
1944 Kind::Orderbook => {
1945 let ob_seed = if self.inner.orderbook_rest_seed {
1946 ob_rest_seed(&hub, key.exchange, acct, &raw_s, self.inner.orderbook_seed_depth).await
1947 } else {
1948 Vec::new()
1949 };
1950 match self.inner.persistence.depth_for(&key.kind) {
1951 Some(PersistDepth::Indicators) | Some(PersistDepth::Full) => {
1952 spawn_forwarder::<ObSnapshotIndicatorsPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone());
1955 }
1956 _ => spawn_forwarder::<ObSnapshotPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), ob_seed, req.clone()),
1957 }
1958 }
1959 Kind::OrderbookDelta => {
1960 pending_seed = if self.inner.orderbook_rest_seed {
1969 #[cfg(not(target_arch = "wasm32"))]
1970 {
1971 let snapshots = ob_rest_seed(&hub, key.exchange, acct, &raw_s, self.inner.orderbook_seed_depth).await;
1972 snapshots.into_iter().next().map(|point| Event::OrderbookSnapshot {
1973 exchange: key.exchange,
1974 symbol: raw_s.clone(),
1975 point,
1976 })
1977 }
1978 #[cfg(target_arch = "wasm32")]
1979 {
1980 tracing::warn!(
1981 target: "dig3::ob_seed",
1982 exchange = ?key.exchange, symbol = raw_s.as_str(),
1983 "orderbook REST seed for delta stream skipped on wasm32 (possible CORS) — continuing WS-only"
1984 );
1985 None
1986 }
1987 } else {
1988 None
1989 };
1990 match self.inner.persistence.depth_for(&key.kind) {
1991 Some(PersistDepth::Indicators) | Some(PersistDepth::Full) =>
1992 spawn_forwarder::<ObDeltaIndicatorsPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
1993 _ => spawn_forwarder::<ObDeltaPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
1994 }
1995 }
1996 Kind::MarkPrice => match self.inner.persistence.depth_for(&key.kind) {
1997 Some(PersistDepth::Indicators) => {
1998 let seed = if warm_n > 0 {
1999 crate::backfill::mark_price_indicators_recent(&hub, key.exchange, acct, &raw_s, warm_n).await
2000 } else { Vec::new() };
2001 spawn_forwarder::<MarkPriceIndicatorsPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), seed, req.clone());
2002 }
2003 Some(PersistDepth::Full) => {
2004 let seed = if warm_n > 0 {
2005 crate::backfill::mark_price_full_recent(&hub, key.exchange, acct, &raw_s, warm_n).await
2006 } else { Vec::new() };
2007 spawn_forwarder::<MarkPriceFullPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), seed, req.clone());
2008 }
2009 _ => {
2010 let seed = if warm_n > 0 {
2011 crate::backfill::mark_price_recent(&hub, key.exchange, acct, &raw_s, warm_n).await
2012 } else { Vec::new() };
2013 spawn_forwarder::<MarkPricePoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), seed, req.clone());
2014 }
2015 },
2016 Kind::FundingRate => match self.inner.persistence.depth_for(&key.kind) {
2017 Some(PersistDepth::Indicators) => {
2018 let seed = if warm_n > 0 {
2019 crate::backfill::funding_rate_indicators_recent(&hub, key.exchange, acct, &raw_s, warm_n).await
2020 } else { Vec::new() };
2021 spawn_forwarder::<FundingRateIndicatorsPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), seed, req.clone());
2022 }
2023 Some(PersistDepth::Full) => {
2024 let seed = if warm_n > 0 {
2025 crate::backfill::funding_rate_full_recent(&hub, key.exchange, acct, &raw_s, warm_n).await
2026 } else { Vec::new() };
2027 spawn_forwarder::<FundingRateFullPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), seed, req.clone());
2028 }
2029 _ => {
2030 let seed = if warm_n > 0 {
2031 crate::backfill::funding_rate_recent(&hub, key.exchange, acct, &raw_s, warm_n).await
2032 } else { Vec::new() };
2033 spawn_forwarder::<FundingRatePoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), seed, req.clone());
2034 }
2035 },
2036 Kind::OpenInterest => match self.inner.persistence.depth_for(&key.kind) {
2037 Some(PersistDepth::Indicators) | Some(PersistDepth::Full) => {
2038 let seed = if warm_n > 0 {
2040 crate::backfill::open_interest_indicators_recent(&hub, key.exchange, acct, &raw_s, warm_n).await
2041 } else { Vec::new() };
2042 spawn_forwarder::<OpenInterestIndicatorsPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), seed, req.clone());
2043 }
2044 _ => {
2045 let seed = if warm_n > 0 {
2046 crate::backfill::open_interest_recent(&hub, key.exchange, acct, &raw_s, warm_n).await
2047 } else { Vec::new() };
2048 spawn_forwarder::<OpenInterestPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), seed, req.clone());
2049 }
2050 },
2051 Kind::Liquidation => match self.inner.persistence.depth_for(&key.kind) {
2052 Some(PersistDepth::Indicators) => {
2053 let seed = if warm_n > 0 {
2054 crate::backfill::liquidation_indicators_recent(&hub, key.exchange, acct, &raw_s, warm_n).await
2055 } else { Vec::new() };
2056 spawn_forwarder::<LiquidationIndicatorsPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), seed, req.clone());
2057 }
2058 Some(PersistDepth::Full) => {
2059 let seed = if warm_n > 0 {
2060 crate::backfill::liquidation_full_recent(&hub, key.exchange, acct, &raw_s, warm_n).await
2061 } else { Vec::new() };
2062 spawn_forwarder::<LiquidationFullPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), seed, req.clone());
2063 }
2064 _ => {
2065 let seed = if warm_n > 0 {
2066 crate::backfill::liquidations_recent(&hub, key.exchange, acct, &raw_s, warm_n).await
2067 } else { Vec::new() };
2068 spawn_forwarder::<LiquidationPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), seed, req.clone());
2069 }
2070 },
2071 Kind::BlockTrade => spawn_forwarder::<BlockTradePoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
2072 Kind::AuctionEvent => spawn_forwarder::<AuctionEventPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
2073 Kind::IndexPrice => match self.inner.persistence.depth_for(&key.kind) {
2074 Some(PersistDepth::Indicators) | Some(PersistDepth::Full) => spawn_forwarder::<IndexPriceIndicatorsPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
2075 _ => spawn_forwarder::<IndexPricePoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
2076 },
2077 Kind::CompositeIndex => spawn_forwarder::<CompositeIndexPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
2078 Kind::OptionGreeks => spawn_forwarder::<OptionGreeksPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
2079 Kind::VolatilityIndex => spawn_forwarder::<VolatilityIndexPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
2080 Kind::HistoricalVolatility => spawn_forwarder::<HistoricalVolatilityPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
2081 Kind::LongShortRatio => spawn_forwarder::<LongShortRatioPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
2085 Kind::TakerVolume => spawn_forwarder::<TakerVolumePoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
2086 Kind::LiquidationBucket => spawn_forwarder::<LiquidationBucketPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
2087 Kind::Basis => spawn_forwarder::<BasisPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
2088 Kind::InsuranceFund => {
2089 let seed = if warm_n > 0 {
2090 crate::backfill::insurance_fund_recent(&hub, key.exchange, acct, &raw_s, warm_n).await
2091 } else { Vec::new() };
2092 spawn_forwarder::<InsuranceFundPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), seed, req.clone());
2093 }
2094 Kind::OrderbookL3 => spawn_forwarder::<OrderbookL3Point>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
2095 Kind::SettlementEvent => spawn_forwarder::<SettlementEventPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
2096 Kind::MarketWarning => spawn_forwarder::<MarketWarningPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
2097 Kind::RiskLimit => spawn_forwarder::<RiskLimitPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
2098 Kind::PredictedFunding => spawn_forwarder::<PredictedFundingPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
2099 Kind::FundingSettlement => spawn_forwarder::<FundingSettlementPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
2100 Kind::MarkPriceKline(iv) => {
2101 let seed = if warm_n > 0 {
2102 crate::backfill::mark_price_klines_recent(&hub, key.exchange, acct, &raw_s, iv.as_str(), warm_n).await
2103 } else { Vec::new() };
2104 spawn_forwarder::<MarkPriceKlinePoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), seed, req.clone());
2105 }
2106 Kind::IndexPriceKline(iv) => {
2107 let seed = if warm_n > 0 {
2108 crate::backfill::index_price_klines_recent(&hub, key.exchange, acct, &raw_s, iv.as_str(), warm_n).await
2109 } else { Vec::new() };
2110 spawn_forwarder::<IndexPriceKlinePoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), seed, req.clone());
2111 }
2112 Kind::PremiumIndexKline(iv) => {
2113 let seed = if warm_n > 0 {
2114 crate::backfill::premium_index_klines_recent(&hub, key.exchange, acct, &raw_s, iv.as_str(), warm_n).await
2115 } else { Vec::new() };
2116 spawn_forwarder::<PremiumIndexKlinePoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), seed, req.clone());
2117 }
2118 Kind::OrderUpdate => spawn_forwarder::<OrderUpdatePoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
2120 Kind::BalanceUpdate => spawn_forwarder::<BalanceUpdatePoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
2121 Kind::PositionUpdate => spawn_forwarder::<PositionUpdatePoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req),
2122 Kind::RangeBar(_) | Kind::TickBar(_) | Kind::VolumeBar(_) | Kind::Footprint(_)
2125 | Kind::RenkoBar(_, _) | Kind::PnfBar(_, _) | Kind::KagiBar(_)
2126 | Kind::CvdLine | Kind::ThreeLineBreak { .. } | Kind::TpoProfile(_, _)
2127 | Kind::DollarBar { .. } | Kind::TickImbalanceBar { .. }
2128 | Kind::VolumeImbalanceBar { .. } | Kind::RunBar { .. } => {
2129 unreachable!("derived kinds dispatched before forwarder match")
2130 }
2131 }
2132
2133 self.inner.muxes.insert(
2134 key.clone(),
2135 Multiplexer { tx: bcast_tx.clone(), consumers, shutdown: Some(shutdown_tx), grace_cancel: None },
2136 );
2137
2138 Ok((bcast_tx, pending_seed))
2139 }
2140}
2141
2142impl Station {
2143 #[cfg(not(target_arch = "wasm32"))]
2158 fn acquire_or_spawn_derived<'a, D: DerivedStream>(
2159 &'a self,
2160 key: &'a SeriesKey,
2161 entry: &'a Entry,
2162 canonical: &'a digdigdig3::core::types::Symbol,
2163 raw_symbol: &'a str,
2164 ) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<broadcast::Sender<Event>>> + Send + 'a>>
2165 where
2166 Event: EventFrom<D::Output>,
2167 {
2168 Box::pin(async move {
2169 self.acquire_or_spawn_derived_body::<D>(key, entry, canonical, raw_symbol).await
2170 })
2171 }
2172
2173 #[cfg(target_arch = "wasm32")]
2176 fn acquire_or_spawn_derived<'a, D: DerivedStream>(
2177 &'a self,
2178 key: &'a SeriesKey,
2179 entry: &'a Entry,
2180 canonical: &'a digdigdig3::core::types::Symbol,
2181 raw_symbol: &'a str,
2182 ) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<broadcast::Sender<Event>>> + 'a>>
2183 where
2184 Event: EventFrom<D::Output>,
2185 {
2186 Box::pin(async move {
2187 self.acquire_or_spawn_derived_body::<D>(key, entry, canonical, raw_symbol).await
2188 })
2189 }
2190
2191 async fn acquire_or_spawn_derived_body<D: DerivedStream>(
2193 &self,
2194 key: &SeriesKey,
2195 entry: &Entry,
2196 canonical: &digdigdig3::core::types::Symbol,
2197 raw_symbol: &str,
2198 ) -> Result<broadcast::Sender<Event>>
2199 where
2200 Event: EventFrom<D::Output>,
2201 {
2202 let mut upstream_rxs: Vec<broadcast::Receiver<Event>> = Vec::new();
2203 let mut upstream_keys: Vec<SeriesKey> = Vec::new();
2204
2205 let resolved_deps: Vec<Stream> = D::deps_for_key(key);
2209 for dep_stream in &resolved_deps {
2210 let dep_kind = dep_stream.to_kind();
2211 debug_assert!(
2212 !dep_kind.is_derived(),
2213 "DerivedStream::deps_for_key() must not list derived kinds (no derived-of-derived)"
2214 );
2215 let dep_key = SeriesKey {
2216 exchange: key.exchange,
2217 account_type: key.account_type,
2218 symbol: raw_symbol.to_string(),
2219 kind: dep_kind,
2220 };
2221 let (up_tx, _) = self
2227 .acquire_or_spawn(&dep_key, entry, canonical, raw_symbol, dep_stream)
2228 .await?;
2229 upstream_rxs.push(up_tx.subscribe());
2230 upstream_keys.push(dep_key);
2231 }
2232
2233 let warm_n = entry.warm_override.unwrap_or(self.inner.warm_start_capacity);
2243 let is_kline_sufficient_kind = matches!(
2255 key.kind,
2256 Kind::RenkoBar(_, _) | Kind::PnfBar(_, _) | Kind::KagiBar(_) | Kind::ThreeLineBreak { .. }
2257 | Kind::RangeBar(_) | Kind::VolumeBar(_) | Kind::DollarBar { .. }
2258 );
2259 let mut kline_seed_baseline: Option<(i64, f64, f64)> = None;
2265 let mut agg_seed_per_dep: Vec<Vec<Event>> = Vec::with_capacity(resolved_deps.len());
2266 for dep_stream in &resolved_deps {
2267 let dep_kind = dep_stream.to_kind();
2268 let mut seed_events: Vec<Event> = Vec::new();
2269 if warm_n > 0 && is_kline_sufficient_kind && matches!(dep_kind, Kind::Kline(_)) {
2270 let n_kline_pages = ((warm_n + 999) / 1000).clamp(1, 100);
2280 let (kline_bars, k_outcome) = crate::backfill::klines_paginated(
2281 &self.inner.hub, key.exchange, entry.account_type, raw_symbol,
2282 "1m", 1000, n_kline_pages,
2283 ).await;
2284 let kline_interval_ms = interval_to_ms("1m").unwrap_or(60_000);
2285 if let Some((last, closed)) = kline_bars.split_last() {
2301 for bar in closed {
2302 seed_events.extend(crate::derived::kline_to_synthetic_trades(
2303 bar, kline_interval_ms, key.exchange, raw_symbol,
2304 ));
2305 }
2306 kline_seed_baseline = Some((last.open_time, last.volume, last.close));
2307 }
2308 self.inner.seed_outcomes.insert(key.clone(), k_outcome);
2309 } else if warm_n > 0 && matches!(dep_kind, Kind::Trade) {
2310 let caps_opt = self.inner.hub.capabilities(key.exchange);
2311 let use_agg = caps_opt.as_ref().map(|c| c.has_agg_trades).unwrap_or(false);
2312 let tier = self.inner.hub
2318 .trade_history_capabilities(key.exchange)
2319 .map(|c| c.tier_for(entry.account_type))
2320 .unwrap_or(digdigdig3::core::types::TradeHistoryTier::RecentOnly { max_trades: 1000 });
2321 let trade_outcome = if matches!(tier, digdigdig3::core::types::TradeHistoryTier::RecentOnly { .. }) {
2322 let (trades, outcome) = crate::backfill::trades_recent(
2325 &self.inner.hub, key.exchange, entry.account_type, raw_symbol, warm_n,
2326 ).await;
2327 for pt in trades {
2328 seed_events.push(Event::Trade {
2329 exchange: key.exchange,
2330 symbol: raw_symbol.to_string(),
2331 point: pt,
2332 });
2333 }
2334 outcome
2335 } else if use_agg {
2336 let n_pages = ((warm_n + 999) / 1000).max(1);
2343 let (agg, outcome) = crate::backfill::agg_trades_paginated(
2344 &self.inner.hub, key.exchange, entry.account_type, raw_symbol,
2345 1000, n_pages,
2346 ).await;
2347 for ap in agg {
2348 seed_events.push(Event::Trade {
2349 exchange: key.exchange,
2350 symbol: raw_symbol.to_string(),
2351 point: TradePoint {
2352 ts_ms: ap.ts_ms,
2353 price: ap.price,
2354 quantity: ap.quantity,
2355 side: ap.side,
2356 trade_id_hash: 0,
2357 },
2358 });
2359 }
2360 outcome
2361 } else {
2362 let (trades, outcome) = crate::backfill::trades_recent(
2364 &self.inner.hub, key.exchange, entry.account_type, raw_symbol, warm_n,
2365 ).await;
2366 for pt in trades {
2367 seed_events.push(Event::Trade {
2368 exchange: key.exchange,
2369 symbol: raw_symbol.to_string(),
2370 point: pt,
2371 });
2372 }
2373 outcome
2374 };
2375 self.inner.seed_outcomes.insert(key.clone(), trade_outcome);
2376 } else if warm_n > 0 && matches!(dep_kind, Kind::MarkPrice) {
2377 let pts = crate::backfill::mark_price_recent(
2380 &self.inner.hub, key.exchange, entry.account_type, raw_symbol, warm_n,
2381 ).await;
2382 for pt in pts {
2383 seed_events.push(Event::MarkPrice {
2384 exchange: key.exchange,
2385 symbol: raw_symbol.to_string(),
2386 point: pt,
2387 });
2388 }
2389 } else if warm_n > 0 && matches!(dep_kind, Kind::IndexPrice) {
2390 let pts = crate::backfill::mark_price_recent(
2393 &self.inner.hub, key.exchange, entry.account_type, raw_symbol, warm_n,
2394 ).await;
2395 for pt in pts {
2398 if pt.index.is_finite() {
2399 seed_events.push(Event::IndexPrice {
2400 exchange: key.exchange,
2401 symbol: raw_symbol.to_string(),
2402 point: IndexPricePoint { ts_ms: pt.ts_ms, price: pt.index },
2403 });
2404 }
2405 }
2406 }
2407 agg_seed_per_dep.push(seed_events);
2408 }
2409
2410 let ring_capacity_hint = agg_seed_per_dep
2422 .iter()
2423 .map(|v| v.len())
2424 .max()
2425 .unwrap_or(warm_n)
2426 .max(self.inner.warm_start_capacity);
2427
2428 let (bcast_tx, _) = broadcast::channel::<Event>(512);
2429 let consumers = Arc::new(AtomicUsize::new(1));
2430 let (shutdown_tx, shutdown_rx) = oneshot::channel::<()>();
2431 let (seed_done_tx, seed_done_rx) = oneshot::channel::<()>();
2432
2433 spawn_derived_forwarder::<D>(
2434 self,
2435 key,
2436 upstream_rxs,
2437 upstream_keys,
2438 bcast_tx.clone(),
2439 shutdown_rx,
2440 raw_symbol.to_string(),
2441 agg_seed_per_dep,
2442 ring_capacity_hint,
2443 seed_done_tx,
2444 kline_seed_baseline,
2445 );
2446
2447 self.inner.muxes.insert(
2448 key.clone(),
2449 Multiplexer { tx: bcast_tx.clone(), consumers, shutdown: Some(shutdown_tx), grace_cancel: None },
2450 );
2451
2452 const SEED_WAIT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(15);
2465 #[cfg(not(target_arch = "wasm32"))]
2466 {
2467 tokio::select! {
2468 res = seed_done_rx => {
2469 if res.is_err() {
2470 tracing::debug!(?key, "derived: seed_done sender dropped before signaling — proceeding anyway");
2471 }
2472 }
2473 _ = tokio::time::sleep(SEED_WAIT_TIMEOUT) => {
2474 tracing::debug!(?key, "derived: cold-start seed wait timed out — proceeding with partial/empty seed");
2475 }
2476 }
2477 }
2478 #[cfg(target_arch = "wasm32")]
2479 {
2480 tokio::select! {
2481 res = seed_done_rx => {
2482 if res.is_err() {
2483 tracing::debug!(?key, "derived: seed_done sender dropped before signaling — proceeding anyway");
2484 }
2485 }
2486 _ = gloo_timers::future::sleep(SEED_WAIT_TIMEOUT) => {
2487 tracing::debug!(?key, "derived: cold-start seed wait timed out — proceeding with partial/empty seed");
2488 }
2489 }
2490 }
2491
2492 Ok(bcast_tx)
2493 }
2494}
2495
2496#[cfg(not(target_arch = "wasm32"))]
2497impl Station {
2498 async fn acquire_or_spawn_polled(
2504 &self,
2505 key: &SeriesKey,
2506 entry: &Entry,
2507 poll_spec: crate::series::PollSpec,
2508 raw_symbol: &str,
2509 ) -> Result<broadcast::Sender<Event>> {
2510 use crate::station::Multiplexer;
2511
2512 let (bcast_tx, _) = broadcast::channel::<Event>(1024);
2513 let consumers = Arc::new(AtomicUsize::new(1));
2514 let (shutdown_tx, shutdown_rx) = oneshot::channel::<()>();
2515 let label = raw_symbol.to_string();
2516
2517 match &key.kind {
2518 Kind::LongShortRatio => {
2519 let source = polling::lsr_poll_source(entry.exchange)
2520 .ok_or_else(|| StationError::StreamNotSupported(format!(
2521 "LongShortRatio REST polling not supported for {:?}",
2522 entry.exchange
2523 )))?;
2524 polling::spawn_poller::<LongShortRatioPoint, _>(
2525 self, key, source, poll_spec, bcast_tx.clone(), shutdown_rx, label,
2526 );
2527 }
2528 Kind::HistoricalVolatility => {
2529 let source = polling::hv_poll_source(entry.exchange)
2530 .ok_or_else(|| StationError::StreamNotSupported(format!(
2531 "HistoricalVolatility REST polling not supported for {:?} \
2532 (Deribit only)",
2533 entry.exchange
2534 )))?;
2535 polling::spawn_poller::<HistoricalVolatilityPoint, _>(
2536 self, key, source, poll_spec, bcast_tx.clone(), shutdown_rx, label,
2537 );
2538 }
2539 Kind::TakerVolume => {
2540 let source = polling::taker_volume_poll_source(&self.inner.hub, entry.exchange)
2541 .ok_or_else(|| StationError::StreamNotSupported(format!(
2542 "TakerVolume REST polling not supported for {:?}",
2543 entry.exchange
2544 )))?;
2545 polling::spawn_poller::<TakerVolumePoint, _>(
2546 self, key, source, poll_spec, bcast_tx.clone(), shutdown_rx, label,
2547 );
2548 }
2549 Kind::LiquidationBucket => {
2550 let source = polling::liquidation_bucket_poll_source(&self.inner.hub, entry.exchange)
2551 .ok_or_else(|| StationError::StreamNotSupported(format!(
2552 "LiquidationBucket REST polling not supported for {:?}",
2553 entry.exchange
2554 )))?;
2555 polling::spawn_poller::<LiquidationBucketPoint, _>(
2556 self, key, source, poll_spec, bcast_tx.clone(), shutdown_rx, label,
2557 );
2558 }
2559 other => {
2560 return Err(StationError::StreamNotSupported(format!(
2561 "acquire_or_spawn_polled: no poll source for {:?}",
2562 other
2563 )));
2564 }
2565 }
2566
2567 self.inner.muxes.insert(
2568 key.clone(),
2569 Multiplexer {
2570 tx: bcast_tx.clone(),
2571 consumers,
2572 shutdown: Some(shutdown_tx),
2573 grace_cancel: None,
2574 },
2575 );
2576 Ok(bcast_tx)
2577 }
2578
2579 async fn acquire_or_spawn_polled_with_source<T, S>(
2588 &self,
2589 key: &SeriesKey,
2590 poll_spec: crate::series::PollSpec,
2591 raw_symbol: &str,
2592 source: S,
2593 ) -> Result<broadcast::Sender<Event>>
2594 where
2595 T: crate::series::DataPoint + 'static,
2596 S: crate::polling::PollSource<T>,
2597 Event: EventFrom<T>,
2598 {
2599 use crate::station::Multiplexer;
2600
2601 let (bcast_tx, _) = broadcast::channel::<Event>(1024);
2602 let consumers = Arc::new(AtomicUsize::new(1));
2603 let (shutdown_tx, shutdown_rx) = oneshot::channel::<()>();
2604 let label = raw_symbol.to_string();
2605
2606 polling::spawn_poller::<T, S>(
2607 self, key, source, poll_spec, bcast_tx.clone(), shutdown_rx, label,
2608 );
2609
2610 self.inner.muxes.insert(
2611 key.clone(),
2612 Multiplexer {
2613 tx: bcast_tx.clone(),
2614 consumers,
2615 shutdown: Some(shutdown_tx),
2616 grace_cancel: None,
2617 },
2618 );
2619 Ok(bcast_tx)
2620 }
2621}
2622
2623impl StationInner {
2624 pub(crate) fn release_consumer(self: &Arc<Self>, key: &SeriesKey) {
2625 let (became_zero, grace) = {
2626 let Some(mux) = self.muxes.get(key) else { return; };
2627 let prev = mux.consumers.fetch_sub(1, Ordering::SeqCst);
2628 (prev <= 1, self.unsubscribe_grace)
2629 };
2630
2631 if !became_zero {
2632 return;
2633 }
2634
2635 if grace.is_zero() {
2636 if let Some((_, mut mux)) = self.muxes.remove(key) {
2638 if let Some(tx) = mux.shutdown.take() {
2639 let _ = tx.send(());
2640 }
2641 }
2642 return;
2643 }
2644
2645 let (cancel_tx, cancel_rx) = oneshot::channel::<()>();
2648
2649 {
2653 let Some(mut mux) = self.muxes.get_mut(key) else { return; };
2654 mux.grace_cancel = Some(cancel_tx);
2655 }
2656
2657 let inner = Arc::clone(self);
2658 let key = key.clone();
2659
2660 let grace_fut = Box::pin(async move {
2661 #[cfg(not(target_arch = "wasm32"))]
2663 let timed_out = tokio::select! {
2664 _ = cancel_rx => false,
2665 _ = tokio::time::sleep(grace) => true,
2666 };
2667 #[cfg(target_arch = "wasm32")]
2668 let timed_out = tokio::select! {
2669 _ = cancel_rx => false,
2670 _ = gloo_timers::future::sleep(grace) => true,
2671 };
2672
2673 if timed_out {
2674 let still_zero = inner
2680 .muxes
2681 .get(&key)
2682 .map(|m| m.consumers.load(Ordering::SeqCst) == 0)
2683 .unwrap_or(false);
2684 if still_zero {
2685 if let Some((_, mut mux)) = inner.muxes.remove(&key) {
2686 if let Some(tx) = mux.shutdown.take() {
2687 let _ = tx.send(());
2688 }
2689 }
2690 inner.series_handles.remove(&key);
2691 }
2692 }
2693 });
2694 #[cfg(not(target_arch = "wasm32"))]
2695 tokio::spawn(grace_fut);
2696 #[cfg(target_arch = "wasm32")]
2697 wasm_bindgen_futures::spawn_local(grace_fut);
2698 }
2699}
2700
2701fn spawn_derived_forwarder<D: DerivedStream + 'static>(
2710 station: &Station,
2711 key: &SeriesKey,
2712 upstream_rxs: Vec<broadcast::Receiver<Event>>,
2713 upstream_keys: Vec<SeriesKey>,
2714 bcast_tx: broadcast::Sender<Event>,
2715 mut shutdown_rx: oneshot::Receiver<()>,
2716 symbol_label: String,
2717 agg_seed_per_dep: Vec<Vec<Event>>,
2720 ring_capacity_hint: usize,
2726 seed_done_tx: oneshot::Sender<()>,
2733 kline_seed_baseline: Option<(i64, f64, f64)>,
2740) where
2741 Event: EventFrom<D::Output>,
2742{
2743 let inner = station.inner.clone();
2744 let key = key.clone();
2745 let storage_root = inner.storage_root.clone();
2746 let persistence = inner.persistence.clone();
2747 let warm = inner.warm_start_capacity;
2748 let exchange = key.exchange;
2749
2750 let shared_series: Arc<RwLock<Series<D::Output>>> =
2757 Arc::new(RwLock::new(Series::new(ring_capacity_hint)));
2758 let erased: Arc<dyn Any + Send + Sync> = Arc::new(Arc::clone(&shared_series));
2762 inner.series_handles.insert(key.clone(), erased);
2763
2764 let (exit_ack_tx, exit_ack_rx) = oneshot::channel::<()>();
2769 inner.exit_acks.insert(key.clone(), exit_ack_rx);
2770
2771 {
2772 let derived_fut = Box::pin(async move {
2773 #[cfg(not(target_arch = "wasm32"))]
2775 let mut disk: Option<DiskStore<D::Output>> = None;
2776 #[cfg(not(target_arch = "wasm32"))]
2777 if persistence.is_enabled_for(&key.kind) {
2778 match DiskStore::<D::Output>::with_idx_every_and_retention(
2779 &storage_root, key.clone(), 1024, persistence.retention_days,
2780 ).await {
2781 Ok(store) => disk = Some(store),
2782 Err(e) => tracing::warn!(?e, ?key, "derived: disk store open failed"),
2783 }
2784 }
2785 #[cfg(target_arch = "wasm32")]
2788 let _ = (&storage_root, &persistence, &warm);
2789
2790 #[cfg(not(target_arch = "wasm32"))]
2796 let mut flush_rx = if disk.is_some() {
2797 let (handle, rx) = FlushHandle::channel();
2798 inner.flush_handles.insert(key.clone(), handle);
2799 Some(rx)
2800 } else {
2801 None
2802 };
2803 #[cfg(target_arch = "wasm32")]
2804 let mut flush_rx: Option<mpsc::Receiver<FlushAck>> = None;
2805
2806 #[cfg(not(target_arch = "wasm32"))]
2811 if let Some(d) = disk.as_ref() {
2812 match d.read_tail(warm).await {
2813 Ok(tail) => {
2814 for point in &tail {
2815 shared_series.write().await.upsert_by_ts(point.clone());
2816 let _ = bcast_tx.send(Event::from_point(
2817 exchange,
2818 key.account_type,
2819 &symbol_label,
2820 &key.kind,
2821 point.clone(),
2822 ));
2823 }
2824 }
2825 Err(e) => tracing::debug!(?e, ?key, "derived: disk warm-seed read_tail failed"),
2826 }
2827 }
2828
2829 let mut state = D::new_for_key(&key);
2830 if let Some((open_time, volume, close)) = kline_seed_baseline {
2831 state.seed_kline_baseline(open_time, volume, close);
2832 }
2833
2834 for (dep_idx, seed_events) in agg_seed_per_dep.iter().enumerate() {
2839 if seed_events.is_empty() {
2840 continue;
2841 }
2842 let emitted = state.seed_from_events(seed_events, dep_idx);
2843 for point in emitted {
2844 #[cfg(not(target_arch = "wasm32"))]
2845 if let Some(d) = disk.as_mut() {
2846 if let Err(e) = d.append(&point) {
2847 tracing::warn!(?e, ?key, "derived agg-seed: disk append failed");
2848 }
2849 }
2850 shared_series.write().await.upsert_by_ts(point.clone());
2851 let _ = bcast_tx.send(Event::from_point(
2852 exchange,
2853 key.account_type,
2854 &symbol_label,
2855 &key.kind,
2856 point,
2857 ));
2858 }
2859 }
2860
2861 if D::deps() == [Stream::Trade] {
2876 if let Some(dep_key) = upstream_keys.first() {
2877 let trade_handle = inner
2878 .series_handles
2879 .get(dep_key)
2880 .and_then(|e| {
2881 e.downcast_ref::<Arc<RwLock<Series<TradePoint>>>>().map(Arc::clone)
2882 });
2883 if let Some(handle) = trade_handle {
2884 let buffered = handle.read().await.snapshot(); for pt in buffered {
2886 let ev = Event::Trade {
2887 exchange,
2888 symbol: symbol_label.clone(),
2889 point: pt,
2890 };
2891 if let Some(point) = state.on_upstream_event(&ev, 0) {
2892 #[cfg(not(target_arch = "wasm32"))]
2893 if let Some(d) = disk.as_mut() {
2894 if let Err(e) = d.append(&point) {
2895 tracing::warn!(?e, ?key, "derived seed: disk append failed");
2896 }
2897 }
2898 shared_series.write().await.upsert_by_ts(point.clone());
2899 let _ = bcast_tx.send(Event::from_point(
2900 exchange,
2901 key.account_type,
2902 &symbol_label,
2903 &key.kind,
2904 point,
2905 ));
2906 }
2907 }
2908 }
2909 }
2910 }
2911
2912 let _ = seed_done_tx.send(());
2918
2919 let tagged: Vec<_> = upstream_rxs
2922 .into_iter()
2923 .enumerate()
2924 .map(|(idx, rx)| {
2925 tokio_stream::wrappers::BroadcastStream::new(rx)
2926 .filter_map(move |res| async move {
2927 match res {
2928 Ok(ev) => Some((idx, ev)),
2929 Err(tokio_stream::wrappers::errors::BroadcastStreamRecvError::Lagged(n)) => {
2930 tracing::warn!(dep_idx = idx, lagged = n, "derived: upstream lagged — events dropped");
2931 None
2932 }
2933 }
2934 })
2935 .boxed()
2937 })
2938 .collect();
2939
2940 let mut merged = futures_util::stream::select_all(tagged);
2941
2942 loop {
2943 #[cfg(not(target_arch = "wasm32"))]
2944 let item_opt = tokio::select! {
2945 _ = &mut shutdown_rx => break,
2946 ack = recv_flush_request(&mut flush_rx) => {
2947 let result = flush_disk_store(&mut disk).await;
2948 let _ = ack.send(result);
2949 continue;
2950 }
2951 item = merged.next() => item,
2952 };
2953 #[cfg(target_arch = "wasm32")]
2957 let item_opt = tokio::select! {
2958 _ = &mut shutdown_rx => break,
2959 ack = recv_flush_request(&mut flush_rx) => {
2960 let _ = ack.send(Ok(()));
2961 continue;
2962 }
2963 item = merged.next() => item,
2964 };
2965
2966 let Some((dep_idx, ev)) = item_opt else {
2967 tracing::info!(?key, "derived: all upstreams closed — exiting");
2969 break;
2970 };
2971
2972 if let Some(point) = state.on_upstream_event(&ev, dep_idx) {
2973 #[cfg(not(target_arch = "wasm32"))]
2974 if let Some(d) = disk.as_mut() {
2975 if let Err(e) = d.append(&point) {
2976 tracing::warn!(?e, "derived: disk store append failed");
2977 }
2978 }
2979 shared_series.write().await.push(point.clone());
2980 let _ = bcast_tx.send(Event::from_point(exchange, key.account_type, &symbol_label, &key.kind, point));
2981 }
2982 }
2983
2984 #[cfg(not(target_arch = "wasm32"))]
2987 if flush_rx.is_some() {
2988 inner.flush_handles.remove(&key);
2989 }
2990 #[cfg(not(target_arch = "wasm32"))]
2991 if let Some(mut d) = disk { let _ = d.flush().await; }
2992
2993 for up_key in &upstream_keys {
2996 inner.release_consumer(up_key);
2997 }
2998
2999 let still_consumers = inner
3001 .muxes
3002 .get(&key)
3003 .map(|m| m.consumers.load(Ordering::SeqCst))
3004 .unwrap_or(0);
3005 if still_consumers == 0 {
3006 inner.muxes.remove(&key);
3007 }
3008 inner.series_handles.remove(&key);
3012 inner.exit_acks.remove(&key);
3016 let _ = exit_ack_tx.send(());
3017 });
3018 #[cfg(not(target_arch = "wasm32"))]
3019 tokio::spawn(derived_fut);
3020 #[cfg(target_arch = "wasm32")]
3021 wasm_bindgen_futures::spawn_local(derived_fut);
3022 }
3023}
3024
3025pub(crate) async fn recv_flush_request(rx: &mut Option<mpsc::Receiver<FlushAck>>) -> FlushAck {
3035 match rx {
3036 Some(rx) => match rx.recv().await {
3037 Some(ack) => ack,
3038 None => std::future::pending().await,
3039 },
3040 None => std::future::pending().await,
3041 }
3042}
3043
3044pub(crate) async fn flush_disk_store<T: DataPoint>(disk: &mut Option<DiskStore<T>>) -> std::io::Result<()> {
3047 match disk {
3048 Some(d) => {
3049 #[cfg(not(target_arch = "wasm32"))]
3050 {
3051 d.flush().await
3052 }
3053 #[cfg(target_arch = "wasm32")]
3054 {
3055 d.flush().await.map_err(std::io::Error::from)
3056 }
3057 }
3058 None => Ok(()),
3059 }
3060}
3061
3062fn spawn_forwarder<T: DataPoint + 'static>(
3071 station: &Station,
3072 key: &SeriesKey,
3073 ws: Arc<dyn digdigdig3::core::traits::WebSocketConnector>,
3074 bcast_tx: broadcast::Sender<Event>,
3075 mut shutdown_rx: oneshot::Receiver<()>,
3076 symbol_label: String,
3077 rest_seed: Vec<T>,
3080 sub_req: SubscriptionRequest,
3084) where
3085 Event: EventFrom<T>,
3086{
3087 let inner = station.inner.clone();
3088 let key = key.clone();
3089 let storage_root = inner.storage_root.clone();
3090 let persistence = inner.persistence.clone();
3091 let warm = inner.warm_start_capacity;
3092 let exchange = key.exchange;
3093 let gap_cfg = inner.gap_heal;
3094 let hub_for_heal = inner.hub.clone();
3095
3096 let shared_series: Arc<RwLock<Series<T>>> = Arc::new(RwLock::new(Series::new(warm)));
3099 let erased: Arc<dyn Any + Send + Sync> = Arc::new(Arc::clone(&shared_series));
3102 inner.series_handles.insert(key.clone(), erased);
3103
3104 let (exit_ack_tx, exit_ack_rx) = oneshot::channel::<()>();
3109 inner.exit_acks.insert(key.clone(), exit_ack_rx);
3110
3111 {
3112 let forwarder_fut = Box::pin(async move {
3113 #[cfg(not(target_arch = "wasm32"))]
3116 let mut disk: Option<DiskStore<T>> = None;
3117 #[cfg(not(target_arch = "wasm32"))]
3118 if persistence.is_enabled_for(&key.kind) {
3119 match DiskStore::<T>::with_idx_every_and_retention(
3120 &storage_root, key.clone(), 1024, persistence.retention_days,
3121 ).await {
3122 Ok(store) => disk = Some(store),
3123 Err(e) => tracing::warn!(?e, ?key, "disk store open failed"),
3124 }
3125 }
3126 #[cfg(target_arch = "wasm32")]
3132 let mut disk: Option<DiskStore<T>> = None;
3133 #[cfg(target_arch = "wasm32")]
3134 if persistence.is_enabled_for(&key.kind) {
3135 match DiskStore::<T>::new(key.clone()).await {
3136 Ok(store) => disk = Some(store),
3137 Err(e) => tracing::warn!(?e, ?key, "wasm OPFS disk store open failed"),
3138 }
3139 }
3140 #[cfg(target_arch = "wasm32")]
3142 let _ = &storage_root;
3143
3144 let mut flush_rx = if disk.is_some() {
3150 let (handle, rx) = FlushHandle::channel();
3151 inner.flush_handles.insert(key.clone(), handle);
3152 Some(rx)
3153 } else {
3154 None
3155 };
3156
3157 let mut last_emitted_ms: i64 = 0;
3161
3162 if warm > 0 {
3164 let disk_tail: Vec<T> = if let Some(d) = disk.as_ref() {
3166 d.read_tail(warm).await.unwrap_or_default()
3167 } else {
3168 Vec::new()
3169 };
3170 let seed_points: Vec<T> = if disk_tail.is_empty() && !rest_seed.is_empty() {
3171 rest_seed
3172 } else {
3173 disk_tail
3174 };
3175 for p in &seed_points {
3176 let _ = bcast_tx.send(Event::from_point(exchange, key.account_type, &symbol_label, &key.kind, p.clone()));
3177 last_emitted_ms = last_emitted_ms.max(p.timestamp_ms());
3178 }
3179 shared_series.write().await.seed(seed_points);
3180 }
3181
3182 let mut stream = ws.event_stream();
3183 #[cfg(not(target_arch = "wasm32"))]
3187 let silence_timeout = std::time::Duration::from_secs(
3188 std::env::var("DIG3_WS_SILENCE_SECS").ok().and_then(|s| s.parse().ok()).unwrap_or(60),
3189 );
3190 #[cfg(target_arch = "wasm32")]
3194 let silence_timeout_ms: u32 = 60_000;
3195 #[cfg(not(target_arch = "wasm32"))]
3199 let debug_slow_ms: u64 = std::env::var("DIG3_DEBUG_SLOW_CONSUMER_MS")
3200 .ok()
3201 .and_then(|s| s.parse().ok())
3202 .unwrap_or(0);
3203 #[cfg(target_arch = "wasm32")]
3208 let mut wasm_flush_counter: u32 = 0;
3209 #[cfg(target_arch = "wasm32")]
3210 const WASM_FLUSH_EVERY: u32 = 64;
3211
3212 loop {
3213 #[cfg(not(target_arch = "wasm32"))]
3218 let item_opt = tokio::select! {
3219 _ = &mut shutdown_rx => break,
3220 ack = recv_flush_request(&mut flush_rx) => {
3221 let result = flush_disk_store(&mut disk).await;
3222 let _ = ack.send(result);
3223 continue;
3224 }
3225 res = tokio::time::timeout(silence_timeout, stream.next()) => res,
3226 };
3227 #[cfg(target_arch = "wasm32")]
3228 let item_opt: std::result::Result<
3232 Option<std::result::Result<_, digdigdig3::core::types::WebSocketError>>,
3233 (),
3234 > = tokio::select! {
3235 _ = &mut shutdown_rx => break,
3236 ack = recv_flush_request(&mut flush_rx) => {
3237 let result = flush_disk_store(&mut disk).await;
3238 let _ = ack.send(result);
3239 continue;
3240 }
3241 _ = gloo_timers::future::sleep(std::time::Duration::from_millis(silence_timeout_ms as u64)) => Err(()),
3242 item = stream.next() => Ok(item),
3243 };
3244
3245 let trigger_heal_reason: Option<&'static str> = match &item_opt {
3246 Err(_) => Some("silence_timeout"),
3247 Ok(None) => Some("stream_ended"),
3248 Ok(Some(Err(_))) => Some("stream_err"),
3249 Ok(Some(Ok(_))) => None,
3250 };
3251
3252 if let Some(reason) = trigger_heal_reason {
3253 let is_kline_family = matches!(
3265 &key.kind,
3266 Kind::Kline(_) | Kind::MarkPriceKline(_)
3267 | Kind::IndexPriceKline(_) | Kind::PremiumIndexKline(_)
3268 );
3269
3270 if !is_kline_family {
3271 tracing::info!(
3272 target: "dig3::gap_heal",
3273 ?key, reason,
3274 "non-kline stream disconnect — forwarder exiting (no resub for non-kline kinds)"
3275 );
3276 break;
3277 }
3278
3279 tracing::info!(target: "dig3::gap_heal", ?key, reason, "ws disconnect detected → heal + resub");
3280 {
3285 let mut series_guard = shared_series.write().await;
3286 run_kline_heal::<T>(
3287 &hub_for_heal, &key, &gap_cfg, &symbol_label,
3288 last_emitted_ms, exchange,
3289 &mut *series_guard, &mut disk, &bcast_tx,
3290 ).await;
3291 last_emitted_ms = last_emitted_ms.max(
3292 series_guard.last().map(|p| p.timestamp_ms()).unwrap_or(0)
3293 );
3294 }
3295 let unsub_res = ws.unsubscribe(sub_req.clone()).await;
3300 let sub_res = ws.subscribe(sub_req.clone()).await;
3301 tracing::info!(
3302 target: "dig3::gap_heal",
3303 ?key,
3304 unsub_ok = unsub_res.is_ok(),
3305 sub_ok = sub_res.is_ok(),
3306 "resub cycle complete"
3307 );
3308 if let Err(e) = unsub_res {
3309 tracing::debug!(target: "dig3::gap_heal", ?key, ?e, "unsubscribe failed (best-effort)");
3310 }
3311 if let Err(e) = sub_res {
3312 tracing::warn!(target: "dig3::gap_heal", ?key, ?e, "resubscribe failed");
3313 }
3314 drop(stream);
3321 stream = ws.event_stream();
3322 continue;
3323 }
3324
3325 let ev = match item_opt {
3326 Ok(Some(Ok(ev))) => ev,
3327 _ => unreachable!(),
3328 };
3329
3330 if !event_matches_key(&ev, &key) {
3331 continue;
3332 }
3333 let Some(point) = T::from_stream_event(&ev) else {
3334 continue;
3335 };
3336
3337 #[cfg(not(target_arch = "wasm32"))]
3340 if let Some(d) = disk.as_mut() {
3341 if let Err(e) = d.append(&point) {
3342 tracing::warn!(?e, "disk store append failed");
3343 }
3344 }
3345 #[cfg(target_arch = "wasm32")]
3346 if let Some(d) = disk.as_mut() {
3347 d.append(&point);
3348 }
3349 #[cfg(target_arch = "wasm32")]
3351 {
3352 wasm_flush_counter = wasm_flush_counter.wrapping_add(1);
3353 if wasm_flush_counter % WASM_FLUSH_EVERY == 0 {
3354 if let Some(d) = disk.as_mut() {
3355 if let Err(e) = d.flush().await {
3356 tracing::warn!(?e, "wasm OPFS periodic flush failed");
3357 }
3358 }
3359 }
3360 }
3361 let pt_ts = point.timestamp_ms();
3362 {
3365 let mut series_guard = shared_series.write().await;
3366 if matches!(&key.kind, Kind::Kline(_) | Kind::MarkPriceKline(_) | Kind::IndexPriceKline(_) | Kind::PremiumIndexKline(_)) {
3367 series_guard.upsert_by_ts(point.clone());
3368 } else {
3369 series_guard.push(point.clone());
3370 }
3371 }
3372 last_emitted_ms = last_emitted_ms.max(pt_ts);
3373
3374 let _ = bcast_tx.send(Event::from_point(exchange, key.account_type, &symbol_label, &key.kind, point));
3375
3376 #[cfg(not(target_arch = "wasm32"))]
3377 if debug_slow_ms > 0 {
3378 tokio::time::sleep(std::time::Duration::from_millis(debug_slow_ms)).await;
3379 }
3380 }
3381
3382 if flush_rx.is_some() {
3387 inner.flush_handles.remove(&key);
3388 }
3389 if let Some(mut d) = disk { let _ = d.flush().await; }
3391 let still_consumers = inner
3402 .muxes
3403 .get(&key)
3404 .map(|m| m.consumers.load(Ordering::SeqCst))
3405 .unwrap_or(0);
3406 if still_consumers == 0 {
3407 inner.muxes.remove(&key);
3408 }
3409 inner.series_handles.remove(&key);
3412 inner.exit_acks.remove(&key);
3420 let _ = exit_ack_tx.send(());
3421 });
3422 #[cfg(not(target_arch = "wasm32"))]
3423 tokio::spawn(forwarder_fut);
3424 #[cfg(target_arch = "wasm32")]
3425 wasm_bindgen_futures::spawn_local(forwarder_fut);
3426 }
3427}
3428
3429async fn run_kline_heal<T: DataPoint + 'static>(
3432 hub: &Arc<ExchangeHub>,
3433 key: &SeriesKey,
3434 cfg: &crate::GapHealConfig,
3435 symbol_label: &str,
3436 last_emitted_ms: i64,
3437 exchange: digdigdig3::core::types::ExchangeId,
3438 series: &mut Series<T>,
3439 disk: &mut Option<DiskStore<T>>,
3440 bcast_tx: &broadcast::Sender<Event>,
3441) where
3442 Event: EventFrom<T>,
3443{
3444 if !cfg.enabled {
3445 return;
3446 }
3447 let Kind::Kline(iv) = &key.kind else { return; };
3448
3449 let now_ms = chrono::Utc::now().timestamp_millis();
3450 let limit = crate::gap_heal::heal_limit(cfg, iv.as_str(), last_emitted_ms, now_ms);
3451
3452 tracing::info!(
3453 target: "dig3::gap_heal",
3454 ?key,
3455 last_emitted_ms,
3456 limit,
3457 "kline heal: pulling REST"
3458 );
3459
3460 let pulled: Vec<T> = cast_vec(
3461 crate::gap_heal::heal_klines(hub, key.exchange, key.account_type, &key.symbol, iv.as_str(), last_emitted_ms, limit).await
3462 );
3463 let pulled_count = pulled.len();
3464
3465 let new_to_emit = crate::gap_heal::select_heal_window(pulled.clone(), last_emitted_ms);
3469 let emitted_count = new_to_emit.len();
3470
3471 for p in pulled {
3472 if let Some(d) = disk.as_mut() {
3473 let _ = d.append(&p);
3474 }
3475 series.upsert_by_ts(p);
3476 }
3477 for p in new_to_emit {
3478 let _ = bcast_tx.send(Event::from_point(exchange, key.account_type, symbol_label, &key.kind, p));
3479 }
3480 if let Some(d) = disk.as_mut() { let _ = d.flush().await; }
3481
3482 tracing::info!(
3483 target: "dig3::gap_heal",
3484 ?key,
3485 pulled_count,
3486 emitted_count,
3487 "kline heal: applied"
3488 );
3489}
3490
3491fn cast_vec<A: 'static, B: 'static>(v: Vec<A>) -> Vec<B> {
3496 if std::any::TypeId::of::<A>() == std::any::TypeId::of::<B>() {
3497 let mut v = std::mem::ManuallyDrop::new(v);
3500 let (ptr, len, cap) = (v.as_mut_ptr() as *mut B, v.len(), v.capacity());
3501 unsafe { Vec::from_raw_parts(ptr, len, cap) }
3502 } else {
3503 Vec::new()
3504 }
3505}
3506
3507fn event_matches_key(ev: &StreamEvent, key: &SeriesKey) -> bool {
3510 let want = key.symbol.as_str();
3511 let got: Option<&str> = event_raw_symbol(ev);
3512 match got {
3513 Some("") => true,
3515 Some(s) => s.eq_ignore_ascii_case(want),
3516 None => true,
3517 }
3518}
3519
3520fn event_raw_symbol(ev: &StreamEvent) -> Option<&str> {
3523 match ev {
3524 StreamEvent::Trade { symbol, .. } => Some(symbol),
3525 StreamEvent::AggTrade { symbol, .. } => Some(symbol),
3526 StreamEvent::Ticker { symbol, .. } => Some(symbol),
3527 StreamEvent::Kline { symbol, .. } => Some(symbol),
3528 StreamEvent::OrderbookSnapshot { symbol, .. } => Some(symbol),
3529 StreamEvent::OrderbookDelta { symbol, .. } => Some(symbol),
3530 StreamEvent::MarkPrice { symbol, .. } => Some(symbol),
3531 StreamEvent::FundingRate { symbol, .. } => Some(symbol),
3532 StreamEvent::OpenInterestUpdate { symbol, .. } => Some(symbol),
3533 StreamEvent::Liquidation { symbol, .. } => Some(symbol),
3534 StreamEvent::LongShortRatio { symbol, .. } => Some(symbol),
3535 StreamEvent::MarkPriceKline { symbol, .. } => Some(symbol),
3536 StreamEvent::IndexPriceKline { symbol, .. } => Some(symbol),
3537 StreamEvent::PremiumIndexKline { symbol, .. } => Some(symbol),
3538 StreamEvent::IndexPrice { symbol, .. } => Some(symbol),
3539 StreamEvent::HistoricalVolatility { symbol, .. } => Some(symbol),
3540 StreamEvent::InsuranceFund { symbol, .. } => Some(symbol),
3541 StreamEvent::Basis { symbol, .. } => Some(symbol),
3542 StreamEvent::OptionGreeks { symbol, .. } => Some(symbol),
3543 StreamEvent::VolatilityIndex { symbol, .. } => Some(symbol),
3544 StreamEvent::BlockTrade { symbol, .. } => Some(symbol),
3545 StreamEvent::AuctionEvent { symbol, .. } => Some(symbol),
3546 StreamEvent::MarketWarning { symbol, .. } => symbol.as_deref(),
3547 StreamEvent::OrderbookL3 { symbol, .. } => Some(symbol),
3548 StreamEvent::SettlementEvent { symbol, .. } => Some(symbol),
3549 StreamEvent::RiskLimit { symbol, .. } => Some(symbol),
3550 StreamEvent::PredictedFunding { symbol, .. } => Some(symbol),
3551 StreamEvent::FundingSettlement { symbol, .. } => Some(symbol),
3552 StreamEvent::CompositeIndex { symbol, .. } => Some(symbol),
3553 StreamEvent::OrderUpdate { symbol: _, event: _ }
3555 | StreamEvent::BalanceUpdate(_)
3556 | StreamEvent::PositionUpdate { symbol: _, event: _ } => None,
3557 StreamEvent::Batch(_) => None,
3560 }
3561}
3562
3563pub(crate) trait EventFrom<T> {
3566 fn from_point(
3567 exchange: digdigdig3::core::types::ExchangeId,
3568 account_type: digdigdig3::core::types::AccountType,
3569 symbol: &str,
3570 kind: &Kind,
3571 p: T,
3572 ) -> Self;
3573}
3574
3575impl EventFrom<TradePoint> for Event {
3576 fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: TradePoint) -> Self {
3577 Event::Trade { exchange, symbol: symbol.to_string(), point }
3578 }
3579}
3580impl EventFrom<AggTradePoint> for Event {
3581 fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: AggTradePoint) -> Self {
3582 Event::AggTrade { exchange, symbol: symbol.to_string(), point }
3583 }
3584}
3585impl EventFrom<BarPoint> for Event {
3586 fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, kind: &Kind, point: BarPoint) -> Self {
3587 let timeframe = match kind { Kind::Kline(iv) => iv.clone(), _ => KlineInterval::new("") };
3588 Event::Bar { exchange, symbol: symbol.to_string(), timeframe, point }
3589 }
3590}
3591impl EventFrom<TickerPoint> for Event {
3592 fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: TickerPoint) -> Self {
3593 Event::Ticker { exchange, symbol: symbol.to_string(), point }
3594 }
3595}
3596impl EventFrom<ObSnapshotPoint> for Event {
3597 fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: ObSnapshotPoint) -> Self {
3598 Event::OrderbookSnapshot { exchange, symbol: symbol.to_string(), point }
3599 }
3600}
3601impl EventFrom<ObDeltaPoint> for Event {
3602 fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: ObDeltaPoint) -> Self {
3603 Event::OrderbookDelta { exchange, symbol: symbol.to_string(), point }
3604 }
3605}
3606impl EventFrom<MarkPricePoint> for Event {
3607 fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: MarkPricePoint) -> Self {
3608 Event::MarkPrice { exchange, symbol: symbol.to_string(), point }
3609 }
3610}
3611impl EventFrom<FundingRatePoint> for Event {
3612 fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: FundingRatePoint) -> Self {
3613 Event::FundingRate { exchange, symbol: symbol.to_string(), point }
3614 }
3615}
3616impl EventFrom<OpenInterestPoint> for Event {
3617 fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: OpenInterestPoint) -> Self {
3618 Event::OpenInterest { exchange, symbol: symbol.to_string(), point }
3619 }
3620}
3621impl EventFrom<LiquidationPoint> for Event {
3622 fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: LiquidationPoint) -> Self {
3623 Event::Liquidation { exchange, symbol: symbol.to_string(), point }
3624 }
3625}
3626impl EventFrom<BlockTradePoint> for Event {
3627 fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: BlockTradePoint) -> Self {
3628 Event::BlockTrade { exchange, symbol: symbol.to_string(), point }
3629 }
3630}
3631impl EventFrom<AuctionEventPoint> for Event {
3632 fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: AuctionEventPoint) -> Self {
3633 Event::AuctionEvent { exchange, symbol: symbol.to_string(), point }
3634 }
3635}
3636impl EventFrom<IndexPricePoint> for Event {
3637 fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: IndexPricePoint) -> Self {
3638 Event::IndexPrice { exchange, symbol: symbol.to_string(), point }
3639 }
3640}
3641impl EventFrom<CompositeIndexPoint> for Event {
3642 fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: CompositeIndexPoint) -> Self {
3643 Event::CompositeIndex { exchange, symbol: symbol.to_string(), point }
3644 }
3645}
3646impl EventFrom<OptionGreeksPoint> for Event {
3647 fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: OptionGreeksPoint) -> Self {
3648 Event::OptionGreeks { exchange, symbol: symbol.to_string(), point }
3649 }
3650}
3651impl EventFrom<VolatilityIndexPoint> for Event {
3652 fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: VolatilityIndexPoint) -> Self {
3653 Event::VolatilityIndex { exchange, symbol: symbol.to_string(), point }
3654 }
3655}
3656impl EventFrom<HistoricalVolatilityPoint> for Event {
3657 fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: HistoricalVolatilityPoint) -> Self {
3658 Event::HistoricalVolatility { exchange, symbol: symbol.to_string(), point }
3659 }
3660}
3661impl EventFrom<LongShortRatioPoint> for Event {
3662 fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: LongShortRatioPoint) -> Self {
3663 Event::LongShortRatio { exchange, symbol: symbol.to_string(), point }
3664 }
3665}
3666impl EventFrom<TakerVolumePoint> for Event {
3667 fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: TakerVolumePoint) -> Self {
3668 Event::TakerVolume { exchange, symbol: symbol.to_string(), point }
3669 }
3670}
3671impl EventFrom<LiquidationBucketPoint> for Event {
3672 fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: LiquidationBucketPoint) -> Self {
3673 Event::LiquidationBucket { exchange, symbol: symbol.to_string(), point }
3674 }
3675}
3676impl EventFrom<BasisPoint> for Event {
3677 fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: BasisPoint) -> Self {
3678 Event::Basis { exchange, symbol: symbol.to_string(), point }
3679 }
3680}
3681impl EventFrom<InsuranceFundPoint> for Event {
3682 fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: InsuranceFundPoint) -> Self {
3683 Event::InsuranceFund { exchange, symbol: symbol.to_string(), point }
3684 }
3685}
3686impl EventFrom<OrderbookL3Point> for Event {
3687 fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: OrderbookL3Point) -> Self {
3688 Event::OrderbookL3 { exchange, symbol: symbol.to_string(), point }
3689 }
3690}
3691impl EventFrom<SettlementEventPoint> for Event {
3692 fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: SettlementEventPoint) -> Self {
3693 Event::SettlementEvent { exchange, symbol: symbol.to_string(), point }
3694 }
3695}
3696impl EventFrom<MarketWarningPoint> for Event {
3697 fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: MarketWarningPoint) -> Self {
3698 Event::MarketWarning { exchange, symbol: symbol.to_string(), point }
3699 }
3700}
3701impl EventFrom<RiskLimitPoint> for Event {
3702 fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: RiskLimitPoint) -> Self {
3703 Event::RiskLimit { exchange, symbol: symbol.to_string(), point }
3704 }
3705}
3706impl EventFrom<PredictedFundingPoint> for Event {
3707 fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: PredictedFundingPoint) -> Self {
3708 Event::PredictedFunding { exchange, symbol: symbol.to_string(), point }
3709 }
3710}
3711impl EventFrom<FundingSettlementPoint> for Event {
3712 fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: FundingSettlementPoint) -> Self {
3713 Event::FundingSettlement { exchange, symbol: symbol.to_string(), point }
3714 }
3715}
3716impl EventFrom<MarkPriceKlinePoint> for Event {
3717 fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, kind: &Kind, point: MarkPriceKlinePoint) -> Self {
3718 let timeframe = match kind { Kind::MarkPriceKline(iv) => iv.clone(), _ => KlineInterval::new("") };
3719 Event::MarkPriceKline { exchange, symbol: symbol.to_string(), timeframe, point }
3720 }
3721}
3722impl EventFrom<IndexPriceKlinePoint> for Event {
3723 fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, kind: &Kind, point: IndexPriceKlinePoint) -> Self {
3724 let timeframe = match kind { Kind::IndexPriceKline(iv) => iv.clone(), _ => KlineInterval::new("") };
3725 Event::IndexPriceKline { exchange, symbol: symbol.to_string(), timeframe, point }
3726 }
3727}
3728impl EventFrom<PremiumIndexKlinePoint> for Event {
3729 fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, kind: &Kind, point: PremiumIndexKlinePoint) -> Self {
3730 let timeframe = match kind { Kind::PremiumIndexKline(iv) => iv.clone(), _ => KlineInterval::new("") };
3731 Event::PremiumIndexKline { exchange, symbol: symbol.to_string(), timeframe, point }
3732 }
3733}
3734impl EventFrom<OrderUpdatePoint> for Event {
3735 fn from_point(exchange: digdigdig3::core::types::ExchangeId, account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: OrderUpdatePoint) -> Self {
3736 Event::OrderUpdate { exchange, account_type, symbol: symbol.to_string(), point }
3737 }
3738}
3739impl EventFrom<BalanceUpdatePoint> for Event {
3740 fn from_point(exchange: digdigdig3::core::types::ExchangeId, account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: BalanceUpdatePoint) -> Self {
3741 Event::BalanceUpdate { exchange, account_type, symbol: symbol.to_string(), point }
3742 }
3743}
3744impl EventFrom<PositionUpdatePoint> for Event {
3745 fn from_point(exchange: digdigdig3::core::types::ExchangeId, account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: PositionUpdatePoint) -> Self {
3746 Event::PositionUpdate { exchange, account_type, symbol: symbol.to_string(), point }
3747 }
3748}
3749impl EventFrom<FootprintPoint> for Event {
3750 fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: FootprintPoint) -> Self {
3751 Event::Footprint { exchange, symbol: symbol.to_string(), point }
3752 }
3753}
3754impl EventFrom<PnfColumnPoint> for Event {
3760 fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: PnfColumnPoint) -> Self {
3761 Event::PnfBar { exchange, symbol: symbol.to_string(), point }
3762 }
3763}
3764impl EventFrom<KagiSegmentPoint> for Event {
3765 fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: KagiSegmentPoint) -> Self {
3766 Event::KagiBar { exchange, symbol: symbol.to_string(), point }
3767 }
3768}
3769impl EventFrom<ScalarBarPoint> for Event {
3770 fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: ScalarBarPoint) -> Self {
3771 Event::CvdLine { exchange, symbol: symbol.to_string(), point }
3772 }
3773}
3774impl EventFrom<TpoSessionPoint> for Event {
3775 fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: TpoSessionPoint) -> Self {
3776 Event::TpoProfile { exchange, symbol: symbol.to_string(), point }
3777 }
3778}
3779impl EventFrom<RenkoBrickPoint> for Event {
3780 fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: RenkoBrickPoint) -> Self {
3781 Event::RenkoBar { exchange, symbol: symbol.to_string(), point }
3782 }
3783}
3784impl EventFrom<ThreeLineBreakLinePoint> for Event {
3785 fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: ThreeLineBreakLinePoint) -> Self {
3786 Event::ThreeLineBreakUpdate { exchange, symbol: symbol.to_string(), point }
3787 }
3788}
3789
3790
3791impl EventFrom<TickerIndicatorsPoint> for Event {
3799 fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, p: TickerIndicatorsPoint) -> Self {
3800 Event::Ticker { exchange, symbol: symbol.to_string(), point: TickerPoint {
3802 ts_ms: p.ts_ms, last: p.last, bid: p.bid, ask: p.ask,
3803 high_24h: p.high_24h, low_24h: p.low_24h,
3804 vol_24h: p.vol_24h, quote_vol_24h: p.quote_vol_24h,
3805 change_pct_24h: p.change_pct_24h,
3806 }}
3807 }
3808}
3809impl EventFrom<TickerFullPoint> for Event {
3810 fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, p: TickerFullPoint) -> Self {
3811 Event::Ticker { exchange, symbol: symbol.to_string(), point: TickerPoint {
3812 ts_ms: p.ts_ms, last: p.last, bid: p.bid, ask: p.ask,
3813 high_24h: p.high_24h, low_24h: p.low_24h,
3814 vol_24h: p.vol_24h, quote_vol_24h: p.quote_vol_24h,
3815 change_pct_24h: p.price_change_pct_24h,
3816 }}
3817 }
3818}
3819impl EventFrom<MarkPriceIndicatorsPoint> for Event {
3820 fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, p: MarkPriceIndicatorsPoint) -> Self {
3821 Event::MarkPrice { exchange, symbol: symbol.to_string(), point: MarkPricePoint {
3822 ts_ms: p.ts_ms, mark: p.mark, index: p.index,
3823 }}
3824 }
3825}
3826impl EventFrom<MarkPriceFullPoint> for Event {
3827 fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, p: MarkPriceFullPoint) -> Self {
3828 Event::MarkPrice { exchange, symbol: symbol.to_string(), point: MarkPricePoint {
3829 ts_ms: p.ts_ms, mark: p.mark, index: p.index,
3830 }}
3831 }
3832}
3833impl EventFrom<FundingRateIndicatorsPoint> for Event {
3834 fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, p: FundingRateIndicatorsPoint) -> Self {
3835 Event::FundingRate { exchange, symbol: symbol.to_string(), point: FundingRatePoint {
3836 ts_ms: p.ts_ms, rate: p.rate,
3837 next_funding_time_ms: p.next_funding_time_ms.unwrap_or(0),
3838 }}
3839 }
3840}
3841impl EventFrom<FundingRateFullPoint> for Event {
3842 fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, p: FundingRateFullPoint) -> Self {
3843 Event::FundingRate { exchange, symbol: symbol.to_string(), point: FundingRatePoint {
3844 ts_ms: p.ts_ms, rate: p.rate,
3845 next_funding_time_ms: p.next_funding_time_ms.unwrap_or(0),
3846 }}
3847 }
3848}
3849impl EventFrom<OpenInterestIndicatorsPoint> for Event {
3850 fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, p: OpenInterestIndicatorsPoint) -> Self {
3851 Event::OpenInterest { exchange, symbol: symbol.to_string(), point: OpenInterestPoint {
3852 ts_ms: p.ts_ms, open_interest: p.open_interest,
3853 open_interest_value: p.open_interest_value,
3854 }}
3855 }
3856}
3857impl EventFrom<LiquidationIndicatorsPoint> for Event {
3858 fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, p: LiquidationIndicatorsPoint) -> Self {
3859 Event::Liquidation { exchange, symbol: symbol.to_string(), point: LiquidationPoint {
3860 ts_ms: p.ts_ms, price: p.price, quantity: p.quantity,
3861 value: p.value, side: p.side,
3862 }}
3863 }
3864}
3865impl EventFrom<LiquidationFullPoint> for Event {
3866 fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, p: LiquidationFullPoint) -> Self {
3867 Event::Liquidation { exchange, symbol: symbol.to_string(), point: LiquidationPoint {
3868 ts_ms: p.ts_ms, price: p.price, quantity: p.quantity,
3869 value: p.value, side: p.side,
3870 }}
3871 }
3872}
3873impl EventFrom<IndexPriceIndicatorsPoint> for Event {
3874 fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, p: IndexPriceIndicatorsPoint) -> Self {
3875 Event::IndexPrice { exchange, symbol: symbol.to_string(), point: IndexPricePoint {
3876 ts_ms: p.ts_ms, price: p.price,
3877 }}
3878 }
3879}
3880impl EventFrom<ObSnapshotIndicatorsPoint> for Event {
3881 fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, p: ObSnapshotIndicatorsPoint) -> Self {
3882 Event::OrderbookSnapshot { exchange, symbol: symbol.to_string(), point: ObSnapshotPoint {
3883 ts_ms: p.ts_ms, bids: p.bids, asks: p.asks,
3884 }}
3885 }
3886}
3887impl EventFrom<ObDeltaIndicatorsPoint> for Event {
3888 fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, p: ObDeltaIndicatorsPoint) -> Self {
3889 Event::OrderbookDelta { exchange, symbol: symbol.to_string(), point: ObDeltaPoint {
3890 ts_ms: p.ts_ms, bid_changes: p.bid_changes, ask_changes: p.ask_changes,
3891 }}
3892 }
3893}
3894
3895async fn ob_rest_seed(
3898 hub: &Arc<digdigdig3::connector_manager::ExchangeHub>,
3899 exchange: digdigdig3::core::types::ExchangeId,
3900 account: digdigdig3::core::types::AccountType,
3901 symbol: &str,
3902 depth: usize,
3903) -> Vec<ObSnapshotPoint> {
3904 let Some(rest) = hub.rest(exchange) else {
3905 tracing::warn!(
3906 target: "dig3::ob_seed",
3907 ?exchange, symbol,
3908 "orderbook REST seed: connector not initialized — continuing WS-only"
3909 );
3910 return Vec::new();
3911 };
3912 let depth_u16 = depth.min(u16::MAX as usize) as u16;
3913 match rest
3914 .get_orderbook(
3915 digdigdig3::core::types::SymbolInput::Raw(symbol),
3916 Some(depth_u16),
3917 account,
3918 )
3919 .await
3920 {
3921 Ok(ob) if ob.bids.is_empty() && ob.asks.is_empty() => {
3922 tracing::warn!(
3923 target: "dig3::ob_seed",
3924 ?exchange, symbol,
3925 "orderbook REST seed returned empty snapshot — continuing WS-only"
3926 );
3927 Vec::new()
3928 }
3929 Ok(ob) => {
3930 tracing::debug!(
3931 target: "dig3::ob_seed",
3932 ?exchange, symbol,
3933 bids = ob.bids.len(),
3934 asks = ob.asks.len(),
3935 "orderbook REST seed ok"
3936 );
3937 vec![ObSnapshotPoint::from_orderbook(&ob)]
3938 }
3939 Err(e) => {
3940 tracing::warn!(
3941 target: "dig3::ob_seed",
3942 ?exchange, symbol, ?e,
3943 "orderbook REST seed failed — continuing WS-only"
3944 );
3945 Vec::new()
3946 }
3947 }
3948}
3949
3950fn ws_request_for(
3951 kind: &Kind,
3952 sym: Symbol,
3953 account: digdigdig3::core::types::AccountType,
3954) -> SubscriptionRequest {
3955 let stream_type = match kind {
3956 Kind::Trade => StreamType::Trade,
3957 Kind::AggTrade => StreamType::AggTrade,
3958 Kind::Kline(iv) => StreamType::Kline { interval: iv.as_str().to_string() },
3959 Kind::Ticker => StreamType::Ticker,
3960 Kind::Orderbook => StreamType::Orderbook,
3961 Kind::OrderbookDelta => StreamType::OrderbookDelta,
3962 Kind::MarkPrice => StreamType::MarkPrice,
3963 Kind::FundingRate => StreamType::FundingRate,
3964 Kind::OpenInterest => StreamType::OpenInterest,
3965 Kind::Liquidation => StreamType::Liquidation,
3966 Kind::BlockTrade => StreamType::BlockTrade,
3967 Kind::AuctionEvent => StreamType::AuctionEvent,
3968 Kind::IndexPrice => StreamType::IndexPrice,
3969 Kind::CompositeIndex => StreamType::CompositeIndex,
3970 Kind::OptionGreeks => StreamType::OptionGreeks,
3971 Kind::VolatilityIndex => StreamType::VolatilityIndex,
3972 Kind::HistoricalVolatility => StreamType::HistoricalVolatility,
3973 Kind::LongShortRatio => StreamType::LongShortRatio,
3977 Kind::TakerVolume | Kind::LiquidationBucket => {
3978 unreachable!("TakerVolume/LiquidationBucket are poll-only — ws_request_for must not be called for them")
3979 }
3980 Kind::Basis => StreamType::Basis,
3981 Kind::InsuranceFund => StreamType::InsuranceFund,
3982 Kind::OrderbookL3 => StreamType::OrderbookL3,
3983 Kind::SettlementEvent => StreamType::SettlementEvent,
3984 Kind::MarketWarning => StreamType::MarketWarning,
3985 Kind::RiskLimit => StreamType::RiskLimit,
3986 Kind::PredictedFunding => StreamType::PredictedFunding,
3987 Kind::FundingSettlement => StreamType::FundingSettlement,
3988 Kind::MarkPriceKline(iv) => StreamType::MarkPriceKline { interval: iv.as_str().to_string() },
3989 Kind::IndexPriceKline(iv) => StreamType::IndexPriceKline { interval: iv.as_str().to_string() },
3990 Kind::PremiumIndexKline(iv) => StreamType::PremiumIndexKline { interval: iv.as_str().to_string() },
3991 Kind::OrderUpdate => StreamType::OrderUpdate,
3992 Kind::BalanceUpdate => StreamType::BalanceUpdate,
3993 Kind::PositionUpdate => StreamType::PositionUpdate,
3994 Kind::RangeBar(_) | Kind::TickBar(_) | Kind::VolumeBar(_) | Kind::Footprint(_)
3997 | Kind::RenkoBar(_, _) | Kind::PnfBar(_, _) | Kind::KagiBar(_)
3998 | Kind::CvdLine | Kind::ThreeLineBreak { .. } | Kind::TpoProfile(_, _)
3999 | Kind::DollarBar { .. } | Kind::TickImbalanceBar { .. }
4000 | Kind::VolumeImbalanceBar { .. } | Kind::RunBar { .. } => {
4001 unreachable!("derived kinds must not call ws_request_for")
4002 }
4003 };
4004 SubscriptionRequest {
4005 symbol: sym,
4006 stream_type,
4007 account_type: account,
4008 depth: None,
4009 update_speed_ms: None,
4010 }
4011}
4012
4013fn parse_symbol(s: &str) -> Symbol {
4014 if let Some((b, q)) = s.split_once(['-', '/', '_']) {
4015 return Symbol::new(b, q);
4016 }
4017 let upper = s.to_uppercase();
4018 for q in ["USDT", "USDC", "USD", "BTC", "ETH", "BUSD", "EUR", "JPY"] {
4019 if let Some(base) = upper.strip_suffix(q) {
4020 if !base.is_empty() {
4021 return Symbol::new(base, q);
4022 }
4023 }
4024 }
4025 Symbol::new(&upper, "")
4026}
4027
4028pub(crate) fn caps_explicitly_unsupported(caps: &ConnectorCapabilities, kind: &Kind) -> bool {
4043 match kind {
4044 Kind::Trade => !caps.has_recent_trades && !caps.has_ws_trades,
4045 Kind::AggTrade => !caps.has_agg_trades && !caps.has_ws_trades,
4046 Kind::Kline(_) => !caps.has_klines && !caps.has_ws_klines,
4047 Kind::Ticker => !caps.has_ticker && !caps.has_ws_ticker,
4048 Kind::Orderbook => !caps.has_orderbook && !caps.has_ws_orderbook,
4049 Kind::OrderbookDelta => !caps.has_orderbook && !caps.has_ws_orderbook,
4050 Kind::MarkPrice => !caps.has_premium_index && !caps.has_ws_mark_price,
4051 Kind::FundingRate => !caps.has_funding_rate_history && !caps.has_ws_funding_rate,
4052 Kind::OpenInterest => !caps.has_open_interest_history,
4053 Kind::Liquidation => !caps.has_liquidation_history,
4054 Kind::MarkPriceKline(_) => !caps.has_mark_price_klines,
4055 Kind::IndexPriceKline(_) => !caps.has_index_price_klines,
4056 Kind::PremiumIndexKline(_) => !caps.has_premium_index_klines,
4057 Kind::LongShortRatio => !caps.has_long_short_ratio_history,
4059 Kind::TakerVolume => !caps.has_taker_volume_history,
4060 Kind::LiquidationBucket => !caps.has_liquidation_bucket_history,
4061 Kind::InsuranceFund => !caps.has_insurance_fund,
4062 Kind::RangeBar(_)
4064 | Kind::TickBar(_)
4065 | Kind::VolumeBar(_)
4066 | Kind::Footprint(_)
4067 | Kind::RenkoBar(_, _)
4068 | Kind::PnfBar(_, _)
4069 | Kind::KagiBar(_)
4070 | Kind::CvdLine
4071 | Kind::ThreeLineBreak { .. }
4072 | Kind::TpoProfile(_, _)
4073 | Kind::DollarBar { .. }
4074 | Kind::TickImbalanceBar { .. }
4075 | Kind::VolumeImbalanceBar { .. }
4076 | Kind::RunBar { .. }
4077 | Kind::Basis
4078 | Kind::FundingSettlement => false,
4079 Kind::BlockTrade
4082 | Kind::AuctionEvent
4083 | Kind::IndexPrice
4084 | Kind::CompositeIndex
4085 | Kind::OptionGreeks
4086 | Kind::VolatilityIndex
4087 | Kind::HistoricalVolatility
4088 | Kind::OrderbookL3
4089 | Kind::SettlementEvent
4090 | Kind::MarketWarning
4091 | Kind::RiskLimit
4092 | Kind::PredictedFunding
4093 | Kind::OrderUpdate
4094 | Kind::BalanceUpdate
4095 | Kind::PositionUpdate => false,
4096 }
4097}
4098
4099#[cfg(test)]
4100mod flush_persistence_tests {
4101 use super::*;
4102 use crate::series::Kind as SeriesKind;
4103 use digdigdig3::core::types::{AccountType, ExchangeId};
4104
4105 fn test_key(symbol: &str) -> SeriesKey {
4106 SeriesKey::new(ExchangeId::Binance, AccountType::Spot, symbol, SeriesKind::Trade)
4107 }
4108
4109 #[tokio::test]
4115 async fn flush_persistence_round_trips_through_registered_handle() {
4116 let tmp = std::env::temp_dir().join(format!(
4117 "dig3-flush-persistence-test-{}-{}",
4118 std::process::id(),
4119 chrono::Utc::now().timestamp_nanos_opt().unwrap_or(0),
4120 ));
4121
4122 let station = Station::builder()
4123 .storage_root(&tmp)
4124 .build()
4125 .await
4126 .expect("Station::build is pure-local (no network) — must succeed");
4127
4128 let key = test_key("BTCUSDT");
4129 let (handle, mut flush_rx) = FlushHandle::channel();
4130 station.inner.flush_handles.insert(key.clone(), handle);
4131
4132 let synthetic = tokio::spawn(async move {
4136 let ack = flush_rx.recv().await.expect("flush_persistence must send a request");
4137 let _ = ack.send(Ok(()));
4138 });
4139
4140 let result = station.flush_persistence().await;
4141 assert!(result.is_ok(), "expected Ok(()), got {result:?}");
4142
4143 synthetic.await.expect("synthetic forwarder task panicked");
4144
4145 assert!(!station.inner.flush_handles.contains_key(&key));
4152
4153 let _ = std::fs::remove_dir_all(&tmp);
4154 }
4155
4156 #[tokio::test]
4160 async fn flush_persistence_treats_dead_handle_as_already_flushed() {
4161 let tmp = std::env::temp_dir().join(format!(
4162 "dig3-flush-persistence-dead-test-{}-{}",
4163 std::process::id(),
4164 chrono::Utc::now().timestamp_nanos_opt().unwrap_or(0),
4165 ));
4166
4167 let station = Station::builder()
4168 .storage_root(&tmp)
4169 .build()
4170 .await
4171 .expect("Station::build is pure-local (no network) — must succeed");
4172
4173 let key = test_key("ETHUSDT");
4174 let (handle, flush_rx) = FlushHandle::channel();
4175 drop(flush_rx); station.inner.flush_handles.insert(key.clone(), handle);
4177
4178 let result = station.flush_persistence().await;
4179 assert!(result.is_ok(), "dead handle must not surface as an error: {result:?}");
4180
4181 assert!(!station.inner.flush_handles.contains_key(&key));
4183
4184 let _ = std::fs::remove_dir_all(&tmp);
4185 }
4186
4187 #[tokio::test]
4190 async fn flush_persistence_aggregates_forwarder_reported_errors() {
4191 let tmp = std::env::temp_dir().join(format!(
4192 "dig3-flush-persistence-err-test-{}-{}",
4193 std::process::id(),
4194 chrono::Utc::now().timestamp_nanos_opt().unwrap_or(0),
4195 ));
4196
4197 let station = Station::builder()
4198 .storage_root(&tmp)
4199 .build()
4200 .await
4201 .expect("Station::build is pure-local (no network) — must succeed");
4202
4203 let key = test_key("SOLUSDT");
4204 let (handle, mut flush_rx) = FlushHandle::channel();
4205 station.inner.flush_handles.insert(key.clone(), handle);
4206
4207 let synthetic = tokio::spawn(async move {
4208 let ack = flush_rx.recv().await.expect("flush_persistence must send a request");
4209 let _ = ack.send(Err(std::io::Error::other("simulated disk failure")));
4210 });
4211
4212 let result = station.flush_persistence().await;
4213 synthetic.await.expect("synthetic forwarder task panicked");
4214
4215 let errors = result.expect_err("a reported flush error must surface");
4216 assert_eq!(errors.len(), 1);
4217 assert_eq!(errors[0].0, key);
4218
4219 let _ = std::fs::remove_dir_all(&tmp);
4220 }
4221}
4222
4223#[cfg(test)]
4224mod force_unsubscribe_tests {
4225 use super::*;
4226 use crate::series::Kind as SeriesKind;
4227 use digdigdig3::core::types::{AccountType, ExchangeId};
4228
4229 fn test_key(symbol: &str) -> SeriesKey {
4230 SeriesKey::new(ExchangeId::Binance, AccountType::Spot, symbol, SeriesKind::Trade)
4231 }
4232
4233 #[tokio::test]
4235 async fn force_unsubscribe_is_idempotent_when_no_mux_exists() {
4236 let tmp = std::env::temp_dir().join(format!(
4237 "dig3-force-unsub-noop-test-{}-{}",
4238 std::process::id(),
4239 chrono::Utc::now().timestamp_nanos_opt().unwrap_or(0),
4240 ));
4241 let station = Station::builder()
4242 .storage_root(&tmp)
4243 .build()
4244 .await
4245 .expect("Station::build is pure-local (no network) — must succeed");
4246
4247 let key = test_key("BTCUSDT");
4248 let result = station.force_unsubscribe_and_await(&key).await;
4249 assert!(result.is_ok());
4250
4251 let _ = std::fs::remove_dir_all(&tmp);
4252 }
4253
4254 #[tokio::test]
4267 async fn force_unsubscribe_fires_shutdown_and_awaits_exit_ack() {
4268 let tmp = std::env::temp_dir().join(format!(
4269 "dig3-force-unsub-mechanics-test-{}-{}",
4270 std::process::id(),
4271 chrono::Utc::now().timestamp_nanos_opt().unwrap_or(0),
4272 ));
4273 let station = Station::builder()
4274 .storage_root(&tmp)
4275 .build()
4276 .await
4277 .expect("Station::build is pure-local (no network) — must succeed");
4278
4279 let key = test_key("ETHUSDT");
4280
4281 let (bcast_tx, _) = broadcast::channel::<Event>(16);
4282 let (shutdown_tx, shutdown_rx) = oneshot::channel::<()>();
4283 let (exit_ack_tx, exit_ack_rx) = oneshot::channel::<()>();
4284
4285 station.inner.muxes.insert(
4289 key.clone(),
4290 Multiplexer {
4291 tx: bcast_tx,
4292 consumers: Arc::new(AtomicUsize::new(2)),
4293 shutdown: Some(shutdown_tx),
4294 grace_cancel: None,
4295 },
4296 );
4297 station.inner.exit_acks.insert(key.clone(), exit_ack_rx);
4298 let dummy_series: Arc<dyn Any + Send + Sync> =
4299 Arc::new(Arc::new(RwLock::new(Series::<TradePoint>::new(4))));
4300 station.inner.series_handles.insert(key.clone(), dummy_series);
4301
4302 let shutdown_observed = Arc::new(std::sync::atomic::AtomicBool::new(false));
4307 let shutdown_observed_clone = Arc::clone(&shutdown_observed);
4308 let synthetic = tokio::spawn(async move {
4309 let _ = shutdown_rx.await;
4310 shutdown_observed_clone.store(true, Ordering::SeqCst);
4311 let _ = exit_ack_tx.send(());
4312 });
4313
4314 let result = station.force_unsubscribe_and_await(&key).await;
4315 assert!(result.is_ok());
4316
4317 assert!(!station.inner.muxes.contains_key(&key));
4320 assert!(!station.inner.series_handles.contains_key(&key));
4321 assert!(shutdown_observed.load(Ordering::SeqCst));
4324
4325 synthetic.await.expect("synthetic forwarder task panicked");
4326 let _ = std::fs::remove_dir_all(&tmp);
4327 }
4328
4329 #[tokio::test]
4334 async fn force_unsubscribe_treats_already_exited_forwarder_as_done() {
4335 let tmp = std::env::temp_dir().join(format!(
4336 "dig3-force-unsub-already-exited-test-{}-{}",
4337 std::process::id(),
4338 chrono::Utc::now().timestamp_nanos_opt().unwrap_or(0),
4339 ));
4340 let station = Station::builder()
4341 .storage_root(&tmp)
4342 .build()
4343 .await
4344 .expect("Station::build is pure-local (no network) — must succeed");
4345
4346 let key = test_key("SOLUSDT");
4347 let result = station.force_unsubscribe_and_await(&key).await;
4350 assert!(result.is_ok());
4351
4352 let _ = std::fs::remove_dir_all(&tmp);
4353 }
4354
4355 #[tokio::test]
4360 async fn force_unsubscribe_does_not_hang_on_dropped_ack_sender() {
4361 let tmp = std::env::temp_dir().join(format!(
4362 "dig3-force-unsub-dropped-sender-test-{}-{}",
4363 std::process::id(),
4364 chrono::Utc::now().timestamp_nanos_opt().unwrap_or(0),
4365 ));
4366 let station = Station::builder()
4367 .storage_root(&tmp)
4368 .build()
4369 .await
4370 .expect("Station::build is pure-local (no network) — must succeed");
4371
4372 let key = test_key("DOGEUSDT");
4373 let (bcast_tx, _) = broadcast::channel::<Event>(16);
4374 let (shutdown_tx, _shutdown_rx) = oneshot::channel::<()>();
4375 let (exit_ack_tx, exit_ack_rx) = oneshot::channel::<()>();
4376 drop(exit_ack_tx); station.inner.muxes.insert(
4379 key.clone(),
4380 Multiplexer {
4381 tx: bcast_tx,
4382 consumers: Arc::new(AtomicUsize::new(1)),
4383 shutdown: Some(shutdown_tx),
4384 grace_cancel: None,
4385 },
4386 );
4387 station.inner.exit_acks.insert(key.clone(), exit_ack_rx);
4388
4389 let result = tokio::time::timeout(
4390 std::time::Duration::from_secs(5),
4391 station.force_unsubscribe_and_await(&key),
4392 )
4393 .await
4394 .expect("force_unsubscribe_and_await must not hang on a dropped ack sender");
4395 assert!(result.is_ok());
4396
4397 let _ = std::fs::remove_dir_all(&tmp);
4398 }
4399}
4400
4401#[cfg(test)]
4402mod rewarm_derived_tests {
4403 use super::*;
4413 use crate::series::Kind as SeriesKind;
4414 use digdigdig3::core::types::{AccountType, ExchangeId};
4415
4416 fn test_key(symbol: &str, kind: SeriesKind) -> SeriesKey {
4417 SeriesKey::new(ExchangeId::Binance, AccountType::Spot, symbol, kind)
4418 }
4419
4420 fn trade_event(ts_ms: i64, price: f64, quantity: f64, side: u8) -> Event {
4421 Event::Trade {
4422 exchange: ExchangeId::Binance,
4423 symbol: "BTCUSDT".to_string(),
4424 point: TradePoint { ts_ms, price, quantity, side, trade_id_hash: 0 },
4425 }
4426 }
4427
4428 async fn station_with_tmp() -> (Station, PathBuf) {
4429 let tmp = std::env::temp_dir().join(format!(
4430 "dig3-rewarm-derived-test-{}-{}",
4431 std::process::id(),
4432 chrono::Utc::now().timestamp_nanos_opt().unwrap_or(0),
4433 ));
4434 let station = Station::builder()
4435 .storage_root(&tmp)
4436 .build()
4437 .await
4438 .expect("Station::build is pure-local (no network) — must succeed");
4439 (station, tmp)
4440 }
4441
4442 fn insert_series<T: DataPoint + Send + Sync + 'static>(
4443 station: &Station,
4444 key: &SeriesKey,
4445 points: Vec<T>,
4446 ) {
4447 let mut series = Series::<T>::new(points.len().max(16));
4448 series.extend(points);
4449 let handle: Arc<dyn Any + Send + Sync> = Arc::new(Arc::new(RwLock::new(series)));
4450 station.inner.series_handles.insert(key.clone(), handle);
4451 }
4452
4453 #[tokio::test]
4459 async fn renko_rewarm_keeps_existing_bricks_and_shares_grid() {
4460 let (station, tmp) = station_with_tmp().await;
4461 let key = test_key("BTCUSDT", SeriesKind::RenkoBar(100_000_000, 1));
4463
4464 let existing = vec![RenkoBrickPoint {
4466 open_time: 10_000,
4467 bottom: 100.0,
4468 top: 101.0,
4469 up: true,
4470 volume: 5.0,
4471 trades_count: 3,
4472 }];
4473 insert_series(&station, &key, existing.clone());
4474
4475 let (older_via_public, _outcome) = station.rewarm_renko(&key, "BTCUSDT", 100).await;
4480 assert!(older_via_public.is_empty(), "no REST connector in unit test — fold input is empty");
4481
4482 let events = vec![
4492 trade_event(1_000, 97.5, 1.0, 0),
4493 trade_event(2_000, 98.5, 1.0, 0),
4494 trade_event(3_000, 99.5, 1.0, 0),
4495 ];
4496 let grid_floor = existing[0].bottom; let emissions = Station::rewarm_fold::<TradeToRenkoBarDerived>(&key, &events, |state| {
4498 state.preset_grid_anchor(grid_floor);
4499 })
4500 .await;
4501 let rebuilt = Station::rewarm_dedup(emissions, |p: &RenkoBrickPoint| p.open_time);
4502 assert!(!rebuilt.is_empty(), "expected at least one brick from the older window");
4503 let older = station.rewarm_splice(&key, rebuilt, |p: &RenkoBrickPoint| p.open_time).await;
4504 assert!(!older.is_empty(), "expected at least one prepended brick");
4505 for p in &older {
4506 let steps = (p.bottom - 100.0) / 1.0;
4510 assert!(
4511 (steps - steps.round()).abs() < 1e-9,
4512 "brick bottom {} is not grid-aligned with existing bottom 100.0",
4513 p.bottom
4514 );
4515 }
4516 assert!(older.iter().all(|p| p.open_time < 10_000));
4518
4519 let handle = station.inner.series_handles.get(&key).and_then(|e| {
4522 e.downcast_ref::<Arc<RwLock<Series<RenkoBrickPoint>>>>().map(Arc::clone)
4523 }).expect("series handle must still exist after splice");
4524 let after = handle.read().await.snapshot();
4525 let spliced_existing = after.last().expect("existing brick must be at the tail");
4526 assert_eq!(spliced_existing.open_time, existing[0].open_time);
4527 assert_eq!(spliced_existing.bottom, existing[0].bottom);
4528 assert_eq!(spliced_existing.top, existing[0].top);
4529 assert_eq!(spliced_existing.up, existing[0].up);
4530 assert_eq!(spliced_existing.volume, existing[0].volume);
4531 assert_eq!(spliced_existing.trades_count, existing[0].trades_count);
4532
4533 let _ = std::fs::remove_dir_all(&tmp);
4534 }
4535
4536 #[tokio::test]
4537 async fn renko_rewarm_no_existing_series_returns_empty() {
4538 let (station, tmp) = station_with_tmp().await;
4539 let key = test_key("BTCUSDT", SeriesKind::RenkoBar(100_000_000, 1));
4540 let (older, _outcome) = station.rewarm_renko(&key, "BTCUSDT", 100).await;
4542 assert!(older.is_empty());
4543 let _ = std::fs::remove_dir_all(&tmp);
4544 }
4545
4546 #[tokio::test]
4552 async fn tick_bar_rewarm_existing_untouched_prepended_strictly_older() {
4553 let (station, tmp) = station_with_tmp().await;
4554 let key = test_key("BTCUSDT", SeriesKind::TickBar(2));
4556
4557 let existing = vec![BarPoint {
4558 open_time: 50_000,
4559 open: 200.0,
4560 high: 201.0,
4561 low: 199.0,
4562 close: 200.5,
4563 volume: 4.0,
4564 quote_volume: 800.0,
4565 trades_count: 2,
4566 }];
4567 insert_series(&station, &key, existing.clone());
4568
4569 let (older, _outcome) = station
4570 .rewarm_derived_standalone::<TradeToTickBarDerived>(&key, "BTCUSDT", 100)
4571 .await;
4572
4573 assert!(older.is_empty(), "no REST connector in unit test — fold input is empty");
4580
4581 let events = vec![
4584 trade_event(10_000, 190.0, 1.0, 0),
4585 trade_event(11_000, 191.0, 1.0, 0),
4586 trade_event(12_000, 192.0, 1.0, 1),
4587 trade_event(13_000, 193.0, 1.0, 1),
4588 ];
4589 let emissions = Station::rewarm_fold::<TradeToTickBarDerived>(&key, &events, |_s| {}).await;
4590 let rebuilt = Station::rewarm_dedup(emissions, |p: &BarPoint| p.open_time);
4591 assert_eq!(rebuilt.len(), 2);
4593 let spliced = station.rewarm_splice(&key, rebuilt, |p: &BarPoint| p.open_time).await;
4594 assert_eq!(spliced.len(), 2, "both synthetic bars are older than the existing head");
4595 assert!(spliced.iter().all(|p| p.open_time < 50_000), "prepended bars must be strictly older");
4596
4597 let handle = station.inner.series_handles.get(&key).and_then(|e| {
4599 e.downcast_ref::<Arc<RwLock<Series<BarPoint>>>>().map(Arc::clone)
4600 }).expect("series handle must still exist after splice");
4601 let after = handle.read().await.snapshot();
4602 let spliced_existing = after.last().expect("existing bar must be at the tail");
4603 assert_eq!(spliced_existing.open_time, existing[0].open_time);
4604 assert_eq!(spliced_existing.open, existing[0].open);
4605 assert_eq!(spliced_existing.close, existing[0].close);
4606 assert_eq!(spliced_existing.trades_count, existing[0].trades_count);
4607 assert!(after[..after.len() - 1].iter().all(|p| p.open_time != existing[0].open_time));
4610
4611 let _ = std::fs::remove_dir_all(&tmp);
4612 }
4613
4614 #[tokio::test]
4618 async fn tick_bar_rewarm_boundary_collision_existing_wins() {
4619 let (station, tmp) = station_with_tmp().await;
4620 let key = test_key("BTCUSDT", SeriesKind::TickBar(2));
4621
4622 let existing = vec![BarPoint {
4623 open_time: 5_000,
4624 open: 100.0,
4625 high: 100.0,
4626 low: 100.0,
4627 close: 100.0,
4628 volume: 1.0,
4629 quote_volume: 100.0,
4630 trades_count: 1,
4631 }];
4632 insert_series(&station, &key, existing.clone());
4633
4634 let rebuilt = vec![
4637 BarPoint {
4638 open_time: 1_000,
4639 open: 90.0, high: 90.0, low: 90.0, close: 90.0,
4640 volume: 1.0, quote_volume: 90.0, trades_count: 1,
4641 },
4642 BarPoint {
4643 open_time: 5_000, open: 999.0, high: 999.0, low: 999.0, close: 999.0,
4645 volume: 999.0, quote_volume: 999.0, trades_count: 99,
4646 },
4647 ];
4648 let spliced = station.rewarm_splice(&key, rebuilt, |p: &BarPoint| p.open_time).await;
4649 assert_eq!(spliced.len(), 1);
4651 assert_eq!(spliced[0].open_time, 1_000);
4652
4653 let handle = station.inner.series_handles.get(&key).and_then(|e| {
4654 e.downcast_ref::<Arc<RwLock<Series<BarPoint>>>>().map(Arc::clone)
4655 }).expect("series handle must still exist");
4656 let after = handle.read().await.snapshot();
4657 let head = after.iter().find(|p| p.open_time == 5_000).expect("existing head must survive");
4659 assert_eq!(head.open, 100.0, "existing bar must never be overwritten by the fold's copy");
4660
4661 let _ = std::fs::remove_dir_all(&tmp);
4662 }
4663
4664 #[tokio::test]
4670 async fn cvd_rewarm_offsets_to_meet_existing_first_value() {
4671 let (station, tmp) = station_with_tmp().await;
4672 let key = test_key("BTCUSDT", SeriesKind::CvdLine);
4673
4674 let existing = vec![ScalarBarPoint { ts_ms: 10_000, value: 500.0 }];
4675 insert_series(&station, &key, existing.clone());
4676
4677 let events = vec![
4680 trade_event(1_000, 100.0, 1.0, 0), trade_event(2_000, 100.0, 1.0, 1), trade_event(3_000, 100.0, 1.0, 0), ];
4684 let emissions = Station::rewarm_fold::<TradeToCvdLineDerived>(&key, &events, |_s| {}).await;
4685 let mut rebuilt = Station::rewarm_dedup(emissions, |p: &ScalarBarPoint| p.ts_ms);
4686 assert_eq!(rebuilt.len(), 3);
4687 let raw_last = rebuilt.last().unwrap().value;
4688 assert!((raw_last - 1.0).abs() < 1e-9, "raw fold should end at +1.0 before offset");
4689
4690 let offset = existing[0].value - raw_last;
4691 for p in &mut rebuilt {
4692 p.value += offset;
4693 }
4694 let spliced = station.rewarm_splice(&key, rebuilt, |p: &ScalarBarPoint| p.ts_ms).await;
4695 assert_eq!(spliced.len(), 3);
4696 let last_prepended = spliced.last().unwrap();
4697 assert!(
4698 (last_prepended.value - existing[0].value).abs() < 1e-9,
4699 "last prepended CVD sample must equal existing first value exactly"
4700 );
4701
4702 let handle = station.inner.series_handles.get(&key).and_then(|e| {
4704 e.downcast_ref::<Arc<RwLock<Series<ScalarBarPoint>>>>().map(Arc::clone)
4705 }).expect("series handle must still exist");
4706 let after = handle.read().await.snapshot();
4707 let tail = after.last().expect("existing sample must be at the tail");
4708 assert_eq!(tail.ts_ms, existing[0].ts_ms);
4709 assert_eq!(tail.value, existing[0].value);
4710
4711 let _ = std::fs::remove_dir_all(&tmp);
4712 }
4713
4714 #[tokio::test]
4715 async fn cvd_rewarm_public_helper_matches_manual_offset() {
4716 let (station, tmp) = station_with_tmp().await;
4717 let key = test_key("BTCUSDT", SeriesKind::CvdLine);
4718 let existing = vec![ScalarBarPoint { ts_ms: 10_000, value: -25.0 }];
4719 insert_series(&station, &key, existing.clone());
4720
4721 let (older, _outcome) = station.rewarm_cvd(&key, "BTCUSDT", 100).await;
4725 assert!(older.is_empty());
4726
4727 let handle = station.inner.series_handles.get(&key).and_then(|e| {
4728 e.downcast_ref::<Arc<RwLock<Series<ScalarBarPoint>>>>().map(Arc::clone)
4729 }).expect("series handle must still exist");
4730 let after = handle.read().await.snapshot();
4731 assert_eq!(after.len(), 1);
4732 assert_eq!(after[0].value, existing[0].value);
4733
4734 let _ = std::fs::remove_dir_all(&tmp);
4735 }
4736
4737 #[tokio::test]
4742 async fn run_bar_rewarm_returns_unsupported_error() {
4743 let (station, tmp) = station_with_tmp().await;
4744 let key = test_key("BTCUSDT", SeriesKind::RunBar { alpha_x100: 20, min_ticks: 10 });
4745 let result = station.rewarm_derived(&key, "BTCUSDT", 100).await;
4746 assert!(matches!(result, Err(StationError::RewarmUnsupported(_))));
4747 let _ = std::fs::remove_dir_all(&tmp);
4748 }
4749
4750 #[tokio::test]
4751 async fn tick_imbalance_bar_rewarm_returns_unsupported_error() {
4752 let (station, tmp) = station_with_tmp().await;
4753 let key = test_key("BTCUSDT", SeriesKind::TickImbalanceBar { alpha_x100: 20, min_ticks: 10 });
4754 let result = station.rewarm_derived(&key, "BTCUSDT", 100).await;
4755 assert!(matches!(result, Err(StationError::RewarmUnsupported(_))));
4756 let _ = std::fs::remove_dir_all(&tmp);
4757 }
4758
4759 #[tokio::test]
4760 async fn volume_imbalance_bar_rewarm_returns_unsupported_error() {
4761 let (station, tmp) = station_with_tmp().await;
4762 let key = test_key("BTCUSDT", SeriesKind::VolumeImbalanceBar { alpha_x100: 20, min_ticks: 10 });
4763 let result = station.rewarm_derived(&key, "BTCUSDT", 100).await;
4764 assert!(matches!(result, Err(StationError::RewarmUnsupported(_))));
4765 let _ = std::fs::remove_dir_all(&tmp);
4766 }
4767
4768 #[tokio::test]
4769 async fn footprint_rewarm_derived_returns_unsupported_error_pointing_at_dedicated_fn() {
4770 let (station, tmp) = station_with_tmp().await;
4771 let key = test_key("BTCUSDT", SeriesKind::Footprint(KlineInterval::new("1m")));
4772 let result = station.rewarm_derived(&key, "BTCUSDT", 100).await;
4773 assert!(matches!(result, Err(StationError::RewarmUnsupported(_))));
4774 let _ = std::fs::remove_dir_all(&tmp);
4775 }
4776
4777 #[test]
4782 fn rewarm_dedup_keeps_last_emission_per_identity() {
4783 let points = vec![
4784 ScalarBarPoint { ts_ms: 1, value: 1.0 },
4785 ScalarBarPoint { ts_ms: 1, value: 2.0 }, ScalarBarPoint { ts_ms: 2, value: 3.0 },
4787 ];
4788 let out = Station::rewarm_dedup(points, |p: &ScalarBarPoint| p.ts_ms);
4789 assert_eq!(out.len(), 2);
4790 assert_eq!(out[0].value, 2.0, "second emission at ts=1 must supersede the first");
4791 assert_eq!(out[1].value, 3.0);
4792 }
4793}