Skip to main content

digdigdig3_station/
station.rs

1use std::any::Any;
2use std::path::PathBuf;
3use std::sync::atomic::{AtomicUsize, Ordering};
4use std::sync::Arc;
5
6use crate::ws_health::WsHealth;
7
8use dashmap::DashMap;
9use digdigdig3::connector_manager::ExchangeHub;
10use digdigdig3::core::types::{
11    AccountType, ConnectorCapabilities, ExchangeId, StreamEvent, StreamType, SubscriptionRequest,
12    Symbol, SymbolInfo,
13};
14use digdigdig3::core::websocket::KlineInterval;
15use digdigdig3::core::utils::SymbolNormalizer;
16use futures_util::StreamExt;
17use tokio::sync::{broadcast, mpsc, oneshot, RwLock};
18
19use crate::data::{
20    AggTradePoint, AuctionEventPoint, BalanceUpdatePoint, BarPoint, BasisPoint, BlockTradePoint,
21    CompositeIndexPoint,
22    FootprintPoint,
23    FundingRatePoint, FundingRateIndicatorsPoint, FundingRateFullPoint,
24    FundingSettlementPoint, HistoricalVolatilityPoint,
25    IndexPriceKlinePoint, IndexPricePoint, IndexPriceIndicatorsPoint,
26    InsuranceFundPoint,
27    LiquidationPoint, LiquidationIndicatorsPoint, LiquidationFullPoint,
28    LiquidationBucketPoint,
29    LongShortRatioPoint,
30    MarkPriceKlinePoint, MarkPricePoint, MarkPriceIndicatorsPoint, MarkPriceFullPoint,
31    MarketWarningPoint,
32    ObDeltaPoint, ObDeltaIndicatorsPoint,
33    ObSnapshotPoint, ObSnapshotIndicatorsPoint,
34    OpenInterestPoint, OpenInterestIndicatorsPoint,
35    OptionGreeksPoint,
36    OrderUpdatePoint, OrderbookL3Point, PositionUpdatePoint,
37    PredictedFundingPoint, PremiumIndexKlinePoint, RiskLimitPoint, SettlementEventPoint,
38    TakerVolumePoint,
39    TickerPoint, TickerIndicatorsPoint, TickerFullPoint,
40    TradePoint, VolatilityIndexPoint,
41    KagiSegmentPoint, PnfColumnPoint, RenkoBrickPoint, ScalarBarPoint,
42    ThreeLineBreakLinePoint, TpoSessionPoint,
43};
44use crate::persistence::PersistDepth;
45use crate::derived::{
46    BasisDerived, DerivedStream, FundingSettlementDerived, TradeToBarDerived,
47    TradeToRangeBarDerived, TradeToTickBarDerived, TradeToVolumeBarDerived,
48    TradeToFootprintDerived, TradeToRenkoBarDerived, TradeToPnfBarDerived,
49    TradeToKagiBarDerived, TradeToCvdLineDerived, TradeToThreeLineBreakDerived,
50    TpoFromKline1mDerived,
51    TpoFromTradeDerived, TradeToDollarBarDerived, TradeToTickImbalanceDerived,
52    TradeToVolumeImbalanceDerived, TradeToRunBarDerived, interval_to_ms,
53};
54use crate::series::TpoSource;
55#[cfg(not(target_arch = "wasm32"))]
56use crate::polling;
57#[cfg(not(target_arch = "wasm32"))]
58use crate::polling::PollSource;
59use crate::series::DiskStore;
60use crate::series::{DataPoint, Kind, Series, SeriesKey};
61use crate::subscription::{Entry, Event, FailedStream, MultiplexRef, Stream};
62use crate::{
63    PersistenceConfig, Result, StationBuilder, StationError, SubscribeReport, SubscriptionHandle,
64    SubscriptionSet,
65};
66
67
68/// Phase 5 Station. Unified `Series<T>` + `DiskStore<T>` plumbing under all
69/// stream classes. One multiplexer actor per `SeriesKey` (= exchange × account
70/// × symbol × kind). N consumers share via `broadcast::channel`.
71pub struct Station {
72    pub(crate) inner: Arc<StationInner>,
73}
74
75pub(crate) struct StationInner {
76    pub(crate) hub: Arc<ExchangeHub>,
77    pub(crate) storage_root: PathBuf,
78    pub(crate) persistence: PersistenceConfig,
79    pub(crate) muxes: DashMap<SeriesKey, Multiplexer>,
80    /// Sync-accessible series handles for render-time consumers.
81    ///
82    /// Each active forwarder stores `Arc<RwLock<Series<T>>>` here (type-erased
83    /// to `Arc<dyn Any + Send + Sync>`). `Station::series<T>()` retrieves and
84    /// downcasts. Entries are removed when the forwarder exits (same lifecycle
85    /// as `muxes`).
86    pub(crate) series_handles: DashMap<SeriesKey, Arc<dyn Any + Send + Sync>>,
87    pub(crate) warm_start_capacity: usize,
88    pub(crate) gap_heal: crate::GapHealConfig,
89    /// How long to keep a forwarder alive after its last consumer drops.
90    /// `Duration::ZERO` = immediate shutdown (default).
91    pub(crate) unsubscribe_grace: std::time::Duration,
92    /// Issue a one-shot REST `get_orderbook` seed on first subscribe to
93    /// `Orderbook` / `OrderbookDelta`. False = WS-only (default).
94    pub(crate) orderbook_rest_seed: bool,
95    /// Depth for the REST seed. Passed as `Some(depth as u16)` to `get_orderbook`.
96    pub(crate) orderbook_seed_depth: usize,
97    /// Broadcast channel for connector lifecycle events (`ConnectorReady`,
98    /// `SymbolsLoaded`). Independent of per-`SeriesKey` data muxes.
99    /// Capacity 256 — lag drops oldest.
100    pub(crate) connector_tx: broadcast::Sender<crate::subscription::Event>,
101    /// Cache for `get_exchange_info` results, keyed by `(exchange, account_type)`.
102    /// Populated by `warmup()`; re-emits without REST on repeated calls.
103    pub(crate) exchange_info_cache: DashMap<(ExchangeId, AccountType), Vec<SymbolInfo>>,
104    /// Per-`SeriesKey` flush notifiers for forwarders that opened a
105    /// `DiskStore`. Registered at spawn time, unregistered by the forwarder
106    /// itself right before it exits. Used by [`Station::flush_persistence`]
107    /// to force every open store to drain + flush synchronously from the
108    /// caller's perspective — needed on wasm where `beforeunload` /
109    /// `visibilitychange` can close the tab before `Drop` gets a chance to
110    /// await an async OPFS flush. Native `BufWriter` already flushes on
111    /// `Drop`, but the API is uniform across both targets.
112    pub(crate) flush_handles: DashMap<SeriesKey, FlushHandle>,
113    /// Per-`SeriesKey` forwarder-exit acknowledgement receivers, registered
114    /// at spawn time (same lifecycle as `flush_handles`). The paired
115    /// `oneshot::Sender` half is held privately by the forwarder task and
116    /// fired exactly once, right before it returns (any exit reason:
117    /// shutdown signal, upstream closed, WS silence with no resub path).
118    /// Used by [`Station::force_unsubscribe_and_await`] to await confirmation
119    /// that a forwarder has actually torn down (mux entry removed, upstream
120    /// refs released) rather than merely requesting shutdown and hoping.
121    pub(crate) exit_acks: DashMap<SeriesKey, ExitAck>,
122    /// Cold-seed [`crate::SeedOutcome`] recorded per `SeriesKey`, populated
123    /// by `acquire_or_spawn_derived_body`'s trade-history seed fetch right
124    /// before the forwarder spawns. `Station::subscribe` drains the entry
125    /// for each key it just acquired into `SubscribeReport::seed_outcomes` —
126    /// this is a side-channel (not threaded through `acquire_or_spawn`'s
127    /// return type) so non-derived paths (WS-only, poll-only, orderbook
128    /// REST seed) are unaffected. Entries for keys with no recorded seed
129    /// (WS-only cold start, or a re-acquire of an already-live mux) are
130    /// simply absent — `subscribe()` only forwards what's present.
131    pub(crate) seed_outcomes: DashMap<SeriesKey, crate::SeedOutcome>,
132}
133
134/// Receiver half of a forwarder-exit acknowledgement. See
135/// [`StationInner::exit_acks`].
136pub(crate) type ExitAck = oneshot::Receiver<()>;
137
138/// Request to flush a forwarder's `DiskStore` and report the outcome.
139///
140/// Sent on [`FlushHandle::0`]; the forwarder replies on the embedded
141/// `oneshot::Sender` with the result of `DiskStore::flush().await`.
142pub(crate) type FlushAck = oneshot::Sender<std::io::Result<()>>;
143
144/// Per-forwarder flush notifier, registered in `StationInner::flush_handles`.
145///
146/// Each forwarder that opens a `DiskStore<T>` creates one of these at spawn
147/// time and keeps the paired `mpsc::Receiver<FlushAck>` in its own select
148/// loop. Sending on the channel with `SendError` (receiver dropped — the
149/// forwarder already exited) is treated as "already gone" by
150/// [`Station::flush_persistence`], never as a hard error.
151#[derive(Clone)]
152pub(crate) struct FlushHandle(mpsc::Sender<FlushAck>);
153
154impl FlushHandle {
155    /// Build a fresh handle + the receiver half the forwarder keeps in its
156    /// select loop. Capacity 1 — a flush request is a rendezvous, not a queue;
157    /// `flush_persistence` awaits each ack before moving to the next entry.
158    pub(crate) fn channel() -> (Self, mpsc::Receiver<FlushAck>) {
159        let (tx, rx) = mpsc::channel(1);
160        (Self(tx), rx)
161    }
162}
163
164/// One broadcast-fanout actor per `SeriesKey`. Each consumer increments
165/// `consumers`; on the last drop the actor shuts down.
166pub(crate) struct Multiplexer {
167    pub(crate) tx: broadcast::Sender<Event>,
168    pub(crate) consumers: Arc<AtomicUsize>,
169    pub(crate) shutdown: Option<oneshot::Sender<()>>,
170    /// Cancel sender for a pending grace-period timer task.
171    /// `Some` only while the forwarder is in the grace window
172    /// (refcount == 0 but shutdown not yet fired). Sending `()` on this
173    /// channel cancels the timer and keeps the forwarder alive.
174    /// A new subscribe arriving before the timer fires sends on this channel
175    /// and increments consumers.
176    pub(crate) grace_cancel: Option<oneshot::Sender<()>>,
177}
178
179impl std::fmt::Debug for Station {
180    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
181        f.debug_struct("Station")
182            .field("storage_root", &self.inner.storage_root)
183            .field("persistence", &self.inner.persistence)
184            .field("muxes", &self.inner.muxes.len())
185            .finish()
186    }
187}
188
189/// Typed prepended-points result of [`Station::rewarm_derived`], one
190/// variant per supported output shape. The caller downstream (mlc bridge)
191/// matches on this to build its own per-kind `LiveUpdate` batch event —
192/// this crate does not emit any bridge event itself.
193#[derive(Debug, Clone)]
194pub enum RewarmedPoints {
195    Renko(Vec<RenkoBrickPoint>),
196    Pnf(Vec<PnfColumnPoint>),
197    Kagi(Vec<KagiSegmentPoint>),
198    ThreeLineBreak(Vec<ThreeLineBreakLinePoint>),
199    /// Shared output shape for range/tick/volume/dollar bar.
200    Bar(Vec<BarPoint>),
201    Cvd(Vec<ScalarBarPoint>),
202    Tpo(Vec<TpoSessionPoint>),
203}
204
205impl RewarmedPoints {
206    /// Number of prepended points, regardless of concrete kind.
207    pub fn len(&self) -> usize {
208        match self {
209            RewarmedPoints::Renko(v) => v.len(),
210            RewarmedPoints::Pnf(v) => v.len(),
211            RewarmedPoints::Kagi(v) => v.len(),
212            RewarmedPoints::ThreeLineBreak(v) => v.len(),
213            RewarmedPoints::Bar(v) => v.len(),
214            RewarmedPoints::Cvd(v) => v.len(),
215            RewarmedPoints::Tpo(v) => v.len(),
216        }
217    }
218
219    pub fn is_empty(&self) -> bool {
220        self.len() == 0
221    }
222}
223
224/// [`Station::rewarm_derived`]'s full result: the prepended points plus the
225/// [`crate::SeedOutcome`] of the underlying trade/kline window fetch that
226/// produced them.
227///
228/// Additive wrapper around `RewarmedPoints` (which stays a plain enum for
229/// callers that only care about the points) — added so the bridge can gate
230/// its ladder on `seed.truncated_by` instead of inferring "cannot deepen"
231/// from an empty diff after the fact. A `RecentOnly`/`RestWindow` ceiling is
232/// visible on the FIRST deepen attempt via `seed.truncated_by`, letting the
233/// caller mark the key exhausted immediately instead of retrying.
234#[derive(Debug, Clone)]
235pub struct RewarmOutcome {
236    pub points: RewarmedPoints,
237    pub seed: crate::SeedOutcome,
238}
239
240impl RewarmOutcome {
241    /// Number of prepended points, regardless of concrete kind.
242    pub fn len(&self) -> usize { self.points.len() }
243    pub fn is_empty(&self) -> bool { self.points.is_empty() }
244    /// True when the underlying seed fetch reports a capability ceiling
245    /// (`RecentOnly` or `RestWindow`) rather than a transient/empty result —
246    /// the caller should stop retrying this key at deeper depths.
247    pub fn is_capability_exhausted(&self) -> bool {
248        matches!(
249            self.seed.truncated_by,
250            Some(crate::TruncationReason::VenueRecentOnly | crate::TruncationReason::VenueWindowCap)
251        )
252    }
253}
254
255impl Station {
256    pub fn builder() -> StationBuilder { StationBuilder::new() }
257    pub fn storage_root(&self) -> &std::path::Path { &self.inner.storage_root }
258    pub fn active_streams(&self) -> usize { self.inner.muxes.len() }
259
260    /// Shared `ExchangeHub` backing this Station's connectors.
261    ///
262    /// Exposed so a consumer that also needs raw REST history (e.g. a chart
263    /// doing scroll-left pagination) can route `backfill::fetch_history` /
264    /// `backfill::klines_recent` through the SAME connector pool the Station's
265    /// live subscriptions use — instead of dialing a second, parallel hub.
266    /// One pool means one dial-wave, one rate-limit budget, one warm-up.
267    ///
268    /// `ExchangeHub::clone` is O(1) (Arc-pooled internally).
269    pub fn hub(&self) -> Arc<ExchangeHub> {
270        self.inner.hub.clone()
271    }
272
273    /// Return a sync handle to the in-memory ring for `key`.
274    ///
275    /// Returns `Some(Arc<RwLock<Series<T>>>)` when a forwarder for `key` is
276    /// currently live (active subscription or within the 30 s grace window) and
277    /// the concrete element type matches `T`.  Returns `None` when no active
278    /// forwarder exists for this key or when the stored type differs from `T`
279    /// (type mismatch is silently treated as absent rather than panicking).
280    ///
281    /// Render-time consumers (chart panels, dashboards) use this to peek at the
282    /// running ring without awaiting an `Event`.  The handle is independent of
283    /// `SubscriptionHandle::recv()` — events still flow through there for
284    /// state-mutation paths; this getter is read-only.
285    pub fn series<T: DataPoint + 'static>(&self, key: &SeriesKey)
286        -> Option<Arc<RwLock<Series<T>>>>
287    {
288        let erased = self.inner.series_handles.get(key)?;
289        // Downcast Arc<dyn Any + Send + Sync> → Arc<RwLock<Series<T>>>.
290        // `Arc::downcast` is not available for trait objects; use `Any::downcast_ref`
291        // on the inner value to verify the type, then clone the concrete Arc.
292        erased
293            .downcast_ref::<Arc<RwLock<Series<T>>>>()
294            .map(Arc::clone)
295    }
296
297    /// Register a consumer with the given quota. Drop the returned
298    /// [`ConsumerHandle`] to release all of the consumer's active
299    /// subscriptions atomically.
300    ///
301    /// This is opt-in: [`Station::subscribe`] continues to work without
302    /// quotas. A consumer that wants caps registers and uses
303    /// [`ConsumerHandle::subscribe`]; one that does not keeps calling
304    /// [`Station::subscribe`] directly.
305    pub fn register_consumer(
306        &self,
307        quota: crate::quota::ConsumerQuota,
308    ) -> crate::quota::ConsumerHandle {
309        let rest_bucket = quota.max_rest_per_window.map(|cap| {
310            crate::quota::TokenBucket::new(cap, quota.rest_window)
311        });
312        crate::quota::ConsumerHandle {
313            station: Arc::clone(&self.inner),
314            quota,
315            rest_bucket: Arc::new(tokio::sync::Mutex::new(rest_bucket)),
316            refs: tokio::sync::Mutex::new((0, Vec::new())),
317        }
318    }
319
320    pub(crate) async fn from_builder(b: StationBuilder) -> Result<Self> {
321        let _ = digdigdig3::core::install_default_crypto_provider();
322        // Native: pre-create the storage root. wasm32: OPFS directories are created
323        // lazily by the OPFS DiskStore on first append — std::fs is Unsupported here
324        // (it would fail Station::build for any persistence-enabled Station on wasm).
325        #[cfg(not(target_arch = "wasm32"))]
326        if b.persistence.enabled {
327            std::fs::create_dir_all(&b.storage_root).map_err(StationError::Io)?;
328        }
329        let (connector_tx, _) = broadcast::channel(256);
330        Ok(Self {
331            inner: Arc::new(StationInner {
332                hub: Arc::new(ExchangeHub::new()),
333                storage_root: b.storage_root,
334                persistence: b.persistence,
335                muxes: DashMap::new(),
336                series_handles: DashMap::new(),
337                warm_start_capacity: b.warm_start.max(1),
338                gap_heal: b.gap_heal,
339                unsubscribe_grace: b.unsubscribe_grace,
340                orderbook_rest_seed: b.orderbook_rest_seed,
341                orderbook_seed_depth: b.orderbook_seed_depth,
342                connector_tx,
343                exchange_info_cache: DashMap::new(),
344                flush_handles: DashMap::new(),
345                exit_acks: DashMap::new(),
346                seed_outcomes: DashMap::new(),
347            }),
348        })
349    }
350
351    /// A broadcast receiver for connector lifecycle events
352    /// (`ConnectorReady` / `SymbolsLoaded`). Returns events emitted from any
353    /// source — `warmup()`, on-demand subscribe-time connector init, REST
354    /// exchange-info refresh.
355    ///
356    /// Independent of `SubscriptionHandle` event streams. Capacity 256.
357    /// Lag drops oldest.
358    pub fn connector_events(&self) -> broadcast::Receiver<crate::subscription::Event> {
359        self.inner.connector_tx.subscribe()
360    }
361
362    /// Snapshot of cached `SymbolInfo` for `exchange` across all account types.
363    /// Empty if `warmup` hasn't yet been called or REST hasn't completed.
364    pub fn symbols(&self, exchange: ExchangeId) -> Vec<SymbolInfo> {
365        let mut out = Vec::new();
366        for entry in self.inner.exchange_info_cache.iter() {
367            if entry.key().0 == exchange {
368                out.extend_from_slice(entry.value());
369            }
370        }
371        out
372    }
373
374    /// Snapshot the live health metrics for the WS forwarder backing `key`.
375    ///
376    /// Returns `None` if no forwarder exists for this key (no active or
377    /// grace-window subscription).
378    ///
379    /// Sync, non-blocking — suitable for periodic diagnostics polls or
380    /// per-frame UI overlays (latency badges). Each field is a best-effort
381    /// snapshot:
382    ///
383    /// - `connected`: always accurate — derived from mux presence.
384    /// - `rtt_ms`: `None` until per-connector RTT handle wiring is added
385    ///   (incremental; OKX is the first candidate).
386    /// - `last_message_ms`: `None` until per-forwarder atomic timestamp
387    ///   wiring is added (incremental).
388    pub fn ws_health(&self, key: &SeriesKey) -> Option<WsHealth> {
389        // Presence in muxes == a live forwarder (active or grace-window).
390        self.inner.muxes.get(key)?;
391        Some(WsHealth {
392            connected: true,
393            rtt_ms: None,
394            last_message_ms: None,
395        })
396    }
397
398    /// Aggregate health across all forwarders for `exchange`.
399    ///
400    /// - `rtt_ms`: median of non-`None` RTT values across forwarders.
401    /// - `last_message_ms`: max of non-`None` last-message timestamps
402    ///   (most-recent message seen on any forwarder for this exchange).
403    /// - `connected`: `true` if at least one forwarder is connected.
404    ///
405    /// Returns `None` if there are no active forwarders for `exchange`.
406    pub fn ws_health_for_exchange(
407        &self,
408        exchange: ExchangeId,
409    ) -> Option<WsHealth> {
410        let mut any_connected = false;
411        let mut rtts: Vec<u64> = Vec::new();
412        let mut last_msgs: Vec<i64> = Vec::new();
413
414        for entry in self.inner.muxes.iter() {
415            if entry.key().exchange != exchange {
416                continue;
417            }
418            // Each mux entry == one live forwarder.
419            let h = WsHealth {
420                connected: true,
421                rtt_ms: None,
422                last_message_ms: None,
423            };
424            any_connected = true;
425            if let Some(rtt) = h.rtt_ms {
426                rtts.push(rtt);
427            }
428            if let Some(ts) = h.last_message_ms {
429                last_msgs.push(ts);
430            }
431        }
432
433        if !any_connected {
434            return None;
435        }
436
437        let rtt_ms = if rtts.is_empty() {
438            None
439        } else {
440            rtts.sort_unstable();
441            Some(rtts[rtts.len() / 2])
442        };
443
444        Some(WsHealth {
445            connected: true,
446            rtt_ms,
447            last_message_ms: last_msgs.into_iter().max(),
448        })
449    }
450
451    /// Drain + flush every open `DiskStore` across all live forwarders
452    /// (primitive, derived, and poller) and wait for each to acknowledge.
453    ///
454    /// This exists chiefly for wasm OPFS: the browser can tear down the tab
455    /// on `beforeunload` / `visibilitychange` before an async `Drop` flush
456    /// would ever run, so the mlc-shell calls this from the unload handler to
457    /// force every buffered store to disk first. Native `BufWriter` already
458    /// flushes synchronously on `Drop`; this method is still safe and useful
459    /// there (e.g. before a controlled shutdown) and keeps the API identical
460    /// across targets.
461    ///
462    /// A forwarder that has already exited (its flush-request receiver
463    /// dropped) is treated as "already flushed" — not an error. Only a
464    /// `DiskStore::flush().await` call that actually ran and returned `Err`
465    /// is collected in the result.
466    pub async fn flush_persistence(&self) -> std::result::Result<(), Vec<(SeriesKey, std::io::Error)>> {
467        let handles: Vec<(SeriesKey, FlushHandle)> = self
468            .inner
469            .flush_handles
470            .iter()
471            .map(|entry| (entry.key().clone(), entry.value().clone()))
472            .collect();
473
474        let mut errors = Vec::new();
475
476        for (key, handle) in handles {
477            let (ack_tx, ack_rx) = oneshot::channel();
478            match handle.0.send(ack_tx).await {
479                Ok(()) => match ack_rx.await {
480                    Ok(Ok(())) => {}
481                    Ok(Err(io_err)) => errors.push((key, io_err)),
482                    // Forwarder dropped the ack sender without replying
483                    // (exited mid-flush) — treat as already gone, not an error.
484                    Err(_) => {}
485                },
486                // Forwarder's receiver already dropped (exited before this
487                // request was sent) — treat as already flushed.
488                Err(_) => {}
489            }
490        }
491
492        // Best-effort sweep of entries whose forwarder has since exited but
493        // never got a chance to unregister (e.g. it exited concurrently with
494        // this call, after the snapshot above but before the send). A stale
495        // entry is harmless (the next flush_persistence call will just see
496        // the same SendError and skip it again), so this is opportunistic
497        // cleanup rather than a correctness requirement.
498        self.inner
499            .flush_handles
500            .retain(|_, handle| !handle.0.is_closed());
501
502        if errors.is_empty() {
503            Ok(())
504        } else {
505            Err(errors)
506        }
507    }
508
509    /// Force a live (or grace-window) forwarder for `key` to tear down
510    /// immediately, and wait until it has actually exited.
511    ///
512    /// Unlike letting all consumers drop (which relies on
513    /// `unsubscribe_grace` — 30s in mlc's config — before the forwarder
514    /// actually shuts down), this cancels any pending grace timer, fires the
515    /// forwarder's shutdown signal unconditionally regardless of remaining
516    /// consumer count, and awaits a forwarder-exit acknowledgement before
517    /// returning. This makes cold-respawn-with-a-larger-`warm_override`
518    /// deterministic instead of racing a 30s timer.
519    ///
520    /// Idempotent: if no mux exists for `key` (already torn down, or never
521    /// subscribed), returns `Ok(())` immediately.
522    ///
523    /// Race-safety: if the forwarder exits on its own (e.g. upstream closed)
524    /// between the shutdown signal and the ack being registered, the ack
525    /// entry being gone from `exit_acks` is treated as "already exited" —
526    /// this method never hangs waiting for an ack that will never come.
527    pub async fn force_unsubscribe_and_await(&self, key: &SeriesKey) -> Result<()> {
528        // Cancel any pending grace-period timer, mirroring the reuse path in
529        // `acquire_or_spawn` (station.rs ~:857) — a timer left running would
530        // otherwise race the shutdown we are about to fire, though the
531        // idempotent guards on both sides make this a non-issue either way.
532        // Fire shutdown unconditionally regardless of `consumers` count —
533        // this is the key difference from `release_consumer`, which only
534        // fires shutdown once the count reaches zero.
535        let removed = {
536            let Some(mut mux) = self.inner.muxes.get_mut(key) else {
537                // No live mux — nothing to tear down.
538                return Ok(());
539            };
540            if let Some(cancel) = mux.grace_cancel.take() {
541                let _ = cancel.send(());
542            }
543            mux.shutdown.take()
544        };
545
546        self.inner.muxes.remove(key);
547        self.inner.series_handles.remove(key);
548
549        if let Some(shutdown_tx) = removed {
550            let _ = shutdown_tx.send(());
551        }
552
553        // Await the forwarder's exit ack. If the registry entry is already
554        // gone (forwarder exited concurrently, e.g. upstream closed right
555        // before we got here) there is nothing to await — treat as done.
556        // A `RecvError` (sender dropped without sending — forwarder task
557        // panicked or was aborted) is likewise treated as "exited" rather
558        // than a hard error, since the goal is "forwarder is gone", not
559        // "forwarder exited cleanly".
560        if let Some((_, ack_rx)) = self.inner.exit_acks.remove(key) {
561            let _ = ack_rx.await;
562        }
563
564        Ok(())
565    }
566
567    /// Deepen the warm window of a LIVE footprint subscription in place —
568    /// no teardown, no live gap. Fetches a `warm_n`-deep aggTrade window
569    /// from REST (same pager as the cold-start seed), replays it through a
570    /// FRESH `TradeToFootprintDerived` state machine, and splices the
571    /// resulting bars in FRONT of the existing ring: only bars at/older
572    /// than the current ring head bucket are taken (the partial boundary
573    /// bucket is replaced by its full rebuild); everything newer stays
574    /// owned by the live forwarder, so there is no seam race with
575    /// in-progress buckets.
576    ///
577    /// Footprint-only by design: its buckets are pure time buckets, so
578    /// bars rebuilt from a longer trade window agree with already-ringed
579    /// bars at every bucket boundary. Path-dependent derived kinds (renko/
580    /// pnf/kagi/3lb/range/volume/tick/...) MUST keep the teardown+respawn
581    /// deepen — their bar boundaries depend on where the seed starts.
582    ///
583    /// Returns the spliced-in points (oldest→newest, all `open_time <=`
584    /// the pre-call ring head; empty when the key has no live ring, the
585    /// venue yields no history, or nothing at/older than the ring head was
586    /// produced). No `Event` is broadcast here — the caller decides how to
587    /// deliver the batch downstream (a per-point broadcast of thousands of
588    /// historical bars is exactly the consumer-side stall this API
589    /// avoids).
590    pub async fn rewarm_footprint(
591        &self,
592        key: &SeriesKey,
593        raw_symbol: &str,
594        warm_n: usize,
595    ) -> Vec<FootprintPoint> {
596        // No live ring → nothing to splice into; caller should fall back
597        // to a cold subscribe at the deeper depth.
598        let Some(handle) = self.inner.series_handles.get(key).and_then(|e| {
599            e.downcast_ref::<Arc<RwLock<Series<FootprintPoint>>>>().map(Arc::clone)
600        }) else {
601            return Vec::new();
602        };
603        if warm_n == 0 {
604            return Vec::new();
605        }
606
607        // REST fetch + derive happen OUTSIDE the ring lock — this is the
608        // long part (100+ sequential pages on a 100k window in a browser);
609        // the live forwarder keeps appending to the ring meanwhile.
610        let caps_opt = self.inner.hub.capabilities(key.exchange);
611        let use_agg = caps_opt.as_ref().map(|c| c.has_agg_trades).unwrap_or(false);
612        let mut seed_events: Vec<Event> = Vec::new();
613        if use_agg {
614            let n_pages = ((warm_n + 999) / 1000).max(1);
615            let (agg, _outcome) = crate::backfill::agg_trades_paginated(
616                &self.inner.hub, key.exchange, key.account_type, raw_symbol, 1000, n_pages,
617            )
618            .await;
619            for ap in agg {
620                seed_events.push(Event::Trade {
621                    exchange: key.exchange,
622                    symbol: raw_symbol.to_string(),
623                    point: TradePoint {
624                        ts_ms: ap.ts_ms,
625                        price: ap.price,
626                        quantity: ap.quantity,
627                        side: ap.side,
628                        trade_id_hash: 0,
629                    },
630                });
631            }
632        } else {
633            let (trades, _outcome) = crate::backfill::trades_recent(
634                &self.inner.hub, key.exchange, key.account_type, raw_symbol, warm_n,
635            )
636            .await;
637            for pt in trades {
638                seed_events.push(Event::Trade {
639                    exchange: key.exchange,
640                    symbol: raw_symbol.to_string(),
641                    point: pt,
642                });
643            }
644        }
645        if seed_events.is_empty() {
646            return Vec::new();
647        }
648        // Chunked derive with a MACROTASK yield between chunks: on wasm the
649        // whole rewarm runs on the browser main thread, and one synchronous
650        // pass over a 100k-trade window stalls RAF/input for seconds
651        // (microtask yields — tokio yield_now — do NOT let the browser
652        // paint; only a timer macrotask does).
653        let mut state = TradeToFootprintDerived::new_for_key(key);
654        let mut emissions: Vec<FootprintPoint> = Vec::new();
655        for chunk in seed_events.chunks(5_000) {
656            emissions.extend(state.seed_from_events(chunk, 0));
657            #[cfg(target_arch = "wasm32")]
658            gloo_timers::future::sleep(std::time::Duration::from_millis(0)).await;
659            #[cfg(not(target_arch = "wasm32"))]
660            tokio::task::yield_now().await;
661        }
662        // The state machine emits the in-progress bar on EVERY trade — the
663        // cold-seed path collapses those through the ring's `upsert_by_ts`,
664        // so a splice must dedup the same way: keep the LAST emission per
665        // bucket open_time (chronological input → later supersedes).
666        let mut rebuilt: Vec<FootprintPoint> = Vec::new();
667        for p in emissions {
668            match rebuilt.last_mut() {
669                Some(last) if last.timestamp_ms() == p.timestamp_ms() => *last = p,
670                _ => rebuilt.push(p),
671            }
672        }
673        if rebuilt.is_empty() {
674            return Vec::new();
675        }
676
677        // Splice under ONE write lock: snapshot → cut at the ring head
678        // bucket → rebuild a larger ring. The in-lock work is a few
679        // thousand clones (microseconds); a live bar landing between the
680        // fetch above and this lock only grows the tail, which is kept
681        // verbatim.
682        let mut guard = handle.write().await;
683        let existing = guard.snapshot();
684        let head_ts = existing.first().map(|p| p.timestamp_ms());
685        let older: Vec<FootprintPoint> = rebuilt
686            .into_iter()
687            .filter(|p| head_ts.is_none_or(|h| p.timestamp_ms() <= h))
688            .collect();
689        if older.is_empty() {
690            return Vec::new();
691        }
692        let boundary_replaced = head_ts
693            .is_some_and(|h| older.last().is_some_and(|p| p.timestamp_ms() == h));
694        let keep_from = usize::from(boundary_replaced);
695        let merged_len = older.len() + existing.len().saturating_sub(keep_from);
696        let mut fresh = Series::new(merged_len.max(guard.capacity()));
697        fresh.extend(older.iter().cloned());
698        fresh.extend(existing.into_iter().skip(keep_from));
699        *guard = fresh;
700        older
701    }
702
703    /// Generalized version of [`Station::rewarm_footprint`] for path-
704    /// dependent derived kinds — renko / pnf / kagi / three-line-break /
705    /// range / volume / tick / dollar bar / cvd / tpo.
706    ///
707    /// Same call shape and same mechanics as `rewarm_footprint`: fetch a
708    /// deeper raw trade window, run a THROWAWAY fold over it with a FRESH
709    /// state machine (optionally seeded from the existing series' own
710    /// output so path-dependent grids/state line up with the live ring),
711    /// then splice the result in FRONT of the existing ring under one
712    /// write lock. Existing points are moved verbatim — never re-derived.
713    ///
714    /// Returns `Ok(RewarmOutcome)` — prepended points plus the seed fetch's
715    /// [`crate::SeedOutcome`] — on the kinds this mechanism supports.
716    /// Returns `Err(StationError::RewarmUnsupported)` for
717    /// `TickImbalanceBar` / `VolumeImbalanceBar` / `RunBar` — their fold
718    /// state is an EMA carried across the ENTIRE history, which cannot be
719    /// reconstructed from already-emitted output; callers must fall back to
720    /// the teardown + resubscribe-at-depth path for those three. Also
721    /// `Err` for `Footprint` (use `rewarm_footprint` instead) and any
722    /// non-derived kind.
723    pub async fn rewarm_derived(
724        &self,
725        key: &SeriesKey,
726        raw_symbol: &str,
727        warm_n: usize,
728    ) -> Result<RewarmOutcome> {
729        match &key.kind {
730            Kind::RenkoBar(_, _) => {
731                let (v, seed) = self.rewarm_renko(key, raw_symbol, warm_n).await;
732                Ok(RewarmOutcome { points: RewarmedPoints::Renko(v), seed })
733            }
734            Kind::PnfBar(_, _) => {
735                let (v, seed) = self.rewarm_pnf(key, raw_symbol, warm_n).await;
736                Ok(RewarmOutcome { points: RewarmedPoints::Pnf(v), seed })
737            }
738            Kind::KagiBar(_) => {
739                let (v, seed) = self.rewarm_derived_standalone::<TradeToKagiBarDerived>(key, raw_symbol, warm_n).await;
740                Ok(RewarmOutcome { points: RewarmedPoints::Kagi(v), seed })
741            }
742            Kind::ThreeLineBreak { .. } => {
743                let (v, seed) = self.rewarm_derived_standalone::<TradeToThreeLineBreakDerived>(key, raw_symbol, warm_n).await;
744                Ok(RewarmOutcome { points: RewarmedPoints::ThreeLineBreak(v), seed })
745            }
746            Kind::RangeBar(_) => {
747                let (v, seed) = self.rewarm_derived_standalone::<TradeToRangeBarDerived>(key, raw_symbol, warm_n).await;
748                Ok(RewarmOutcome { points: RewarmedPoints::Bar(v), seed })
749            }
750            Kind::VolumeBar(_) => {
751                let (v, seed) = self.rewarm_derived_standalone::<TradeToVolumeBarDerived>(key, raw_symbol, warm_n).await;
752                Ok(RewarmOutcome { points: RewarmedPoints::Bar(v), seed })
753            }
754            Kind::TickBar(_) => {
755                let (v, seed) = self.rewarm_derived_standalone::<TradeToTickBarDerived>(key, raw_symbol, warm_n).await;
756                Ok(RewarmOutcome { points: RewarmedPoints::Bar(v), seed })
757            }
758            Kind::DollarBar { .. } => {
759                let (v, seed) = self.rewarm_derived_standalone::<TradeToDollarBarDerived>(key, raw_symbol, warm_n).await;
760                Ok(RewarmOutcome { points: RewarmedPoints::Bar(v), seed })
761            }
762            Kind::CvdLine => {
763                let (v, seed) = self.rewarm_cvd(key, raw_symbol, warm_n).await;
764                Ok(RewarmOutcome { points: RewarmedPoints::Cvd(v), seed })
765            }
766            Kind::TpoProfile(_, _) => {
767                let (v, seed) = self.rewarm_tpo(key, raw_symbol, warm_n).await;
768                Ok(RewarmOutcome { points: RewarmedPoints::Tpo(v), seed })
769            }
770            Kind::TickImbalanceBar { .. } => Err(StationError::RewarmUnsupported(
771                "TickImbalanceBar carries EMA state (E[T]/E[|theta|]) over the entire \
772                 history — not reconstructible from emitted output; use teardown+resubscribe."
773                    .to_string(),
774            )),
775            Kind::VolumeImbalanceBar { .. } => Err(StationError::RewarmUnsupported(
776                "VolumeImbalanceBar carries EMA state over the entire history — not \
777                 reconstructible from emitted output; use teardown+resubscribe."
778                    .to_string(),
779            )),
780            Kind::RunBar { .. } => Err(StationError::RewarmUnsupported(
781                "RunBar carries EMA state over the entire history — not reconstructible \
782                 from emitted output; use teardown+resubscribe."
783                    .to_string(),
784            )),
785            Kind::Footprint(_) => Err(StationError::RewarmUnsupported(
786                "Footprint has its own dedicated rewarm_footprint — call that instead."
787                    .to_string(),
788            )),
789            other => Err(StationError::RewarmUnsupported(format!(
790                "{other:?} is not a prepend-fold-capable derived kind"
791            ))),
792        }
793    }
794
795    /// Fetch the deeper raw trade window shared by every `rewarm_*` helper:
796    /// paginated aggTrades (falling back to `trades_recent` when the venue
797    /// lacks aggTrade history), same pager `rewarm_footprint` uses.
798    ///
799    /// Returns the seed events alongside the [`crate::SeedOutcome`] so
800    /// callers can tell a live venue-capability ceiling
801    /// (`TruncationReason::VenueRecentOnly` / `VenueWindowCap`) apart from a
802    /// genuinely empty window — a `RecentOnly` venue reports the ceiling on
803    /// the FIRST deepen attempt instead of after repeated hammering.
804    async fn rewarm_fetch_trade_window(
805        &self,
806        key: &SeriesKey,
807        raw_symbol: &str,
808        warm_n: usize,
809    ) -> (Vec<Event>, crate::SeedOutcome) {
810        if warm_n == 0 {
811            return (Vec::new(), crate::SeedOutcome::full(0, 0, crate::SeedSource::AggTradesPaginated));
812        }
813        let caps_opt = self.inner.hub.capabilities(key.exchange);
814        let use_agg = caps_opt.as_ref().map(|c| c.has_agg_trades).unwrap_or(false);
815        let mut seed_events: Vec<Event> = Vec::new();
816        let outcome;
817        if use_agg {
818            let n_pages = ((warm_n + 999) / 1000).max(1);
819            let (agg, agg_outcome) = crate::backfill::agg_trades_paginated(
820                &self.inner.hub, key.exchange, key.account_type, raw_symbol, 1000, n_pages,
821            )
822            .await;
823            for ap in agg {
824                seed_events.push(Event::Trade {
825                    exchange: key.exchange,
826                    symbol: raw_symbol.to_string(),
827                    point: TradePoint {
828                        ts_ms: ap.ts_ms,
829                        price: ap.price,
830                        quantity: ap.quantity,
831                        side: ap.side,
832                        trade_id_hash: 0,
833                    },
834                });
835            }
836            outcome = agg_outcome;
837        } else {
838            let (trades, trades_outcome) = crate::backfill::trades_recent(
839                &self.inner.hub, key.exchange, key.account_type, raw_symbol, warm_n,
840            )
841            .await;
842            for pt in trades {
843                seed_events.push(Event::Trade {
844                    exchange: key.exchange,
845                    symbol: raw_symbol.to_string(),
846                    point: pt,
847                });
848            }
849            outcome = trades_outcome;
850        }
851        (seed_events, outcome)
852    }
853
854    /// Kline-approx window for the kline-sufficient kinds
855    /// (renko/pnf/kagi/tlb/range/volume/dollar) — matches the cold-seed
856    /// composition in `acquire_or_spawn_derived_body`, so a deepened rewarm
857    /// fold sees the same data quality as a cold seed would at that depth.
858    ///
859    /// These kinds are PURE KLINE CONVERTERS (no `Stream::Trade` dependency
860    /// at all — see `DerivedStream::deps_for_key` on each of the seven), so
861    /// this is their ENTIRE rewarm fetch; there is no trade-history tail to
862    /// bridge with. `klines_paginated` returns oldest→newest and its window
863    /// reaches "now", so the LAST bar may be the exchange's still-forming
864    /// candle — excluded from the 4-leg fold for the same reason the
865    /// cold-seed excludes it (see `acquire_or_spawn_derived_body`'s
866    /// kline-sufficient seed branch): folding it in full would double-count
867    /// against whatever the live series already derived from the WS Δ-tick
868    /// path for that same bar. Dropping it here (rather than handing a
869    /// baseline forward, which a THROWAWAY rewarm fold has no live
870    /// continuation to hand it to) is the audited ≤1-bar seam already
871    /// accepted elsewhere in this design (see `rewarm_derived_standalone`'s
872    /// doc comment).
873    async fn rewarm_fetch_kline_sufficient_window(
874        &self,
875        key: &SeriesKey,
876        raw_symbol: &str,
877        warm_n: usize,
878    ) -> (Vec<Event>, crate::SeedOutcome) {
879        if warm_n == 0 {
880            return (Vec::new(), crate::SeedOutcome::full(0, 0, crate::SeedSource::Klines));
881        }
882        let n_kline_pages = ((warm_n + 999) / 1000).clamp(1, 100);
883        let (kline_bars, outcome) = crate::backfill::klines_paginated(
884            &self.inner.hub, key.exchange, key.account_type, raw_symbol, "1m", 1000, n_kline_pages,
885        )
886        .await;
887        let kline_interval_ms = interval_to_ms("1m").unwrap_or(60_000);
888        let mut events = Vec::new();
889        if let Some((_last, closed)) = kline_bars.split_last() {
890            for bar in closed {
891                events.extend(crate::derived::kline_to_synthetic_trades(
892                    bar, kline_interval_ms, key.exchange, raw_symbol,
893                ));
894            }
895        }
896        (events, outcome)
897    }
898
899    /// Chunked throwaway fold over `events` through a fresh `D` state
900    /// machine, yielding a wasm-safe macrotask between chunks — identical
901    /// cadence to `rewarm_footprint`. `seed` runs once on the freshly
902    /// constructed state, before any events are fed, so callers can inject
903    /// per-kind grid/state presets (renko/pnf).
904    async fn rewarm_fold<D: DerivedStream>(
905        key: &SeriesKey,
906        events: &[Event],
907        seed: impl FnOnce(&mut D),
908    ) -> Vec<D::Output> {
909        let mut state = D::new_for_key(key);
910        seed(&mut state);
911        let mut emissions: Vec<D::Output> = Vec::new();
912        for chunk in events.chunks(5_000) {
913            emissions.extend(state.seed_from_events(chunk, 0));
914            #[cfg(target_arch = "wasm32")]
915            gloo_timers::future::sleep(std::time::Duration::from_millis(0)).await;
916            #[cfg(not(target_arch = "wasm32"))]
917            tokio::task::yield_now().await;
918        }
919        emissions
920    }
921
922    /// Splice `rebuilt` (already deduped-per-identity, chronological) in
923    /// front of the existing ring for `key`, under one write lock —
924    /// same overall shape as the tail of `rewarm_footprint`, but with the
925    /// OPPOSITE boundary policy: **existing wins**. Footprint's buckets are
926    /// pure time-buckets so its fresher rebuild of the boundary bucket is
927    /// strictly more complete and replaces the live one; every derived kind
928    /// here is path-dependent (renko/pnf grid, kagi shoulder/waist, TLB
929    /// ring, running counters, CVD cumulative value, TPO session) — its
930    /// live bar reflects state the throwaway fold cannot see (everything
931    /// that happened after the older-window fetch), so the live copy must
932    /// never be overwritten. `ident` extracts the per-kind bar identity
933    /// used for the boundary-collision check (NOT necessarily
934    /// `timestamp_ms()` — e.g. pnf uses `column_id`); ordering itself still
935    /// uses `timestamp_ms()`.
936    async fn rewarm_splice<T: DataPoint + Clone, Id: PartialEq>(
937        &self,
938        key: &SeriesKey,
939        rebuilt: Vec<T>,
940        ident: impl Fn(&T) -> Id,
941    ) -> Vec<T> {
942        let Some(handle) = self.inner.series_handles.get(key).and_then(|e| {
943            e.downcast_ref::<Arc<RwLock<Series<T>>>>().map(Arc::clone)
944        }) else {
945            return Vec::new();
946        };
947        if rebuilt.is_empty() {
948            return Vec::new();
949        }
950        let mut guard = handle.write().await;
951        let existing = guard.snapshot();
952        let head_ts = existing.first().map(|p| p.timestamp_ms());
953        let mut older: Vec<T> = rebuilt
954            .into_iter()
955            .filter(|p| head_ts.is_none_or(|h| p.timestamp_ms() <= h))
956            .collect();
957        if older.is_empty() {
958            return Vec::new();
959        }
960        // Boundary-collision guard: if the fold's last (newest) older point
961        // shares identity with the existing head, the fold's copy is a
962        // partial/stale rebuild of a bar the live forwarder already owns —
963        // drop it. Existing never gets overwritten.
964        let head_ident = existing.first().map(&ident);
965        if head_ident.as_ref().is_some_and(|h| older.last().is_some_and(|p| &ident(p) == h)) {
966            older.pop();
967        }
968        if older.is_empty() {
969            return Vec::new();
970        }
971        let merged_len = older.len() + existing.len();
972        let mut fresh = Series::new(merged_len.max(guard.capacity()));
973        fresh.extend(older.iter().cloned());
974        fresh.extend(existing);
975        *guard = fresh;
976        older
977    }
978
979    /// Dedup a chronological emission stream down to "last emission per
980    /// identity supersedes" — same rule `rewarm_footprint` applies per
981    /// `open_time` bucket, generalized to an arbitrary `ident` extractor
982    /// (most kinds re-emit the in-progress bar on every trade; only the
983    /// final emission per identity reflects the bar's closed/complete
984    /// state).
985    fn rewarm_dedup<T: Clone, Id: PartialEq>(emissions: Vec<T>, ident: impl Fn(&T) -> Id) -> Vec<T> {
986        let mut out: Vec<T> = Vec::new();
987        for p in emissions {
988            match out.last() {
989                Some(last) if ident(last) == ident(&p) => {
990                    let idx = out.len() - 1;
991                    out[idx] = p;
992                }
993                _ => out.push(p),
994            }
995        }
996        out
997    }
998
999    /// Renko rewarm: preset the throwaway fold's grid anchor from the
1000    /// existing series' first brick's lower boundary (`bottom`) so the
1001    /// prepended bricks land on the SAME grid as the live series — no
1002    /// floor-snap off wherever the older window happens to start.
1003    ///
1004    /// Returns the prepended points alongside the [`crate::SeedOutcome`] of
1005    /// the underlying kline-sufficient window fetch (kline side is
1006    /// depth-defining — see [`Self::rewarm_fetch_kline_sufficient_window`])
1007    /// — a `RecentOnly`/`RestWindow` ceiling on that fetch is the caller's
1008    /// signal that this key cannot be deepened further no matter how many
1009    /// times it retries.
1010    async fn rewarm_renko(
1011        &self,
1012        key: &SeriesKey,
1013        raw_symbol: &str,
1014        warm_n: usize,
1015    ) -> (Vec<RenkoBrickPoint>, crate::SeedOutcome) {
1016        let empty_outcome = crate::SeedOutcome::full(warm_n, 0, crate::SeedSource::AggTradesPaginated);
1017        let Some(handle) = self.inner.series_handles.get(key).and_then(|e| {
1018            e.downcast_ref::<Arc<RwLock<Series<RenkoBrickPoint>>>>().map(Arc::clone)
1019        }) else {
1020            return (Vec::new(), empty_outcome);
1021        };
1022        let grid_floor = {
1023            let guard = handle.read().await;
1024            match guard.snapshot().first() {
1025                Some(first) => first.bottom,
1026                None => return (Vec::new(), empty_outcome),
1027            }
1028        };
1029        let (events, outcome) = self.rewarm_fetch_kline_sufficient_window(key, raw_symbol, warm_n).await;
1030        if events.is_empty() {
1031            return (Vec::new(), outcome);
1032        }
1033        let emissions = Self::rewarm_fold::<TradeToRenkoBarDerived>(key, &events, |state| {
1034            state.preset_grid_anchor(grid_floor);
1035        })
1036        .await;
1037        let rebuilt = Self::rewarm_dedup(emissions, |p: &RenkoBrickPoint| p.open_time);
1038        if rebuilt.is_empty() {
1039            return (Vec::new(), outcome);
1040        }
1041        (self.rewarm_splice(key, rebuilt, |p: &RenkoBrickPoint| p.open_time).await, outcome)
1042    }
1043
1044    /// PnF rewarm: preset the throwaway fold's box grid from the existing
1045    /// series' first column's `(bottom, top, is_x)` span — same reasoning
1046    /// as renko.
1047    async fn rewarm_pnf(
1048        &self,
1049        key: &SeriesKey,
1050        raw_symbol: &str,
1051        warm_n: usize,
1052    ) -> (Vec<PnfColumnPoint>, crate::SeedOutcome) {
1053        let empty_outcome = crate::SeedOutcome::full(warm_n, 0, crate::SeedSource::AggTradesPaginated);
1054        let Some(handle) = self.inner.series_handles.get(key).and_then(|e| {
1055            e.downcast_ref::<Arc<RwLock<Series<PnfColumnPoint>>>>().map(Arc::clone)
1056        }) else {
1057            return (Vec::new(), empty_outcome);
1058        };
1059        let (bottom, top, is_x) = {
1060            let guard = handle.read().await;
1061            match guard.snapshot().first() {
1062                Some(first) => (first.bottom, first.top, first.is_x),
1063                None => return (Vec::new(), empty_outcome),
1064            }
1065        };
1066        let (events, outcome) = self.rewarm_fetch_kline_sufficient_window(key, raw_symbol, warm_n).await;
1067        if events.is_empty() {
1068            return (Vec::new(), outcome);
1069        }
1070        let emissions = Self::rewarm_fold::<TradeToPnfBarDerived>(key, &events, |state| {
1071            state.preset_grid(bottom, top, is_x);
1072        })
1073        .await;
1074        let rebuilt = Self::rewarm_dedup(emissions, |p: &PnfColumnPoint| p.column_id);
1075        if rebuilt.is_empty() {
1076            return (Vec::new(), outcome);
1077        }
1078        (self.rewarm_splice(key, rebuilt, |p: &PnfColumnPoint| p.column_id).await, outcome)
1079    }
1080
1081    /// Standalone prepend-fold for kinds whose throwaway fold needs NO
1082    /// state injected from the existing series' output — kagi (shoulder/
1083    /// waist reconstruction deferred; v1 accepts the audited ≤1-segment
1084    /// seam), three-line-break (ring rebuilds itself within
1085    /// `lines_back` lines), and range/tick/volume/dollar bar (counters
1086    /// reset cleanly at the fold's own first trade; v1 accepts the
1087    /// audited ≤1-bar seam). `D::Output` identity for dedup + splice is
1088    /// `timestamp_ms()` for every one of these (renko/pnf are the only
1089    /// kinds with a non-timestamp identity).
1090    ///
1091    /// Kagi/three-line-break/range/volume/dollar are kline-sufficient (see
1092    /// `is_kline_sufficient_kind` in `acquire_or_spawn_derived_body`) and get
1093    /// the deep kline-approx window instead of deep trade pagination — tick
1094    /// bar genuinely needs per-trade counts and keeps the plain trade-window
1095    /// fetch.
1096    async fn rewarm_derived_standalone<D>(
1097        &self,
1098        key: &SeriesKey,
1099        raw_symbol: &str,
1100        warm_n: usize,
1101    ) -> (Vec<D::Output>, crate::SeedOutcome)
1102    where
1103        D: DerivedStream,
1104        D::Output: Clone,
1105    {
1106        let empty_outcome = crate::SeedOutcome::full(warm_n, 0, crate::SeedSource::AggTradesPaginated);
1107        if self.inner.series_handles.get(key).is_none() {
1108            return (Vec::new(), empty_outcome);
1109        }
1110        let is_kline_sufficient_kind = matches!(
1111            key.kind,
1112            Kind::KagiBar(_) | Kind::ThreeLineBreak { .. }
1113            | Kind::RangeBar(_) | Kind::VolumeBar(_) | Kind::DollarBar { .. }
1114        );
1115        let (events, outcome) = if is_kline_sufficient_kind {
1116            self.rewarm_fetch_kline_sufficient_window(key, raw_symbol, warm_n).await
1117        } else {
1118            self.rewarm_fetch_trade_window(key, raw_symbol, warm_n).await
1119        };
1120        if events.is_empty() {
1121            return (Vec::new(), outcome);
1122        }
1123        let emissions = Self::rewarm_fold::<D>(key, &events, |_state| {}).await;
1124        let rebuilt = Self::rewarm_dedup(emissions, |p: &D::Output| p.timestamp_ms());
1125        if rebuilt.is_empty() {
1126            return (Vec::new(), outcome);
1127        }
1128        (self.rewarm_splice(key, rebuilt, |p: &D::Output| p.timestamp_ms()).await, outcome)
1129    }
1130
1131    /// CVD rewarm: fold the older window standalone (starts its own
1132    /// cumulative sum at 0 from the fold's first trade), then OFFSET every
1133    /// prepended sample by a constant so the LAST prepended sample's value
1134    /// equals the existing FIRST sample's value. Existing samples are
1135    /// never touched — only the prepended segment is shifted to meet them
1136    /// exactly at the seam.
1137    async fn rewarm_cvd(
1138        &self,
1139        key: &SeriesKey,
1140        raw_symbol: &str,
1141        warm_n: usize,
1142    ) -> (Vec<ScalarBarPoint>, crate::SeedOutcome) {
1143        let empty_outcome = crate::SeedOutcome::full(warm_n, 0, crate::SeedSource::AggTradesPaginated);
1144        let Some(handle) = self.inner.series_handles.get(key).and_then(|e| {
1145            e.downcast_ref::<Arc<RwLock<Series<ScalarBarPoint>>>>().map(Arc::clone)
1146        }) else {
1147            return (Vec::new(), empty_outcome);
1148        };
1149        let existing_first_value = {
1150            let guard = handle.read().await;
1151            match guard.snapshot().first() {
1152                Some(first) => first.value,
1153                None => return (Vec::new(), empty_outcome),
1154            }
1155        };
1156        let (events, outcome) = self.rewarm_fetch_trade_window(key, raw_symbol, warm_n).await;
1157        if events.is_empty() {
1158            return (Vec::new(), outcome);
1159        }
1160        let emissions = Self::rewarm_fold::<TradeToCvdLineDerived>(key, &events, |_state| {}).await;
1161        let mut rebuilt = Self::rewarm_dedup(emissions, |p: &ScalarBarPoint| p.ts_ms);
1162        if rebuilt.is_empty() {
1163            return (Vec::new(), outcome);
1164        }
1165        // Offset so the fold's LAST (newest) sample ends exactly at the
1166        // existing series' first value — the seam is exact by
1167        // construction, not approximate.
1168        let offset = existing_first_value - rebuilt.last().map(|p| p.value).unwrap_or(0.0);
1169        for p in &mut rebuilt {
1170            p.value += offset;
1171        }
1172        (self.rewarm_splice(key, rebuilt, |p: &ScalarBarPoint| p.ts_ms).await, outcome)
1173    }
1174
1175    /// TPO rewarm: sessions are wall-clock UTC-day buckets (like footprint's
1176    /// time buckets), so grid alignment is inherent — no state injection.
1177    /// The one boundary hazard is the throwaway fold's own trailing
1178    /// in-progress session colliding with the existing head session; that
1179    /// case is exactly what `rewarm_splice`'s identity-based boundary guard
1180    /// (`open_time == session_date_ms`) already drops, so no extra
1181    /// bookkeeping is needed beyond routing through the same helper.
1182    async fn rewarm_tpo(
1183        &self,
1184        key: &SeriesKey,
1185        raw_symbol: &str,
1186        warm_n: usize,
1187    ) -> (Vec<TpoSessionPoint>, crate::SeedOutcome) {
1188        let empty_outcome = crate::SeedOutcome::full(warm_n, 0, crate::SeedSource::Klines);
1189        if self.inner.series_handles.get(key).is_none() {
1190            return (Vec::new(), empty_outcome);
1191        }
1192        let (rebuilt, outcome): (Vec<TpoSessionPoint>, crate::SeedOutcome) = match &key.kind {
1193            Kind::TpoProfile(_, TpoSource::Kline1m) => {
1194                let (kline_bars, kline_outcome) = crate::backfill::klines_paginated(
1195                    &self.inner.hub, key.exchange, key.account_type, raw_symbol, "1m", 1000,
1196                    ((warm_n + 999) / 1000).max(1),
1197                )
1198                .await;
1199                if kline_bars.is_empty() {
1200                    return (Vec::new(), kline_outcome);
1201                }
1202                let interval = digdigdig3::core::websocket::KlineInterval::new("1m");
1203                let events: Vec<Event> = kline_bars
1204                    .into_iter()
1205                    .map(|point| Event::Bar {
1206                        exchange: key.exchange,
1207                        symbol: raw_symbol.to_string(),
1208                        timeframe: interval.clone(),
1209                        point,
1210                    })
1211                    .collect();
1212                let emissions = Self::rewarm_fold::<TpoFromKline1mDerived>(key, &events, |_state| {}).await;
1213                (Self::rewarm_dedup(emissions, |p: &TpoSessionPoint| p.open_time), kline_outcome)
1214            }
1215            Kind::TpoProfile(_, TpoSource::TradeBucket) => {
1216                let (events, trade_outcome) = self.rewarm_fetch_trade_window(key, raw_symbol, warm_n).await;
1217                if events.is_empty() {
1218                    return (Vec::new(), trade_outcome);
1219                }
1220                let emissions = Self::rewarm_fold::<TpoFromTradeDerived>(key, &events, |_state| {}).await;
1221                (Self::rewarm_dedup(emissions, |p: &TpoSessionPoint| p.open_time), trade_outcome)
1222            }
1223            _ => return (Vec::new(), empty_outcome),
1224        };
1225        if rebuilt.is_empty() {
1226            return (Vec::new(), outcome);
1227        }
1228        (self.rewarm_splice(key, rebuilt, |p: &TpoSessionPoint| p.open_time).await, outcome)
1229    }
1230
1231    /// Eagerly connect to every exchange in `exchanges` and pre-load their
1232    /// full symbol list. Subscribes nothing — produces only
1233    /// `Event::ConnectorReady` (one per exchange that finishes
1234    /// `connect_public`) and `Event::SymbolsLoaded` (one per exchange whose
1235    /// REST `get_exchange_info` succeeds for at least one account type) on
1236    /// the broadcast channel returned by `Station::connector_events()`.
1237    ///
1238    /// Idempotent: running concurrently or repeatedly is safe.
1239    /// Already-connected exchanges short-circuit. Already-cached symbol lists
1240    /// re-broadcast from cache without REST.
1241    ///
1242    /// Runs to completion — returns a [`WarmupReport`] of outcomes.
1243    pub async fn warmup(&self, exchanges: &[ExchangeId]) -> crate::subscription::WarmupReport {
1244        use crate::subscription::{Event, WarmupReport};
1245
1246        let mut ok = Vec::new();
1247        let mut failed: Vec<(ExchangeId, String)> = Vec::new();
1248
1249        // Phase 1: connect all exchanges (spawn concurrent tasks).
1250        #[cfg(not(target_arch = "wasm32"))]
1251        {
1252            let mut join_set: tokio::task::JoinSet<(ExchangeId, Result<()>)> =
1253                tokio::task::JoinSet::new();
1254
1255            for &eid in exchanges {
1256                if self.inner.hub.is_connected(eid) {
1257                    let _ = self.inner.connector_tx.send(Event::ConnectorReady { exchange: eid });
1258                    ok.push(eid);
1259                } else {
1260                    let hub = Arc::clone(&self.inner.hub);
1261                    join_set.spawn(async move {
1262                        (eid, hub.connect_public(eid, false).await.map_err(|e| {
1263                            crate::StationError::Core(e.to_string())
1264                        }))
1265                    });
1266                }
1267            }
1268
1269            while let Some(res) = join_set.join_next().await {
1270                match res {
1271                    Ok((eid, Ok(()))) => {
1272                        let _ = self.inner.connector_tx.send(Event::ConnectorReady { exchange: eid });
1273                        ok.push(eid);
1274                    }
1275                    Ok((eid, Err(e))) => {
1276                        tracing::warn!(?eid, ?e, "warmup: connect_public failed");
1277                        failed.push((eid, e.to_string()));
1278                    }
1279                    Err(join_err) => {
1280                        tracing::warn!(?join_err, "warmup: task panicked");
1281                    }
1282                }
1283            }
1284        }
1285        // wasm32: no JoinSet — run sequentially (wasm is single-threaded).
1286        #[cfg(target_arch = "wasm32")]
1287        {
1288            for &eid in exchanges {
1289                if self.inner.hub.is_connected(eid) {
1290                    let _ = self.inner.connector_tx.send(Event::ConnectorReady { exchange: eid });
1291                    ok.push(eid);
1292                } else {
1293                    match self.inner.hub.connect_public(eid, false).await {
1294                        Ok(()) => {
1295                            let _ = self.inner.connector_tx.send(Event::ConnectorReady { exchange: eid });
1296                            ok.push(eid);
1297                        }
1298                        Err(e) => {
1299                            tracing::warn!(?eid, ?e, "warmup: connect_public failed");
1300                            failed.push((eid, e.to_string()));
1301                        }
1302                    }
1303                }
1304            }
1305        }
1306
1307        // Phase 2: fetch exchange info for all successfully connected exchanges.
1308        const ACCOUNT_TYPES: &[AccountType] = &[AccountType::Spot, AccountType::FuturesCross];
1309
1310        for eid in ok.iter().copied() {
1311            let Some(connector) = self.inner.hub.rest(eid) else {
1312                // REST connector absent — skip exchange-info silently.
1313                continue;
1314            };
1315            for &at in ACCOUNT_TYPES {
1316                // Cache hit: re-emit from cache, skip REST.
1317                if let Some(cached) = self.inner.exchange_info_cache.get(&(eid, at)) {
1318                    let symbols = cached.value().clone();
1319                    let _ = self.inner.connector_tx.send(Event::SymbolsLoaded {
1320                        exchange: eid,
1321                        account_type: at,
1322                        symbols,
1323                    });
1324                    continue;
1325                }
1326                // Cache miss: call REST.
1327                match connector.get_exchange_info(at).await {
1328                    Ok(symbols) if !symbols.is_empty() => {
1329                        self.inner.exchange_info_cache.insert((eid, at), symbols.clone());
1330                        let _ = self.inner.connector_tx.send(Event::SymbolsLoaded {
1331                            exchange: eid,
1332                            account_type: at,
1333                            symbols,
1334                        });
1335                    }
1336                    Ok(_empty) => {
1337                        // Empty list — account type not supported by this exchange, skip.
1338                    }
1339                    Err(e) => {
1340                        use digdigdig3::core::types::ExchangeError;
1341                        match &e {
1342                            ExchangeError::WireAbsent(_)
1343                            | ExchangeError::NotImplemented(_) => {
1344                                // Expected for exchanges without futures or without
1345                                // exchange-info REST — silent skip.
1346                            }
1347                            other => {
1348                                tracing::warn!(?eid, ?at, ?other, "warmup: get_exchange_info failed");
1349                                failed.push((eid, other.to_string()));
1350                            }
1351                        }
1352                    }
1353                }
1354            }
1355        }
1356
1357        WarmupReport { ok, failed }
1358    }
1359
1360    /// Subscribe to every (exchange, symbol, account, stream) combination in
1361    /// `set`. Continue-on-error: per-stream failures are collected in
1362    /// [`SubscribeReport::failed`] and do not abort the rest of the batch.
1363    ///
1364    /// The returned `handle` carries events for every stream in `ok`. A
1365    /// stream whose subscribe failed will simply not emit events through
1366    /// the handle.
1367    ///
1368    /// The whole call returns `Err` ONLY for batch-level failures (empty
1369    /// set). Per-stream failures (StreamNotSupported, connect_websocket,
1370    /// symbol normalize) are reported via `report.failed`.
1371    pub async fn subscribe(&self, set: SubscriptionSet) -> Result<SubscribeReport> {
1372        if set.is_empty() {
1373            return Err(StationError::Subscribe("empty SubscriptionSet".into()));
1374        }
1375
1376        let (tx, rx) = mpsc::unbounded_channel::<Event>();
1377        let mut refs: Vec<MultiplexRef> = Vec::new();
1378        let mut ok: Vec<SeriesKey> = Vec::new();
1379        let mut failed: Vec<FailedStream> = Vec::new();
1380        let mut seed_outcomes: Vec<(SeriesKey, crate::SeedOutcome)> = Vec::new();
1381
1382        for entry in set.entries {
1383            // REST connector — needed for warm-start backfill (`get_recent_trades` /
1384            // `get_klines`). Hub memoizes internally; idempotent. Errors here are
1385            // logged-and-continued: WS-only subscribe still works without REST.
1386            if let Err(e) = self
1387                .inner
1388                .hub
1389                .connect_public(entry.exchange, false)
1390                .await
1391            {
1392                tracing::debug!(?e, ?entry.exchange, "connect_public failed; warm-start REST backfill will be skipped");
1393            }
1394
1395            // WS connect: skip only if ALL streams in this entry are poll-only
1396            // (REST polling — never touch a WS connector). Derived streams DO
1397            // need WS: they subscribe to WS-backed upstreams (Basis ← MarkPrice
1398            // + IndexPrice; TradeToBar/Range/Tick/Volume/Footprint ← Trade), so
1399            // the WS connector must be up before the derived forwarder spawns —
1400            // otherwise the recursive upstream acquire fails with "ws handle
1401            // missing post-connect". Mixed entries (e.g. [Trade, LongShortRatio])
1402            // also need WS for the non-poll stream. Per-stream failures are
1403            // reported in `failed`; only poll-only streams are excluded from that.
1404            let needs_ws = entry.streams.iter().any(|s| {
1405                s.to_kind().is_poll_only().is_none()
1406            });
1407
1408            if needs_ws {
1409                // For authenticated entries (private streams), open an
1410                // authenticated WS connection.  Falls back to public WS on
1411                // wasm32 where private WS auth is unavailable.
1412                let ws_connect_result = if let Some(ref creds) = entry.credentials {
1413                    #[cfg(not(target_arch = "wasm32"))]
1414                    {
1415                        self.inner
1416                            .hub
1417                            .connect_websocket_with_credentials(
1418                                entry.exchange,
1419                                entry.account_type,
1420                                creds.clone(),
1421                            )
1422                            .await
1423                    }
1424                    #[cfg(target_arch = "wasm32")]
1425                    {
1426                        let _ = creds;
1427                        Err(digdigdig3::core::types::ExchangeError::NotImplemented(
1428                            "private WS streams not supported on wasm32".into(),
1429                        ))
1430                    }
1431                } else {
1432                    self.inner
1433                        .hub
1434                        .connect_websocket(entry.exchange, entry.account_type, false)
1435                        .await
1436                };
1437                if let Err(e) = ws_connect_result
1438                {
1439                    let err_msg = format!("connect_websocket: {e}");
1440                    for s in &entry.streams {
1441                        // Only poll-only streams survive a WS-connect failure —
1442                        // derived streams need WS upstreams, so a WS failure fails
1443                        // them too.
1444                        if s.to_kind().is_poll_only().is_some() {
1445                            continue;
1446                        }
1447                        failed.push(FailedStream {
1448                            exchange: entry.exchange,
1449                            account_type: entry.account_type,
1450                            symbol: entry.symbol.clone(),
1451                            stream: s.clone(),
1452                            error: StationError::Core(err_msg.clone()),
1453                        });
1454                    }
1455                    // Only `continue` if there are no poll-only streams that can
1456                    // still be acquired without WS.
1457                    let has_non_ws = entry.streams.iter().any(|s| {
1458                        s.to_kind().is_poll_only().is_some()
1459                    });
1460                    if !has_non_ws {
1461                        continue;
1462                    }
1463                }
1464            }
1465
1466            // Resolve to (canonical, raw exchange-native) pair.
1467            //
1468            // - `add_raw`: passthrough. `entry.symbol` is the wire format
1469            //   already; canonical Symbol is built with empty base/quote +
1470            //   the raw string as its `raw` field. This is the only path
1471            //   that works for exotic instruments where BASE-QUOTE doesn't
1472            //   apply (Deribit options "BTC-23MAY26-86000-C", dated
1473            //   futures, index symbols, etc.).
1474            // - `add` (canonical): parse "BTC-USDT"-style input, translate
1475            //   to exchange-native via SymbolNormalizer.
1476            let (canonical, raw) = if entry.is_raw {
1477                (
1478                    Symbol::with_raw("", "", entry.symbol.clone()),
1479                    entry.symbol.clone(),
1480                )
1481            } else {
1482                let canonical = parse_symbol(&entry.symbol);
1483                match SymbolNormalizer::to_exchange(
1484                    entry.exchange,
1485                    &canonical,
1486                    entry.account_type,
1487                ) {
1488                    Ok(r) => (canonical, r),
1489                    Err(e) => {
1490                        let err_msg = format!("symbol normalize: {e}");
1491                        for s in &entry.streams {
1492                            failed.push(FailedStream {
1493                                exchange: entry.exchange,
1494                                account_type: entry.account_type,
1495                                symbol: entry.symbol.clone(),
1496                                stream: s.clone(),
1497                                error: StationError::Subscribe(err_msg.clone()),
1498                            });
1499                        }
1500                        continue;
1501                    }
1502                }
1503            };
1504
1505            // Part B seam: resolve display symbol → wire id for connectors where
1506            // the WS subscribe frame coin differs from the caller-facing display
1507            // name (HyperLiquid spot: "HYPE/USDC" → "@107"). The REST connector
1508            // is always present before subscriptions (connect_public / connect_full
1509            // is called before subscribe), and REST connectors self-warm their
1510            // universe cache on first use (OnceCell). For all other venues the
1511            // default impl is a passthrough (zero allocation, zero round-trip).
1512            let raw = if let Some(rest) = self.inner.hub.rest(entry.exchange) {
1513                rest.resolve_market_symbol(&raw, entry.account_type).await
1514            } else {
1515                raw
1516            };
1517
1518            for s in &entry.streams {
1519                let kind = s.to_kind();
1520                let key = SeriesKey {
1521                    exchange: entry.exchange,
1522                    account_type: entry.account_type,
1523                    symbol: raw.clone(),
1524                    kind: kind.clone(),
1525                };
1526
1527                // Task B: fail-fast capability gate — short-circuit OBVIOUS
1528                // unsupported cases before paying for a WS subscribe roundtrip.
1529                // Only fires when the connector has declared BOTH REST and WS
1530                // false for the Kind. Conservative — when uncertain we proceed.
1531                if let Some(caps) = self.inner.hub.capabilities(entry.exchange) {
1532                    if caps_explicitly_unsupported(&caps, &kind) {
1533                        let reason = format!(
1534                            "{:?} capability not declared on {:?} — neither WS nor REST available",
1535                            kind, entry.exchange,
1536                        );
1537                        tracing::debug!(
1538                            ?key, reason,
1539                            "capability fail-fast: skipping acquire_or_spawn"
1540                        );
1541                        failed.push(FailedStream {
1542                            exchange: entry.exchange,
1543                            account_type: entry.account_type,
1544                            symbol: entry.symbol.clone(),
1545                            stream: s.clone(),
1546                            error: StationError::StreamNotSupported(reason),
1547                        });
1548                        continue;
1549                    }
1550                }
1551
1552                let (bcast_tx, pending_seed) = match self
1553                    .acquire_or_spawn(&key, &entry, &canonical, &raw, s)
1554                    .await
1555                {
1556                    Ok(pair) => pair,
1557                    Err(e) => {
1558                        // WireAbsent on a per-(exchange, kind) basis: log
1559                        // at debug, record in `failed`, move on. Other errors
1560                        // get an info-level log so they are not lost.
1561                        if e.is_not_supported() {
1562                            tracing::debug!(?key, ?e, "stream not supported; skipping");
1563                        } else {
1564                            tracing::info!(?key, ?e, "subscribe failed; skipping");
1565                        }
1566                        failed.push(FailedStream {
1567                            exchange: entry.exchange,
1568                            account_type: entry.account_type,
1569                            symbol: entry.symbol.clone(),
1570                            stream: s.clone(),
1571                            error: e,
1572                        });
1573                        continue;
1574                    }
1575                };
1576
1577                let mut bcast_rx = bcast_tx.subscribe();
1578                let tx_clone = tx.clone();
1579                // Per-handle symbol label: relay rewrites Event.symbol from the
1580                // raw exchange-native form (carried on the broadcast) to the
1581                // user-input form THIS handle subscribed with. Two handles on
1582                // the same multiplex with different input forms each see their
1583                // own label.
1584                let label = entry.symbol.clone();
1585                {
1586                    let relay_fut = Box::pin(async move {
1587                        // Lagged is a back-pressure signal, NOT a fatal error.
1588                        // Cold-start scenario on single-threaded executors
1589                        // (wasm `spawn_local`): the relay task is queued
1590                        // AFTER `spawn_local(forwarder)`, so the forwarder
1591                        // can emit hundreds of events into its
1592                        // `broadcast::channel(512)` buffer before the relay
1593                        // gets its first CPU slice. If the producer is fast
1594                        // (derived TpoFromTrade emits one point per upstream
1595                        // Trade — ~30-100/s on BTCUSDT) the buffer overflows
1596                        // and `recv()` returns `Lagged(n)`.  Breaking here
1597                        // tore down the handle silently, so consumers saw
1598                        // `events_total=0`.  Keep going — skip the lost
1599                        // events and stream the live tail.
1600                        loop {
1601                            match bcast_rx.recv().await {
1602                                Ok(mut ev) => {
1603                                    ev.set_symbol(label.clone());
1604                                    if tx_clone.send(ev).is_err() {
1605                                        break;
1606                                    }
1607                                }
1608                                Err(tokio::sync::broadcast::error::RecvError::Lagged(_n)) => {
1609                                    // Skip the dropped window, keep reading the live tail.
1610                                    continue;
1611                                }
1612                                Err(tokio::sync::broadcast::error::RecvError::Closed) => {
1613                                    break;
1614                                }
1615                            }
1616                        }
1617                    });
1618                    #[cfg(not(target_arch = "wasm32"))]
1619                    tokio::spawn(relay_fut);
1620                    #[cfg(target_arch = "wasm32")]
1621                    wasm_bindgen_futures::spawn_local(relay_fut);
1622                }
1623
1624                // For OrderbookDelta with REST seed: emit the snapshot NOW,
1625                // after the relay task has subscribed to the broadcast channel.
1626                // This guarantees the snapshot reaches the consumer's
1627                // SubscriptionHandle::recv() — previously it was sent before
1628                // any receiver existed and was silently dropped.
1629                if let Some(seed_ev) = pending_seed {
1630                    if bcast_tx.send(seed_ev).is_err() {
1631                        tracing::debug!(
1632                            target: "dig3::ob_seed",
1633                            ?key,
1634                            "ob delta seed: send failed after relay wired (unexpected)"
1635                        );
1636                    }
1637                }
1638
1639                refs.push(MultiplexRef {
1640                    station: Arc::downgrade(&self.inner),
1641                    key: key.clone(),
1642                });
1643                // Drain any cold-seed outcome recorded for this key by
1644                // `acquire_or_spawn_derived_body` (side-channel — see
1645                // `StationInner::seed_outcomes`). Absent for WS-only /
1646                // poll-only / already-live-mux acquires.
1647                if let Some((_, outcome)) = self.inner.seed_outcomes.remove(&key) {
1648                    seed_outcomes.push((key.clone(), outcome));
1649                }
1650                ok.push(key);
1651            }
1652        }
1653
1654        Ok(SubscribeReport {
1655            handle: SubscriptionHandle { rx, _refs: refs },
1656            ok,
1657            failed,
1658            seed_outcomes,
1659        })
1660    }
1661
1662    /// Acquire (or spawn) the multiplexer for `key`. Spawn includes:
1663    /// - opening DiskStore<T> if persistence is on,
1664    /// - seeding broadcast with last-N (warm-start) before any live event,
1665    /// - issuing WS subscribe + forwarder task that runs until shutdown.
1666    async fn acquire_or_spawn(
1667        &self,
1668        key: &SeriesKey,
1669        entry: &Entry,
1670        canonical: &Symbol,
1671        raw_symbol: &str,
1672        stream: &Stream,
1673    ) -> Result<(broadcast::Sender<Event>, Option<Event>)> {
1674        if let Some(mut mux) = self.inner.muxes.get_mut(key) {
1675            // Cancel any pending grace-period timer — the forwarder is being
1676            // reused before the grace window expired. Sending on grace_cancel
1677            // unblocks the timer task's select! arm, which then exits without
1678            // firing the shutdown signal.
1679            if let Some(cancel) = mux.grace_cancel.take() {
1680                let _ = cancel.send(());
1681            }
1682            mux.consumers.fetch_add(1, Ordering::SeqCst);
1683            return Ok((mux.tx.clone(), None));
1684        }
1685
1686        // --- Derived stream path (no WS, no REST) ---
1687        // Must come BEFORE the ws handle resolution so we never call
1688        // ws.subscribe() for a derived kind.
1689        //
1690        // Exception: Kind::Basis and Kind::FundingSettlement are "derivable"
1691        // but also have native REST history endpoints on some exchanges.  On
1692        // native builds the per-(exchange, account) capability is checked first;
1693        // if available the stream is routed through a REST poller instead of the
1694        // RAM-derive path.  On wasm (no tokio::time::interval) the derive path
1695        // is always used.
1696        if key.kind.is_derived() {
1697            #[cfg(not(target_arch = "wasm32"))]
1698            {
1699                if let Kind::Basis = &key.kind {
1700                    let hub = &self.inner.hub;
1701                    if let Some(source) = polling::basis_poll_source(hub, key.exchange) {
1702                        tracing::info!(
1703                            target: "dig3::station::native_source",
1704                            exchange = ?key.exchange,
1705                            symbol   = %key.symbol,
1706                            "Kind::Basis: native REST basis-history available — using poll path"
1707                        );
1708                        let poll_spec = crate::series::PollSpec {
1709                            cadence: source.cadence(),
1710                            jitter_pct: 10,
1711                        };
1712                        return self
1713                            .acquire_or_spawn_polled_with_source::<BasisPoint, _>(
1714                                key, poll_spec, raw_symbol, source,
1715                            )
1716                            .await
1717                            .map(|tx| (tx, None));
1718                    } else {
1719                        tracing::info!(
1720                            target: "dig3::station::native_source",
1721                            exchange = ?key.exchange,
1722                            symbol   = %key.symbol,
1723                            "Kind::Basis: no native source — deriving from mark+index in RAM"
1724                        );
1725                    }
1726                }
1727                if let Kind::FundingSettlement = &key.kind {
1728                    let hub = &self.inner.hub;
1729                    if let Some(source) = polling::funding_poll_source(hub, key.exchange) {
1730                        tracing::info!(
1731                            target: "dig3::station::native_source",
1732                            exchange = ?key.exchange,
1733                            symbol   = %key.symbol,
1734                            "Kind::FundingSettlement: native REST funding-rate history available — using poll path"
1735                        );
1736                        let poll_spec = crate::series::PollSpec {
1737                            cadence: source.cadence(),
1738                            jitter_pct: 10,
1739                        };
1740                        return self
1741                            .acquire_or_spawn_polled_with_source::<FundingSettlementPoint, _>(
1742                                key, poll_spec, raw_symbol, source,
1743                            )
1744                            .await
1745                            .map(|tx| (tx, None));
1746                    } else {
1747                        tracing::info!(
1748                            target: "dig3::station::native_source",
1749                            exchange = ?key.exchange,
1750                            symbol   = %key.symbol,
1751                            "Kind::FundingSettlement: no native source — deriving from funding-rate flips in RAM"
1752                        );
1753                    }
1754                }
1755            }
1756
1757            return match &key.kind {
1758                Kind::Basis => {
1759                    self.acquire_or_spawn_derived::<BasisDerived>(key, entry, canonical, raw_symbol).await.map(|tx| (tx, None))
1760                }
1761                Kind::FundingSettlement => {
1762                    self.acquire_or_spawn_derived::<FundingSettlementDerived>(key, entry, canonical, raw_symbol).await.map(|tx| (tx, None))
1763                }
1764                Kind::RangeBar(_) => {
1765                    self.acquire_or_spawn_derived::<TradeToRangeBarDerived>(key, entry, canonical, raw_symbol).await.map(|tx| (tx, None))
1766                }
1767                Kind::TickBar(_) => {
1768                    self.acquire_or_spawn_derived::<TradeToTickBarDerived>(key, entry, canonical, raw_symbol).await.map(|tx| (tx, None))
1769                }
1770                Kind::VolumeBar(_) => {
1771                    self.acquire_or_spawn_derived::<TradeToVolumeBarDerived>(key, entry, canonical, raw_symbol).await.map(|tx| (tx, None))
1772                }
1773                Kind::Footprint(_) => {
1774                    self.acquire_or_spawn_derived::<TradeToFootprintDerived>(key, entry, canonical, raw_symbol).await.map(|tx| (tx, None))
1775                }
1776                Kind::RenkoBar(_, _) => {
1777                    self.acquire_or_spawn_derived::<TradeToRenkoBarDerived>(key, entry, canonical, raw_symbol).await.map(|tx| (tx, None))
1778                }
1779                Kind::PnfBar(_, _) => {
1780                    self.acquire_or_spawn_derived::<TradeToPnfBarDerived>(key, entry, canonical, raw_symbol).await.map(|tx| (tx, None))
1781                }
1782                Kind::KagiBar(_) => {
1783                    self.acquire_or_spawn_derived::<TradeToKagiBarDerived>(key, entry, canonical, raw_symbol).await.map(|tx| (tx, None))
1784                }
1785                Kind::CvdLine => {
1786                    self.acquire_or_spawn_derived::<TradeToCvdLineDerived>(key, entry, canonical, raw_symbol).await.map(|tx| (tx, None))
1787                }
1788                Kind::ThreeLineBreak { .. } => {
1789                    self.acquire_or_spawn_derived::<TradeToThreeLineBreakDerived>(key, entry, canonical, raw_symbol).await.map(|tx| (tx, None))
1790                }
1791                Kind::TpoProfile(_, TpoSource::Kline1m) => {
1792                    self.acquire_or_spawn_derived::<TpoFromKline1mDerived>(key, entry, canonical, raw_symbol).await.map(|tx| (tx, None))
1793                }
1794                Kind::TpoProfile(_, TpoSource::TradeBucket) => {
1795                    self.acquire_or_spawn_derived::<TpoFromTradeDerived>(key, entry, canonical, raw_symbol).await.map(|tx| (tx, None))
1796                }
1797                Kind::DollarBar { .. } => {
1798                    self.acquire_or_spawn_derived::<TradeToDollarBarDerived>(key, entry, canonical, raw_symbol).await.map(|tx| (tx, None))
1799                }
1800                Kind::TickImbalanceBar { .. } => {
1801                    self.acquire_or_spawn_derived::<TradeToTickImbalanceDerived>(key, entry, canonical, raw_symbol).await.map(|tx| (tx, None))
1802                }
1803                Kind::VolumeImbalanceBar { .. } => {
1804                    self.acquire_or_spawn_derived::<TradeToVolumeImbalanceDerived>(key, entry, canonical, raw_symbol).await.map(|tx| (tx, None))
1805                }
1806                Kind::RunBar { .. } => {
1807                    self.acquire_or_spawn_derived::<TradeToRunBarDerived>(key, entry, canonical, raw_symbol).await.map(|tx| (tx, None))
1808                }
1809                _ => unreachable!("is_derived() returned true for unhandled kind — update acquire_or_spawn dispatch"),
1810            };
1811        }
1812
1813        // --- Poll-only stream path (REST periodic polling, no WS) ---
1814        // Must come BEFORE the ws.subscribe call so we never try to subscribe
1815        // a WS channel for streams that have no WS feed.
1816        #[cfg(not(target_arch = "wasm32"))]
1817        if let Some(poll_spec) = key.kind.is_poll_only() {
1818            return self.acquire_or_spawn_polled(key, entry, poll_spec, raw_symbol).await.map(|tx| (tx, None));
1819        }
1820        // On wasm, poll-only kinds are not supported (no tokio::time::interval).
1821        #[cfg(target_arch = "wasm32")]
1822        if key.kind.is_poll_only().is_some() {
1823            return Err(StationError::StreamNotSupported(format!(
1824                "poll-only streams not supported on wasm32 ({:?})",
1825                key.kind
1826            )));
1827        }
1828
1829        let sym = Symbol::with_raw(&canonical.base, &canonical.quote, raw_symbol.to_string());
1830        let req = ws_request_for(&key.kind, sym, entry.account_type);
1831
1832        let ws = self
1833            .inner
1834            .hub
1835            .ws(entry.exchange, entry.account_type)
1836            .ok_or_else(|| StationError::Core("ws handle missing post-connect".into()))?;
1837        // `transport.rs::subscribe` eagerly invokes `subscribe_frame` and
1838        // propagates any frame-construction failure (WireAbsent and
1839        // NotImplemented included). Map those to
1840        // `StreamNotSupported` so `Station::subscribe(set)` can bucket
1841        // them into `SubscribeReport::failed` without spawning a forwarder
1842        // that would loop in heal/resub forever (this is what caused
1843        // MLI's 0.3.6 OOM — see release-0.3.7-plan.md).
1844        //
1845        // Special case — Kind::Kline(iv): if the venue does not natively
1846        // support this interval on its WS, fall back to TradeToBarDerived
1847        // (trade-aggregation engine) rather than returning a hard error.
1848        // The fallback is attempted only when ws.subscribe fails with
1849        // WireAbsent / NotImplemented; native kline paths are
1850        // unchanged. If the interval string is unknown (interval_to_ms
1851        // returns None) we cannot build the aggregator either — return a
1852        // clear StreamNotSupported to the caller.
1853        if let Err(e) = ws.subscribe(req.clone()).await {
1854            use digdigdig3::core::types::WebSocketError;
1855            let is_not_supported = matches!(
1856                e,
1857                WebSocketError::WireAbsent(_) | WebSocketError::NotImplemented(_)
1858            );
1859            if is_not_supported {
1860                if let Kind::Kline(iv) = &key.kind {
1861                    // Validate the interval before attempting the aggregator.
1862                    if interval_to_ms(iv.as_str()).is_none() {
1863                        return Err(StationError::StreamNotSupported(format!(
1864                            "Kline interval {:?} is unknown — cannot aggregate from trades",
1865                            iv.as_str()
1866                        )));
1867                    }
1868                    tracing::debug!(
1869                        target: "dig3::station::derived",
1870                        exchange = ?key.exchange,
1871                        symbol   = %key.symbol,
1872                        interval = %iv,
1873                        "native Kline WS not supported — falling back to TradeToBarDerived"
1874                    );
1875                    return self
1876                        .acquire_or_spawn_derived::<TradeToBarDerived>(key, entry, canonical, raw_symbol)
1877                        .await
1878                        .map(|tx| (tx, None));
1879                }
1880            }
1881            return Err(match e {
1882                WebSocketError::WireAbsent(msg)
1883                | WebSocketError::NotImplemented(msg) => {
1884                    StationError::StreamNotSupported(msg)
1885                }
1886                other => StationError::Subscribe(format!("ws.subscribe: {other}")),
1887            });
1888        }
1889
1890        let (bcast_tx, _) = broadcast::channel::<Event>(1024);
1891        let consumers = Arc::new(AtomicUsize::new(1));
1892        let (shutdown_tx, shutdown_rx) = oneshot::channel::<()>();
1893
1894        let _ = stream; // kept for future per-Stream parameter customizations
1895
1896        // For each kind, compute the REST backfill seed (used when disk is
1897        // empty), then spawn the typed forwarder. Backfill is best-effort —
1898        // empty Vec on any failure or unsupported endpoint.
1899        let warm_n = self.inner.warm_start_capacity;
1900        let hub = self.inner.hub.clone();
1901        let acct = entry.account_type;
1902        let raw_s = raw_symbol.to_string();
1903
1904        // Populated by Kind::OrderbookDelta when orderbook_rest_seed=true.
1905        // Returned to Station::subscribe so it can be sent AFTER the relay task
1906        // has subscribed to the broadcast channel (fixes the race where the snapshot
1907        // was dropped because no receivers existed yet).
1908        let mut pending_seed: Option<Event> = None;
1909
1910        match &key.kind {
1911            Kind::Trade => {
1912                let seed = if warm_n > 0 {
1913                    crate::backfill::trades_recent(&hub, key.exchange, acct, &raw_s, warm_n).await.0
1914                } else { Vec::new() };
1915                spawn_forwarder::<TradePoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), seed, req.clone());
1916            }
1917            Kind::Kline(interval) => {
1918                let seed = if warm_n > 0 {
1919                    crate::backfill::klines_recent(&hub, key.exchange, acct, &raw_s, interval.as_str(), warm_n).await
1920                } else { Vec::new() };
1921                spawn_forwarder::<BarPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), seed, req.clone());
1922            }
1923            Kind::AggTrade => {
1924                let seed = if warm_n > 0 {
1925                    crate::backfill::agg_trades_recent(&hub, key.exchange, acct, &raw_s, warm_n).await.0
1926                } else { Vec::new() };
1927                spawn_forwarder::<AggTradePoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), seed, req.clone());
1928            }
1929            Kind::Ticker => match self.inner.persistence.depth_for(&key.kind) {
1930                Some(PersistDepth::Indicators) => {
1931                    let seed = if warm_n > 0 {
1932                        crate::backfill::tickers_indicators_recent(&hub, key.exchange, acct, &raw_s, warm_n).await
1933                    } else { Vec::new() };
1934                    spawn_forwarder::<TickerIndicatorsPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), seed, req.clone());
1935                }
1936                Some(PersistDepth::Full) => {
1937                    let seed = if warm_n > 0 {
1938                        crate::backfill::tickers_full_recent(&hub, key.exchange, acct, &raw_s, warm_n).await
1939                    } else { Vec::new() };
1940                    spawn_forwarder::<TickerFullPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), seed, req.clone());
1941                }
1942                _ => spawn_forwarder::<TickerPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
1943            },
1944            Kind::Orderbook => {
1945                let ob_seed = if self.inner.orderbook_rest_seed {
1946                    ob_rest_seed(&hub, key.exchange, acct, &raw_s, self.inner.orderbook_seed_depth).await
1947                } else {
1948                    Vec::new()
1949                };
1950                match self.inner.persistence.depth_for(&key.kind) {
1951                    Some(PersistDepth::Indicators) | Some(PersistDepth::Full) => {
1952                        // ObSnapshotIndicatorsPoint carries cts + prev_change_id.
1953                        // Seed is discarded for extended types (seed uses Compact layout).
1954                        spawn_forwarder::<ObSnapshotIndicatorsPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone());
1955                    }
1956                    _ => spawn_forwarder::<ObSnapshotPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), ob_seed, req.clone()),
1957                }
1958            }
1959            Kind::OrderbookDelta => {
1960                // Seed via REST snapshot: gives downstream assemblers a seeded
1961                // full-book state before deltas arrive. The seed event is NOT
1962                // emitted here — it is returned as `pending_seed` and emitted by
1963                // `Station::subscribe` AFTER the consumer's relay task has subscribed
1964                // to the broadcast channel. Emitting before any receiver exists (the
1965                // old behaviour) caused the snapshot to be silently dropped.
1966                //
1967                // Note: wasm32 skips REST seed (possible CORS), same as before.
1968                pending_seed = if self.inner.orderbook_rest_seed {
1969                    #[cfg(not(target_arch = "wasm32"))]
1970                    {
1971                        let snapshots = ob_rest_seed(&hub, key.exchange, acct, &raw_s, self.inner.orderbook_seed_depth).await;
1972                        snapshots.into_iter().next().map(|point| Event::OrderbookSnapshot {
1973                            exchange: key.exchange,
1974                            symbol: raw_s.clone(),
1975                            point,
1976                        })
1977                    }
1978                    #[cfg(target_arch = "wasm32")]
1979                    {
1980                        tracing::warn!(
1981                            target: "dig3::ob_seed",
1982                            exchange = ?key.exchange, symbol = raw_s.as_str(),
1983                            "orderbook REST seed for delta stream skipped on wasm32 (possible CORS) — continuing WS-only"
1984                        );
1985                        None
1986                    }
1987                } else {
1988                    None
1989                };
1990                match self.inner.persistence.depth_for(&key.kind) {
1991                    Some(PersistDepth::Indicators) | Some(PersistDepth::Full) =>
1992                        spawn_forwarder::<ObDeltaIndicatorsPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
1993                    _ => spawn_forwarder::<ObDeltaPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
1994                }
1995            }
1996            Kind::MarkPrice => match self.inner.persistence.depth_for(&key.kind) {
1997                Some(PersistDepth::Indicators) => {
1998                    let seed = if warm_n > 0 {
1999                        crate::backfill::mark_price_indicators_recent(&hub, key.exchange, acct, &raw_s, warm_n).await
2000                    } else { Vec::new() };
2001                    spawn_forwarder::<MarkPriceIndicatorsPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), seed, req.clone());
2002                }
2003                Some(PersistDepth::Full) => {
2004                    let seed = if warm_n > 0 {
2005                        crate::backfill::mark_price_full_recent(&hub, key.exchange, acct, &raw_s, warm_n).await
2006                    } else { Vec::new() };
2007                    spawn_forwarder::<MarkPriceFullPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), seed, req.clone());
2008                }
2009                _ => {
2010                    let seed = if warm_n > 0 {
2011                        crate::backfill::mark_price_recent(&hub, key.exchange, acct, &raw_s, warm_n).await
2012                    } else { Vec::new() };
2013                    spawn_forwarder::<MarkPricePoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), seed, req.clone());
2014                }
2015            },
2016            Kind::FundingRate => match self.inner.persistence.depth_for(&key.kind) {
2017                Some(PersistDepth::Indicators) => {
2018                    let seed = if warm_n > 0 {
2019                        crate::backfill::funding_rate_indicators_recent(&hub, key.exchange, acct, &raw_s, warm_n).await
2020                    } else { Vec::new() };
2021                    spawn_forwarder::<FundingRateIndicatorsPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), seed, req.clone());
2022                }
2023                Some(PersistDepth::Full) => {
2024                    let seed = if warm_n > 0 {
2025                        crate::backfill::funding_rate_full_recent(&hub, key.exchange, acct, &raw_s, warm_n).await
2026                    } else { Vec::new() };
2027                    spawn_forwarder::<FundingRateFullPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), seed, req.clone());
2028                }
2029                _ => {
2030                    let seed = if warm_n > 0 {
2031                        crate::backfill::funding_rate_recent(&hub, key.exchange, acct, &raw_s, warm_n).await
2032                    } else { Vec::new() };
2033                    spawn_forwarder::<FundingRatePoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), seed, req.clone());
2034                }
2035            },
2036            Kind::OpenInterest => match self.inner.persistence.depth_for(&key.kind) {
2037                Some(PersistDepth::Indicators) | Some(PersistDepth::Full) => {
2038                    // OpenInterestFullPoint is a type alias to OpenInterestIndicatorsPoint.
2039                    let seed = if warm_n > 0 {
2040                        crate::backfill::open_interest_indicators_recent(&hub, key.exchange, acct, &raw_s, warm_n).await
2041                    } else { Vec::new() };
2042                    spawn_forwarder::<OpenInterestIndicatorsPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), seed, req.clone());
2043                }
2044                _ => {
2045                    let seed = if warm_n > 0 {
2046                        crate::backfill::open_interest_recent(&hub, key.exchange, acct, &raw_s, warm_n).await
2047                    } else { Vec::new() };
2048                    spawn_forwarder::<OpenInterestPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), seed, req.clone());
2049                }
2050            },
2051            Kind::Liquidation => match self.inner.persistence.depth_for(&key.kind) {
2052                Some(PersistDepth::Indicators) => {
2053                    let seed = if warm_n > 0 {
2054                        crate::backfill::liquidation_indicators_recent(&hub, key.exchange, acct, &raw_s, warm_n).await
2055                    } else { Vec::new() };
2056                    spawn_forwarder::<LiquidationIndicatorsPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), seed, req.clone());
2057                }
2058                Some(PersistDepth::Full) => {
2059                    let seed = if warm_n > 0 {
2060                        crate::backfill::liquidation_full_recent(&hub, key.exchange, acct, &raw_s, warm_n).await
2061                    } else { Vec::new() };
2062                    spawn_forwarder::<LiquidationFullPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), seed, req.clone());
2063                }
2064                _ => {
2065                    let seed = if warm_n > 0 {
2066                        crate::backfill::liquidations_recent(&hub, key.exchange, acct, &raw_s, warm_n).await
2067                    } else { Vec::new() };
2068                    spawn_forwarder::<LiquidationPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), seed, req.clone());
2069                }
2070            },
2071            Kind::BlockTrade => spawn_forwarder::<BlockTradePoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
2072            Kind::AuctionEvent => spawn_forwarder::<AuctionEventPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
2073            Kind::IndexPrice => match self.inner.persistence.depth_for(&key.kind) {
2074                Some(PersistDepth::Indicators) | Some(PersistDepth::Full) => spawn_forwarder::<IndexPriceIndicatorsPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
2075                _ => spawn_forwarder::<IndexPricePoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
2076            },
2077            Kind::CompositeIndex => spawn_forwarder::<CompositeIndexPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
2078            Kind::OptionGreeks => spawn_forwarder::<OptionGreeksPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
2079            Kind::VolatilityIndex => spawn_forwarder::<VolatilityIndexPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
2080            Kind::HistoricalVolatility => spawn_forwarder::<HistoricalVolatilityPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
2081            // LongShortRatio / TakerVolume / LiquidationBucket: poll-only, unreachable in
2082            // normal operation (the is_poll_only() branch above handles these first).
2083            // Kept as defensive fallbacks so the match arm is exhaustive.
2084            Kind::LongShortRatio => spawn_forwarder::<LongShortRatioPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
2085            Kind::TakerVolume => spawn_forwarder::<TakerVolumePoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
2086            Kind::LiquidationBucket => spawn_forwarder::<LiquidationBucketPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
2087            Kind::Basis => spawn_forwarder::<BasisPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
2088            Kind::InsuranceFund => {
2089                let seed = if warm_n > 0 {
2090                    crate::backfill::insurance_fund_recent(&hub, key.exchange, acct, &raw_s, warm_n).await
2091                } else { Vec::new() };
2092                spawn_forwarder::<InsuranceFundPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), seed, req.clone());
2093            }
2094            Kind::OrderbookL3 => spawn_forwarder::<OrderbookL3Point>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
2095            Kind::SettlementEvent => spawn_forwarder::<SettlementEventPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
2096            Kind::MarketWarning => spawn_forwarder::<MarketWarningPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
2097            Kind::RiskLimit => spawn_forwarder::<RiskLimitPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
2098            Kind::PredictedFunding => spawn_forwarder::<PredictedFundingPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
2099            Kind::FundingSettlement => spawn_forwarder::<FundingSettlementPoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
2100            Kind::MarkPriceKline(iv) => {
2101                let seed = if warm_n > 0 {
2102                    crate::backfill::mark_price_klines_recent(&hub, key.exchange, acct, &raw_s, iv.as_str(), warm_n).await
2103                } else { Vec::new() };
2104                spawn_forwarder::<MarkPriceKlinePoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), seed, req.clone());
2105            }
2106            Kind::IndexPriceKline(iv) => {
2107                let seed = if warm_n > 0 {
2108                    crate::backfill::index_price_klines_recent(&hub, key.exchange, acct, &raw_s, iv.as_str(), warm_n).await
2109                } else { Vec::new() };
2110                spawn_forwarder::<IndexPriceKlinePoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), seed, req.clone());
2111            }
2112            Kind::PremiumIndexKline(iv) => {
2113                let seed = if warm_n > 0 {
2114                    crate::backfill::premium_index_klines_recent(&hub, key.exchange, acct, &raw_s, iv.as_str(), warm_n).await
2115                } else { Vec::new() };
2116                spawn_forwarder::<PremiumIndexKlinePoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), seed, req.clone());
2117            }
2118            // Private streams — no warm-start seed, no persistence (ephemeral by design).
2119            Kind::OrderUpdate => spawn_forwarder::<OrderUpdatePoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
2120            Kind::BalanceUpdate => spawn_forwarder::<BalanceUpdatePoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req.clone()),
2121            Kind::PositionUpdate => spawn_forwarder::<PositionUpdatePoint>(self, key, ws, bcast_tx.clone(), shutdown_rx, key.symbol.clone(), Vec::new(), req),
2122            // Derived kinds — handled above by acquire_or_spawn_derived before
2123            // reaching this match. These arms satisfy exhaustiveness only.
2124            Kind::RangeBar(_) | Kind::TickBar(_) | Kind::VolumeBar(_) | Kind::Footprint(_)
2125            | Kind::RenkoBar(_, _) | Kind::PnfBar(_, _) | Kind::KagiBar(_)
2126            | Kind::CvdLine | Kind::ThreeLineBreak { .. } | Kind::TpoProfile(_, _)
2127            | Kind::DollarBar { .. } | Kind::TickImbalanceBar { .. }
2128            | Kind::VolumeImbalanceBar { .. } | Kind::RunBar { .. } => {
2129                unreachable!("derived kinds dispatched before forwarder match")
2130            }
2131        }
2132
2133        self.inner.muxes.insert(
2134            key.clone(),
2135            Multiplexer { tx: bcast_tx.clone(), consumers, shutdown: Some(shutdown_tx), grace_cancel: None },
2136        );
2137
2138        Ok((bcast_tx, pending_seed))
2139    }
2140}
2141
2142impl Station {
2143    /// Acquire (or spawn) a derived-stream multiplexer for `key`.
2144    ///
2145    /// Recursively calls `acquire_or_spawn` for each upstream dep (which
2146    /// follows the normal WS path), subscribes to each upstream broadcast,
2147    /// then spawns `spawn_derived_forwarder<D>` to run the computation.
2148    ///
2149    /// Ref-counting: each upstream `acquire_or_spawn` call increments the
2150    /// upstream `consumers` counter by 1 (for the derived forwarder's benefit).
2151    /// When the derived forwarder exits it calls `inner.release_consumer` on
2152    /// each upstream key, propagating shutdown upward if no other consumer
2153    /// holds the upstream.
2154    /// Acquire (or spawn) a derived multiplexer — **native** path.
2155    /// Returns `Pin<Box<dyn Future + Send>>` so the future can be awaited
2156    /// from `acquire_or_spawn` which is itself spawned via `tokio::spawn` on native.
2157    #[cfg(not(target_arch = "wasm32"))]
2158    fn acquire_or_spawn_derived<'a, D: DerivedStream>(
2159        &'a self,
2160        key: &'a SeriesKey,
2161        entry: &'a Entry,
2162        canonical: &'a digdigdig3::core::types::Symbol,
2163        raw_symbol: &'a str,
2164    ) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<broadcast::Sender<Event>>> + Send + 'a>>
2165    where
2166        Event: EventFrom<D::Output>,
2167    {
2168        Box::pin(async move {
2169            self.acquire_or_spawn_derived_body::<D>(key, entry, canonical, raw_symbol).await
2170        })
2171    }
2172
2173    /// Acquire (or spawn) a derived multiplexer — **wasm32** path.
2174    /// No `Send` bound — wasm is single-threaded and all futures are `!Send`.
2175    #[cfg(target_arch = "wasm32")]
2176    fn acquire_or_spawn_derived<'a, D: DerivedStream>(
2177        &'a self,
2178        key: &'a SeriesKey,
2179        entry: &'a Entry,
2180        canonical: &'a digdigdig3::core::types::Symbol,
2181        raw_symbol: &'a str,
2182    ) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<broadcast::Sender<Event>>> + 'a>>
2183    where
2184        Event: EventFrom<D::Output>,
2185    {
2186        Box::pin(async move {
2187            self.acquire_or_spawn_derived_body::<D>(key, entry, canonical, raw_symbol).await
2188        })
2189    }
2190
2191    /// Shared body for `acquire_or_spawn_derived` — called from both cfg variants.
2192    async fn acquire_or_spawn_derived_body<D: DerivedStream>(
2193        &self,
2194        key: &SeriesKey,
2195        entry: &Entry,
2196        canonical: &digdigdig3::core::types::Symbol,
2197        raw_symbol: &str,
2198    ) -> Result<broadcast::Sender<Event>>
2199    where
2200        Event: EventFrom<D::Output>,
2201    {
2202        let mut upstream_rxs: Vec<broadcast::Receiver<Event>> = Vec::new();
2203        let mut upstream_keys: Vec<SeriesKey> = Vec::new();
2204
2205        // Use the key-aware dep list so derived streams whose deps carry a
2206        // runtime parameter (e.g. TpoFromKline1mDerived → Kline("1m")) can
2207        // resolve their actual upstream `SeriesKey`.
2208        let resolved_deps: Vec<Stream> = D::deps_for_key(key);
2209        for dep_stream in &resolved_deps {
2210            let dep_kind = dep_stream.to_kind();
2211            debug_assert!(
2212                !dep_kind.is_derived(),
2213                "DerivedStream::deps_for_key() must not list derived kinds (no derived-of-derived)"
2214            );
2215            let dep_key = SeriesKey {
2216                exchange: key.exchange,
2217                account_type: key.account_type,
2218                symbol: raw_symbol.to_string(),
2219                kind: dep_kind,
2220            };
2221            // Recursive call — follows the normal WS path for each upstream kind.
2222            // The pending_seed (second tuple element) is intentionally ignored here:
2223            // derived streams subscribe to the upstream broadcast directly, not through
2224            // a consumer relay, so the seed will be replayed naturally through the
2225            // upstream forwarder's warm-start mechanism.
2226            let (up_tx, _) = self
2227                .acquire_or_spawn(&dep_key, entry, canonical, raw_symbol, dep_stream)
2228                .await?;
2229            upstream_rxs.push(up_tx.subscribe());
2230            upstream_keys.push(dep_key);
2231        }
2232
2233        // AggTrade dense seed for derived cold-start (P0-C closure). Build a
2234        // per-dep seed Event vec: for Trade deps, fetch AggTrade history from
2235        // REST (falling back to recent_trades when has_agg_trades=false), convert
2236        // to TradePoint→Event::Trade. Passed into spawn_derived_forwarder which
2237        // feeds them through D::seed_from_events before the live loop begins.
2238        //
2239        // Per-subscribe override: `entry.warm_override` replaces the Station-
2240        // wide `warm_start_capacity` for THIS entry's cold-start depth (see
2241        // `SubscriptionSet::add_with_warm`). `None` keeps existing behavior.
2242        let warm_n = entry.warm_override.unwrap_or(self.inner.warm_start_capacity);
2243        // Kline-sufficient kinds (Renko/PnF/Kagi/3LB/Range/Volume/Dollar) are
2244        // PURE KLINE CONVERTERS — history AND live both come from
2245        // `Stream::Kline("1m")` exclusively; they carry no `Stream::Trade`
2246        // dependency at all (see `DerivedStream::deps_for_key` on each of
2247        // the seven). Everything the fold needs (price path, and for
2248        // volume/dollar, volume/quote-volume) is fully recoverable from 1m
2249        // klines via `kline_to_synthetic_trades` for history and
2250        // `KlineDeltaState` for the live Δ-tick path. Tick/imbalance/run
2251        // bars, CVD, and footprint genuinely need real trades (aggressor
2252        // side, per-trade counts, intra-bar ladders) and are excluded here —
2253        // their paths are untouched.
2254        let is_kline_sufficient_kind = matches!(
2255            key.kind,
2256            Kind::RenkoBar(_, _) | Kind::PnfBar(_, _) | Kind::KagiBar(_) | Kind::ThreeLineBreak { .. }
2257            | Kind::RangeBar(_) | Kind::VolumeBar(_) | Kind::DollarBar { .. }
2258        );
2259        // Baseline handed to the live `KlineDeltaState` (see
2260        // `DerivedStream::seed_kline_baseline`) from the cold-seed's LAST
2261        // fetched kline — set only for `is_kline_sufficient_kind`, and only
2262        // when at least one kline was fetched. `spawn_derived_forwarder`
2263        // applies it right after `D::new_for_key`, before any live event.
2264        let mut kline_seed_baseline: Option<(i64, f64, f64)> = None;
2265        let mut agg_seed_per_dep: Vec<Vec<Event>> = Vec::with_capacity(resolved_deps.len());
2266        for dep_stream in &resolved_deps {
2267            let dep_kind = dep_stream.to_kind();
2268            let mut seed_events: Vec<Event> = Vec::new();
2269            if warm_n > 0 && is_kline_sufficient_kind && matches!(dep_kind, Kind::Kline(_)) {
2270                // Sole seed source for these 7 kinds. n_kline_pages formula:
2271                // warm_n is expressed in the same "trade count" unit as the
2272                // plain aggTrade warm depth (e.g. 50_000). One 1m kline page
2273                // = 1000 bars = ~16.7h. We want the kline window to scale
2274                // with the caller's requested depth while staying within a
2275                // sane REST budget, so: 1 page per 1000 "warm units"
2276                // requested, clamped to [1, 100] pages (100 pages × 1000 ×
2277                // 1m ≈ 69 days — plenty for "weeks of 1m", while bounding
2278                // worst-case REST calls per chart-open).
2279                let n_kline_pages = ((warm_n + 999) / 1000).clamp(1, 100);
2280                let (kline_bars, k_outcome) = crate::backfill::klines_paginated(
2281                    &self.inner.hub, key.exchange, entry.account_type, raw_symbol,
2282                    "1m", 1000, n_kline_pages,
2283                ).await;
2284                let kline_interval_ms = interval_to_ms("1m").unwrap_or(60_000);
2285                // `klines_paginated` returns oldest→newest. The query window
2286                // reaches "now" (see its `next_end_time` seed), so the LAST
2287                // bar may be the exchange's still-forming (unclosed) 1m
2288                // candle. Fold every bar EXCEPT the last as a full 4-leg
2289                // synthetic-trade path — those are certainly closed, honest
2290                // history. Hand the last bar's (open_time, volume, close) to
2291                // the live Δ-tick adapter as its seed baseline instead of
2292                // folding it too: the first live WS kline update sharing
2293                // that SAME open_time then computes a correct incremental
2294                // delta against this baseline rather than double-counting
2295                // (full 4-leg fold + live delta) or dropping it. If that
2296                // last bar is in fact already closed, this drops one bar's
2297                // OHLC path from the deep seed — the same audited ≤1-bar
2298                // seam already accepted elsewhere in this design (see
2299                // `rewarm_derived_standalone`'s doc comment).
2300                if let Some((last, closed)) = kline_bars.split_last() {
2301                    for bar in closed {
2302                        seed_events.extend(crate::derived::kline_to_synthetic_trades(
2303                            bar, kline_interval_ms, key.exchange, raw_symbol,
2304                        ));
2305                    }
2306                    kline_seed_baseline = Some((last.open_time, last.volume, last.close));
2307                }
2308                self.inner.seed_outcomes.insert(key.clone(), k_outcome);
2309            } else if warm_n > 0 && matches!(dep_kind, Kind::Trade) {
2310                let caps_opt = self.inner.hub.capabilities(key.exchange);
2311                let use_agg = caps_opt.as_ref().map(|c| c.has_agg_trades).unwrap_or(false);
2312                // Capability-aware trade-history tier for this (exchange,
2313                // account) — consulted BEFORE any REST call so a
2314                // `RecentOnly` venue never "attempts" pagination, and a
2315                // `RestWindow` venue's walk clamps at the wall instead of
2316                // discovering it by an empty page (Task 4).
2317                let tier = self.inner.hub
2318                    .trade_history_capabilities(key.exchange)
2319                    .map(|c| c.tier_for(entry.account_type))
2320                    .unwrap_or(digdigdig3::core::types::TradeHistoryTier::RecentOnly { max_trades: 1000 });
2321                let trade_outcome = if matches!(tier, digdigdig3::core::types::TradeHistoryTier::RecentOnly { .. }) {
2322                    // RecentOnly: no pagination attempt at all — a single
2323                    // shallow call is the venue's entire history.
2324                    let (trades, outcome) = crate::backfill::trades_recent(
2325                        &self.inner.hub, key.exchange, entry.account_type, raw_symbol, warm_n,
2326                    ).await;
2327                    for pt in trades {
2328                        seed_events.push(Event::Trade {
2329                            exchange: key.exchange,
2330                            symbol: raw_symbol.to_string(),
2331                            point: pt,
2332                        });
2333                    }
2334                    outcome
2335                } else if use_agg {
2336                    // Paginated aggTrade fetch — derive page count from warm_n
2337                    // so warm_start(5000) gives 5 pages (~20-60s of BTC history,
2338                    // enough for several closed range/volume/tick bars). Single-
2339                    // page agg_trades_recent only covered ~1-5s, never enough.
2340                    // `agg_trades_paginated` itself clamps pagination at the
2341                    // `RestWindow` wall when the tier declares one.
2342                    let n_pages = ((warm_n + 999) / 1000).max(1);
2343                    let (agg, outcome) = crate::backfill::agg_trades_paginated(
2344                        &self.inner.hub, key.exchange, entry.account_type, raw_symbol,
2345                        1000, n_pages,
2346                    ).await;
2347                    for ap in agg {
2348                        seed_events.push(Event::Trade {
2349                            exchange: key.exchange,
2350                            symbol: raw_symbol.to_string(),
2351                            point: TradePoint {
2352                                ts_ms: ap.ts_ms,
2353                                price: ap.price,
2354                                quantity: ap.quantity,
2355                                side: ap.side,
2356                                trade_id_hash: 0,
2357                            },
2358                        });
2359                    }
2360                    outcome
2361                } else {
2362                    // Fall back to recent trades when no aggTrades available.
2363                    let (trades, outcome) = crate::backfill::trades_recent(
2364                        &self.inner.hub, key.exchange, entry.account_type, raw_symbol, warm_n,
2365                    ).await;
2366                    for pt in trades {
2367                        seed_events.push(Event::Trade {
2368                            exchange: key.exchange,
2369                            symbol: raw_symbol.to_string(),
2370                            point: pt,
2371                        });
2372                    }
2373                    outcome
2374                };
2375                self.inner.seed_outcomes.insert(key.clone(), trade_outcome);
2376            } else if warm_n > 0 && matches!(dep_kind, Kind::MarkPrice) {
2377                // Seed MarkPrice dep (used by BasisDerived dep index 0) from REST snapshot
2378                // so Basis can emit immediately on cold-start without waiting for WS.
2379                let pts = crate::backfill::mark_price_recent(
2380                    &self.inner.hub, key.exchange, entry.account_type, raw_symbol, warm_n,
2381                ).await;
2382                for pt in pts {
2383                    seed_events.push(Event::MarkPrice {
2384                        exchange: key.exchange,
2385                        symbol: raw_symbol.to_string(),
2386                        point: pt,
2387                    });
2388                }
2389            } else if warm_n > 0 && matches!(dep_kind, Kind::IndexPrice) {
2390                // Seed IndexPrice dep (used by BasisDerived dep index 1) from REST snapshot
2391                // so Basis can emit immediately on cold-start without waiting for WS.
2392                let pts = crate::backfill::mark_price_recent(
2393                    &self.inner.hub, key.exchange, entry.account_type, raw_symbol, warm_n,
2394                ).await;
2395                // get_premium_index returns MarkPrice which carries both mark + index.
2396                // Map to IndexPricePoint via the index field.
2397                for pt in pts {
2398                    if pt.index.is_finite() {
2399                        seed_events.push(Event::IndexPrice {
2400                            exchange: key.exchange,
2401                            symbol: raw_symbol.to_string(),
2402                            point: IndexPricePoint { ts_ms: pt.ts_ms, price: pt.index },
2403                        });
2404                    }
2405                }
2406            }
2407            agg_seed_per_dep.push(seed_events);
2408        }
2409
2410        // Ring-capacity hint: the in-memory Series<D::Output> ring must be
2411        // sized at least as large as the seed we just computed, or the ring
2412        // evicts the deep seed before spawn_derived_forwarder even reaches
2413        // the live loop (risk flagged in the design doc's risk register —
2414        // MUST ship in the same commit as the warm_override, not after).
2415        // A price-path deep seed can synthesize far more points than
2416        // warm_start_capacity (e.g. 100 kline pages × 1000 bars × 4 synthetic
2417        // legs = 400k raw trade-events feeding the state machine, though the
2418        // OUTPUT point count — bricks/columns/segments/lines — is typically
2419        // much smaller). We size on the largest per-dep seed length observed,
2420        // never below the Station-wide warm_start_capacity floor.
2421        let ring_capacity_hint = agg_seed_per_dep
2422            .iter()
2423            .map(|v| v.len())
2424            .max()
2425            .unwrap_or(warm_n)
2426            .max(self.inner.warm_start_capacity);
2427
2428        let (bcast_tx, _) = broadcast::channel::<Event>(512);
2429        let consumers = Arc::new(AtomicUsize::new(1));
2430        let (shutdown_tx, shutdown_rx) = oneshot::channel::<()>();
2431        let (seed_done_tx, seed_done_rx) = oneshot::channel::<()>();
2432
2433        spawn_derived_forwarder::<D>(
2434            self,
2435            key,
2436            upstream_rxs,
2437            upstream_keys,
2438            bcast_tx.clone(),
2439            shutdown_rx,
2440            raw_symbol.to_string(),
2441            agg_seed_per_dep,
2442            ring_capacity_hint,
2443            seed_done_tx,
2444            kline_seed_baseline,
2445        );
2446
2447        self.inner.muxes.insert(
2448            key.clone(),
2449            Multiplexer { tx: bcast_tx.clone(), consumers, shutdown: Some(shutdown_tx), grace_cancel: None },
2450        );
2451
2452        // Gate return on the forwarder's cold-start seeding (disk warm-seed +
2453        // agg-seed + RAM warm-seed) so a consumer that immediately peeks
2454        // `Station::series::<D::Output>(key)` after `subscribe()` sees the
2455        // seeded ring instead of an empty one (see docs/plans — pnf_seed_smoke
2456        // regression: peek-then-live read n=0 right after subscribe while the
2457        // ring held thousands of points 20s later). Bounded by a 15s timeout
2458        // so a wedged/slow forwarder (e.g. REST paging stalls) can never hang
2459        // `subscribe()` forever — on timeout or a dropped sender we proceed
2460        // with whatever the ring holds at that moment. Same wasm-safe timer
2461        // idiom as the grace-period race above (`tokio::time::sleep` native /
2462        // `gloo_timers::future::sleep` wasm) since `tokio::time::timeout` is
2463        // not available on wasm32.
2464        const SEED_WAIT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(15);
2465        #[cfg(not(target_arch = "wasm32"))]
2466        {
2467            tokio::select! {
2468                res = seed_done_rx => {
2469                    if res.is_err() {
2470                        tracing::debug!(?key, "derived: seed_done sender dropped before signaling — proceeding anyway");
2471                    }
2472                }
2473                _ = tokio::time::sleep(SEED_WAIT_TIMEOUT) => {
2474                    tracing::debug!(?key, "derived: cold-start seed wait timed out — proceeding with partial/empty seed");
2475                }
2476            }
2477        }
2478        #[cfg(target_arch = "wasm32")]
2479        {
2480            tokio::select! {
2481                res = seed_done_rx => {
2482                    if res.is_err() {
2483                        tracing::debug!(?key, "derived: seed_done sender dropped before signaling — proceeding anyway");
2484                    }
2485                }
2486                _ = gloo_timers::future::sleep(SEED_WAIT_TIMEOUT) => {
2487                    tracing::debug!(?key, "derived: cold-start seed wait timed out — proceeding with partial/empty seed");
2488                }
2489            }
2490        }
2491
2492        Ok(bcast_tx)
2493    }
2494}
2495
2496#[cfg(not(target_arch = "wasm32"))]
2497impl Station {
2498    /// Acquire (or spawn) a poll-driven multiplexer for `key`.
2499    ///
2500    /// Called when `key.kind.is_poll_only()` returns `Some(PollSpec)`. Skips
2501    /// `ws.subscribe` entirely and instead spawns a `spawn_poller<T, S>` actor
2502    /// driven by `tokio::time::interval`.
2503    async fn acquire_or_spawn_polled(
2504        &self,
2505        key: &SeriesKey,
2506        entry: &Entry,
2507        poll_spec: crate::series::PollSpec,
2508        raw_symbol: &str,
2509    ) -> Result<broadcast::Sender<Event>> {
2510        use crate::station::Multiplexer;
2511
2512        let (bcast_tx, _) = broadcast::channel::<Event>(1024);
2513        let consumers = Arc::new(AtomicUsize::new(1));
2514        let (shutdown_tx, shutdown_rx) = oneshot::channel::<()>();
2515        let label = raw_symbol.to_string();
2516
2517        match &key.kind {
2518            Kind::LongShortRatio => {
2519                let source = polling::lsr_poll_source(entry.exchange)
2520                    .ok_or_else(|| StationError::StreamNotSupported(format!(
2521                        "LongShortRatio REST polling not supported for {:?}",
2522                        entry.exchange
2523                    )))?;
2524                polling::spawn_poller::<LongShortRatioPoint, _>(
2525                    self, key, source, poll_spec, bcast_tx.clone(), shutdown_rx, label,
2526                );
2527            }
2528            Kind::HistoricalVolatility => {
2529                let source = polling::hv_poll_source(entry.exchange)
2530                    .ok_or_else(|| StationError::StreamNotSupported(format!(
2531                        "HistoricalVolatility REST polling not supported for {:?} \
2532                         (Deribit only)",
2533                        entry.exchange
2534                    )))?;
2535                polling::spawn_poller::<HistoricalVolatilityPoint, _>(
2536                    self, key, source, poll_spec, bcast_tx.clone(), shutdown_rx, label,
2537                );
2538            }
2539            Kind::TakerVolume => {
2540                let source = polling::taker_volume_poll_source(&self.inner.hub, entry.exchange)
2541                    .ok_or_else(|| StationError::StreamNotSupported(format!(
2542                        "TakerVolume REST polling not supported for {:?}",
2543                        entry.exchange
2544                    )))?;
2545                polling::spawn_poller::<TakerVolumePoint, _>(
2546                    self, key, source, poll_spec, bcast_tx.clone(), shutdown_rx, label,
2547                );
2548            }
2549            Kind::LiquidationBucket => {
2550                let source = polling::liquidation_bucket_poll_source(&self.inner.hub, entry.exchange)
2551                    .ok_or_else(|| StationError::StreamNotSupported(format!(
2552                        "LiquidationBucket REST polling not supported for {:?}",
2553                        entry.exchange
2554                    )))?;
2555                polling::spawn_poller::<LiquidationBucketPoint, _>(
2556                    self, key, source, poll_spec, bcast_tx.clone(), shutdown_rx, label,
2557                );
2558            }
2559            other => {
2560                return Err(StationError::StreamNotSupported(format!(
2561                    "acquire_or_spawn_polled: no poll source for {:?}",
2562                    other
2563                )));
2564            }
2565        }
2566
2567        self.inner.muxes.insert(
2568            key.clone(),
2569            Multiplexer {
2570                tx: bcast_tx.clone(),
2571                consumers,
2572                shutdown: Some(shutdown_tx),
2573                grace_cancel: None,
2574            },
2575        );
2576        Ok(bcast_tx)
2577    }
2578
2579    /// Acquire (or spawn) a poll-driven multiplexer for `key` using an
2580    /// explicitly supplied `PollSource` implementation.
2581    ///
2582    /// Unlike `acquire_or_spawn_polled`, this method does NOT require the
2583    /// `kind` to return `Some` from `is_poll_only()`. It is used for kinds
2584    /// that are normally "derived" (e.g. `Kind::Basis`,
2585    /// `Kind::FundingSettlement`) but have a native REST history endpoint on
2586    /// the current exchange.
2587    async fn acquire_or_spawn_polled_with_source<T, S>(
2588        &self,
2589        key: &SeriesKey,
2590        poll_spec: crate::series::PollSpec,
2591        raw_symbol: &str,
2592        source: S,
2593    ) -> Result<broadcast::Sender<Event>>
2594    where
2595        T: crate::series::DataPoint + 'static,
2596        S: crate::polling::PollSource<T>,
2597        Event: EventFrom<T>,
2598    {
2599        use crate::station::Multiplexer;
2600
2601        let (bcast_tx, _) = broadcast::channel::<Event>(1024);
2602        let consumers = Arc::new(AtomicUsize::new(1));
2603        let (shutdown_tx, shutdown_rx) = oneshot::channel::<()>();
2604        let label = raw_symbol.to_string();
2605
2606        polling::spawn_poller::<T, S>(
2607            self, key, source, poll_spec, bcast_tx.clone(), shutdown_rx, label,
2608        );
2609
2610        self.inner.muxes.insert(
2611            key.clone(),
2612            Multiplexer {
2613                tx: bcast_tx.clone(),
2614                consumers,
2615                shutdown: Some(shutdown_tx),
2616                grace_cancel: None,
2617            },
2618        );
2619        Ok(bcast_tx)
2620    }
2621}
2622
2623impl StationInner {
2624    pub(crate) fn release_consumer(self: &Arc<Self>, key: &SeriesKey) {
2625        let (became_zero, grace) = {
2626            let Some(mux) = self.muxes.get(key) else { return; };
2627            let prev = mux.consumers.fetch_sub(1, Ordering::SeqCst);
2628            (prev <= 1, self.unsubscribe_grace)
2629        };
2630
2631        if !became_zero {
2632            return;
2633        }
2634
2635        if grace.is_zero() {
2636            // Immediate shutdown — existing behaviour.
2637            if let Some((_, mut mux)) = self.muxes.remove(key) {
2638                if let Some(tx) = mux.shutdown.take() {
2639                    let _ = tx.send(());
2640                }
2641            }
2642            return;
2643        }
2644
2645        // Grace period: spawn a timer task. Store a cancel channel in the mux
2646        // so `acquire_or_spawn` can cancel it when a new subscriber arrives.
2647        let (cancel_tx, cancel_rx) = oneshot::channel::<()>();
2648
2649        // Store cancel_tx in the mux before spawning to avoid a race where
2650        // acquire_or_spawn could observe grace_cancel == None before the task
2651        // starts (extremely unlikely but theoretically possible on native).
2652        {
2653            let Some(mut mux) = self.muxes.get_mut(key) else { return; };
2654            mux.grace_cancel = Some(cancel_tx);
2655        }
2656
2657        let inner = Arc::clone(self);
2658        let key = key.clone();
2659
2660        let grace_fut = Box::pin(async move {
2661            // Race: grace timer vs cancel signal from acquire_or_spawn.
2662            #[cfg(not(target_arch = "wasm32"))]
2663            let timed_out = tokio::select! {
2664                _ = cancel_rx => false,
2665                _ = tokio::time::sleep(grace) => true,
2666            };
2667            #[cfg(target_arch = "wasm32")]
2668            let timed_out = tokio::select! {
2669                _ = cancel_rx => false,
2670                _ = gloo_timers::future::sleep(grace) => true,
2671            };
2672
2673            if timed_out {
2674                // Grace expired without a new subscriber — fire shutdown.
2675                // Double-check consumers == 0 as a safety net (the cancel
2676                // channel send happens before fetch_add, so a race that
2677                // increments consumers before we reach here is possible in
2678                // theory; the guard prevents a spurious kill).
2679                let still_zero = inner
2680                    .muxes
2681                    .get(&key)
2682                    .map(|m| m.consumers.load(Ordering::SeqCst) == 0)
2683                    .unwrap_or(false);
2684                if still_zero {
2685                    if let Some((_, mut mux)) = inner.muxes.remove(&key) {
2686                        if let Some(tx) = mux.shutdown.take() {
2687                            let _ = tx.send(());
2688                        }
2689                    }
2690                    inner.series_handles.remove(&key);
2691                }
2692            }
2693        });
2694        #[cfg(not(target_arch = "wasm32"))]
2695        tokio::spawn(grace_fut);
2696        #[cfg(target_arch = "wasm32")]
2697        wasm_bindgen_futures::spawn_local(grace_fut);
2698    }
2699}
2700
2701/// Derived-stream actor. Consumes from N upstream broadcast channels via
2702/// `futures_util::stream::select_all`, runs the `DerivedStream` state machine,
2703/// and emits output to the derived stream's own broadcast channel.
2704///
2705/// On exit (shutdown signal or all upstreams closed):
2706/// - flushes disk store
2707/// - decrements consumer ref-count on each upstream key (RAII propagation)
2708/// - removes own mux entry if no consumers remain
2709fn spawn_derived_forwarder<D: DerivedStream + 'static>(
2710    station: &Station,
2711    key: &SeriesKey,
2712    upstream_rxs: Vec<broadcast::Receiver<Event>>,
2713    upstream_keys: Vec<SeriesKey>,
2714    bcast_tx: broadcast::Sender<Event>,
2715    mut shutdown_rx: oneshot::Receiver<()>,
2716    symbol_label: String,
2717    // Per-dep AggTrade/Trade seed events for P0-C cold-start. One Vec per dep
2718    // in `D::deps()` order. Empty vec = no seed for that dep.
2719    agg_seed_per_dep: Vec<Vec<Event>>,
2720    // Ring capacity for the in-memory Series<D::Output> — sized at spawn time
2721    // to be at least as large as the computed seed (see the call site in
2722    // `acquire_or_spawn_derived_body`), so a deep kline-approx seed is never
2723    // evicted by the ring before the live loop begins. Always
2724    // >= `warm_start_capacity`.
2725    ring_capacity_hint: usize,
2726    // Fired exactly once, right before the live select loop begins (i.e. once
2727    // disk warm-seed + agg-seed + RAM warm-seed have all landed in the ring).
2728    // `acquire_or_spawn_derived_body` awaits this (with a timeout) so
2729    // `Station::subscribe` returns only after cold-start seeding is visible
2730    // to `Station::series::<T>()` peekers. Send-error (receiver already
2731    // dropped, e.g. on timeout) is intentionally ignored.
2732    seed_done_tx: oneshot::Sender<()>,
2733    // Cold-seed → live seam baseline for the kline-sufficient kinds'
2734    // `KlineDeltaState` (see `DerivedStream::seed_kline_baseline`):
2735    // `(open_time, volume, close)` of the LAST kline the cold-seed fetched
2736    // (possibly still-forming). `None` for every other derived kind, and
2737    // for kline-sufficient kinds when the cold-seed fetched zero klines
2738    // (e.g. `warm_n == 0`).
2739    kline_seed_baseline: Option<(i64, f64, f64)>,
2740) where
2741    Event: EventFrom<D::Output>,
2742{
2743    let inner = station.inner.clone();
2744    let key = key.clone();
2745    let storage_root = inner.storage_root.clone();
2746    let persistence = inner.persistence.clone();
2747    let warm = inner.warm_start_capacity;
2748    let exchange = key.exchange;
2749
2750    // Create the shared series arc and register it in series_handles so
2751    // Station::series<T>() can hand it to render-time consumers synchronously
2752    // — same mechanism as spawn_forwarder (~:1893). Without this, a derived
2753    // kind (Renko/PnF/Kagi/3LB/Footprint/CVD/TPO/Basis/FundingSettlement)
2754    // never appears via Station::series<T>() and every re-subscribe re-seeds
2755    // from scratch.
2756    let shared_series: Arc<RwLock<Series<D::Output>>> =
2757        Arc::new(RwLock::new(Series::new(ring_capacity_hint)));
2758    // Type-erase: store Arc<RwLock<Series<D::Output>>> inside an
2759    // Arc<dyn Any+Send+Sync>. The outer Arc is what Any::downcast_ref will
2760    // find the concrete type on.
2761    let erased: Arc<dyn Any + Send + Sync> = Arc::new(Arc::clone(&shared_series));
2762    inner.series_handles.insert(key.clone(), erased);
2763
2764    // Register the exit-ack receiver now, before the derived forwarder body
2765    // starts — same rationale as spawn_forwarder: a caller racing
2766    // `force_unsubscribe_and_await` against this spawn must always find the
2767    // entry once `acquire_or_spawn` has returned.
2768    let (exit_ack_tx, exit_ack_rx) = oneshot::channel::<()>();
2769    inner.exit_acks.insert(key.clone(), exit_ack_rx);
2770
2771    {
2772        let derived_fut = Box::pin(async move {
2773            // Open disk store if persistence is on for this kind (native only).
2774            #[cfg(not(target_arch = "wasm32"))]
2775            let mut disk: Option<DiskStore<D::Output>> = None;
2776            #[cfg(not(target_arch = "wasm32"))]
2777            if persistence.is_enabled_for(&key.kind) {
2778                match DiskStore::<D::Output>::with_idx_every_and_retention(
2779                    &storage_root, key.clone(), 1024, persistence.retention_days,
2780                ).await {
2781                    Ok(store) => disk = Some(store),
2782                    Err(e) => tracing::warn!(?e, ?key, "derived: disk store open failed"),
2783                }
2784            }
2785            // On wasm, no disk store — `warm` is only consumed by the
2786            // native-only `d.read_tail(warm)` call below.
2787            #[cfg(target_arch = "wasm32")]
2788            let _ = (&storage_root, &persistence, &warm);
2789
2790            // Register a flush handle so `Station::flush_persistence()` can
2791            // force this forwarder's DiskStore to drain + flush on demand.
2792            // Native-only — wasm derived forwarders open no disk store (see
2793            // above), so `flush_rx` stays `None` and the select branch below
2794            // is a permanent no-op there, matching current wasm behavior.
2795            #[cfg(not(target_arch = "wasm32"))]
2796            let mut flush_rx = if disk.is_some() {
2797                let (handle, rx) = FlushHandle::channel();
2798                inner.flush_handles.insert(key.clone(), handle);
2799                Some(rx)
2800            } else {
2801                None
2802            };
2803            #[cfg(target_arch = "wasm32")]
2804            let mut flush_rx: Option<mpsc::Receiver<FlushAck>> = None;
2805
2806            // Disk warm-seed (Task C): emit persisted derived bars from the
2807            // previous session BEFORE the live loop begins. Bridges cross-session
2808            // history so consumers see past derived bars without waiting for new
2809            // live derives. Same pattern as spawn_forwarder (polling.rs ~line 145).
2810            #[cfg(not(target_arch = "wasm32"))]
2811            if let Some(d) = disk.as_ref() {
2812                match d.read_tail(warm).await {
2813                    Ok(tail) => {
2814                        for point in &tail {
2815                            shared_series.write().await.upsert_by_ts(point.clone());
2816                            let _ = bcast_tx.send(Event::from_point(
2817                                exchange,
2818                                key.account_type,
2819                                &symbol_label,
2820                                &key.kind,
2821                                point.clone(),
2822                            ));
2823                        }
2824                    }
2825                    Err(e) => tracing::debug!(?e, ?key, "derived: disk warm-seed read_tail failed"),
2826                }
2827            }
2828
2829            let mut state = D::new_for_key(&key);
2830            if let Some((open_time, volume, close)) = kline_seed_baseline {
2831                state.seed_kline_baseline(open_time, volume, close);
2832            }
2833
2834            // AggTrade dense seed (P0-C). Feed pre-fetched REST AggTrade/Trade
2835            // events through the derived state machine, broadcasting emitted bars
2836            // so consumers see the cold-start window immediately. Applied per-dep
2837            // in deps() order before the live loop.
2838            for (dep_idx, seed_events) in agg_seed_per_dep.iter().enumerate() {
2839                if seed_events.is_empty() {
2840                    continue;
2841                }
2842                let emitted = state.seed_from_events(seed_events, dep_idx);
2843                for point in emitted {
2844                    #[cfg(not(target_arch = "wasm32"))]
2845                    if let Some(d) = disk.as_mut() {
2846                        if let Err(e) = d.append(&point) {
2847                            tracing::warn!(?e, ?key, "derived agg-seed: disk append failed");
2848                        }
2849                    }
2850                    shared_series.write().await.upsert_by_ts(point.clone());
2851                    let _ = bcast_tx.send(Event::from_point(
2852                        exchange,
2853                        key.account_type,
2854                        &symbol_label,
2855                        &key.kind,
2856                        point,
2857                    ));
2858                }
2859            }
2860
2861            // RAM warm-seed (Gap C): a trade-derived aggregator (Range/Tick/
2862            // Volume/Footprint bars — deps() == [Trade]) reconstructs its initial
2863            // bar state from the trades ALREADY buffered in the upstream Trade
2864            // Series ring, so subscribing while trades are flowing for another
2865            // consumer (tape / footprint panel / another chart) is not
2866            // cold-empty. We replay the buffered TradePoints through
2867            // `state.on_upstream_event` BEFORE the live loop, then emit/persist
2868            // the resulting bars so the new consumer sees them immediately.
2869            //
2870            // Only applies when the sole dependency is Stream::Trade — Basis
2871            // (MarkPrice+IndexPrice) and FundingSettlement (FundingRate) are NOT
2872            // trade-seeded (replaying trades through them is meaningless). The ring
2873            // is only non-trivial when the Station was built with warm_start(N>0);
2874            // otherwise it holds ≤1 point and the seed is a near-no-op.
2875            if D::deps() == [Stream::Trade] {
2876                if let Some(dep_key) = upstream_keys.first() {
2877                    let trade_handle = inner
2878                        .series_handles
2879                        .get(dep_key)
2880                        .and_then(|e| {
2881                            e.downcast_ref::<Arc<RwLock<Series<TradePoint>>>>().map(Arc::clone)
2882                        });
2883                    if let Some(handle) = trade_handle {
2884                        let buffered = handle.read().await.snapshot(); // oldest→newest
2885                        for pt in buffered {
2886                            let ev = Event::Trade {
2887                                exchange,
2888                                symbol: symbol_label.clone(),
2889                                point: pt,
2890                            };
2891                            if let Some(point) = state.on_upstream_event(&ev, 0) {
2892                                #[cfg(not(target_arch = "wasm32"))]
2893                                if let Some(d) = disk.as_mut() {
2894                                    if let Err(e) = d.append(&point) {
2895                                        tracing::warn!(?e, ?key, "derived seed: disk append failed");
2896                                    }
2897                                }
2898                                shared_series.write().await.upsert_by_ts(point.clone());
2899                                let _ = bcast_tx.send(Event::from_point(
2900                                    exchange,
2901                                    key.account_type,
2902                                    &symbol_label,
2903                                    &key.kind,
2904                                    point,
2905                                ));
2906                            }
2907                        }
2908                    }
2909                }
2910            }
2911
2912            // Cold-start seeding (disk warm-seed + agg-seed + RAM warm-seed)
2913            // is now fully applied to `shared_series` — signal the awaiting
2914            // `acquire_or_spawn_derived_body` so `Station::subscribe` can
2915            // return with the ring already populated. Ignore send error:
2916            // the receiver may have already timed out and dropped.
2917            let _ = seed_done_tx.send(());
2918
2919            // Convert each upstream Receiver into a tagged BroadcastStream so
2920            // the state machine can branch by dep_idx cheaply.
2921            let tagged: Vec<_> = upstream_rxs
2922                .into_iter()
2923                .enumerate()
2924                .map(|(idx, rx)| {
2925                    tokio_stream::wrappers::BroadcastStream::new(rx)
2926                        .filter_map(move |res| async move {
2927                            match res {
2928                                Ok(ev) => Some((idx, ev)),
2929                                Err(tokio_stream::wrappers::errors::BroadcastStreamRecvError::Lagged(n)) => {
2930                                    tracing::warn!(dep_idx = idx, lagged = n, "derived: upstream lagged — events dropped");
2931                                    None
2932                                }
2933                            }
2934                        })
2935                        // Box to make all stream types uniform for select_all.
2936                        .boxed()
2937                })
2938                .collect();
2939
2940            let mut merged = futures_util::stream::select_all(tagged);
2941
2942            loop {
2943                #[cfg(not(target_arch = "wasm32"))]
2944                let item_opt = tokio::select! {
2945                    _ = &mut shutdown_rx => break,
2946                    ack = recv_flush_request(&mut flush_rx) => {
2947                        let result = flush_disk_store(&mut disk).await;
2948                        let _ = ack.send(result);
2949                        continue;
2950                    }
2951                    item = merged.next() => item,
2952                };
2953                // wasm derived forwarders open no disk store, but the flush
2954                // branch is still wired for API uniformity — `flush_rx` is
2955                // always `None` here so it never fires.
2956                #[cfg(target_arch = "wasm32")]
2957                let item_opt = tokio::select! {
2958                    _ = &mut shutdown_rx => break,
2959                    ack = recv_flush_request(&mut flush_rx) => {
2960                        let _ = ack.send(Ok(()));
2961                        continue;
2962                    }
2963                    item = merged.next() => item,
2964                };
2965
2966                let Some((dep_idx, ev)) = item_opt else {
2967                    // All upstreams closed their senders — derived stream is done.
2968                    tracing::info!(?key, "derived: all upstreams closed — exiting");
2969                    break;
2970                };
2971
2972                if let Some(point) = state.on_upstream_event(&ev, dep_idx) {
2973                    #[cfg(not(target_arch = "wasm32"))]
2974                    if let Some(d) = disk.as_mut() {
2975                        if let Err(e) = d.append(&point) {
2976                            tracing::warn!(?e, "derived: disk store append failed");
2977                        }
2978                    }
2979                    shared_series.write().await.push(point.clone());
2980                    let _ = bcast_tx.send(Event::from_point(exchange, key.account_type, &symbol_label, &key.kind, point));
2981                }
2982            }
2983
2984            // Unregister the flush handle BEFORE the final flush — mirrors
2985            // spawn_forwarder's teardown ordering.
2986            #[cfg(not(target_arch = "wasm32"))]
2987            if flush_rx.is_some() {
2988                inner.flush_handles.remove(&key);
2989            }
2990            #[cfg(not(target_arch = "wasm32"))]
2991            if let Some(mut d) = disk { let _ = d.flush().await; }
2992
2993            // Release upstream consumer refs — propagates shutdown upward if the
2994            // derived forwarder was the only consumer of its upstream muxes.
2995            for up_key in &upstream_keys {
2996                inner.release_consumer(up_key);
2997            }
2998
2999            // Remove own mux entry if no consumers remain.
3000            let still_consumers = inner
3001                .muxes
3002                .get(&key)
3003                .map(|m| m.consumers.load(Ordering::SeqCst))
3004                .unwrap_or(0);
3005            if still_consumers == 0 {
3006                inner.muxes.remove(&key);
3007            }
3008            // Remove the series handle so Station::series<T>() returns None once
3009            // the forwarder is gone (same lifecycle as the mux entry). Mirrors
3010            // spawn_forwarder's teardown (~:2159).
3011            inner.series_handles.remove(&key);
3012            // Fire the exit ack LAST — mirrors spawn_forwarder's teardown
3013            // ordering so `force_unsubscribe_and_await` only unblocks once
3014            // mux + series-handle + upstream refs are all torn down.
3015            inner.exit_acks.remove(&key);
3016            let _ = exit_ack_tx.send(());
3017        });
3018        #[cfg(not(target_arch = "wasm32"))]
3019        tokio::spawn(derived_fut);
3020        #[cfg(target_arch = "wasm32")]
3021        wasm_bindgen_futures::spawn_local(derived_fut);
3022    }
3023}
3024
3025/// Await the next flush request from an `Option<mpsc::Receiver<FlushAck>>`.
3026///
3027/// `None` (no `DiskStore` open for this forwarder — persistence disabled or
3028/// open failed) never resolves, so this branch is effectively disabled in
3029/// the enclosing `select!` without needing a separate `if`-guarded arm.
3030/// `Some(rx)` whose channel has since closed (should not happen — the
3031/// forwarder itself owns `rx` for its own lifetime) also never resolves;
3032/// `flush_persistence` treats a closed *sender* as "already gone" on its own
3033/// side, so this is purely defensive.
3034pub(crate) async fn recv_flush_request(rx: &mut Option<mpsc::Receiver<FlushAck>>) -> FlushAck {
3035    match rx {
3036        Some(rx) => match rx.recv().await {
3037            Some(ack) => ack,
3038            None => std::future::pending().await,
3039        },
3040        None => std::future::pending().await,
3041    }
3042}
3043
3044/// Flush an open `DiskStore<T>`, normalizing both targets' error types to
3045/// `std::io::Result<()>`. `None` (no store open) is treated as a no-op success.
3046pub(crate) async fn flush_disk_store<T: DataPoint>(disk: &mut Option<DiskStore<T>>) -> std::io::Result<()> {
3047    match disk {
3048        Some(d) => {
3049            #[cfg(not(target_arch = "wasm32"))]
3050            {
3051                d.flush().await
3052            }
3053            #[cfg(target_arch = "wasm32")]
3054            {
3055                d.flush().await.map_err(std::io::Error::from)
3056            }
3057        }
3058        None => Ok(()),
3059    }
3060}
3061
3062/// Generic per-kind forwarder. Owns:
3063/// - DiskStore<T> (Option; on if persistence enabled),
3064/// - in-memory Series<T> (capacity = warm_start_capacity, kept as scratch),
3065/// - WS event stream.
3066///
3067/// On spawn: emits warm-start tail from DiskStore (if any) as `Event`s to
3068/// broadcast. Then transitions to live mode: each StreamEvent → DataPoint::from
3069/// → write disk → push memory → emit broadcast Event.
3070fn spawn_forwarder<T: DataPoint + 'static>(
3071    station: &Station,
3072    key: &SeriesKey,
3073    ws: Arc<dyn digdigdig3::core::traits::WebSocketConnector>,
3074    bcast_tx: broadcast::Sender<Event>,
3075    mut shutdown_rx: oneshot::Receiver<()>,
3076    symbol_label: String,
3077    // REST-backfill seed used when on-disk history is empty. Empty Vec
3078    // disables the REST fallback.
3079    rest_seed: Vec<T>,
3080    // Original subscribe request. Held so the forwarder can issue
3081    // unsubscribe + subscribe on disconnect to force a fresh subscription
3082    // state at the exchange.
3083    sub_req: SubscriptionRequest,
3084) where
3085    Event: EventFrom<T>,
3086{
3087    let inner = station.inner.clone();
3088    let key = key.clone();
3089    let storage_root = inner.storage_root.clone();
3090    let persistence = inner.persistence.clone();
3091    let warm = inner.warm_start_capacity;
3092    let exchange = key.exchange;
3093    let gap_cfg = inner.gap_heal;
3094    let hub_for_heal = inner.hub.clone();
3095
3096    // Create the shared series arc and register it in series_handles so
3097    // Station::series<T>() can hand it to render-time consumers synchronously.
3098    let shared_series: Arc<RwLock<Series<T>>> = Arc::new(RwLock::new(Series::new(warm)));
3099    // Type-erase: store Arc<RwLock<Series<T>>> inside an Arc<dyn Any+Send+Sync>.
3100    // The outer Arc is what Any::downcast_ref will find the concrete type on.
3101    let erased: Arc<dyn Any + Send + Sync> = Arc::new(Arc::clone(&shared_series));
3102    inner.series_handles.insert(key.clone(), erased);
3103
3104    // Register the exit-ack receiver now, before the forwarder body starts —
3105    // mirrors series_handles: a caller racing `force_unsubscribe_and_await`
3106    // against this very spawn must always find the entry once `acquire_or_spawn`
3107    // has returned.
3108    let (exit_ack_tx, exit_ack_rx) = oneshot::channel::<()>();
3109    inner.exit_acks.insert(key.clone(), exit_ack_rx);
3110
3111    {
3112    let forwarder_fut = Box::pin(async move {
3113        // Open disk store if persistence is on for this kind.
3114        // Native: std::fs-backed DiskStore. Wasm: OPFS-backed DiskStore.
3115        #[cfg(not(target_arch = "wasm32"))]
3116        let mut disk: Option<DiskStore<T>> = None;
3117        #[cfg(not(target_arch = "wasm32"))]
3118        if persistence.is_enabled_for(&key.kind) {
3119            match DiskStore::<T>::with_idx_every_and_retention(
3120                &storage_root, key.clone(), 1024, persistence.retention_days,
3121            ).await {
3122                Ok(store) => disk = Some(store),
3123                Err(e) => tracing::warn!(?e, ?key, "disk store open failed"),
3124            }
3125        }
3126        // Wasm: OPFS DiskStore (Wave 4-E).
3127        // TODO(wasm): sweep_retention — OPFS DiskStore has no retention sweep
3128        // yet (native-only for now, see series/store.rs task notes). Entry
3129        // removal on OPFS is not wired here; `persistence.retention_days` is
3130        // read on native only.
3131        #[cfg(target_arch = "wasm32")]
3132        let mut disk: Option<DiskStore<T>> = None;
3133        #[cfg(target_arch = "wasm32")]
3134        if persistence.is_enabled_for(&key.kind) {
3135            match DiskStore::<T>::new(key.clone()).await {
3136                Ok(store) => disk = Some(store),
3137                Err(e) => tracing::warn!(?e, ?key, "wasm OPFS disk store open failed"),
3138            }
3139        }
3140        // Suppress unused warning on wasm when persistence is disabled.
3141        #[cfg(target_arch = "wasm32")]
3142        let _ = &storage_root;
3143
3144        // Register a flush handle so `Station::flush_persistence()` can force
3145        // this forwarder's DiskStore to drain + flush on demand (chiefly for
3146        // wasm OPFS — see `flush_persistence` doc comment). Only registered
3147        // when persistence actually opened a store; unregistered right
3148        // before the forwarder exits.
3149        let mut flush_rx = if disk.is_some() {
3150            let (handle, rx) = FlushHandle::channel();
3151            inner.flush_handles.insert(key.clone(), handle);
3152            Some(rx)
3153        } else {
3154            None
3155        };
3156
3157        // In-memory ring (warm capacity) — shared with Station::series<T>().
3158        // The forwarder is the sole writer; render-time consumers hold read
3159        // guards for snapshot access without awaiting an Event.
3160        let mut last_emitted_ms: i64 = 0;
3161
3162        // Warm-start. Priority: disk tail > REST seed.
3163        if warm > 0 {
3164            // Wave 4-D: both native and wasm read real disk/OPFS tail.
3165            let disk_tail: Vec<T> = if let Some(d) = disk.as_ref() {
3166                d.read_tail(warm).await.unwrap_or_default()
3167            } else {
3168                Vec::new()
3169            };
3170            let seed_points: Vec<T> = if disk_tail.is_empty() && !rest_seed.is_empty() {
3171                rest_seed
3172            } else {
3173                disk_tail
3174            };
3175            for p in &seed_points {
3176                let _ = bcast_tx.send(Event::from_point(exchange, key.account_type, &symbol_label, &key.kind, p.clone()));
3177                last_emitted_ms = last_emitted_ms.max(p.timestamp_ms());
3178            }
3179            shared_series.write().await.seed(seed_points);
3180        }
3181
3182        let mut stream = ws.event_stream();
3183        // Silence threshold: if `event_stream().next()` produces no event for
3184        // this long, the underlying WS is presumed dropped. Mirrors MLC
3185        // `ws_manager` behaviour (60s). Tunable via env for tests.
3186        #[cfg(not(target_arch = "wasm32"))]
3187        let silence_timeout = std::time::Duration::from_secs(
3188            std::env::var("DIG3_WS_SILENCE_SECS").ok().and_then(|s| s.parse().ok()).unwrap_or(60),
3189        );
3190        // Wasm (Wave 4-F): 60s silence watchdog via gloo_timers.
3191        // DIG3_WS_SILENCE_SECS is not readable on wasm32 (std::env absent);
3192        // default 60 s is hardcoded. Configurable at compile time if needed.
3193        #[cfg(target_arch = "wasm32")]
3194        let silence_timeout_ms: u32 = 60_000;
3195        // Debug-only: artificially slow down the per-event loop. Used by e2e
3196        // tests to force broadcast-channel overflow → `Lagged` error →
3197        // `stream_err` branch. Production callers leave this unset (0 ms).
3198        #[cfg(not(target_arch = "wasm32"))]
3199        let debug_slow_ms: u64 = std::env::var("DIG3_DEBUG_SLOW_CONSUMER_MS")
3200            .ok()
3201            .and_then(|s| s.parse().ok())
3202            .unwrap_or(0);
3203        // Wasm (Wave 4-E): flush OPFS every N events to bound data loss.
3204        // A periodic gloo interval is not used here — instead a simple
3205        // flush-every-N-appends strategy matches the forwarder's sync
3206        // append pattern without an extra concurrent task.
3207        #[cfg(target_arch = "wasm32")]
3208        let mut wasm_flush_counter: u32 = 0;
3209        #[cfg(target_arch = "wasm32")]
3210        const WASM_FLUSH_EVERY: u32 = 64;
3211
3212        loop {
3213            // Single select arm — exits via shutdown or detects disconnect
3214            // (None / Err / silence). All three cases = disconnect = heal.
3215            // Native: tokio::time::timeout for silence detection.
3216            // Wasm (4-F): gloo_timers::future::sleep race for silence detection.
3217            #[cfg(not(target_arch = "wasm32"))]
3218            let item_opt = tokio::select! {
3219                _ = &mut shutdown_rx => break,
3220                ack = recv_flush_request(&mut flush_rx) => {
3221                    let result = flush_disk_store(&mut disk).await;
3222                    let _ = ack.send(result);
3223                    continue;
3224                }
3225                res = tokio::time::timeout(silence_timeout, stream.next()) => res,
3226            };
3227            #[cfg(target_arch = "wasm32")]
3228            // On wasm: race stream.next() against a gloo_timers sleep.
3229            // Returns Ok(Some(...)) for a real event, Ok(None) for stream end,
3230            // Err(()) for the silence timeout expiring.
3231            let item_opt: std::result::Result<
3232                Option<std::result::Result<_, digdigdig3::core::types::WebSocketError>>,
3233                (),
3234            > = tokio::select! {
3235                _ = &mut shutdown_rx => break,
3236                ack = recv_flush_request(&mut flush_rx) => {
3237                    let result = flush_disk_store(&mut disk).await;
3238                    let _ = ack.send(result);
3239                    continue;
3240                }
3241                _ = gloo_timers::future::sleep(std::time::Duration::from_millis(silence_timeout_ms as u64)) => Err(()),
3242                item = stream.next() => Ok(item),
3243            };
3244
3245            let trigger_heal_reason: Option<&'static str> = match &item_opt {
3246                Err(_) => Some("silence_timeout"),
3247                Ok(None) => Some("stream_ended"),
3248                Ok(Some(Err(_))) => Some("stream_err"),
3249                Ok(Some(Ok(_))) => None,
3250            };
3251
3252            if let Some(reason) = trigger_heal_reason {
3253                // Heal + resub is kline-only. For non-kline kinds:
3254                // - REST cannot bridge the gap (no public endpoint for
3255                //   trade/OB/ticker/mark/funding/OI/liq history live-feed).
3256                // - Resub spam on a WireAbsent stream was the trigger for
3257                //   MLI's 0.3.6 OOM — see release-0.3.7-plan.md.
3258                // - The transport-level UniversalWsTransport auto-reconnects
3259                //   internally; the forwarder does not need to resub manually.
3260                //
3261                // Non-kline behavior: log + exit the forwarder. The mux entry
3262                // is removed below so a later subscribe for the same key can
3263                // re-spawn cleanly.
3264                let is_kline_family = matches!(
3265                    &key.kind,
3266                    Kind::Kline(_) | Kind::MarkPriceKline(_)
3267                    | Kind::IndexPriceKline(_) | Kind::PremiumIndexKline(_)
3268                );
3269
3270                if !is_kline_family {
3271                    tracing::info!(
3272                        target: "dig3::gap_heal",
3273                        ?key, reason,
3274                        "non-kline stream disconnect — forwarder exiting (no resub for non-kline kinds)"
3275                    );
3276                    break;
3277                }
3278
3279                tracing::info!(target: "dig3::gap_heal", ?key, reason, "ws disconnect detected → heal + resub");
3280                // 1. REST heal (kline-only; no-op for non-kline kinds, which
3281                //    have already returned above). Wave 4-B: enabled on both
3282                //    targets. On wasm REST succeeds for the 9 proxy-override
3283                //    venues; silently returns empty for others until Wave 4-C.
3284                {
3285                    let mut series_guard = shared_series.write().await;
3286                    run_kline_heal::<T>(
3287                        &hub_for_heal, &key, &gap_cfg, &symbol_label,
3288                        last_emitted_ms, exchange,
3289                        &mut *series_guard, &mut disk, &bcast_tx,
3290                    ).await;
3291                    last_emitted_ms = last_emitted_ms.max(
3292                        series_guard.last().map(|p| p.timestamp_ms()).unwrap_or(0)
3293                    );
3294                }
3295                // 2. Force a fresh subscription state at the exchange.
3296                //    Unsubscribe is best-effort (the server may have already
3297                //    dropped us). Resubscribe must succeed or we log + retry
3298                //    on the next disconnect cycle.
3299                let unsub_res = ws.unsubscribe(sub_req.clone()).await;
3300                let sub_res = ws.subscribe(sub_req.clone()).await;
3301                tracing::info!(
3302                    target: "dig3::gap_heal",
3303                    ?key,
3304                    unsub_ok = unsub_res.is_ok(),
3305                    sub_ok = sub_res.is_ok(),
3306                    "resub cycle complete"
3307                );
3308                if let Err(e) = unsub_res {
3309                    tracing::debug!(target: "dig3::gap_heal", ?key, ?e, "unsubscribe failed (best-effort)");
3310                }
3311                if let Err(e) = sub_res {
3312                    tracing::warn!(target: "dig3::gap_heal", ?key, ?e, "resubscribe failed");
3313                }
3314                // 3. Re-attach broadcast receiver — picks up post-resub events.
3315                //    Explicit drop of the old stream first: BroadcastStream
3316                //    holds a Receiver whose internal ring buffer occupies
3317                //    `event_tx` capacity until dropped. Letting the old one
3318                //    go before subscribing a new one keeps receiver_count
3319                //    minimal across heal cycles.
3320                drop(stream);
3321                stream = ws.event_stream();
3322                continue;
3323            }
3324
3325            let ev = match item_opt {
3326                Ok(Some(Ok(ev))) => ev,
3327                _ => unreachable!(),
3328            };
3329
3330            if !event_matches_key(&ev, &key) {
3331                continue;
3332            }
3333            let Some(point) = T::from_stream_event(&ev) else {
3334                continue;
3335            };
3336
3337            // Wave 4-E: append on both targets (native std::fs, wasm OPFS).
3338            // Native append returns Result; wasm append returns () (infallible — buffers in memory).
3339            #[cfg(not(target_arch = "wasm32"))]
3340            if let Some(d) = disk.as_mut() {
3341                if let Err(e) = d.append(&point) {
3342                    tracing::warn!(?e, "disk store append failed");
3343                }
3344            }
3345            #[cfg(target_arch = "wasm32")]
3346            if let Some(d) = disk.as_mut() {
3347                d.append(&point);
3348            }
3349            // Wasm (Wave 4-E): flush OPFS buffer periodically to bound data loss.
3350            #[cfg(target_arch = "wasm32")]
3351            {
3352                wasm_flush_counter = wasm_flush_counter.wrapping_add(1);
3353                if wasm_flush_counter % WASM_FLUSH_EVERY == 0 {
3354                    if let Some(d) = disk.as_mut() {
3355                        if let Err(e) = d.flush().await {
3356                            tracing::warn!(?e, "wasm OPFS periodic flush failed");
3357                        }
3358                    }
3359                }
3360            }
3361            let pt_ts = point.timestamp_ms();
3362            // Klines: multiple in-flight updates share open_time — upsert
3363            // keeps the ring deduplicated. Other kinds are monotonic.
3364            {
3365                let mut series_guard = shared_series.write().await;
3366                if matches!(&key.kind, Kind::Kline(_) | Kind::MarkPriceKline(_) | Kind::IndexPriceKline(_) | Kind::PremiumIndexKline(_)) {
3367                    series_guard.upsert_by_ts(point.clone());
3368                } else {
3369                    series_guard.push(point.clone());
3370                }
3371            }
3372            last_emitted_ms = last_emitted_ms.max(pt_ts);
3373
3374            let _ = bcast_tx.send(Event::from_point(exchange, key.account_type, &symbol_label, &key.kind, point));
3375
3376            #[cfg(not(target_arch = "wasm32"))]
3377            if debug_slow_ms > 0 {
3378                tokio::time::sleep(std::time::Duration::from_millis(debug_slow_ms)).await;
3379            }
3380        }
3381
3382        // Unregister the flush handle BEFORE the final flush — once this
3383        // entry is gone, `flush_persistence` will no longer try to reach
3384        // this (about-to-exit) forwarder and will treat it as already flushed.
3385        // The final flush below still runs unconditionally right after.
3386        if flush_rx.is_some() {
3387            inner.flush_handles.remove(&key);
3388        }
3389        // Final flush on both targets (Wave 4-E: wasm flushes OPFS on shutdown).
3390        if let Some(mut d) = disk { let _ = d.flush().await; }
3391        // Remove the mux entry so a subsequent `subscribe` for the same key
3392        // can re-spawn a fresh forwarder. Without this, the dead mux would
3393        // sit in `inner.muxes`, and re-subscribe would attach to a broadcast
3394        // tx whose forwarder has already exited (no events ever arrive).
3395        //
3396        // Only remove if no consumer is left attached — otherwise a still-
3397        // alive consumer would think it has a working stream while we
3398        // silently tore it down. (`release_consumer` already removes on
3399        // refcount==0; here we cover the other path: forwarder ended on
3400        // its own before all consumers dropped.)
3401        let still_consumers = inner
3402            .muxes
3403            .get(&key)
3404            .map(|m| m.consumers.load(Ordering::SeqCst))
3405            .unwrap_or(0);
3406        if still_consumers == 0 {
3407            inner.muxes.remove(&key);
3408        }
3409        // Remove the series handle so Station::series<T>() returns None once
3410        // the forwarder is gone (same lifecycle as the mux entry).
3411        inner.series_handles.remove(&key);
3412        // Fire the exit ack LAST — after every other teardown step has
3413        // completed, so a caller awaiting `force_unsubscribe_and_await` only
3414        // unblocks once the mux and series-handle entries are already gone.
3415        // `force_unsubscribe_and_await` itself already removed this entry
3416        // before firing shutdown; a plain grace/refcount exit (the common
3417        // case) removes it here instead. Ignore send error: no awaiter
3418        // present is normal.
3419        inner.exit_acks.remove(&key);
3420        let _ = exit_ack_tx.send(());
3421    });
3422    #[cfg(not(target_arch = "wasm32"))]
3423    tokio::spawn(forwarder_fut);
3424    #[cfg(target_arch = "wasm32")]
3425    wasm_bindgen_futures::spawn_local(forwarder_fut);
3426    }
3427}
3428
3429/// Run a kline auto-heal triggered by WS disconnect. Called only for
3430/// `Kind::Kline`; no-op for other kinds. Wave 4-B: enabled on both targets.
3431async fn run_kline_heal<T: DataPoint + 'static>(
3432    hub: &Arc<ExchangeHub>,
3433    key: &SeriesKey,
3434    cfg: &crate::GapHealConfig,
3435    symbol_label: &str,
3436    last_emitted_ms: i64,
3437    exchange: digdigdig3::core::types::ExchangeId,
3438    series: &mut Series<T>,
3439    disk: &mut Option<DiskStore<T>>,
3440    bcast_tx: &broadcast::Sender<Event>,
3441) where
3442    Event: EventFrom<T>,
3443{
3444    if !cfg.enabled {
3445        return;
3446    }
3447    let Kind::Kline(iv) = &key.kind else { return; };
3448
3449    let now_ms = chrono::Utc::now().timestamp_millis();
3450    let limit = crate::gap_heal::heal_limit(cfg, iv.as_str(), last_emitted_ms, now_ms);
3451
3452    tracing::info!(
3453        target: "dig3::gap_heal",
3454        ?key,
3455        last_emitted_ms,
3456        limit,
3457        "kline heal: pulling REST"
3458    );
3459
3460    let pulled: Vec<T> = cast_vec(
3461        crate::gap_heal::heal_klines(hub, key.exchange, key.account_type, &key.symbol, iv.as_str(), last_emitted_ms, limit).await
3462    );
3463    let pulled_count = pulled.len();
3464
3465    // ALL pulled bars get upserted (last-write-wins overwrites any in-flight
3466    // broken live bar). Only bars strictly newer than last_emitted_ms are
3467    // EMITTED to consumers (the older ones already reached them as live).
3468    let new_to_emit = crate::gap_heal::select_heal_window(pulled.clone(), last_emitted_ms);
3469    let emitted_count = new_to_emit.len();
3470
3471    for p in pulled {
3472        if let Some(d) = disk.as_mut() {
3473            let _ = d.append(&p);
3474        }
3475        series.upsert_by_ts(p);
3476    }
3477    for p in new_to_emit {
3478        let _ = bcast_tx.send(Event::from_point(exchange, key.account_type, symbol_label, &key.kind, p));
3479    }
3480    if let Some(d) = disk.as_mut() { let _ = d.flush().await; }
3481
3482    tracing::info!(
3483        target: "dig3::gap_heal",
3484        ?key,
3485        pulled_count,
3486        emitted_count,
3487        "kline heal: applied"
3488    );
3489}
3490
3491/// Cast `Vec<A>` to `Vec<B>` when A == B at runtime via TypeId. Used to
3492/// bridge the kind-specific REST return type (`Vec<BarPoint>`) back to the
3493/// generic forwarder's `Vec<T>`. Safe when called at a site where the kind
3494/// match arm guarantees A == B.
3495fn cast_vec<A: 'static, B: 'static>(v: Vec<A>) -> Vec<B> {
3496    if std::any::TypeId::of::<A>() == std::any::TypeId::of::<B>() {
3497        // SAFETY: confirmed TypeId equality immediately above; memory layout
3498        // and ownership are identical between `Vec<A>` and `Vec<B>`.
3499        let mut v = std::mem::ManuallyDrop::new(v);
3500        let (ptr, len, cap) = (v.as_mut_ptr() as *mut B, v.len(), v.capacity());
3501        unsafe { Vec::from_raw_parts(ptr, len, cap) }
3502    } else {
3503        Vec::new()
3504    }
3505}
3506
3507/// Symbol-level routing: drop events whose `symbol` field doesn't match our key.
3508/// Every public-data variant now carries `symbol: String` on the variant itself.
3509fn event_matches_key(ev: &StreamEvent, key: &SeriesKey) -> bool {
3510    let want = key.symbol.as_str();
3511    let got: Option<&str> = event_raw_symbol(ev);
3512    match got {
3513        // Empty string = parser couldn't extract; let event through (dispatch is by SeriesKey at the channel level).
3514        Some("") => true,
3515        Some(s) => s.eq_ignore_ascii_case(want),
3516        None => true,
3517    }
3518}
3519
3520/// Extract the raw exchange-native symbol carried on a `StreamEvent` variant,
3521/// or `None` for private events that don't carry one in this dispatch model.
3522fn event_raw_symbol(ev: &StreamEvent) -> Option<&str> {
3523    match ev {
3524        StreamEvent::Trade { symbol, .. } => Some(symbol),
3525        StreamEvent::AggTrade { symbol, .. } => Some(symbol),
3526        StreamEvent::Ticker { symbol, .. } => Some(symbol),
3527        StreamEvent::Kline { symbol, .. } => Some(symbol),
3528        StreamEvent::OrderbookSnapshot { symbol, .. } => Some(symbol),
3529        StreamEvent::OrderbookDelta { symbol, .. } => Some(symbol),
3530        StreamEvent::MarkPrice { symbol, .. } => Some(symbol),
3531        StreamEvent::FundingRate { symbol, .. } => Some(symbol),
3532        StreamEvent::OpenInterestUpdate { symbol, .. } => Some(symbol),
3533        StreamEvent::Liquidation { symbol, .. } => Some(symbol),
3534        StreamEvent::LongShortRatio { symbol, .. } => Some(symbol),
3535        StreamEvent::MarkPriceKline { symbol, .. } => Some(symbol),
3536        StreamEvent::IndexPriceKline { symbol, .. } => Some(symbol),
3537        StreamEvent::PremiumIndexKline { symbol, .. } => Some(symbol),
3538        StreamEvent::IndexPrice { symbol, .. } => Some(symbol),
3539        StreamEvent::HistoricalVolatility { symbol, .. } => Some(symbol),
3540        StreamEvent::InsuranceFund { symbol, .. } => Some(symbol),
3541        StreamEvent::Basis { symbol, .. } => Some(symbol),
3542        StreamEvent::OptionGreeks { symbol, .. } => Some(symbol),
3543        StreamEvent::VolatilityIndex { symbol, .. } => Some(symbol),
3544        StreamEvent::BlockTrade { symbol, .. } => Some(symbol),
3545        StreamEvent::AuctionEvent { symbol, .. } => Some(symbol),
3546        StreamEvent::MarketWarning { symbol, .. } => symbol.as_deref(),
3547        StreamEvent::OrderbookL3 { symbol, .. } => Some(symbol),
3548        StreamEvent::SettlementEvent { symbol, .. } => Some(symbol),
3549        StreamEvent::RiskLimit { symbol, .. } => Some(symbol),
3550        StreamEvent::PredictedFunding { symbol, .. } => Some(symbol),
3551        StreamEvent::FundingSettlement { symbol, .. } => Some(symbol),
3552        StreamEvent::CompositeIndex { symbol, .. } => Some(symbol),
3553        // Private events — private dispatch isn't symbol-routed at the SeriesKey level.
3554        StreamEvent::OrderUpdate { symbol: _, event: _ }
3555        | StreamEvent::BalanceUpdate(_)
3556        | StreamEvent::PositionUpdate { symbol: _, event: _ } => None,
3557        // Batch is flattened by the transport-layer dispatcher before reaching station;
3558        // data modules see only leaf variants. This arm exists for exhaustiveness only.
3559        StreamEvent::Batch(_) => None,
3560    }
3561}
3562
3563/// Trait wired by each `DataPoint` so the forwarder can build the right Event
3564/// variant. `pub(crate)` so `polling.rs` can use it in `spawn_poller`.
3565pub(crate) trait EventFrom<T> {
3566    fn from_point(
3567        exchange: digdigdig3::core::types::ExchangeId,
3568        account_type: digdigdig3::core::types::AccountType,
3569        symbol: &str,
3570        kind: &Kind,
3571        p: T,
3572    ) -> Self;
3573}
3574
3575impl EventFrom<TradePoint> for Event {
3576    fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: TradePoint) -> Self {
3577        Event::Trade { exchange, symbol: symbol.to_string(), point }
3578    }
3579}
3580impl EventFrom<AggTradePoint> for Event {
3581    fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: AggTradePoint) -> Self {
3582        Event::AggTrade { exchange, symbol: symbol.to_string(), point }
3583    }
3584}
3585impl EventFrom<BarPoint> for Event {
3586    fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, kind: &Kind, point: BarPoint) -> Self {
3587        let timeframe = match kind { Kind::Kline(iv) => iv.clone(), _ => KlineInterval::new("") };
3588        Event::Bar { exchange, symbol: symbol.to_string(), timeframe, point }
3589    }
3590}
3591impl EventFrom<TickerPoint> for Event {
3592    fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: TickerPoint) -> Self {
3593        Event::Ticker { exchange, symbol: symbol.to_string(), point }
3594    }
3595}
3596impl EventFrom<ObSnapshotPoint> for Event {
3597    fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: ObSnapshotPoint) -> Self {
3598        Event::OrderbookSnapshot { exchange, symbol: symbol.to_string(), point }
3599    }
3600}
3601impl EventFrom<ObDeltaPoint> for Event {
3602    fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: ObDeltaPoint) -> Self {
3603        Event::OrderbookDelta { exchange, symbol: symbol.to_string(), point }
3604    }
3605}
3606impl EventFrom<MarkPricePoint> for Event {
3607    fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: MarkPricePoint) -> Self {
3608        Event::MarkPrice { exchange, symbol: symbol.to_string(), point }
3609    }
3610}
3611impl EventFrom<FundingRatePoint> for Event {
3612    fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: FundingRatePoint) -> Self {
3613        Event::FundingRate { exchange, symbol: symbol.to_string(), point }
3614    }
3615}
3616impl EventFrom<OpenInterestPoint> for Event {
3617    fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: OpenInterestPoint) -> Self {
3618        Event::OpenInterest { exchange, symbol: symbol.to_string(), point }
3619    }
3620}
3621impl EventFrom<LiquidationPoint> for Event {
3622    fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: LiquidationPoint) -> Self {
3623        Event::Liquidation { exchange, symbol: symbol.to_string(), point }
3624    }
3625}
3626impl EventFrom<BlockTradePoint> for Event {
3627    fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: BlockTradePoint) -> Self {
3628        Event::BlockTrade { exchange, symbol: symbol.to_string(), point }
3629    }
3630}
3631impl EventFrom<AuctionEventPoint> for Event {
3632    fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: AuctionEventPoint) -> Self {
3633        Event::AuctionEvent { exchange, symbol: symbol.to_string(), point }
3634    }
3635}
3636impl EventFrom<IndexPricePoint> for Event {
3637    fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: IndexPricePoint) -> Self {
3638        Event::IndexPrice { exchange, symbol: symbol.to_string(), point }
3639    }
3640}
3641impl EventFrom<CompositeIndexPoint> for Event {
3642    fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: CompositeIndexPoint) -> Self {
3643        Event::CompositeIndex { exchange, symbol: symbol.to_string(), point }
3644    }
3645}
3646impl EventFrom<OptionGreeksPoint> for Event {
3647    fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: OptionGreeksPoint) -> Self {
3648        Event::OptionGreeks { exchange, symbol: symbol.to_string(), point }
3649    }
3650}
3651impl EventFrom<VolatilityIndexPoint> for Event {
3652    fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: VolatilityIndexPoint) -> Self {
3653        Event::VolatilityIndex { exchange, symbol: symbol.to_string(), point }
3654    }
3655}
3656impl EventFrom<HistoricalVolatilityPoint> for Event {
3657    fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: HistoricalVolatilityPoint) -> Self {
3658        Event::HistoricalVolatility { exchange, symbol: symbol.to_string(), point }
3659    }
3660}
3661impl EventFrom<LongShortRatioPoint> for Event {
3662    fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: LongShortRatioPoint) -> Self {
3663        Event::LongShortRatio { exchange, symbol: symbol.to_string(), point }
3664    }
3665}
3666impl EventFrom<TakerVolumePoint> for Event {
3667    fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: TakerVolumePoint) -> Self {
3668        Event::TakerVolume { exchange, symbol: symbol.to_string(), point }
3669    }
3670}
3671impl EventFrom<LiquidationBucketPoint> for Event {
3672    fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: LiquidationBucketPoint) -> Self {
3673        Event::LiquidationBucket { exchange, symbol: symbol.to_string(), point }
3674    }
3675}
3676impl EventFrom<BasisPoint> for Event {
3677    fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: BasisPoint) -> Self {
3678        Event::Basis { exchange, symbol: symbol.to_string(), point }
3679    }
3680}
3681impl EventFrom<InsuranceFundPoint> for Event {
3682    fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: InsuranceFundPoint) -> Self {
3683        Event::InsuranceFund { exchange, symbol: symbol.to_string(), point }
3684    }
3685}
3686impl EventFrom<OrderbookL3Point> for Event {
3687    fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: OrderbookL3Point) -> Self {
3688        Event::OrderbookL3 { exchange, symbol: symbol.to_string(), point }
3689    }
3690}
3691impl EventFrom<SettlementEventPoint> for Event {
3692    fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: SettlementEventPoint) -> Self {
3693        Event::SettlementEvent { exchange, symbol: symbol.to_string(), point }
3694    }
3695}
3696impl EventFrom<MarketWarningPoint> for Event {
3697    fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: MarketWarningPoint) -> Self {
3698        Event::MarketWarning { exchange, symbol: symbol.to_string(), point }
3699    }
3700}
3701impl EventFrom<RiskLimitPoint> for Event {
3702    fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: RiskLimitPoint) -> Self {
3703        Event::RiskLimit { exchange, symbol: symbol.to_string(), point }
3704    }
3705}
3706impl EventFrom<PredictedFundingPoint> for Event {
3707    fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: PredictedFundingPoint) -> Self {
3708        Event::PredictedFunding { exchange, symbol: symbol.to_string(), point }
3709    }
3710}
3711impl EventFrom<FundingSettlementPoint> for Event {
3712    fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: FundingSettlementPoint) -> Self {
3713        Event::FundingSettlement { exchange, symbol: symbol.to_string(), point }
3714    }
3715}
3716impl EventFrom<MarkPriceKlinePoint> for Event {
3717    fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, kind: &Kind, point: MarkPriceKlinePoint) -> Self {
3718        let timeframe = match kind { Kind::MarkPriceKline(iv) => iv.clone(), _ => KlineInterval::new("") };
3719        Event::MarkPriceKline { exchange, symbol: symbol.to_string(), timeframe, point }
3720    }
3721}
3722impl EventFrom<IndexPriceKlinePoint> for Event {
3723    fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, kind: &Kind, point: IndexPriceKlinePoint) -> Self {
3724        let timeframe = match kind { Kind::IndexPriceKline(iv) => iv.clone(), _ => KlineInterval::new("") };
3725        Event::IndexPriceKline { exchange, symbol: symbol.to_string(), timeframe, point }
3726    }
3727}
3728impl EventFrom<PremiumIndexKlinePoint> for Event {
3729    fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, kind: &Kind, point: PremiumIndexKlinePoint) -> Self {
3730        let timeframe = match kind { Kind::PremiumIndexKline(iv) => iv.clone(), _ => KlineInterval::new("") };
3731        Event::PremiumIndexKline { exchange, symbol: symbol.to_string(), timeframe, point }
3732    }
3733}
3734impl EventFrom<OrderUpdatePoint> for Event {
3735    fn from_point(exchange: digdigdig3::core::types::ExchangeId, account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: OrderUpdatePoint) -> Self {
3736        Event::OrderUpdate { exchange, account_type, symbol: symbol.to_string(), point }
3737    }
3738}
3739impl EventFrom<BalanceUpdatePoint> for Event {
3740    fn from_point(exchange: digdigdig3::core::types::ExchangeId, account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: BalanceUpdatePoint) -> Self {
3741        Event::BalanceUpdate { exchange, account_type, symbol: symbol.to_string(), point }
3742    }
3743}
3744impl EventFrom<PositionUpdatePoint> for Event {
3745    fn from_point(exchange: digdigdig3::core::types::ExchangeId, account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: PositionUpdatePoint) -> Self {
3746        Event::PositionUpdate { exchange, account_type, symbol: symbol.to_string(), point }
3747    }
3748}
3749impl EventFrom<FootprintPoint> for Event {
3750    fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: FootprintPoint) -> Self {
3751        Event::Footprint { exchange, symbol: symbol.to_string(), point }
3752    }
3753}
3754// Renko / PnF / Kagi / CVD / TPO emit through derived aggregators only —
3755// EventFrom funnels their Point types into the corresponding Event
3756// variant. The `kind` is consulted ONLY where the variant carries a
3757// runtime parameter that comes from the SeriesKey (none of the five do —
3758// the parameter is encoded into the Point itself).
3759impl EventFrom<PnfColumnPoint> for Event {
3760    fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: PnfColumnPoint) -> Self {
3761        Event::PnfBar { exchange, symbol: symbol.to_string(), point }
3762    }
3763}
3764impl EventFrom<KagiSegmentPoint> for Event {
3765    fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: KagiSegmentPoint) -> Self {
3766        Event::KagiBar { exchange, symbol: symbol.to_string(), point }
3767    }
3768}
3769impl EventFrom<ScalarBarPoint> for Event {
3770    fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: ScalarBarPoint) -> Self {
3771        Event::CvdLine { exchange, symbol: symbol.to_string(), point }
3772    }
3773}
3774impl EventFrom<TpoSessionPoint> for Event {
3775    fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: TpoSessionPoint) -> Self {
3776        Event::TpoProfile { exchange, symbol: symbol.to_string(), point }
3777    }
3778}
3779impl EventFrom<RenkoBrickPoint> for Event {
3780    fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: RenkoBrickPoint) -> Self {
3781        Event::RenkoBar { exchange, symbol: symbol.to_string(), point }
3782    }
3783}
3784impl EventFrom<ThreeLineBreakLinePoint> for Event {
3785    fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, point: ThreeLineBreakLinePoint) -> Self {
3786        Event::ThreeLineBreakUpdate { exchange, symbol: symbol.to_string(), point }
3787    }
3788}
3789
3790
3791// ── Extended-depth EventFrom impls ────────────────────────────────────────────
3792//
3793// Extended point types (Indicators / Full) are written to disk at depth; they
3794// are NOT carried separately in the broadcast channel — broadcast consumers
3795// always receive the Compact event variant. These impls convert the enriched
3796// point back to the Compact Event variant so the forwarder compile-checks.
3797
3798impl EventFrom<TickerIndicatorsPoint> for Event {
3799    fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, p: TickerIndicatorsPoint) -> Self {
3800        // Broadcast consumers see a Compact Ticker; disk has the Indicators record.
3801        Event::Ticker { exchange, symbol: symbol.to_string(), point: TickerPoint {
3802            ts_ms: p.ts_ms, last: p.last, bid: p.bid, ask: p.ask,
3803            high_24h: p.high_24h, low_24h: p.low_24h,
3804            vol_24h: p.vol_24h, quote_vol_24h: p.quote_vol_24h,
3805            change_pct_24h: p.change_pct_24h,
3806        }}
3807    }
3808}
3809impl EventFrom<TickerFullPoint> for Event {
3810    fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, p: TickerFullPoint) -> Self {
3811        Event::Ticker { exchange, symbol: symbol.to_string(), point: TickerPoint {
3812            ts_ms: p.ts_ms, last: p.last, bid: p.bid, ask: p.ask,
3813            high_24h: p.high_24h, low_24h: p.low_24h,
3814            vol_24h: p.vol_24h, quote_vol_24h: p.quote_vol_24h,
3815            change_pct_24h: p.price_change_pct_24h,
3816        }}
3817    }
3818}
3819impl EventFrom<MarkPriceIndicatorsPoint> for Event {
3820    fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, p: MarkPriceIndicatorsPoint) -> Self {
3821        Event::MarkPrice { exchange, symbol: symbol.to_string(), point: MarkPricePoint {
3822            ts_ms: p.ts_ms, mark: p.mark, index: p.index,
3823        }}
3824    }
3825}
3826impl EventFrom<MarkPriceFullPoint> for Event {
3827    fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, p: MarkPriceFullPoint) -> Self {
3828        Event::MarkPrice { exchange, symbol: symbol.to_string(), point: MarkPricePoint {
3829            ts_ms: p.ts_ms, mark: p.mark, index: p.index,
3830        }}
3831    }
3832}
3833impl EventFrom<FundingRateIndicatorsPoint> for Event {
3834    fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, p: FundingRateIndicatorsPoint) -> Self {
3835        Event::FundingRate { exchange, symbol: symbol.to_string(), point: FundingRatePoint {
3836            ts_ms: p.ts_ms, rate: p.rate,
3837            next_funding_time_ms: p.next_funding_time_ms.unwrap_or(0),
3838        }}
3839    }
3840}
3841impl EventFrom<FundingRateFullPoint> for Event {
3842    fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, p: FundingRateFullPoint) -> Self {
3843        Event::FundingRate { exchange, symbol: symbol.to_string(), point: FundingRatePoint {
3844            ts_ms: p.ts_ms, rate: p.rate,
3845            next_funding_time_ms: p.next_funding_time_ms.unwrap_or(0),
3846        }}
3847    }
3848}
3849impl EventFrom<OpenInterestIndicatorsPoint> for Event {
3850    fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, p: OpenInterestIndicatorsPoint) -> Self {
3851        Event::OpenInterest { exchange, symbol: symbol.to_string(), point: OpenInterestPoint {
3852            ts_ms: p.ts_ms, open_interest: p.open_interest,
3853            open_interest_value: p.open_interest_value,
3854        }}
3855    }
3856}
3857impl EventFrom<LiquidationIndicatorsPoint> for Event {
3858    fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, p: LiquidationIndicatorsPoint) -> Self {
3859        Event::Liquidation { exchange, symbol: symbol.to_string(), point: LiquidationPoint {
3860            ts_ms: p.ts_ms, price: p.price, quantity: p.quantity,
3861            value: p.value, side: p.side,
3862        }}
3863    }
3864}
3865impl EventFrom<LiquidationFullPoint> for Event {
3866    fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, p: LiquidationFullPoint) -> Self {
3867        Event::Liquidation { exchange, symbol: symbol.to_string(), point: LiquidationPoint {
3868            ts_ms: p.ts_ms, price: p.price, quantity: p.quantity,
3869            value: p.value, side: p.side,
3870        }}
3871    }
3872}
3873impl EventFrom<IndexPriceIndicatorsPoint> for Event {
3874    fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, p: IndexPriceIndicatorsPoint) -> Self {
3875        Event::IndexPrice { exchange, symbol: symbol.to_string(), point: IndexPricePoint {
3876            ts_ms: p.ts_ms, price: p.price,
3877        }}
3878    }
3879}
3880impl EventFrom<ObSnapshotIndicatorsPoint> for Event {
3881    fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, p: ObSnapshotIndicatorsPoint) -> Self {
3882        Event::OrderbookSnapshot { exchange, symbol: symbol.to_string(), point: ObSnapshotPoint {
3883            ts_ms: p.ts_ms, bids: p.bids, asks: p.asks,
3884        }}
3885    }
3886}
3887impl EventFrom<ObDeltaIndicatorsPoint> for Event {
3888    fn from_point(exchange: digdigdig3::core::types::ExchangeId, _account_type: digdigdig3::core::types::AccountType, symbol: &str, _kind: &Kind, p: ObDeltaIndicatorsPoint) -> Self {
3889        Event::OrderbookDelta { exchange, symbol: symbol.to_string(), point: ObDeltaPoint {
3890            ts_ms: p.ts_ms, bid_changes: p.bid_changes, ask_changes: p.ask_changes,
3891        }}
3892    }
3893}
3894
3895/// Fetch a REST orderbook snapshot and convert to `Vec<ObSnapshotPoint>`.
3896/// Returns an empty Vec on any failure (no REST, exchange error, empty book).
3897async fn ob_rest_seed(
3898    hub: &Arc<digdigdig3::connector_manager::ExchangeHub>,
3899    exchange: digdigdig3::core::types::ExchangeId,
3900    account: digdigdig3::core::types::AccountType,
3901    symbol: &str,
3902    depth: usize,
3903) -> Vec<ObSnapshotPoint> {
3904    let Some(rest) = hub.rest(exchange) else {
3905        tracing::warn!(
3906            target: "dig3::ob_seed",
3907            ?exchange, symbol,
3908            "orderbook REST seed: connector not initialized — continuing WS-only"
3909        );
3910        return Vec::new();
3911    };
3912    let depth_u16 = depth.min(u16::MAX as usize) as u16;
3913    match rest
3914        .get_orderbook(
3915            digdigdig3::core::types::SymbolInput::Raw(symbol),
3916            Some(depth_u16),
3917            account,
3918        )
3919        .await
3920    {
3921        Ok(ob) if ob.bids.is_empty() && ob.asks.is_empty() => {
3922            tracing::warn!(
3923                target: "dig3::ob_seed",
3924                ?exchange, symbol,
3925                "orderbook REST seed returned empty snapshot — continuing WS-only"
3926            );
3927            Vec::new()
3928        }
3929        Ok(ob) => {
3930            tracing::debug!(
3931                target: "dig3::ob_seed",
3932                ?exchange, symbol,
3933                bids = ob.bids.len(),
3934                asks = ob.asks.len(),
3935                "orderbook REST seed ok"
3936            );
3937            vec![ObSnapshotPoint::from_orderbook(&ob)]
3938        }
3939        Err(e) => {
3940            tracing::warn!(
3941                target: "dig3::ob_seed",
3942                ?exchange, symbol, ?e,
3943                "orderbook REST seed failed — continuing WS-only"
3944            );
3945            Vec::new()
3946        }
3947    }
3948}
3949
3950fn ws_request_for(
3951    kind: &Kind,
3952    sym: Symbol,
3953    account: digdigdig3::core::types::AccountType,
3954) -> SubscriptionRequest {
3955    let stream_type = match kind {
3956        Kind::Trade => StreamType::Trade,
3957        Kind::AggTrade => StreamType::AggTrade,
3958        Kind::Kline(iv) => StreamType::Kline { interval: iv.as_str().to_string() },
3959        Kind::Ticker => StreamType::Ticker,
3960        Kind::Orderbook => StreamType::Orderbook,
3961        Kind::OrderbookDelta => StreamType::OrderbookDelta,
3962        Kind::MarkPrice => StreamType::MarkPrice,
3963        Kind::FundingRate => StreamType::FundingRate,
3964        Kind::OpenInterest => StreamType::OpenInterest,
3965        Kind::Liquidation => StreamType::Liquidation,
3966        Kind::BlockTrade => StreamType::BlockTrade,
3967        Kind::AuctionEvent => StreamType::AuctionEvent,
3968        Kind::IndexPrice => StreamType::IndexPrice,
3969        Kind::CompositeIndex => StreamType::CompositeIndex,
3970        Kind::OptionGreeks => StreamType::OptionGreeks,
3971        Kind::VolatilityIndex => StreamType::VolatilityIndex,
3972        Kind::HistoricalVolatility => StreamType::HistoricalVolatility,
3973        // LongShortRatio / TakerVolume / LiquidationBucket: poll-only kinds. The WS arms
3974        // are unreachable in normal operation (acquire_or_spawn_polled handles them first),
3975        // but kept as defensive fallbacks so the match is exhaustive.
3976        Kind::LongShortRatio => StreamType::LongShortRatio,
3977        Kind::TakerVolume | Kind::LiquidationBucket => {
3978            unreachable!("TakerVolume/LiquidationBucket are poll-only — ws_request_for must not be called for them")
3979        }
3980        Kind::Basis => StreamType::Basis,
3981        Kind::InsuranceFund => StreamType::InsuranceFund,
3982        Kind::OrderbookL3 => StreamType::OrderbookL3,
3983        Kind::SettlementEvent => StreamType::SettlementEvent,
3984        Kind::MarketWarning => StreamType::MarketWarning,
3985        Kind::RiskLimit => StreamType::RiskLimit,
3986        Kind::PredictedFunding => StreamType::PredictedFunding,
3987        Kind::FundingSettlement => StreamType::FundingSettlement,
3988        Kind::MarkPriceKline(iv) => StreamType::MarkPriceKline { interval: iv.as_str().to_string() },
3989        Kind::IndexPriceKline(iv) => StreamType::IndexPriceKline { interval: iv.as_str().to_string() },
3990        Kind::PremiumIndexKline(iv) => StreamType::PremiumIndexKline { interval: iv.as_str().to_string() },
3991        Kind::OrderUpdate => StreamType::OrderUpdate,
3992        Kind::BalanceUpdate => StreamType::BalanceUpdate,
3993        Kind::PositionUpdate => StreamType::PositionUpdate,
3994        // Derived kinds — never reach ws_request_for (acquire_or_spawn dispatches
3995        // them through acquire_or_spawn_derived before calling this function).
3996        Kind::RangeBar(_) | Kind::TickBar(_) | Kind::VolumeBar(_) | Kind::Footprint(_)
3997        | Kind::RenkoBar(_, _) | Kind::PnfBar(_, _) | Kind::KagiBar(_)
3998        | Kind::CvdLine | Kind::ThreeLineBreak { .. } | Kind::TpoProfile(_, _)
3999        | Kind::DollarBar { .. } | Kind::TickImbalanceBar { .. }
4000        | Kind::VolumeImbalanceBar { .. } | Kind::RunBar { .. } => {
4001            unreachable!("derived kinds must not call ws_request_for")
4002        }
4003    };
4004    SubscriptionRequest {
4005        symbol: sym,
4006        stream_type,
4007        account_type: account,
4008        depth: None,
4009        update_speed_ms: None,
4010    }
4011}
4012
4013fn parse_symbol(s: &str) -> Symbol {
4014    if let Some((b, q)) = s.split_once(['-', '/', '_']) {
4015        return Symbol::new(b, q);
4016    }
4017    let upper = s.to_uppercase();
4018    for q in ["USDT", "USDC", "USD", "BTC", "ETH", "BUSD", "EUR", "JPY"] {
4019        if let Some(base) = upper.strip_suffix(q) {
4020            if !base.is_empty() {
4021                return Symbol::new(base, q);
4022            }
4023        }
4024    }
4025    Symbol::new(&upper, "")
4026}
4027
4028/// Conservative capability gate for `Station::subscribe`.
4029///
4030/// Returns `true` ONLY when the connector has declared BOTH the REST capability
4031/// AND the WS capability false for a Kind — i.e. the exchange exposes the data
4032/// via neither REST nor WS and the stream would certainly fail.  When in doubt
4033/// (flag combination unclear, or the Kind has a derive fallback) this returns
4034/// `false` and the normal `acquire_or_spawn` error path handles it.
4035///
4036/// Derived Kinds (`RangeBar`, `TickBar`, `VolumeBar`, `Footprint`, `Basis`,
4037/// `FundingSettlement`) are never considered unsupported here — they derive
4038/// from other streams and may succeed even when no native channel exists.
4039///
4040/// Poll-only Kinds (`LongShortRatio`, `TakerVolume`, `LiquidationBucket`)
4041/// are checked against their REST history flag only (they have no WS path).
4042pub(crate) fn caps_explicitly_unsupported(caps: &ConnectorCapabilities, kind: &Kind) -> bool {
4043    match kind {
4044        Kind::Trade => !caps.has_recent_trades && !caps.has_ws_trades,
4045        Kind::AggTrade => !caps.has_agg_trades && !caps.has_ws_trades,
4046        Kind::Kline(_) => !caps.has_klines && !caps.has_ws_klines,
4047        Kind::Ticker => !caps.has_ticker && !caps.has_ws_ticker,
4048        Kind::Orderbook => !caps.has_orderbook && !caps.has_ws_orderbook,
4049        Kind::OrderbookDelta => !caps.has_orderbook && !caps.has_ws_orderbook,
4050        Kind::MarkPrice => !caps.has_premium_index && !caps.has_ws_mark_price,
4051        Kind::FundingRate => !caps.has_funding_rate_history && !caps.has_ws_funding_rate,
4052        Kind::OpenInterest => !caps.has_open_interest_history,
4053        Kind::Liquidation => !caps.has_liquidation_history,
4054        Kind::MarkPriceKline(_) => !caps.has_mark_price_klines,
4055        Kind::IndexPriceKline(_) => !caps.has_index_price_klines,
4056        Kind::PremiumIndexKline(_) => !caps.has_premium_index_klines,
4057        // Poll-only: REST-flag only (no WS path exists for these).
4058        Kind::LongShortRatio => !caps.has_long_short_ratio_history,
4059        Kind::TakerVolume => !caps.has_taker_volume_history,
4060        Kind::LiquidationBucket => !caps.has_liquidation_bucket_history,
4061        Kind::InsuranceFund => !caps.has_insurance_fund,
4062        // Derived Kinds: let acquire_or_spawn_derived handle them.
4063        Kind::RangeBar(_)
4064        | Kind::TickBar(_)
4065        | Kind::VolumeBar(_)
4066        | Kind::Footprint(_)
4067        | Kind::RenkoBar(_, _)
4068        | Kind::PnfBar(_, _)
4069        | Kind::KagiBar(_)
4070        | Kind::CvdLine
4071        | Kind::ThreeLineBreak { .. }
4072        | Kind::TpoProfile(_, _)
4073        | Kind::DollarBar { .. }
4074        | Kind::TickImbalanceBar { .. }
4075        | Kind::VolumeImbalanceBar { .. }
4076        | Kind::RunBar { .. }
4077        | Kind::Basis
4078        | Kind::FundingSettlement => false,
4079        // The remaining Kinds have no dedicated capability flag — let the WS
4080        // error path surface failures rather than pre-empting incorrectly.
4081        Kind::BlockTrade
4082        | Kind::AuctionEvent
4083        | Kind::IndexPrice
4084        | Kind::CompositeIndex
4085        | Kind::OptionGreeks
4086        | Kind::VolatilityIndex
4087        | Kind::HistoricalVolatility
4088        | Kind::OrderbookL3
4089        | Kind::SettlementEvent
4090        | Kind::MarketWarning
4091        | Kind::RiskLimit
4092        | Kind::PredictedFunding
4093        | Kind::OrderUpdate
4094        | Kind::BalanceUpdate
4095        | Kind::PositionUpdate => false,
4096    }
4097}
4098
4099#[cfg(test)]
4100mod flush_persistence_tests {
4101    use super::*;
4102    use crate::series::Kind as SeriesKind;
4103    use digdigdig3::core::types::{AccountType, ExchangeId};
4104
4105    fn test_key(symbol: &str) -> SeriesKey {
4106        SeriesKey::new(ExchangeId::Binance, AccountType::Spot, symbol, SeriesKind::Trade)
4107    }
4108
4109    /// Exercises the registry mechanics in isolation: a synthetic task plays
4110    /// the "forwarder" role (owns a `mpsc::Receiver<FlushAck>`, registers a
4111    /// `FlushHandle` under a `SeriesKey`, replies on each ack), without
4112    /// spinning up a real WS/derived/poller forwarder (those need a live
4113    /// network or broadcast setup that a unit test cannot provide).
4114    #[tokio::test]
4115    async fn flush_persistence_round_trips_through_registered_handle() {
4116        let tmp = std::env::temp_dir().join(format!(
4117            "dig3-flush-persistence-test-{}-{}",
4118            std::process::id(),
4119            chrono::Utc::now().timestamp_nanos_opt().unwrap_or(0),
4120        ));
4121
4122        let station = Station::builder()
4123            .storage_root(&tmp)
4124            .build()
4125            .await
4126            .expect("Station::build is pure-local (no network) — must succeed");
4127
4128        let key = test_key("BTCUSDT");
4129        let (handle, mut flush_rx) = FlushHandle::channel();
4130        station.inner.flush_handles.insert(key.clone(), handle);
4131
4132        // Synthetic forwarder: waits for exactly one flush request, then acks
4133        // success. Mirrors the `recv_flush_request` + ack pattern real
4134        // forwarders run inside their select loop.
4135        let synthetic = tokio::spawn(async move {
4136            let ack = flush_rx.recv().await.expect("flush_persistence must send a request");
4137            let _ = ack.send(Ok(()));
4138        });
4139
4140        let result = station.flush_persistence().await;
4141        assert!(result.is_ok(), "expected Ok(()), got {result:?}");
4142
4143        synthetic.await.expect("synthetic forwarder task panicked");
4144
4145        // The synthetic task's `flush_rx` is dropped when the task returns,
4146        // which closes the paired sender. `flush_persistence`'s opportunistic
4147        // cleanup sweep (`retain(|_, h| !h.is_closed())`) then reaps the
4148        // now-dead entry on this same call — matching what a real forwarder
4149        // achieves via its own explicit `flush_handles.remove(&key)` right
4150        // before exit. Either path converges on the entry being gone.
4151        assert!(!station.inner.flush_handles.contains_key(&key));
4152
4153        let _ = std::fs::remove_dir_all(&tmp);
4154    }
4155
4156    /// A registry entry whose receiver has already been dropped (forwarder
4157    /// exited without unregistering — defensive case) must be treated as
4158    /// "already flushed", not surfaced as an error.
4159    #[tokio::test]
4160    async fn flush_persistence_treats_dead_handle_as_already_flushed() {
4161        let tmp = std::env::temp_dir().join(format!(
4162            "dig3-flush-persistence-dead-test-{}-{}",
4163            std::process::id(),
4164            chrono::Utc::now().timestamp_nanos_opt().unwrap_or(0),
4165        ));
4166
4167        let station = Station::builder()
4168            .storage_root(&tmp)
4169            .build()
4170            .await
4171            .expect("Station::build is pure-local (no network) — must succeed");
4172
4173        let key = test_key("ETHUSDT");
4174        let (handle, flush_rx) = FlushHandle::channel();
4175        drop(flush_rx); // simulate a forwarder that already exited
4176        station.inner.flush_handles.insert(key.clone(), handle);
4177
4178        let result = station.flush_persistence().await;
4179        assert!(result.is_ok(), "dead handle must not surface as an error: {result:?}");
4180
4181        // Opportunistic cleanup: flush_persistence sweeps closed senders.
4182        assert!(!station.inner.flush_handles.contains_key(&key));
4183
4184        let _ = std::fs::remove_dir_all(&tmp);
4185    }
4186
4187    /// Aggregation: a genuine `Err` from the forwarder's `DiskStore::flush()`
4188    /// must be collected in the returned `Vec`, keyed by `SeriesKey`.
4189    #[tokio::test]
4190    async fn flush_persistence_aggregates_forwarder_reported_errors() {
4191        let tmp = std::env::temp_dir().join(format!(
4192            "dig3-flush-persistence-err-test-{}-{}",
4193            std::process::id(),
4194            chrono::Utc::now().timestamp_nanos_opt().unwrap_or(0),
4195        ));
4196
4197        let station = Station::builder()
4198            .storage_root(&tmp)
4199            .build()
4200            .await
4201            .expect("Station::build is pure-local (no network) — must succeed");
4202
4203        let key = test_key("SOLUSDT");
4204        let (handle, mut flush_rx) = FlushHandle::channel();
4205        station.inner.flush_handles.insert(key.clone(), handle);
4206
4207        let synthetic = tokio::spawn(async move {
4208            let ack = flush_rx.recv().await.expect("flush_persistence must send a request");
4209            let _ = ack.send(Err(std::io::Error::other("simulated disk failure")));
4210        });
4211
4212        let result = station.flush_persistence().await;
4213        synthetic.await.expect("synthetic forwarder task panicked");
4214
4215        let errors = result.expect_err("a reported flush error must surface");
4216        assert_eq!(errors.len(), 1);
4217        assert_eq!(errors[0].0, key);
4218
4219        let _ = std::fs::remove_dir_all(&tmp);
4220    }
4221}
4222
4223#[cfg(test)]
4224mod force_unsubscribe_tests {
4225    use super::*;
4226    use crate::series::Kind as SeriesKind;
4227    use digdigdig3::core::types::{AccountType, ExchangeId};
4228
4229    fn test_key(symbol: &str) -> SeriesKey {
4230        SeriesKey::new(ExchangeId::Binance, AccountType::Spot, symbol, SeriesKind::Trade)
4231    }
4232
4233    /// No mux registered for `key` — must be a no-op `Ok(())`, never hang.
4234    #[tokio::test]
4235    async fn force_unsubscribe_is_idempotent_when_no_mux_exists() {
4236        let tmp = std::env::temp_dir().join(format!(
4237            "dig3-force-unsub-noop-test-{}-{}",
4238            std::process::id(),
4239            chrono::Utc::now().timestamp_nanos_opt().unwrap_or(0),
4240        ));
4241        let station = Station::builder()
4242            .storage_root(&tmp)
4243            .build()
4244            .await
4245            .expect("Station::build is pure-local (no network) — must succeed");
4246
4247        let key = test_key("BTCUSDT");
4248        let result = station.force_unsubscribe_and_await(&key).await;
4249        assert!(result.is_ok());
4250
4251        let _ = std::fs::remove_dir_all(&tmp);
4252    }
4253
4254    /// Full mechanics: register a synthetic mux (fake "forwarder" — a real
4255    /// WS/derived/poller forwarder needs live network or a broadcast setup a
4256    /// unit test cannot provide) with a `shutdown` sender AND an `exit_acks`
4257    /// entry. A synthetic task plays the forwarder role: waits for the
4258    /// shutdown signal, then fires the exit ack — mirroring the real
4259    /// teardown ordering in `spawn_forwarder`/`spawn_derived_forwarder`/
4260    /// `spawn_poller`. `force_unsubscribe_and_await` must:
4261    /// - remove the mux + series_handles entries immediately,
4262    /// - fire shutdown regardless of `consumers` count (set to 2 here, i.e.
4263    ///   still "in use" by the ref-count that `release_consumer` would
4264    ///   otherwise respect),
4265    /// - await the exit ack before returning.
4266    #[tokio::test]
4267    async fn force_unsubscribe_fires_shutdown_and_awaits_exit_ack() {
4268        let tmp = std::env::temp_dir().join(format!(
4269            "dig3-force-unsub-mechanics-test-{}-{}",
4270            std::process::id(),
4271            chrono::Utc::now().timestamp_nanos_opt().unwrap_or(0),
4272        ));
4273        let station = Station::builder()
4274            .storage_root(&tmp)
4275            .build()
4276            .await
4277            .expect("Station::build is pure-local (no network) — must succeed");
4278
4279        let key = test_key("ETHUSDT");
4280
4281        let (bcast_tx, _) = broadcast::channel::<Event>(16);
4282        let (shutdown_tx, shutdown_rx) = oneshot::channel::<()>();
4283        let (exit_ack_tx, exit_ack_rx) = oneshot::channel::<()>();
4284
4285        // consumers = 2 — deliberately "still referenced" so a plain
4286        // `release_consumer` call would NOT fire shutdown. force_unsubscribe
4287        // must fire it anyway, unconditionally.
4288        station.inner.muxes.insert(
4289            key.clone(),
4290            Multiplexer {
4291                tx: bcast_tx,
4292                consumers: Arc::new(AtomicUsize::new(2)),
4293                shutdown: Some(shutdown_tx),
4294                grace_cancel: None,
4295            },
4296        );
4297        station.inner.exit_acks.insert(key.clone(), exit_ack_rx);
4298        let dummy_series: Arc<dyn Any + Send + Sync> =
4299            Arc::new(Arc::new(RwLock::new(Series::<TradePoint>::new(4))));
4300        station.inner.series_handles.insert(key.clone(), dummy_series);
4301
4302        // Synthetic forwarder: waits for the shutdown signal, then does its
4303        // (simulated) teardown work and fires the exit ack — same sequence
4304        // real forwarders follow (shutdown observed → drain/cleanup →
4305        // exit_acks.remove + send).
4306        let shutdown_observed = Arc::new(std::sync::atomic::AtomicBool::new(false));
4307        let shutdown_observed_clone = Arc::clone(&shutdown_observed);
4308        let synthetic = tokio::spawn(async move {
4309            let _ = shutdown_rx.await;
4310            shutdown_observed_clone.store(true, Ordering::SeqCst);
4311            let _ = exit_ack_tx.send(());
4312        });
4313
4314        let result = station.force_unsubscribe_and_await(&key).await;
4315        assert!(result.is_ok());
4316
4317        // Mux + series_handles must be gone immediately (before the await on
4318        // the ack even needed the synthetic task to run its cleanup).
4319        assert!(!station.inner.muxes.contains_key(&key));
4320        assert!(!station.inner.series_handles.contains_key(&key));
4321        // The await must not have returned before the synthetic forwarder
4322        // actually observed the shutdown signal.
4323        assert!(shutdown_observed.load(Ordering::SeqCst));
4324
4325        synthetic.await.expect("synthetic forwarder task panicked");
4326        let _ = std::fs::remove_dir_all(&tmp);
4327    }
4328
4329    /// Race-safety: if the forwarder has already exited and removed its own
4330    /// `exit_acks` entry (and the mux) before `force_unsubscribe_and_await`
4331    /// runs, the call must return `Ok(())` immediately rather than hang
4332    /// waiting for an ack that will never arrive.
4333    #[tokio::test]
4334    async fn force_unsubscribe_treats_already_exited_forwarder_as_done() {
4335        let tmp = std::env::temp_dir().join(format!(
4336            "dig3-force-unsub-already-exited-test-{}-{}",
4337            std::process::id(),
4338            chrono::Utc::now().timestamp_nanos_opt().unwrap_or(0),
4339        ));
4340        let station = Station::builder()
4341            .storage_root(&tmp)
4342            .build()
4343            .await
4344            .expect("Station::build is pure-local (no network) — must succeed");
4345
4346        let key = test_key("SOLUSDT");
4347        // No mux, no exit_ack registered — simulates a forwarder that has
4348        // already fully torn down (e.g. upstream closed concurrently).
4349        let result = station.force_unsubscribe_and_await(&key).await;
4350        assert!(result.is_ok());
4351
4352        let _ = std::fs::remove_dir_all(&tmp);
4353    }
4354
4355    /// If the mux's `exit_acks` receiver is present but its paired sender
4356    /// was dropped without sending (forwarder task panicked/aborted before
4357    /// reaching its exit-ack send), the await must resolve (with a recv
4358    /// error) rather than hang forever.
4359    #[tokio::test]
4360    async fn force_unsubscribe_does_not_hang_on_dropped_ack_sender() {
4361        let tmp = std::env::temp_dir().join(format!(
4362            "dig3-force-unsub-dropped-sender-test-{}-{}",
4363            std::process::id(),
4364            chrono::Utc::now().timestamp_nanos_opt().unwrap_or(0),
4365        ));
4366        let station = Station::builder()
4367            .storage_root(&tmp)
4368            .build()
4369            .await
4370            .expect("Station::build is pure-local (no network) — must succeed");
4371
4372        let key = test_key("DOGEUSDT");
4373        let (bcast_tx, _) = broadcast::channel::<Event>(16);
4374        let (shutdown_tx, _shutdown_rx) = oneshot::channel::<()>();
4375        let (exit_ack_tx, exit_ack_rx) = oneshot::channel::<()>();
4376        drop(exit_ack_tx); // simulate a forwarder that panicked before ack-send
4377
4378        station.inner.muxes.insert(
4379            key.clone(),
4380            Multiplexer {
4381                tx: bcast_tx,
4382                consumers: Arc::new(AtomicUsize::new(1)),
4383                shutdown: Some(shutdown_tx),
4384                grace_cancel: None,
4385            },
4386        );
4387        station.inner.exit_acks.insert(key.clone(), exit_ack_rx);
4388
4389        let result = tokio::time::timeout(
4390            std::time::Duration::from_secs(5),
4391            station.force_unsubscribe_and_await(&key),
4392        )
4393        .await
4394        .expect("force_unsubscribe_and_await must not hang on a dropped ack sender");
4395        assert!(result.is_ok());
4396
4397        let _ = std::fs::remove_dir_all(&tmp);
4398    }
4399}
4400
4401#[cfg(test)]
4402mod rewarm_derived_tests {
4403    //! Unit tests for `Station::rewarm_derived` and its private per-kind
4404    //! helpers. Network I/O (`rewarm_fetch_trade_window` /
4405    //! `rewarm_fetch_kline_sufficient_window`) is exercised implicitly through
4406    //! `rewarm_fold` + `rewarm_splice` + `rewarm_dedup` directly with
4407    //! synthetic events — a real REST fetch needs a live exchange
4408    //! connection a unit test cannot provide (same limitation documented in
4409    //! `derived_stream_integration.rs`). `rewarm_derived`'s dispatcher and
4410    //! `StationError::RewarmUnsupported` short-circuit ARE exercised
4411    //! end-to-end since they never touch the network.
4412    use super::*;
4413    use crate::series::Kind as SeriesKind;
4414    use digdigdig3::core::types::{AccountType, ExchangeId};
4415
4416    fn test_key(symbol: &str, kind: SeriesKind) -> SeriesKey {
4417        SeriesKey::new(ExchangeId::Binance, AccountType::Spot, symbol, kind)
4418    }
4419
4420    fn trade_event(ts_ms: i64, price: f64, quantity: f64, side: u8) -> Event {
4421        Event::Trade {
4422            exchange: ExchangeId::Binance,
4423            symbol: "BTCUSDT".to_string(),
4424            point: TradePoint { ts_ms, price, quantity, side, trade_id_hash: 0 },
4425        }
4426    }
4427
4428    async fn station_with_tmp() -> (Station, PathBuf) {
4429        let tmp = std::env::temp_dir().join(format!(
4430            "dig3-rewarm-derived-test-{}-{}",
4431            std::process::id(),
4432            chrono::Utc::now().timestamp_nanos_opt().unwrap_or(0),
4433        ));
4434        let station = Station::builder()
4435            .storage_root(&tmp)
4436            .build()
4437            .await
4438            .expect("Station::build is pure-local (no network) — must succeed");
4439        (station, tmp)
4440    }
4441
4442    fn insert_series<T: DataPoint + Send + Sync + 'static>(
4443        station: &Station,
4444        key: &SeriesKey,
4445        points: Vec<T>,
4446    ) {
4447        let mut series = Series::<T>::new(points.len().max(16));
4448        series.extend(points);
4449        let handle: Arc<dyn Any + Send + Sync> = Arc::new(Arc::new(RwLock::new(series)));
4450        station.inner.series_handles.insert(key.clone(), handle);
4451    }
4452
4453    // -----------------------------------------------------------------------
4454    // 1. Renko rewarm: existing bricks byte-identical after splice, AND
4455    //    prepended bricks sit on the SAME grid.
4456    // -----------------------------------------------------------------------
4457
4458    #[tokio::test]
4459    async fn renko_rewarm_keeps_existing_bricks_and_shares_grid() {
4460        let (station, tmp) = station_with_tmp().await;
4461        // box_size = 1.0 (1e8 fixed point), reversal = 1.
4462        let key = test_key("BTCUSDT", SeriesKind::RenkoBar(100_000_000, 1));
4463
4464        // Existing live series: one brick already on the grid anchored at 100.0.
4465        let existing = vec![RenkoBrickPoint {
4466            open_time: 10_000,
4467            bottom: 100.0,
4468            top: 101.0,
4469            up: true,
4470            volume: 5.0,
4471            trades_count: 3,
4472        }];
4473        insert_series(&station, &key, existing.clone());
4474
4475        // No REST connector in this unit test — the public helper's network
4476        // fetch returns an empty window, so it must return empty rather
4477        // than touching the existing series (same no-network contract as
4478        // the other rewarm_* public helpers).
4479        let (older_via_public, _outcome) = station.rewarm_renko(&key, "BTCUSDT", 100).await;
4480        assert!(older_via_public.is_empty(), "no REST connector in unit test — fold input is empty");
4481
4482        // Drive the fold + splice path directly with synthetic OLDER
4483        // trades (bypassing the network fetch) to prove the real grid-seed
4484        // + seam behavior: `rewarm_fetch_kline_sufficient_window` (what
4485        // `rewarm_renko` actually calls) hands the fold `Event::Trade` legs
4486        // from `kline_to_synthetic_trades` — NOT `Event::Bar` — so this
4487        // must feed `Event::Trade` to match production. An OLDER trade
4488        // window that walks price up from 97.0 toward the existing grid,
4489        // seeded with the existing first brick's grid floor (100.0) exactly
4490        // as `rewarm_renko` would.
4491        let events = vec![
4492            trade_event(1_000, 97.5, 1.0, 0),
4493            trade_event(2_000, 98.5, 1.0, 0),
4494            trade_event(3_000, 99.5, 1.0, 0),
4495        ];
4496        let grid_floor = existing[0].bottom; // 100.0 — same preset rewarm_renko would compute
4497        let emissions = Station::rewarm_fold::<TradeToRenkoBarDerived>(&key, &events, |state| {
4498            state.preset_grid_anchor(grid_floor);
4499        })
4500        .await;
4501        let rebuilt = Station::rewarm_dedup(emissions, |p: &RenkoBrickPoint| p.open_time);
4502        assert!(!rebuilt.is_empty(), "expected at least one brick from the older window");
4503        let older = station.rewarm_splice(&key, rebuilt, |p: &RenkoBrickPoint| p.open_time).await;
4504        assert!(!older.is_empty(), "expected at least one prepended brick");
4505        for p in &older {
4506            // Every prepended brick must sit on the SAME 1.0-wide grid as
4507            // the existing series (bottom is an integer number of box
4508            // widths away from the existing first brick's bottom).
4509            let steps = (p.bottom - 100.0) / 1.0;
4510            assert!(
4511                (steps - steps.round()).abs() < 1e-9,
4512                "brick bottom {} is not grid-aligned with existing bottom 100.0",
4513                p.bottom
4514            );
4515        }
4516        // The prepended bricks must all be older than the existing head.
4517        assert!(older.iter().all(|p| p.open_time < 10_000));
4518
4519        // Existing bricks are byte-identical after splice: read back the
4520        // series ring and check that entry.
4521        let handle = station.inner.series_handles.get(&key).and_then(|e| {
4522            e.downcast_ref::<Arc<RwLock<Series<RenkoBrickPoint>>>>().map(Arc::clone)
4523        }).expect("series handle must still exist after splice");
4524        let after = handle.read().await.snapshot();
4525        let spliced_existing = after.last().expect("existing brick must be at the tail");
4526        assert_eq!(spliced_existing.open_time, existing[0].open_time);
4527        assert_eq!(spliced_existing.bottom, existing[0].bottom);
4528        assert_eq!(spliced_existing.top, existing[0].top);
4529        assert_eq!(spliced_existing.up, existing[0].up);
4530        assert_eq!(spliced_existing.volume, existing[0].volume);
4531        assert_eq!(spliced_existing.trades_count, existing[0].trades_count);
4532
4533        let _ = std::fs::remove_dir_all(&tmp);
4534    }
4535
4536    #[tokio::test]
4537    async fn renko_rewarm_no_existing_series_returns_empty() {
4538        let (station, tmp) = station_with_tmp().await;
4539        let key = test_key("BTCUSDT", SeriesKind::RenkoBar(100_000_000, 1));
4540        // No series_handles entry at all — must fall back to empty (cold path).
4541        let (older, _outcome) = station.rewarm_renko(&key, "BTCUSDT", 100).await;
4542        assert!(older.is_empty());
4543        let _ = std::fs::remove_dir_all(&tmp);
4544    }
4545
4546    // -----------------------------------------------------------------------
4547    // 2. Counter-bar (tick_bar) rewarm: existing bars untouched, prepended
4548    //    bars strictly older, no identity collision at the seam.
4549    // -----------------------------------------------------------------------
4550
4551    #[tokio::test]
4552    async fn tick_bar_rewarm_existing_untouched_prepended_strictly_older() {
4553        let (station, tmp) = station_with_tmp().await;
4554        // n = 2 trades per bar.
4555        let key = test_key("BTCUSDT", SeriesKind::TickBar(2));
4556
4557        let existing = vec![BarPoint {
4558            open_time: 50_000,
4559            open: 200.0,
4560            high: 201.0,
4561            low: 199.0,
4562            close: 200.5,
4563            volume: 4.0,
4564            quote_volume: 800.0,
4565            trades_count: 2,
4566        }];
4567        insert_series(&station, &key, existing.clone());
4568
4569        let (older, _outcome) = station
4570            .rewarm_derived_standalone::<TradeToTickBarDerived>(&key, "BTCUSDT", 100)
4571            .await;
4572
4573        // rewarm_fetch_trade_window needs a connected REST client to return
4574        // anything; without one it returns an empty window, so the fold
4575        // itself produces nothing and splice short-circuits to empty. That
4576        // is a legitimate outcome for a unit test with no live network —
4577        // assert the no-network contract explicitly, then re-verify the
4578        // same invariant by driving the fold directly with synthetic events.
4579        assert!(older.is_empty(), "no REST connector in unit test — fold input is empty");
4580
4581        // Drive the fold + splice path directly with synthetic OLDER trades
4582        // (bypassing the network fetch) to prove the real seam behavior.
4583        let events = vec![
4584            trade_event(10_000, 190.0, 1.0, 0),
4585            trade_event(11_000, 191.0, 1.0, 0),
4586            trade_event(12_000, 192.0, 1.0, 1),
4587            trade_event(13_000, 193.0, 1.0, 1),
4588        ];
4589        let emissions = Station::rewarm_fold::<TradeToTickBarDerived>(&key, &events, |_s| {}).await;
4590        let rebuilt = Station::rewarm_dedup(emissions, |p: &BarPoint| p.open_time);
4591        // 4 trades / 2 per bar = 2 closed bars.
4592        assert_eq!(rebuilt.len(), 2);
4593        let spliced = station.rewarm_splice(&key, rebuilt, |p: &BarPoint| p.open_time).await;
4594        assert_eq!(spliced.len(), 2, "both synthetic bars are older than the existing head");
4595        assert!(spliced.iter().all(|p| p.open_time < 50_000), "prepended bars must be strictly older");
4596
4597        // Existing bar must be byte-identical (untouched) after splice.
4598        let handle = station.inner.series_handles.get(&key).and_then(|e| {
4599            e.downcast_ref::<Arc<RwLock<Series<BarPoint>>>>().map(Arc::clone)
4600        }).expect("series handle must still exist after splice");
4601        let after = handle.read().await.snapshot();
4602        let spliced_existing = after.last().expect("existing bar must be at the tail");
4603        assert_eq!(spliced_existing.open_time, existing[0].open_time);
4604        assert_eq!(spliced_existing.open, existing[0].open);
4605        assert_eq!(spliced_existing.close, existing[0].close);
4606        assert_eq!(spliced_existing.trades_count, existing[0].trades_count);
4607        // No identity collision: no prepended bar shares open_time with the
4608        // existing head.
4609        assert!(after[..after.len() - 1].iter().all(|p| p.open_time != existing[0].open_time));
4610
4611        let _ = std::fs::remove_dir_all(&tmp);
4612    }
4613
4614    /// Boundary-collision guard: when the throwaway fold's last emission
4615    /// happens to share identity with the existing head, existing wins —
4616    /// the fold's copy is dropped, never overwrites the live bar.
4617    #[tokio::test]
4618    async fn tick_bar_rewarm_boundary_collision_existing_wins() {
4619        let (station, tmp) = station_with_tmp().await;
4620        let key = test_key("BTCUSDT", SeriesKind::TickBar(2));
4621
4622        let existing = vec![BarPoint {
4623            open_time: 5_000,
4624            open: 100.0,
4625            high: 100.0,
4626            low: 100.0,
4627            close: 100.0,
4628            volume: 1.0,
4629            quote_volume: 100.0,
4630            trades_count: 1,
4631        }];
4632        insert_series(&station, &key, existing.clone());
4633
4634        // Fold output deliberately includes a point with the SAME identity
4635        // (open_time) as the existing head, plus one strictly-older point.
4636        let rebuilt = vec![
4637            BarPoint {
4638                open_time: 1_000,
4639                open: 90.0, high: 90.0, low: 90.0, close: 90.0,
4640                volume: 1.0, quote_volume: 90.0, trades_count: 1,
4641            },
4642            BarPoint {
4643                open_time: 5_000, // collides with existing head
4644                open: 999.0, high: 999.0, low: 999.0, close: 999.0,
4645                volume: 999.0, quote_volume: 999.0, trades_count: 99,
4646            },
4647        ];
4648        let spliced = station.rewarm_splice(&key, rebuilt, |p: &BarPoint| p.open_time).await;
4649        // The colliding point must be dropped from the returned prepend set.
4650        assert_eq!(spliced.len(), 1);
4651        assert_eq!(spliced[0].open_time, 1_000);
4652
4653        let handle = station.inner.series_handles.get(&key).and_then(|e| {
4654            e.downcast_ref::<Arc<RwLock<Series<BarPoint>>>>().map(Arc::clone)
4655        }).expect("series handle must still exist");
4656        let after = handle.read().await.snapshot();
4657        // Existing head must be UNCHANGED (open == 100.0, not 999.0).
4658        let head = after.iter().find(|p| p.open_time == 5_000).expect("existing head must survive");
4659        assert_eq!(head.open, 100.0, "existing bar must never be overwritten by the fold's copy");
4660
4661        let _ = std::fs::remove_dir_all(&tmp);
4662    }
4663
4664    // -----------------------------------------------------------------------
4665    // 3. CVD rewarm: prepended segment's last value == existing first
4666    //    value; existing samples untouched.
4667    // -----------------------------------------------------------------------
4668
4669    #[tokio::test]
4670    async fn cvd_rewarm_offsets_to_meet_existing_first_value() {
4671        let (station, tmp) = station_with_tmp().await;
4672        let key = test_key("BTCUSDT", SeriesKind::CvdLine);
4673
4674        let existing = vec![ScalarBarPoint { ts_ms: 10_000, value: 500.0 }];
4675        insert_series(&station, &key, existing.clone());
4676
4677        // Older window: +1 -1 +1 (buy, sell, buy) => raw fold ends at +1.0
4678        // (starting from 0). After offset, must land exactly on 500.0.
4679        let events = vec![
4680            trade_event(1_000, 100.0, 1.0, 0), // buy: +1
4681            trade_event(2_000, 100.0, 1.0, 1), // sell: -1
4682            trade_event(3_000, 100.0, 1.0, 0), // buy: +1
4683        ];
4684        let emissions = Station::rewarm_fold::<TradeToCvdLineDerived>(&key, &events, |_s| {}).await;
4685        let mut rebuilt = Station::rewarm_dedup(emissions, |p: &ScalarBarPoint| p.ts_ms);
4686        assert_eq!(rebuilt.len(), 3);
4687        let raw_last = rebuilt.last().unwrap().value;
4688        assert!((raw_last - 1.0).abs() < 1e-9, "raw fold should end at +1.0 before offset");
4689
4690        let offset = existing[0].value - raw_last;
4691        for p in &mut rebuilt {
4692            p.value += offset;
4693        }
4694        let spliced = station.rewarm_splice(&key, rebuilt, |p: &ScalarBarPoint| p.ts_ms).await;
4695        assert_eq!(spliced.len(), 3);
4696        let last_prepended = spliced.last().unwrap();
4697        assert!(
4698            (last_prepended.value - existing[0].value).abs() < 1e-9,
4699            "last prepended CVD sample must equal existing first value exactly"
4700        );
4701
4702        // Existing sample must be untouched.
4703        let handle = station.inner.series_handles.get(&key).and_then(|e| {
4704            e.downcast_ref::<Arc<RwLock<Series<ScalarBarPoint>>>>().map(Arc::clone)
4705        }).expect("series handle must still exist");
4706        let after = handle.read().await.snapshot();
4707        let tail = after.last().expect("existing sample must be at the tail");
4708        assert_eq!(tail.ts_ms, existing[0].ts_ms);
4709        assert_eq!(tail.value, existing[0].value);
4710
4711        let _ = std::fs::remove_dir_all(&tmp);
4712    }
4713
4714    #[tokio::test]
4715    async fn cvd_rewarm_public_helper_matches_manual_offset() {
4716        let (station, tmp) = station_with_tmp().await;
4717        let key = test_key("BTCUSDT", SeriesKind::CvdLine);
4718        let existing = vec![ScalarBarPoint { ts_ms: 10_000, value: -25.0 }];
4719        insert_series(&station, &key, existing.clone());
4720
4721        // No REST connector in this unit test — rewarm_cvd's network fetch
4722        // returns an empty window, so the public helper itself must return
4723        // empty rather than panicking or touching the existing series.
4724        let (older, _outcome) = station.rewarm_cvd(&key, "BTCUSDT", 100).await;
4725        assert!(older.is_empty());
4726
4727        let handle = station.inner.series_handles.get(&key).and_then(|e| {
4728            e.downcast_ref::<Arc<RwLock<Series<ScalarBarPoint>>>>().map(Arc::clone)
4729        }).expect("series handle must still exist");
4730        let after = handle.read().await.snapshot();
4731        assert_eq!(after.len(), 1);
4732        assert_eq!(after[0].value, existing[0].value);
4733
4734        let _ = std::fs::remove_dir_all(&tmp);
4735    }
4736
4737    // -----------------------------------------------------------------------
4738    // 4. Unsupported kind (run_bar) returns the explicit unsupported error.
4739    // -----------------------------------------------------------------------
4740
4741    #[tokio::test]
4742    async fn run_bar_rewarm_returns_unsupported_error() {
4743        let (station, tmp) = station_with_tmp().await;
4744        let key = test_key("BTCUSDT", SeriesKind::RunBar { alpha_x100: 20, min_ticks: 10 });
4745        let result = station.rewarm_derived(&key, "BTCUSDT", 100).await;
4746        assert!(matches!(result, Err(StationError::RewarmUnsupported(_))));
4747        let _ = std::fs::remove_dir_all(&tmp);
4748    }
4749
4750    #[tokio::test]
4751    async fn tick_imbalance_bar_rewarm_returns_unsupported_error() {
4752        let (station, tmp) = station_with_tmp().await;
4753        let key = test_key("BTCUSDT", SeriesKind::TickImbalanceBar { alpha_x100: 20, min_ticks: 10 });
4754        let result = station.rewarm_derived(&key, "BTCUSDT", 100).await;
4755        assert!(matches!(result, Err(StationError::RewarmUnsupported(_))));
4756        let _ = std::fs::remove_dir_all(&tmp);
4757    }
4758
4759    #[tokio::test]
4760    async fn volume_imbalance_bar_rewarm_returns_unsupported_error() {
4761        let (station, tmp) = station_with_tmp().await;
4762        let key = test_key("BTCUSDT", SeriesKind::VolumeImbalanceBar { alpha_x100: 20, min_ticks: 10 });
4763        let result = station.rewarm_derived(&key, "BTCUSDT", 100).await;
4764        assert!(matches!(result, Err(StationError::RewarmUnsupported(_))));
4765        let _ = std::fs::remove_dir_all(&tmp);
4766    }
4767
4768    #[tokio::test]
4769    async fn footprint_rewarm_derived_returns_unsupported_error_pointing_at_dedicated_fn() {
4770        let (station, tmp) = station_with_tmp().await;
4771        let key = test_key("BTCUSDT", SeriesKind::Footprint(KlineInterval::new("1m")));
4772        let result = station.rewarm_derived(&key, "BTCUSDT", 100).await;
4773        assert!(matches!(result, Err(StationError::RewarmUnsupported(_))));
4774        let _ = std::fs::remove_dir_all(&tmp);
4775    }
4776
4777    // -----------------------------------------------------------------------
4778    // rewarm_dedup: last emission per identity supersedes.
4779    // -----------------------------------------------------------------------
4780
4781    #[test]
4782    fn rewarm_dedup_keeps_last_emission_per_identity() {
4783        let points = vec![
4784            ScalarBarPoint { ts_ms: 1, value: 1.0 },
4785            ScalarBarPoint { ts_ms: 1, value: 2.0 }, // same identity, supersedes
4786            ScalarBarPoint { ts_ms: 2, value: 3.0 },
4787        ];
4788        let out = Station::rewarm_dedup(points, |p: &ScalarBarPoint| p.ts_ms);
4789        assert_eq!(out.len(), 2);
4790        assert_eq!(out[0].value, 2.0, "second emission at ts=1 must supersede the first");
4791        assert_eq!(out[1].value, 3.0);
4792    }
4793}