digdigdig3_station/
persistence.rs

1//! Persistence configuration for the Station builder.
2//!
3//! Actual disk I/O lives in [`crate::series::DiskStore`], generic over
4//! [`crate::series::DataPoint`]. This module carries per-kind depth toggles
5//! threaded through the builder.
6
7use serde::{Deserialize, Serialize};
8
9// ─── PersistDepth ────────────────────────────────────────────────────────────
10
11/// How much of a payload to persist per Kind.
12///
13/// Different consumers want different granularity:
14/// - `Compact`: lean record used by the bar-aligned mli/mlq loaders (existing
15///   `<Kind>Point` types). Smallest disk footprint. Backward-compatible —
16///   existing `.dat` files are produced AND read by this depth.
17/// - `Indicators`: enriched record for indicator/feature engineering — adds
18///   the modeling-relevant fields the exchange emits (bid/ask qty, weighted
19///   avg, open, last qty, mark/index, OI, funding, count). Costs more disk
20///   per record but no loss of common signals at replay.
21/// - `Full`: every stable wire field on the payload struct. Largest record;
22///   intended for historical research persistence where nothing should be
23///   dropped at replay.
24///
25/// Default is `Compact` for every Kind for backward compatibility.
26#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
27pub enum PersistDepth {
28    Compact,
29    Indicators,
30    Full,
31}
32
33impl Default for PersistDepth {
34    fn default() -> Self {
35        PersistDepth::Compact
36    }
37}
38
39// ─── PersistenceConfig ───────────────────────────────────────────────────────
40
41/// Per-kind persistence toggles.
42///
43/// `None` means "do not persist this kind". `Some(depth)` means persist at the
44/// given depth. Master `enabled` gates everything; individual kinds can still be
45/// `Some(_)` but produce no writes when `enabled = false`.
46///
47/// All fields that previously existed as `bool` now carry
48/// `Option<PersistDepth>`. For backward compatibility:
49/// - `Default` produces `None` for every kind (same as all-`false` before).
50/// - `PersistenceConfig::on()` sets every field to `Some(Compact)` —
51///   identical behavior to the previous `true` / all-on path.
52#[derive(Debug, Clone, Serialize, Deserialize)]
53pub struct PersistenceConfig {
54    pub enabled: bool,
55    /// Sweep window for day files, in days. `None` = keep every day file
56    /// forever (default, no behavior change). `Some(n)` deletes `.dat`/`.idx`/
57    /// `.blob` triplets older than `today - n` on `DiskStore` open and on UTC
58    /// day rotation — see [`crate::series::DiskStore`].
59    #[serde(default)]
60    pub retention_days: Option<u32>,
61    pub trades: Option<PersistDepth>,
62    pub agg_trades: Option<PersistDepth>,
63    pub klines: Option<PersistDepth>,
64    pub tickers: Option<PersistDepth>,
65    pub orderbook_snapshots: Option<PersistDepth>,
66    pub orderbook_deltas: Option<PersistDepth>,
67    pub mark_price: Option<PersistDepth>,
68    pub funding_rate: Option<PersistDepth>,
69    pub open_interest: Option<PersistDepth>,
70    pub liquidations: Option<PersistDepth>,
71    // Extended kinds — opt-in, default off.
72    pub index_price: Option<PersistDepth>,
73    pub block_trade: Option<PersistDepth>,
74    pub auction_event: Option<PersistDepth>,
75    pub composite_index: Option<PersistDepth>,
76    pub volatility_index: Option<PersistDepth>,
77    pub historical_volatility: Option<PersistDepth>,
78    pub long_short_ratio: Option<PersistDepth>,
79    pub taker_volume: Option<PersistDepth>,
80    pub liquidation_bucket: Option<PersistDepth>,
81    pub basis: Option<PersistDepth>,
82    pub insurance_fund: Option<PersistDepth>,
83    pub settlement_event: Option<PersistDepth>,
84    pub predicted_funding: Option<PersistDepth>,
85    pub funding_settlement: Option<PersistDepth>,
86    pub risk_limit: Option<PersistDepth>,
87    pub option_greeks: Option<PersistDepth>,
88    pub mark_price_kline: Option<PersistDepth>,
89    pub index_price_kline: Option<PersistDepth>,
90    pub premium_index_kline: Option<PersistDepth>,
91    pub market_warning: Option<PersistDepth>,
92    pub orderbook_l3: Option<PersistDepth>,
93}
94
95impl Default for PersistenceConfig {
96    fn default() -> Self {
97        Self {
98            enabled: false,
99            retention_days: None,
100            trades: None,
101            agg_trades: None,
102            klines: None,
103            tickers: None,
104            orderbook_snapshots: None,
105            orderbook_deltas: None,
106            mark_price: None,
107            funding_rate: None,
108            open_interest: None,
109            liquidations: None,
110            index_price: None,
111            block_trade: None,
112            auction_event: None,
113            composite_index: None,
114            volatility_index: None,
115            historical_volatility: None,
116            long_short_ratio: None,
117            taker_volume: None,
118            liquidation_bucket: None,
119            basis: None,
120            insurance_fund: None,
121            settlement_event: None,
122            predicted_funding: None,
123            funding_settlement: None,
124            risk_limit: None,
125            option_greeks: None,
126            mark_price_kline: None,
127            index_price_kline: None,
128            premium_index_kline: None,
129            market_warning: None,
130            orderbook_l3: None,
131        }
132    }
133}
134
135impl PersistenceConfig {
136    /// Enable persistence for every supported kind at `Compact` depth.
137    ///
138    /// Produces identical output to the previous `on()` that set all bools to
139    /// `true`. Compact = existing `<Kind>Point` layout, no behavior change.
140    pub fn on() -> Self {
141        let c = Some(PersistDepth::Compact);
142        Self {
143            enabled: true,
144            retention_days: Some(30),
145            trades: c,
146            agg_trades: c,
147            klines: c,
148            tickers: c,
149            orderbook_snapshots: c,
150            orderbook_deltas: c,
151            mark_price: c,
152            funding_rate: c,
153            open_interest: c,
154            liquidations: c,
155            // Extended kinds default off even in on() — they were previously
156            // guarded by `self.enabled` only, so on() must still enable them.
157            index_price: c,
158            block_trade: c,
159            auction_event: c,
160            composite_index: c,
161            volatility_index: c,
162            historical_volatility: c,
163            long_short_ratio: c,
164            taker_volume: c,
165            liquidation_bucket: c,
166            basis: c,
167            insurance_fund: c,
168            settlement_event: c,
169            predicted_funding: c,
170            funding_settlement: c,
171            risk_limit: c,
172            option_greeks: c,
173            mark_price_kline: c,
174            index_price_kline: c,
175            premium_index_kline: c,
176            market_warning: c,
177            orderbook_l3: c,
178        }
179    }
180
181    /// Set the day-file retention window. `None` disables the sweep
182    /// (keep everything forever); `Some(n)` sweeps files older than
183    /// `today - n` days on `DiskStore` open + day rotation.
184    pub fn retention_days(mut self, days: Option<u32>) -> Self { self.retention_days = days; self }
185
186    // Builder methods — accept `Option<PersistDepth>` so callers can do
187    // `.tickers(Some(PersistDepth::Indicators))` or `.tickers(None)`.
188
189    pub fn trades(mut self, depth: Option<PersistDepth>) -> Self { self.trades = depth; self }
190    pub fn agg_trades(mut self, depth: Option<PersistDepth>) -> Self { self.agg_trades = depth; self }
191    pub fn klines(mut self, depth: Option<PersistDepth>) -> Self { self.klines = depth; self }
192    pub fn tickers(mut self, depth: Option<PersistDepth>) -> Self { self.tickers = depth; self }
193    pub fn orderbook_snapshots(mut self, depth: Option<PersistDepth>) -> Self { self.orderbook_snapshots = depth; self }
194    pub fn orderbook_deltas(mut self, depth: Option<PersistDepth>) -> Self { self.orderbook_deltas = depth; self }
195    pub fn mark_price(mut self, depth: Option<PersistDepth>) -> Self { self.mark_price = depth; self }
196    pub fn funding_rate(mut self, depth: Option<PersistDepth>) -> Self { self.funding_rate = depth; self }
197    pub fn open_interest(mut self, depth: Option<PersistDepth>) -> Self { self.open_interest = depth; self }
198    pub fn liquidations(mut self, depth: Option<PersistDepth>) -> Self { self.liquidations = depth; self }
199    pub fn index_price(mut self, depth: Option<PersistDepth>) -> Self { self.index_price = depth; self }
200    pub fn block_trade(mut self, depth: Option<PersistDepth>) -> Self { self.block_trade = depth; self }
201    pub fn auction_event(mut self, depth: Option<PersistDepth>) -> Self { self.auction_event = depth; self }
202    pub fn composite_index(mut self, depth: Option<PersistDepth>) -> Self { self.composite_index = depth; self }
203    pub fn volatility_index(mut self, depth: Option<PersistDepth>) -> Self { self.volatility_index = depth; self }
204    pub fn historical_volatility(mut self, depth: Option<PersistDepth>) -> Self { self.historical_volatility = depth; self }
205    pub fn long_short_ratio(mut self, depth: Option<PersistDepth>) -> Self { self.long_short_ratio = depth; self }
206    pub fn taker_volume(mut self, depth: Option<PersistDepth>) -> Self { self.taker_volume = depth; self }
207    pub fn liquidation_bucket(mut self, depth: Option<PersistDepth>) -> Self { self.liquidation_bucket = depth; self }
208    pub fn basis(mut self, depth: Option<PersistDepth>) -> Self { self.basis = depth; self }
209    pub fn insurance_fund(mut self, depth: Option<PersistDepth>) -> Self { self.insurance_fund = depth; self }
210    pub fn settlement_event(mut self, depth: Option<PersistDepth>) -> Self { self.settlement_event = depth; self }
211    pub fn predicted_funding(mut self, depth: Option<PersistDepth>) -> Self { self.predicted_funding = depth; self }
212    pub fn funding_settlement(mut self, depth: Option<PersistDepth>) -> Self { self.funding_settlement = depth; self }
213    pub fn risk_limit(mut self, depth: Option<PersistDepth>) -> Self { self.risk_limit = depth; self }
214    pub fn option_greeks(mut self, depth: Option<PersistDepth>) -> Self { self.option_greeks = depth; self }
215    pub fn mark_price_kline(mut self, depth: Option<PersistDepth>) -> Self { self.mark_price_kline = depth; self }
216    pub fn index_price_kline(mut self, depth: Option<PersistDepth>) -> Self { self.index_price_kline = depth; self }
217    pub fn premium_index_kline(mut self, depth: Option<PersistDepth>) -> Self { self.premium_index_kline = depth; self }
218    pub fn market_warning(mut self, depth: Option<PersistDepth>) -> Self { self.market_warning = depth; self }
219    pub fn orderbook_l3(mut self, depth: Option<PersistDepth>) -> Self { self.orderbook_l3 = depth; self }
220
221    /// Should `kind` be persisted given the current config?
222    pub fn is_enabled_for(&self, kind: &crate::series::Kind) -> bool {
223        self.depth_for(kind).is_some()
224    }
225
226    /// The configured depth for `kind`, or `None` if persistence is disabled
227    /// for this kind (either master `enabled=false` or the kind's toggle is
228    /// `None`).
229    pub fn depth_for(&self, kind: &crate::series::Kind) -> Option<PersistDepth> {
230        if !self.enabled {
231            return None;
232        }
233        use crate::series::Kind::*;
234        match kind {
235            Trade => self.trades,
236            AggTrade => self.agg_trades,
237            Kline(_) => self.klines,
238            Ticker => self.tickers,
239            Orderbook => self.orderbook_snapshots,
240            OrderbookDelta => self.orderbook_deltas,
241            MarkPrice => self.mark_price,
242            FundingRate => self.funding_rate,
243            OpenInterest => self.open_interest,
244            Liquidation => self.liquidations,
245            IndexPrice => self.index_price,
246            BlockTrade => self.block_trade,
247            AuctionEvent => self.auction_event,
248            CompositeIndex => self.composite_index,
249            VolatilityIndex => self.volatility_index,
250            HistoricalVolatility => self.historical_volatility,
251            LongShortRatio => self.long_short_ratio,
252            TakerVolume => self.taker_volume,
253            LiquidationBucket => self.liquidation_bucket,
254            Basis => self.basis,
255            InsuranceFund => self.insurance_fund,
256            SettlementEvent => self.settlement_event,
257            PredictedFunding => self.predicted_funding,
258            FundingSettlement => self.funding_settlement,
259            RiskLimit => self.risk_limit,
260            OptionGreeks => self.option_greeks,
261            MarkPriceKline(_) => self.mark_price_kline,
262            IndexPriceKline(_) => self.index_price_kline,
263            PremiumIndexKline(_) => self.premium_index_kline,
264            MarketWarning => self.market_warning,
265            OrderbookL3 => self.orderbook_l3,
266            // Mechanical bar aggregators — follow global enabled state at Compact.
267            RangeBar(_) | TickBar(_) | VolumeBar(_) | Footprint(_)
268            | RenkoBar(_, _) | PnfBar(_, _) | KagiBar(_)
269            | CvdLine | ThreeLineBreak { .. } | TpoProfile(_, _)
270            | DollarBar { .. } | TickImbalanceBar { .. }
271            | VolumeImbalanceBar { .. } | RunBar { .. } => {
272                if self.enabled { Some(PersistDepth::Compact) } else { None }
273            }
274            // Private streams are ephemeral — never persisted.
275            OrderUpdate | BalanceUpdate | PositionUpdate => None,
276        }
277    }
278}
digdigdig3_station/persistence.rs

digdigdig3_station/
persistence.rs
