Skip to main content

object_log/
budget.rs

1//! Durable-ops budget controller (TD-004).
2
3use std::time::{Duration, Instant};
4
5/// How the engine resolves budget pressure vs admitted latency.
6#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
7pub enum BudgetMode {
8    /// Deadline flushes always run (may overdraft). Default.
9    #[default]
10    LatencyPriority,
11    /// New produces may wait for tokens; admitted deadline flushes still complete.
12    BudgetPriority,
13    /// Refuse admission when predicted media ops cannot be reserved.
14    FailClosed,
15}
16
17/// Configuration for the media-ops budget controller.
18#[derive(Clone, Copy, Debug)]
19pub struct BudgetConfig {
20    /// Master switch. Default **true** (Fjord-first).
21    pub enabled: bool,
22    /// How deadline flushes interact with an empty token bucket.
23    pub mode: BudgetMode,
24    /// Hard cap on durable ops/sec (None = no hard cap beyond capacity×fraction).
25    pub budget_per_sec_cap: Option<f64>,
26    /// Soft floor on effective budget when set (must be ≤ cap).
27    pub budget_per_sec_floor: Option<f64>,
28    /// Fraction of capacity used as soft budget target.
29    pub budget_fraction: f64,
30    /// Capacity when no probe/EWMA is available (S3 default path).
31    pub default_capacity_per_sec: f64,
32    /// Token fill ratio above which early-flush (effective linger = 0) is allowed.
33    pub early_flush_fill_ratio: f64,
34    /// Suppress consecutive early-flushes for this long after one fires.
35    pub early_flush_cooldown: Duration,
36    /// Early-flush only when the mutable queue holds at most this many bytes.
37    /// Combined with [`Self::early_flush_idle`]: bulk continuous ingest should
38    /// not early-flush. `0` disables the byte gate (idle gate may still apply).
39    pub early_flush_max_queued_bytes: usize,
40    /// Early-flush only if no produce has been enqueued for at least this long
41    /// (true idle). Sustained produce keeps resetting the timer so bulk waits
42    /// full linger / `flush`. `ZERO` disables the idle gate.
43    pub early_flush_idle: Duration,
44    /// Max wait for tokens under [`BudgetMode::BudgetPriority`].
45    pub admission_timeout: Duration,
46}
47
48impl Default for BudgetConfig {
49    fn default() -> Self {
50        Self {
51            enabled: true,
52            mode: BudgetMode::LatencyPriority,
53            budget_per_sec_cap: None,
54            budget_per_sec_floor: None,
55            budget_fraction: 0.5,
56            // Conservative S3-class default; Local probe/EWMA will refine.
57            default_capacity_per_sec: 50.0,
58            early_flush_fill_ratio: 0.5,
59            early_flush_cooldown: Duration::from_millis(5),
60            // ~4 MiB: single-op / sparse OK when also idle.
61            early_flush_max_queued_bytes: 4 * 1024 * 1024,
62            // Require a quiet gap so continuous bulk cannot be sliced into 4MiB seals.
63            early_flush_idle: Duration::from_millis(10),
64            admission_timeout: Duration::from_millis(100),
65        }
66    }
67}
68
69impl BudgetConfig {
70    /// Validate floor ≤ cap when both set.
71    pub fn validate(&self) -> Result<(), String> {
72        if let (Some(cap), Some(floor)) = (self.budget_per_sec_cap, self.budget_per_sec_floor)
73            && floor > cap
74        {
75            return Err(format!(
76                "budget_per_sec_floor ({floor}) > budget_per_sec_cap ({cap})"
77            ));
78        }
79        if !(0.0..=1.0).contains(&self.budget_fraction) {
80            return Err("budget_fraction must be in [0, 1]".into());
81        }
82        if self.default_capacity_per_sec < 0.0 {
83            return Err("default_capacity_per_sec must be >= 0".into());
84        }
85        Ok(())
86    }
87
88    /// Compute effective durable_ops/sec from optional measurements.
89    pub fn effective_budget_per_sec(
90        &self,
91        startup_capacity: Option<f64>,
92        ongoing_capacity: Option<f64>,
93    ) -> f64 {
94        let capacity = ongoing_capacity
95            .or(startup_capacity)
96            .unwrap_or(self.default_capacity_per_sec)
97            .max(0.0);
98        let mut effective = capacity * self.budget_fraction;
99        if let Some(cap) = self.budget_per_sec_cap {
100            effective = effective.min(cap);
101        }
102        if let Some(floor) = self.budget_per_sec_floor {
103            effective = effective.max(floor);
104            if let Some(cap) = self.budget_per_sec_cap {
105                effective = effective.min(cap);
106            }
107        }
108        effective
109    }
110}
111
112/// Runtime token-bucket + EWMA state (held under the engine queue lock or own mutex).
113#[derive(Debug)]
114pub struct BudgetRuntime {
115    pub config: BudgetConfig,
116    pub tokens: f64,
117    pub effective_budget_per_sec: f64,
118    pub last_refill: Instant,
119    pub predicted_media_ops: f64,
120    pub last_early_flush: Option<Instant>,
121    pub media_ops_total: u64,
122    pub overdraft_total: u64,
123    pub flushes_total: u64,
124    pub undersized_deadline_flushes: u64,
125    pub startup_capacity: Option<f64>,
126    pub ongoing_capacity: Option<f64>,
127    /// EWMA of observed media ops per flush.
128    media_ops_ewma: f64,
129    ewma_inited: bool,
130}
131
132impl BudgetRuntime {
133    pub fn new(config: BudgetConfig) -> Self {
134        let effective = config.effective_budget_per_sec(None, None);
135        Self {
136            config,
137            tokens: effective.max(1.0),
138            effective_budget_per_sec: effective,
139            last_refill: Instant::now(),
140            predicted_media_ops: 1.0,
141            last_early_flush: None,
142            media_ops_total: 0,
143            overdraft_total: 0,
144            flushes_total: 0,
145            undersized_deadline_flushes: 0,
146            startup_capacity: None,
147            ongoing_capacity: None,
148            media_ops_ewma: 1.0,
149            ewma_inited: false,
150        }
151    }
152
153    pub fn refill(&mut self, now: Instant) {
154        let dt = now.duration_since(self.last_refill).as_secs_f64();
155        if dt <= 0.0 {
156            return;
157        }
158        let burst = self.effective_budget_per_sec.max(1.0);
159        self.tokens = (self.tokens + dt * self.effective_budget_per_sec).min(burst);
160        self.last_refill = now;
161    }
162
163    pub fn fill_ratio(&self) -> f64 {
164        let burst = self.effective_budget_per_sec.max(1.0);
165        (self.tokens / burst).clamp(0.0, 1.0)
166    }
167
168    /// Whether headroom + idle allow early flush (effective linger 0).
169    ///
170    /// - `queued_bytes` must be ≤ `early_flush_max_queued_bytes` (if that cap &gt; 0)
171    /// - `last_enqueue` must be at least `early_flush_idle` ago (if idle &gt; 0)
172    ///
173    /// Sustained produce keeps `last_enqueue` fresh → no early-flush → linger/`flush` pack.
174    pub fn allow_early_flush(
175        &self,
176        now: Instant,
177        queued_bytes: usize,
178        last_enqueue: Option<Instant>,
179    ) -> bool {
180        if !self.config.enabled {
181            return false;
182        }
183        let max_q = self.config.early_flush_max_queued_bytes;
184        if max_q > 0 && queued_bytes > max_q {
185            return false;
186        }
187        let idle_need = self.config.early_flush_idle;
188        if !idle_need.is_zero() {
189            let Some(last) = last_enqueue else {
190                return false;
191            };
192            if now.duration_since(last) < idle_need {
193                return false;
194            }
195        }
196        if self.fill_ratio() < self.config.early_flush_fill_ratio {
197            return false;
198        }
199        if let Some(last) = self.last_early_flush
200            && now.duration_since(last) < self.config.early_flush_cooldown
201        {
202            return false;
203        }
204        true
205    }
206
207    pub fn note_early_flush(&mut self, now: Instant) {
208        self.last_early_flush = Some(now);
209    }
210
211    /// Consume media ops after a flush (always succeeds; may overdraft).
212    pub fn consume_after_flush(&mut self, media_ops: u64, now: Instant) {
213        self.refill(now);
214        let cost = media_ops as f64;
215        if cost <= self.tokens {
216            self.tokens -= cost;
217        } else {
218            self.overdraft_total += (cost - self.tokens).ceil() as u64;
219            self.tokens = 0.0;
220        }
221        self.media_ops_total += media_ops;
222        self.flushes_total += 1;
223        // EWMA of cost
224        const ALPHA: f64 = 0.2;
225        if !self.ewma_inited {
226            self.media_ops_ewma = cost.max(0.1);
227            self.ewma_inited = true;
228        } else {
229            self.media_ops_ewma = ALPHA * cost + (1.0 - ALPHA) * self.media_ops_ewma;
230        }
231        self.predicted_media_ops = self.media_ops_ewma.max(0.1);
232        // Ongoing capacity estimate from implied rate if we had overdraft recently — keep simple:
233        // if we observed media ops, capacity estimate ≈ max(default, ops in last sec proxy)
234        // Use inverse of nothing; leave ongoing for optional future. Bootstrap prediction only.
235    }
236
237    /// Admission check for Durable/Sequenced produces under fail_closed / budget_priority.
238    /// Returns false if produce should be rejected (fail_closed) or caller should wait.
239    pub fn can_admit_now(&mut self, now: Instant) -> bool {
240        if !self.config.enabled {
241            return true;
242        }
243        self.refill(now);
244        match self.config.mode {
245            BudgetMode::LatencyPriority => true,
246            BudgetMode::BudgetPriority | BudgetMode::FailClosed => {
247                self.tokens >= self.predicted_media_ops * 0.5
248            }
249        }
250    }
251
252    pub fn reserve_for_fail_closed(&mut self, now: Instant) -> bool {
253        if !self.config.enabled || self.config.mode != BudgetMode::FailClosed {
254            return true;
255        }
256        self.refill(now);
257        let need = self.predicted_media_ops;
258        if self.tokens >= need {
259            self.tokens -= need;
260            true
261        } else {
262            false
263        }
264    }
265}
266
267/// Why an effective value was chosen (for inspectability).
268#[derive(Clone, Copy, Debug, PartialEq, Eq)]
269pub enum EffectiveReason {
270    /// Operator config alone.
271    Configured,
272    /// Limited by config cap.
273    ConfigCap,
274    /// From startup probe.
275    StartupProbe,
276    /// From ongoing measurement.
277    Ongoing,
278    /// Library / profile default capacity.
279    DefaultCapacity,
280    /// Budget controller disabled.
281    Disabled,
282}
283
284/// One inspectable knob with layered sources.
285#[derive(Clone, Debug, PartialEq)]
286pub struct EffectiveKnob<T: Clone> {
287    /// Operator-configured value, if any.
288    pub configured: Option<T>,
289    /// Startup measurement, if any.
290    pub startup_measured: Option<T>,
291    /// Ongoing measurement, if any.
292    pub ongoing_measured: Option<T>,
293    /// Value currently in force.
294    pub effective: T,
295    /// Why `effective` was chosen.
296    pub reason: EffectiveReason,
297}
298
299/// Point-in-time controller + buffer view.
300#[derive(Clone, Debug)]
301pub struct PipelineSnapshot {
302    /// Effective durable-ops/sec budget.
303    pub budget_per_sec: EffectiveKnob<f64>,
304    /// Effective linger currently used for scheduling (0 = early flush).
305    pub effective_linger_ms: EffectiveKnob<u64>,
306    /// Operator max linger (ms).
307    pub max_linger_ms: u64,
308    /// Token fill ratio in [0, 1].
309    pub token_fill_ratio: f64,
310    /// Cumulative media ops consumed.
311    pub media_ops_total: u64,
312    /// Cumulative overdraft media ops.
313    pub overdraft_total: u64,
314    /// Successful flushes.
315    pub flushes_total: u64,
316    /// Deadline flushes that were under size soft target (counted when metered).
317    pub undersized_deadline_flushes: u64,
318    /// Predicted media ops per flush (EWMA).
319    pub predicted_media_ops: f64,
320    /// Whether budget controller is enabled.
321    pub budget_enabled: bool,
322    /// Active budget conflict mode.
323    pub budget_mode: BudgetMode,
324}