nodedb_mem/
over_release.rs

1// SPDX-License-Identifier: Apache-2.0
2
3//! Over-release detection on the live reservation-release path.
4//!
5//! [`atomic_saturating_sub`](crate::budget::atomic_saturating_sub) clamps a
6//! below-zero release to zero so a wrapped counter never reads as 100 %
7//! utilization. The clamp hides the drift from every `allocated()` reader —
8//! [`OverRelease`] is the counter that keeps it visible, split per budget
9//! layer so an operator can tell which layer's release accounting drifted.
10
11use std::sync::atomic::{AtomicUsize, Ordering};
12
13use nodedb_types::{DatabaseId, TenantId};
14
15use crate::engine::EngineId;
16
17/// The reservation a release belongs to, carried into the warning so a
18/// reader can attribute drift to a database, tenant, and engine.
19#[derive(Debug, Clone, Copy)]
20pub(crate) struct ReleaseIdentity {
21    pub db: DatabaseId,
22    pub tenant: TenantId,
23    pub engine: EngineId,
24}
25
26/// A budget layer in the four-level reservation hierarchy.
27#[derive(Debug, Clone, Copy, PartialEq, Eq)]
28pub enum Layer {
29    Global,
30    Database,
31    Tenant,
32    Engine,
33}
34
35/// Per-layer over-release counters shared by every releaser through
36/// `Arc<GlobalCounter>`.
37///
38/// A nonzero counter means some call site released more bytes than that
39/// layer ever held — the "memory release exceeds allocation" symptom, now
40/// counted at the site that actually releases memory instead of at a dead
41/// code path.
42#[derive(Debug, Default)]
43pub struct OverRelease {
44    global: AtomicUsize,
45    database: AtomicUsize,
46    tenant: AtomicUsize,
47    engine: AtomicUsize,
48}
49
50impl OverRelease {
51    /// Build a fresh set of counters, all zero.
52    pub fn new() -> Self {
53        Self::default()
54    }
55
56    /// Record an over-release event against `layer`. `shortfall` is a
57    /// nonzero test, not a byte total — the counter counts events, and
58    /// never accumulates the shortfall bytes themselves. A zero shortfall
59    /// is a no-op.
60    pub fn record(&self, layer: Layer, shortfall: usize) {
61        if shortfall == 0 {
62            return;
63        }
64        self.counter(layer).fetch_add(1, Ordering::Relaxed);
65    }
66
67    /// Number of over-release events recorded against the global layer.
68    pub fn global(&self) -> usize {
69        self.global.load(Ordering::Relaxed)
70    }
71
72    /// Number of over-release events recorded against the database layer.
73    pub fn database(&self) -> usize {
74        self.database.load(Ordering::Relaxed)
75    }
76
77    /// Number of over-release events recorded against the tenant layer.
78    pub fn tenant(&self) -> usize {
79        self.tenant.load(Ordering::Relaxed)
80    }
81
82    /// Number of over-release events recorded against the engine layer.
83    pub fn engine(&self) -> usize {
84        self.engine.load(Ordering::Relaxed)
85    }
86
87    /// Total over-release events across every layer.
88    ///
89    /// Sums four independent relaxed loads — not a single atomic read, so
90    /// a concurrent `record` on another layer can land between them. The
91    /// count settles once all releasers finish; it is not a point-in-time
92    /// snapshot under concurrent load.
93    pub fn total(&self) -> usize {
94        self.global() + self.database() + self.tenant() + self.engine()
95    }
96
97    fn counter(&self, layer: Layer) -> &AtomicUsize {
98        match layer {
99            Layer::Global => &self.global,
100            Layer::Database => &self.database,
101            Layer::Tenant => &self.tenant,
102            Layer::Engine => &self.engine,
103        }
104    }
105}
106
107/// Release `size` bytes from one layer's counter and record any shortfall.
108///
109/// The shared call site `ReservationToken::drop` and `ReserveScope::drop`
110/// both use to release one layer of the four-level hierarchy. Returns the
111/// shortfall so a caller that also warns (`ReservationToken::drop`) does
112/// not need to recompute it.
113pub(crate) fn release_layer(
114    counter: &AtomicUsize,
115    size: usize,
116    layer: Layer,
117    over_release: &OverRelease,
118    id: ReleaseIdentity,
119) -> usize {
120    let shortfall = crate::budget::atomic_saturating_sub(counter, size);
121    if shortfall == 0 {
122        return 0;
123    }
124    over_release.record(layer, shortfall);
125    tracing::warn!(
126        layer = ?layer,
127        requested = size,
128        shortfall,
129        db = ?id.db,
130        tenant = ?id.tenant,
131        engine = ?id.engine,
132        "memory release exceeds allocation (WAL replay or accounting drift)"
133    );
134    shortfall
135}
136
137#[cfg(test)]
138mod tests {
139    use super::*;
140
141    #[test]
142    fn record_ignores_zero_shortfall() {
143        let counters = OverRelease::new();
144        counters.record(Layer::Engine, 0);
145        assert_eq!(counters.engine(), 0);
146        assert_eq!(counters.total(), 0);
147    }
148
149    #[test]
150    fn record_counts_events_not_bytes() {
151        let counters = OverRelease::new();
152        counters.record(Layer::Engine, 10);
153        counters.record(Layer::Engine, 9999);
154        assert_eq!(
155            counters.engine(),
156            2,
157            "record counts the number of over-release events, not shortfall bytes"
158        );
159    }
160
161    #[test]
162    fn each_layer_tracks_independently() {
163        let counters = OverRelease::new();
164        counters.record(Layer::Global, 1);
165        counters.record(Layer::Database, 1);
166        counters.record(Layer::Tenant, 1);
167        counters.record(Layer::Engine, 1);
168
169        assert_eq!(counters.global(), 1);
170        assert_eq!(counters.database(), 1);
171        assert_eq!(counters.tenant(), 1);
172        assert_eq!(counters.engine(), 1);
173        assert_eq!(counters.total(), 4);
174    }
175}
nodedb_mem/over_release.rs

nodedb_mem/
over_release.rs
