kernel/
degradation.rs

1// This Source Code Form is subject to the terms of the Mozilla Public
2// License, v. 2.0. If a copy of the MPL was not distributed with this
3// file, You can obtain one at https://mozilla.org/MPL/2.0/.
4// Copyright (c) 2026 2youg1 and the sprawling contributors
5
6//! Degrade truthfully: which resource is the bottleneck, the readings
7//! that show it, and what the person can do about it.
8//! Readings are sampled by the caller; this module only decides. The one
9//! state that stops the city taking on new work is a disk close to full,
10//! and the refusal carries that [`Degradation::DiskLow`] so the door can
11//! say how much to free.
12//! The part `crates/kernel/spec/Degradation.lean` specifies this module.
13
14use std::time::Duration;
15
16/// How many commit floors the durable watermark may lag before the disk
17/// counts as slow. Relative, so a spinning disk whose fsync is simply
18/// slow is not reported as degraded.
19pub const DISK_SLOW_FACTOR: u32 = 8;
20/// The lag below which the disk never counts as slow, so a fast device's
21/// jitter of a few commit floors does not flap the state.
22pub const DISK_SLOW_MIN: Duration = Duration::from_millis(20);
23/// One 60 Hz frame: the scheduling delay a person starts to see.
24pub const CPU_SATURATED_DELAY: Duration = Duration::from_micros(16_667);
25/// The free-space floor is this fraction of the volume, before clamping.
26pub const FREE_SPACE_FLOOR_DIVISOR: u64 = 50;
27/// Room for the next snapshot and git's objects even on a small volume.
28pub const FREE_SPACE_FLOOR_MIN: u64 = 256 * 1024 * 1024;
29/// A large volume does not hold back tens of gigabytes as its floor.
30pub const FREE_SPACE_FLOOR_MAX: u64 = 4 * 1024 * 1024 * 1024;
31
32/// One sample of the resources the harness depends on.
33#[derive(Debug, Clone, Copy, PartialEq, Eq)]
34pub struct ResourceReadings {
35    /// How long the oldest appended-but-not-durable event has waited.
36    pub durable_lag: Duration,
37    /// The measured fsync median of the city's volume: this step's floor.
38    pub commit_floor: Duration,
39    pub volume: VolumeSpace,
40    /// Runs waiting because available memory would not hold one more.
41    pub queued_runs: u32,
42    /// The accounting thread's median wake-to-run delay.
43    pub schedule_delay: Duration,
44}
45
46/// The city's volume: free space and capacity, read together.
47#[derive(Debug, Clone, Copy, PartialEq, Eq)]
48pub struct VolumeSpace {
49    pub free_bytes: u64,
50    pub total_bytes: u64,
51}
52
53/// A resource that is the bottleneck, carrying the readings that decided it.
54#[derive(Debug, Clone, Copy, PartialEq, Eq)]
55pub enum Degradation {
56    DiskSlow {
57        durable_lag: Duration,
58        commit_floor: Duration,
59    },
60    DiskLow {
61        free_bytes: u64,
62        floor_bytes: u64,
63    },
64    MemoryTight {
65        queued_runs: u32,
66    },
67    CpuSaturated {
68        schedule_delay: Duration,
69    },
70}
71
72/// What the person can do to end a degradation.
73#[derive(Debug, Clone, Copy, PartialEq, Eq)]
74pub enum Recovery {
75    ReduceDiskLoad,
76    FreeDiskSpace { at_least_bytes: u64 },
77    FreeMemory,
78    ReduceCpuLoad,
79}
80
81impl Degradation {
82    pub fn recovery(&self) -> Recovery {
83        match *self {
84            Degradation::DiskSlow { .. } => Recovery::ReduceDiskLoad,
85            Degradation::DiskLow {
86                free_bytes,
87                floor_bytes,
88            } => Recovery::FreeDiskSpace {
89                at_least_bytes: floor_bytes.saturating_sub(free_bytes),
90            },
91            Degradation::MemoryTight { .. } => Recovery::FreeMemory,
92            Degradation::CpuSaturated { .. } => Recovery::ReduceCpuLoad,
93        }
94    }
95}
96
97/// Every degradation the readings show, in declaration order, without
98/// allocating.
99pub fn assess(readings: &ResourceReadings) -> impl Iterator<Item = Degradation> {
100    [
101        disk_slow(readings),
102        disk_low(readings.volume),
103        memory_tight(readings),
104        cpu_saturated(readings),
105    ]
106    .into_iter()
107    .flatten()
108}
109
110/// Whether the city takes on new work. A volume below its free-space
111/// floor refuses with the [`Degradation::DiskLow`] it shows; every other
112/// degradation admits, because it only slows work down.
113///
114/// # Errors
115/// The `DiskLow` degradation when `volume` is below its floor.
116pub fn admit_work(volume: VolumeSpace) -> Result<(), Degradation> {
117    disk_low(volume).map_or(Ok(()), Err)
118}
119
120// `clamp` panics when its bounds are crossed; these two are constants,
121// so the build refuses a pair that would.
122const _: () = assert!(FREE_SPACE_FLOOR_MIN <= FREE_SPACE_FLOOR_MAX);
123
124/// The free-space floor a volume of `volume_bytes` derives.
125pub fn free_space_floor(volume_bytes: u64) -> u64 {
126    (volume_bytes / FREE_SPACE_FLOOR_DIVISOR).clamp(FREE_SPACE_FLOOR_MIN, FREE_SPACE_FLOOR_MAX)
127}
128
129fn disk_slow(r: &ResourceReadings) -> Option<Degradation> {
130    let relative_line = r.commit_floor.saturating_mul(DISK_SLOW_FACTOR);
131    (r.durable_lag > relative_line.max(DISK_SLOW_MIN)).then_some(Degradation::DiskSlow {
132        durable_lag: r.durable_lag,
133        commit_floor: r.commit_floor,
134    })
135}
136
137fn disk_low(volume: VolumeSpace) -> Option<Degradation> {
138    let floor_bytes = free_space_floor(volume.total_bytes);
139    (volume.free_bytes < floor_bytes).then_some(Degradation::DiskLow {
140        free_bytes: volume.free_bytes,
141        floor_bytes,
142    })
143}
144
145fn memory_tight(r: &ResourceReadings) -> Option<Degradation> {
146    (r.queued_runs > 0).then_some(Degradation::MemoryTight {
147        queued_runs: r.queued_runs,
148    })
149}
150
151fn cpu_saturated(r: &ResourceReadings) -> Option<Degradation> {
152    (r.schedule_delay > CPU_SATURATED_DELAY).then_some(Degradation::CpuSaturated {
153        schedule_delay: r.schedule_delay,
154    })
155}
156
157#[cfg(test)]
158mod tests;
kernel/degradation.rs

kernel/
degradation.rs
