rust_web_server/disk_guard/
mod.rs

1//! Keep rws from filling a disk (18.15.0): before writing to a file it
2//! keeps growing — the access log, the disk cache — rws asks
3//! [`has_room`]. Below `RWS_CONFIG_MIN_FREE_DISK` free on that file's
4//! filesystem the write is skipped (and counted) until space comes back:
5//! a full disk costs log lines and cache entries, never the server or
6//! whatever shares the disk with it.
7//!
8//! `RWS_CONFIG_MIN_FREE_DISK`: a size (`2G`, `500M`) or a share of the
9//! filesystem (`10%`); unset or empty means the larger of 1% and 1 GiB (the
10//! space counted is what an unprivileged process may use, so the root
11//! reserve most filesystems keep is already set aside); `0` turns the guard
12//! off. Free space is read (`statvfs`) at most every
13//! [`RECHECK`] per directory. One line is logged when a directory runs low
14//! and one when it recovers.
15
16use std::collections::HashMap;
17use std::path::{Path, PathBuf};
18use std::sync::atomic::{AtomicU64, Ordering};
19use std::sync::{Mutex, OnceLock};
20use std::time::{Duration, Instant};
21
22/// How long a free-space reading is reused.
23pub const RECHECK: Duration = Duration::from_secs(5);
24
25/// `RWS_CONFIG_MIN_FREE_DISK`.
26pub const RWS_CONFIG_MIN_FREE_DISK: &str = "RWS_CONFIG_MIN_FREE_DISK";
27
28#[derive(Clone, Copy, Debug, PartialEq, Eq)]
29enum Floor {
30    Off,
31    Bytes(u64),
32    Percent(u64),
33    /// The default: the larger of 1% and 1 GiB.
34    Default,
35}
36
37fn floor() -> Floor {
38    static FLOOR: OnceLock<Floor> = OnceLock::new();
39    *FLOOR.get_or_init(|| {
40        let value = std::env::var(RWS_CONFIG_MIN_FREE_DISK).unwrap_or_default();
41        // Tests don't depend on how full the machine running them is.
42        if cfg!(test) && value.trim().is_empty() {
43            return Floor::Off;
44        }
45        parse_floor(&value).unwrap_or_else(|| {
46            eprintln!("{RWS_CONFIG_MIN_FREE_DISK}: not a size or a percentage; using the default (1%, at least 1G)");
47            Floor::Default
48        })
49    })
50}
51
52fn parse_floor(value: &str) -> Option<Floor> {
53    let value = value.trim();
54    if value.is_empty() {
55        return Some(Floor::Default);
56    }
57    if let Some(percent) = value.strip_suffix('%') {
58        let percent: u64 = percent.trim().parse().ok().filter(|p| *p <= 100)?;
59        return Some(if percent == 0 { Floor::Off } else { Floor::Percent(percent) });
60    }
61    let bytes = crate::log::access_sink::parse_size(value)?;
62    Some(if bytes == 0 { Floor::Off } else { Floor::Bytes(bytes) })
63}
64
65/// Bytes that must stay free on a filesystem of `total` bytes.
66fn required(floor: Floor, total: u64) -> u64 {
67    match floor {
68        Floor::Off => 0,
69        Floor::Bytes(bytes) => bytes,
70        Floor::Percent(percent) => total / 100 * percent,
71        Floor::Default => (total / 100).max(1 << 30).min(total / 2),
72    }
73}
74
75/// `(available to unprivileged writers, total)` bytes of the filesystem
76/// holding `dir`.
77#[cfg(all(unix, any(feature = "http1", feature = "http2")))]
78fn space(dir: &Path) -> Option<(u64, u64)> {
79    use std::os::unix::ffi::OsStrExt;
80    let path = std::ffi::CString::new(dir.as_os_str().as_bytes()).ok()?;
81    // SAFETY: `stat` is plain old data written by statvfs; `path` is a valid
82    // NUL-terminated string that outlives the call.
83    let mut stat: libc::statvfs = unsafe { std::mem::zeroed() };
84    if unsafe { libc::statvfs(path.as_ptr(), &mut stat) } != 0 {
85        return None;
86    }
87    let block = stat.f_frsize as u64;
88    Some((stat.f_bavail as u64 * block, stat.f_blocks as u64 * block))
89}
90
91#[cfg(not(all(unix, any(feature = "http1", feature = "http2"))))]
92fn space(_dir: &Path) -> Option<(u64, u64)> {
93    None
94}
95
96struct Reading {
97    at: Instant,
98    free: u64,
99    total: u64,
100    low: bool,
101}
102
103fn readings() -> &'static Mutex<HashMap<PathBuf, Reading>> {
104    static READINGS: OnceLock<Mutex<HashMap<PathBuf, Reading>>> = OnceLock::new();
105    READINGS.get_or_init(Mutex::default)
106}
107
108/// The directory whose filesystem `file` lives on.
109fn directory_of(file: &Path) -> PathBuf {
110    if file.is_dir() {
111        return file.to_path_buf();
112    }
113    match file.parent() {
114        Some(parent) if !parent.as_os_str().is_empty() => parent.to_path_buf(),
115        _ => PathBuf::from("."),
116    }
117}
118
119/// Whether there's room to keep writing to `file` (or into the directory
120/// `file`). `true` when the guard is off or the space can't be read.
121pub fn has_room(file: &Path) -> bool {
122    has_room_with(file, floor())
123}
124
125fn has_room_with(file: &Path, floor: Floor) -> bool {
126    if floor == Floor::Off {
127        return true;
128    }
129    let dir = directory_of(file);
130    let mut readings = readings().lock().unwrap_or_else(|e| e.into_inner());
131    if let Some(reading) = readings.get(&dir) {
132        if reading.at.elapsed() < RECHECK && !cfg!(test) {
133            return !reading.low;
134        }
135    }
136    let Some((free, total)) = space(&dir) else { return true };
137    let low = free < required(floor, total);
138    let was_low = readings.get(&dir).is_some_and(|r| r.low);
139    if low && !was_low {
140        eprintln!(
141            "disk: {} has {} MiB free, under the {} MiB {RWS_CONFIG_MIN_FREE_DISK} floor; rws stops writing there until space is freed",
142            dir.display(),
143            free >> 20,
144            required(floor, total) >> 20
145        );
146    } else if !low && was_low {
147        eprintln!("disk: {} has {} MiB free again; rws writes there again", dir.display(), free >> 20);
148    }
149    readings.insert(dir, Reading { at: Instant::now(), free, total, low });
150    !low
151}
152
153static DROPPED_LOG_LINES: AtomicU64 = AtomicU64::new(0);
154static SKIPPED_CACHE_WRITES: AtomicU64 = AtomicU64::new(0);
155
156/// Access-log lines not written for want of space.
157pub(crate) fn dropped_log_lines(lines: u64) {
158    DROPPED_LOG_LINES.fetch_add(lines, Ordering::Relaxed);
159}
160
161/// Disk-cache entries not stored for want of space.
162pub(crate) fn skipped_cache_write() {
163    SKIPPED_CACHE_WRITES.fetch_add(1, Ordering::Relaxed);
164}
165
166/// Every directory checked so far: `(directory, free bytes, total bytes,
167/// low)`, sorted.
168pub fn status() -> Vec<(String, u64, u64, bool)> {
169    let readings = readings().lock().unwrap_or_else(|e| e.into_inner());
170    let mut out: Vec<_> = readings.iter().map(|(dir, r)| (dir.display().to_string(), r.free, r.total, r.low)).collect();
171    out.sort();
172    out
173}
174
175/// `rws_disk_free_bytes{dir}`, `rws_disk_low{dir}` and the drop counters;
176/// empty until a directory has been checked.
177pub(crate) fn prometheus_text() -> String {
178    let status = status();
179    if status.is_empty() {
180        return String::new();
181    }
182    let label = |dir: &str| dir.replace('\\', "\\\\").replace('"', "\\\"");
183    let mut out = String::from("# HELP rws_disk_free_bytes Free space where rws writes (access log, disk cache)\n# TYPE rws_disk_free_bytes gauge\n");
184    for (dir, free, _, _) in &status {
185        out.push_str(&format!("rws_disk_free_bytes{{dir=\"{}\"}} {free}\n", label(dir)));
186    }
187    out.push_str(&format!("# HELP rws_disk_low 1 while a directory is under {RWS_CONFIG_MIN_FREE_DISK} and rws isn't writing there\n# TYPE rws_disk_low gauge\n"));
188    for (dir, _, _, low) in &status {
189        out.push_str(&format!("rws_disk_low{{dir=\"{}\"}} {}\n", label(dir), *low as u8));
190    }
191    out.push_str("# HELP rws_access_log_dropped_lines_total Access-log lines not written because the disk was low\n# TYPE rws_access_log_dropped_lines_total counter\n");
192    out.push_str(&format!("rws_access_log_dropped_lines_total {}\n", DROPPED_LOG_LINES.load(Ordering::Relaxed)));
193    out.push_str("# HELP rws_cache_skipped_writes_total Disk-cache entries not stored because the disk was low\n# TYPE rws_cache_skipped_writes_total counter\n");
194    out.push_str(&format!("rws_cache_skipped_writes_total {}\n", SKIPPED_CACHE_WRITES.load(Ordering::Relaxed)));
195    out
196}
197
198#[cfg(test)]
199mod tests;
rust_web_server/disk_guard/mod.rs

rust_web_server/disk_guard/
mod.rs
