rws_server/disk_guard/
mod.rs

1//! Keep rws from filling a disk (18.15.0): before writing to a file it
2//! keeps growing — the access log, the disk cache — rws asks
3//! [`has_room`]. Below `RWS_CONFIG_MIN_FREE_DISK` free on that file's
4//! filesystem the write is skipped (and counted) until space comes back:
5//! a full disk costs log lines and cache entries, never the server or
6//! whatever shares the disk with it.
7//!
8//! `RWS_CONFIG_MIN_FREE_DISK`: a size (`2G`, `500M`) or a share of the
9//! filesystem (`10%`); unset or empty means the larger of 1% and 1 GiB (the
10//! space counted is what an unprivileged process may use, so the root
11//! reserve most filesystems keep is already set aside); `0` turns the guard
12//! off. Free space is read (`statvfs`) at most every
13//! [`RECHECK`] per directory. One line is logged when a directory runs low
14//! and one when it recovers.
15
16use std::collections::HashMap;
17use std::path::{Path, PathBuf};
18use std::sync::atomic::{AtomicU64, Ordering};
19use std::sync::{Mutex, OnceLock};
20use std::time::{Duration, Instant};
21
22/// How long a free-space reading is reused.
23pub const RECHECK: Duration = Duration::from_secs(5);
24
25/// `RWS_CONFIG_MIN_FREE_DISK`.
26pub const RWS_CONFIG_MIN_FREE_DISK: &str = "RWS_CONFIG_MIN_FREE_DISK";
27
28#[derive(Clone, Copy, Debug, PartialEq, Eq)]
29enum Floor {
30    Off,
31    Bytes(u64),
32    Percent(u64),
33    /// The default: the larger of 1% and 1 GiB.
34    Default,
35}
36
37fn floor() -> Floor {
38    static FLOOR: OnceLock<Floor> = OnceLock::new();
39    *FLOOR.get_or_init(|| {
40        let value = std::env::var(RWS_CONFIG_MIN_FREE_DISK).unwrap_or_default();
41        // Tests don't depend on how full the machine running them is.
42        if cfg!(test) && value.trim().is_empty() {
43            return Floor::Off;
44        }
45        parse_floor(&value).unwrap_or_else(|| {
46            eprintln!("{RWS_CONFIG_MIN_FREE_DISK}: not a size or a percentage; using the default (1%, at least 1G)");
47            Floor::Default
48        })
49    })
50}
51
52fn parse_floor(value: &str) -> Option<Floor> {
53    let value = value.trim();
54    if value.is_empty() {
55        return Some(Floor::Default);
56    }
57    if let Some(percent) = value.strip_suffix('%') {
58        let percent: u64 = percent.trim().parse().ok().filter(|p| *p <= 100)?;
59        return Some(if percent == 0 { Floor::Off } else { Floor::Percent(percent) });
60    }
61    let bytes = parse_size(value)?;
62    Some(if bytes == 0 { Floor::Off } else { Floor::Bytes(bytes) })
63}
64
65/// A byte count: digits, optionally followed by `K`, `M` or `G` (×1024).
66#[doc(hidden)]
67pub fn parse_size(value: &str) -> Option<u64> {
68    let value = value.trim();
69    let (digits, multiplier) = match value.as_bytes().last()? {
70        b'k' | b'K' => (&value[..value.len() - 1], 1 << 10),
71        b'm' | b'M' => (&value[..value.len() - 1], 1 << 20),
72        b'g' | b'G' => (&value[..value.len() - 1], 1 << 30),
73        _ => (value, 1),
74    };
75    digits.trim().parse::<u64>().ok()?.checked_mul(multiplier)
76}
77
78/// Bytes that must stay free on a filesystem of `total` bytes.
79fn required(floor: Floor, total: u64) -> u64 {
80    match floor {
81        Floor::Off => 0,
82        Floor::Bytes(bytes) => bytes,
83        Floor::Percent(percent) => total / 100 * percent,
84        Floor::Default => (total / 100).max(1 << 30).min(total / 2),
85    }
86}
87
88/// `(available to unprivileged writers, total)` bytes of the filesystem
89/// holding `dir`.
90#[cfg(all(unix, any(feature = "http1", feature = "http2")))]
91fn space(dir: &Path) -> Option<(u64, u64)> {
92    use std::os::unix::ffi::OsStrExt;
93    let path = std::ffi::CString::new(dir.as_os_str().as_bytes()).ok()?;
94    // SAFETY: `stat` is plain old data written by statvfs; `path` is a valid
95    // NUL-terminated string that outlives the call.
96    let mut stat: libc::statvfs = unsafe { std::mem::zeroed() };
97    if unsafe { libc::statvfs(path.as_ptr(), &mut stat) } != 0 {
98        return None;
99    }
100    let block = stat.f_frsize as u64;
101    Some((stat.f_bavail as u64 * block, stat.f_blocks as u64 * block))
102}
103
104#[cfg(not(all(unix, any(feature = "http1", feature = "http2"))))]
105fn space(_dir: &Path) -> Option<(u64, u64)> {
106    None
107}
108
109struct Reading {
110    at: Instant,
111    free: u64,
112    total: u64,
113    low: bool,
114}
115
116fn readings() -> &'static Mutex<HashMap<PathBuf, Reading>> {
117    static READINGS: OnceLock<Mutex<HashMap<PathBuf, Reading>>> = OnceLock::new();
118    READINGS.get_or_init(Mutex::default)
119}
120
121/// The directory whose filesystem `file` lives on.
122fn directory_of(file: &Path) -> PathBuf {
123    if file.is_dir() {
124        return file.to_path_buf();
125    }
126    match file.parent() {
127        Some(parent) if !parent.as_os_str().is_empty() => parent.to_path_buf(),
128        _ => PathBuf::from("."),
129    }
130}
131
132/// Whether there's room to keep writing to `file` (or into the directory
133/// `file`). `true` when the guard is off or the space can't be read.
134pub fn has_room(file: &Path) -> bool {
135    has_room_with(file, floor())
136}
137
138fn has_room_with(file: &Path, floor: Floor) -> bool {
139    if floor == Floor::Off {
140        return true;
141    }
142    let dir = directory_of(file);
143    let mut readings = readings().lock().unwrap_or_else(|e| e.into_inner());
144    if let Some(reading) = readings.get(&dir) {
145        if reading.at.elapsed() < RECHECK && !cfg!(test) {
146            return !reading.low;
147        }
148    }
149    let Some((free, total)) = space(&dir) else { return true };
150    let low = free < required(floor, total);
151    let was_low = readings.get(&dir).is_some_and(|r| r.low);
152    if low && !was_low {
153        eprintln!(
154            "disk: {} has {} MiB free, under the {} MiB {RWS_CONFIG_MIN_FREE_DISK} floor; rws stops writing there until space is freed",
155            dir.display(),
156            free >> 20,
157            required(floor, total) >> 20
158        );
159    } else if !low && was_low {
160        eprintln!("disk: {} has {} MiB free again; rws writes there again", dir.display(), free >> 20);
161    }
162    readings.insert(dir, Reading { at: Instant::now(), free, total, low });
163    !low
164}
165
166static DROPPED_LOG_LINES: AtomicU64 = AtomicU64::new(0);
167static SKIPPED_CACHE_WRITES: AtomicU64 = AtomicU64::new(0);
168
169/// Access-log lines not written for want of space.
170#[doc(hidden)]
171pub fn dropped_log_lines(lines: u64) {
172    DROPPED_LOG_LINES.fetch_add(lines, Ordering::Relaxed);
173}
174
175/// Disk-cache entries not stored for want of space.
176#[doc(hidden)]
177pub fn skipped_cache_write() {
178    SKIPPED_CACHE_WRITES.fetch_add(1, Ordering::Relaxed);
179}
180
181/// Every directory checked so far: `(directory, free bytes, total bytes,
182/// low)`, sorted.
183pub fn status() -> Vec<(String, u64, u64, bool)> {
184    let readings = readings().lock().unwrap_or_else(|e| e.into_inner());
185    let mut out: Vec<_> = readings.iter().map(|(dir, r)| (dir.display().to_string(), r.free, r.total, r.low)).collect();
186    out.sort();
187    out
188}
189
190/// `rws_disk_free_bytes{dir}`, `rws_disk_low{dir}` and the drop counters;
191/// empty until a directory has been checked.
192#[doc(hidden)]
193pub fn prometheus_text() -> String {
194    let status = status();
195    if status.is_empty() {
196        return String::new();
197    }
198    let label = |dir: &str| dir.replace('\\', "\\\\").replace('"', "\\\"");
199    let mut out = String::from("# HELP rws_disk_free_bytes Free space where rws writes (access log, disk cache)\n# TYPE rws_disk_free_bytes gauge\n");
200    for (dir, free, _, _) in &status {
201        out.push_str(&format!("rws_disk_free_bytes{{dir=\"{}\"}} {free}\n", label(dir)));
202    }
203    out.push_str(&format!("# HELP rws_disk_low 1 while a directory is under {RWS_CONFIG_MIN_FREE_DISK} and rws isn't writing there\n# TYPE rws_disk_low gauge\n"));
204    for (dir, _, _, low) in &status {
205        out.push_str(&format!("rws_disk_low{{dir=\"{}\"}} {}\n", label(dir), *low as u8));
206    }
207    out.push_str("# HELP rws_access_log_dropped_lines_total Access-log lines not written because the disk was low\n# TYPE rws_access_log_dropped_lines_total counter\n");
208    out.push_str(&format!("rws_access_log_dropped_lines_total {}\n", DROPPED_LOG_LINES.load(Ordering::Relaxed)));
209    out.push_str("# HELP rws_cache_skipped_writes_total Disk-cache entries not stored because the disk was low\n# TYPE rws_cache_skipped_writes_total counter\n");
210    out.push_str(&format!("rws_cache_skipped_writes_total {}\n", SKIPPED_CACHE_WRITES.load(Ordering::Relaxed)));
211    out
212}
213
214#[cfg(test)]
215mod tests;
rws_server/disk_guard/mod.rs

rws_server/disk_guard/
mod.rs
