Skip to main content

sys_lib/
readiness.rs

1//! The server readiness report: the operating-system settings that decide
2//! whether a server performs and stays up under load, each with the reason and
3//! the fix, the way enterprise databases warn at start-up.
4//!
5//! **Read-only:** sys-lib reports and never changes a setting; [`Check::fix`]
6//! tells the operator what to run. Each check runs only where it applies (on
7//! that OS, for that [`Profile`]); elsewhere it is left out, not reported as
8//! passing. A value sys-lib cannot read is [`Status::Unknown`] with the reason
9//! in [`Check::found`], never a guess. Every threshold is a documented
10//! constant in this module.
11//!
12//! ```
13//! use sys_lib::readiness::{self, Profile};
14//!
15//! let report = readiness::report_for(Profile::Database, std::env::temp_dir());
16//! println!("{report}");
17//! for c in report.warnings() {
18//!     eprintln!("{}: {} (found {}, want {}): {}", c.id, c.title, c.found, c.recommended, c.fix);
19//! }
20//! if !report.passed() {
21//!     // refuse to start, or log loudly
22//! }
23//! ```
24
25use core::fmt::{self, Write as _};
26use std::path::Path;
27
28use crate::{
29    Bytes,
30    disk::{Disk, DiskKind},
31};
32
33/// What the server is for, which decides the checks and their thresholds.
34#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)]
35#[non_exhaustive]
36pub enum Profile {
37    /// Any long-running service: limits, clock, space.
38    General,
39    /// A database: memory management, write-back, NUMA, storage.
40    Database,
41    /// A network server: limits, listen backlog, ports.
42    WebServer,
43    /// A machine about to run benchmarks: frequency, power, clock.
44    Benchmark,
45}
46
47impl Profile {
48    /// A short lowercase name: `"database"`.
49    #[must_use]
50    pub const fn name(self) -> &'static str {
51        match self {
52            Self::General => "general",
53            Self::Database => "database",
54            Self::WebServer => "webserver",
55            Self::Benchmark => "benchmark",
56        }
57    }
58}
59
60/// The outcome of one check.
61#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)]
62#[non_exhaustive]
63pub enum Status {
64    /// As recommended.
65    Pass,
66    /// Works, but costs performance or headroom.
67    Warn,
68    /// Likely to fail under load, or losing data.
69    Fail,
70    /// Could not be read; [`Check::found`] says why.
71    Unknown,
72}
73
74impl Status {
75    /// `"PASS"`, `"WARN"`, `"FAIL"`, `"UNKNOWN"`.
76    #[must_use]
77    pub const fn name(self) -> &'static str {
78        match self {
79            Self::Pass => "PASS",
80            Self::Warn => "WARN",
81            Self::Fail => "FAIL",
82            Self::Unknown => "UNKNOWN",
83        }
84    }
85}
86
87/// One setting, what it is, and what it should be.
88#[derive(Clone, Debug, PartialEq, Eq)]
89#[non_exhaustive]
90pub struct Check {
91    /// A stable id: `"linux.thp"`, `"limits.nofile"`, `"fs.atime"`.
92    pub id: &'static str,
93    /// What it is: `"Transparent huge pages"`.
94    pub title: &'static str,
95    /// The outcome.
96    pub status: Status,
97    /// The value found, or why it could not be read.
98    pub found: String,
99    /// The value this profile wants.
100    pub recommended: &'static str,
101    /// Why it matters, in one sentence.
102    pub why: &'static str,
103    /// The command or setting that changes it.
104    pub fix: &'static str,
105}
106
107/// Every check that applies here, for one profile.
108#[derive(Clone, Debug, PartialEq, Eq)]
109#[non_exhaustive]
110pub struct Report {
111    /// The profile checked.
112    pub profile: Profile,
113    /// The checks, in a fixed order.
114    pub checks: Vec<Check>,
115}
116
117impl Report {
118    /// No check failed. Warnings and unknowns still pass.
119    #[must_use]
120    pub fn passed(&self) -> bool {
121        !self.checks.iter().any(|c| c.status == Status::Fail)
122    }
123
124    /// The checks that warn.
125    pub fn warnings(&self) -> impl Iterator<Item = &Check> {
126        self.checks.iter().filter(|c| c.status == Status::Warn)
127    }
128
129    /// The checks that fail.
130    pub fn failures(&self) -> impl Iterator<Item = &Check> {
131        self.checks.iter().filter(|c| c.status == Status::Fail)
132    }
133
134    /// One `key=value` line per check: `check=linux.thp status=WARN found=always`.
135    ///
136    /// # Errors
137    ///
138    /// Whatever `out` returns.
139    pub fn write_kv(&self, out: &mut impl fmt::Write) -> fmt::Result {
140        for (i, c) in self.checks.iter().enumerate() {
141            if i > 0 {
142                out.write_char('\n')?;
143            }
144            crate::fmt::kv(out, "check", c.id)?;
145            out.write_char(' ')?;
146            crate::fmt::kv(out, "status", c.status.name())?;
147            out.write_char(' ')?;
148            crate::fmt::kv(out, "found", &c.found)?;
149            out.write_char(' ')?;
150            crate::fmt::kv(out, "recommended", c.recommended)?;
151        }
152        Ok(())
153    }
154
155    /// One JSON object: the profile, whether it passed, and every check with
156    /// its reason and fix.
157    ///
158    /// # Errors
159    ///
160    /// Whatever `out` returns.
161    pub fn write_json(&self, out: &mut impl fmt::Write) -> fmt::Result {
162        use crate::fmt::json_str;
163        out.write_str("{\"profile\":")?;
164        json_str(out, self.profile.name())?;
165        write!(out, ",\"passed\":{},\"checks\":[", self.passed())?;
166        for (i, c) in self.checks.iter().enumerate() {
167            if i > 0 {
168                out.write_char(',')?;
169            }
170            out.write_str("{\"id\":")?;
171            json_str(out, c.id)?;
172            out.write_str(",\"title\":")?;
173            json_str(out, c.title)?;
174            out.write_str(",\"status\":")?;
175            json_str(out, c.status.name())?;
176            out.write_str(",\"found\":")?;
177            json_str(out, &c.found)?;
178            out.write_str(",\"recommended\":")?;
179            json_str(out, c.recommended)?;
180            out.write_str(",\"why\":")?;
181            json_str(out, c.why)?;
182            out.write_str(",\"fix\":")?;
183            json_str(out, c.fix)?;
184            out.write_char('}')?;
185        }
186        out.write_str("]}")
187    }
188}
189
190/// A table: status, id, title, found, recommended, and under each check that
191/// did not pass, its fix.
192impl fmt::Display for Report {
193    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
194        // Each column as wide as its longest entry, so long values never run
195        // into the next column; the found column is capped (a long value then
196        // only pushes its own row).
197        let width = |col: fn(&Check) -> usize, cap: usize| {
198            self.checks.iter().map(col).max().unwrap_or(0).min(cap)
199        };
200        let id = width(|c| c.id.len(), 32);
201        let title = width(|c| c.title.chars().count(), 40);
202        let found = width(|c| c.found.chars().count(), 32);
203        for (i, c) in self.checks.iter().enumerate() {
204            if i > 0 {
205                f.write_char('\n')?;
206            }
207            write!(
208                f,
209                "{:<7} {:<id$}  {:<title$}  {:<found$}  {}",
210                c.status.name(),
211                c.id,
212                c.title,
213                c.found,
214                c.recommended
215            )?;
216            if c.status != Status::Pass && !c.fix.is_empty() {
217                write!(f, "\n        {}", c.fix)?;
218            }
219        }
220        Ok(())
221    }
222}
223
224/// The report for `profile`, read now. Never fails: a value that cannot be
225/// read is a check with [`Status::Unknown`]. Costs 0.2 to 2 ms [est.].
226#[must_use]
227pub fn report(profile: Profile) -> Report {
228    evaluate(profile, &gather(None))
229}
230
231/// [`report`], plus the checks of the directory the server keeps its data in:
232/// its filesystem, mount options, free space, kind of storage, and I/O
233/// scheduler.
234#[must_use]
235pub fn report_for(profile: Profile, data_dir: impl AsRef<Path>) -> Report {
236    evaluate(profile, &gather(Some(data_dir.as_ref())))
237}
238
239// ---------------------------------------------------------------------------
240// What a platform reports
241// ---------------------------------------------------------------------------
242
243/// One setting as a platform reports it.
244#[derive(Clone, Debug, Default, PartialEq, Eq)]
245pub(crate) enum Fact<T> {
246    /// The check does not apply on this platform: left out of the report.
247    #[default]
248    Skip,
249    /// The value.
250    Value(T),
251    /// The check applies, and the value could not be read: why.
252    Unknown(String),
253}
254
255impl<T> Fact<T> {
256    /// A reading's result as a fact: an error becomes `Unknown` with its text.
257    pub(crate) fn of(r: crate::Result<T>) -> Fact<T> {
258        match r {
259            Ok(v) => Fact::Value(v),
260            Err(e) => Fact::Unknown(e.to_string()),
261        }
262    }
263
264    pub(crate) fn map<U>(self, f: impl FnOnce(T) -> U) -> Fact<U> {
265        match self {
266            Fact::Skip => Fact::Skip,
267            Fact::Value(v) => Fact::Value(f(v)),
268            Fact::Unknown(why) => Fact::Unknown(why),
269        }
270    }
271}
272
273/// The Linux write-back settings; a `_bytes` value, when not 0, replaces its ratio.
274#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
275pub(crate) struct Dirty {
276    pub(crate) ratio: u64,
277    pub(crate) background_ratio: u64,
278    pub(crate) bytes: u64,
279    pub(crate) background_bytes: u64,
280}
281
282/// The CPU frequency policy: the Linux governors in use, or the Windows power
283/// plan.
284#[derive(Clone, Debug, Default, PartialEq, Eq)]
285pub(crate) struct Governor {
286    /// `"performance"`, `"powersave,performance"` (CPUs differ), `"High performance"`.
287    pub(crate) name: String,
288    /// Every CPU runs at its highest frequency policy.
289    pub(crate) performance: bool,
290}
291
292/// The locked-memory allowance.
293#[allow(dead_code)] // each platform constructs its own variant
294#[derive(Clone, Copy, Debug, PartialEq, Eq)]
295pub(crate) enum MemLock {
296    /// `RLIMIT_MEMLOCK`: `None` is unlimited.
297    Limit { soft: Option<u64> },
298    /// Windows: the token holds `SeLockMemoryPrivilege`.
299    Privilege(bool),
300}
301
302/// The system's settings, as a platform reports them; `Skip` where a check
303/// does not apply there.
304#[derive(Clone, Debug, Default, PartialEq, Eq)]
305pub(crate) struct Tuning {
306    /// `transparent_hugepage/enabled` and `defrag`, the selected words.
307    pub(crate) thp: Fact<(String, String)>,
308    pub(crate) swappiness: Fact<u64>,
309    /// `overcommit_memory` and `overcommit_ratio`.
310    pub(crate) overcommit: Fact<(u64, u64)>,
311    pub(crate) max_map_count: Fact<u64>,
312    pub(crate) dirty: Fact<Dirty>,
313    /// `numa_balancing` and `zone_reclaim_mode`.
314    pub(crate) numa: Fact<(u64, u64)>,
315    /// Open handles have no per-process limit (Windows).
316    pub(crate) handles_unlimited: bool,
317    /// The system-wide open-file ceiling (`fs.file-max`, `kern.maxfiles`).
318    pub(crate) files_system: Fact<u64>,
319    /// The per-process ceiling (`fs.nr_open`, `kern.maxfilesperproc`).
320    pub(crate) files_process: Fact<u64>,
321    pub(crate) memlock: Fact<MemLock>,
322    /// `somaxconn`, and the SYN backlog where the OS has one.
323    pub(crate) backlog: Fact<(u64, Option<u64>)>,
324    /// The ephemeral port range, inclusive.
325    pub(crate) ports: Fact<(u32, u32)>,
326    pub(crate) governor: Fact<Governor>,
327    pub(crate) clock_source: Fact<String>,
328}
329
330/// The data directory's settings, as a platform reports them.
331#[derive(Clone, Debug, Default, PartialEq, Eq)]
332pub(crate) struct DataDirTuning {
333    /// How access times are kept: `"noatime"`, `"relatime"`, `"strictatime"`.
334    pub(crate) atime: Fact<String>,
335    /// The disk's I/O scheduler (Linux).
336    pub(crate) scheduler: Fact<String>,
337}
338
339/// Everything [`evaluate`] reads, gathered once, so the thresholds can be
340/// tested on any machine.
341#[derive(Clone, Debug, Default)]
342pub(crate) struct Inputs {
343    pub(crate) tuning: Tuning,
344    pub(crate) open_files_soft: Fact<Option<u64>>,
345    pub(crate) clock_synchronized: Fact<bool>,
346    pub(crate) swap_total: Fact<Bytes>,
347    pub(crate) numa_nodes: usize,
348    pub(crate) container_limits: Option<String>,
349    pub(crate) data_dir: Option<DataDir>,
350    pub(crate) bench: Option<crate::bench::Environment>,
351}
352
353/// The data directory.
354#[derive(Clone, Debug)]
355pub(crate) struct DataDir {
356    pub(crate) disk: Fact<Disk>,
357    pub(crate) tuning: DataDirTuning,
358}
359
360fn gather(data_dir: Option<&Path>) -> Inputs {
361    let tuning = crate::sys::tuning();
362    let open_files_soft = Fact::of(crate::process::limits()).map(|l| l.open_files.soft);
363    let clock_synchronized = match crate::host::clock_sync() {
364        Ok(Some(c)) => Fact::Value(c.synchronized),
365        Ok(None) => Fact::Unknown("the OS does not say".to_owned()),
366        Err(e) => Fact::Unknown(e.to_string()),
367    };
368    let swap_total = Fact::of(crate::mem::swap()).map(|s| s.total);
369    let mut limits = Vec::new();
370    if let Ok(Some(l)) = crate::mem::limit() {
371        limits.push(format!("memory {}", l.hard));
372    }
373    if let Ok(Some(q)) = crate::cpu::quota() {
374        limits.push(format!("cpu {:.2}", q.cpus));
375    }
376    let container_limits = (!limits.is_empty()).then(|| limits.join(", "));
377    let data_dir = data_dir.map(|path| {
378        let disk = Fact::of(crate::disk::of(path));
379        let tuning = match &disk {
380            Fact::Value(d) => crate::sys::data_dir_tuning(path, d),
381            _ => DataDirTuning::default(),
382        };
383        DataDir { disk, tuning }
384    });
385    Inputs {
386        tuning,
387        open_files_soft,
388        clock_synchronized,
389        swap_total,
390        numa_nodes: crate::cpu::numa_nodes().len(),
391        container_limits,
392        data_dir,
393        bench: None,
394    }
395}
396
397// ---------------------------------------------------------------------------
398// The thresholds
399// ---------------------------------------------------------------------------
400
401/// The least `vm.swappiness` a database wants.
402pub const SWAPPINESS_MIN: u64 = 1;
403/// The most `vm.swappiness` a database wants.
404pub const SWAPPINESS_MAX: u64 = 10;
405/// The least `vm.max_map_count` a database wants (memory-mapped storage).
406pub const MAX_MAP_COUNT_MIN: u64 = 262_144;
407/// The most `dirty_background_ratio` a database wants, in percent.
408pub const DIRTY_BACKGROUND_MAX: u64 = 5;
409/// The most `dirty_ratio` a database wants, in percent.
410pub const DIRTY_MAX: u64 = 15;
411/// The open-file soft limit a database or network server wants.
412pub const OPEN_FILES_SERVER: u64 = 65_536;
413/// The open-file soft limit any service wants.
414pub const OPEN_FILES_GENERAL: u64 = 4_096;
415/// Below this soft limit a database or network server fails under load.
416pub const OPEN_FILES_FLOOR: u64 = 1_024;
417/// The least listen backlog a database or network server wants.
418pub const BACKLOG_MIN: u64 = 4_096;
419/// The least ephemeral ports a database or network server wants.
420pub const PORTS_MIN: u32 = 28_000;
421/// The free space under which a data directory warns, in percent.
422pub const FREE_WARN: f32 = 15.0;
423/// The free space under which a data directory fails, in percent.
424pub const FREE_FAIL: f32 = 5.0;
425
426#[cfg(target_os = "macos")]
427mod fixes {
428    pub(super) const NOFILE: &str =
429        "launchctl limit maxfiles 65536 200000, or SoftResourceLimits in the launchd plist";
430    pub(super) const MEMLOCK: &str = "launchctl limit memlock unlimited unlimited";
431    pub(super) const BACKLOG: &str = "sysctl -w kern.ipc.somaxconn=4096";
432    pub(super) const PORTS: &str =
433        "sysctl -w net.inet.ip.portrange.first=10000 net.inet.ip.portrange.last=65535";
434    pub(super) const GOVERNOR: &str = "";
435    pub(super) const CLOCK: &str = "enable Set time automatically in System Settings";
436    pub(super) const CLOCK_SOURCE: &str = "";
437    pub(super) const TURBO: &str = "";
438    pub(super) const SWAP: &str = "macOS manages swap itself";
439}
440
441#[cfg(windows)]
442mod fixes {
443    pub(super) const NOFILE: &str = "";
444    pub(super) const MEMLOCK: &str =
445        "grant Lock pages in memory (SeLockMemoryPrivilege) to the service account in secpol.msc";
446    pub(super) const BACKLOG: &str = "";
447    pub(super) const PORTS: &str = "netsh int ipv4 set dynamicport tcp start=10000 num=55535";
448    pub(super) const GOVERNOR: &str = "powercfg /setactive SCHEME_MIN";
449    pub(super) const CLOCK: &str = "w32tm /resync, and check the Windows Time service";
450    pub(super) const CLOCK_SOURCE: &str = "";
451    pub(super) const TURBO: &str = "powercfg /setacvalueindex SCHEME_CURRENT SUB_PROCESSOR PERFBOOSTMODE 0, then powercfg /setactive SCHEME_CURRENT";
452    pub(super) const SWAP: &str = "System Properties > Advanced > Performance > Virtual memory";
453}
454
455#[cfg(target_os = "freebsd")]
456mod fixes {
457    pub(super) const NOFILE: &str =
458        "openfiles in /etc/login.conf (then cap_mkdb /etc/login.conf), within kern.maxfilesperproc";
459    pub(super) const MEMLOCK: &str =
460        "memorylocked=unlimited in /etc/login.conf (then cap_mkdb /etc/login.conf)";
461    pub(super) const BACKLOG: &str = "sysctl kern.ipc.soacceptqueue=4096, and in /etc/sysctl.conf";
462    pub(super) const PORTS: &str =
463        "sysctl net.inet.ip.portrange.first=10000 net.inet.ip.portrange.last=65535";
464    pub(super) const GOVERNOR: &str = "";
465    pub(super) const CLOCK: &str =
466        "sysrc ntpd_enable=YES ntpd_sync_on_start=YES, then service ntpd start";
467    pub(super) const CLOCK_SOURCE: &str = "sysctl kern.timecounter.hardware=TSC-low";
468    pub(super) const TURBO: &str = "";
469    pub(super) const SWAP: &str = "add a swap partition or file: swapon(8) and /etc/fstab";
470}
471
472#[cfg(not(any(target_os = "macos", target_os = "freebsd", windows)))]
473mod fixes {
474    pub(super) const NOFILE: &str =
475        "LimitNOFILE=65536 in the systemd unit, or nofile in /etc/security/limits.conf";
476    pub(super) const MEMLOCK: &str =
477        "LimitMEMLOCK=infinity in the systemd unit, or memlock in /etc/security/limits.conf";
478    pub(super) const BACKLOG: &str =
479        "sysctl -w net.core.somaxconn=4096 net.ipv4.tcp_max_syn_backlog=4096";
480    pub(super) const PORTS: &str = "sysctl -w net.ipv4.ip_local_port_range=\"10000 65535\"";
481    pub(super) const GOVERNOR: &str = "cpupower frequency-set -g performance";
482    pub(super) const CLOCK: &str = "enable chrony or systemd-timesyncd: timedatectl set-ntp true";
483    pub(super) const CLOCK_SOURCE: &str =
484        "echo tsc > /sys/devices/system/clocksource/clocksource0/current_clocksource";
485    pub(super) const TURBO: &str = "echo 1 > /sys/devices/system/cpu/intel_pstate/no_turbo, or echo 0 > /sys/devices/system/cpu/cpufreq/boost";
486    pub(super) const SWAP: &str = "add a small swap file or zram: swapon, or zram-generator";
487}
488
489fn check(
490    id: &'static str,
491    title: &'static str,
492    status: Status,
493    found: impl Into<String>,
494    recommended: &'static str,
495    why: &'static str,
496    fix: &'static str,
497) -> Check {
498    Check {
499        id,
500        title,
501        status,
502        found: found.into(),
503        recommended,
504        why,
505        fix,
506    }
507}
508
509/// Adds a check from a fact: left out when it does not apply, `Unknown` when
510/// it could not be read, otherwise `judge` gives the status and the text found.
511fn push<T>(
512    out: &mut Vec<Check>,
513    fact: &Fact<T>,
514    meta: (
515        &'static str,
516        &'static str,
517        &'static str,
518        &'static str,
519        &'static str,
520    ),
521    judge: impl FnOnce(&T) -> (Status, String),
522) {
523    let (id, title, recommended, why, fix) = meta;
524    match fact {
525        Fact::Skip => {}
526        Fact::Unknown(reason) => {
527            out.push(check(
528                id,
529                title,
530                Status::Unknown,
531                reason.clone(),
532                recommended,
533                why,
534                fix,
535            ));
536        }
537        Fact::Value(v) => {
538            let (status, found) = judge(v);
539            out.push(check(id, title, status, found, recommended, why, fix));
540        }
541    }
542}
543
544const fn pass_if(ok: bool) -> Status {
545    if ok { Status::Pass } else { Status::Warn }
546}
547
548fn has(profiles: &[Profile], profile: Profile) -> bool {
549    profiles.contains(&profile)
550}
551
552#[allow(clippy::too_many_lines)] // one table, read top to bottom
553pub(crate) fn evaluate(profile: Profile, inputs: &Inputs) -> Report {
554    use Profile::{Benchmark, Database, WebServer};
555
556    let t = &inputs.tuning;
557    let mut out = Vec::new();
558    let db = profile == Database;
559    let server = matches!(profile, Database | WebServer);
560
561    if db {
562        push(
563            &mut out,
564            &t.thp,
565            (
566                "linux.thp",
567                "Transparent huge pages",
568                "madvise or never",
569                "With always, the kernel stalls allocations to compact memory and bloats sparse heaps, which shows as latency spikes.",
570                "echo madvise > /sys/kernel/mm/transparent_hugepage/enabled",
571            ),
572            |(enabled, defrag)| {
573                let ok = enabled != "always" && defrag != "always";
574                (pass_if(ok), format!("{enabled}, defrag {defrag}"))
575            },
576        );
577        push(
578            &mut out,
579            &t.swappiness,
580            (
581                "linux.swappiness",
582                "Swap tendency (vm.swappiness)",
583                "1 to 10",
584                "A high value swaps out the buffer pool to keep page cache, which a database already manages itself.",
585                "sysctl -w vm.swappiness=1",
586            ),
587            |&v| {
588                (
589                    pass_if((SWAPPINESS_MIN..=SWAPPINESS_MAX).contains(&v)),
590                    v.to_string(),
591                )
592            },
593        );
594        push(
595            &mut out,
596            &t.overcommit,
597            (
598                "linux.overcommit",
599                "Memory overcommit",
600                "0 or 2",
601                "With 1 the kernel grants any allocation and the OOM killer ends the database when memory runs out.",
602                "sysctl -w vm.overcommit_memory=0",
603            ),
604            |&(mode, ratio)| {
605                let found = if mode == 2 {
606                    format!("{mode}, ratio {ratio}")
607                } else {
608                    mode.to_string()
609                };
610                (
611                    if mode == 1 {
612                        Status::Fail
613                    } else {
614                        Status::Pass
615                    },
616                    found,
617                )
618            },
619        );
620        push(
621            &mut out,
622            &t.max_map_count,
623            (
624                "linux.max_map_count",
625                "Memory map count",
626                "at least 262144",
627                "Memory-mapped storage engines need a map per file region and fail allocations past the limit.",
628                "sysctl -w vm.max_map_count=262144",
629            ),
630            |&v| (pass_if(v >= MAX_MAP_COUNT_MIN), v.to_string()),
631        );
632        push(
633            &mut out,
634            &t.dirty,
635            (
636                "linux.dirty",
637                "Dirty page write-back",
638                "background at most 5%, foreground at most 15%",
639                "Large dirty thresholds let gigabytes of writes pile up and then stall every writer while they flush.",
640                "sysctl -w vm.dirty_background_ratio=5 vm.dirty_ratio=15",
641            ),
642            |d| {
643                if d.bytes != 0 || d.background_bytes != 0 {
644                    // Set in bytes: the operator chose a size on purpose.
645                    return (
646                        Status::Pass,
647                        format!(
648                            "background {} bytes, foreground {} bytes",
649                            d.background_bytes, d.bytes
650                        ),
651                    );
652                }
653                (
654                    pass_if(d.background_ratio <= DIRTY_BACKGROUND_MAX && d.ratio <= DIRTY_MAX),
655                    format!(
656                        "background {}%, foreground {}%",
657                        d.background_ratio, d.ratio
658                    ),
659                )
660            },
661        );
662        if inputs.numa_nodes > 1 {
663            push(
664                &mut out,
665                &t.numa,
666                (
667                    "linux.numa",
668                    "NUMA balancing and zone reclaim",
669                    "both 0",
670                    "Automatic balancing migrates the buffer pool between nodes, and zone reclaim drops cache instead of using remote memory.",
671                    "sysctl -w kernel.numa_balancing=0 vm.zone_reclaim_mode=0",
672                ),
673                |&(balancing, reclaim)| {
674                    (
675                        pass_if(balancing == 0 && reclaim == 0),
676                        format!("balancing {balancing}, zone reclaim {reclaim}"),
677                    )
678                },
679            );
680        }
681    }
682
683    // Open files: every profile.
684    let (want, recommended) = if server {
685        (OPEN_FILES_SERVER, "at least 65536")
686    } else {
687        (OPEN_FILES_GENERAL, "at least 4096")
688    };
689    const NOFILE_WHY: &str = "Every connection and every open table or segment file takes a descriptor; at the limit, accepts and opens fail.";
690    if t.handles_unlimited {
691        out.push(check(
692            "limits.nofile",
693            "Open-file limit",
694            Status::Pass,
695            "unlimited (handles)",
696            recommended,
697            NOFILE_WHY,
698            fixes::NOFILE,
699        ));
700    } else {
701        push(
702            &mut out,
703            &inputs.open_files_soft,
704            (
705                "limits.nofile",
706                "Open-file limit",
707                recommended,
708                NOFILE_WHY,
709                fixes::NOFILE,
710            ),
711            |&soft| match soft {
712                None => (Status::Pass, "unlimited".to_owned()),
713                Some(n) => {
714                    let mut found = n.to_string();
715                    if let Fact::Value(max) = t.files_system {
716                        // Kernels default fs.file-max to LONG_MAX.
717                        if max >= 1 << 62 {
718                            found.push_str(", system unlimited");
719                        } else {
720                            found.push_str(&format!(", system {max}"));
721                        }
722                    }
723                    let status = if n >= want {
724                        Status::Pass
725                    } else if server && n < OPEN_FILES_FLOOR {
726                        Status::Fail
727                    } else {
728                        Status::Warn
729                    };
730                    (status, found)
731                }
732            },
733        );
734    }
735
736    if db {
737        push(
738            &mut out,
739            &t.memlock,
740            (
741                "limits.memlock",
742                "Locked-memory limit",
743                "unlimited, when the buffer pool is locked",
744                "A database that locks its buffer pool in memory, to keep it out of swap, can lock no more than this.",
745                fixes::MEMLOCK,
746            ),
747            |m| match *m {
748                MemLock::Limit { soft: None } => (Status::Pass, "unlimited".to_owned()),
749                MemLock::Limit { soft: Some(n) } => (Status::Warn, Bytes::new(n).to_string()),
750                MemLock::Privilege(true) => (Status::Pass, "granted".to_owned()),
751                MemLock::Privilege(false) => (Status::Warn, "not granted".to_owned()),
752            },
753        );
754    }
755
756    if server {
757        push(
758            &mut out,
759            &t.backlog,
760            (
761                "net.backlog",
762                "Listen backlog",
763                "at least 4096",
764                "Connections beyond the backlog are dropped during bursts, and clients see timeouts.",
765                fixes::BACKLOG,
766            ),
767            |&(somaxconn, syn)| {
768                let least = syn.map_or(somaxconn, |s| s.min(somaxconn));
769                let found = match syn {
770                    Some(s) => format!("{somaxconn}, SYN {s}"),
771                    None => somaxconn.to_string(),
772                };
773                (pass_if(least >= BACKLOG_MIN), found)
774            },
775        );
776        push(
777            &mut out,
778            &t.ports,
779            (
780                "net.ports",
781                "Ephemeral ports",
782                "at least 28000 ports",
783                "Each outgoing connection holds a port through TIME_WAIT; a narrow range runs out under load.",
784                fixes::PORTS,
785            ),
786            |&(first, last)| {
787                let count = last.saturating_sub(first).saturating_add(1);
788                (
789                    pass_if(count >= PORTS_MIN),
790                    format!("{first} to {last} ({count})"),
791                )
792            },
793        );
794    }
795
796    if has(&[Database, WebServer, Benchmark], profile) {
797        push(
798            &mut out,
799            &t.governor,
800            (
801                "cpu.governor",
802                "CPU frequency policy",
803                "performance",
804                "Power-saving policies add wake-up latency and make throughput depend on load history.",
805                fixes::GOVERNOR,
806            ),
807            |g| (pass_if(g.performance), g.name.clone()),
808        );
809    }
810
811    if has(&[Database, Benchmark], profile) {
812        push(
813            &mut out,
814            &t.clock_source,
815            (
816                "clock.source",
817                "Kernel clock source",
818                "tsc",
819                "Reading hpet or acpi_pm costs microseconds, which every timestamp and timeout pays.",
820                fixes::CLOCK_SOURCE,
821            ),
822            |s| (pass_if(!crate::bench::slow_clock(s)), s.clone()),
823        );
824    }
825
826    if profile != Benchmark {
827        push(
828            &mut out,
829            &inputs.clock_synchronized,
830            (
831                "clock.sync",
832                "Clock synchronized",
833                "synchronized",
834                "Leases, distributed locks, certificates, and log ordering assume clocks agree.",
835                fixes::CLOCK,
836            ),
837            |&ok| {
838                (
839                    pass_if(ok),
840                    if ok {
841                        "synchronized"
842                    } else {
843                        "not synchronized"
844                    }
845                    .to_owned(),
846                )
847            },
848        );
849    }
850
851    if db {
852        push(
853            &mut out,
854            &inputs.swap_total,
855            (
856                "mem.swap",
857                "Swap present",
858                "some swap or zram, small",
859                "Without any swap, a brief memory spike goes straight to the OOM killer instead of slowing down.",
860                fixes::SWAP,
861            ),
862            |&total| {
863                let status = if total.raw() == 0 {
864                    Status::Warn
865                } else {
866                    Status::Pass
867                };
868                (status, total.to_string())
869            },
870        );
871    }
872
873    if let Some(dir) = &inputs.data_dir {
874        data_dir_checks(&mut out, profile, dir);
875    }
876
877    if let Some(limits) = &inputs.container_limits {
878        out.push(check(
879            "container.limits",
880            "Container limits",
881            Status::Pass,
882            limits.clone(),
883            "reported",
884            "Size thread pools and caches from these limits, not from the host.",
885            "",
886        ));
887    }
888
889    if profile == Benchmark {
890        let env = inputs
891            .bench
892            .clone()
893            .unwrap_or_else(crate::bench::environment);
894        bench_checks(&mut out, &env);
895    }
896
897    Report {
898        profile,
899        checks: out,
900    }
901}
902
903fn data_dir_checks(out: &mut Vec<Check>, profile: Profile, dir: &DataDir) {
904    let db = profile == Profile::Database;
905    const FS_WHY: &str = "Network filesystems break locking and fsync guarantees, and some local ones are slow under database write patterns.";
906    let disk = match &dir.disk {
907        Fact::Skip => return,
908        Fact::Unknown(reason) => {
909            out.push(check(
910                "fs.type",
911                "Data directory filesystem",
912                Status::Unknown,
913                reason.clone(),
914                RECOMMENDED_FS,
915                FS_WHY,
916                "",
917            ));
918            return;
919        }
920        Fact::Value(d) => d,
921    };
922    let fs = disk.filesystem.to_ascii_lowercase();
923    let network = disk.kind == DiskKind::Network;
924    let status = if network {
925        if db { Status::Fail } else { Status::Warn }
926    } else if db {
927        pass_if(RECOMMENDED_FS_NAMES.contains(&fs.as_str()))
928    } else {
929        Status::Pass
930    };
931    out.push(check(
932        "fs.type",
933        "Data directory filesystem",
934        status,
935        format!("{} on {}", disk.filesystem, disk.device),
936        RECOMMENDED_FS,
937        FS_WHY,
938        "move the data directory to a local disk with a recommended filesystem",
939    ));
940
941    push(
942        out,
943        &dir.tuning.atime,
944        (
945            "fs.atime",
946            "Data directory access times",
947            "noatime or relatime",
948            "Updating the access time on every read turns reads into writes.",
949            "mount with noatime: add it to the options in /etc/fstab and remount",
950        ),
951        |a| (pass_if(a != "strictatime"), a.clone()),
952    );
953
954    if let Some(space) = disk.space {
955        let free = space.available.percent_of(space.total).value();
956        let status = if free < FREE_FAIL {
957            Status::Fail
958        } else if free < FREE_WARN {
959            Status::Warn
960        } else {
961            Status::Pass
962        };
963        out.push(check(
964            "disk.space",
965            "Data directory free space",
966            status,
967            format!("{free:.0}% free ({} of {})", space.available, space.total),
968            "at least 15% free",
969            "A full disk stops writes and can corrupt files that are half written; copy-on-write filesystems need headroom.",
970            "free space or grow the volume",
971        ));
972    }
973
974    if db {
975        let (status, found) = match disk.kind {
976            DiskKind::Ssd => (Status::Pass, "ssd"),
977            DiskKind::Hdd => (Status::Warn, "hdd"),
978            DiskKind::Ram => (Status::Warn, "memory (lost on reboot)"),
979            DiskKind::Network => (Status::Warn, "network"),
980            DiskKind::Removable => (Status::Warn, "removable"),
981            DiskKind::Optical => (Status::Warn, "optical"),
982            DiskKind::Virtual => (Status::Pass, "virtual"),
983            _ => (Status::Unknown, "unknown"),
984        };
985        out.push(check(
986            "disk.kind",
987            "Data directory storage",
988            status,
989            found,
990            "ssd or nvme",
991            "Random reads and fsync latency on rotating disks are a hundred times slower.",
992            "move the data directory to solid-state storage",
993        ));
994        if disk.kind == DiskKind::Ssd {
995            push(
996                out,
997                &dir.tuning.scheduler,
998                (
999                    "disk.scheduler",
1000                    "Data disk I/O scheduler",
1001                    "none or mq-deadline",
1002                    "Fair-queueing schedulers add latency that solid-state drives do not need.",
1003                    "echo none > /sys/block/<disk>/queue/scheduler, or a udev rule",
1004                ),
1005                |s| (pass_if(s == "none" || s == "mq-deadline"), s.clone()),
1006            );
1007        }
1008    }
1009}
1010
1011#[cfg(target_os = "macos")]
1012const RECOMMENDED_FS: &str = "local APFS";
1013#[cfg(windows)]
1014const RECOMMENDED_FS: &str = "local NTFS or ReFS";
1015#[cfg(target_os = "freebsd")]
1016const RECOMMENDED_FS: &str = "local ZFS or UFS";
1017#[cfg(not(any(target_os = "macos", target_os = "freebsd", windows)))]
1018const RECOMMENDED_FS: &str = "local ext4 or xfs";
1019
1020/// The filesystems a database profile passes, lowercase.
1021const RECOMMENDED_FS_NAMES: &[&str] = &["ext4", "xfs", "apfs", "ntfs", "refs", "zfs", "ufs"];
1022
1023fn bench_checks(out: &mut Vec<Check>, env: &crate::bench::Environment) {
1024    use crate::bench::Warning;
1025    let warned = |w| env.warnings.contains(&w);
1026    if let Some(turbo) = env.turbo {
1027        out.push(check(
1028            "cpu.turbo",
1029            "Turbo boost",
1030            pass_if(!turbo),
1031            if turbo { "enabled" } else { "disabled" },
1032            "disabled",
1033            "Turbo frequency depends on temperature and on what other cores do, so runs differ.",
1034            fixes::TURBO,
1035        ));
1036    }
1037    if let Some(throttled) = env.throttled {
1038        out.push(check(
1039            "cpu.throttled",
1040            "Thermal throttling",
1041            pass_if(!throttled),
1042            if throttled { "throttled" } else { "not throttled" },
1043            "not throttled",
1044            "A throttled CPU runs slower than its rated frequency for reasons the benchmark does not control.",
1045            "improve cooling, or let the machine cool down",
1046        ));
1047    }
1048    if let Some(ac) = env.on_ac {
1049        out.push(check(
1050            "power.ac",
1051            "Mains power",
1052            pass_if(ac),
1053            if ac { "on mains" } else { "on battery" },
1054            "on mains",
1055            "On battery the OS lowers frequencies and parks cores.",
1056            "connect the charger",
1057        ));
1058    }
1059    out.push(check(
1060        "cpu.idle",
1061        "Machine idle",
1062        pass_if(!warned(Warning::Busy)),
1063        if warned(Warning::Busy) {
1064            "busy"
1065        } else {
1066            "idle"
1067        },
1068        "idle",
1069        "Other work competes for cores, caches, and memory bandwidth.",
1070        "stop other workloads before measuring",
1071    ));
1072    out.push(check(
1073        "virt.vm",
1074        "Bare metal",
1075        pass_if(!env.virtualized),
1076        if env.virtualized {
1077            "virtual machine"
1078        } else {
1079            "bare metal"
1080        },
1081        "bare metal",
1082        "Other guests share the hardware, and the hypervisor adds exits and steal time.",
1083        "measure on bare metal, or pin the VM to dedicated cores",
1084    ));
1085}
1086
1087#[cfg(test)]
1088mod tests {
1089    use super::*;
1090    use crate::disk::Space;
1091
1092    fn tuned() -> Tuning {
1093        Tuning {
1094            thp: Fact::Value(("madvise".into(), "madvise".into())),
1095            swappiness: Fact::Value(1),
1096            overcommit: Fact::Value((0, 50)),
1097            max_map_count: Fact::Value(262_144),
1098            dirty: Fact::Value(Dirty {
1099                ratio: 10,
1100                background_ratio: 5,
1101                ..Dirty::default()
1102            }),
1103            numa: Fact::Value((0, 0)),
1104            handles_unlimited: false,
1105            files_system: Fact::Value(1 << 20),
1106            files_process: Fact::Value(1 << 20),
1107            memlock: Fact::Value(MemLock::Limit { soft: None }),
1108            backlog: Fact::Value((4096, Some(4096))),
1109            ports: Fact::Value((10_000, 65_535)),
1110            governor: Fact::Value(Governor {
1111                name: "performance".into(),
1112                performance: true,
1113            }),
1114            clock_source: Fact::Value("tsc".into()),
1115        }
1116    }
1117
1118    fn inputs(tuning: Tuning) -> Inputs {
1119        Inputs {
1120            tuning,
1121            open_files_soft: Fact::Value(Some(1 << 20)),
1122            clock_synchronized: Fact::Value(true),
1123            swap_total: Fact::Value(Bytes::new(1 << 30)),
1124            numa_nodes: 2,
1125            container_limits: None,
1126            data_dir: None,
1127            bench: None,
1128        }
1129    }
1130
1131    fn status(r: &Report, id: &str) -> Option<Status> {
1132        r.checks.iter().find(|c| c.id == id).map(|c| c.status)
1133    }
1134
1135    #[test]
1136    fn test_readiness_tuned_database_passes_everything() {
1137        let r = evaluate(Profile::Database, &inputs(tuned()));
1138        assert!(r.passed());
1139        assert!(r.checks.iter().all(|c| c.status == Status::Pass), "{r}");
1140        assert_eq!(status(&r, "linux.numa"), Some(Status::Pass));
1141    }
1142
1143    #[test]
1144    fn test_readiness_untuned_database_warns_and_fails() {
1145        let mut t = tuned();
1146        t.thp = Fact::Value(("always".into(), "madvise".into()));
1147        t.swappiness = Fact::Value(60);
1148        t.overcommit = Fact::Value((1, 50));
1149        t.max_map_count = Fact::Value(65_530);
1150        t.dirty = Fact::Value(Dirty {
1151            ratio: 20,
1152            background_ratio: 10,
1153            ..Dirty::default()
1154        });
1155        t.backlog = Fact::Value((4096, Some(512)));
1156        t.ports = Fact::Value((40_000, 60_999));
1157        t.clock_source = Fact::Value("hpet".into());
1158        t.memlock = Fact::Value(MemLock::Limit {
1159            soft: Some(8 << 20),
1160        });
1161        let mut i = inputs(t);
1162        i.open_files_soft = Fact::Value(Some(1024));
1163        i.swap_total = Fact::Value(Bytes::new(0));
1164        let r = evaluate(Profile::Database, &i);
1165        assert!(!r.passed());
1166        assert_eq!(status(&r, "linux.overcommit"), Some(Status::Fail));
1167        for id in [
1168            "linux.thp",
1169            "linux.swappiness",
1170            "linux.max_map_count",
1171            "linux.dirty",
1172            "net.backlog",
1173            "net.ports",
1174            "clock.source",
1175            "limits.memlock",
1176            "limits.nofile",
1177            "mem.swap",
1178        ] {
1179            assert_eq!(status(&r, id), Some(Status::Warn), "{id}");
1180        }
1181        assert_eq!(r.failures().count(), 1);
1182        // 32768..=60999 is 28232 ports: enough.
1183        let mut t = tuned();
1184        t.ports = Fact::Value((32_768, 60_999));
1185        assert_eq!(
1186            status(&evaluate(Profile::WebServer, &inputs(t)), "net.ports"),
1187            Some(Status::Pass)
1188        );
1189    }
1190
1191    #[test]
1192    fn test_readiness_open_files_floor_fails_servers_only() {
1193        let mut i = inputs(tuned());
1194        i.open_files_soft = Fact::Value(Some(256));
1195        assert_eq!(
1196            status(&evaluate(Profile::WebServer, &i), "limits.nofile"),
1197            Some(Status::Fail)
1198        );
1199        assert_eq!(
1200            status(&evaluate(Profile::General, &i), "limits.nofile"),
1201            Some(Status::Warn)
1202        );
1203        let mut t = tuned();
1204        t.handles_unlimited = true;
1205        let mut i = inputs(t);
1206        i.open_files_soft = Fact::Skip;
1207        assert_eq!(
1208            status(&evaluate(Profile::Database, &i), "limits.nofile"),
1209            Some(Status::Pass)
1210        );
1211    }
1212
1213    #[test]
1214    fn test_readiness_skip_is_left_out_and_unknown_reported() {
1215        let t = Tuning {
1216            swappiness: Fact::Unknown("permission denied".into()),
1217            ..Tuning::default()
1218        };
1219        let mut i = inputs(t);
1220        i.numa_nodes = 1;
1221        let r = evaluate(Profile::Database, &i);
1222        assert_eq!(status(&r, "linux.thp"), None);
1223        assert_eq!(status(&r, "linux.numa"), None);
1224        let c = r
1225            .checks
1226            .iter()
1227            .find(|c| c.id == "linux.swappiness")
1228            .unwrap();
1229        assert_eq!(c.status, Status::Unknown);
1230        assert_eq!(c.found, "permission denied");
1231        assert!(r.passed());
1232    }
1233
1234    #[test]
1235    fn test_readiness_profiles_choose_their_checks() {
1236        let general = evaluate(Profile::General, &inputs(tuned()));
1237        assert_eq!(status(&general, "linux.thp"), None);
1238        assert_eq!(status(&general, "net.backlog"), None);
1239        assert_eq!(status(&general, "cpu.governor"), None);
1240        assert_eq!(status(&general, "clock.sync"), Some(Status::Pass));
1241        let web = evaluate(Profile::WebServer, &inputs(tuned()));
1242        assert_eq!(status(&web, "net.backlog"), Some(Status::Pass));
1243        assert_eq!(status(&web, "linux.swappiness"), None);
1244    }
1245
1246    fn disk(fs: &str, kind: DiskKind, free: u64) -> Disk {
1247        Disk {
1248            mount_point: "/var/lib".into(),
1249            device: "/dev/nvme0n1p2".into(),
1250            label: None,
1251            filesystem: fs.into(),
1252            kind,
1253            space: Some(Space {
1254                total: Bytes::new(100),
1255                free: Bytes::new(free),
1256                available: Bytes::new(free),
1257            }),
1258            read_only: false,
1259            removable: false,
1260        }
1261    }
1262
1263    #[test]
1264    fn test_readiness_data_directory() {
1265        let mut i = inputs(tuned());
1266        i.data_dir = Some(DataDir {
1267            disk: Fact::Value(disk("ext4", DiskKind::Ssd, 50)),
1268            tuning: DataDirTuning {
1269                atime: Fact::Value("relatime".into()),
1270                scheduler: Fact::Value("none".into()),
1271            },
1272        });
1273        let r = evaluate(Profile::Database, &i);
1274        for id in [
1275            "fs.type",
1276            "fs.atime",
1277            "disk.space",
1278            "disk.kind",
1279            "disk.scheduler",
1280        ] {
1281            assert_eq!(status(&r, id), Some(Status::Pass), "{id}");
1282        }
1283        i.data_dir = Some(DataDir {
1284            disk: Fact::Value(disk("nfs4", DiskKind::Network, 3)),
1285            tuning: DataDirTuning {
1286                atime: Fact::Value("strictatime".into()),
1287                scheduler: Fact::Skip,
1288            },
1289        });
1290        let r = evaluate(Profile::Database, &i);
1291        assert_eq!(status(&r, "fs.type"), Some(Status::Fail));
1292        assert_eq!(status(&r, "disk.space"), Some(Status::Fail));
1293        assert_eq!(status(&r, "fs.atime"), Some(Status::Warn));
1294        assert_eq!(status(&r, "disk.scheduler"), None);
1295        assert_eq!(
1296            status(&evaluate(Profile::General, &i), "fs.type"),
1297            Some(Status::Warn)
1298        );
1299        i.data_dir = Some(DataDir {
1300            disk: Fact::Value(disk("btrfs", DiskKind::Hdd, 10)),
1301            tuning: DataDirTuning::default(),
1302        });
1303        let r = evaluate(Profile::Database, &i);
1304        assert_eq!(status(&r, "fs.type"), Some(Status::Warn));
1305        assert_eq!(status(&r, "disk.space"), Some(Status::Warn));
1306        assert_eq!(status(&r, "disk.kind"), Some(Status::Warn));
1307    }
1308
1309    #[test]
1310    fn test_readiness_benchmark_profile() {
1311        let mut i = inputs(tuned());
1312        i.bench = Some(crate::bench::environment());
1313        let r = evaluate(Profile::Benchmark, &i);
1314        assert!(status(&r, "cpu.idle").is_some() && status(&r, "virt.vm").is_some());
1315        assert_eq!(status(&r, "clock.sync"), None);
1316        assert_eq!(status(&r, "cpu.governor"), Some(Status::Pass));
1317    }
1318
1319    #[test]
1320    fn test_readiness_outputs() {
1321        let mut t = tuned();
1322        t.swappiness = Fact::Value(60);
1323        let r = evaluate(Profile::Database, &inputs(t));
1324        let text = r.to_string();
1325        assert!(text.contains("WARN    linux.swappiness  "), "{text}");
1326        assert!(text.contains("\n        sysctl -w vm.swappiness=1"));
1327        let mut kv = String::new();
1328        r.write_kv(&mut kv).unwrap();
1329        assert!(kv.contains("check=linux.swappiness status=WARN found=60 recommended=\"1 to 10\""));
1330        let mut j = String::new();
1331        r.write_json(&mut j).unwrap();
1332        assert!(j.starts_with("{\"profile\":\"database\",\"passed\":true,\"checks\":[{\"id\":"));
1333        assert!(j.ends_with("}]}"));
1334    }
1335
1336    #[test]
1337    fn test_readiness_live_never_fails() {
1338        for p in [
1339            Profile::General,
1340            Profile::Database,
1341            Profile::WebServer,
1342            Profile::Benchmark,
1343        ] {
1344            let r = report_for(p, std::env::temp_dir());
1345            assert_eq!(r.profile, p);
1346            assert!(r.checks.iter().any(|c| c.id == "limits.nofile"), "{r}");
1347            for c in &r.checks {
1348                assert!(!c.title.is_empty() && !c.why.is_empty(), "{c:?}");
1349            }
1350        }
1351    }
1352}