gunnar_server/

import_room.rs

//! **An import's memory, against the memory this server may really use** (IMPORT-2, 2026-10-27).
//!
//! # What was measured
//!
//! torvalds/linux (11 988 094 objects, a 6 602 805 008 B pack) imported through
//! `Mirror.Add --create` + `Mirror.Sync` into the shipped `gunnar-server` on a
//! Mini-shaped KVM guest (4 vCPU, 16 GB, no swap, 4 x 10 GB md-raid0), lane
//! KVM-IMPORT on dwarf, gunnar master a0b0ee50: DONE in 724 s, peak RSS 11.9 GiB,
//! of which ~6.2 GiB is the mmapped pack (file-backed, reclaimable) and
//! **5.05 GiB anonymous** at the peak. That box fits it.
//!
//! The same import in a 6 GiB cgroup on that VM (`MemoryMax=6G`): the guard read the
//! VM's 14.6 GB available and never paused, and the kernel OOM-killed gunnar-server
//! in the index at **anon-rss 6 238 872 kB** — no named error, the job gone with it.
//!
//! # Why the box was not the whole question
//!
//! The indexer's memory guard (`znippy_plugin_git::memguard`) reads `/proc/meminfo`,
//! and so did every byte budget derived from it. Inside a cgroup — a container, a
//! `MemoryMax=` unit — `/proc/meminfo` is the HOST's memory: on oden a
//! `--memory=16g` container read 500 GB, so the guard never paused and the import
//! went to 16.08 GB inside a 16 GB limit (import-big, 2026-10-26). A limit the
//! guard cannot see is a limit nothing keeps the import under.
//!
//! # What this module decides
//!
//! * [`MemoryLimit`]: the memory this process may use — `MemTotal`, or the
//!   cgroup v2 `memory.max` when that is smaller — and which of the two it was.
//! * [`import_room`]: before a fetched pack is indexed, whether indexing it fits
//!   that limit with [`MARGIN_PERCENT`] to spare. If not, the import is refused BY
//!   NAME (`[import-memory]`) before the index starts, and the repository it made
//!   is removed (`mirror_fetch::discard_created`), instead of an index that runs
//!   the box out of memory.
//! * [`cgroup_meminfo`]: under a cgroup limit, a `/proc/meminfo`-shaped text of the
//!   CGROUP's memory (`MemTotal` = `memory.max`, `MemAvailable` = room left plus
//!   reclaimable file pages), which `mirror_fetch` keeps fresh in a file while the
//!   import indexes and hands to the store's memory guard, so the guard pauses and
//!   flushes against the limit that is really there.

use std::path::{Path, PathBuf};

/// Anonymous bytes indexing one object of a fetched pack holds at its peak.
///
/// MEASURED (see the module doc): 5172 MiB anonymous at the peak for 11 988 094
/// objects is 452 B per object, server included. 512 is that figure rounded up,
/// the side that keeps the box alive.
pub const IMPORT_BYTES_PER_OBJECT: u64 = 512;

/// What an import holds whatever the pack's size: the server itself (~50 MiB
/// anonymous when idle, measured), the resolver's workers and their lookahead
/// (4 x 32 MiB), held delta bases (51 MiB peak measured for linux), row batches.
pub const IMPORT_FIXED_BYTES: u64 = 512 * 1024 * 1024;

/// The share of the limit an import must leave free: the same 20 % floor the
/// indexer's memory guard pauses under (`znippy_plugin_git::memguard::DEFAULT_BUDGET_PCT`).
pub const MARGIN_PERCENT: u64 = 20;

/// Where a [`MemoryLimit`] came from.
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum LimitSource {
    /// `/proc/meminfo` `MemTotal`: no cgroup limit, or a looser one.
    Box,
    /// The cgroup v2 `memory.max` of this process, in this directory.
    Cgroup(PathBuf),
}

/// The memory this process may use, and why that number.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct MemoryLimit {
    pub bytes: u64,
    pub source: LimitSource,
}

/// One reading of a cgroup v2 directory's memory files.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct CgroupMemory {
    /// `memory.max`.
    pub max: u64,
    /// `memory.current`: everything charged, page cache included.
    pub current: u64,
    /// Clean file pages the kernel can drop inside the cgroup: `active_file +
    /// inactive_file - file_dirty - file_writeback` out of `memory.stat`.
    pub reclaimable: u64,
}

impl CgroupMemory {
    /// What the cgroup still has: the room under `memory.max` plus the clean page
    /// cache it could drop. Never more than `max`.
    pub fn available(&self) -> u64 {
        self.max
            .saturating_sub(self.current)
            .saturating_add(self.reclaimable)
            .min(self.max)
    }
}

impl MemoryLimit {
    /// This process's limit: `/proc/meminfo`, `/proc/self/cgroup`, `/sys/fs/cgroup`.
    /// `None` when neither the box nor a cgroup can be read.
    pub fn read() -> Option<MemoryLimit> {
        Self::read_from(Path::new("/proc/meminfo"), own_cgroup_dir().as_deref())
    }

    /// The same rule over named files: what a test hands a fake box and a fake cgroup.
    pub fn read_from(meminfo: &Path, cgroup_dir: Option<&Path>) -> Option<MemoryLimit> {
        let total = std::fs::read_to_string(meminfo)
            .ok()
            .and_then(|t| crate::dashboard::probe::parse_meminfo(&t))
            .map(|(total, _)| total)
            .filter(|t| *t > 0);
        let cgroup = cgroup_dir.and_then(tightest_cgroup);
        match (total, cgroup) {
            (Some(total), Some((dir, c))) if c.max < total => Some(MemoryLimit {
                bytes: c.max,
                source: LimitSource::Cgroup(dir),
            }),
            (None, Some((dir, c))) => Some(MemoryLimit {
                bytes: c.max,
                source: LimitSource::Cgroup(dir),
            }),
            (Some(total), _) => Some(MemoryLimit {
                bytes: total,
                source: LimitSource::Box,
            }),
            (None, None) => None,
        }
    }

    /// The cgroup directory when the limit is a cgroup's.
    pub fn cgroup_dir(&self) -> Option<&Path> {
        match &self.source {
            LimitSource::Cgroup(dir) => Some(dir),
            LimitSource::Box => None,
        }
    }

    /// What the limit is, in words: `16.0G (MemTotal)`, `6.0G (cgroup memory.max)`.
    pub fn words(&self) -> String {
        let source = match self.source {
            LimitSource::Box => "this box's MemTotal",
            LimitSource::Cgroup(_) => "this server's cgroup memory.max",
        };
        format!("{} ({source})", crate::budget::human(self.bytes))
    }
}

/// This process's own cgroup v2 directory, out of `/proc/self/cgroup`'s `0::` line.
pub fn own_cgroup_dir() -> Option<PathBuf> {
    let own = std::fs::read_to_string("/proc/self/cgroup").ok()?;
    let rel = own
        .lines()
        .find_map(|line| line.strip_prefix("0::"))?
        .trim()
        .trim_start_matches('/');
    Some(Path::new("/sys/fs/cgroup").join(rel))
}

/// **The tightest limit over `leaf`** (RA5-40): `leaf` and every parent that holds a
/// `memory.max` (cgroup v2's root holds none, and the walk ends there), the one with the
/// smallest `memory.max`, with its directory. A slice's `MemoryMax=` or a pod's limit above
/// the container is a limit as much as the leaf's own. `None`: no limit anywhere.
pub fn tightest_cgroup(leaf: &Path) -> Option<(PathBuf, CgroupMemory)> {
    let mut best: Option<(PathBuf, CgroupMemory)> = None;
    let mut dir = Some(leaf);
    while let Some(d) = dir {
        if !d.join("memory.max").exists() {
            break;
        }
        if let Some(c) = cgroup_memory_in(d) {
            if best.as_ref().is_none_or(|(_, b)| c.max < b.max) {
                best = Some((d.to_path_buf(), c));
            }
        }
        dir = d.parent();
    }
    best
}

/// `memory.max`, `memory.current` and the reclaimable file pages of `dir`. `None`
/// when a file is missing or `memory.max` is `max` (no limit).
pub fn cgroup_memory_in(dir: &Path) -> Option<CgroupMemory> {
    let max = std::fs::read_to_string(dir.join("memory.max")).ok()?;
    let max: u64 = match max.trim() {
        "max" => return None,
        n => n.parse().ok()?,
    };
    let current: u64 = std::fs::read_to_string(dir.join("memory.current"))
        .ok()?
        .trim()
        .parse()
        .ok()?;
    let stat = std::fs::read_to_string(dir.join("memory.stat")).unwrap_or_default();
    let field = |name: &str| -> u64 {
        stat.lines()
            .find_map(|l| l.strip_prefix(name).and_then(|r| r.strip_prefix(' ')))
            .and_then(|v| v.trim().parse().ok())
            .unwrap_or(0)
    };
    let reclaimable = (field("active_file") + field("inactive_file"))
        .saturating_sub(field("file_dirty") + field("file_writeback"));
    Some(CgroupMemory {
        max,
        current,
        reclaimable,
    })
}

/// A `/proc/meminfo`-shaped text of a cgroup's memory, for a reader that only
/// reads `/proc/meminfo` (the indexer's memory guard): `MemTotal` is the cgroup's
/// `memory.max`, `MemAvailable` its [`CgroupMemory::available`].
pub fn cgroup_meminfo(c: &CgroupMemory) -> String {
    format!(
        "MemTotal:       {} kB\nMemAvailable:   {} kB\n",
        c.max / 1024,
        c.available() / 1024
    )
}

/// Anonymous memory indexing a fetched pack of `objects` objects is budgeted at.
pub const fn import_need(objects: u64) -> u64 {
    IMPORT_FIXED_BYTES + objects * IMPORT_BYTES_PER_OBJECT
}

/// The most an import may hold under `limit`: the limit less [`MARGIN_PERCENT`].
pub const fn import_ceiling(limit: u64) -> u64 {
    limit - limit / 100 * MARGIN_PERCENT
}

/// **Does indexing a pack of `objects` objects fit `limit`, with the margin?**
///
/// `Err` is the refusal, said by name before the index starts: what the pack is,
/// what indexing it needs, and what this server may use.
pub fn import_room(path: &str, objects: u64, limit: &MemoryLimit) -> Result<(), String> {
    let need = import_need(objects);
    let ceiling = import_ceiling(limit.bytes);
    if need <= ceiling {
        return Ok(());
    }
    Err(format!(
        "[import-memory] {path}: the fetched pack has {objects} objects, and indexing it needs \
         about {} of memory; this server may use {}, and an import keeps {MARGIN_PERCENT} % of \
         that free, so at most {}. Nothing was imported and the repository was not kept. A \
         server with more memory imports it.",
        crate::budget::human(need),
        limit.words(),
        crate::budget::human(ceiling),
    ))
}

#[cfg(test)]
mod tests {
    use super::*;

    const GIB: u64 = 1024 * 1024 * 1024;
    /// torvalds/linux as GitHub served it on 2026-10-27.
    const LINUX_OBJECTS: u64 = 11_988_094;

    fn fake_box(dir: &Path, total: u64) -> PathBuf {
        let p = dir.join("meminfo");
        std::fs::write(
            &p,
            format!(
                "MemTotal: {} kB\nMemFree: 1 kB\nMemAvailable: {} kB\n",
                total / 1024,
                total / 1024
            ),
        )
        .unwrap();
        p
    }

    fn fake_cgroup(dir: &Path, max: &str, current: u64, stat: &str) -> PathBuf {
        let cg = dir.join("cg");
        std::fs::create_dir_all(&cg).unwrap();
        std::fs::write(cg.join("memory.max"), format!("{max}\n")).unwrap();
        std::fs::write(cg.join("memory.current"), format!("{current}\n")).unwrap();
        std::fs::write(cg.join("memory.stat"), stat).unwrap();
        cg
    }

    /// ★ A cgroup limit below the box is the limit, and says it is the cgroup's.
    #[test]
    fn a_cgroup_limit_below_the_box_is_the_limit() {
        let d = tempfile::tempdir().unwrap();
        let meminfo = fake_box(d.path(), 500 * GIB);
        let cg = fake_cgroup(d.path(), &(6 * GIB).to_string(), GIB, "anon 1\n");
        let limit = MemoryLimit::read_from(&meminfo, Some(&cg)).expect("a limit");
        assert_eq!(
            limit.bytes,
            6 * GIB,
            "a 6 GiB cgroup on a 500 GiB host may use 6 GiB: {limit:?}"
        );
        assert_eq!(limit.source, LimitSource::Cgroup(cg.clone()));
        assert!(
            limit.words().contains("cgroup memory.max"),
            "{}",
            limit.words()
        );
    }

    /// No cgroup limit (`max`), or one above the box: the box is the limit.
    #[test]
    fn without_a_cgroup_limit_the_box_is_the_limit() {
        let d = tempfile::tempdir().unwrap();
        let meminfo = fake_box(d.path(), 16 * GIB);
        let cg = fake_cgroup(d.path(), "max", GIB, "");
        assert_eq!(
            MemoryLimit::read_from(&meminfo, Some(&cg)),
            Some(MemoryLimit {
                bytes: 16 * GIB,
                source: LimitSource::Box
            })
        );
        let loose = fake_cgroup(d.path(), &(64 * GIB).to_string(), GIB, "");
        assert_eq!(
            MemoryLimit::read_from(&meminfo, Some(&loose)).map(|l| l.bytes),
            Some(16 * GIB)
        );
        assert_eq!(
            MemoryLimit::read_from(&meminfo, None).map(|l| l.source),
            Some(LimitSource::Box)
        );
    }

    /// ★ The model covers what the Mini VM measured, and linux fits the Mini.
    #[test]
    fn linux_fits_the_mini_with_the_margin_and_the_model_covers_the_measurement() {
        // The 5 s sampler's peak on the Mini VM, and the anon-rss the 6 GiB cgroup was
        // OOM-killed at (still climbing): the budget must cover both.
        for (measured, what) in [
            (5172 * 1024 * 1024, "the Mini VM's anonymous peak"),
            (6_238_872 * 1024, "the anon-rss of the OOM kill"),
        ] {
            assert!(
                import_need(LINUX_OBJECTS) >= measured,
                "the budget for linux ({}) is below {what} ({})",
                crate::budget::human(import_need(LINUX_OBJECTS)),
                crate::budget::human(measured)
            );
        }
        let mini = MemoryLimit {
            bytes: 16 * GIB,
            source: LimitSource::Box,
        };
        assert_eq!(import_room("alice/linux", LINUX_OBJECTS, &mini), Ok(()));
    }

    /// ★ The same import inside a 6 GiB cgroup on a big host is refused by name,
    /// before the index, and the sentence says why in numbers.
    #[test]
    fn linux_in_a_6g_cgroup_is_refused_by_name_before_the_index() {
        let d = tempfile::tempdir().unwrap();
        let meminfo = fake_box(d.path(), 500 * GIB);
        let cg = fake_cgroup(d.path(), &(6 * GIB).to_string(), GIB, "");
        let limit = MemoryLimit::read_from(&meminfo, Some(&cg)).unwrap();
        let err =
            import_room("alice/linux", LINUX_OBJECTS, &limit).expect_err("must not fit 6 GiB");
        for word in [
            "[import-memory]",
            "alice/linux",
            "11988094 objects",
            "6.0G",
            "cgroup memory.max",
            "4.8G",
        ] {
            assert!(err.contains(word), "the refusal must say {word:?}: {err}");
        }
        // …and a small repository in the same cgroup is not refused.
        assert_eq!(import_room("alice/small", 100_000, &limit), Ok(()));
    }

    /// ★ **RA5-40: a limit on a PARENT cgroup is the limit.** IMPORT-2 read only the leaf's
    /// `memory.max`; a systemd slice's `MemoryMax=` (or a pod above the container) was missed,
    /// and the import was admitted against MemTotal, into the OOM kill it exists to prevent. The
    /// walk goes up while a `memory.max` is there, and the smallest one is the limit.
    #[test]
    fn a_parent_cgroups_limit_is_the_limit() {
        let d = tempfile::tempdir().unwrap();
        let meminfo = fake_box(d.path(), 500 * GIB);
        let slice = fake_cgroup(d.path(), &(6 * GIB).to_string(), GIB, "anon 1\n");
        let leaf = fake_cgroup(&slice, "max", GIB, "anon 1\n");
        let limit = MemoryLimit::read_from(&meminfo, Some(&leaf)).expect("a limit");
        assert_eq!(limit.bytes, 6 * GIB, "the slice's MemoryMax");
        assert_eq!(
            limit.cgroup_dir(),
            Some(slice.as_path()),
            "named as the slice's"
        );
        assert!(import_room("alice/linux", LINUX_OBJECTS, &limit).is_err());
        // A leaf tighter than its slice is the limit.
        let tight = fake_cgroup(&slice.join("cg"), &(2 * GIB).to_string(), GIB, "anon 1\n");
        assert_eq!(
            MemoryLimit::read_from(&meminfo, Some(&tight))
                .unwrap()
                .bytes,
            2 * GIB
        );
    }

    /// The 4 GB twin shape cannot take linux, and says so.
    #[test]
    fn a_4g_box_refuses_linux() {
        let twin = MemoryLimit {
            bytes: 4 * GIB,
            source: LimitSource::Box,
        };
        let err = import_room("alice/linux", LINUX_OBJECTS, &twin).unwrap_err();
        assert!(err.contains("MemTotal"), "{err}");
    }

    /// ★ The cgroup meminfo the guard reads is the cgroup's, not the host's: a
    /// cgroup near its limit reads near-empty; clean page cache counts as room.
    #[test]
    fn the_cgroup_meminfo_is_the_cgroups_memory() {
        let d = tempfile::tempdir().unwrap();
        let cg = fake_cgroup(
            d.path(),
            &(6 * GIB).to_string(),
            6 * GIB - 100 * 1024 * 1024,
            "anon 5000000000\nactive_file 209715200\ninactive_file 104857600\nfile_dirty 104857600\nfile_writeback 0\n",
        );
        let c = cgroup_memory_in(&cg).expect("a limited cgroup");
        // 100 MiB of room + (200 + 100 - 100) MiB clean page cache.
        assert_eq!(c.available(), 300 * 1024 * 1024);
        let text = cgroup_meminfo(&c);
        assert_eq!(
            crate::dashboard::probe::parse_meminfo(&text),
            Some((6 * GIB, 300 * 1024 * 1024)),
            "{text}"
        );
        assert_eq!(cgroup_memory_in(&fake_cgroup(d.path(), "max", 1, "")), None);
    }

    // The indexer's own guard reading that text inside the cgroup is tested where the engine may
    // be named: `storage/znippy_engine.rs`, `the_indexers_guard_sees_the_cgroup_through_the_text`.
}