gunnar_server/

limits.rs

//! **The descriptor limit: raised once, read once, and the handle cache sized
//! to fit under it.**
//!
//! # The bug this is written against — MEASURED 2026-08-29 on the rig
//!
//! The 202nd repository touched since boot failed on every filesystem shape:
//!
//! ```text
//! opening the git store at /data/repos/rig/z…-202.git/gunnar-git.znippy:
//!   open journal …/objects.pack.journal: No file descriptors available (os error 24)
//! ```
//!
//! and with it the ssh accept loop died — every later connection got
//! `Connection reset by peer` until reboot while gRPC kept answering. Two
//! facts made it: an open znippy store held **five** descriptors
//! (`objects.pack` ×3, `objects.pack.journal`, `objects.tail` — read off
//! `/proc/self/fd` in `tests/fd_budget.rs`; **three** since 2026-08-30, see
//! [`FDS_PER_OPEN_REPO`]) and the handle cache kept
//! [`DEFAULT_OPEN_REPOS`](crate::stores::DEFAULT_OPEN_REPOS) = 256 of them,
//! while PID 1 runs with the kernel's default `RLIMIT_NOFILE` soft limit of
//! **1024** — nothing in the server or in `korp-start` ever raised it. 256 × 5
//! is 1280: the cache alone was over the limit, and at ~200 repositories the
//! next open, and the next `accept`, had nothing left.
//!
//! # Both halves
//!
//! * [`nofile`] raises the soft limit once per process — as PID 1 (which holds
//!   `CAP_SYS_RESOURCE`) to [`PID1_NOFILE`] on both soft and hard, elsewhere to
//!   whatever the hard limit already allows. Pure Rust: `rustix::process`.
//!   A limit that is still under [`COMFORTABLE_NOFILE`] afterwards is a
//!   `WARN` on `gunnar::limits`, which the dashboard shows as a row.
//! * [`open_repo_budget`] sizes the handle cache from the limit that resulted:
//!   `(soft − RESERVED_FDS) / FDS_PER_OPEN_REPO`, clamped to
//!   `[MIN_OPEN_REPOS, DEFAULT_OPEN_REPOS]`. Under 1024 that is 256 handles
//!   (765 descriptors) with 256 left for listeners, sessions, the control
//!   database and the log — and the 1500-repository test passes under
//!   `ulimit -n 1024`. Under a raised limit it is the old 256.
//!
//! The listener half — an `accept` that fails with `EMFILE` must not end the
//! accept loop — is `gunnar-ssh`'s and lives in its `server.rs`.

use std::sync::OnceLock;

use rustix::process::{getrlimit, setrlimit, Resource, Rlimit};

/// What PID 1 sets both limits to. The kernel's `fs.nr_open` default is
/// 1 048 576 and a `setrlimit` above it is refused, so this is the ceiling a
/// stock kernel accepts; a smaller `nr_open` is honoured by the fallback in
/// [`raise`].
pub const PID1_NOFILE: u64 = 1 << 20;

/// Below this many descriptors the server says so on the dashboard. 65 536 is
/// what a distribution's `LimitNOFILE=` for a network daemon typically reads;
/// 1024 is what `sshd` on a laptop gets and is where the bug lived.
pub const COMFORTABLE_NOFILE: u64 = 65_536;

/// Descriptors one open repository handle costs on the znippy engine — read
/// off `/proc/self/fd` (`tests/fd_budget.rs`, and per-store in the engine's own
/// `znippy-plugin-git/tests/fd_per_store.rs`): `objects.pack`,
/// `objects.pack.journal`, `objects.tail`. The gix arm holds fewer; budgeting
/// for the larger of the two is the safe side.
///
/// **Five until 2026-08-30**, because `objects.pack` was opened three
/// times in one process — the writer's read/write handle, the absorber's
/// read-only `pread` fallback, and the index workers' read-only handle. All
/// three read and write positionally on an append-only file, so the two extra
/// descriptors bought a kernel object and a page-cache alias per open
/// repository and no isolation; they are one shared open file description now
/// (`ArchiveWrite::blobs_shared`). It is a **divisor**, so the whole of
/// [`open_repo_budget`] moves with it: under the kernel default soft limit of
/// 1024 the ceiling goes 153 → 256 handles, +67 % concurrently-open
/// repositories at the same `RLIMIT_NOFILE`.
pub const FDS_PER_OPEN_REPO: u64 = 3;

/// Descriptors kept back from the handle cache for everything that is not a
/// cached repository: the two listeners, every live session (one socket, plus
/// the transfer's own store handle while it runs), the control database, the
/// log tap, the scrape socket.
pub const RESERVED_FDS: u64 = 256;

/// The handle cache never shrinks under this, whatever the limit says: a
/// server that can keep no repository open re-opens (a `refold`) on every
/// request, and a limit that low is the WARN row's job, not the cache's.
pub const MIN_OPEN_REPOS: usize = 8;

/// The descriptor limit this process ended up with.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct Nofile {
    /// The soft limit after [`nofile`] had its one go at raising it.
    pub soft: u64,
    /// The hard limit after the same.
    pub hard: u64,
    /// What the soft limit was before.
    pub was: u64,
}

impl Nofile {
    /// Was the soft limit raised at all?
    pub fn raised(&self) -> bool {
        self.soft > self.was
    }
}

/// One `Rlimit` as two numbers; `None` means unlimited, spelled `u64::MAX`.
fn as_pair(limit: Rlimit) -> (u64, u64) {
    (
        limit.current.unwrap_or(u64::MAX),
        limit.maximum.unwrap_or(u64::MAX),
    )
}

/// Raise the descriptor limit as far as this process may.
///
/// PID 1 asks for [`PID1_NOFILE`] on both limits and, if the kernel's
/// `fs.nr_open` refuses that, for the hard limit it already has. Anyone else
/// may only move the soft limit up to the hard one, and does. Never lowers.
fn raise(is_pid1: bool) -> Nofile {
    let before = getrlimit(Resource::Nofile);
    let (was, hard) = as_pair(before);
    let want: &[(u64, u64)] = if is_pid1 {
        &[(PID1_NOFILE, PID1_NOFILE), (hard, hard)]
    } else {
        &[(hard, hard)]
    };
    for &(soft, max) in want {
        if soft <= was && max <= hard {
            break;
        }
        let asked = Rlimit {
            current: (soft != u64::MAX).then_some(soft),
            maximum: (max != u64::MAX).then_some(max),
        };
        if setrlimit(Resource::Nofile, asked).is_ok() {
            break;
        }
    }
    let (soft, hard) = as_pair(getrlimit(Resource::Nofile));
    Nofile { soft, hard, was }
}

/// **The process's descriptor limit, raised once.**
///
/// The first caller does the raising and the logging; every later call reads
/// the same answer. [`crate::stores::ServerStores::open`] is the first caller
/// on every server path, so the handle cache is always sized after the raise,
/// and `serve::run` calls it again for nothing but the guarantee that the WARN
/// row exists before the listeners bind.
pub fn nofile() -> Nofile {
    static LIMIT: OnceLock<Nofile> = OnceLock::new();
    *LIMIT.get_or_init(|| {
        let limit = raise(appliance_pid1::net::is_init());
        announce(&limit);
        limit
    })
}

/// Say what the limit is — one INFO line, or one WARN row when it is low.
fn announce(limit: &Nofile) {
    if limit.soft < COMFORTABLE_NOFILE {
        tracing::warn!(
            target: "gunnar::limits",
            event = "limits.nofile.low",
            soft = limit.soft,
            hard = limit.hard,
            was = limit.was,
            open_repos = open_repo_budget(limit.soft),
            "fd limit {} is low (hard {}); the handle cache is bounded to fit but a busy box \
             will re-open repositories — raise LimitNOFILE / ulimit -n",
            limit.soft,
            limit.hard
        );
    } else {
        tracing::info!(
            target: "gunnar::limits",
            event = "limits.nofile",
            soft = limit.soft,
            hard = limit.hard,
            was = limit.was,
            raised = limit.raised(),
            open_repos = open_repo_budget(limit.soft),
            "fd limit {} (was {}); the handle cache keeps up to {} repositories open",
            limit.soft,
            limit.was,
            open_repo_budget(limit.soft)
        );
    }
}

/// How many repository handles fit under a soft limit of `soft` descriptors.
///
/// `(soft − RESERVED_FDS) / FDS_PER_OPEN_REPO`, clamped to
/// `[MIN_OPEN_REPOS, DEFAULT_OPEN_REPOS]`: 1024 → 256 (the cap, exactly —
/// 256 × 3 + 256 reserved is 1024 to the descriptor), 512 → 85, 256 → 8 (the
/// floor). It was 1024 → 153 while a store cost five.
pub fn open_repo_budget(soft: u64) -> usize {
    let fits = soft.saturating_sub(RESERVED_FDS) / FDS_PER_OPEN_REPO;
    usize::try_from(fits)
        .unwrap_or(usize::MAX)
        .clamp(MIN_OPEN_REPOS, crate::stores::DEFAULT_OPEN_REPOS)
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn the_budget_fits_under_the_kernel_default_and_caps_at_the_old_bound() {
        // The rig's PID 1: 1024 descriptors. At five per store that was 153
        // handles = 765 descriptors; at three it is the cap, 256 handles = 768
        // descriptors, with RESERVED_FDS = 256 left over to the descriptor.
        // The bug fixed here was 256 handles *at five*, which is 1280.
        assert_eq!(open_repo_budget(1024), 256);
        assert!(open_repo_budget(1024) as u64 * FDS_PER_OPEN_REPO + RESERVED_FDS <= 1024);
        // Below the cap the division is what decides, and the reservation is
        // still whole: 512 → (512−256)/3 = 85.
        assert_eq!(open_repo_budget(512), 85);
        assert!(open_repo_budget(512) as u64 * FDS_PER_OPEN_REPO + RESERVED_FDS <= 512);
        assert_eq!(open_repo_budget(2048), crate::stores::DEFAULT_OPEN_REPOS);
        assert_eq!(open_repo_budget(65_536), crate::stores::DEFAULT_OPEN_REPOS);
        assert_eq!(
            open_repo_budget(u64::MAX),
            crate::stores::DEFAULT_OPEN_REPOS
        );
        // A limit too small to hold anything still keeps the floor; the WARN
        // row is what says the box is misconfigured.
        assert_eq!(open_repo_budget(256), MIN_OPEN_REPOS);
        assert_eq!(open_repo_budget(0), MIN_OPEN_REPOS);
    }

    #[test]
    fn a_non_pid1_process_ends_at_its_hard_limit_and_never_lower() {
        let before = as_pair(getrlimit(Resource::Nofile));
        let limit = raise(false);
        assert_eq!(limit.was, before.0);
        assert!(
            limit.soft >= before.0,
            "raise lowered the soft limit: {limit:?}"
        );
        assert_eq!(
            limit.soft, limit.hard,
            "an unprivileged raise stops at the hard limit"
        );
        assert_eq!(
            limit.hard, before.1,
            "an unprivileged raise must not move the hard limit"
        );
        assert_eq!(limit.raised(), limit.soft > before.0);
        // Idempotent: a second go changes nothing.
        let again = raise(false);
        assert_eq!((again.soft, again.hard), (limit.soft, limit.hard));
    }

    #[test]
    fn nofile_is_one_answer_per_process() {
        assert_eq!(nofile(), nofile());
        assert!(nofile().soft >= MIN_OPEN_REPOS as u64);
    }
}