gunnar_server/

readiness.rs

//! **Is this server ready to serve git?** — one flag, set once, and the honest
//! answer while it is clear.
//!
//! # The boot window — MEASURED 2026-08-30
//!
//! After a power cut the appliance answered ssh :2222 and grpc :50051
//! **0.6–0.7 s before its stores had finished their post-crash open**, and a
//! read in that window came back `the repository is unavailable; try again` —
//! which is [`gunnar_wire::Error::Store`]'s sentence, arrives with a *denial*
//! exit status, and is what a client hears when a repository is genuinely
//! broken. So nothing could tell "not yet" from "gone": the first
//! power-cut door reported **64 acked pushes missing** that a boot one second
//! later found every one of, and the same window wedged a whole bench run's
//! clone and fetch phases.
//!
//! # Where the 0.6–0.7 s comes from
//!
//! Not from `ServerStores::open` — that runs before `serve` is called at all.
//! It is [`crate::disktest`]. `serve::run` spawns the start-up disk test on
//! `/data` and then binds the listeners **immediately**, without waiting for
//! it; the test writes a gigabyte sequentially, reads it back with `O_DIRECT`
//! and then does 4 KiB `fsync`s for its own timing loop. The rig measured that
//! run at `4k sync 4517 ops/s (0.7 s)` on the rig — the same 0.6–0.7 s. For
//! that whole window a repository's first post-crash store open is contending
//! for a disk that another thread of the same process is deliberately
//! saturating.
//!
//! # The fix is not to bind later
//!
//! PID 1 must answer quickly: an operator watching a console needs the box to
//! be *there*, `gunnar control status` has to work on a server nobody holds a
//! key for, and the dashboard band is drawn from the same process. So the
//! listeners still bind at the same instant. What changes is the **answer**:
//! while this flag is clear, every git transfer is refused in git's own
//! protocol with `EXIT_BUSY` (75, `EX_TEMPFAIL`) and a sentence containing the
//! word **starting** ([`gunnar_ssh::STARTING_MESSAGE`]), and the flag is
//! reported by `Bootstrap.Status` — the one call an unenrolled server answers —
//! so `gunnar control status` says it too.
//!
//! Boot stays fast and stores stay lazy: this gates on the one boot task that
//! competes with the store for the disk, not on opening every repository. A
//! box with a thousand repositories does not pay a thousand opens to say hello.

/// What [`begin`] said it was waiting for, so [`detail`] can name it instead of
/// guessing. Empty until a boot window is opened.
static WAITING_FOR: std::sync::Mutex<String> = std::sync::Mutex::new(String::new());

use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
use std::sync::OnceLock;

/// The process-wide readiness flag.
///
/// An `Arc<AtomicBool>` and not a trait, because `gunnar-ssh` may not read the
/// operating system: "have the boot tasks finished" is this crate's fact to
/// publish, not that crate's to discover. It is handed to the listener with
/// `GunnarSsh::with_readiness`.
/// It starts **true**, and that is the safe default rather than an oversight.
/// Only a boot path that actually opens a window closes it ([`begin`]): a
/// library caller that builds a listener directly — every integration test in
/// this workspace, `gunnar-solo` on a developer machine — has nothing competing
/// for the disk and must not be made to refuse every transfer because a boot it
/// never performed never finished.
pub fn flag() -> Arc<AtomicBool> {
    static FLAG: OnceLock<Arc<AtomicBool>> = OnceLock::new();
    Arc::clone(FLAG.get_or_init(|| Arc::new(AtomicBool::new(true))))
}

/// **Open the boot window**: from here until [`mark_ready`], git transfers are
/// refused with `starting` and `EX_TEMPFAIL`.
///
/// Called by the one boot task that competes with the store for the disk. If
/// nothing calls this, the server is ready from its first bound socket, which
/// is the truth on every path that has no such task.
pub fn begin(what: &str) {
    // Remember WHAT, so the status door can name it. `begin` has always taken the
    // task's name and thrown it away, while `detail` spelled "the start-up disk
    // test" into the answer — true only because that is the single caller today.
    // The moment a second boot task opens this window that sentence becomes a lie,
    // and an operator asking "what are you waiting for" is the one person who must
    // not be lied to.
    if let Ok(mut waiting) = WAITING_FOR.lock() {
        waiting.clear();
        waiting.push_str(what);
    }
    flag().store(false, Ordering::Relaxed);
    tracing::info!(
        target: "gunnar::limits",
        event = "serve.starting",
        waiting_for = what,
        "git transfers are refused with `starting` (EX_TEMPFAIL, a RETRY) until {} finishes: it \
         is saturating the same disk a repository's first post-crash open has to read",
        what
    );
}

/// Whether this server is serving git yet.
pub fn is_ready() -> bool {
    flag().load(Ordering::Relaxed)
}

/// **Say the server is ready**, once, with one INFO row naming what it waited
/// for and how long.
///
/// Idempotent: a second call is ignored rather than logged again, so a boot
/// path that can finish two ways (the disk test ran, or it was skipped) may
/// call it from both.
pub fn mark_ready(waited_for: &str, since: std::time::Instant) {
    if flag().swap(true, Ordering::Relaxed) {
        return;
    }

    tracing::info!(
        target: "gunnar::limits",
        event = "serve.ready",
        waited_for,
        ms = since.elapsed().as_millis() as u64,
        "serving git: {} finished after {} ms. Until this row every git transfer was refused \
         with `starting` and EX_TEMPFAIL, which is a RETRY and not a missing repository.",
        waited_for,
        since.elapsed().as_millis()
    );
}

/// **Where the git listener actually bound.**
///
/// A pair of addresses and not a flag, because the defect this closes WAS a
/// flag. [`is_ready`] is a *boot latch*: it starts **true** and is closed only
/// by a boot task that opens a window, so `Bootstrap.Status.serving_git` has
/// always answered "no boot task is still running" — a question about this
/// process's start-up, never about a socket. MEASURED 2026-09-20 on the live
/// estate: 10.13.8.101 answered `git serving` with a build, a role and an
/// applied sequence, while every `git-receive-pack` to
/// `ssh://git@10.13.8.101:2222` failed `Connection refused (os error 111)` —
/// an RST, so nothing was bound there at all. A health line that cannot be
/// false is not a health line.
///
/// `asked` is what `--ssh-bind` named; `bound` is what `local_addr()` returned.
/// They differ on exactly one path today, and it is the one that matters:
/// `serve::start` binds a **twin**'s listener to `127.0.0.1:0` whatever was
/// asked for, because a twin serves no git to anyone.
///
/// Carried per `CoreControlPlane` rather than in a static: two servers in one
/// test process are two listeners, and a process-wide slot would let the
/// second one answer for the first.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct GitListener {
    /// The address `--ssh-bind` named.
    pub asked: std::net::SocketAddr,
    /// The address the listener actually took.
    pub bound: std::net::SocketAddr,
}

impl GitListener {
    /// **Is git reachable at the door that was asked for?**
    ///
    /// The addresses, not a flag: the listener serves git when the socket it
    /// took is the socket that was named. A port of `0` in `asked` is a
    /// deliberate "any port" — every integration test in this workspace binds
    /// that way — so only the interface is compared there.
    pub fn serves_git(&self) -> bool {
        self.bound.ip() == self.asked.ip()
            && (self.asked.port() == 0 || self.bound.port() == self.asked.port())
    }

    /// One sentence for an operator, naming both addresses.
    fn detail(&self) -> String {
        if self.serves_git() {
            format!("serving git on {}", self.bound)
        } else {
            format!(
                "NOT serving git: the listener was asked for {} and took {} instead, which is \
                 what a TWIN does — it serves no git to anyone, so a clone, a fetch or a \
                 replication push to {} is refused by the operating system before gunnar sees \
                 the connection",
                self.asked, self.bound, self.asked
            )
        }
    }
}

/// **The predicate `Bootstrap.Status.serving_git` answers**: the boot window is
/// closed AND the git listener is bound where it was asked to be.
///
/// Both halves can be false, and each is false for a different box: an
/// appliance for 0.6 s after a power cut (the window), a twin for its whole
/// life (the listener). `None` — nothing handed this plane a listener, which is
/// `gunnar-solo` and every library caller that built one itself — keeps the
/// answer this door has always given, because an unbound-because-unasked is not
/// an unbound-because-twin and a status door must not turn silence into a
/// refusal.
pub fn is_serving_git(listener: Option<GitListener>) -> bool {
    is_ready() && listener.map(|l| l.serves_git()).unwrap_or(true)
}

/// One sentence for `Bootstrap.Status`, so an operator with no key can ask.
///
/// [`detail_with`] with no listener: the boot window alone, which is the whole
/// truth for a caller that has no listener to report.
pub fn detail() -> String {
    detail_with(None)
}

/// One sentence for `Bootstrap.Status`, given the listener this server bound.
pub fn detail_with(listener: Option<GitListener>) -> String {
    if is_ready() {
        // The boot window is closed, so the answer belongs to the LISTENER and
        // not to this latch.
        return match listener {
            Some(listener) => listener.detail(),
            None => "serving git".to_string(),
        };
    }
    // A poisoned lock must not take the status door down with it: a server that
    // cannot say WHICH task it waits for can still say that it is waiting.
    let what = WAITING_FOR
        .lock()
        .ok()
        .map(|w| w.clone())
        .filter(|w| !w.is_empty())
        .unwrap_or_else(|| "a boot task".to_string());
    format!(
        "starting: {what} is still running, so git transfers are refused with EX_TEMPFAIL \
         (retry); the control plane answers now"
    )
}

#[cfg(test)]
mod tests {
    use super::*;

    /// The flag is one flag, and marking it twice says so once.
    /// Ready by default, closed only by a boot that opens a window, and one
    /// latch for the whole process.
    #[test]
    fn readiness_is_one_latch_that_starts_open() {
        let a = flag();
        let b = flag();
        assert!(Arc::ptr_eq(&a, &b), "two flags is two truths");
        assert!(
            is_ready(),
            "a process with no boot task serves from its first socket"
        );
        begin("a test");
        assert!(!is_ready());
        assert!(detail().contains("starting"));
        // The door names the task that actually opened the window.
        assert!(
            detail().contains("a test"),
            "detail() must name what begin() is waiting for: {}",
            detail()
        );
        assert!(
            !detail().contains("disk test"),
            "detail() spelled the disk test into an answer about another task: {}",
            detail()
        );
        mark_ready("a test", std::time::Instant::now());
        assert!(is_ready());
        assert!(detail().contains("serving"));
        // Idempotent.
        mark_ready("a test again", std::time::Instant::now());
        assert!(is_ready());
    }

    /// **A twin's listener is not the listener it was asked for, and that is
    /// what `serving_git` has to say.**
    ///
    /// The shapes are the live ones: the appliance asked for `0.0.0.0:2222` and
    /// took it; the twin asked for the same and `serve::start` gave it
    /// `127.0.0.1:<ephemeral>` instead. Before this, both answered the same
    /// `true`, because the only thing consulted was a boot latch that no twin
    /// ever closes.
    #[test]
    fn serving_git_is_the_listener_and_a_twin_says_no() {
        let asked: std::net::SocketAddr = "0.0.0.0:2222".parse().expect("an address");
        let primary = GitListener {
            asked,
            bound: asked,
        };
        assert!(
            primary.serves_git(),
            "the door that was asked for is the door that was taken"
        );
        // The boot window is the OTHER half, and it is a process-wide latch
        // that the test beside this one opens and closes: a primary serves git
        // exactly when that window is shut, which is what this says without
        // racing it.
        assert_eq!(
            is_serving_git(Some(primary)),
            is_ready(),
            "with a listener that serves, the boot window is the only thing left to say no"
        );

        // What `serve.rs` does for `stores.is_twin() || config.twin_role`.
        let twin = GitListener {
            asked,
            bound: "127.0.0.1:41337".parse().expect("an address"),
        };
        assert!(
            !twin.serves_git(),
            "a listener on loopback is not the 0.0.0.0:2222 an operator was told about"
        );
        assert!(
            !is_serving_git(Some(twin)),
            "THE DEFECT: the twin answered `git serving` while :2222 answered ECONNREFUSED"
        );
        let why = twin.detail();
        assert!(why.contains("NOT serving git"), "{why}");
        assert!(
            why.contains("TWIN"),
            "an operator must be told WHY, not just no: {why}"
        );
        assert!(
            why.contains("0.0.0.0:2222") && why.contains("127.0.0.1:41337"),
            "{why}"
        );

        // An ephemeral port is what a test asks for, and taking one is not a
        // failure to serve.
        let ephemeral = GitListener {
            asked: "127.0.0.1:0".parse().expect("an address"),
            bound: "127.0.0.1:52001".parse().expect("an address"),
        };
        assert!(
            ephemeral.serves_git(),
            "port 0 means any port, and any port was taken"
        );

        // Nothing handed this plane a listener: the old answer, unchanged —
        // the boot window and nothing else.
        assert_eq!(is_serving_git(None), is_ready(), "silence is not a refusal");
        assert!(primary.detail().contains("serving git on 0.0.0.0:2222"));
    }
}