gunnar_server/

live_update.rs

//! **Live update, gunnar's side** (owner 2026-10-12, TURNAROUND §8) — generic primitives only;
//! the decisions are korp-installer's (`korp_installer::live`), the wire is nordisk-pairing's
//! (`nordisk_pairing::live`, the installer's log door, its own certificate only):
//!
//! * [`running`] — what this box runs: BASE, app pin, slot, the off switch.
//! * [`stage`] — a new signed app verified and written to the slot NOT running; then this
//!   process stops GRACEFULLY (the ordinary stop) and, instead of powering off, re-execs
//!   korp-start ([`after_stop`]): PID 1 stays PID 1, `/data` stays mounted, korp-start boots the
//!   staged slot through its own checks.
//! * [`spawn_confirm`] — the new app confirms once it serves ([`confirm_plan`]); a slot still
//!   unconfirmed at the deadline re-execs korp-start, which rolls it back.

use std::time::Duration;

/// How long the new app serves before it confirms its slot.
pub const CONFIRM_AFTER: Duration = Duration::from_secs(60);
/// The health deadline: a trial slot not serving by then is rolled back.
pub const CONFIRM_DEADLINE: Duration = Duration::from_secs(600);
/// How long the graceful stop of a live update has before the staging thread re-execs itself.
pub const STOP_GRACE: Duration = Duration::from_secs(30);

/// What the trial watcher does next ([`confirm_plan`]).
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ConfirmStep {
    /// Nothing on trial: nothing to do.
    Nothing,
    /// Not serving long enough yet: look again.
    Wait,
    /// Serving for [`CONFIRM_AFTER`]: confirm the slot.
    Confirm,
    /// Not serving by [`CONFIRM_DEADLINE`]: re-exec korp-start (it rolls back).
    RollBack,
}

/// Pure: **the trial watcher's step** — `on_trial` (the setting says `booted`), how long this
/// process has run, and since when it serves (`None`: not yet).
pub fn confirm_plan(on_trial: bool, up: Duration, serving_since: Option<Duration>) -> ConfirmStep {
    if !on_trial {
        return ConfirmStep::Nothing;
    }
    match serving_since {
        Some(since) if up.saturating_sub(since) >= CONFIRM_AFTER => ConfirmStep::Confirm,
        Some(_) => ConfirmStep::Wait,
        None if up > CONFIRM_DEADLINE => ConfirmStep::RollBack,
        None => ConfirmStep::Wait,
    }
}

/// What a clean stop ends in.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum AfterStop {
    /// The ordinary end on the appliance.
    PowerOff,
    /// A live update is staged: re-exec korp-start.
    ReExec,
}

/// Pure: **after the graceful stop** — a staged live update re-execs, everything else powers
/// off as before.
pub fn after_stop(staged: bool) -> AfterStop {
    if staged {
        AfterStop::ReExec
    } else {
        AfterStop::PowerOff
    }
}

/// A live update staged in this process: the stop ends in a re-exec ([`after_stop`]).
static STAGED: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);

/// Whether a live update waits for the stop.
pub fn staged() -> bool {
    STAGED.load(std::sync::atomic::Ordering::SeqCst)
}

#[cfg(all(feature = "live-app", target_os = "linux"))]
mod appliance {
    use super::*;
    use korp_installer::live::{self, Esp};

    fn policy() -> live::Policy {
        let cl = korp_installer::cmdline::effective();
        let signed = korp_installer::cmdline::decide(&cl, std::env::var(korp_installer::cmdline::REQUIRE_SIGNED_ENV).ok().as_deref()).require_signed;
        live::Policy { live_update: cl.live_update_on(), require_signed: signed }
    }

    /// **What this box runs** (`GET /live/running`).
    pub fn running() -> Result<nordisk_pairing::live::Running, String> {
        let esp = Esp::boot();
        let s = esp.read_setting()?;
        let pin = std::fs::read_to_string(esp.records(s.active).pin).map_err(|e| format!("[live-running] the running slot's pin: {e}"))?;
        Ok(nordisk_pairing::live::Running { base: live::running_base(&esp)?, app: pin.trim().to_string(), slot: live::render_setting(&s).trim().to_string(), live_update: policy().live_update })
    }

    /// **A new app** (`POST /live/stage`): verified against this box's enrolled root, floor and
    /// BASE, written to the slot not running, the setting last; then the GRACEFUL stop is
    /// asked (SIGTERM to PID 1 itself, a second from now, after this answer left) and the
    /// stop ends in the re-exec.
    pub fn stage(st: &nordisk_pairing::live::Stage) -> Result<String, String> {
        let esp = Esp::boot();
        let pubkey = std::fs::read(esp.root.join("korp/root.pub")).ok().and_then(|b| <[u8; 32]>::try_from(b.as_slice()).ok());
        let floor = korp_installer::rollback::FloorStore::floor(&mut korp_installer::rollback::FloorFile::at(esp.root.join("korp/rollback.floor"))).map_err(|e| format!("[live-floor] {e:#}"))?;
        let base = live::running_base(&esp)?;
        let p = live::LivePayload { archive: &st.archive, pin_hex: &st.pin_hex, signature: Some(&st.signature), declared_base: Some(&st.base) };
        let slot = live::stage_on(&esp, &p, pubkey.as_ref(), floor, &base, policy(), &mut live::write_slot_partition)?;
        STAGED.store(true, std::sync::atomic::Ordering::SeqCst);
        tracing::warn!(target: "gunnar::live", event = "live.staged", slot = ?slot, app = %st.pin_hex, "live update: a new app is staged in slot {slot:?} — this box stops gracefully and re-execs korp-start into it");
        // A28 M18, stays a thread: it IS the stop — a signal, the grace, the re-exec; execve ends it.
        std::thread::spawn(|| {
            std::thread::sleep(Duration::from_secs(1));
            let _ = nix::sys::signal::kill(nix::unistd::Pid::this(), nix::sys::signal::Signal::SIGTERM);
            // The graceful stop has STOP_GRACE to reach `main`'s re-exec. A process that is
            // still here after it (MEASURED on the dwarf mock: the listeners shut down and the
            // runtime never let go) is re-exec'd from this thread: the listeners are down, the
            // writers were given their grace, the disks are synced — execve replaces every thread.
            std::thread::sleep(STOP_GRACE);
            tracing::warn!(target: "gunnar::live", event = "live.reexec_after_grace", "live update: the graceful stop did not reach its end within {} s — synced, re-exec from here", STOP_GRACE.as_secs());
            gunnar_common::logqueue::flush(Duration::from_secs(1));
            let e = reexec_now();
            tracing::error!(target: "gunnar::live", "live update: the re-exec FAILED: {e}");
        });
        Ok(format!("staged into slot {slot:?} ({} bytes, app {}); stopping gracefully, then re-exec", st.archive.len(), st.pin_hex))
    }

    /// **The re-exec** after the graceful stop: synced, then korp-start (`/init`) again. Only
    /// returns when the execve failed.
    pub fn reexec_now() -> String {
        nix::unistd::sync();
        let argv = [std::ffi::CString::new("/init").expect("no NUL")];
        let envp: Vec<std::ffi::CString> = std::env::vars().filter_map(|(k, v)| std::ffi::CString::new(format!("{k}={v}")).ok()).collect();
        match live::reexec(std::path::Path::new("/init"), &argv, &envp) {
            Ok(never) => match never {},
            Err(e) => e,
        }
    }

    /// **The trial watcher**: on a slot started on trial, confirm once the app has served for
    /// [`CONFIRM_AFTER`]; not serving by [`CONFIRM_DEADLINE`], re-exec (the rollback).
    pub fn spawn_confirm(serving: fn() -> bool) {
        let esp = Esp::boot();
        let on_trial = esp.read_setting().map(|s| s.trial == live::Trial::Booted).unwrap_or(false);
        if !on_trial {
            return;
        }
        tracing::warn!(target: "gunnar::live", event = "live.trial", "live update: this app runs on TRIAL — it confirms after serving {} s, or is rolled back at {} s", CONFIRM_AFTER.as_secs(), CONFIRM_DEADLINE.as_secs());
        // A jera job (A28 M18, A32-14): listed, a look that panics is said and a FAILED row and the
        // next one runs; a box that cannot start one says so (the raw `thread::spawn` panicked PID 1). Its first round only
        // waits, as the thread's first sleep did.
        let start = std::time::Instant::now();
        let mut since: Option<Duration> = None;
        let mut first = true;
        let say: gunnar_common::jobs::Say = std::sync::Arc::new(|l: &str| tracing::error!(target: "gunnar::live", "{l}"));
        let started = gunnar_common::jobs::start(super::TRIAL_DUTY, Duration::from_secs(5), say, move || {
            if std::mem::take(&mut first) {
                return gunnar_common::jobs::Round::Again;
            }
            if since.is_none() && serving() {
                since = Some(start.elapsed());
            }
            match confirm_plan(true, start.elapsed(), since) {
                ConfirmStep::Wait | ConfirmStep::Nothing => gunnar_common::jobs::Round::Again,
                ConfirmStep::Confirm => {
                    match live::confirm_on(&esp) {
                        Ok(s) => tracing::info!(target: "gunnar::live", event = "live.confirmed", "live update: confirmed — slot {:?} is this box's app now", s.active),
                        Err(e) => tracing::error!(target: "gunnar::live", "live update: the confirm was NOT written ({e}) — the next start rolls this slot back"),
                    }
                    gunnar_common::jobs::Round::Done
                }
                ConfirmStep::RollBack => {
                    tracing::error!(target: "gunnar::live", event = "live.rollback", "live update: the new app did not serve within {} s — re-exec, korp-start rolls back", CONFIRM_DEADLINE.as_secs());
                    let e = reexec_now();
                    tracing::error!(target: "gunnar::live", "live update: the rollback re-exec failed: {e}");
                    gunnar_common::jobs::Round::Done
                }
            }
        });
        if let Err(e) = started {
            tracing::error!(target: "gunnar::live", "ALERT {} no trial watch ({e}): this trial is never confirmed, and the next start rolls it back", super::TRIAL_DUTY);
        }
    }
}

#[cfg(all(feature = "live-app", target_os = "linux"))]
pub use appliance::{reexec_now, running, spawn_confirm, stage};

/// The jera job that watches a live update's trial (A28 M18, A32-14).
pub const TRIAL_DUTY: &str = "live-trial";

#[cfg(test)]
mod tests {
    use super::*;

    /// ★ **The trial's watch is a duty** (A28 M18): the raw `thread::spawn` panicked PID 1 when
    /// no thread could start, and a watch that panicked left the trial never confirmed.
    #[test]
    fn the_trial_watch_is_a_duty() {
        let src = include_str!("live_update.rs");
        let code = &src[..src.find("#[cfg(test)]").unwrap()];
        assert!(code.contains("gunnar_common::jobs::start(super::TRIAL_DUTY"), "the watch starts as a jera job (A32-14)");
    }

    /// ★ The new app confirms only after it SERVED for a while; one that never serves is
    /// rolled back at the deadline; nothing on trial does nothing.
    #[test]
    fn the_new_app_confirms_after_serving_and_is_rolled_back_at_the_deadline() {
        let s = Duration::from_secs;
        assert_eq!(confirm_plan(false, s(9999), None), ConfirmStep::Nothing);
        assert_eq!(confirm_plan(true, s(10), None), ConfirmStep::Wait);
        assert_eq!(confirm_plan(true, s(30), Some(s(20))), ConfirmStep::Wait, "serving, not long enough");
        assert_eq!(confirm_plan(true, s(90), Some(s(20))), ConfirmStep::Confirm);
        assert_eq!(confirm_plan(true, CONFIRM_DEADLINE + s(1), None), ConfirmStep::RollBack, "never served: rolled back");
        assert_eq!(confirm_plan(true, CONFIRM_DEADLINE + s(1), Some(CONFIRM_DEADLINE)), ConfirmStep::Wait, "serving at the deadline: it gets its minute");
    }

    /// ★ A staged live update ends the graceful stop in a re-exec, never a power-off.
    #[test]
    fn a_staged_update_reexecs_after_the_graceful_stop() {
        assert_eq!(after_stop(true), AfterStop::ReExec);
        assert_eq!(after_stop(false), AfterStop::PowerOff);
    }
}