gunnar_server/
live_update.rs

1//! **Live update, gunnar's side** (owner 2026-10-12, TURNAROUND §8) — generic primitives only;
2//! the decisions are korp-installer's (`korp_installer::live`), the wire is nordisk-pairing's
3//! (`nordisk_pairing::live`, the installer's log door, its own certificate only):
4//!
5//! * [`running`] — what this box runs: BASE, app pin, slot, the off switch.
6//! * [`stage`] — a new signed app verified and written to the slot NOT running; then this
7//!   process stops GRACEFULLY (the ordinary stop) and, instead of powering off, re-execs
8//!   korp-start ([`after_stop`]): PID 1 stays PID 1, `/data` stays mounted, korp-start boots the
9//!   staged slot through its own checks.
10//! * [`spawn_confirm`] — the new app confirms once it serves ([`confirm_plan`]); a slot still
11//!   unconfirmed at the deadline re-execs korp-start, which rolls it back.
12
13use std::time::Duration;
14
15/// How long the new app serves before it confirms its slot.
16pub const CONFIRM_AFTER: Duration = Duration::from_secs(60);
17/// The health deadline: a trial slot not serving by then is rolled back.
18pub const CONFIRM_DEADLINE: Duration = Duration::from_secs(600);
19/// How long the graceful stop of a live update has before the staging thread re-execs itself.
20pub const STOP_GRACE: Duration = Duration::from_secs(30);
21
22/// What the trial watcher does next ([`confirm_plan`]).
23#[derive(Debug, Clone, Copy, PartialEq, Eq)]
24pub enum ConfirmStep {
25    /// Nothing on trial: nothing to do.
26    Nothing,
27    /// Not serving long enough yet: look again.
28    Wait,
29    /// Serving for [`CONFIRM_AFTER`]: confirm the slot.
30    Confirm,
31    /// Not serving by [`CONFIRM_DEADLINE`]: re-exec korp-start (it rolls back).
32    RollBack,
33}
34
35/// Pure: **the trial watcher's step** — `on_trial` (the setting says `booted`), how long this
36/// process has run, and since when it serves (`None`: not yet).
37pub fn confirm_plan(on_trial: bool, up: Duration, serving_since: Option<Duration>) -> ConfirmStep {
38    if !on_trial {
39        return ConfirmStep::Nothing;
40    }
41    match serving_since {
42        Some(since) if up.saturating_sub(since) >= CONFIRM_AFTER => ConfirmStep::Confirm,
43        Some(_) => ConfirmStep::Wait,
44        None if up > CONFIRM_DEADLINE => ConfirmStep::RollBack,
45        None => ConfirmStep::Wait,
46    }
47}
48
49/// What a clean stop ends in.
50#[derive(Debug, Clone, Copy, PartialEq, Eq)]
51pub enum AfterStop {
52    /// The ordinary end on the appliance.
53    PowerOff,
54    /// A live update is staged: re-exec korp-start.
55    ReExec,
56}
57
58/// Pure: **after the graceful stop** — a staged live update re-execs, everything else powers
59/// off as before.
60pub fn after_stop(staged: bool) -> AfterStop {
61    if staged {
62        AfterStop::ReExec
63    } else {
64        AfterStop::PowerOff
65    }
66}
67
68/// A live update staged in this process: the stop ends in a re-exec ([`after_stop`]).
69static STAGED: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);
70
71/// Whether a live update waits for the stop.
72pub fn staged() -> bool {
73    STAGED.load(std::sync::atomic::Ordering::SeqCst)
74}
75
76#[cfg(all(feature = "live-app", target_os = "linux"))]
77mod appliance {
78    use super::*;
79    use korp_installer::live::{self, Esp};
80
81    fn policy() -> live::Policy {
82        let cl = korp_installer::cmdline::effective();
83        let signed = korp_installer::cmdline::decide(
84            &cl,
85            std::env::var(korp_installer::cmdline::REQUIRE_SIGNED_ENV)
86                .ok()
87                .as_deref(),
88        )
89        .require_signed;
90        live::Policy {
91            live_update: cl.live_update_on(),
92            require_signed: signed,
93        }
94    }
95
96    /// **What this box runs** (`GET /live/running`).
97    pub fn running() -> Result<nordisk_pairing::live::Running, String> {
98        let esp = Esp::boot();
99        let s = esp.read_setting()?;
100        let pin = std::fs::read_to_string(esp.records(s.active).pin)
101            .map_err(|e| format!("[live-running] the running slot's pin: {e}"))?;
102        Ok(nordisk_pairing::live::Running {
103            base: live::running_base(&esp)?,
104            app: pin.trim().to_string(),
105            slot: live::render_setting(&s).trim().to_string(),
106            live_update: policy().live_update,
107        })
108    }
109
110    /// **A new app** (`POST /live/stage`): verified against this box's enrolled root, floor and
111    /// BASE, written to the slot not running, the setting last; then the GRACEFUL stop is
112    /// asked (SIGTERM to PID 1 itself, a second from now, after this answer left) and the
113    /// stop ends in the re-exec.
114    pub fn stage(st: &nordisk_pairing::live::Stage) -> Result<String, String> {
115        let esp = Esp::boot();
116        let pubkey = std::fs::read(esp.root.join("korp/root.pub"))
117            .ok()
118            .and_then(|b| <[u8; 32]>::try_from(b.as_slice()).ok());
119        let floor = korp_installer::rollback::FloorStore::floor(
120            &mut korp_installer::rollback::FloorFile::at(esp.root.join("korp/rollback.floor")),
121        )
122        .map_err(|e| format!("[live-floor] {e:#}"))?;
123        let base = live::running_base(&esp)?;
124        let p = live::LivePayload {
125            archive: &st.archive,
126            pin_hex: &st.pin_hex,
127            signature: Some(&st.signature),
128            declared_base: Some(&st.base),
129        };
130        let slot = live::stage_on(
131            &esp,
132            &p,
133            pubkey.as_ref(),
134            floor,
135            &base,
136            policy(),
137            &mut live::write_slot_partition,
138        )?;
139        STAGED.store(true, std::sync::atomic::Ordering::SeqCst);
140        tracing::warn!(target: "gunnar::live", event = "live.staged", slot = ?slot, app = %st.pin_hex, "live update: a new app is staged in slot {slot:?} — this box stops gracefully and re-execs korp-start into it");
141        // A28 M18, stays a thread: it IS the stop — a signal, the grace, the re-exec; execve ends it.
142        std::thread::spawn(|| {
143            std::thread::sleep(Duration::from_secs(1));
144            let _ =
145                nix::sys::signal::kill(nix::unistd::Pid::this(), nix::sys::signal::Signal::SIGTERM);
146            // The graceful stop has STOP_GRACE to reach `main`'s re-exec. A process that is
147            // still here after it (MEASURED on the dwarf mock: the listeners shut down and the
148            // runtime never let go) is re-exec'd from this thread: the listeners are down, the
149            // writers were given their grace, the disks are synced — execve replaces every thread.
150            std::thread::sleep(STOP_GRACE);
151            tracing::warn!(target: "gunnar::live", event = "live.reexec_after_grace", "live update: the graceful stop did not reach its end within {} s — synced, re-exec from here", STOP_GRACE.as_secs());
152            gunnar_common::logqueue::flush(Duration::from_secs(1));
153            let e = reexec_now();
154            tracing::error!(target: "gunnar::live", "live update: the re-exec FAILED: {e}");
155        });
156        Ok(format!(
157            "staged into slot {slot:?} ({} bytes, app {}); stopping gracefully, then re-exec",
158            st.archive.len(),
159            st.pin_hex
160        ))
161    }
162
163    /// **The re-exec** after the graceful stop: synced, then korp-start (`/init`) again. Only
164    /// returns when the execve failed.
165    pub fn reexec_now() -> String {
166        nix::unistd::sync();
167        let argv = [std::ffi::CString::new("/init").expect("no NUL")];
168        let envp: Vec<std::ffi::CString> = std::env::vars()
169            .filter_map(|(k, v)| std::ffi::CString::new(format!("{k}={v}")).ok())
170            .collect();
171        match live::reexec(std::path::Path::new("/init"), &argv, &envp) {
172            Ok(never) => match never {},
173            Err(e) => e,
174        }
175    }
176
177    /// **The trial watcher**: on a slot started on trial, confirm once the app has served for
178    /// [`CONFIRM_AFTER`]; not serving by [`CONFIRM_DEADLINE`], re-exec (the rollback).
179    pub fn spawn_confirm(serving: fn() -> bool) {
180        let esp = Esp::boot();
181        let on_trial = esp
182            .read_setting()
183            .map(|s| s.trial == live::Trial::Booted)
184            .unwrap_or(false);
185        if !on_trial {
186            return;
187        }
188        tracing::warn!(target: "gunnar::live", event = "live.trial", "live update: this app runs on TRIAL — it confirms after serving {} s, or is rolled back at {} s", CONFIRM_AFTER.as_secs(), CONFIRM_DEADLINE.as_secs());
189        // A jera job (A28 M18, A32-14): listed, a look that panics is said and a FAILED row and the
190        // next one runs; a box that cannot start one says so (the raw `thread::spawn` panicked PID 1). Its first round only
191        // waits, as the thread's first sleep did.
192        let start = std::time::Instant::now();
193        let mut since: Option<Duration> = None;
194        let mut first = true;
195        let say: gunnar_common::jobs::Say =
196            std::sync::Arc::new(|l: &str| tracing::error!(target: "gunnar::live", "{l}"));
197        let started = gunnar_common::jobs::start(
198            super::TRIAL_DUTY,
199            Duration::from_secs(5),
200            say,
201            move || {
202                if std::mem::take(&mut first) {
203                    return gunnar_common::jobs::Round::Again;
204                }
205                if since.is_none() && serving() {
206                    since = Some(start.elapsed());
207                }
208                match confirm_plan(true, start.elapsed(), since) {
209                    ConfirmStep::Wait | ConfirmStep::Nothing => gunnar_common::jobs::Round::Again,
210                    ConfirmStep::Confirm => {
211                        match live::confirm_on(&esp) {
212                            Ok(s) => {
213                                tracing::info!(target: "gunnar::live", event = "live.confirmed", "live update: confirmed — slot {:?} is this box's app now", s.active)
214                            }
215                            Err(e) => {
216                                tracing::error!(target: "gunnar::live", "live update: the confirm was NOT written ({e}) — the next start rolls this slot back")
217                            }
218                        }
219                        gunnar_common::jobs::Round::Done
220                    }
221                    ConfirmStep::RollBack => {
222                        tracing::error!(target: "gunnar::live", event = "live.rollback", "live update: the new app did not serve within {} s — re-exec, korp-start rolls back", CONFIRM_DEADLINE.as_secs());
223                        let e = reexec_now();
224                        tracing::error!(target: "gunnar::live", "live update: the rollback re-exec failed: {e}");
225                        gunnar_common::jobs::Round::Done
226                    }
227                }
228            },
229        );
230        if let Err(e) = started {
231            tracing::error!(target: "gunnar::live", "ALERT {} no trial watch ({e}): this trial is never confirmed, and the next start rolls it back", super::TRIAL_DUTY);
232        }
233    }
234}
235
236#[cfg(all(feature = "live-app", target_os = "linux"))]
237pub use appliance::{reexec_now, running, spawn_confirm, stage};
238
239/// The jera job that watches a live update's trial (A28 M18, A32-14).
240pub const TRIAL_DUTY: &str = "live-trial";
241
242#[cfg(test)]
243mod tests {
244    use super::*;
245
246    /// ★ **The trial's watch is a duty** (A28 M18): the raw `thread::spawn` panicked PID 1 when
247    /// no thread could start, and a watch that panicked left the trial never confirmed.
248    #[test]
249    fn the_trial_watch_is_a_duty() {
250        let src = include_str!("live_update.rs");
251        let code = &src[..src.find("#[cfg(test)]").unwrap()];
252        assert!(
253            code.split_whitespace()
254                .collect::<String>()
255                .contains("gunnar_common::jobs::start(super::TRIAL_DUTY"),
256            "the watch starts as a jera job (A32-14)"
257        );
258    }
259
260    /// ★ The new app confirms only after it SERVED for a while; one that never serves is
261    /// rolled back at the deadline; nothing on trial does nothing.
262    #[test]
263    fn the_new_app_confirms_after_serving_and_is_rolled_back_at_the_deadline() {
264        let s = Duration::from_secs;
265        assert_eq!(confirm_plan(false, s(9999), None), ConfirmStep::Nothing);
266        assert_eq!(confirm_plan(true, s(10), None), ConfirmStep::Wait);
267        assert_eq!(
268            confirm_plan(true, s(30), Some(s(20))),
269            ConfirmStep::Wait,
270            "serving, not long enough"
271        );
272        assert_eq!(confirm_plan(true, s(90), Some(s(20))), ConfirmStep::Confirm);
273        assert_eq!(
274            confirm_plan(true, CONFIRM_DEADLINE + s(1), None),
275            ConfirmStep::RollBack,
276            "never served: rolled back"
277        );
278        assert_eq!(
279            confirm_plan(true, CONFIRM_DEADLINE + s(1), Some(CONFIRM_DEADLINE)),
280            ConfirmStep::Wait,
281            "serving at the deadline: it gets its minute"
282        );
283    }
284
285    /// ★ A staged live update ends the graceful stop in a re-exec, never a power-off.
286    #[test]
287    fn a_staged_update_reexecs_after_the_graceful_stop() {
288        assert_eq!(after_stop(true), AfterStop::ReExec);
289        assert_eq!(after_stop(false), AfterStop::PowerOff);
290    }
291}
gunnar_server/live_update.rs

gunnar_server/
live_update.rs
