polyc_runtime/

stager.rs

//! Update stager — the apply substrate the hot and warm update paths build on.
//!
//! The stager runs one job: land a staged update safely. It downloads the
//! update to a *side* location that never touches the live path, verifies its
//! signature *before* anything is applied, flips the live pointer to the new
//! version, and — if a post-apply health check fails — flips the pointer back
//! to the previous version. The previous version stays staged the whole time,
//! so a rollback is a pointer flip rather than a re-download.
//!
//! # The seams
//!
//! The stager owns the *state machine*, not the I/O. Everything that touches
//! the outside world is an injectable seam, so the machine can be exercised
//! without a real download, a real key, or a real process boot:
//!
//! - [`UpdateSource`] downloads the release to a side location.
//! - [`SignatureVerifier`] checks the ed25519 signature over the staged bytes.
//!   In production this wraps the project's ed25519 key custody (the same
//!   `verify` the event log and approval flow use); the stager only ever sees a
//!   yes/no answer, so a foundation crate needs no signing dependency to run it.
//! - [`Activator`] materializes the pointer flip — a symlink swap, an image
//!   tag, or a process swap. It is called once to apply and, in reverse, once
//!   to roll back.
//! - [`HealthCheck`] probes the freshly-applied version. [`Health::Healthy`]
//!   commits; anything else triggers the auto-rollback.
//!
//! Each seam has a blanket implementation for the matching closure, so a caller
//! can pass a closure where a full type would be overkill.
//!
//! # The state machine
//!
//! [`Stager::stage_and_apply`] walks a fixed sequence:
//!
//! ```text
//!   download ──▶ verify ──▶ apply (pointer flip) ──▶ health check
//!                  │                                     │
//!                  │ (unverified)                        ├─ healthy  ──▶ Committed
//!                  ▼                                     │
//!               refused                                  └─ unhealthy ──▶ auto-rollback ──▶ RolledBack
//!               (nothing applied)                            (pointer flips back)
//! ```
//!
//! The invariant the tests pin: an unverified bundle is never applied (the live
//! pointer is untouched), the previous version stays staged, and a failed
//! health check leaves the system honestly back on the previous version.
//!
//! The staged artifact carries the [`StagedBundle`] it delivers, so a caller
//! that already classified the release (see
//! [`crate::compat`]) hands the same value straight through to apply.

use std::path::PathBuf;

use thiserror::Error;

use crate::compat::StagedBundle;

/// A release identity — the pointer the live slot resolves to and flips between.
///
/// Opaque and cheap to clone; in production it is a content address, a version
/// tag, or an image digest, and the [`Activator`] decides how a pointer flip to
/// it materializes.
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct ReleaseId(String);

impl ReleaseId {
    /// Wrap a release identity string.
    #[must_use]
    pub fn new(id: impl Into<String>) -> Self {
        Self(id.into())
    }

    /// The underlying identity string.
    #[must_use]
    pub const fn as_str(&self) -> &str {
        self.0.as_str()
    }
}

impl std::fmt::Display for ReleaseId {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        f.write_str(&self.0)
    }
}

/// An update downloaded to a side location, before verification.
///
/// Holds everything the stager needs to verify and apply the release without
/// re-reading the source: the release identity, the side path it landed at
/// (never the live path), the [`StagedBundle`] it delivers, and the detached
/// ed25519 signature over `signed_bytes` under `signer_public_key`.
#[derive(Debug, Clone)]
pub struct StagedArtifact {
    /// Which release this artifact is.
    pub release: ReleaseId,
    /// The side location the update was downloaded to — never the live path.
    pub staged_path: PathBuf,
    /// The config-as-data bundle this artifact delivers, already classified
    /// against the running runtime by [`crate::compat`].
    pub bundle: StagedBundle,
    /// The canonical bytes the signature commits to.
    pub signed_bytes: Vec<u8>,
    /// The detached ed25519 signature over [`Self::signed_bytes`].
    pub signature: Vec<u8>,
    /// The encoded public key the signature is expected to verify against.
    pub signer_public_key: Vec<u8>,
}

/// The result of a post-apply health check.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Health {
    /// The applied version is serving; commit it.
    Healthy,
    /// The applied version failed to come up; the enclosed reason is recorded
    /// and surfaced in the rollback outcome.
    Unhealthy(String),
}

/// What happened to a `stage_and_apply` call that got as far as a health check.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Outcome {
    /// The new version passed its health check and is now the live version.
    Committed {
        /// The version now serving.
        version: ReleaseId,
    },
    /// The new version failed its health check; the pointer flipped back and the
    /// previous version is live again.
    RolledBack {
        /// The version still serving after the rollback — the one that was live
        /// before the attempt.
        stayed_on: ReleaseId,
        /// The health-check failure that triggered the rollback.
        reason: String,
    },
}

impl Outcome {
    /// A one-line, user-facing summary of the outcome.
    ///
    /// Honest about a rollback — it names the version still serving and why the
    /// update did not stick — so the surface presenting it never has to word the
    /// state twice.
    #[must_use]
    pub fn summary(&self) -> String {
        match self {
            Self::Committed { version } => format!("Now running {version}."),
            Self::RolledBack { stayed_on, reason } => {
                format!(
                    "Stayed on {stayed_on} — the new version failed its health check ({reason})."
                )
            }
        }
    }
}

/// Why a `stage_and_apply` call could not reach a health check.
#[derive(Debug, Error)]
pub enum StageError {
    /// The source could not download the release to its side location.
    #[error("staging the update failed: {0}")]
    Download(String),
    /// The staged bundle's signature did not verify against a trusted key, so it
    /// was refused before anything was applied.
    #[error("the update's signature did not verify against a trusted key")]
    Unverified,
    /// The pointer flip itself failed — either applying the new version or
    /// flipping back to the previous one.
    #[error("activating the staged update failed: {0}")]
    Activate(String),
}

/// Downloads a release to a side location, never the live path.
pub trait UpdateSource {
    /// Fetch `release` into a side location and return the staged artifact.
    ///
    /// # Errors
    ///
    /// Returns [`StageError::Download`] when the release cannot be fetched or
    /// written to its side location.
    fn fetch(&self, release: &ReleaseId) -> Result<StagedArtifact, StageError>;
}

impl<F> UpdateSource for F
where
    F: Fn(&ReleaseId) -> Result<StagedArtifact, StageError>,
{
    fn fetch(&self, release: &ReleaseId) -> Result<StagedArtifact, StageError> {
        self(release)
    }
}

/// Verifies the ed25519 signature over a staged artifact.
///
/// The contract is a plain yes/no: `true` iff the signature over
/// [`StagedArtifact::signed_bytes`] verifies against a trusted key. A `false`
/// return means the bundle is refused and never applied. Keeping the answer a
/// boolean lets a foundation crate run the interlock without depending on the
/// signing crate — production wraps the project's ed25519 `verify` here.
pub trait SignatureVerifier {
    /// Whether `artifact`'s signature verifies against a trusted key.
    fn verify(&self, artifact: &StagedArtifact) -> bool;
}

impl<F> SignatureVerifier for F
where
    F: Fn(&StagedArtifact) -> bool,
{
    fn verify(&self, artifact: &StagedArtifact) -> bool {
        self(artifact)
    }
}

/// Materializes a pointer flip to a release — a symlink swap, an image tag, or a
/// process swap.
///
/// Called once to apply the new version and, in reverse, once to roll back to
/// the previous one. The previous version stays staged, so a rollback flip never
/// re-downloads.
pub trait Activator {
    /// Flip the live pointer to `release`.
    ///
    /// # Errors
    ///
    /// Returns [`StageError::Activate`] when the pointer cannot be flipped.
    fn activate(&self, release: &ReleaseId) -> Result<(), StageError>;
}

impl<F> Activator for F
where
    F: Fn(&ReleaseId) -> Result<(), StageError>,
{
    fn activate(&self, release: &ReleaseId) -> Result<(), StageError> {
        self(release)
    }
}

/// Probes a freshly-applied version to decide whether it commits or rolls back.
pub trait HealthCheck {
    /// Probe the applied version. [`Health::Healthy`] commits; anything else
    /// triggers the auto-rollback.
    fn check(&self) -> Health;
}

impl<F> HealthCheck for F
where
    F: Fn() -> Health,
{
    fn check(&self) -> Health {
        self()
    }
}

/// The apply substrate: stage → verify → apply → (commit | auto-rollback).
///
/// Holds the live version and the previous version behind it, plus the four
/// seams that touch the outside world. [`Stager::stage_and_apply`] drives the
/// whole sequence; [`Stager::live`] and [`Stager::previous`] expose the current
/// pointer state.
pub struct Stager<S, V, A, H> {
    source: S,
    verifier: V,
    activator: A,
    health: H,
    live: ReleaseId,
    previous: Option<ReleaseId>,
}

impl<S, V, A, H> Stager<S, V, A, H>
where
    S: UpdateSource,
    V: SignatureVerifier,
    A: Activator,
    H: HealthCheck,
{
    /// Build a stager over its four seams, starting from `initial_live` as the
    /// version already serving.
    pub const fn new(
        source: S,
        verifier: V,
        activator: A,
        health: H,
        initial_live: ReleaseId,
    ) -> Self {
        Self {
            source,
            verifier,
            activator,
            health,
            live: initial_live,
            previous: None,
        }
    }

    /// The version currently serving — the committed live pointer.
    #[must_use]
    pub const fn live(&self) -> &ReleaseId {
        &self.live
    }

    /// The previous version, kept staged so a rollback is a pointer flip. `None`
    /// until the first update commits.
    #[must_use]
    pub const fn previous(&self) -> Option<&ReleaseId> {
        self.previous.as_ref()
    }

    /// Stage `release`, verify it, apply it, and commit or auto-roll-back on the
    /// health check.
    ///
    /// The sequence is fixed:
    ///
    /// 1. download the release to a side location;
    /// 2. verify its signature — an unverified bundle returns
    ///    [`StageError::Unverified`] with the live pointer untouched;
    /// 3. flip the live pointer to the new version;
    /// 4. run the health check — on [`Health::Healthy`] commit and return
    ///    [`Outcome::Committed`]; on [`Health::Unhealthy`] flip the pointer back
    ///    to the previous version and return [`Outcome::RolledBack`].
    ///
    /// The committed live pointer ([`Stager::live`]) only advances in step 4 on
    /// a passed health check, so a refused or rolled-back attempt leaves the
    /// stager exactly where it started.
    ///
    /// # Errors
    ///
    /// Returns [`StageError::Download`] if the source cannot stage the release,
    /// [`StageError::Unverified`] if the signature does not verify, or
    /// [`StageError::Activate`] if a pointer flip fails (applying the new
    /// version, or — worse — flipping back during a rollback).
    pub fn stage_and_apply(&mut self, release: &ReleaseId) -> Result<Outcome, StageError> {
        // 1. Download to a side location — never the live path.
        let artifact = self.source.fetch(release)?;

        // 2. Verify BEFORE anything is applied. An unverified bundle is refused
        //    here, with the live pointer and staged previous both untouched.
        if !self.verifier.verify(&artifact) {
            return Err(StageError::Unverified);
        }

        // 3. Apply: flip the live pointer to the new version. The version that
        //    was live stays staged as the rollback target. `self.live` is not
        //    advanced yet — it only commits once the health check passes.
        let rollback_to = self.live.clone();
        self.activator.activate(&artifact.release)?;

        // 4. Health check decides commit vs. auto-rollback.
        match self.health.check() {
            Health::Healthy => {
                self.previous = Some(rollback_to);
                self.live = artifact.release.clone();
                Ok(Outcome::Committed {
                    version: artifact.release,
                })
            }
            Health::Unhealthy(reason) => {
                // Auto-rollback: flip the pointer back to the previous version.
                // The committed `self.live` never moved, so we are honestly back
                // where we started.
                self.activator.activate(&rollback_to)?;
                Ok(Outcome::RolledBack {
                    stayed_on: rollback_to,
                    reason,
                })
            }
        }
    }
}

#[cfg(test)]
mod tests {
    #![allow(clippy::pedantic, clippy::nursery, missing_docs)]

    use std::cell::RefCell;
    use std::rc::Rc;

    use super::*;
    use crate::compat::RuntimeTarget;

    /// A recording activator whose flip log is shared with the test after it is
    /// moved into the stager, so the sequence of pointer flips is observable.
    #[derive(Clone)]
    struct RecordingActivator {
        flips: Rc<RefCell<Vec<ReleaseId>>>,
        fail_on: Option<ReleaseId>,
    }

    impl RecordingActivator {
        fn new() -> Self {
            Self {
                flips: Rc::new(RefCell::new(Vec::new())),
                fail_on: None,
            }
        }

        fn failing_on(release: ReleaseId) -> Self {
            Self {
                flips: Rc::new(RefCell::new(Vec::new())),
                fail_on: Some(release),
            }
        }

        fn flips(&self) -> Vec<ReleaseId> {
            self.flips.borrow().clone()
        }
    }

    impl Activator for RecordingActivator {
        fn activate(&self, release: &ReleaseId) -> Result<(), StageError> {
            if self.fail_on.as_ref() == Some(release) {
                return Err(StageError::Activate(format!("cannot flip to {release}")));
            }
            self.flips.borrow_mut().push(release.clone());
            Ok(())
        }
    }

    fn artifact_for(release: &ReleaseId) -> StagedArtifact {
        StagedArtifact {
            release: release.clone(),
            // A side path — the download never touches the live path.
            staged_path: PathBuf::from(format!("/var/lib/polychrome/staged/{release}")),
            bundle: StagedBundle::new(
                RuntimeTarget::new(3, 7, "polychrome.dev/v1"),
                format!("catalog-{release}"),
            ),
            signed_bytes: format!("bytes-of-{release}").into_bytes(),
            signature: vec![0xAB; 4],
            signer_public_key: vec![0xCD; 4],
        }
    }

    /// A source that always stages the requested release from a side location.
    fn ok_source(release: &ReleaseId) -> Result<StagedArtifact, StageError> {
        Ok(artifact_for(release))
    }

    #[test]
    fn verified_healthy_update_commits_and_keeps_previous_staged() {
        let activator = RecordingActivator::new();
        let mut stager = Stager::new(
            ok_source,
            |_: &StagedArtifact| true,
            activator.clone(),
            || Health::Healthy,
            ReleaseId::new("v1"),
        );

        let outcome = stager.stage_and_apply(&ReleaseId::new("v2")).unwrap();

        assert_eq!(
            outcome,
            Outcome::Committed {
                version: ReleaseId::new("v2")
            }
        );
        // The live pointer advanced to the new version...
        assert_eq!(stager.live(), &ReleaseId::new("v2"));
        // ...and the version that was live stays staged as the rollback target.
        assert_eq!(stager.previous(), Some(&ReleaseId::new("v1")));
        // A healthy apply flips the pointer exactly once, forward.
        assert_eq!(activator.flips(), vec![ReleaseId::new("v2")]);
    }

    #[test]
    fn unverified_bundle_is_never_applied() {
        let activator = RecordingActivator::new();
        let mut stager = Stager::new(
            ok_source,
            // Signature does not verify.
            |_: &StagedArtifact| false,
            activator.clone(),
            // A health check that would panic proves it is never reached.
            || panic!("health check must not run for an unverified bundle"),
            ReleaseId::new("v1"),
        );

        let err = stager.stage_and_apply(&ReleaseId::new("v2")).unwrap_err();

        assert!(matches!(err, StageError::Unverified));
        // Nothing was applied: the pointer never flipped and the live version is
        // unchanged.
        assert!(activator.flips().is_empty());
        assert_eq!(stager.live(), &ReleaseId::new("v1"));
        assert_eq!(stager.previous(), None);
    }

    #[test]
    fn failed_health_check_auto_rolls_back_to_previous() {
        let activator = RecordingActivator::new();
        let mut stager = Stager::new(
            ok_source,
            |_: &StagedArtifact| true,
            activator.clone(),
            || Health::Unhealthy("readiness probe timed out".to_owned()),
            ReleaseId::new("v1"),
        );

        let outcome = stager.stage_and_apply(&ReleaseId::new("v2")).unwrap();

        assert_eq!(
            outcome,
            Outcome::RolledBack {
                stayed_on: ReleaseId::new("v1"),
                reason: "readiness probe timed out".to_owned(),
            }
        );
        // The committed live pointer never moved off the previous version.
        assert_eq!(stager.live(), &ReleaseId::new("v1"));
        assert_eq!(stager.previous(), None);
        // The pointer flipped to the new version, then back — a rollback is a
        // pointer flip, not a re-download.
        assert_eq!(
            activator.flips(),
            vec![ReleaseId::new("v2"), ReleaseId::new("v1")]
        );
    }

    #[test]
    fn rollback_outcome_surfaces_an_honest_summary() {
        let outcome = Outcome::RolledBack {
            stayed_on: ReleaseId::new("v1"),
            reason: "readiness probe timed out".to_owned(),
        };
        let summary = outcome.summary();

        assert_eq!(
            summary,
            "Stayed on v1 — the new version failed its health check (readiness probe timed out).",
        );
        // Copy rules: no apology or filler words.
        for banned in ["sorry", "please", "unfortunately"] {
            assert!(
                !summary.to_lowercase().contains(banned),
                "rollback summary must not contain {banned:?}",
            );
        }
    }

    #[test]
    fn committed_outcome_summary_names_the_new_version() {
        let summary = Outcome::Committed {
            version: ReleaseId::new("v2"),
        }
        .summary();
        assert_eq!(summary, "Now running v2.");
    }

    #[test]
    fn download_failure_applies_nothing() {
        let activator = RecordingActivator::new();
        let mut stager = Stager::new(
            |_: &ReleaseId| Err(StageError::Download("side location is full".to_owned())),
            |_: &StagedArtifact| panic!("verify must not run when the download fails"),
            activator.clone(),
            || panic!("health check must not run when the download fails"),
            ReleaseId::new("v1"),
        );

        let err = stager.stage_and_apply(&ReleaseId::new("v2")).unwrap_err();

        assert!(matches!(err, StageError::Download(_)));
        assert!(activator.flips().is_empty());
        assert_eq!(stager.live(), &ReleaseId::new("v1"));
    }

    #[test]
    fn apply_flip_failure_leaves_live_untouched() {
        // The activator refuses to flip to the new version; the live pointer
        // must stay on the previous one and nothing commits.
        let activator = RecordingActivator::failing_on(ReleaseId::new("v2"));
        let mut stager = Stager::new(
            ok_source,
            |_: &StagedArtifact| true,
            activator.clone(),
            || Health::Healthy,
            ReleaseId::new("v1"),
        );

        let err = stager.stage_and_apply(&ReleaseId::new("v2")).unwrap_err();

        assert!(matches!(err, StageError::Activate(_)));
        assert!(activator.flips().is_empty());
        assert_eq!(stager.live(), &ReleaseId::new("v1"));
        assert_eq!(stager.previous(), None);
    }

    #[test]
    fn a_second_update_restages_the_prior_live_as_previous() {
        let activator = RecordingActivator::new();
        let mut stager = Stager::new(
            ok_source,
            |_: &StagedArtifact| true,
            activator.clone(),
            || Health::Healthy,
            ReleaseId::new("v1"),
        );

        stager.stage_and_apply(&ReleaseId::new("v2")).unwrap();
        stager.stage_and_apply(&ReleaseId::new("v3")).unwrap();

        assert_eq!(stager.live(), &ReleaseId::new("v3"));
        // The rollback target tracks the most recent committed version.
        assert_eq!(stager.previous(), Some(&ReleaseId::new("v2")));
        assert_eq!(
            activator.flips(),
            vec![ReleaseId::new("v2"), ReleaseId::new("v3")]
        );
    }

    #[test]
    fn staged_artifact_carries_its_classified_bundle() {
        // The download seam lands the artifact on a side path and carries the
        // StagedBundle from the compat classifier straight through to apply.
        let artifact = artifact_for(&ReleaseId::new("v2"));
        assert!(
            artifact
                .staged_path
                .starts_with("/var/lib/polychrome/staged")
        );
        assert_eq!(artifact.bundle.catalog_hash, "catalog-v2");
    }
}