#![forbid(unsafe_code)]
#![deny(missing_docs)]
use crate::{LifecycleSnapshot, LifecycleState, Ownership, ServiceManager, ServiceSpec};
use std::{
fmt,
panic::{catch_unwind, AssertUnwindSafe},
time::{Duration, Instant},
};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum LifecycleUpdatePhase {
Inspect,
Quiesce,
Commit,
RestoreNew,
PostInstallCheck,
QuiesceForRollback,
RestoreOld,
FinalInspect,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct LifecycleFailure {
pub phase: LifecycleUpdatePhase,
pub service_id: String,
pub detail: String,
pub ownership: Option<Ownership>,
pub state: Option<LifecycleState>,
}
impl LifecycleFailure {
pub(crate) fn new(
spec: &ServiceSpec,
phase: LifecycleUpdatePhase,
detail: impl fmt::Display,
snapshot: Option<&LifecycleSnapshot>,
) -> Self {
Self {
phase,
service_id: spec.id().as_str().to_owned(),
detail: bounded_detail(detail.to_string()),
ownership: snapshot.map(|s| s.ownership),
state: snapshot.map(|s| s.state),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct PostInstallCheckError(String);
impl PostInstallCheckError {
pub fn new(detail: impl Into<String>) -> Self {
Self(bounded_detail(detail.into()))
}
}
impl fmt::Display for PostInstallCheckError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(&self.0)
}
}
impl std::error::Error for PostInstallCheckError {}
pub trait PostInstallCheck: fmt::Debug {
fn check(
&self,
spec: &ServiceSpec,
snapshot: &LifecycleSnapshot,
remaining: Duration,
) -> Result<(), PostInstallCheckError>;
}
#[derive(Debug, Default)]
pub struct NoPostInstallCheck;
impl PostInstallCheck for NoPostInstallCheck {
fn check(
&self,
_spec: &ServiceSpec,
_snapshot: &LifecycleSnapshot,
_remaining: Duration,
) -> Result<(), PostInstallCheckError> {
Ok(())
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
#[non_exhaustive]
pub enum PostTransitionVerification {
#[default]
Verify,
TrustManagerReport,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
#[non_exhaustive]
pub enum OwnershipLossTolerance {
#[default]
Deny,
TolerateAlreadyStopped,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub struct CommitEnginePolicy {
pub post_transition_verification: PostTransitionVerification,
pub ownership_loss_tolerance: OwnershipLossTolerance,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum Desired {
Running,
Stopped,
}
const INVALID_DESIRED_STATE: &str =
"desired lifecycle state is not a valid Running or Stopped target";
pub(crate) fn remaining(deadline: Instant) -> Duration {
deadline.saturating_duration_since(Instant::now())
}
pub(crate) fn bounded_detail(mut detail: String) -> String {
detail = detail.chars().filter(|c| !c.is_control()).collect();
crate::truncate_utf8_bytes(detail, crate::MAX_DIAGNOSTIC_BYTES)
}
pub(crate) fn safe_inspect<M: ServiceManager>(
manager: &mut M,
spec: &ServiceSpec,
phase: LifecycleUpdatePhase,
) -> Result<LifecycleSnapshot, LifecycleFailure> {
match catch_unwind(AssertUnwindSafe(|| manager.inspect(spec))) {
Ok(Ok(snapshot)) => Ok(snapshot),
Ok(Err(error)) => Err(LifecycleFailure::new(spec, phase, error, None)),
Err(_) => Err(LifecycleFailure::new(
spec,
phase,
"service manager panicked while inspecting",
None,
)),
}
}
fn ownership_tolerated(
policy: &CommitEnginePolicy,
wanted: LifecycleState,
snapshot: &LifecycleSnapshot,
) -> bool {
policy.ownership_loss_tolerance == OwnershipLossTolerance::TolerateAlreadyStopped
&& wanted == LifecycleState::Stopped
&& snapshot.state == LifecycleState::Stopped
}
pub(crate) fn transition_to<M: ServiceManager>(
manager: &mut M,
spec: &ServiceSpec,
wanted: LifecycleState,
timeout: Duration,
phase: LifecycleUpdatePhase,
policy: &CommitEnginePolicy,
) -> Result<(), LifecycleFailure> {
if timeout.is_zero() {
return Err(LifecycleFailure::new(
spec,
phase,
"lifecycle operation deadline exhausted",
None,
));
}
let desired = match wanted {
LifecycleState::Running => Desired::Running,
LifecycleState::Stopped => Desired::Stopped,
LifecycleState::Transitioning | LifecycleState::Unknown => {
return Err(LifecycleFailure::new(
spec,
phase,
INVALID_DESIRED_STATE,
None,
))
}
};
let current = manager
.inspect(spec)
.map_err(|e| LifecycleFailure::new(spec, phase, e, None))?;
if current.ownership != Ownership::Owned {
if ownership_tolerated(policy, wanted, ¤t) {
return Ok(());
}
return Err(LifecycleFailure::new(
spec,
phase,
"service ownership is no longer Owned",
Some(¤t),
));
}
let result = match (desired, current.state) {
(Desired::Running, LifecycleState::Running)
| (Desired::Stopped, LifecycleState::Stopped) => return Ok(()),
(Desired::Running, LifecycleState::Stopped) => manager.start(spec, timeout),
(Desired::Stopped, LifecycleState::Running) => manager.stop(spec, timeout),
(_, LifecycleState::Transitioning | LifecycleState::Unknown) => {
return Err(LifecycleFailure::new(
spec,
phase,
"service state is Unknown or Transitioning",
Some(¤t),
))
}
}
.map_err(|e| LifecycleFailure::new(spec, phase, e, Some(¤t)))?;
if !result.completed {
return Err(LifecycleFailure::new(
spec,
phase,
"service transition did not complete",
Some(¤t),
));
}
if policy.post_transition_verification == PostTransitionVerification::Verify {
let after = manager
.inspect(spec)
.map_err(|e| LifecycleFailure::new(spec, phase, e, None))?;
validate_observed(spec, &after, wanted, phase, policy)?;
}
Ok(())
}
pub(crate) fn safe_transition_to<M: ServiceManager>(
manager: &mut M,
spec: &ServiceSpec,
wanted: LifecycleState,
timeout: Duration,
phase: LifecycleUpdatePhase,
policy: &CommitEnginePolicy,
) -> Result<(), LifecycleFailure> {
match catch_unwind(AssertUnwindSafe(|| {
transition_to(manager, spec, wanted, timeout, phase, policy)
})) {
Ok(result) => result,
Err(_) => Err(LifecycleFailure::new(
spec,
phase,
"service manager panicked while transitioning",
None,
)),
}
}
pub(crate) fn validate_observed(
spec: &ServiceSpec,
snapshot: &LifecycleSnapshot,
wanted: LifecycleState,
phase: LifecycleUpdatePhase,
policy: &CommitEnginePolicy,
) -> Result<(), LifecycleFailure> {
let ownership_ok =
snapshot.ownership == Ownership::Owned || ownership_tolerated(policy, wanted, snapshot);
if !ownership_ok || snapshot.state != wanted {
return Err(LifecycleFailure::new(
spec,
phase,
"service ownership/state does not match the requested state",
Some(snapshot),
));
}
Ok(())
}
pub(crate) fn quiesce_service<M: ServiceManager>(
manager: &mut M,
spec: &ServiceSpec,
before: &LifecycleSnapshot,
timeout: Duration,
) -> Result<(), LifecycleFailure> {
let stop_result = catch_unwind(AssertUnwindSafe(|| manager.stop(spec, timeout)));
match stop_result {
Ok(Err(error)) => Err(LifecycleFailure::new(
spec,
LifecycleUpdatePhase::Quiesce,
error,
Some(before),
)),
Err(_) => Err(LifecycleFailure::new(
spec,
LifecycleUpdatePhase::Quiesce,
"service manager panicked while stopping",
Some(before),
)),
Ok(Ok(result)) if !result.completed => Err(LifecycleFailure::new(
spec,
LifecycleUpdatePhase::Quiesce,
"stop did not complete",
Some(before),
)),
Ok(Ok(_)) => {
let after = safe_inspect(manager, spec, LifecycleUpdatePhase::Quiesce)?;
if after.ownership == Ownership::Owned && after.state == LifecycleState::Stopped {
Ok(())
} else {
Err(LifecycleFailure::new(
spec,
LifecycleUpdatePhase::Quiesce,
"ownership/state changed after stop",
Some(&after),
))
}
}
}
}
pub(crate) fn quiesce_for_rollback<M: ServiceManager>(
manager: &mut M,
spec: &ServiceSpec,
deadline: Instant,
) -> Option<LifecycleFailure> {
let budget = remaining(deadline);
if budget.is_zero() {
return Some(LifecycleFailure::new(
spec,
LifecycleUpdatePhase::QuiesceForRollback,
"post-commit deadline exhausted before rollback quiescence",
None,
));
}
let observed = match manager.inspect(spec) {
Ok(snapshot) => Some(snapshot),
Err(error) => {
return Some(LifecycleFailure::new(
spec,
LifecycleUpdatePhase::QuiesceForRollback,
format!("refusing to quiesce unobservable service before rollback: {error}"),
None,
))
}
};
if observed
.as_ref()
.is_some_and(|s| s.ownership == Ownership::Owned && s.state == LifecycleState::Stopped)
{
return None;
}
if let Some(observation) = observed.as_ref() {
if observation.ownership != Ownership::Owned {
return Some(LifecycleFailure::new(
spec,
LifecycleUpdatePhase::QuiesceForRollback,
"refusing to quiesce non-owned service before rollback",
Some(observation),
));
}
}
match manager.stop(spec, budget) {
Err(error) => Some(LifecycleFailure::new(
spec,
LifecycleUpdatePhase::QuiesceForRollback,
error,
observed.as_ref(),
)),
Ok(result) if !result.completed => Some(LifecycleFailure::new(
spec,
LifecycleUpdatePhase::QuiesceForRollback,
"stop did not complete before artifact rollback",
observed.as_ref(),
)),
Ok(_) => match manager.inspect(spec) {
Ok(after)
if after.ownership == Ownership::Owned
&& after.state == LifecycleState::Stopped =>
{
None
}
Ok(after) => Some(LifecycleFailure::new(
spec,
LifecycleUpdatePhase::QuiesceForRollback,
"stopped state could not be confirmed before artifact rollback",
Some(&after),
)),
Err(error) => Some(LifecycleFailure::new(
spec,
LifecycleUpdatePhase::QuiesceForRollback,
error,
None,
)),
},
}
}
pub(crate) fn restore_and_check<M: ServiceManager, C: PostInstallCheck>(
manager: &mut M,
spec: &ServiceSpec,
wanted: LifecycleState,
check: &C,
deadline: Instant,
policy: &CommitEnginePolicy,
) -> Result<(), LifecycleFailure> {
safe_transition_to(
manager,
spec,
wanted,
remaining(deadline),
LifecycleUpdatePhase::RestoreNew,
policy,
)?;
let snapshot = safe_inspect(manager, spec, LifecycleUpdatePhase::RestoreNew)?;
if policy.post_transition_verification == PostTransitionVerification::Verify {
validate_observed(
spec,
&snapshot,
wanted,
LifecycleUpdatePhase::RestoreNew,
policy,
)?;
}
let budget = remaining(deadline);
if budget.is_zero() {
return Err(LifecycleFailure::new(
spec,
LifecycleUpdatePhase::PostInstallCheck,
"post-commit deadline exhausted before post-install check",
Some(&snapshot),
));
}
check.check(spec, &snapshot, budget).map_err(|e| {
LifecycleFailure::new(
spec,
LifecycleUpdatePhase::PostInstallCheck,
e,
Some(&snapshot),
)
})
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{HealthState, ServiceError, ServiceId, TransitionResult};
use std::time::Duration;
fn spec() -> ServiceSpec {
ServiceSpec::new(
ServiceId::new("commit-engine").unwrap(),
std::env::current_exe().unwrap(),
Vec::new(),
None,
)
.unwrap()
}
#[derive(Debug)]
struct RunningManager;
impl ServiceManager for RunningManager {
fn inspect(&self, spec: &ServiceSpec) -> Result<LifecycleSnapshot, ServiceError> {
Ok(LifecycleSnapshot {
id: spec.id().clone(),
ownership: Ownership::Owned,
state: LifecycleState::Running,
health: HealthState::Unknown,
was_registered: true,
was_running: true,
})
}
fn install(&mut self, _spec: &ServiceSpec) -> Result<TransitionResult, ServiceError> {
unreachable!("not exercised")
}
fn start(
&mut self,
_spec: &ServiceSpec,
_timeout: Duration,
) -> Result<TransitionResult, ServiceError> {
unreachable!("not exercised")
}
fn stop(
&mut self,
_spec: &ServiceSpec,
_timeout: Duration,
) -> Result<TransitionResult, ServiceError> {
unreachable!("not exercised")
}
fn restart(
&mut self,
_spec: &ServiceSpec,
_timeout: Duration,
) -> Result<TransitionResult, ServiceError> {
unreachable!("not exercised")
}
fn uninstall(&mut self, _spec: &ServiceSpec) -> Result<TransitionResult, ServiceError> {
unreachable!("not exercised")
}
}
#[test]
fn non_running_stopped_desired_state_is_a_typed_failure() {
for desired in [LifecycleState::Transitioning, LifecycleState::Unknown] {
let mut manager = RunningManager;
let spec = spec();
let failure = transition_to(
&mut manager,
&spec,
desired,
Duration::from_secs(1),
LifecycleUpdatePhase::RestoreNew,
&CommitEnginePolicy::default(),
)
.expect_err("an invalid desired state must fail");
assert_eq!(failure.phase, LifecycleUpdatePhase::RestoreNew);
assert_eq!(failure.detail, INVALID_DESIRED_STATE);
assert!(failure.detail.len() <= crate::MAX_DIAGNOSTIC_BYTES);
}
}
}