Skip to main content

eggup_service/
lifecycle_update.rs

1use crate::{
2    commit_engine::{quiesce_for_rollback, restore_and_check, safe_inspect, safe_transition_to},
3    CommitEnginePolicy, LifecycleFailure, LifecycleSnapshot, LifecycleState, LifecycleUpdatePhase,
4    Ownership, PostInstallCheck, PostInstallCheckError, RestoreIntent, ServiceManager, ServiceSpec,
5};
6use eggup_core::{
7    CommitOwnership, Error as CoreError, PostCommitFailurePolicy, TransactionDisposition,
8    TransactionReceipt, ValidatedTransaction,
9};
10use std::{
11    fmt,
12    panic::{catch_unwind, AssertUnwindSafe},
13    time::{Duration, Instant},
14};
15
16/// Terminal service restoration status for a composite update.
17#[derive(Debug, Clone, Copy, PartialEq, Eq)]
18pub enum LifecycleRestorationStatus {
19    /// Service lifecycle was restored and confirmed for the terminal artifact generation.
20    Restored,
21    /// A lifecycle transition or confirmation failed.
22    Failed,
23    /// Artifact state is uncertain, so automatic restoration was suppressed.
24    NotAttemptedRecoveryRequired,
25}
26
27/// Mechanism-level choices and bounds for one service-aware transaction.
28#[derive(Debug, Clone, Copy)]
29pub struct LifecycleUpdatePolicy {
30    /// Desired service state after a successful artifact update.
31    pub restore: RestoreIntent,
32    /// Core behavior if post-commit lifecycle/check work fails.
33    pub post_commit_failure: PostCommitFailurePolicy,
34    /// Maximum wait for stopping the prior service generation.
35    pub quiesce_timeout: Duration,
36    /// Shared budget for post-commit transition and caller check work.
37    pub post_commit_timeout: Duration,
38    /// Maximum wait to restore prior lifecycle state after rollback or Core pre-receipt error.
39    pub rollback_restore_timeout: Duration,
40}
41
42impl LifecycleUpdatePolicy {
43    /// Rejects zero or overlong timeouts before any manager mutation.
44    ///
45    /// # Errors
46    ///
47    /// [`LifecycleUpdateError::InvalidPolicy`] when `quiesce_timeout`,
48    /// `post_commit_timeout`, or `rollback_restore_timeout` is zero or above
49    /// [`crate::MAX_TRANSITION_TIMEOUT`]. The error names the offending field.
50    pub fn validate(&self) -> Result<(), LifecycleUpdateError> {
51        for (field, value) in [
52            ("quiesce_timeout", self.quiesce_timeout),
53            ("post_commit_timeout", self.post_commit_timeout),
54            ("rollback_restore_timeout", self.rollback_restore_timeout),
55        ] {
56            if value.is_zero() || value > crate::MAX_TRANSITION_TIMEOUT {
57                return Err(LifecycleUpdateError::InvalidPolicy { field });
58            }
59        }
60        Ok(())
61    }
62}
63
64/// Composite result that retains the exact Core transaction evidence.
65#[derive(Debug)]
66pub struct LifecycleUpdateReceipt {
67    /// Immutable pre-update observation.
68    pub before: LifecycleSnapshot,
69    /// Final read-only observation when available.
70    pub final_snapshot: Option<LifecycleSnapshot>,
71    /// Exact Core artifact transaction receipt.
72    pub transaction: TransactionReceipt,
73    /// Whether a previously running service was quiesced before Core mutation.
74    pub pre_commit_quiesced: bool,
75    /// Lifecycle restoration state at the terminal artifact disposition.
76    pub restoration: LifecycleRestorationStatus,
77    /// Post-commit lifecycle or caller-check failure, when one occurred.
78    pub post_commit_failure: Option<LifecycleFailure>,
79    /// Failure stopping a possibly running new generation before Core rollback.
80    pub rollback_quiesce_failure: Option<LifecycleFailure>,
81    /// Failure restoring the old service state after successful artifact rollback.
82    pub rollback_restore_failure: Option<LifecycleFailure>,
83    /// Failure to obtain the optional final read-only observation.
84    pub observation_failure: Option<LifecycleFailure>,
85    /// The commit-engine policy that governed this transaction.
86    ///
87    /// This makes the selected [`CommitEnginePolicy`] inputs observable in the
88    /// returned evidence, including the permissive ownership-loss tolerance,
89    /// which otherwise has no failure to attach itself to.
90    pub engine_policy: CommitEnginePolicy,
91}
92
93impl LifecycleUpdateReceipt {
94    /// Core's authoritative artifact disposition.
95    pub fn artifact_disposition(&self) -> TransactionDisposition {
96        self.transaction.disposition()
97    }
98
99    /// Whether Core retained recovery evidence requiring manual artifact recovery.
100    pub fn manual_artifact_recovery_required(&self) -> bool {
101        self.transaction.disposition() == TransactionDisposition::RecoveryRequired
102    }
103
104    /// Whether the new artifact generation remains installed.
105    pub fn new_generation_retained(&self) -> bool {
106        self.transaction.disposition() == TransactionDisposition::Committed
107    }
108
109    /// Whether service lifecycle was restored and confirmed for the terminal generation.
110    pub fn service_state_restored(&self) -> bool {
111        self.restoration == LifecycleRestorationStatus::Restored
112    }
113}
114
115/// Failure before a composite terminal transaction receipt could be returned.
116#[derive(Debug)]
117#[non_exhaustive]
118pub enum LifecycleUpdateError {
119    /// One timeout is zero or exceeds the service transition limit.
120    InvalidPolicy {
121        /// Name of the invalid field.
122        field: &'static str,
123    },
124    /// Preflight failed before service or artifact mutation.
125    Preflight {
126        /// Structured failure evidence.
127        failure: LifecycleFailure,
128    },
129    /// Pre-commit service quiescence failed; attempted restoration is retained separately.
130    Quiesce {
131        /// Original quiescence failure.
132        failure: LifecycleFailure,
133        /// Failure restoring the pre-update state after the stop attempt, if any.
134        restore_failure: Option<LifecycleFailure>,
135    },
136    /// Core returned an error before a terminal receipt; service restoration was attempted.
137    CoreBeforeReceipt {
138        /// Exact Core error.
139        core: CoreError,
140        /// Failure restoring pre-update lifecycle state, if any.
141        restore_failure: Option<LifecycleFailure>,
142    },
143}
144
145impl fmt::Display for LifecycleUpdateError {
146    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
147        match self {
148            Self::InvalidPolicy { field } => write!(f, "invalid lifecycle update timeout: {field}"),
149            Self::Preflight { failure } => write!(f, "lifecycle preflight failed: {failure:?}"),
150            Self::Quiesce { failure, .. } => write!(f, "service quiescence failed: {failure:?}"),
151            Self::CoreBeforeReceipt { core, .. } => {
152                write!(f, "Core returned before a transaction receipt: {core}")
153            }
154        }
155    }
156}
157impl std::error::Error for LifecycleUpdateError {}
158
159/// Quiesce an owned service, compose Core M007, and restore lifecycle state based on the artifact result.
160///
161/// The existing registration must be `Owned` and either `Running` or `Stopped`.
162/// `Preserve` keeps a stopped service stopped after success. A rolled-back
163/// transaction restores the pre-update lifecycle state regardless of `restore`.
164/// `RecoveryRequired` suppresses automatic service starts. No registration
165/// install/refresh is performed. The service adapter uses Core's transaction
166/// rollback rather than maintaining artifact backups of its own.
167///
168/// This entry point runs the unified commit engine under
169/// [`CommitEnginePolicy::default`], the stricter of the two former engine
170/// copies' behaviours. Use [`commit_with_lifecycle_with_engine_policy`] to
171/// select the named policy inputs explicitly.
172///
173/// # Errors
174///
175/// [`LifecycleUpdateError`] when the update cannot be completed safely: the
176/// policy is invalid, the service is not `Owned`, the commit engine could
177/// not commit, roll back, or restore the service to its pre-update state. The
178/// error carries the transaction disposition, the observed lifecycle state,
179/// and whether manual artifact recovery is required.
180pub fn commit_with_lifecycle<M: ServiceManager, C: PostInstallCheck>(
181    manager: &mut M,
182    spec: &ServiceSpec,
183    transaction: ValidatedTransaction,
184    ownership: CommitOwnership<'_>,
185    policy: LifecycleUpdatePolicy,
186    check: &C,
187) -> Result<LifecycleUpdateReceipt, LifecycleUpdateError> {
188    commit_with_lifecycle_with_engine_policy(
189        manager,
190        spec,
191        transaction,
192        ownership,
193        policy,
194        check,
195        CommitEnginePolicy::default(),
196    )
197}
198
199/// Quiesce an owned service under an explicitly selected commit-engine policy.
200///
201/// Identical to [`commit_with_lifecycle`] except that the two named
202/// commit-engine inputs โ€” post-transition verification and ownership-loss
203/// tolerance โ€” are chosen by the caller instead of taking their stricter
204/// defaults. The selected policy is observable in the returned receipt's
205/// `engine_policy` field.
206///
207/// # Errors
208///
209/// [`LifecycleUpdateError`] when the update cannot be completed safely: the
210/// policy is invalid, the service is not `Owned`, the commit engine could
211/// not commit, roll back, or restore the service to its pre-update state. The
212/// error carries the transaction disposition, the observed lifecycle state,
213/// and whether manual artifact recovery is required.
214pub fn commit_with_lifecycle_with_engine_policy<M: ServiceManager, C: PostInstallCheck>(
215    manager: &mut M,
216    spec: &ServiceSpec,
217    transaction: ValidatedTransaction,
218    ownership: CommitOwnership<'_>,
219    policy: LifecycleUpdatePolicy,
220    check: &C,
221    engine: CommitEnginePolicy,
222) -> Result<LifecycleUpdateReceipt, LifecycleUpdateError> {
223    policy.validate()?;
224    let before = safe_inspect(manager, spec, LifecycleUpdatePhase::Inspect)
225        .map_err(|failure| LifecycleUpdateError::Preflight { failure })?;
226    if before.ownership != Ownership::Owned
227        || !matches!(
228            before.state,
229            LifecycleState::Running | LifecycleState::Stopped
230        )
231    {
232        return Err(LifecycleUpdateError::Preflight {
233            failure: LifecycleFailure::new(
234                spec,
235                LifecycleUpdatePhase::Inspect,
236                "service must be Owned and Running or Stopped",
237                Some(&before),
238            ),
239        });
240    }
241
242    let mut pre_commit_quiesced = false;
243    if before.state == LifecycleState::Running {
244        let stop_failure =
245            crate::commit_engine::quiesce_service(manager, spec, &before, policy.quiesce_timeout)
246                .err();
247        if let Some(failure) = stop_failure {
248            let restore_failure = safe_transition_to(
249                manager,
250                spec,
251                LifecycleState::Running,
252                policy.rollback_restore_timeout,
253                LifecycleUpdatePhase::Quiesce,
254                &engine,
255            )
256            .err();
257            return Err(LifecycleUpdateError::Quiesce {
258                failure,
259                restore_failure,
260            });
261        }
262        pre_commit_quiesced = true;
263    }
264
265    let mut post_commit_failure = None;
266    let mut rollback_quiesce_failure = None;
267    let core_result =
268        transaction.commit_with_post_commit(ownership, policy.post_commit_failure, || {
269            let deadline = Instant::now() + policy.post_commit_timeout;
270            let work = catch_unwind(AssertUnwindSafe(|| {
271                restore_success_state(
272                    manager,
273                    spec,
274                    &before,
275                    policy.restore,
276                    check,
277                    deadline,
278                    &engine,
279                )
280            }));
281            let failure = match work {
282                Ok(Ok(())) => return Ok(()),
283                Ok(Err(failure)) => failure,
284                Err(_) => LifecycleFailure::new(
285                    spec,
286                    LifecycleUpdatePhase::PostInstallCheck,
287                    "post-commit service work panicked",
288                    None,
289                ),
290            };
291            post_commit_failure = Some(failure.clone());
292
293            if policy.post_commit_failure == PostCommitFailurePolicy::RollBack {
294                rollback_quiesce_failure = match catch_unwind(AssertUnwindSafe(|| {
295                    quiesce_for_rollback(manager, spec, deadline)
296                })) {
297                    Ok(failure) => failure,
298                    Err(_) => Some(LifecycleFailure::new(
299                        spec,
300                        LifecycleUpdatePhase::QuiesceForRollback,
301                        "service manager panicked while quiescing before artifact rollback",
302                        None,
303                    )),
304                };
305            }
306            Err(PostInstallCheckError::new(failure.detail))
307        });
308
309    let transaction = match core_result {
310        Ok(receipt) => receipt,
311        Err(core) => {
312            let restore_failure = if pre_commit_quiesced {
313                safe_transition_to(
314                    manager,
315                    spec,
316                    before.state,
317                    policy.rollback_restore_timeout,
318                    LifecycleUpdatePhase::RestoreOld,
319                    &engine,
320                )
321                .err()
322            } else {
323                None
324            };
325            return Err(LifecycleUpdateError::CoreBeforeReceipt {
326                core,
327                restore_failure,
328            });
329        }
330    };
331
332    let mut rollback_restore_failure = None;
333    let mut restoration = LifecycleRestorationStatus::Restored;
334    match transaction.disposition() {
335        TransactionDisposition::Committed => {
336            if post_commit_failure
337                .as_ref()
338                .is_some_and(|f| f.phase == LifecycleUpdatePhase::RestoreNew)
339            {
340                restoration = LifecycleRestorationStatus::Failed;
341            }
342        }
343        TransactionDisposition::RolledBack => {
344            if let Err(failure) = safe_transition_to(
345                manager,
346                spec,
347                before.state,
348                policy.rollback_restore_timeout,
349                LifecycleUpdatePhase::RestoreOld,
350                &engine,
351            ) {
352                rollback_restore_failure = Some(failure);
353                restoration = LifecycleRestorationStatus::Failed;
354            }
355        }
356        TransactionDisposition::RecoveryRequired => {
357            restoration = LifecycleRestorationStatus::NotAttemptedRecoveryRequired;
358        }
359    }
360
361    let final_snapshot = match safe_inspect(manager, spec, LifecycleUpdatePhase::FinalInspect) {
362        Ok(snapshot) => Some(snapshot),
363        Err(observation_failure) => {
364            // A final read-only observation is nonessential and never changes Core disposition.
365            return Ok(LifecycleUpdateReceipt {
366                before,
367                final_snapshot: None,
368                transaction,
369                pre_commit_quiesced,
370                restoration,
371                post_commit_failure,
372                rollback_quiesce_failure,
373                rollback_restore_failure,
374                observation_failure: Some(observation_failure),
375                engine_policy: engine,
376            });
377        }
378    };
379
380    Ok(LifecycleUpdateReceipt {
381        before,
382        final_snapshot,
383        transaction,
384        pre_commit_quiesced,
385        restoration,
386        post_commit_failure,
387        rollback_quiesce_failure,
388        rollback_restore_failure,
389        observation_failure: None,
390        engine_policy: engine,
391    })
392}
393
394fn restore_success_state<M: ServiceManager, C: PostInstallCheck>(
395    manager: &mut M,
396    spec: &ServiceSpec,
397    before: &LifecycleSnapshot,
398    intent: RestoreIntent,
399    check: &C,
400    deadline: Instant,
401    engine: &CommitEnginePolicy,
402) -> Result<(), LifecycleFailure> {
403    let wanted = match intent {
404        RestoreIntent::Preserve => before.state,
405        RestoreIntent::EnsureRunning => LifecycleState::Running,
406        RestoreIntent::EnsureStopped => LifecycleState::Stopped,
407    };
408    // `restore_and_check` delegates the transition to the unified engine's
409    // panic-safe transition mechanic, which catches a panicking
410    // `start`/`stop`/`inspect` and records it under the phase that was
411    // actually executing. The enclosing `catch_unwind` cannot do this: it
412    // wraps the transition *and* the post-install check together, so labelling
413    // its panic arm `PostInstallCheck` would misreport a panicking service
414    // transition as a check failure and let the receipt claim `Restored`.
415    restore_and_check(manager, spec, wanted, check, deadline, engine)
416}
417
418#[cfg(test)]
419mod tests {
420    use super::*;
421    use crate::{
422        ManagerFailureKind, NoPostInstallCheck, ServiceError, ServiceId, ServiceOperation,
423        TestDoubleManager, TransitionResult,
424    };
425    use eggup_core::{
426        AbsentOnlyVerifier, AbsentPolicy, AllValidators, ArtifactMember, ArtifactSet,
427        CommitOwnership as CoreCommitOwnership, InstallPlan, MemberId, Ownership as CoreOwnership,
428        PermissionsIntent, ProductId, ReleaseId,
429    };
430    use std::{
431        cell::{Cell, RefCell},
432        fs,
433        path::{Path, PathBuf},
434        rc::Rc,
435        sync::atomic::{AtomicU64, Ordering},
436        thread,
437    };
438
439    static NEXT: AtomicU64 = AtomicU64::new(0);
440
441    #[test]
442    fn lifecycle_failure_detail_is_utf8_safe_bounded_and_control_free() {
443        let spec = ServiceSpec::new(
444            ServiceId::new("diagnostic").unwrap(),
445            std::env::current_exe().unwrap(),
446            Vec::new(),
447            None,
448        )
449        .unwrap();
450        let mut input = "a".repeat(510);
451        input.push('๐Ÿงก');
452        input.push_str("\nmanager detail");
453        let failure = LifecycleFailure::new(&spec, LifecycleUpdatePhase::Inspect, input, None);
454        assert!(failure.detail.len() <= 512);
455        assert!(failure.detail.is_char_boundary(failure.detail.len()));
456        assert!(!failure.detail.chars().any(char::is_control));
457    }
458
459    struct FixedVerifier(CoreOwnership);
460    impl eggup_core::OwnershipVerifier for FixedVerifier {
461        fn verify(&self, _member: &MemberId, _destination: &Path) -> CoreOwnership {
462            self.0
463        }
464    }
465
466    #[derive(Default)]
467    struct ScriptedManager {
468        inner: TestDoubleManager,
469        events: Rc<RefCell<Vec<&'static str>>>,
470        fail_stop: bool,
471        incomplete_stop: bool,
472        fail_start_calls: Vec<usize>,
473        panic_start_calls: Vec<usize>,
474        start_calls: usize,
475        stop_calls: usize,
476        drift_to_foreign_after_stop: bool,
477        fail_final_inspect: bool,
478        check_finished: Option<Rc<Cell<bool>>>,
479        final_inspect_failed: Cell<bool>,
480        start_delay: Duration,
481    }
482
483    impl ScriptedManager {
484        fn with_service(spec: ServiceSpec, state: LifecycleState) -> Self {
485            let mut inner = TestDoubleManager::new();
486            inner.put(spec, state);
487            Self {
488                inner,
489                ..Self::default()
490            }
491        }
492    }
493
494    impl ServiceManager for ScriptedManager {
495        fn inspect(&self, spec: &ServiceSpec) -> Result<LifecycleSnapshot, ServiceError> {
496            self.events.borrow_mut().push("inspect");
497            if self.fail_final_inspect
498                && self.check_finished.as_ref().is_some_and(|f| f.get())
499                && !self.final_inspect_failed.get()
500            {
501                // The final observation is the first inspect after the check returns.
502                self.final_inspect_failed.set(true);
503                return Err(ServiceError::manager(
504                    ManagerFailureKind::Protocol,
505                    "injected final inspection failure",
506                ));
507            }
508            self.inner.inspect(spec)
509        }
510
511        fn install(&mut self, spec: &ServiceSpec) -> Result<TransitionResult, ServiceError> {
512            self.events.borrow_mut().push("install");
513            self.inner.install(spec)
514        }
515
516        fn start(
517            &mut self,
518            spec: &ServiceSpec,
519            timeout: Duration,
520        ) -> Result<TransitionResult, ServiceError> {
521            self.events.borrow_mut().push("start");
522            self.start_calls += 1;
523            if self.start_delay > Duration::ZERO {
524                thread::sleep(self.start_delay);
525            }
526            if self.fail_start_calls.contains(&self.start_calls) {
527                return Err(ServiceError::manager(
528                    ManagerFailureKind::Protocol,
529                    "injected start failure",
530                ));
531            }
532            if self.panic_start_calls.contains(&self.start_calls) {
533                panic!("test panic payload must not escape");
534            }
535            self.inner.start(spec, timeout)
536        }
537
538        fn stop(
539            &mut self,
540            spec: &ServiceSpec,
541            timeout: Duration,
542        ) -> Result<TransitionResult, ServiceError> {
543            self.events.borrow_mut().push("stop");
544            self.stop_calls += 1;
545            if self.fail_stop {
546                return Err(ServiceError::manager(
547                    ManagerFailureKind::Protocol,
548                    "injected stop failure",
549                ));
550            }
551            if self.incomplete_stop {
552                return Ok(TransitionResult {
553                    operation: ServiceOperation::Stop,
554                    completed: false,
555                    detail: "injected incomplete stop".into(),
556                });
557            }
558            let result = self.inner.stop(spec, timeout)?;
559            if self.drift_to_foreign_after_stop {
560                let foreign = ServiceSpec::new(
561                    spec.id().clone(),
562                    crate::absolute("/different/service-executable"),
563                    spec.args().to_vec(),
564                    spec.config().map(Path::to_path_buf),
565                )?;
566                self.inner.put(foreign, LifecycleState::Stopped);
567            }
568            Ok(result)
569        }
570
571        fn restart(
572            &mut self,
573            spec: &ServiceSpec,
574            timeout: Duration,
575        ) -> Result<TransitionResult, ServiceError> {
576            self.events.borrow_mut().push("restart");
577            self.inner.restart(spec, timeout)
578        }
579
580        fn uninstall(&mut self, spec: &ServiceSpec) -> Result<TransitionResult, ServiceError> {
581            self.events.borrow_mut().push("uninstall");
582            self.inner.uninstall(spec)
583        }
584    }
585
586    fn setup(old_generation: bool) -> (PathBuf, PathBuf, ValidatedTransaction) {
587        let base = std::env::temp_dir().join(format!(
588            "eggup-service-lifecycle-update-{}-{}",
589            std::process::id(),
590            NEXT.fetch_add(1, Ordering::Relaxed)
591        ));
592        let source_root = base.join("input");
593        let install_root = base.join("install");
594        fs::create_dir_all(&source_root).unwrap();
595        fs::create_dir_all(&install_root).unwrap();
596        let main = source_root.join("main");
597        let sidecar = source_root.join("sidecar");
598        fs::write(&main, b"new-main").unwrap();
599        fs::write(&sidecar, b"new-sidecar").unwrap();
600        if old_generation {
601            fs::write(install_root.join("service.bin"), b"old-main").unwrap();
602            fs::write(install_root.join("sidecar.bin"), b"old-sidecar").unwrap();
603        }
604        let members = ArtifactSet::new(vec![
605            ArtifactMember::new(MemberId::new("main").unwrap(), &main, "service.bin")
606                .unwrap()
607                .with_permissions(PermissionsIntent::Executable)
608                .with_integrity(eggup_core::IntegrityRequirement::Sha256(
609                    eggup_core::hash_file(&main).unwrap(),
610                )),
611            ArtifactMember::new(MemberId::new("sidecar").unwrap(), &sidecar, "sidecar.bin")
612                .unwrap()
613                .with_integrity(eggup_core::IntegrityRequirement::Sha256(
614                    eggup_core::hash_file(&sidecar).unwrap(),
615                )),
616        ])
617        .unwrap();
618        let prepared = InstallPlan::new(
619            ProductId::new("service-test").unwrap(),
620            ReleaseId::new("r2").unwrap(),
621            &install_root,
622            members,
623        )
624        .unwrap()
625        .prepare()
626        .unwrap();
627        let validated = prepared
628            .verify_integrity()
629            .unwrap()
630            .validate(&AllValidators::new())
631            .unwrap();
632        (base, install_root, validated)
633    }
634
635    fn spec(install_root: &Path) -> ServiceSpec {
636        ServiceSpec::new(
637            ServiceId::new("m005-test-service").unwrap(),
638            install_root.join("service.bin"),
639            vec!["--worker".into()],
640            None,
641        )
642        .unwrap()
643    }
644
645    fn policy(restore: RestoreIntent, failure: PostCommitFailurePolicy) -> LifecycleUpdatePolicy {
646        LifecycleUpdatePolicy {
647            restore,
648            post_commit_failure: failure,
649            quiesce_timeout: Duration::from_secs(2),
650            post_commit_timeout: Duration::from_secs(2),
651            rollback_restore_timeout: Duration::from_secs(2),
652        }
653    }
654
655    #[derive(Debug)]
656    struct PassCheck {
657        complete: Option<Rc<Cell<bool>>>,
658        observed_remaining: Option<Rc<Cell<u128>>>,
659        events: Option<Rc<RefCell<Vec<&'static str>>>>,
660        fail: bool,
661    }
662    impl PostInstallCheck for PassCheck {
663        fn check(
664            &self,
665            _spec: &ServiceSpec,
666            _snapshot: &LifecycleSnapshot,
667            remaining: Duration,
668        ) -> Result<(), PostInstallCheckError> {
669            if let Some(events) = &self.events {
670                events.borrow_mut().push("check");
671            }
672            if let Some(value) = &self.observed_remaining {
673                value.set(remaining.as_nanos());
674            }
675            if let Some(complete) = &self.complete {
676                complete.set(true);
677            }
678            if self.fail {
679                Err(PostInstallCheckError::new("injected health check failure"))
680            } else {
681                Ok(())
682            }
683        }
684    }
685
686    #[derive(Debug)]
687    struct CompleteGenerationCheck;
688    impl PostInstallCheck for CompleteGenerationCheck {
689        fn check(
690            &self,
691            spec: &ServiceSpec,
692            _snapshot: &LifecycleSnapshot,
693            _remaining: Duration,
694        ) -> Result<(), PostInstallCheckError> {
695            let root = spec
696                .executable()
697                .parent()
698                .expect("service executable parent");
699            if fs::read(root.join("service.bin")).ok().as_deref() == Some(b"new-main")
700                && fs::read(root.join("sidecar.bin")).ok().as_deref() == Some(b"new-sidecar")
701            {
702                Ok(())
703            } else {
704                Err(PostInstallCheckError::new(
705                    "complete artifact generation is not visible",
706                ))
707            }
708        }
709    }
710
711    #[derive(Debug)]
712    struct PanicCheck {
713        called: Rc<Cell<bool>>,
714    }
715    impl PostInstallCheck for PanicCheck {
716        fn check(
717            &self,
718            _spec: &ServiceSpec,
719            _snapshot: &LifecycleSnapshot,
720            _remaining: Duration,
721        ) -> Result<(), PostInstallCheckError> {
722            self.called.set(true);
723            panic!("test panic payload must not escape");
724        }
725    }
726
727    #[derive(Debug)]
728    struct SabotageCheck(PathBuf);
729    impl PostInstallCheck for SabotageCheck {
730        fn check(
731            &self,
732            _spec: &ServiceSpec,
733            _snapshot: &LifecycleSnapshot,
734            _remaining: Duration,
735        ) -> Result<(), PostInstallCheckError> {
736            let dest = self.0.join("service.bin");
737            fs::remove_file(&dest).unwrap();
738            fs::create_dir(&dest).unwrap();
739            Err(PostInstallCheckError::new("injected rollback failure"))
740        }
741    }
742
743    fn absent_ownership() -> CoreCommitOwnership<'static> {
744        // The verifier is a static test value, so the returned ownership can be static.
745        static VERIFIER: AbsentOnlyVerifier = AbsentOnlyVerifier;
746        CoreCommitOwnership::new(&VERIFIER, AbsentPolicy::AllowCreate)
747    }
748
749    #[test]
750    fn rejects_zero_and_overlong_budgets_before_mutation() {
751        let mut invalid = policy(
752            RestoreIntent::Preserve,
753            PostCommitFailurePolicy::KeepInstalled,
754        );
755        invalid.quiesce_timeout = Duration::ZERO;
756        assert!(matches!(
757            invalid.validate(),
758            Err(LifecycleUpdateError::InvalidPolicy { .. })
759        ));
760        invalid.quiesce_timeout = Duration::from_secs(1);
761        invalid.post_commit_timeout = crate::MAX_TRANSITION_TIMEOUT + Duration::from_nanos(1);
762        assert!(matches!(
763            invalid.validate(),
764            Err(LifecycleUpdateError::InvalidPolicy { .. })
765        ));
766    }
767
768    #[test]
769    fn preserve_and_explicit_restore_intents_produce_expected_success_state() {
770        for (pre_state, intent, expected) in [
771            (
772                LifecycleState::Running,
773                RestoreIntent::Preserve,
774                LifecycleState::Running,
775            ),
776            (
777                LifecycleState::Stopped,
778                RestoreIntent::Preserve,
779                LifecycleState::Stopped,
780            ),
781            (
782                LifecycleState::Stopped,
783                RestoreIntent::EnsureRunning,
784                LifecycleState::Running,
785            ),
786            (
787                LifecycleState::Running,
788                RestoreIntent::EnsureStopped,
789                LifecycleState::Stopped,
790            ),
791        ] {
792            let (base, root, tx) = setup(false);
793            let service = spec(&root);
794            let mut manager = ScriptedManager::with_service(service.clone(), pre_state);
795            let receipt = commit_with_lifecycle(
796                &mut manager,
797                &service,
798                tx,
799                absent_ownership(),
800                policy(intent, PostCommitFailurePolicy::KeepInstalled),
801                &NoPostInstallCheck,
802            )
803            .unwrap();
804            assert_eq!(
805                receipt.artifact_disposition(),
806                TransactionDisposition::Committed
807            );
808            assert_eq!(receipt.final_snapshot.as_ref().unwrap().state, expected);
809            assert_eq!(
810                receipt.pre_commit_quiesced,
811                pre_state == LifecycleState::Running
812            );
813            let events = manager.events.borrow();
814            assert_eq!(events.iter().filter(|e| **e == "install").count(), 0);
815            if pre_state == LifecycleState::Stopped && intent == RestoreIntent::Preserve {
816                assert_eq!(events.iter().filter(|e| **e == "stop").count(), 0);
817                assert_eq!(events.iter().filter(|e| **e == "start").count(), 0);
818            }
819            let _ = fs::remove_dir_all(base);
820        }
821    }
822
823    #[test]
824    fn post_install_check_observes_the_complete_new_multi_member_generation() {
825        let (base, root, tx) = setup(false);
826        let service = spec(&root);
827        let mut manager = ScriptedManager::with_service(service.clone(), LifecycleState::Stopped);
828        let receipt = commit_with_lifecycle(
829            &mut manager,
830            &service,
831            tx,
832            absent_ownership(),
833            policy(
834                RestoreIntent::Preserve,
835                PostCommitFailurePolicy::KeepInstalled,
836            ),
837            &CompleteGenerationCheck,
838        )
839        .unwrap();
840        assert_eq!(
841            receipt.artifact_disposition(),
842            TransactionDisposition::Committed
843        );
844        assert!(receipt.post_commit_failure.is_none());
845        let _ = fs::remove_dir_all(base);
846    }
847
848    #[test]
849    fn absent_foreign_unknown_and_transitioning_preconditions_fail_before_artifact_commit() {
850        for kind in 0..4 {
851            let (base, root, tx) = setup(false);
852            let service = spec(&root);
853            let mut manager = ScriptedManager::default();
854            match kind {
855                0 => {} // absent
856                1 => {
857                    let foreign = ServiceSpec::new(
858                        service.id().clone(),
859                        crate::absolute("/foreign"),
860                        service.args().to_vec(),
861                        None,
862                    )
863                    .unwrap();
864                    manager.inner.put(foreign, LifecycleState::Stopped);
865                }
866                2 => manager.inner.put_malformed(service.id().clone()),
867                _ => manager
868                    .inner
869                    .put(service.clone(), LifecycleState::Transitioning),
870            }
871            assert!(matches!(
872                commit_with_lifecycle(
873                    &mut manager,
874                    &service,
875                    tx,
876                    absent_ownership(),
877                    policy(
878                        RestoreIntent::Preserve,
879                        PostCommitFailurePolicy::KeepInstalled
880                    ),
881                    &NoPostInstallCheck
882                ),
883                Err(LifecycleUpdateError::Preflight { .. })
884            ));
885            assert!(!root.join("service.bin").exists());
886            assert_eq!(
887                manager
888                    .events
889                    .borrow()
890                    .iter()
891                    .filter(|e| **e == "stop")
892                    .count(),
893                0
894            );
895            let _ = fs::remove_dir_all(base);
896        }
897    }
898
899    #[test]
900    fn stop_error_incomplete_result_and_ownership_drift_prevent_core_mutation() {
901        for mode in 0..3 {
902            let (base, root, tx) = setup(false);
903            let service = spec(&root);
904            let mut manager =
905                ScriptedManager::with_service(service.clone(), LifecycleState::Running);
906            match mode {
907                0 => manager.fail_stop = true,
908                1 => manager.incomplete_stop = true,
909                _ => manager.drift_to_foreign_after_stop = true,
910            }
911            let result = commit_with_lifecycle(
912                &mut manager,
913                &service,
914                tx,
915                absent_ownership(),
916                policy(
917                    RestoreIntent::Preserve,
918                    PostCommitFailurePolicy::KeepInstalled,
919                ),
920                &NoPostInstallCheck,
921            );
922            assert!(matches!(result, Err(LifecycleUpdateError::Quiesce { .. })));
923            assert!(!root.join("service.bin").exists());
924            let _ = fs::remove_dir_all(base);
925        }
926    }
927
928    #[test]
929    fn keep_installed_check_failure_retains_core_commit_and_service_evidence() {
930        let (base, root, tx) = setup(false);
931        let service = spec(&root);
932        let mut manager = ScriptedManager::with_service(service.clone(), LifecycleState::Running);
933        let check = PassCheck {
934            complete: None,
935            observed_remaining: None,
936            events: None,
937            fail: true,
938        };
939        let receipt = commit_with_lifecycle(
940            &mut manager,
941            &service,
942            tx,
943            absent_ownership(),
944            policy(
945                RestoreIntent::Preserve,
946                PostCommitFailurePolicy::KeepInstalled,
947            ),
948            &check,
949        )
950        .unwrap();
951        assert_eq!(
952            receipt.artifact_disposition(),
953            TransactionDisposition::Committed
954        );
955        assert!(receipt.transaction.post_commit_failure().is_some());
956        assert_eq!(
957            receipt.post_commit_failure.as_ref().unwrap().phase,
958            LifecycleUpdatePhase::PostInstallCheck
959        );
960        assert!(receipt.new_generation_retained());
961        assert_eq!(
962            receipt.final_snapshot.as_ref().unwrap().state,
963            LifecycleState::Running
964        );
965        assert!(root.join("service.bin").exists());
966        let _ = fs::remove_dir_all(base);
967    }
968
969    #[test]
970    fn rollback_check_failure_stops_new_generation_before_core_rollback_then_restores_old_state() {
971        let (base, root, tx) = setup(true);
972        let service = spec(&root);
973        let mut manager = ScriptedManager::with_service(service.clone(), LifecycleState::Running);
974        let events = manager.events.clone();
975        let check = PassCheck {
976            complete: None,
977            observed_remaining: None,
978            events: Some(events.clone()),
979            fail: true,
980        };
981        let verifier = FixedVerifier(CoreOwnership::Owned);
982        let receipt = commit_with_lifecycle(
983            &mut manager,
984            &service,
985            tx,
986            CoreCommitOwnership::new(&verifier, AbsentPolicy::AllowCreate),
987            policy(RestoreIntent::Preserve, PostCommitFailurePolicy::RollBack),
988            &check,
989        )
990        .unwrap();
991        assert_eq!(
992            receipt.artifact_disposition(),
993            TransactionDisposition::RolledBack
994        );
995        assert_eq!(fs::read(root.join("service.bin")).unwrap(), b"old-main");
996        assert_eq!(fs::read(root.join("sidecar.bin")).unwrap(), b"old-sidecar");
997        assert_eq!(
998            receipt.final_snapshot.as_ref().unwrap().state,
999            LifecycleState::Running
1000        );
1001        assert!(receipt.post_commit_failure.is_some());
1002        let events = events.borrow();
1003        let check_i = events.iter().position(|e| *e == "check").unwrap();
1004        let stops_after_check = events
1005            .iter()
1006            .enumerate()
1007            .filter(|(i, e)| *i > check_i && **e == "stop")
1008            .count();
1009        assert_eq!(stops_after_check, 1);
1010        let _ = fs::remove_dir_all(base);
1011    }
1012
1013    #[test]
1014    fn caller_check_panic_is_bounded_and_rollback_quiescence_is_attempted() {
1015        let (base, root, tx) = setup(true);
1016        let service = spec(&root);
1017        let mut manager = ScriptedManager::with_service(service.clone(), LifecycleState::Running);
1018        let called = Rc::new(Cell::new(false));
1019        let receipt = commit_with_lifecycle(
1020            &mut manager,
1021            &service,
1022            tx,
1023            CoreCommitOwnership::new(
1024                &FixedVerifier(CoreOwnership::Owned),
1025                AbsentPolicy::AllowCreate,
1026            ),
1027            policy(RestoreIntent::Preserve, PostCommitFailurePolicy::RollBack),
1028            &PanicCheck {
1029                called: called.clone(),
1030            },
1031        )
1032        .unwrap();
1033        assert!(called.get());
1034        assert_eq!(
1035            receipt.artifact_disposition(),
1036            TransactionDisposition::RolledBack
1037        );
1038        assert_eq!(
1039            receipt.post_commit_failure.as_ref().unwrap().phase,
1040            LifecycleUpdatePhase::PostInstallCheck
1041        );
1042        assert_eq!(
1043            receipt.final_snapshot.as_ref().unwrap().state,
1044            LifecycleState::Running
1045        );
1046        assert!(
1047            manager.stop_calls >= 2,
1048            "old and new generations should each be quiesced"
1049        );
1050        let _ = fs::remove_dir_all(base);
1051    }
1052
1053    #[test]
1054    fn panicking_restore_transition_is_recorded_as_restore_new_not_a_check() {
1055        let (base, root, tx) = setup(true);
1056        let service = spec(&root);
1057        // Running before the update, so pre-commit quiesce stops it and the
1058        // post-commit restore starts it again. That restore-phase `start` is
1059        // the one that panics, so the true phase is RestoreNew.
1060        let mut manager = ScriptedManager::with_service(service.clone(), LifecycleState::Running);
1061        manager.panic_start_calls = vec![1];
1062        let called = Rc::new(Cell::new(false));
1063        let receipt = commit_with_lifecycle(
1064            &mut manager,
1065            &service,
1066            tx,
1067            CoreCommitOwnership::new(
1068                &FixedVerifier(CoreOwnership::Owned),
1069                AbsentPolicy::AllowCreate,
1070            ),
1071            policy(
1072                RestoreIntent::Preserve,
1073                PostCommitFailurePolicy::KeepInstalled,
1074            ),
1075            &PanicCheck {
1076                called: called.clone(),
1077            },
1078        )
1079        .unwrap();
1080        assert!(
1081            !called.get(),
1082            "the post-install check must not run after the restore transition panicked"
1083        );
1084        assert_eq!(
1085            receipt.post_commit_failure.as_ref().unwrap().phase,
1086            LifecycleUpdatePhase::RestoreNew
1087        );
1088        assert_eq!(
1089            receipt.restoration,
1090            crate::LifecycleRestorationStatus::Failed
1091        );
1092        assert!(
1093            !receipt.service_state_restored(),
1094            "a panicking restore must never be reported as Restored"
1095        );
1096        let _ = fs::remove_dir_all(base);
1097    }
1098
1099    #[test]
1100    fn core_pre_receipt_error_restores_pre_update_state() {
1101        let (base, root, tx) = setup(false);
1102        let service = spec(&root);
1103        let mut manager = ScriptedManager::with_service(service.clone(), LifecycleState::Running);
1104        // Core propagates lock-acquisition failure as Err before it can issue a receipt.
1105        fs::write(root.join(".eggup-mutation.lock"), b"contended lock record").unwrap();
1106        let error = commit_with_lifecycle(
1107            &mut manager,
1108            &service,
1109            tx,
1110            CoreCommitOwnership::new(
1111                &FixedVerifier(CoreOwnership::Unknown),
1112                AbsentPolicy::AllowCreate,
1113            ),
1114            policy(
1115                RestoreIntent::EnsureStopped,
1116                PostCommitFailurePolicy::KeepInstalled,
1117            ),
1118            &NoPostInstallCheck,
1119        )
1120        .unwrap_err();
1121        assert!(matches!(
1122            error,
1123            LifecycleUpdateError::CoreBeforeReceipt {
1124                restore_failure: None,
1125                ..
1126            }
1127        ));
1128        assert_eq!(
1129            manager.inner.inspect(&service).unwrap().state,
1130            LifecycleState::Running
1131        );
1132        assert!(!root.join("service.bin").exists());
1133        let _ = fs::remove_dir_all(base);
1134    }
1135
1136    #[test]
1137    fn post_rollback_restore_failure_remains_separate_from_core_rolled_back_receipt() {
1138        let (base, root, tx) = setup(true);
1139        let service = spec(&root);
1140        let mut manager = ScriptedManager::with_service(service.clone(), LifecycleState::Running);
1141        manager.fail_start_calls = vec![1, 2]; // new-generation start, then old-generation restoration
1142        let check = PassCheck {
1143            complete: None,
1144            observed_remaining: None,
1145            events: None,
1146            fail: true,
1147        };
1148        let receipt = commit_with_lifecycle(
1149            &mut manager,
1150            &service,
1151            tx,
1152            CoreCommitOwnership::new(
1153                &FixedVerifier(CoreOwnership::Owned),
1154                AbsentPolicy::AllowCreate,
1155            ),
1156            policy(RestoreIntent::Preserve, PostCommitFailurePolicy::RollBack),
1157            &check,
1158        )
1159        .unwrap();
1160        assert_eq!(
1161            receipt.artifact_disposition(),
1162            TransactionDisposition::RolledBack
1163        );
1164        assert!(receipt.rollback_restore_failure.is_some());
1165        assert_eq!(receipt.restoration, LifecycleRestorationStatus::Failed);
1166        assert_eq!(fs::read(root.join("service.bin")).unwrap(), b"old-main");
1167        let _ = fs::remove_dir_all(base);
1168    }
1169
1170    #[test]
1171    fn recovery_required_suppresses_automatic_old_generation_restart() {
1172        let (base, root, tx) = setup(true);
1173        let service = spec(&root);
1174        let mut manager = ScriptedManager::with_service(service.clone(), LifecycleState::Running);
1175        let receipt = commit_with_lifecycle(
1176            &mut manager,
1177            &service,
1178            tx,
1179            CoreCommitOwnership::new(
1180                &FixedVerifier(CoreOwnership::Owned),
1181                AbsentPolicy::AllowCreate,
1182            ),
1183            policy(RestoreIntent::Preserve, PostCommitFailurePolicy::RollBack),
1184            &SabotageCheck(root.clone()),
1185        )
1186        .unwrap();
1187        assert_eq!(
1188            receipt.artifact_disposition(),
1189            TransactionDisposition::RecoveryRequired
1190        );
1191        assert!(receipt.manual_artifact_recovery_required());
1192        assert_eq!(
1193            receipt.restoration,
1194            LifecycleRestorationStatus::NotAttemptedRecoveryRequired
1195        );
1196        assert_eq!(
1197            manager.start_calls, 1,
1198            "only the new generation may have been started"
1199        );
1200        assert_eq!(
1201            receipt.final_snapshot.as_ref().unwrap().state,
1202            LifecycleState::Stopped
1203        );
1204        let _ = fs::remove_dir_all(base);
1205    }
1206
1207    #[test]
1208    fn one_post_commit_deadline_is_passed_to_the_check_and_final_observation_failure_is_separate() {
1209        let (base, root, tx) = setup(false);
1210        let service = spec(&root);
1211        let complete = Rc::new(Cell::new(false));
1212        let remaining = Rc::new(Cell::new(0));
1213        let mut manager = ScriptedManager::with_service(service.clone(), LifecycleState::Stopped);
1214        manager.start_delay = Duration::from_millis(25);
1215        manager.fail_final_inspect = true;
1216        manager.check_finished = Some(complete.clone());
1217        let check = PassCheck {
1218            complete: Some(complete.clone()),
1219            observed_remaining: Some(remaining.clone()),
1220            events: None,
1221            fail: false,
1222        };
1223        let receipt = commit_with_lifecycle(
1224            &mut manager,
1225            &service,
1226            tx,
1227            absent_ownership(),
1228            policy(
1229                RestoreIntent::EnsureRunning,
1230                PostCommitFailurePolicy::KeepInstalled,
1231            ),
1232            &check,
1233        )
1234        .unwrap();
1235        assert!(remaining.get() < Duration::from_secs(2).as_nanos());
1236        assert!(receipt.observation_failure.is_some());
1237        assert!(receipt.final_snapshot.is_none());
1238        assert_eq!(
1239            receipt.artifact_disposition(),
1240            TransactionDisposition::Committed
1241        );
1242        let _ = fs::remove_dir_all(base);
1243    }
1244
1245    #[test]
1246    fn service_transition_failure_keep_installed_does_not_retry_hidden_policy() {
1247        let (base, root, tx) = setup(false);
1248        let service = spec(&root);
1249        let mut manager = ScriptedManager::with_service(service.clone(), LifecycleState::Running);
1250        manager.fail_start_calls = vec![1];
1251        let receipt = commit_with_lifecycle(
1252            &mut manager,
1253            &service,
1254            tx,
1255            absent_ownership(),
1256            policy(
1257                RestoreIntent::Preserve,
1258                PostCommitFailurePolicy::KeepInstalled,
1259            ),
1260            &NoPostInstallCheck,
1261        )
1262        .unwrap();
1263        assert_eq!(
1264            receipt.artifact_disposition(),
1265            TransactionDisposition::Committed
1266        );
1267        assert_eq!(
1268            receipt.post_commit_failure.as_ref().unwrap().phase,
1269            LifecycleUpdatePhase::RestoreNew
1270        );
1271        assert_eq!(manager.start_calls, 1);
1272        assert_eq!(
1273            receipt.final_snapshot.as_ref().unwrap().state,
1274            LifecycleState::Stopped
1275        );
1276        assert!(root.join("service.bin").exists());
1277        let _ = fs::remove_dir_all(base);
1278    }
1279}