1use crate::{
2 commit_engine::{quiesce_for_rollback, restore_and_check, safe_inspect, safe_transition_to},
3 CommitEnginePolicy, LifecycleFailure, LifecycleSnapshot, LifecycleState, LifecycleUpdatePhase,
4 Ownership, PostInstallCheck, PostInstallCheckError, RestoreIntent, ServiceManager, ServiceSpec,
5};
6use eggup_core::{
7 CommitOwnership, Error as CoreError, PostCommitFailurePolicy, TransactionDisposition,
8 TransactionReceipt, ValidatedTransaction,
9};
10use std::{
11 fmt,
12 panic::{catch_unwind, AssertUnwindSafe},
13 time::{Duration, Instant},
14};
15
16#[derive(Debug, Clone, Copy, PartialEq, Eq)]
18pub enum LifecycleRestorationStatus {
19 Restored,
21 Failed,
23 NotAttemptedRecoveryRequired,
25}
26
27#[derive(Debug, Clone, Copy)]
29pub struct LifecycleUpdatePolicy {
30 pub restore: RestoreIntent,
32 pub post_commit_failure: PostCommitFailurePolicy,
34 pub quiesce_timeout: Duration,
36 pub post_commit_timeout: Duration,
38 pub rollback_restore_timeout: Duration,
40}
41
42impl LifecycleUpdatePolicy {
43 pub fn validate(&self) -> Result<(), LifecycleUpdateError> {
51 for (field, value) in [
52 ("quiesce_timeout", self.quiesce_timeout),
53 ("post_commit_timeout", self.post_commit_timeout),
54 ("rollback_restore_timeout", self.rollback_restore_timeout),
55 ] {
56 if value.is_zero() || value > crate::MAX_TRANSITION_TIMEOUT {
57 return Err(LifecycleUpdateError::InvalidPolicy { field });
58 }
59 }
60 Ok(())
61 }
62}
63
64#[derive(Debug)]
66pub struct LifecycleUpdateReceipt {
67 pub before: LifecycleSnapshot,
69 pub final_snapshot: Option<LifecycleSnapshot>,
71 pub transaction: TransactionReceipt,
73 pub pre_commit_quiesced: bool,
75 pub restoration: LifecycleRestorationStatus,
77 pub post_commit_failure: Option<LifecycleFailure>,
79 pub rollback_quiesce_failure: Option<LifecycleFailure>,
81 pub rollback_restore_failure: Option<LifecycleFailure>,
83 pub observation_failure: Option<LifecycleFailure>,
85 pub engine_policy: CommitEnginePolicy,
91}
92
93impl LifecycleUpdateReceipt {
94 pub fn artifact_disposition(&self) -> TransactionDisposition {
96 self.transaction.disposition()
97 }
98
99 pub fn manual_artifact_recovery_required(&self) -> bool {
101 self.transaction.disposition() == TransactionDisposition::RecoveryRequired
102 }
103
104 pub fn new_generation_retained(&self) -> bool {
106 self.transaction.disposition() == TransactionDisposition::Committed
107 }
108
109 pub fn service_state_restored(&self) -> bool {
111 self.restoration == LifecycleRestorationStatus::Restored
112 }
113}
114
115#[derive(Debug)]
117#[non_exhaustive]
118pub enum LifecycleUpdateError {
119 InvalidPolicy {
121 field: &'static str,
123 },
124 Preflight {
126 failure: LifecycleFailure,
128 },
129 Quiesce {
131 failure: LifecycleFailure,
133 restore_failure: Option<LifecycleFailure>,
135 },
136 CoreBeforeReceipt {
138 core: CoreError,
140 restore_failure: Option<LifecycleFailure>,
142 },
143}
144
145impl fmt::Display for LifecycleUpdateError {
146 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
147 match self {
148 Self::InvalidPolicy { field } => write!(f, "invalid lifecycle update timeout: {field}"),
149 Self::Preflight { failure } => write!(f, "lifecycle preflight failed: {failure:?}"),
150 Self::Quiesce { failure, .. } => write!(f, "service quiescence failed: {failure:?}"),
151 Self::CoreBeforeReceipt { core, .. } => {
152 write!(f, "Core returned before a transaction receipt: {core}")
153 }
154 }
155 }
156}
157impl std::error::Error for LifecycleUpdateError {}
158
159pub fn commit_with_lifecycle<M: ServiceManager, C: PostInstallCheck>(
181 manager: &mut M,
182 spec: &ServiceSpec,
183 transaction: ValidatedTransaction,
184 ownership: CommitOwnership<'_>,
185 policy: LifecycleUpdatePolicy,
186 check: &C,
187) -> Result<LifecycleUpdateReceipt, LifecycleUpdateError> {
188 commit_with_lifecycle_with_engine_policy(
189 manager,
190 spec,
191 transaction,
192 ownership,
193 policy,
194 check,
195 CommitEnginePolicy::default(),
196 )
197}
198
199pub fn commit_with_lifecycle_with_engine_policy<M: ServiceManager, C: PostInstallCheck>(
215 manager: &mut M,
216 spec: &ServiceSpec,
217 transaction: ValidatedTransaction,
218 ownership: CommitOwnership<'_>,
219 policy: LifecycleUpdatePolicy,
220 check: &C,
221 engine: CommitEnginePolicy,
222) -> Result<LifecycleUpdateReceipt, LifecycleUpdateError> {
223 policy.validate()?;
224 let before = safe_inspect(manager, spec, LifecycleUpdatePhase::Inspect)
225 .map_err(|failure| LifecycleUpdateError::Preflight { failure })?;
226 if before.ownership != Ownership::Owned
227 || !matches!(
228 before.state,
229 LifecycleState::Running | LifecycleState::Stopped
230 )
231 {
232 return Err(LifecycleUpdateError::Preflight {
233 failure: LifecycleFailure::new(
234 spec,
235 LifecycleUpdatePhase::Inspect,
236 "service must be Owned and Running or Stopped",
237 Some(&before),
238 ),
239 });
240 }
241
242 let mut pre_commit_quiesced = false;
243 if before.state == LifecycleState::Running {
244 let stop_failure =
245 crate::commit_engine::quiesce_service(manager, spec, &before, policy.quiesce_timeout)
246 .err();
247 if let Some(failure) = stop_failure {
248 let restore_failure = safe_transition_to(
249 manager,
250 spec,
251 LifecycleState::Running,
252 policy.rollback_restore_timeout,
253 LifecycleUpdatePhase::Quiesce,
254 &engine,
255 )
256 .err();
257 return Err(LifecycleUpdateError::Quiesce {
258 failure,
259 restore_failure,
260 });
261 }
262 pre_commit_quiesced = true;
263 }
264
265 let mut post_commit_failure = None;
266 let mut rollback_quiesce_failure = None;
267 let core_result =
268 transaction.commit_with_post_commit(ownership, policy.post_commit_failure, || {
269 let deadline = Instant::now() + policy.post_commit_timeout;
270 let work = catch_unwind(AssertUnwindSafe(|| {
271 restore_success_state(
272 manager,
273 spec,
274 &before,
275 policy.restore,
276 check,
277 deadline,
278 &engine,
279 )
280 }));
281 let failure = match work {
282 Ok(Ok(())) => return Ok(()),
283 Ok(Err(failure)) => failure,
284 Err(_) => LifecycleFailure::new(
285 spec,
286 LifecycleUpdatePhase::PostInstallCheck,
287 "post-commit service work panicked",
288 None,
289 ),
290 };
291 post_commit_failure = Some(failure.clone());
292
293 if policy.post_commit_failure == PostCommitFailurePolicy::RollBack {
294 rollback_quiesce_failure = match catch_unwind(AssertUnwindSafe(|| {
295 quiesce_for_rollback(manager, spec, deadline)
296 })) {
297 Ok(failure) => failure,
298 Err(_) => Some(LifecycleFailure::new(
299 spec,
300 LifecycleUpdatePhase::QuiesceForRollback,
301 "service manager panicked while quiescing before artifact rollback",
302 None,
303 )),
304 };
305 }
306 Err(PostInstallCheckError::new(failure.detail))
307 });
308
309 let transaction = match core_result {
310 Ok(receipt) => receipt,
311 Err(core) => {
312 let restore_failure = if pre_commit_quiesced {
313 safe_transition_to(
314 manager,
315 spec,
316 before.state,
317 policy.rollback_restore_timeout,
318 LifecycleUpdatePhase::RestoreOld,
319 &engine,
320 )
321 .err()
322 } else {
323 None
324 };
325 return Err(LifecycleUpdateError::CoreBeforeReceipt {
326 core,
327 restore_failure,
328 });
329 }
330 };
331
332 let mut rollback_restore_failure = None;
333 let mut restoration = LifecycleRestorationStatus::Restored;
334 match transaction.disposition() {
335 TransactionDisposition::Committed => {
336 if post_commit_failure
337 .as_ref()
338 .is_some_and(|f| f.phase == LifecycleUpdatePhase::RestoreNew)
339 {
340 restoration = LifecycleRestorationStatus::Failed;
341 }
342 }
343 TransactionDisposition::RolledBack => {
344 if let Err(failure) = safe_transition_to(
345 manager,
346 spec,
347 before.state,
348 policy.rollback_restore_timeout,
349 LifecycleUpdatePhase::RestoreOld,
350 &engine,
351 ) {
352 rollback_restore_failure = Some(failure);
353 restoration = LifecycleRestorationStatus::Failed;
354 }
355 }
356 TransactionDisposition::RecoveryRequired => {
357 restoration = LifecycleRestorationStatus::NotAttemptedRecoveryRequired;
358 }
359 }
360
361 let final_snapshot = match safe_inspect(manager, spec, LifecycleUpdatePhase::FinalInspect) {
362 Ok(snapshot) => Some(snapshot),
363 Err(observation_failure) => {
364 return Ok(LifecycleUpdateReceipt {
366 before,
367 final_snapshot: None,
368 transaction,
369 pre_commit_quiesced,
370 restoration,
371 post_commit_failure,
372 rollback_quiesce_failure,
373 rollback_restore_failure,
374 observation_failure: Some(observation_failure),
375 engine_policy: engine,
376 });
377 }
378 };
379
380 Ok(LifecycleUpdateReceipt {
381 before,
382 final_snapshot,
383 transaction,
384 pre_commit_quiesced,
385 restoration,
386 post_commit_failure,
387 rollback_quiesce_failure,
388 rollback_restore_failure,
389 observation_failure: None,
390 engine_policy: engine,
391 })
392}
393
394fn restore_success_state<M: ServiceManager, C: PostInstallCheck>(
395 manager: &mut M,
396 spec: &ServiceSpec,
397 before: &LifecycleSnapshot,
398 intent: RestoreIntent,
399 check: &C,
400 deadline: Instant,
401 engine: &CommitEnginePolicy,
402) -> Result<(), LifecycleFailure> {
403 let wanted = match intent {
404 RestoreIntent::Preserve => before.state,
405 RestoreIntent::EnsureRunning => LifecycleState::Running,
406 RestoreIntent::EnsureStopped => LifecycleState::Stopped,
407 };
408 restore_and_check(manager, spec, wanted, check, deadline, engine)
416}
417
418#[cfg(test)]
419mod tests {
420 use super::*;
421 use crate::{
422 ManagerFailureKind, NoPostInstallCheck, ServiceError, ServiceId, ServiceOperation,
423 TestDoubleManager, TransitionResult,
424 };
425 use eggup_core::{
426 AbsentOnlyVerifier, AbsentPolicy, AllValidators, ArtifactMember, ArtifactSet,
427 CommitOwnership as CoreCommitOwnership, InstallPlan, MemberId, Ownership as CoreOwnership,
428 PermissionsIntent, ProductId, ReleaseId,
429 };
430 use std::{
431 cell::{Cell, RefCell},
432 fs,
433 path::{Path, PathBuf},
434 rc::Rc,
435 sync::atomic::{AtomicU64, Ordering},
436 thread,
437 };
438
439 static NEXT: AtomicU64 = AtomicU64::new(0);
440
441 #[test]
442 fn lifecycle_failure_detail_is_utf8_safe_bounded_and_control_free() {
443 let spec = ServiceSpec::new(
444 ServiceId::new("diagnostic").unwrap(),
445 std::env::current_exe().unwrap(),
446 Vec::new(),
447 None,
448 )
449 .unwrap();
450 let mut input = "a".repeat(510);
451 input.push('๐งก');
452 input.push_str("\nmanager detail");
453 let failure = LifecycleFailure::new(&spec, LifecycleUpdatePhase::Inspect, input, None);
454 assert!(failure.detail.len() <= 512);
455 assert!(failure.detail.is_char_boundary(failure.detail.len()));
456 assert!(!failure.detail.chars().any(char::is_control));
457 }
458
459 struct FixedVerifier(CoreOwnership);
460 impl eggup_core::OwnershipVerifier for FixedVerifier {
461 fn verify(&self, _member: &MemberId, _destination: &Path) -> CoreOwnership {
462 self.0
463 }
464 }
465
466 #[derive(Default)]
467 struct ScriptedManager {
468 inner: TestDoubleManager,
469 events: Rc<RefCell<Vec<&'static str>>>,
470 fail_stop: bool,
471 incomplete_stop: bool,
472 fail_start_calls: Vec<usize>,
473 panic_start_calls: Vec<usize>,
474 start_calls: usize,
475 stop_calls: usize,
476 drift_to_foreign_after_stop: bool,
477 fail_final_inspect: bool,
478 check_finished: Option<Rc<Cell<bool>>>,
479 final_inspect_failed: Cell<bool>,
480 start_delay: Duration,
481 }
482
483 impl ScriptedManager {
484 fn with_service(spec: ServiceSpec, state: LifecycleState) -> Self {
485 let mut inner = TestDoubleManager::new();
486 inner.put(spec, state);
487 Self {
488 inner,
489 ..Self::default()
490 }
491 }
492 }
493
494 impl ServiceManager for ScriptedManager {
495 fn inspect(&self, spec: &ServiceSpec) -> Result<LifecycleSnapshot, ServiceError> {
496 self.events.borrow_mut().push("inspect");
497 if self.fail_final_inspect
498 && self.check_finished.as_ref().is_some_and(|f| f.get())
499 && !self.final_inspect_failed.get()
500 {
501 self.final_inspect_failed.set(true);
503 return Err(ServiceError::manager(
504 ManagerFailureKind::Protocol,
505 "injected final inspection failure",
506 ));
507 }
508 self.inner.inspect(spec)
509 }
510
511 fn install(&mut self, spec: &ServiceSpec) -> Result<TransitionResult, ServiceError> {
512 self.events.borrow_mut().push("install");
513 self.inner.install(spec)
514 }
515
516 fn start(
517 &mut self,
518 spec: &ServiceSpec,
519 timeout: Duration,
520 ) -> Result<TransitionResult, ServiceError> {
521 self.events.borrow_mut().push("start");
522 self.start_calls += 1;
523 if self.start_delay > Duration::ZERO {
524 thread::sleep(self.start_delay);
525 }
526 if self.fail_start_calls.contains(&self.start_calls) {
527 return Err(ServiceError::manager(
528 ManagerFailureKind::Protocol,
529 "injected start failure",
530 ));
531 }
532 if self.panic_start_calls.contains(&self.start_calls) {
533 panic!("test panic payload must not escape");
534 }
535 self.inner.start(spec, timeout)
536 }
537
538 fn stop(
539 &mut self,
540 spec: &ServiceSpec,
541 timeout: Duration,
542 ) -> Result<TransitionResult, ServiceError> {
543 self.events.borrow_mut().push("stop");
544 self.stop_calls += 1;
545 if self.fail_stop {
546 return Err(ServiceError::manager(
547 ManagerFailureKind::Protocol,
548 "injected stop failure",
549 ));
550 }
551 if self.incomplete_stop {
552 return Ok(TransitionResult {
553 operation: ServiceOperation::Stop,
554 completed: false,
555 detail: "injected incomplete stop".into(),
556 });
557 }
558 let result = self.inner.stop(spec, timeout)?;
559 if self.drift_to_foreign_after_stop {
560 let foreign = ServiceSpec::new(
561 spec.id().clone(),
562 crate::absolute("/different/service-executable"),
563 spec.args().to_vec(),
564 spec.config().map(Path::to_path_buf),
565 )?;
566 self.inner.put(foreign, LifecycleState::Stopped);
567 }
568 Ok(result)
569 }
570
571 fn restart(
572 &mut self,
573 spec: &ServiceSpec,
574 timeout: Duration,
575 ) -> Result<TransitionResult, ServiceError> {
576 self.events.borrow_mut().push("restart");
577 self.inner.restart(spec, timeout)
578 }
579
580 fn uninstall(&mut self, spec: &ServiceSpec) -> Result<TransitionResult, ServiceError> {
581 self.events.borrow_mut().push("uninstall");
582 self.inner.uninstall(spec)
583 }
584 }
585
586 fn setup(old_generation: bool) -> (PathBuf, PathBuf, ValidatedTransaction) {
587 let base = std::env::temp_dir().join(format!(
588 "eggup-service-lifecycle-update-{}-{}",
589 std::process::id(),
590 NEXT.fetch_add(1, Ordering::Relaxed)
591 ));
592 let source_root = base.join("input");
593 let install_root = base.join("install");
594 fs::create_dir_all(&source_root).unwrap();
595 fs::create_dir_all(&install_root).unwrap();
596 let main = source_root.join("main");
597 let sidecar = source_root.join("sidecar");
598 fs::write(&main, b"new-main").unwrap();
599 fs::write(&sidecar, b"new-sidecar").unwrap();
600 if old_generation {
601 fs::write(install_root.join("service.bin"), b"old-main").unwrap();
602 fs::write(install_root.join("sidecar.bin"), b"old-sidecar").unwrap();
603 }
604 let members = ArtifactSet::new(vec![
605 ArtifactMember::new(MemberId::new("main").unwrap(), &main, "service.bin")
606 .unwrap()
607 .with_permissions(PermissionsIntent::Executable)
608 .with_integrity(eggup_core::IntegrityRequirement::Sha256(
609 eggup_core::hash_file(&main).unwrap(),
610 )),
611 ArtifactMember::new(MemberId::new("sidecar").unwrap(), &sidecar, "sidecar.bin")
612 .unwrap()
613 .with_integrity(eggup_core::IntegrityRequirement::Sha256(
614 eggup_core::hash_file(&sidecar).unwrap(),
615 )),
616 ])
617 .unwrap();
618 let prepared = InstallPlan::new(
619 ProductId::new("service-test").unwrap(),
620 ReleaseId::new("r2").unwrap(),
621 &install_root,
622 members,
623 )
624 .unwrap()
625 .prepare()
626 .unwrap();
627 let validated = prepared
628 .verify_integrity()
629 .unwrap()
630 .validate(&AllValidators::new())
631 .unwrap();
632 (base, install_root, validated)
633 }
634
635 fn spec(install_root: &Path) -> ServiceSpec {
636 ServiceSpec::new(
637 ServiceId::new("m005-test-service").unwrap(),
638 install_root.join("service.bin"),
639 vec!["--worker".into()],
640 None,
641 )
642 .unwrap()
643 }
644
645 fn policy(restore: RestoreIntent, failure: PostCommitFailurePolicy) -> LifecycleUpdatePolicy {
646 LifecycleUpdatePolicy {
647 restore,
648 post_commit_failure: failure,
649 quiesce_timeout: Duration::from_secs(2),
650 post_commit_timeout: Duration::from_secs(2),
651 rollback_restore_timeout: Duration::from_secs(2),
652 }
653 }
654
655 #[derive(Debug)]
656 struct PassCheck {
657 complete: Option<Rc<Cell<bool>>>,
658 observed_remaining: Option<Rc<Cell<u128>>>,
659 events: Option<Rc<RefCell<Vec<&'static str>>>>,
660 fail: bool,
661 }
662 impl PostInstallCheck for PassCheck {
663 fn check(
664 &self,
665 _spec: &ServiceSpec,
666 _snapshot: &LifecycleSnapshot,
667 remaining: Duration,
668 ) -> Result<(), PostInstallCheckError> {
669 if let Some(events) = &self.events {
670 events.borrow_mut().push("check");
671 }
672 if let Some(value) = &self.observed_remaining {
673 value.set(remaining.as_nanos());
674 }
675 if let Some(complete) = &self.complete {
676 complete.set(true);
677 }
678 if self.fail {
679 Err(PostInstallCheckError::new("injected health check failure"))
680 } else {
681 Ok(())
682 }
683 }
684 }
685
686 #[derive(Debug)]
687 struct CompleteGenerationCheck;
688 impl PostInstallCheck for CompleteGenerationCheck {
689 fn check(
690 &self,
691 spec: &ServiceSpec,
692 _snapshot: &LifecycleSnapshot,
693 _remaining: Duration,
694 ) -> Result<(), PostInstallCheckError> {
695 let root = spec
696 .executable()
697 .parent()
698 .expect("service executable parent");
699 if fs::read(root.join("service.bin")).ok().as_deref() == Some(b"new-main")
700 && fs::read(root.join("sidecar.bin")).ok().as_deref() == Some(b"new-sidecar")
701 {
702 Ok(())
703 } else {
704 Err(PostInstallCheckError::new(
705 "complete artifact generation is not visible",
706 ))
707 }
708 }
709 }
710
711 #[derive(Debug)]
712 struct PanicCheck {
713 called: Rc<Cell<bool>>,
714 }
715 impl PostInstallCheck for PanicCheck {
716 fn check(
717 &self,
718 _spec: &ServiceSpec,
719 _snapshot: &LifecycleSnapshot,
720 _remaining: Duration,
721 ) -> Result<(), PostInstallCheckError> {
722 self.called.set(true);
723 panic!("test panic payload must not escape");
724 }
725 }
726
727 #[derive(Debug)]
728 struct SabotageCheck(PathBuf);
729 impl PostInstallCheck for SabotageCheck {
730 fn check(
731 &self,
732 _spec: &ServiceSpec,
733 _snapshot: &LifecycleSnapshot,
734 _remaining: Duration,
735 ) -> Result<(), PostInstallCheckError> {
736 let dest = self.0.join("service.bin");
737 fs::remove_file(&dest).unwrap();
738 fs::create_dir(&dest).unwrap();
739 Err(PostInstallCheckError::new("injected rollback failure"))
740 }
741 }
742
743 fn absent_ownership() -> CoreCommitOwnership<'static> {
744 static VERIFIER: AbsentOnlyVerifier = AbsentOnlyVerifier;
746 CoreCommitOwnership::new(&VERIFIER, AbsentPolicy::AllowCreate)
747 }
748
749 #[test]
750 fn rejects_zero_and_overlong_budgets_before_mutation() {
751 let mut invalid = policy(
752 RestoreIntent::Preserve,
753 PostCommitFailurePolicy::KeepInstalled,
754 );
755 invalid.quiesce_timeout = Duration::ZERO;
756 assert!(matches!(
757 invalid.validate(),
758 Err(LifecycleUpdateError::InvalidPolicy { .. })
759 ));
760 invalid.quiesce_timeout = Duration::from_secs(1);
761 invalid.post_commit_timeout = crate::MAX_TRANSITION_TIMEOUT + Duration::from_nanos(1);
762 assert!(matches!(
763 invalid.validate(),
764 Err(LifecycleUpdateError::InvalidPolicy { .. })
765 ));
766 }
767
768 #[test]
769 fn preserve_and_explicit_restore_intents_produce_expected_success_state() {
770 for (pre_state, intent, expected) in [
771 (
772 LifecycleState::Running,
773 RestoreIntent::Preserve,
774 LifecycleState::Running,
775 ),
776 (
777 LifecycleState::Stopped,
778 RestoreIntent::Preserve,
779 LifecycleState::Stopped,
780 ),
781 (
782 LifecycleState::Stopped,
783 RestoreIntent::EnsureRunning,
784 LifecycleState::Running,
785 ),
786 (
787 LifecycleState::Running,
788 RestoreIntent::EnsureStopped,
789 LifecycleState::Stopped,
790 ),
791 ] {
792 let (base, root, tx) = setup(false);
793 let service = spec(&root);
794 let mut manager = ScriptedManager::with_service(service.clone(), pre_state);
795 let receipt = commit_with_lifecycle(
796 &mut manager,
797 &service,
798 tx,
799 absent_ownership(),
800 policy(intent, PostCommitFailurePolicy::KeepInstalled),
801 &NoPostInstallCheck,
802 )
803 .unwrap();
804 assert_eq!(
805 receipt.artifact_disposition(),
806 TransactionDisposition::Committed
807 );
808 assert_eq!(receipt.final_snapshot.as_ref().unwrap().state, expected);
809 assert_eq!(
810 receipt.pre_commit_quiesced,
811 pre_state == LifecycleState::Running
812 );
813 let events = manager.events.borrow();
814 assert_eq!(events.iter().filter(|e| **e == "install").count(), 0);
815 if pre_state == LifecycleState::Stopped && intent == RestoreIntent::Preserve {
816 assert_eq!(events.iter().filter(|e| **e == "stop").count(), 0);
817 assert_eq!(events.iter().filter(|e| **e == "start").count(), 0);
818 }
819 let _ = fs::remove_dir_all(base);
820 }
821 }
822
823 #[test]
824 fn post_install_check_observes_the_complete_new_multi_member_generation() {
825 let (base, root, tx) = setup(false);
826 let service = spec(&root);
827 let mut manager = ScriptedManager::with_service(service.clone(), LifecycleState::Stopped);
828 let receipt = commit_with_lifecycle(
829 &mut manager,
830 &service,
831 tx,
832 absent_ownership(),
833 policy(
834 RestoreIntent::Preserve,
835 PostCommitFailurePolicy::KeepInstalled,
836 ),
837 &CompleteGenerationCheck,
838 )
839 .unwrap();
840 assert_eq!(
841 receipt.artifact_disposition(),
842 TransactionDisposition::Committed
843 );
844 assert!(receipt.post_commit_failure.is_none());
845 let _ = fs::remove_dir_all(base);
846 }
847
848 #[test]
849 fn absent_foreign_unknown_and_transitioning_preconditions_fail_before_artifact_commit() {
850 for kind in 0..4 {
851 let (base, root, tx) = setup(false);
852 let service = spec(&root);
853 let mut manager = ScriptedManager::default();
854 match kind {
855 0 => {} 1 => {
857 let foreign = ServiceSpec::new(
858 service.id().clone(),
859 crate::absolute("/foreign"),
860 service.args().to_vec(),
861 None,
862 )
863 .unwrap();
864 manager.inner.put(foreign, LifecycleState::Stopped);
865 }
866 2 => manager.inner.put_malformed(service.id().clone()),
867 _ => manager
868 .inner
869 .put(service.clone(), LifecycleState::Transitioning),
870 }
871 assert!(matches!(
872 commit_with_lifecycle(
873 &mut manager,
874 &service,
875 tx,
876 absent_ownership(),
877 policy(
878 RestoreIntent::Preserve,
879 PostCommitFailurePolicy::KeepInstalled
880 ),
881 &NoPostInstallCheck
882 ),
883 Err(LifecycleUpdateError::Preflight { .. })
884 ));
885 assert!(!root.join("service.bin").exists());
886 assert_eq!(
887 manager
888 .events
889 .borrow()
890 .iter()
891 .filter(|e| **e == "stop")
892 .count(),
893 0
894 );
895 let _ = fs::remove_dir_all(base);
896 }
897 }
898
899 #[test]
900 fn stop_error_incomplete_result_and_ownership_drift_prevent_core_mutation() {
901 for mode in 0..3 {
902 let (base, root, tx) = setup(false);
903 let service = spec(&root);
904 let mut manager =
905 ScriptedManager::with_service(service.clone(), LifecycleState::Running);
906 match mode {
907 0 => manager.fail_stop = true,
908 1 => manager.incomplete_stop = true,
909 _ => manager.drift_to_foreign_after_stop = true,
910 }
911 let result = commit_with_lifecycle(
912 &mut manager,
913 &service,
914 tx,
915 absent_ownership(),
916 policy(
917 RestoreIntent::Preserve,
918 PostCommitFailurePolicy::KeepInstalled,
919 ),
920 &NoPostInstallCheck,
921 );
922 assert!(matches!(result, Err(LifecycleUpdateError::Quiesce { .. })));
923 assert!(!root.join("service.bin").exists());
924 let _ = fs::remove_dir_all(base);
925 }
926 }
927
928 #[test]
929 fn keep_installed_check_failure_retains_core_commit_and_service_evidence() {
930 let (base, root, tx) = setup(false);
931 let service = spec(&root);
932 let mut manager = ScriptedManager::with_service(service.clone(), LifecycleState::Running);
933 let check = PassCheck {
934 complete: None,
935 observed_remaining: None,
936 events: None,
937 fail: true,
938 };
939 let receipt = commit_with_lifecycle(
940 &mut manager,
941 &service,
942 tx,
943 absent_ownership(),
944 policy(
945 RestoreIntent::Preserve,
946 PostCommitFailurePolicy::KeepInstalled,
947 ),
948 &check,
949 )
950 .unwrap();
951 assert_eq!(
952 receipt.artifact_disposition(),
953 TransactionDisposition::Committed
954 );
955 assert!(receipt.transaction.post_commit_failure().is_some());
956 assert_eq!(
957 receipt.post_commit_failure.as_ref().unwrap().phase,
958 LifecycleUpdatePhase::PostInstallCheck
959 );
960 assert!(receipt.new_generation_retained());
961 assert_eq!(
962 receipt.final_snapshot.as_ref().unwrap().state,
963 LifecycleState::Running
964 );
965 assert!(root.join("service.bin").exists());
966 let _ = fs::remove_dir_all(base);
967 }
968
969 #[test]
970 fn rollback_check_failure_stops_new_generation_before_core_rollback_then_restores_old_state() {
971 let (base, root, tx) = setup(true);
972 let service = spec(&root);
973 let mut manager = ScriptedManager::with_service(service.clone(), LifecycleState::Running);
974 let events = manager.events.clone();
975 let check = PassCheck {
976 complete: None,
977 observed_remaining: None,
978 events: Some(events.clone()),
979 fail: true,
980 };
981 let verifier = FixedVerifier(CoreOwnership::Owned);
982 let receipt = commit_with_lifecycle(
983 &mut manager,
984 &service,
985 tx,
986 CoreCommitOwnership::new(&verifier, AbsentPolicy::AllowCreate),
987 policy(RestoreIntent::Preserve, PostCommitFailurePolicy::RollBack),
988 &check,
989 )
990 .unwrap();
991 assert_eq!(
992 receipt.artifact_disposition(),
993 TransactionDisposition::RolledBack
994 );
995 assert_eq!(fs::read(root.join("service.bin")).unwrap(), b"old-main");
996 assert_eq!(fs::read(root.join("sidecar.bin")).unwrap(), b"old-sidecar");
997 assert_eq!(
998 receipt.final_snapshot.as_ref().unwrap().state,
999 LifecycleState::Running
1000 );
1001 assert!(receipt.post_commit_failure.is_some());
1002 let events = events.borrow();
1003 let check_i = events.iter().position(|e| *e == "check").unwrap();
1004 let stops_after_check = events
1005 .iter()
1006 .enumerate()
1007 .filter(|(i, e)| *i > check_i && **e == "stop")
1008 .count();
1009 assert_eq!(stops_after_check, 1);
1010 let _ = fs::remove_dir_all(base);
1011 }
1012
1013 #[test]
1014 fn caller_check_panic_is_bounded_and_rollback_quiescence_is_attempted() {
1015 let (base, root, tx) = setup(true);
1016 let service = spec(&root);
1017 let mut manager = ScriptedManager::with_service(service.clone(), LifecycleState::Running);
1018 let called = Rc::new(Cell::new(false));
1019 let receipt = commit_with_lifecycle(
1020 &mut manager,
1021 &service,
1022 tx,
1023 CoreCommitOwnership::new(
1024 &FixedVerifier(CoreOwnership::Owned),
1025 AbsentPolicy::AllowCreate,
1026 ),
1027 policy(RestoreIntent::Preserve, PostCommitFailurePolicy::RollBack),
1028 &PanicCheck {
1029 called: called.clone(),
1030 },
1031 )
1032 .unwrap();
1033 assert!(called.get());
1034 assert_eq!(
1035 receipt.artifact_disposition(),
1036 TransactionDisposition::RolledBack
1037 );
1038 assert_eq!(
1039 receipt.post_commit_failure.as_ref().unwrap().phase,
1040 LifecycleUpdatePhase::PostInstallCheck
1041 );
1042 assert_eq!(
1043 receipt.final_snapshot.as_ref().unwrap().state,
1044 LifecycleState::Running
1045 );
1046 assert!(
1047 manager.stop_calls >= 2,
1048 "old and new generations should each be quiesced"
1049 );
1050 let _ = fs::remove_dir_all(base);
1051 }
1052
1053 #[test]
1054 fn panicking_restore_transition_is_recorded_as_restore_new_not_a_check() {
1055 let (base, root, tx) = setup(true);
1056 let service = spec(&root);
1057 let mut manager = ScriptedManager::with_service(service.clone(), LifecycleState::Running);
1061 manager.panic_start_calls = vec![1];
1062 let called = Rc::new(Cell::new(false));
1063 let receipt = commit_with_lifecycle(
1064 &mut manager,
1065 &service,
1066 tx,
1067 CoreCommitOwnership::new(
1068 &FixedVerifier(CoreOwnership::Owned),
1069 AbsentPolicy::AllowCreate,
1070 ),
1071 policy(
1072 RestoreIntent::Preserve,
1073 PostCommitFailurePolicy::KeepInstalled,
1074 ),
1075 &PanicCheck {
1076 called: called.clone(),
1077 },
1078 )
1079 .unwrap();
1080 assert!(
1081 !called.get(),
1082 "the post-install check must not run after the restore transition panicked"
1083 );
1084 assert_eq!(
1085 receipt.post_commit_failure.as_ref().unwrap().phase,
1086 LifecycleUpdatePhase::RestoreNew
1087 );
1088 assert_eq!(
1089 receipt.restoration,
1090 crate::LifecycleRestorationStatus::Failed
1091 );
1092 assert!(
1093 !receipt.service_state_restored(),
1094 "a panicking restore must never be reported as Restored"
1095 );
1096 let _ = fs::remove_dir_all(base);
1097 }
1098
1099 #[test]
1100 fn core_pre_receipt_error_restores_pre_update_state() {
1101 let (base, root, tx) = setup(false);
1102 let service = spec(&root);
1103 let mut manager = ScriptedManager::with_service(service.clone(), LifecycleState::Running);
1104 fs::write(root.join(".eggup-mutation.lock"), b"contended lock record").unwrap();
1106 let error = commit_with_lifecycle(
1107 &mut manager,
1108 &service,
1109 tx,
1110 CoreCommitOwnership::new(
1111 &FixedVerifier(CoreOwnership::Unknown),
1112 AbsentPolicy::AllowCreate,
1113 ),
1114 policy(
1115 RestoreIntent::EnsureStopped,
1116 PostCommitFailurePolicy::KeepInstalled,
1117 ),
1118 &NoPostInstallCheck,
1119 )
1120 .unwrap_err();
1121 assert!(matches!(
1122 error,
1123 LifecycleUpdateError::CoreBeforeReceipt {
1124 restore_failure: None,
1125 ..
1126 }
1127 ));
1128 assert_eq!(
1129 manager.inner.inspect(&service).unwrap().state,
1130 LifecycleState::Running
1131 );
1132 assert!(!root.join("service.bin").exists());
1133 let _ = fs::remove_dir_all(base);
1134 }
1135
1136 #[test]
1137 fn post_rollback_restore_failure_remains_separate_from_core_rolled_back_receipt() {
1138 let (base, root, tx) = setup(true);
1139 let service = spec(&root);
1140 let mut manager = ScriptedManager::with_service(service.clone(), LifecycleState::Running);
1141 manager.fail_start_calls = vec![1, 2]; let check = PassCheck {
1143 complete: None,
1144 observed_remaining: None,
1145 events: None,
1146 fail: true,
1147 };
1148 let receipt = commit_with_lifecycle(
1149 &mut manager,
1150 &service,
1151 tx,
1152 CoreCommitOwnership::new(
1153 &FixedVerifier(CoreOwnership::Owned),
1154 AbsentPolicy::AllowCreate,
1155 ),
1156 policy(RestoreIntent::Preserve, PostCommitFailurePolicy::RollBack),
1157 &check,
1158 )
1159 .unwrap();
1160 assert_eq!(
1161 receipt.artifact_disposition(),
1162 TransactionDisposition::RolledBack
1163 );
1164 assert!(receipt.rollback_restore_failure.is_some());
1165 assert_eq!(receipt.restoration, LifecycleRestorationStatus::Failed);
1166 assert_eq!(fs::read(root.join("service.bin")).unwrap(), b"old-main");
1167 let _ = fs::remove_dir_all(base);
1168 }
1169
1170 #[test]
1171 fn recovery_required_suppresses_automatic_old_generation_restart() {
1172 let (base, root, tx) = setup(true);
1173 let service = spec(&root);
1174 let mut manager = ScriptedManager::with_service(service.clone(), LifecycleState::Running);
1175 let receipt = commit_with_lifecycle(
1176 &mut manager,
1177 &service,
1178 tx,
1179 CoreCommitOwnership::new(
1180 &FixedVerifier(CoreOwnership::Owned),
1181 AbsentPolicy::AllowCreate,
1182 ),
1183 policy(RestoreIntent::Preserve, PostCommitFailurePolicy::RollBack),
1184 &SabotageCheck(root.clone()),
1185 )
1186 .unwrap();
1187 assert_eq!(
1188 receipt.artifact_disposition(),
1189 TransactionDisposition::RecoveryRequired
1190 );
1191 assert!(receipt.manual_artifact_recovery_required());
1192 assert_eq!(
1193 receipt.restoration,
1194 LifecycleRestorationStatus::NotAttemptedRecoveryRequired
1195 );
1196 assert_eq!(
1197 manager.start_calls, 1,
1198 "only the new generation may have been started"
1199 );
1200 assert_eq!(
1201 receipt.final_snapshot.as_ref().unwrap().state,
1202 LifecycleState::Stopped
1203 );
1204 let _ = fs::remove_dir_all(base);
1205 }
1206
1207 #[test]
1208 fn one_post_commit_deadline_is_passed_to_the_check_and_final_observation_failure_is_separate() {
1209 let (base, root, tx) = setup(false);
1210 let service = spec(&root);
1211 let complete = Rc::new(Cell::new(false));
1212 let remaining = Rc::new(Cell::new(0));
1213 let mut manager = ScriptedManager::with_service(service.clone(), LifecycleState::Stopped);
1214 manager.start_delay = Duration::from_millis(25);
1215 manager.fail_final_inspect = true;
1216 manager.check_finished = Some(complete.clone());
1217 let check = PassCheck {
1218 complete: Some(complete.clone()),
1219 observed_remaining: Some(remaining.clone()),
1220 events: None,
1221 fail: false,
1222 };
1223 let receipt = commit_with_lifecycle(
1224 &mut manager,
1225 &service,
1226 tx,
1227 absent_ownership(),
1228 policy(
1229 RestoreIntent::EnsureRunning,
1230 PostCommitFailurePolicy::KeepInstalled,
1231 ),
1232 &check,
1233 )
1234 .unwrap();
1235 assert!(remaining.get() < Duration::from_secs(2).as_nanos());
1236 assert!(receipt.observation_failure.is_some());
1237 assert!(receipt.final_snapshot.is_none());
1238 assert_eq!(
1239 receipt.artifact_disposition(),
1240 TransactionDisposition::Committed
1241 );
1242 let _ = fs::remove_dir_all(base);
1243 }
1244
1245 #[test]
1246 fn service_transition_failure_keep_installed_does_not_retry_hidden_policy() {
1247 let (base, root, tx) = setup(false);
1248 let service = spec(&root);
1249 let mut manager = ScriptedManager::with_service(service.clone(), LifecycleState::Running);
1250 manager.fail_start_calls = vec![1];
1251 let receipt = commit_with_lifecycle(
1252 &mut manager,
1253 &service,
1254 tx,
1255 absent_ownership(),
1256 policy(
1257 RestoreIntent::Preserve,
1258 PostCommitFailurePolicy::KeepInstalled,
1259 ),
1260 &NoPostInstallCheck,
1261 )
1262 .unwrap();
1263 assert_eq!(
1264 receipt.artifact_disposition(),
1265 TransactionDisposition::Committed
1266 );
1267 assert_eq!(
1268 receipt.post_commit_failure.as_ref().unwrap().phase,
1269 LifecycleUpdatePhase::RestoreNew
1270 );
1271 assert_eq!(manager.start_calls, 1);
1272 assert_eq!(
1273 receipt.final_snapshot.as_ref().unwrap().state,
1274 LifecycleState::Stopped
1275 );
1276 assert!(root.join("service.bin").exists());
1277 let _ = fs::remove_dir_all(base);
1278 }
1279}