1use std::collections::{HashMap, HashSet};
2use std::fs;
3use std::path::{Path, PathBuf};
4use std::sync::atomic::{AtomicU64, Ordering};
5
6use crate::domain::{AbsentPolicy, CommitOwnership, MemberId, Ownership};
7use crate::error::{Error, Result};
8use crate::integrity::hash_file;
9use crate::lock::MutationLock;
10use crate::stage::PreparedTransaction;
11
12static NEXT_BACKUP_NONCE: AtomicU64 = AtomicU64::new(0);
13
14#[derive(Debug, Clone, Copy, PartialEq, Eq)]
19pub enum TransactionDisposition {
20 Committed,
22 RolledBack,
24 RecoveryRequired,
26}
27
28#[derive(Debug, Clone, Copy, PartialEq, Eq)]
34pub enum CleanupDisposition {
35 Cleaned,
37 RetainedForRecovery,
39}
40
41#[derive(Debug, Clone, Copy, PartialEq, Eq)]
43pub enum PostCommitFailurePolicy {
44 KeepInstalled,
46 RollBack,
48}
49
50#[derive(Debug, Clone, Copy, PartialEq, Eq)]
54#[non_exhaustive]
55pub enum FailurePhase {
56 Lock,
58 Ownership,
60 StageRevalidation,
62 Backup,
64 Commit,
66 PostCommit,
68 Rollback,
70 Finalize,
72}
73
74impl FailurePhase {
75 pub fn as_str(&self) -> &'static str {
77 match self {
78 Self::Lock => "lock",
79 Self::Ownership => "ownership",
80 Self::StageRevalidation => "stage-revalidation",
81 Self::Backup => "backup",
82 Self::Commit => "commit",
83 Self::PostCommit => "post-commit",
84 Self::Rollback => "rollback",
85 Self::Finalize => "finalize",
86 }
87 }
88}
89
90#[derive(Debug, Clone, Copy, PartialEq, Eq)]
92#[non_exhaustive]
93pub enum FailureCategory {
94 LockContention,
96 OwnershipConflict,
98 Verification,
100 Filesystem,
102 InvalidInput,
104 Injected,
106 PostCommitCheck,
108}
109
110impl FailureCategory {
111 pub fn as_str(&self) -> &'static str {
113 match self {
114 Self::LockContention => "lock-contention",
115 Self::OwnershipConflict => "ownership-conflict",
116 Self::Verification => "verification",
117 Self::Filesystem => "filesystem",
118 Self::InvalidInput => "invalid-input",
119 Self::Injected => "injected",
120 Self::PostCommitCheck => "post-commit-check",
121 }
122 }
123}
124
125#[derive(Debug, Clone, PartialEq, Eq)]
127pub struct FailureReport {
128 phase: FailurePhase,
129 member: Option<MemberId>,
130 category: FailureCategory,
131 detail: String,
132}
133
134impl FailureReport {
135 pub(crate) fn new(
136 phase: FailurePhase,
137 member: Option<MemberId>,
138 category: FailureCategory,
139 detail: impl Into<String>,
140 ) -> Self {
141 let mut detail = detail.into();
142 if detail.len() > 512 {
144 let boundary = detail
145 .char_indices()
146 .map(|(index, _)| index)
147 .take_while(|index| *index <= 512)
148 .last()
149 .unwrap_or(0);
150 detail.truncate(boundary);
151 }
152 Self {
153 phase,
154 member,
155 category,
156 detail,
157 }
158 }
159
160 pub fn phase(&self) -> FailurePhase {
162 self.phase
163 }
164
165 pub fn member(&self) -> Option<&MemberId> {
167 self.member.as_ref()
168 }
169
170 pub fn category(&self) -> FailureCategory {
172 self.category
173 }
174
175 pub fn detail(&self) -> &str {
177 &self.detail
178 }
179}
180
181pub(crate) fn report_for_error(
182 phase: FailurePhase,
183 member: Option<MemberId>,
184 error: &Error,
185) -> FailureReport {
186 let (category, detail) = match error {
187 Error::UpdateInProgress { lock } => (
188 FailureCategory::LockContention,
189 format!("lock contention at {}", lock.display()),
190 ),
191 Error::DestinationConflict { destination } => (
192 FailureCategory::OwnershipConflict,
193 format!("destination conflict at {}", destination.display()),
194 ),
195 Error::VerificationFailed(m) => (FailureCategory::Verification, m.clone()),
196 Error::CandidateExecution(m) => (FailureCategory::Verification, m.clone()),
197 Error::InvalidInput(m) => {
198 let category = if m.contains("injected") {
199 FailureCategory::Injected
200 } else {
201 FailureCategory::InvalidInput
202 };
203 (category, m.clone())
204 }
205 Error::UnknownMember(m) => (FailureCategory::InvalidInput, format!("unknown member {m}")),
206 Error::Io { operation, source } => (
207 FailureCategory::Filesystem,
208 format!("{operation}: {source}"),
209 ),
210 };
211 FailureReport::new(phase, member, category, detail)
212}
213
214#[derive(Debug, Clone, PartialEq, Eq)]
216pub struct TransactionReceipt {
217 product: crate::domain::ProductId,
218 release: crate::domain::ReleaseId,
219 disposition: TransactionDisposition,
220 rollback_performed: bool,
221 rollback_verified: bool,
222 cleanup: CleanupDisposition,
223 recovery_path: Option<PathBuf>,
224 failure: Option<FailureReport>,
225 rollback_failure: Option<FailureReport>,
226 post_commit_failure: Option<FailureReport>,
227}
228
229impl TransactionReceipt {
230 pub fn product(&self) -> &crate::domain::ProductId {
232 &self.product
233 }
234
235 pub fn release(&self) -> &crate::domain::ReleaseId {
237 &self.release
238 }
239
240 pub fn disposition(&self) -> TransactionDisposition {
242 self.disposition
243 }
244
245 pub fn rollback_performed(&self) -> bool {
247 self.rollback_performed
248 }
249
250 pub fn rollback_verified(&self) -> bool {
252 self.rollback_verified
253 }
254
255 pub fn cleanup(&self) -> CleanupDisposition {
257 self.cleanup
258 }
259
260 pub fn recovery_path(&self) -> Option<&Path> {
267 self.recovery_path.as_deref()
268 }
269
270 pub fn failure(&self) -> Option<&FailureReport> {
277 self.failure.as_ref()
278 }
279
280 pub fn rollback_failure(&self) -> Option<&FailureReport> {
285 self.rollback_failure.as_ref()
286 }
287
288 pub fn post_commit_failure(&self) -> Option<&FailureReport> {
290 self.post_commit_failure.as_ref()
291 }
292}
293
294#[derive(Debug, Clone, PartialEq, Eq)]
295#[cfg_attr(not(test), allow(dead_code))]
296pub(crate) enum CommitFault {
297 LockCreation,
298 Backup(MemberId),
299 BeforeFirstCommit,
300 Commit(MemberId),
301 CommitThenRollback(MemberId, MemberId),
302 Finalize,
303 PostCommitRollback,
304}
305
306#[derive(Debug)]
307struct BackupEntry {
308 member: MemberId,
309 destination: PathBuf,
310 backup: Option<PathBuf>,
311}
312
313type PostCommitCheck<'a> = (
314 PostCommitFailurePolicy,
315 Box<dyn FnOnce() -> std::result::Result<(), String> + 'a>,
316);
317
318impl PreparedTransaction {
319 pub(crate) fn commit_inner(
320 self,
321 ownership: CommitOwnership<'_>,
322 verified_digests: &HashMap<MemberId, [u8; 32]>,
323 fault: Option<CommitFault>,
324 post_commit: Option<PostCommitCheck<'_>>,
325 ) -> Result<TransactionReceipt> {
326 if fault == Some(CommitFault::LockCreation) {
327 return Err(Error::invalid("injected lock creation failure"));
328 }
329 let preflight = classify_all(&self, &ownership)?;
332 let mut lock = MutationLock::acquire(
333 self.plan.installation_root(),
334 self.plan.product(),
335 self.plan.release(),
336 )?;
337 if let Err(error) = revalidate_ownership_locked(&self, &ownership, &preflight) {
340 let report = report_for_error(FailurePhase::Ownership, None, &error);
341 let verified = true;
342 return finish_failure_no_mutation(self, lock, verified, report);
343 }
344 if let Err((member, error)) = revalidate_staged_under_lock(&self, verified_digests) {
347 let report = report_for_error(FailurePhase::StageRevalidation, member, &error);
348 return finish_failure_no_mutation(self, lock, true, report);
349 }
350 let backup_root = match create_backup_directory(self.plan.installation_root()) {
351 Ok(p) => p,
352 Err(error) => {
353 let report = report_for_error(FailurePhase::Backup, None, &error);
354 return finish_failure_no_mutation(self, lock, true, report);
355 }
356 };
357 let mut entries = Vec::with_capacity(self.plan.artifacts().len());
358 if let Err((member, error)) = self.backup_members(&mut entries, &backup_root, fault.clone())
359 {
360 let phase = FailurePhase::Backup;
361 let report = report_for_error(phase, member, &error);
362 let (verified, rollback_failure) =
363 restore_entries(&entries, &HashSet::new(), fault.clone());
364 return finish_failure(self, lock, backup_root, verified, rollback_failure, report);
365 }
366 if fault == Some(CommitFault::BeforeFirstCommit) {
367 let error = Error::invalid("injected pre-commit failure");
368 let report = report_for_error(FailurePhase::Commit, None, &error);
369 let (verified, rollback_failure) =
370 restore_entries(&entries, &HashSet::new(), fault.clone());
371 return finish_failure(self, lock, backup_root, verified, rollback_failure, report);
372 }
373
374 let mut committed = HashSet::new();
375 let members = self.plan.artifacts().iter().cloned().collect::<Vec<_>>();
376 for member in members {
377 let commit_should_fail = fault.as_ref().is_some_and(|candidate| match candidate {
378 CommitFault::Commit(id) | CommitFault::CommitThenRollback(id, _) => {
379 id == member.id()
380 }
381 _ => false,
382 });
383 if commit_should_fail {
384 let error = Error::invalid(format!("injected commit failure at {}", member.id()));
385 let report =
386 report_for_error(FailurePhase::Commit, Some(member.id().clone()), &error);
387 let (verified, rollback_failure) =
388 restore_entries(&entries, &committed, fault.clone());
389 return finish_failure(self, lock, backup_root, verified, rollback_failure, report);
390 }
391 let destination = match self.plan.destination(member.id()) {
392 Ok(destination) => destination,
393 Err(error) => {
394 let report =
395 report_for_error(FailurePhase::Commit, Some(member.id().clone()), &error);
396 let (verified, rollback_failure) =
397 restore_entries(&entries, &committed, fault.clone());
398 return finish_failure(
399 self,
400 lock,
401 backup_root,
402 verified,
403 rollback_failure,
404 report,
405 );
406 }
407 };
408 if let Err(error) = require_ready_parent(&destination, self.plan.installation_root()) {
409 let report =
410 report_for_error(FailurePhase::Commit, Some(member.id().clone()), &error);
411 let (verified, rollback_failure) =
412 restore_entries(&entries, &committed, fault.clone());
413 return finish_failure(self, lock, backup_root, verified, rollback_failure, report);
414 }
415 let staged = match self.staged_path(member.id()) {
416 Ok(staged) => staged,
417 Err(error) => {
418 let report =
419 report_for_error(FailurePhase::Commit, Some(member.id().clone()), &error);
420 let (verified, rollback_failure) =
421 restore_entries(&entries, &committed, fault.clone());
422 return finish_failure(
423 self,
424 lock,
425 backup_root,
426 verified,
427 rollback_failure,
428 report,
429 );
430 }
431 };
432 if let Err(source) = fs::rename(&staged, &destination) {
433 let error = Error::io("replacing destination from prepared stage", source);
434 let report =
435 report_for_error(FailurePhase::Commit, Some(member.id().clone()), &error);
436 let (verified, rollback_failure) =
437 restore_entries(&entries, &committed, fault.clone());
438 return finish_failure(self, lock, backup_root, verified, rollback_failure, report);
439 }
440 committed.insert(member.id().clone());
441 }
442
443 let mut post_commit_failure = None;
444 if let Some((policy, check)) = post_commit {
445 let checked = std::panic::catch_unwind(std::panic::AssertUnwindSafe(check));
446 let failure_detail = match checked {
447 Ok(Ok(())) => None,
448 Ok(Err(detail)) => Some(detail),
449 Err(_) => Some("post-commit check panicked".to_string()),
450 };
451 if let Some(detail) = failure_detail {
452 let report = FailureReport::new(
453 FailurePhase::PostCommit,
454 None,
455 FailureCategory::PostCommitCheck,
456 detail,
457 );
458 match policy {
459 PostCommitFailurePolicy::KeepInstalled => {
460 post_commit_failure = Some(report);
461 }
462 PostCommitFailurePolicy::RollBack => {
463 let rollback_fault = if fault == Some(CommitFault::PostCommitRollback) {
464 Some(CommitFault::CommitThenRollback(
465 self.plan
466 .artifacts()
467 .iter()
468 .next()
469 .expect("non-empty artifact set")
470 .id()
471 .clone(),
472 self.plan
473 .artifacts()
474 .iter()
475 .next()
476 .expect("non-empty artifact set")
477 .id()
478 .clone(),
479 ))
480 } else {
481 fault.clone()
482 };
483 let (verified, rollback_failure) =
484 restore_entries(&entries, &committed, rollback_fault);
485 let mut receipt = finish_failure(
486 self,
487 lock,
488 backup_root,
489 verified,
490 rollback_failure,
491 report.clone(),
492 )?;
493 receipt.post_commit_failure = Some(report);
494 return Ok(receipt);
495 }
496 }
497 }
498 }
499
500 if fault == Some(CommitFault::Finalize) {
501 lock.preserve();
502 let report = FailureReport::new(
503 FailurePhase::Finalize,
504 None,
505 FailureCategory::Injected,
506 "injected finalize failure",
507 );
508 return Ok(TransactionReceipt {
509 product: self.plan.product().clone(),
510 release: self.plan.release().clone(),
511 disposition: TransactionDisposition::Committed,
512 rollback_performed: false,
513 rollback_verified: false,
514 cleanup: CleanupDisposition::RetainedForRecovery,
515 recovery_path: Some(backup_root),
516 failure: Some(report),
517 rollback_failure: None,
518 post_commit_failure,
519 });
520 }
521 if let Err(source) = fs::remove_dir_all(&backup_root) {
522 lock.preserve();
523 let error = Error::io("removing backup set after commit", source);
524 let report = report_for_error(FailurePhase::Finalize, None, &error);
525 return Ok(TransactionReceipt {
526 product: self.plan.product().clone(),
527 release: self.plan.release().clone(),
528 disposition: TransactionDisposition::Committed,
529 rollback_performed: false,
530 rollback_verified: false,
531 cleanup: CleanupDisposition::RetainedForRecovery,
532 recovery_path: Some(backup_root),
533 failure: Some(report),
534 rollback_failure: None,
535 post_commit_failure,
536 });
537 }
538 Ok(TransactionReceipt {
539 product: self.plan.product().clone(),
540 release: self.plan.release().clone(),
541 disposition: TransactionDisposition::Committed,
542 rollback_performed: false,
543 rollback_verified: false,
544 cleanup: CleanupDisposition::Cleaned,
545 recovery_path: None,
546 failure: None,
547 rollback_failure: None,
548 post_commit_failure,
549 })
550 }
551
552 fn backup_members(
553 &self,
554 entries: &mut Vec<BackupEntry>,
555 backup_root: &Path,
556 fault: Option<CommitFault>,
557 ) -> std::result::Result<(), (Option<MemberId>, Error)> {
558 let members = self.plan.artifacts().iter().cloned().collect::<Vec<_>>();
559 for member in members {
560 if fault == Some(CommitFault::Backup(member.id().clone())) {
561 return Err((
562 Some(member.id().clone()),
563 Error::invalid(format!("injected backup failure at {}", member.id())),
564 ));
565 }
566 let destination = self
567 .plan
568 .destination(member.id())
569 .map_err(|e| (Some(member.id().clone()), e))?;
570 revalidate_destination(&destination, self.plan.installation_root())
571 .map_err(|e| (Some(member.id().clone()), e))?;
572 let backup = match fs::symlink_metadata(&destination) {
573 Ok(_) => {
574 let path = backup_root.join(member.destination());
575 if let Some(parent) = path.parent() {
576 fs::create_dir_all(parent).map_err(|source| {
577 (
578 Some(member.id().clone()),
579 Error::io("creating backup directory", source),
580 )
581 })?;
582 }
583 fs::rename(&destination, &path).map_err(|source| {
584 (
585 Some(member.id().clone()),
586 Error::io("backing up existing destination", source),
587 )
588 })?;
589 Some(path)
590 }
591 Err(source) if source.kind() == std::io::ErrorKind::NotFound => None,
592 Err(source) => {
593 return Err((
594 Some(member.id().clone()),
595 Error::io("reading destination before backup", source),
596 ));
597 }
598 };
599 entries.push(BackupEntry {
600 member: member.id().clone(),
601 destination,
602 backup,
603 });
604 }
605 Ok(())
606 }
607}
608
609fn classify_all(
610 txn: &PreparedTransaction,
611 ownership: &CommitOwnership<'_>,
612) -> Result<HashMap<MemberId, Ownership>> {
613 let mut out = HashMap::new();
614 for member in txn.artifacts().iter() {
615 let destination = txn.destination(member.id())?;
616 out.insert(
617 member.id().clone(),
618 ownership.verifier.verify(member.id(), &destination),
619 );
620 }
621 Ok(out)
622}
623
624fn revalidate_ownership_locked(
625 txn: &PreparedTransaction,
626 ownership: &CommitOwnership<'_>,
627 preflight: &HashMap<MemberId, Ownership>,
628) -> Result<()> {
629 for member in txn.artifacts().iter() {
630 let destination = txn.destination(member.id())?;
631 let locked = ownership.verifier.verify(member.id(), &destination);
632 let first = preflight
633 .get(member.id())
634 .copied()
635 .unwrap_or(Ownership::Unknown);
636 if locked != first {
639 return Err(Error::DestinationConflict { destination });
640 }
641 match locked {
642 Ownership::Owned => {}
643 Ownership::Absent if ownership.absent == AbsentPolicy::AllowCreate => {}
644 Ownership::Absent | Ownership::Foreign | Ownership::Unknown => {
645 return Err(Error::DestinationConflict { destination });
646 }
647 }
648 require_ready_parent(&destination, txn.plan.installation_root())?;
651 revalidate_destination(&destination, txn.plan.installation_root())?;
652 }
653 Ok(())
654}
655
656fn revalidate_staged_under_lock(
657 txn: &PreparedTransaction,
658 verified_digests: &HashMap<MemberId, [u8; 32]>,
659) -> std::result::Result<(), (Option<MemberId>, Error)> {
660 for member in txn.artifacts().iter() {
661 let expected = verified_digests.get(member.id()).ok_or_else(|| {
662 (
663 Some(member.id().clone()),
664 Error::VerificationFailed(format!(
665 "member {} has no verified digest; commit requires verified integrity",
666 member.id()
667 )),
668 )
669 })?;
670 let staged = txn
671 .staged_path(member.id())
672 .map_err(|e| (Some(member.id().clone()), e))?;
673 let metadata = fs::symlink_metadata(&staged).map_err(|source| {
674 (
675 Some(member.id().clone()),
676 Error::io("reading staged candidate", source),
677 )
678 })?;
679 if !metadata.is_file() {
680 return Err((
681 Some(member.id().clone()),
682 Error::VerificationFailed(format!(
683 "staged member {} is not a regular file",
684 member.id()
685 )),
686 ));
687 }
688 #[cfg(unix)]
689 {
690 use std::os::unix::fs::MetadataExt;
691 if metadata.nlink() != 1 {
692 return Err((
693 Some(member.id().clone()),
694 Error::VerificationFailed(format!(
695 "staged member {} is hard-linked",
696 member.id()
697 )),
698 ));
699 }
700 }
701 let actual = hash_file(&staged).map_err(|e| (Some(member.id().clone()), e))?;
702 if &actual != expected {
703 return Err((
704 Some(member.id().clone()),
705 Error::VerificationFailed(format!(
706 "staged member {} changed after verification",
707 member.id()
708 )),
709 ));
710 }
711 }
712 Ok(())
713}
714
715fn create_backup_directory(root: &Path) -> Result<PathBuf> {
716 use std::time::{SystemTime, UNIX_EPOCH};
717 for _ in 0..32 {
718 let nonce = NEXT_BACKUP_NONCE.fetch_add(1, Ordering::Relaxed);
719 let nanos = SystemTime::now()
720 .duration_since(UNIX_EPOCH)
721 .map(|d| d.as_nanos())
722 .unwrap_or(0);
723 let path = root.join(format!(
724 ".eggup-backup-{}-{nonce}-{nanos}",
725 std::process::id()
726 ));
727 match fs::create_dir(&path) {
728 Ok(()) => {
729 #[cfg(unix)]
730 {
731 use std::os::unix::fs::PermissionsExt;
732 let _ = fs::set_permissions(&path, fs::Permissions::from_mode(0o700));
733 }
734 return Ok(path);
735 }
736 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => continue,
737 Err(source) => return Err(Error::io("creating backup set", source)),
738 }
739 }
740 Err(Error::invalid("could not create a unique backup directory"))
741}
742
743fn revalidate_destination(destination: &Path, root: &Path) -> Result<()> {
744 let root = fs::canonicalize(root)
745 .map_err(|source| Error::io("canonicalizing install root", source))?;
746 let mut ancestor = destination;
747 while !ancestor.exists() {
748 ancestor = ancestor
749 .parent()
750 .ok_or_else(|| Error::invalid("destination has no existing ancestor"))?;
751 }
752 let canonical_ancestor = fs::canonicalize(ancestor)
753 .map_err(|source| Error::io("canonicalizing destination ancestor", source))?;
754 if !canonical_ancestor.starts_with(&root) {
755 return Err(Error::DestinationConflict {
756 destination: destination.to_path_buf(),
757 });
758 }
759 match fs::symlink_metadata(destination) {
760 Ok(metadata) => {
761 if !metadata.is_file() {
762 return Err(Error::DestinationConflict {
763 destination: destination.to_path_buf(),
764 });
765 }
766 #[cfg(unix)]
767 {
768 use std::os::unix::fs::MetadataExt;
769 if metadata.nlink() != 1 {
770 return Err(Error::DestinationConflict {
771 destination: destination.to_path_buf(),
772 });
773 }
774 }
775 }
776 Err(source) if source.kind() == std::io::ErrorKind::NotFound => {}
777 Err(source) => return Err(Error::io("reading destination during revalidation", source)),
778 }
779 Ok(())
780}
781
782fn require_ready_parent(destination: &Path, root: &Path) -> Result<()> {
789 let parent = destination
790 .parent()
791 .ok_or_else(|| Error::invalid("destination has no parent"))?;
792 let parent_meta = fs::symlink_metadata(parent).map_err(|source| {
793 if source.kind() == std::io::ErrorKind::NotFound {
794 Error::invalid(format!(
795 "destination parent does not exist: {}",
796 parent.display()
797 ))
798 } else {
799 Error::io("reading destination parent", source)
800 }
801 })?;
802 if parent_meta.file_type().is_symlink() {
803 return Err(Error::DestinationConflict {
804 destination: destination.to_path_buf(),
805 });
806 }
807 if !parent_meta.is_dir() {
808 return Err(Error::DestinationConflict {
809 destination: destination.to_path_buf(),
810 });
811 }
812 let canonical_root = fs::canonicalize(root)
813 .map_err(|source| Error::io("canonicalizing install root", source))?;
814 let canonical_parent = fs::canonicalize(parent)
815 .map_err(|source| Error::io("canonicalizing destination parent", source))?;
816 if !canonical_parent.starts_with(&canonical_root) {
817 return Err(Error::DestinationConflict {
818 destination: destination.to_path_buf(),
819 });
820 }
821 let mut cursor = Some(parent);
824 while let Some(p) = cursor {
825 if p == root {
826 break;
827 }
828 let meta = fs::symlink_metadata(p)
829 .map_err(|source| Error::io("reading destination ancestor", source))?;
830 if meta.file_type().is_symlink() {
831 return Err(Error::DestinationConflict {
832 destination: destination.to_path_buf(),
833 });
834 }
835 cursor = p.parent();
836 if let Some(next) = cursor {
837 if next.as_os_str().is_empty() {
838 break;
839 }
840 } else {
841 break;
842 }
843 }
844 Ok(())
845}
846
847fn restore_entries(
848 entries: &[BackupEntry],
849 committed: &HashSet<MemberId>,
850 fault: Option<CommitFault>,
851) -> (bool, Option<FailureReport>) {
852 let mut verified = true;
853 let mut rollback_failure: Option<FailureReport> = None;
854 for entry in entries.iter().rev() {
855 let rollback_target = match fault.as_ref() {
856 Some(CommitFault::CommitThenRollback(_, id)) => Some(id),
857 _ => None,
858 };
859 if rollback_target == Some(&entry.member) {
860 verified = false;
861 rollback_failure = Some(FailureReport::new(
862 FailurePhase::Rollback,
863 Some(entry.member.clone()),
864 FailureCategory::Injected,
865 format!("injected rollback failure at {}", entry.member),
866 ));
867 continue;
868 }
869 if entry.backup.is_some() || committed.contains(&entry.member) {
870 if let Ok(metadata) = fs::symlink_metadata(&entry.destination) {
871 if !metadata.is_file() || fs::remove_file(&entry.destination).is_err() {
872 verified = false;
873 if rollback_failure.is_none() {
874 rollback_failure = Some(FailureReport::new(
875 FailurePhase::Rollback,
876 Some(entry.member.clone()),
877 FailureCategory::Filesystem,
878 format!("could not remove partial member {}", entry.member),
879 ));
880 }
881 continue;
882 }
883 }
884 }
885 if let Some(backup) = &entry.backup {
886 if fs::rename(backup, &entry.destination).is_err() {
887 verified = false;
888 if rollback_failure.is_none() {
889 rollback_failure = Some(FailureReport::new(
890 FailurePhase::Rollback,
891 Some(entry.member.clone()),
892 FailureCategory::Filesystem,
893 format!("could not restore backup for {}", entry.member),
894 ));
895 }
896 }
897 }
898 }
899 for entry in entries {
900 let restored = match &entry.backup {
901 Some(_) => fs::symlink_metadata(&entry.destination).is_ok(),
902 None => {
903 !committed.contains(&entry.member)
904 || fs::symlink_metadata(&entry.destination).is_err()
905 }
906 };
907 if !restored {
908 verified = false;
909 }
910 }
911 (verified, rollback_failure)
912}
913
914fn finish_failure_no_mutation(
915 transaction: PreparedTransaction,
916 lock: MutationLock,
917 nothing_mutated: bool,
918 report: FailureReport,
919) -> Result<TransactionReceipt> {
920 debug_assert!(nothing_mutated);
921 drop(lock);
922 Ok(TransactionReceipt {
923 product: transaction.plan.product().clone(),
924 release: transaction.plan.release().clone(),
925 disposition: TransactionDisposition::RolledBack,
926 rollback_performed: false,
927 rollback_verified: true,
928 cleanup: CleanupDisposition::Cleaned,
929 recovery_path: None,
930 failure: Some(report),
931 rollback_failure: None,
932 post_commit_failure: None,
933 })
934}
935
936fn finish_failure(
937 transaction: PreparedTransaction,
938 mut lock: MutationLock,
939 backup_root: PathBuf,
940 rollback_verified: bool,
941 rollback_failure: Option<FailureReport>,
942 report: FailureReport,
943) -> Result<TransactionReceipt> {
944 if rollback_verified && rollback_failure.is_none() && fs::remove_dir_all(&backup_root).is_ok() {
945 return Ok(TransactionReceipt {
946 product: transaction.plan.product().clone(),
947 release: transaction.plan.release().clone(),
948 disposition: TransactionDisposition::RolledBack,
949 rollback_performed: true,
950 rollback_verified: true,
951 cleanup: CleanupDisposition::Cleaned,
952 recovery_path: None,
953 failure: Some(report),
954 rollback_failure: None,
955 post_commit_failure: None,
956 });
957 }
958 lock.preserve();
959 Ok(TransactionReceipt {
960 product: transaction.plan.product().clone(),
961 release: transaction.plan.release().clone(),
962 disposition: TransactionDisposition::RecoveryRequired,
963 rollback_performed: true,
964 rollback_verified: false,
965 cleanup: CleanupDisposition::RetainedForRecovery,
966 recovery_path: Some(backup_root),
967 failure: Some(report),
968 rollback_failure,
969 post_commit_failure: None,
970 })
971}