ic_backup/ops/persistence/command_lifetime_lock/
mod.rs

1//! One-command descriptor custody and exact existing-sidecar quiescence admission.
2
3use super::file_lock::{self, FileLockError};
4use crate::model::command_custody::{CommandCustodyRecord, CommandCustodyRecordError};
5use std::{
6    fs::{self, File},
7    io,
8    path::{Path, PathBuf},
9    process::{Child, Command},
10    thread,
11    time::{Duration, Instant},
12};
13use thiserror::Error;
14
15const QUIESCENCE_GRACE: Duration = Duration::from_millis(250);
16/// Child environment variable containing its inherited custody descriptor number.
17pub const COMMAND_CUSTODY_DESCRIPTOR_ENV: &str = "IC_BACKUP_COMMAND_CUSTODY_FD";
18
19/// Exclusive custody for one journal operation's local command attempt.
20///
21/// Persist [`Self::record`] and dispatch intent before spawning. This primitive
22/// does not consume durable paid-call authority or qualify a backend's retry policy.
23#[derive(Debug)]
24pub struct CommandLifetimeLock {
25    file: File,
26    path: PathBuf,
27    record: CommandCustodyRecord,
28    dispatched: bool,
29}
30
31impl CommandLifetimeLock {
32    /// Acquire an operation's regular no-follow sidecar without waiting.
33    ///
34    /// The journal parent must already exist. The journal leaf may be absent;
35    /// aliases of an explicitly selected parent resolve to one location.
36    ///
37    /// # Errors
38    /// Rejects unsafe journal/sidecar entries, contention, invalid identity and IO failures.
39    pub fn acquire(
40        journal: &Path,
41        operation_sequence: u64,
42    ) -> Result<Self, CommandLifetimeLockError> {
43        let journal = resolve_journal(journal)?;
44        let path = lock_path(&journal, operation_sequence);
45        let file = file_lock::acquire(&path).map_err(|error| project_error(&path, error))?;
46        let (device, inode) = file_identity(&file.metadata()?)?;
47        let record = CommandCustodyRecord::new(journal, operation_sequence, device, inode)?;
48        Ok(Self {
49            file,
50            path,
51            record,
52            dispatched: false,
53        })
54    }
55
56    /// Return exact custody evidence for durable retention before dispatch.
57    #[must_use]
58    pub fn record(&self) -> &CommandCustodyRecord {
59        &self.record
60    }
61
62    /// Return the retained sidecar path; dropping custody never removes it.
63    #[must_use]
64    pub fn path(&self) -> &Path {
65        &self.path
66    }
67
68    /// Attempt one trusted command with owned descriptor inheritance.
69    ///
70    /// Consumes this guard's single spawn allowance before the spawn attempt.
71    /// The command value is consumed and dropped, so its parent-side descriptor
72    /// copy cannot outlive spawn. The owner descriptor remains close-on-exec.
73    /// The selected backend must preserve inherited custody in its descendants.
74    /// Caller code owns argv admission, output/deadline limits and child reaping.
75    ///
76    /// # Errors
77    /// Rejects repeated dispatch, replaced sidecars, unsupported hosts and spawn failures.
78    pub fn spawn(&mut self, mut command: Command) -> Result<Child, CommandLifetimeLockError> {
79        if self.dispatched {
80            return Err(CommandLifetimeLockError::AlreadyDispatched {
81                path: self.path.clone(),
82            });
83        }
84        require_identity(&self.record, &fs::symlink_metadata(&self.path)?, &self.path)?;
85        self.dispatched = true;
86        #[cfg(unix)]
87        {
88            use command_fds::CommandFdExt;
89            use std::os::fd::AsRawFd;
90            let descriptor =
91                rustix::io::fcntl_dupfd_cloexec(&self.file, 3).map_err(io::Error::from)?;
92            command.env(
93                COMMAND_CUSTODY_DESCRIPTOR_ENV,
94                descriptor.as_raw_fd().to_string(),
95            );
96            command.preserved_fds(vec![descriptor]);
97            let child = command.spawn();
98            drop(command);
99            child.map_err(CommandLifetimeLockError::from)
100        }
101        #[cfg(not(unix))]
102        {
103            let _ = command;
104            Err(io::Error::from(io::ErrorKind::Unsupported).into())
105        }
106    }
107
108    /// Close owner custody and admit exact quiescence within a bounded grace period.
109    ///
110    /// Success returns an exclusive non-spawning guard. Contention means another
111    /// inherited holder remains; command exit alone is insufficient. Failures
112    /// retain the sidecar and do not establish that a remote effect failed.
113    ///
114    /// # Errors
115    /// Returns in-flight custody, changed/missing/unsafe sidecars and IO failures.
116    pub fn finish(self) -> Result<CommandQuiescenceGuard, CommandLifetimeLockError> {
117        let Self { file, record, .. } = self;
118        drop(file);
119        let deadline = Instant::now() + QUIESCENCE_GRACE;
120        loop {
121            match CommandQuiescenceGuard::acquire(&record) {
122                Err(CommandLifetimeLockError::InFlight { .. }) if Instant::now() < deadline => {
123                    thread::sleep(Duration::from_millis(5));
124                }
125                result => return result,
126            }
127        }
128    }
129}
130
131/// Fresh exclusive local quiescence for retained exact custody evidence.
132///
133/// This guard cannot spawn commands. It excludes cooperating holders of the same
134/// sidecar until dropped. Drop explicitly releases this non-spawning exclusion,
135/// even if an unrelated fork temporarily retained a descriptor copy. Terminal
136/// completion still requires its own journal proof.
137#[derive(Debug)]
138pub struct CommandQuiescenceGuard {
139    file: File,
140    record: CommandCustodyRecord,
141}
142
143impl CommandQuiescenceGuard {
144    /// Observe an existing sidecar and hold exact local custody without creating it.
145    ///
146    /// # Errors
147    /// Rejects active holders, changed/missing/unsafe identities and filesystem failures.
148    pub fn acquire(expected: &CommandCustodyRecord) -> Result<Self, CommandLifetimeLockError> {
149        let path = lock_path(expected.journal(), expected.operation_sequence());
150        // Reject an already-replaced sidecar before trying to lock its inode.
151        // A transient holder of that different inode must not mask the mismatch.
152        // Recheck the opened file below because the path can change meanwhile.
153        let metadata = fs::symlink_metadata(&path)?;
154        if metadata.is_file() {
155            require_identity(expected, &metadata, &path)?;
156        }
157        let file =
158            file_lock::acquire_existing(&path).map_err(|error| project_error(&path, error))?;
159        require_identity(expected, &file.metadata()?, &path)?;
160        Ok(Self {
161            file,
162            record: expected.clone(),
163        })
164    }
165
166    /// Return the exact retained evidence freshly admitted by this guard.
167    #[must_use]
168    pub fn record(&self) -> &CommandCustodyRecord {
169        &self.record
170    }
171}
172
173impl Drop for CommandQuiescenceGuard {
174    fn drop(&mut self) {
175        // Unlike dispatched command custody, this guard cannot intentionally
176        // transfer its lock to a descendant. Closing alone would keep exclusion
177        // alive until all transient fork/dup references also close.
178        #[cfg(unix)]
179        file_lock::unlock(&self.file);
180    }
181}
182
183fn resolve_journal(path: &Path) -> Result<PathBuf, CommandLifetimeLockError> {
184    let name = path
185        .file_name()
186        .ok_or_else(|| CommandLifetimeLockError::InvalidJournal {
187            path: path.to_path_buf(),
188        })?;
189    let parent = path
190        .parent()
191        .filter(|parent| !parent.as_os_str().is_empty())
192        .unwrap_or_else(|| Path::new("."));
193    let parent = parent.canonicalize()?;
194    if !parent.is_dir() {
195        return Err(io::Error::from(io::ErrorKind::NotADirectory).into());
196    }
197    let journal = parent.join(name);
198    match fs::symlink_metadata(&journal) {
199        Ok(metadata) if !metadata.is_file() => {
200            return Err(CommandLifetimeLockError::InvalidJournal { path: journal });
201        }
202        Err(error) if error.kind() != io::ErrorKind::NotFound => return Err(error.into()),
203        _ => {}
204    }
205    Ok(journal)
206}
207
208fn lock_path(journal: &Path, operation_sequence: u64) -> PathBuf {
209    let mut path = journal.as_os_str().to_os_string();
210    path.push(format!(".command-{operation_sequence}.lock"));
211    PathBuf::from(path)
212}
213
214fn file_identity(metadata: &fs::Metadata) -> io::Result<(u64, u64)> {
215    if !metadata.is_file() {
216        return Err(io::Error::from(io::ErrorKind::InvalidInput));
217    }
218    #[cfg(unix)]
219    {
220        use std::os::unix::fs::MetadataExt;
221        Ok((metadata.dev(), metadata.ino()))
222    }
223    #[cfg(not(unix))]
224    {
225        let _ = metadata;
226        Err(io::Error::from(io::ErrorKind::Unsupported))
227    }
228}
229
230fn require_identity(
231    expected: &CommandCustodyRecord,
232    metadata: &fs::Metadata,
233    path: &Path,
234) -> Result<(), CommandLifetimeLockError> {
235    if !metadata.is_file() {
236        return Err(CommandLifetimeLockError::IdentityChanged {
237            path: path.to_path_buf(),
238        });
239    }
240    let (device, inode) = file_identity(metadata)?;
241    if expected.matches_file(device, inode) {
242        return Ok(());
243    }
244    Err(CommandLifetimeLockError::IdentityChanged {
245        path: path.to_path_buf(),
246    })
247}
248
249fn project_error(path: &Path, error: FileLockError) -> CommandLifetimeLockError {
250    match error {
251        FileLockError::Locked => CommandLifetimeLockError::InFlight {
252            path: path.to_path_buf(),
253        },
254        FileLockError::UnsafeEntry { kind } => CommandLifetimeLockError::UnsafeEntry {
255            path: path.to_path_buf(),
256            kind,
257        },
258        FileLockError::Io(error) => CommandLifetimeLockError::Io(error),
259    }
260}
261
262/// Typed custody admission, dispatch or quiescence failure.
263#[derive(Debug, Error)]
264pub enum CommandLifetimeLockError {
265    /// An owner or inherited command still holds custody.
266    #[error("command custody remains in flight: {path:?}")]
267    InFlight {
268        /// Retained sidecar path.
269        path: PathBuf,
270    },
271    /// The retained sidecar is no longer the expected regular file.
272    #[error("command custody identity changed: {path:?}")]
273    IdentityChanged {
274        /// Changed sidecar path.
275        path: PathBuf,
276    },
277    /// One guard already attempted its single command spawn.
278    #[error("command spawn already attempted: {path:?}")]
279    AlreadyDispatched {
280        /// Retained sidecar path.
281        path: PathBuf,
282    },
283    /// The sidecar has an unsafe entry type.
284    #[error("unsafe command custody entry at {path:?}: {kind}")]
285    UnsafeEntry {
286        /// Sidecar path.
287        path: PathBuf,
288        /// Observed entry kind.
289        kind: String,
290    },
291    /// The selected journal location is unsafe or incomplete.
292    #[error("invalid command journal location: {path:?}")]
293    InvalidJournal {
294        /// Rejected location.
295        path: PathBuf,
296    },
297    /// Retained custody evidence fails model admission.
298    #[error(transparent)]
299    Record(#[from] CommandCustodyRecordError),
300    /// A filesystem or process operation failed.
301    #[error(transparent)]
302    Io(#[from] io::Error),
303}
304
305#[cfg(all(test, unix))]
306mod tests;
ic_backup/ops/persistence/command_lifetime_lock/mod.rs

ic_backup/ops/persistence/command_lifetime_lock/
mod.rs
