Skip to main content

car_inference/
model_management.rs

1//! Crash-safe, identity-bound state for CAR-managed local model artifacts.
2//!
3//! Raw model IDs never enter filenames. Every record is stored under the
4//! SHA-256 of the exact UTF-8 ID and repeats that ID inside a
5//! `deny_unknown_fields` payload so a collision or corrupt file fails closed.
6
7use car_winenv::PlainCanonicalize;
8use std::fs::{File, OpenOptions};
9use std::io::{Read, Seek, Write};
10use std::path::{Path, PathBuf};
11use std::sync::atomic::{AtomicU64, Ordering};
12use std::sync::{Mutex, OnceLock};
13
14use serde::{Deserialize, Serialize};
15use sha2::{Digest, Sha256};
16
17const MAX_MODEL_ID_BYTES: usize = 4 * 1024;
18const MAX_STATE_BYTES: u64 = 64 * 1024;
19/// Refusal reason for a quarantine that disappeared between CAR's own
20/// no-replace rename and the delete — on a first attempt or a resume, at the
21/// path-based guard or at the narrower window before the binding descriptor
22/// opens. Every one of those paths returns before the receipt and journal
23/// unlinks, so the removal journal is still on disk and running the removal
24/// again resumes it. Shared by both raise sites so the wording cannot drift.
25const QUARANTINE_VANISHED_REASON: &str = "quarantine vanished before deletion; the removal journal is retained, so retrying the removal resumes and completes it";
26
27#[derive(Debug, thiserror::Error)]
28pub enum ModelManagementError {
29    #[error("model-management I/O failed at {path}: {source}")]
30    Io {
31        path: PathBuf,
32        #[source]
33        source: std::io::Error,
34    },
35    #[error("invalid model-management state at {path}: {message}")]
36    InvalidState { path: PathBuf, message: String },
37    #[error("model-management identity mismatch at {path}: expected {expected}, found {actual}")]
38    IdentityMismatch {
39        path: PathBuf,
40        expected: String,
41        actual: String,
42    },
43    #[error("no CAR install receipt exists for {model_id}")]
44    MissingReceipt { model_id: String },
45    #[error("unsafe CAR-managed path for {model_id}: {path} ({reason})")]
46    UnsafeManagedPath {
47        model_id: String,
48        path: PathBuf,
49        reason: String,
50    },
51    #[error("local model {model_id} is in use")]
52    ModelInUse { model_id: String },
53    #[error("local model-management operation is already active for {model_id}")]
54    MutationInProgress { model_id: String },
55}
56
57#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
58#[serde(rename_all = "snake_case")]
59pub enum ManagedArtifactKind {
60    Symlink,
61    Directory,
62    File,
63}
64
65#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
66#[serde(deny_unknown_fields)]
67pub struct InstallReceipt {
68    pub model_id: String,
69    pub managed_path: PathBuf,
70    pub artifact_kind: ManagedArtifactKind,
71    pub source_model_id: String,
72    pub source_revision: Option<String>,
73    pub creation_generation: u64,
74    pub shared_cache_references: Vec<PathBuf>,
75    pub adopted: bool,
76}
77
78/// How often a model in continuous use re-stamps its [`UsageStamp`].
79pub const USAGE_STAMP_INTERVAL: std::time::Duration = std::time::Duration::from_secs(10 * 60);
80
81const USAGE_TRACKING_MARKER: &str = "tracking-since.json";
82const USAGE_COVERAGE: &str = "coverage.json";
83
84/// Longest stretch without a beat — awake time, not wall time — that still
85/// counts as one continuous run of a stamping daemon. The daemon beats every
86/// 15 minutes; four missed beats is a stopped or stalled daemon.
87pub const COVERAGE_GAP_SECS: u64 = 60 * 60;
88
89/// The current contiguous run during which a usage-stamping daemon was up.
90/// Silence only counts inside it: idle needs the whole window observed, so
91/// only the run in progress matters, never earlier ones.
92#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
93struct UsageCoverage {
94    run_start: u64,
95    last_beat: u64,
96}
97
98/// The last time a local model was used, written when its activity lease is
99/// taken — the one point every local load passes, whatever the modality.
100#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
101pub struct UsageStamp {
102    pub model_id: String,
103    pub last_used: u64,
104}
105
106#[derive(Debug, Serialize, Deserialize)]
107struct UsageTracking {
108    since: u64,
109}
110
111fn unix_now() -> u64 {
112    std::time::SystemTime::now()
113        .duration_since(std::time::UNIX_EPOCH)
114        .map(|d| d.as_secs())
115        .unwrap_or(0)
116}
117
118#[derive(Debug, Clone)]
119pub struct ModelManagementStore {
120    models_dir: PathBuf,
121    management_state_dir: PathBuf,
122    receipts_root: PathBuf,
123    tombstones_root: PathBuf,
124    installs_root: PathBuf,
125    removals_root: PathBuf,
126    leases_root: PathBuf,
127    mutation_locks_root: PathBuf,
128    /// When each local model was last used by any CAR on this machine
129    /// ([`UsageStamp`]). Beside the leases, not under one daemon's state root,
130    /// because the weights it judges are machine-shared too.
131    usage_root: PathBuf,
132    /// Last stamp this process wrote per model: a lease is taken per request,
133    /// and a durable write per request is not worth a minute of resolution.
134    usage_throttle: std::sync::Arc<Mutex<std::collections::HashMap<String, std::time::Instant>>>,
135    /// This process's last coverage beat: (wall clock, monotonic instant).
136    /// The instant does not advance while the machine sleeps, which is what
137    /// tells sleep (no one could use a model) from a stalled daemon.
138    last_beat: std::sync::Arc<Mutex<Option<(u64, std::time::Instant)>>>,
139    /// Test-only seam that runs between the phases of a directory removal so
140    /// tests can race it the way a same-UID process would. Compiled out of
141    /// release builds and of every target without directory removal.
142    #[cfg(all(test, any(target_os = "macos", target_os = "linux")))]
143    removal_hook: Option<RemovalTestHook>,
144    #[cfg(test)]
145    identity_init_hook: Option<fn(&Path)>,
146}
147
148/// Whether this build can delete a CAR-managed directory through a
149/// descriptor-bound walk (`ModelManagementStore::remove_quarantined_directory`).
150/// Everywhere else recursive pathname deletion stays refused: directory
151/// receipts remain usable evidence but are never removable.
152pub(crate) const fn directory_removal_supported() -> bool {
153    cfg!(any(target_os = "macos", target_os = "linux"))
154}
155
156#[cfg(all(test, any(target_os = "macos", target_os = "linux")))]
157#[derive(Clone, Copy, Debug, PartialEq, Eq)]
158pub(crate) enum RemovalPhase {
159    /// The managed leaf has been renamed into its quarantine; nothing deleted yet.
160    AfterRename,
161    /// The quarantine passed the path-based identity check; the descriptor
162    /// that binds every later operation has not been opened yet.
163    BeforeRootOpen,
164    /// Every entry of the quarantined directory has been captured by descriptor;
165    /// nothing deleted yet.
166    AfterCapture,
167}
168
169#[cfg(all(test, any(target_os = "macos", target_os = "linux")))]
170#[derive(Clone)]
171pub(crate) struct RemovalTestHook(std::sync::Arc<dyn Fn(RemovalPhase, &Path) + Send + Sync>);
172
173#[cfg(all(test, any(target_os = "macos", target_os = "linux")))]
174impl std::fmt::Debug for RemovalTestHook {
175    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
176        f.write_str("RemovalTestHook")
177    }
178}
179
180impl ModelManagementStore {
181    pub fn new(state_root: PathBuf, models_dir: PathBuf) -> Self {
182        let state_root = crate::resource_policy::normalized_state_root_key(&state_root);
183        let management_state_dir = state_root.join("model-management");
184        // `models_dir` is machine-shared even when two daemons use different
185        // CAR_HOME roots. Resolve aliases before deriving lock paths so those
186        // daemons coordinate on one immutable set of lock sentinels.
187        let models_dir = crate::resource_policy::normalized_state_root_key(&models_dir);
188        let shared_coordination_root = models_dir
189            .parent()
190            .unwrap_or(&models_dir)
191            .join(".car-model-management");
192        let store = Self {
193            models_dir,
194            receipts_root: receipts_dir(&state_root),
195            tombstones_root: management_state_dir.join("tombstones"),
196            installs_root: management_state_dir.join("installs"),
197            removals_root: management_state_dir.join("removals"),
198            leases_root: shared_coordination_root.join("activity"),
199            mutation_locks_root: shared_coordination_root.join("mutation"),
200            usage_root: shared_coordination_root.join("usage"),
201            usage_throttle: Default::default(),
202            last_beat: Default::default(),
203            management_state_dir,
204            #[cfg(all(test, any(target_os = "macos", target_os = "linux")))]
205            removal_hook: None,
206            #[cfg(test)]
207            identity_init_hook: None,
208        };
209        // A crash can leave a CAR-owned quarantine after the managed leaf has
210        // already been atomically detached. Resume only those journaled,
211        // identity-matching transactions whose cross-process locks are idle.
212        store.resume_pending_quarantines();
213        store
214    }
215
216    pub fn models_dir(&self) -> &Path {
217        &self.models_dir
218    }
219
220    #[cfg(all(test, any(target_os = "macos", target_os = "linux")))]
221    pub(crate) fn with_removal_hook(
222        mut self,
223        hook: impl Fn(RemovalPhase, &Path) + Send + Sync + 'static,
224    ) -> Self {
225        self.removal_hook = Some(RemovalTestHook(std::sync::Arc::new(hook)));
226        self
227    }
228
229    pub fn management_state_dir(&self) -> &Path {
230        &self.management_state_dir
231    }
232
233    pub fn receipts_root(&self) -> &Path {
234        &self.receipts_root
235    }
236
237    pub fn receipt_path(&self, model_id: &str) -> PathBuf {
238        self.receipts_root.join(hashed_filename(model_id))
239    }
240
241    pub fn tombstones_root(&self) -> &Path {
242        &self.tombstones_root
243    }
244
245    pub fn tombstone_path(&self, model_id: &str) -> PathBuf {
246        self.tombstones_root.join(hashed_filename(model_id))
247    }
248
249    fn removal_journal_path(&self, model_id: &str) -> PathBuf {
250        self.removals_root.join(hashed_filename(model_id))
251    }
252
253    fn install_journal_path(&self, model_id: &str) -> PathBuf {
254        self.installs_root.join(hashed_filename(model_id))
255    }
256
257    pub fn leases_root(&self) -> &Path {
258        &self.leases_root
259    }
260
261    pub fn lease_path(&self, model_id: &str) -> PathBuf {
262        self.leases_root.join(hashed_filename(model_id))
263    }
264
265    pub fn mutation_lock_path(&self, model_id: &str) -> PathBuf {
266        self.mutation_locks_root.join(hashed_filename(model_id))
267    }
268
269    pub fn load_receipt(
270        &self,
271        model_id: &str,
272    ) -> Result<Option<InstallReceipt>, ModelManagementError> {
273        validate_model_id(model_id)?;
274        read_receipt_at(&self.receipt_path(model_id), model_id)
275    }
276
277    pub(crate) fn write_receipt(
278        &self,
279        receipt: &InstallReceipt,
280    ) -> Result<(), ModelManagementError> {
281        validate_model_id(&receipt.model_id)?;
282        if let Some(existing) = self.load_receipt(&receipt.model_id)? {
283            ensure_identity(
284                &self.receipt_path(&receipt.model_id),
285                &receipt.model_id,
286                &existing.model_id,
287            )?;
288        }
289        write_private_json(&self.receipt_path(&receipt.model_id), receipt)
290    }
291
292    /// Return whether CAR has a receipt whose current artifact still matches
293    /// the ownership evidence recorded at install/adoption time.
294    pub fn can_remove(&self, model_id: &str) -> Result<bool, ModelManagementError> {
295        let Some(receipt) = self.load_receipt(model_id)? else {
296            return Ok(false);
297        };
298        self.validate_managed_artifact(&receipt)?;
299        // Recursive pathname deletion cannot be made object-bound portably.
300        // Directory receipts are removable only where CAR has a
301        // descriptor-bound walk (`directory_removal_supported`); elsewhere they
302        // remain usable evidence but are never removable.
303        Ok(
304            receipt.artifact_kind != ManagedArtifactKind::Directory
305                || directory_removal_supported(),
306        )
307    }
308
309    pub fn acquire_lease(&self, model_id: &str) -> Result<ModelLease, ModelManagementError> {
310        validate_model_id(model_id)?;
311        // A load first enters the shared side of the immutable mutation gate,
312        // then takes its durable activity lease. Removal takes the exclusive
313        // side before it starts draining, closing the check/delete TOCTOU.
314        let mutation = self.open_identity_lock(&self.mutation_lock_path(model_id), model_id)?;
315        mutation
316            .lock_shared()
317            .map_err(|source| ModelManagementError::Io {
318                path: self.mutation_lock_path(model_id),
319                source,
320            })?;
321        let mut mutation = OwnedFileLock::new(mutation);
322        validate_lock_identity(&mut mutation, &self.mutation_lock_path(model_id), model_id)?;
323        let file = self.open_identity_lock(&self.lease_path(model_id), model_id)?;
324        file.lock_shared()
325            .map_err(|source| ModelManagementError::Io {
326                path: self.lease_path(model_id),
327                source,
328            })?;
329        let mut file = OwnedFileLock::new(file);
330        validate_lock_identity(&mut file, &self.lease_path(model_id), model_id)?;
331        drop(mutation);
332        self.note_used(model_id);
333        Ok(ModelLease { _file: file })
334    }
335
336    /// Record that `model_id` is being used now. At most one durable write
337    /// per model per [`USAGE_STAMP_INTERVAL`] from this process, and never a
338    /// reason to refuse the load: a failure is logged and the lease stands.
339    /// Stamped before admission, so an attempt admission then refuses still
340    /// counts as use — the safe direction for a signal that gates deletion.
341    fn note_used(&self, model_id: &str) {
342        let now = std::time::Instant::now();
343        {
344            let mut written = self
345                .usage_throttle
346                .lock()
347                .unwrap_or_else(std::sync::PoisonError::into_inner);
348            if written
349                .get(model_id)
350                .is_some_and(|at| now.duration_since(*at) < USAGE_STAMP_INTERVAL)
351            {
352                return;
353            }
354            written.insert(model_id.to_string(), now);
355        }
356        let stamp = UsageStamp {
357            model_id: model_id.to_string(),
358            last_used: unix_now(),
359        };
360        if let Err(error) =
361            write_private_json(&self.usage_root.join(hashed_filename(model_id)), &stamp)
362        {
363            tracing::debug!(model = model_id, %error, "could not record model use");
364        }
365    }
366
367    /// Mark when usage tracking began on this machine, once. Until a model
368    /// has gone unstamped for longer than an idle threshold *since this
369    /// moment*, its silence is no evidence: the day stamping ships, every
370    /// model is unstamped.
371    ///
372    /// The daemon calls this at startup, and nothing else may: the process
373    /// that serves the models is the one whose leases stamp them. A newer CLI
374    /// or test starting the clock while an older daemon serves unstamped
375    /// would make every model look idle a month later.
376    pub fn start_usage_tracking(&self) {
377        let marker = self.usage_root.join(USAGE_TRACKING_MARKER);
378        if std::fs::symlink_metadata(&marker).is_ok() {
379            return;
380        }
381        if let Err(error) = write_private_json(&marker, &UsageTracking { since: unix_now() }) {
382            tracing::debug!(%error, "could not record when model usage tracking began");
383        }
384    }
385
386    /// Backdate the tracking marker, as if this machine had been tracking
387    /// usage since `since`.
388    #[cfg(test)]
389    pub(crate) fn set_usage_tracking_since_for_test(&self, since: u64) {
390        write_private_json(
391            &self.usage_root.join(USAGE_TRACKING_MARKER),
392            &UsageTracking { since },
393        )
394        .unwrap();
395        write_private_json(
396            &self.usage_root.join(USAGE_COVERAGE),
397            &UsageCoverage {
398                run_start: since,
399                last_beat: unix_now(),
400            },
401        )
402        .unwrap();
403    }
404
405    /// Record that a usage-stamping daemon is up now. The run continues from
406    /// the previous beat when that beat was this process's own and less than
407    /// [`COVERAGE_GAP_SECS`] of awake time ago (a sleeping machine pauses the
408    /// clock rather than breaking the run), or when another stamping process
409    /// beat within that window of wall time (a restart). Otherwise a run
410    /// starts now. Daemon only, like [`Self::start_usage_tracking`].
411    pub fn beat(&self) {
412        let now = unix_now();
413        let instant = std::time::Instant::now();
414        let path = self.usage_root.join(USAGE_COVERAGE);
415        let previous = read_private_state(&path)
416            .ok()
417            .and_then(|bytes| serde_json::from_slice::<UsageCoverage>(&bytes).ok());
418        let mut mine = self
419            .last_beat
420            .lock()
421            .unwrap_or_else(std::sync::PoisonError::into_inner);
422        let run_start = match (previous, *mine) {
423            (Some(prev), Some((wall, at))) if prev.last_beat == wall => {
424                if instant.duration_since(at).as_secs() <= COVERAGE_GAP_SECS {
425                    prev.run_start
426                } else {
427                    now
428                }
429            }
430            (Some(prev), _) if now.saturating_sub(prev.last_beat) <= COVERAGE_GAP_SECS => {
431                prev.run_start
432            }
433            _ => now,
434        };
435        let coverage = UsageCoverage {
436            run_start,
437            last_beat: now,
438        };
439        if let Err(error) = write_private_json(&path, &coverage) {
440            tracing::debug!(%error, "could not record model usage coverage");
441        }
442        *mine = Some((now, instant));
443    }
444
445    /// Since when a usage-stamping daemon has been continuously up, if one is.
446    pub fn usage_covered_since(&self) -> Option<u64> {
447        read_private_state(&self.usage_root.join(USAGE_COVERAGE))
448            .ok()
449            .and_then(|bytes| serde_json::from_slice::<UsageCoverage>(&bytes).ok())
450            .map(|coverage| coverage.run_start)
451    }
452
453    /// When usage tracking began on this machine, if it has.
454    pub fn usage_tracking_since(&self) -> Option<u64> {
455        read_private_state(&self.usage_root.join(USAGE_TRACKING_MARKER))
456            .ok()
457            .and_then(|bytes| serde_json::from_slice::<UsageTracking>(&bytes).ok())
458            .map(|marker| marker.since)
459    }
460
461    /// When `model_id` was last used, if it ever was since tracking began.
462    pub fn last_used(&self, model_id: &str) -> Option<u64> {
463        let path = self.usage_root.join(hashed_filename(model_id));
464        read_private_state(&path)
465            .ok()
466            .and_then(|bytes| serde_json::from_slice::<UsageStamp>(&bytes).ok())
467            .filter(|stamp| stamp.model_id == model_id)
468            .map(|stamp| stamp.last_used)
469    }
470
471    pub(crate) fn begin_mutation(
472        &self,
473        model_id: &str,
474    ) -> Result<ModelMutationGuard, ModelManagementError> {
475        validate_model_id(model_id)?;
476        let mut file = self.open_identity_lock(&self.mutation_lock_path(model_id), model_id)?;
477        validate_lock_identity(&mut file, &self.mutation_lock_path(model_id), model_id)?;
478        match file.try_lock() {
479            Ok(()) => Ok(ModelMutationGuard {
480                store: self.clone(),
481                model_id: model_id.to_string(),
482                _file: OwnedFileLock::new(file),
483            }),
484            Err(std::fs::TryLockError::WouldBlock) => {
485                Err(ModelManagementError::MutationInProgress {
486                    model_id: model_id.to_string(),
487                })
488            }
489            Err(std::fs::TryLockError::Error(source)) => Err(ModelManagementError::Io {
490                path: self.mutation_lock_path(model_id),
491                source,
492            }),
493        }
494    }
495
496    pub fn model_in_use(&self, model_id: &str) -> Result<bool, ModelManagementError> {
497        validate_model_id(model_id)?;
498        let mut file = self.open_identity_lock(&self.lease_path(model_id), model_id)?;
499        validate_lock_identity(&mut file, &self.lease_path(model_id), model_id)?;
500        match file.try_lock() {
501            Ok(()) => {
502                let _lock = OwnedFileLock::new(file);
503                Ok(false)
504            }
505            Err(std::fs::TryLockError::WouldBlock) => Ok(true),
506            Err(std::fs::TryLockError::Error(source)) => Err(ModelManagementError::Io {
507                path: self.lease_path(model_id),
508                source,
509            }),
510        }
511    }
512
513    pub fn car_enabled(&self, model_id: &str) -> Result<bool, ModelManagementError> {
514        Ok(self.load_tombstone(model_id)?.is_none())
515    }
516
517    fn load_tombstone(
518        &self,
519        model_id: &str,
520    ) -> Result<Option<RemovalTombstone>, ModelManagementError> {
521        validate_model_id(model_id)?;
522        let path = self.tombstone_path(model_id);
523        let bytes = match read_private_state(&path) {
524            Ok(bytes) => bytes,
525            Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
526            Err(source) => return Err(ModelManagementError::Io { path, source }),
527        };
528        let tombstone: RemovalTombstone =
529            serde_json::from_slice(&bytes).map_err(|error| ModelManagementError::InvalidState {
530                path: path.clone(),
531                message: error.to_string(),
532            })?;
533        ensure_identity(&path, model_id, &tombstone.model_id)?;
534        Ok(Some(tombstone))
535    }
536
537    pub(crate) fn clear_tombstone(&self, model_id: &str) -> Result<(), ModelManagementError> {
538        validate_model_id(model_id)?;
539        let _ = self.car_enabled(model_id)?;
540        let path = self.tombstone_path(model_id);
541        match std::fs::remove_file(&path) {
542            Ok(()) => sync_directory(path.parent().expect("tombstone path has parent"))
543                .map_err(|source| ModelManagementError::Io { path, source }),
544            Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()),
545            Err(source) => Err(ModelManagementError::Io { path, source }),
546        }
547    }
548
549    /// Bind a usable artifact under the CAR models root to a durable receipt.
550    /// The artifact path is derived by CAR; callers cannot pass arbitrary
551    /// deletion targets through the public API.
552    pub(crate) fn record_managed_artifact(
553        &self,
554        model_id: &str,
555        source_model_id: &str,
556        source_revision: Option<String>,
557        creation_generation: u64,
558        adopted: bool,
559        managed_path: PathBuf,
560    ) -> Result<InstallReceipt, ModelManagementError> {
561        validate_model_id(model_id)?;
562        let artifact_kind = artifact_kind(&managed_path, model_id)?;
563        let shared_cache_references = collect_shared_references(&managed_path, model_id)?;
564        let receipt = InstallReceipt {
565            model_id: model_id.to_string(),
566            managed_path,
567            artifact_kind,
568            source_model_id: source_model_id.to_string(),
569            source_revision,
570            creation_generation,
571            shared_cache_references,
572            adopted,
573        };
574        self.validate_managed_artifact(&receipt)?;
575        self.write_receipt(&receipt)?;
576        self.clear_tombstone(model_id)?;
577        Ok(receipt)
578    }
579
580    pub(crate) fn begin_install_intent(
581        &self,
582        receipt: InstallReceipt,
583        staging_path: Option<&Path>,
584    ) -> Result<(), ModelManagementError> {
585        validate_model_id(&receipt.model_id)?;
586        validate_direct_child(&self.models_dir, &receipt.managed_path, &receipt.model_id)?;
587        let staging_path = staging_path.map(Path::to_path_buf);
588        let staging_identity = staging_path
589            .as_deref()
590            .map(|path| artifact_identity(path, &receipt.model_id))
591            .transpose()?;
592        if let Some(path) = staging_path.as_deref() {
593            validate_direct_child(&self.models_dir, path, &receipt.model_id)?;
594        }
595        let intent = InstallJournal {
596            model_id: receipt.model_id.clone(),
597            receipt,
598            staging_path,
599            staging_identity,
600        };
601        let path = self.install_journal_path(&intent.model_id);
602        if let Some(existing) = self.load_install_intent(&intent.model_id)? {
603            if existing == intent {
604                return Ok(());
605            }
606            return Err(ModelManagementError::InvalidState {
607                path,
608                message: "a different CAR install intent already exists for this model".into(),
609            });
610        }
611        if std::fs::symlink_metadata(&intent.receipt.managed_path).is_ok() {
612            return Err(ModelManagementError::UnsafeManagedPath {
613                model_id: intent.model_id,
614                path: intent.receipt.managed_path,
615                reason: "pre-existing unreceipted leaf cannot become CAR-owned through an install intent"
616                    .into(),
617            });
618        }
619        write_private_json(&path, &intent)
620    }
621
622    pub(crate) fn install_receipt_for_publication(
623        &self,
624        model_id: &str,
625        source_model_id: &str,
626        creation_generation: u64,
627        adopted: bool,
628        managed_path: PathBuf,
629        artifact_kind: ManagedArtifactKind,
630        artifact_source: &Path,
631    ) -> Result<InstallReceipt, ModelManagementError> {
632        validate_model_id(model_id)?;
633        let shared_cache_references = match artifact_kind {
634            ManagedArtifactKind::Symlink => {
635                vec![artifact_source.plain_canonicalize().map_err(|source| {
636                    ModelManagementError::Io {
637                        path: artifact_source.to_path_buf(),
638                        source,
639                    }
640                })?]
641            }
642            ManagedArtifactKind::Directory => collect_shared_references(artifact_source, model_id)?,
643            ManagedArtifactKind::File => Vec::new(),
644        };
645        Ok(InstallReceipt {
646            model_id: model_id.to_string(),
647            managed_path,
648            artifact_kind,
649            source_model_id: source_model_id.to_string(),
650            source_revision: None,
651            creation_generation,
652            shared_cache_references,
653            adopted,
654        })
655    }
656
657    pub(crate) fn resume_install_intent(
658        &self,
659        model_id: &str,
660    ) -> Result<Option<InstallReceipt>, ModelManagementError> {
661        let Some(intent) = self.load_install_intent(model_id)? else {
662            return Ok(None);
663        };
664        if std::fs::symlink_metadata(&intent.receipt.managed_path).is_err() {
665            if let (Some(staging), Some(identity)) = (
666                intent.staging_path.as_deref(),
667                intent.staging_identity.as_ref(),
668            ) {
669                if std::fs::symlink_metadata(staging).is_ok() {
670                    ensure_artifact_identity(staging, model_id, identity)?;
671                    atomic_rename_noreplace(staging, &intent.receipt.managed_path).map_err(
672                        |source| ModelManagementError::Io {
673                            path: intent.receipt.managed_path.clone(),
674                            source,
675                        },
676                    )?;
677                    sync_directory(&self.models_dir).map_err(|source| {
678                        ModelManagementError::Io {
679                            path: self.models_dir.clone(),
680                            source,
681                        }
682                    })?;
683                } else {
684                    self.clear_install_intent(model_id)?;
685                    return Ok(None);
686                }
687            } else {
688                // No CAR-created leaf was published. Clearing only the intent
689                // is safe and lets the caller create a fresh projection.
690                self.clear_install_intent(model_id)?;
691                return Ok(None);
692            }
693        }
694        if let Some(identity) = intent.staging_identity.as_ref() {
695            ensure_artifact_identity(&intent.receipt.managed_path, model_id, identity)?;
696        }
697        self.validate_managed_artifact(&intent.receipt)?;
698        self.write_receipt(&intent.receipt)?;
699        self.clear_install_intent(model_id)?;
700        self.clear_tombstone(model_id)?;
701        Ok(Some(intent.receipt))
702    }
703
704    fn load_install_intent(
705        &self,
706        model_id: &str,
707    ) -> Result<Option<InstallJournal>, ModelManagementError> {
708        let path = self.install_journal_path(model_id);
709        let bytes = match read_private_state(&path) {
710            Ok(bytes) => bytes,
711            Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
712            Err(source) => return Err(ModelManagementError::Io { path, source }),
713        };
714        let intent: InstallJournal =
715            serde_json::from_slice(&bytes).map_err(|error| ModelManagementError::InvalidState {
716                path: path.clone(),
717                message: error.to_string(),
718            })?;
719        ensure_identity(&path, model_id, &intent.model_id)?;
720        ensure_identity(&path, model_id, &intent.receipt.model_id)?;
721        validate_direct_child(&self.models_dir, &intent.receipt.managed_path, model_id)?;
722        if let Some(staging) = intent.staging_path.as_deref() {
723            validate_direct_child(&self.models_dir, staging, model_id)?;
724            if !staging
725                .file_name()
726                .and_then(|name| name.to_str())
727                .is_some_and(|name| name.starts_with(".car-install-"))
728            {
729                return Err(ModelManagementError::UnsafeManagedPath {
730                    model_id: model_id.to_string(),
731                    path: staging.to_path_buf(),
732                    reason: "install journal staging path lacks CAR staging identity".into(),
733                });
734            }
735        }
736        Ok(Some(intent))
737    }
738
739    fn clear_install_intent(&self, model_id: &str) -> Result<(), ModelManagementError> {
740        let path = self.install_journal_path(model_id);
741        match std::fs::remove_file(&path) {
742            Ok(()) => sync_directory(&self.installs_root)
743                .map_err(|source| ModelManagementError::Io { path, source }),
744            Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()),
745            Err(source) => Err(ModelManagementError::Io { path, source }),
746        }
747    }
748
749    /// Create or reuse a CAR-owned symlink projection for an artifact that
750    /// physically lives in a shared cache. Shared bytes remain outside CAR's
751    /// ownership and are never removed with the projection.
752    pub(crate) fn materialize_managed_projection(
753        &self,
754        managed_leaf: &str,
755        source: &Path,
756    ) -> Result<PathBuf, ModelManagementError> {
757        validate_managed_leaf(managed_leaf, &self.models_dir)?;
758        create_private_dir(&self.models_dir)?;
759        let models_root = canonical_directory(&self.models_dir)?;
760        let source_parent =
761            source
762                .parent()
763                .ok_or_else(|| ModelManagementError::UnsafeManagedPath {
764                    model_id: managed_leaf.to_string(),
765                    path: source.to_path_buf(),
766                    reason: "source has no parent".into(),
767                })?;
768        if canonical_directory(source_parent)? == models_root
769            && source == self.models_dir.join(managed_leaf)
770        {
771            return Err(ModelManagementError::UnsafeManagedPath {
772                model_id: managed_leaf.to_string(),
773                path: source.to_path_buf(),
774                reason: "pre-existing unreceipted managed projection cannot become CAR-owned"
775                    .into(),
776            });
777        }
778        let target =
779            source
780                .plain_canonicalize()
781                .map_err(|source_error| ModelManagementError::Io {
782                    path: source.to_path_buf(),
783                    source: source_error,
784                })?;
785        let managed = self.models_dir.join(managed_leaf);
786        if std::fs::symlink_metadata(&managed).is_ok() {
787            return Err(ModelManagementError::UnsafeManagedPath {
788                model_id: managed_leaf.to_string(),
789                path: managed,
790                reason: "pre-existing managed projection has no CAR creation receipt".into(),
791            });
792        }
793        static NEXT_PROJECTION: AtomicU64 = AtomicU64::new(1);
794        let tmp = self.models_dir.join(format!(
795            ".managed-projection.{}.{}",
796            std::process::id(),
797            NEXT_PROJECTION.fetch_add(1, Ordering::Relaxed)
798        ));
799        create_symlink(&target, &tmp)?;
800        let result = atomic_rename_noreplace(&tmp, &managed).map_err(|source| {
801            ModelManagementError::UnsafeManagedPath {
802                model_id: managed_leaf.to_string(),
803                path: managed.clone(),
804                reason: format!("managed projection publication raced or failed: {source}"),
805            }
806        });
807        if result.is_err() {
808            let _ = std::fs::remove_file(&tmp);
809        }
810        result?;
811        sync_directory(&self.models_dir).map_err(|source| ModelManagementError::Io {
812            path: self.models_dir.clone(),
813            source,
814        })?;
815        Ok(managed)
816    }
817
818    pub(crate) fn create_install_staging(
819        &self,
820        model_id: &str,
821    ) -> Result<PathBuf, ModelManagementError> {
822        validate_model_id(model_id)?;
823        create_private_dir(&self.models_dir)?;
824        let digest = hex::encode(Sha256::digest(model_id.as_bytes()));
825        for _ in 0..16 {
826            let candidate = self.models_dir.join(format!(
827                ".car-install-{}-{:032x}",
828                &digest[..16],
829                rand::random::<u128>()
830            ));
831            match std::fs::create_dir(&candidate) {
832                Ok(()) => {
833                    harden_private_directory(&candidate)?;
834                    return Ok(candidate);
835                }
836                Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue,
837                Err(source) => {
838                    return Err(ModelManagementError::Io {
839                        path: candidate,
840                        source,
841                    });
842                }
843            }
844        }
845        Err(ModelManagementError::InvalidState {
846            path: self.models_dir.clone(),
847            message: "could not allocate a collision-resistant install staging directory".into(),
848        })
849    }
850
851    pub(crate) fn publish_install_staging(
852        &self,
853        model_id: &str,
854        staging: &Path,
855        managed_leaf: &str,
856    ) -> Result<PathBuf, ModelManagementError> {
857        validate_model_id(model_id)?;
858        validate_direct_child(&self.models_dir, staging, model_id)?;
859        if !staging
860            .file_name()
861            .and_then(|name| name.to_str())
862            .is_some_and(|name| name.starts_with(".car-install-"))
863        {
864            return Err(ModelManagementError::UnsafeManagedPath {
865                model_id: model_id.to_string(),
866                path: staging.to_path_buf(),
867                reason: "install staging path lacks CAR staging identity".into(),
868            });
869        }
870        validate_managed_leaf(managed_leaf, &self.models_dir)?;
871        let managed = self.models_dir.join(managed_leaf);
872        atomic_rename_noreplace(staging, &managed).map_err(|source| {
873            ModelManagementError::UnsafeManagedPath {
874                model_id: model_id.to_string(),
875                path: managed.clone(),
876                reason: format!("could not atomically publish CAR-owned install: {source}"),
877            }
878        })?;
879        sync_directory(&self.models_dir).map_err(|source| ModelManagementError::Io {
880            path: self.models_dir.clone(),
881            source,
882        })?;
883        Ok(managed)
884    }
885
886    pub(crate) fn discard_install_staging(&self, staging: &Path) {
887        // Deliberately fail closed. A same-UID attacker can swap a verified
888        // directory before a pathname-recursive delete. Leave failed staging
889        // journal/evidence for future object-bound cleanup instead of risking
890        // deletion of shared model bytes.
891        let _ = staging;
892    }
893
894    /// Project a hand-installed or shared artifact into a distinct CAR-owned
895    /// leaf. The original path is never promoted into CAR ownership merely by
896    /// adoption, even when it already sits beneath `models_dir`.
897    pub(crate) fn materialize_adopted_projection(
898        &self,
899        model_id: &str,
900        source: &Path,
901    ) -> Result<PathBuf, ModelManagementError> {
902        validate_model_id(model_id)?;
903        let managed = self.adopted_projection_path(model_id)?;
904        let leaf = managed
905            .file_name()
906            .and_then(|name| name.to_str())
907            .expect("adopted projection is an UTF-8 direct child");
908        self.materialize_managed_projection(leaf, source)
909    }
910
911    pub(crate) fn adopted_projection_path(
912        &self,
913        model_id: &str,
914    ) -> Result<PathBuf, ModelManagementError> {
915        validate_model_id(model_id)?;
916        let digest = hex::encode(Sha256::digest(model_id.as_bytes()));
917        Ok(self.models_dir.join(format!(".car-adopted-{digest}")))
918    }
919
920    fn remove_with_mutation(
921        &self,
922        model_id: &str,
923        removal_generation: u64,
924    ) -> Result<RemoveFromCarResult, ModelManagementError> {
925        let _activity = self.lock_idle_activity(model_id)?;
926
927        if let Some(journal) = self.load_removal_journal(model_id)? {
928            write_private_json(
929                &self.tombstone_path(model_id),
930                &RemovalTombstone {
931                    model_id: model_id.to_string(),
932                    removal_generation: journal.removal_generation,
933                    artifact_kind: Some(journal.receipt.artifact_kind),
934                    preserved_shared_cache_references: journal
935                        .receipt
936                        .shared_cache_references
937                        .clone(),
938                },
939            )?;
940            return self.resume_removal_journal(journal);
941        }
942
943        let Some(receipt) = self.load_receipt(model_id)? else {
944            if let Some(tombstone) = self.load_tombstone(model_id)? {
945                if let Some(artifact_kind) = tombstone.artifact_kind {
946                    return Ok(RemoveFromCarResult {
947                        model_id: model_id.to_string(),
948                        artifact_kind,
949                        preserved_shared_cache_references: tombstone
950                            .preserved_shared_cache_references,
951                    });
952                }
953            }
954            return Err(ModelManagementError::MissingReceipt {
955                model_id: model_id.to_string(),
956            });
957        };
958        self.validate_managed_artifact(&receipt)?;
959        if receipt.artifact_kind == ManagedArtifactKind::Directory && !directory_removal_supported()
960        {
961            return Err(ModelManagementError::UnsafeManagedPath {
962                model_id: model_id.to_string(),
963                path: receipt.managed_path,
964                reason: "recursive directory removal is unsupported without object-bound traversal"
965                    .into(),
966            });
967        }
968        let artifact_identity = artifact_identity(&receipt.managed_path, model_id)?;
969        let quarantine = self.allocate_quarantine(model_id)?;
970
971        write_private_json(
972            &self.tombstone_path(model_id),
973            &RemovalTombstone {
974                model_id: model_id.to_string(),
975                removal_generation,
976                artifact_kind: Some(receipt.artifact_kind),
977                preserved_shared_cache_references: receipt.shared_cache_references.clone(),
978            },
979        )?;
980        let journal = RemovalJournal {
981            model_id: model_id.to_string(),
982            removal_generation,
983            receipt,
984            quarantine_path: quarantine,
985            artifact_identity,
986        };
987        write_private_json(&self.removal_journal_path(model_id), &journal)?;
988        self.resume_removal_journal(journal)
989    }
990
991    fn lock_idle_activity(&self, model_id: &str) -> Result<OwnedFileLock, ModelManagementError> {
992        let mut activity = self.open_identity_lock(&self.lease_path(model_id), model_id)?;
993        validate_lock_identity(&mut activity, &self.lease_path(model_id), model_id)?;
994        match activity.try_lock() {
995            Ok(()) => Ok(OwnedFileLock::new(activity)),
996            Err(std::fs::TryLockError::WouldBlock) => Err(ModelManagementError::ModelInUse {
997                model_id: model_id.to_string(),
998            }),
999            Err(std::fs::TryLockError::Error(source)) => Err(ModelManagementError::Io {
1000                path: self.lease_path(model_id),
1001                source,
1002            }),
1003        }
1004    }
1005
1006    fn load_removal_journal(
1007        &self,
1008        model_id: &str,
1009    ) -> Result<Option<RemovalJournal>, ModelManagementError> {
1010        let path = self.removal_journal_path(model_id);
1011        let bytes = match read_private_state(&path) {
1012            Ok(bytes) => bytes,
1013            Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
1014            Err(source) => return Err(ModelManagementError::Io { path, source }),
1015        };
1016        let journal: RemovalJournal =
1017            serde_json::from_slice(&bytes).map_err(|error| ModelManagementError::InvalidState {
1018                path: path.clone(),
1019                message: error.to_string(),
1020            })?;
1021        ensure_identity(&path, model_id, &journal.model_id)?;
1022        ensure_identity(&path, model_id, &journal.receipt.model_id)?;
1023        validate_direct_child(&self.models_dir, &journal.receipt.managed_path, model_id)?;
1024        if journal.receipt.artifact_kind == ManagedArtifactKind::Directory
1025            && !directory_removal_supported()
1026        {
1027            return Err(ModelManagementError::UnsafeManagedPath {
1028                model_id: model_id.to_string(),
1029                path: journal.receipt.managed_path,
1030                reason: "recursive directory removal journal is unsupported".into(),
1031            });
1032        }
1033        validate_direct_child(&self.models_dir, &journal.quarantine_path, model_id)?;
1034        if !journal
1035            .quarantine_path
1036            .file_name()
1037            .and_then(|name| name.to_str())
1038            .is_some_and(|name| name.starts_with(".car-remove-"))
1039        {
1040            return Err(ModelManagementError::UnsafeManagedPath {
1041                model_id: model_id.to_string(),
1042                path: journal.quarantine_path,
1043                reason: "removal journal has an invalid quarantine path".into(),
1044            });
1045        }
1046        Ok(Some(journal))
1047    }
1048
1049    fn allocate_quarantine(&self, model_id: &str) -> Result<PathBuf, ModelManagementError> {
1050        let digest = hex::encode(Sha256::digest(model_id.as_bytes()));
1051        for _ in 0..16 {
1052            let candidate = self.models_dir.join(format!(
1053                ".car-remove-{}-{:032x}",
1054                &digest[..16],
1055                rand::random::<u128>()
1056            ));
1057            if std::fs::symlink_metadata(&candidate)
1058                .is_err_and(|error| error.kind() == std::io::ErrorKind::NotFound)
1059            {
1060                return Ok(candidate);
1061            }
1062        }
1063        Err(ModelManagementError::InvalidState {
1064            path: self.models_dir.clone(),
1065            message: "could not allocate a collision-resistant removal quarantine".into(),
1066        })
1067    }
1068
1069    fn resume_removal_journal(
1070        &self,
1071        journal: RemovalJournal,
1072    ) -> Result<RemoveFromCarResult, ModelManagementError> {
1073        let model_id = &journal.model_id;
1074        if journal.receipt.artifact_kind == ManagedArtifactKind::Directory
1075            && !directory_removal_supported()
1076        {
1077            return Err(ModelManagementError::UnsafeManagedPath {
1078                model_id: model_id.clone(),
1079                path: journal.receipt.managed_path,
1080                reason: "recursive directory removal is unsupported without object-bound traversal"
1081                    .into(),
1082            });
1083        }
1084        let original_exists = std::fs::symlink_metadata(&journal.receipt.managed_path).is_ok();
1085        let quarantine_exists = std::fs::symlink_metadata(&journal.quarantine_path).is_ok();
1086        let mut renamed_now = false;
1087        if original_exists && quarantine_exists {
1088            return Err(ModelManagementError::UnsafeManagedPath {
1089                model_id: model_id.clone(),
1090                path: journal.receipt.managed_path.clone(),
1091                reason: "both managed artifact and removal quarantine exist".into(),
1092            });
1093        }
1094        if original_exists {
1095            self.validate_managed_artifact(&journal.receipt)?;
1096            ensure_artifact_identity(
1097                &journal.receipt.managed_path,
1098                model_id,
1099                &journal.artifact_identity,
1100            )?;
1101            // Revalidate the parent and leaf identity immediately before the
1102            // same-root no-replace rename. Unsupported platforms fail closed.
1103            validate_direct_child(&self.models_dir, &journal.receipt.managed_path, model_id)?;
1104            ensure_artifact_identity(
1105                &journal.receipt.managed_path,
1106                model_id,
1107                &journal.artifact_identity,
1108            )?;
1109            atomic_rename_noreplace(&journal.receipt.managed_path, &journal.quarantine_path)
1110                .map_err(|source| ModelManagementError::Io {
1111                    path: journal.receipt.managed_path.clone(),
1112                    source,
1113                })?;
1114            sync_directory(&self.models_dir).map_err(|source| ModelManagementError::Io {
1115                path: self.models_dir.clone(),
1116                source,
1117            })?;
1118            renamed_now = true;
1119            #[cfg(all(test, any(target_os = "macos", target_os = "linux")))]
1120            if let Some(hook) = &self.removal_hook {
1121                (hook.0)(RemovalPhase::AfterRename, &journal.quarantine_path);
1122            }
1123        }
1124
1125        if std::fs::symlink_metadata(&journal.quarantine_path).is_err() && renamed_now {
1126            // We detached the leaf moments ago and it is already gone from its
1127            // quarantine: something else moved it. Never report that as a
1128            // completed removal; the journal stays so a later resume can look
1129            // again.
1130            return Err(ModelManagementError::UnsafeManagedPath {
1131                model_id: model_id.clone(),
1132                path: journal.quarantine_path,
1133                reason: QUARANTINE_VANISHED_REASON.into(),
1134            });
1135        }
1136
1137        if std::fs::symlink_metadata(&journal.quarantine_path).is_ok() {
1138            // A resumed directory removal may already be partially deleted, so
1139            // the quarantine is matched on kind, device, and inode only.
1140            ensure_removal_identity(
1141                &journal.quarantine_path,
1142                model_id,
1143                &journal.artifact_identity,
1144            )?;
1145            match journal.receipt.artifact_kind {
1146                #[cfg(any(target_os = "macos", target_os = "linux"))]
1147                ManagedArtifactKind::Directory => {
1148                    self.remove_quarantined_directory(
1149                        &journal.quarantine_path,
1150                        &journal.artifact_identity,
1151                        model_id,
1152                    )?;
1153                }
1154                #[cfg(not(any(target_os = "macos", target_os = "linux")))]
1155                ManagedArtifactKind::Directory => {
1156                    return Err(ModelManagementError::UnsafeManagedPath {
1157                        model_id: model_id.clone(),
1158                        path: journal.quarantine_path,
1159                        reason:
1160                            "recursive directory removal is unsupported without object-bound traversal"
1161                                .into(),
1162                    });
1163                }
1164                ManagedArtifactKind::Symlink | ManagedArtifactKind::File => {
1165                    std::fs::remove_file(&journal.quarantine_path).map_err(|source| {
1166                        ModelManagementError::Io {
1167                            path: journal.quarantine_path.clone(),
1168                            source,
1169                        }
1170                    })?;
1171                }
1172            }
1173            sync_directory(&self.models_dir).map_err(|source| ModelManagementError::Io {
1174                path: self.models_dir.clone(),
1175                source,
1176            })?;
1177        }
1178
1179        let receipt_path = self.receipt_path(model_id);
1180        match std::fs::remove_file(&receipt_path) {
1181            Ok(()) => {
1182                sync_directory(&self.receipts_root).map_err(|source| ModelManagementError::Io {
1183                    path: self.receipts_root.clone(),
1184                    source,
1185                })?
1186            }
1187            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
1188            Err(source) => {
1189                return Err(ModelManagementError::Io {
1190                    path: receipt_path,
1191                    source,
1192                });
1193            }
1194        }
1195        let journal_path = self.removal_journal_path(model_id);
1196        match std::fs::remove_file(&journal_path) {
1197            Ok(()) => {
1198                sync_directory(&self.removals_root).map_err(|source| ModelManagementError::Io {
1199                    path: self.removals_root.clone(),
1200                    source,
1201                })?
1202            }
1203            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
1204            Err(source) => {
1205                return Err(ModelManagementError::Io {
1206                    path: journal_path,
1207                    source,
1208                });
1209            }
1210        }
1211        Ok(RemoveFromCarResult {
1212            model_id: model_id.clone(),
1213            artifact_kind: journal.receipt.artifact_kind,
1214            preserved_shared_cache_references: journal.receipt.shared_cache_references,
1215        })
1216    }
1217
1218    fn resume_pending_quarantines(&self) {
1219        let Ok(entries) = std::fs::read_dir(&self.removals_root) else {
1220            return;
1221        };
1222        for entry in entries.flatten() {
1223            if !is_hashed_state_filename(&entry.file_name()) {
1224                continue;
1225            }
1226            let Ok(bytes) = read_private_state(&entry.path()) else {
1227                continue;
1228            };
1229            let Ok(candidate) = serde_json::from_slice::<RemovalJournal>(&bytes) else {
1230                continue;
1231            };
1232            if entry.path() != self.removal_journal_path(&candidate.model_id) {
1233                continue;
1234            }
1235            let Ok(guard) = self.begin_mutation(&candidate.model_id) else {
1236                continue;
1237            };
1238            let Ok(Some(journal)) = self.load_removal_journal(&candidate.model_id) else {
1239                continue;
1240            };
1241            if std::fs::symlink_metadata(&journal.receipt.managed_path).is_ok() {
1242                continue;
1243            }
1244            let _ = guard.resume_existing(journal);
1245        }
1246    }
1247
1248    fn open_identity_lock(
1249        &self,
1250        path: &Path,
1251        model_id: &str,
1252    ) -> Result<File, ModelManagementError> {
1253        let parent = path.parent().expect("lock path has parent");
1254        create_private_dir(parent)?;
1255        // A new identity inode is visible before it can be locked and filled.
1256        // Serialize that window with a permanent, non-JSON gate. The gate itself
1257        // needs no initialization payload, so whichever opener locks it first
1258        // can safely initialize an identity. Never rename or unlink either inode.
1259        let initialization_path = parent.join(".initialization.lock");
1260        let initialization = match open_private_new(&initialization_path) {
1261            Ok(file) => file,
1262            Err(ModelManagementError::Io { source, .. })
1263                if source.kind() == std::io::ErrorKind::AlreadyExists =>
1264            {
1265                open_existing_identity_lock(&initialization_path)?
1266            }
1267            Err(error) => return Err(error),
1268        };
1269        initialization
1270            .lock()
1271            .map_err(|source| ModelManagementError::Io {
1272                path: initialization_path,
1273                source,
1274            })?;
1275        let _initialization = OwnedFileLock::new(initialization);
1276        // This guard drops on return, before callers wait for any existing
1277        // activity/mutation lock; existing lease holders never need this gate.
1278        match open_private_new(path) {
1279            Ok(file) => {
1280                #[cfg(test)]
1281                if let Some(hook) = self.identity_init_hook {
1282                    hook(path);
1283                }
1284                file.lock().map_err(|source| ModelManagementError::Io {
1285                    path: path.to_path_buf(),
1286                    source,
1287                })?;
1288                let mut file = OwnedFileLock::new(file);
1289                let bytes = serde_json::to_vec(&IdentityRecord {
1290                    model_id: model_id.to_string(),
1291                })
1292                .map_err(|error| ModelManagementError::InvalidState {
1293                    path: path.to_path_buf(),
1294                    message: error.to_string(),
1295                })?;
1296                file.write_all(&bytes)
1297                    .and_then(|_| file.sync_all())
1298                    .map_err(|source| ModelManagementError::Io {
1299                        path: path.to_path_buf(),
1300                        source,
1301                    })?;
1302                return file
1303                    .into_unlocked_file()
1304                    .map_err(|source| ModelManagementError::Io {
1305                        path: path.to_path_buf(),
1306                        source,
1307                    });
1308            }
1309            Err(ModelManagementError::Io { source, .. })
1310                if source.kind() == std::io::ErrorKind::AlreadyExists => {}
1311            Err(error) => return Err(error),
1312        }
1313        open_existing_identity_lock(path)
1314    }
1315
1316    fn validate_managed_artifact(
1317        &self,
1318        receipt: &InstallReceipt,
1319    ) -> Result<(), ModelManagementError> {
1320        let models_root = validate_models_root(&self.models_dir, &receipt.model_id)?;
1321        // Both the HF home and the hub itself: `HF_HUB_CACHE` can put the hub
1322        // anywhere, and that is where the shared bytes live.
1323        for hf_root in [crate::hf_cache::hf_home(), crate::hf_cache::hub_dir()] {
1324            let hf_root = crate::resource_policy::normalized_state_root_key(&hf_root);
1325            if models_root.starts_with(&hf_root) || hf_root.starts_with(&models_root) {
1326                return Err(ModelManagementError::UnsafeManagedPath {
1327                    model_id: receipt.model_id.clone(),
1328                    path: receipt.managed_path.clone(),
1329                    reason: "managed models root overlaps the shared Hugging Face cache".into(),
1330                });
1331            }
1332        }
1333        let parent = receipt.managed_path.parent().ok_or_else(|| {
1334            ModelManagementError::UnsafeManagedPath {
1335                model_id: receipt.model_id.clone(),
1336                path: receipt.managed_path.clone(),
1337                reason: "managed path has no parent".into(),
1338            }
1339        })?;
1340        let canonical_parent = canonical_directory(parent)?;
1341        if canonical_parent != models_root || receipt.managed_path == models_root {
1342            return Err(ModelManagementError::UnsafeManagedPath {
1343                model_id: receipt.model_id.clone(),
1344                path: receipt.managed_path.clone(),
1345                reason: "artifact is not a direct child of the managed models root".into(),
1346            });
1347        }
1348        let actual = artifact_kind(&receipt.managed_path, &receipt.model_id)?;
1349        if actual != receipt.artifact_kind {
1350            return Err(ModelManagementError::UnsafeManagedPath {
1351                model_id: receipt.model_id.clone(),
1352                path: receipt.managed_path.clone(),
1353                reason: format!(
1354                    "receipt says {:?}, filesystem is {:?}",
1355                    receipt.artifact_kind, actual
1356                ),
1357            });
1358        }
1359        if actual == ManagedArtifactKind::Symlink {
1360            let target = receipt
1361                .managed_path
1362                .plain_canonicalize()
1363                .map_err(|source| ModelManagementError::Io {
1364                    path: receipt.managed_path.clone(),
1365                    source,
1366                })?;
1367            // A receipt written before canonical paths were plain holds the
1368            // Windows verbatim spelling; compare like with like.
1369            if !receipt
1370                .shared_cache_references
1371                .iter()
1372                .any(|path| car_winenv::plain_path(path) == target)
1373            {
1374                return Err(ModelManagementError::UnsafeManagedPath {
1375                    model_id: receipt.model_id.clone(),
1376                    path: receipt.managed_path.clone(),
1377                    reason: "symlink target does not match a preserved shared-cache reference"
1378                        .into(),
1379                });
1380            }
1381        } else {
1382            let current_references =
1383                collect_shared_references(&receipt.managed_path, &receipt.model_id)?;
1384            let recorded: Vec<PathBuf> = receipt
1385                .shared_cache_references
1386                .iter()
1387                .map(|path| car_winenv::plain_path(path))
1388                .collect();
1389            if current_references != recorded {
1390                return Err(ModelManagementError::UnsafeManagedPath {
1391                    model_id: receipt.model_id.clone(),
1392                    path: receipt.managed_path.clone(),
1393                    reason: "shared-cache references no longer match the install receipt".into(),
1394                });
1395            }
1396        }
1397        Ok(())
1398    }
1399}
1400
1401// Closing a descriptor alone does not release flock when a concurrently
1402// spawned child inherited the open-file description. End ownership explicitly
1403// at the Rust guard boundary, including temporary initialization/activity locks.
1404struct OwnedFileLock(Option<File>);
1405
1406impl OwnedFileLock {
1407    fn new(file: File) -> Self {
1408        Self(Some(file))
1409    }
1410
1411    fn into_unlocked_file(mut self) -> std::io::Result<File> {
1412        self.unlock()?;
1413        Ok(self.0.take().expect("owned lock has a file"))
1414    }
1415}
1416
1417impl std::ops::Deref for OwnedFileLock {
1418    type Target = File;
1419
1420    fn deref(&self) -> &File {
1421        self.0.as_ref().expect("owned lock has a file")
1422    }
1423}
1424
1425impl std::ops::DerefMut for OwnedFileLock {
1426    fn deref_mut(&mut self) -> &mut File {
1427        self.0.as_mut().expect("owned lock has a file")
1428    }
1429}
1430
1431impl Drop for OwnedFileLock {
1432    fn drop(&mut self) {
1433        if let Some(file) = &self.0 {
1434            let _ = file.unlock();
1435        }
1436    }
1437}
1438
1439pub(crate) struct ModelMutationGuard {
1440    store: ModelManagementStore,
1441    model_id: String,
1442    _file: OwnedFileLock,
1443}
1444
1445/// The exclusive side of a model's activity lease, held by a removal while
1446/// it deletes: no request anywhere on the machine holds a lease on the model
1447/// while this lives.
1448pub(crate) struct IdleActivity {
1449    _file: OwnedFileLock,
1450}
1451
1452impl ModelMutationGuard {
1453    /// Take the model's activity lease exclusively, as a receipt-backed
1454    /// removal does for the whole of its deletion; `ModelInUse` while any
1455    /// process holds a lease. Under this guard a new lease cannot start (a
1456    /// lease enters the shared side of the mutation lock first), so once this
1457    /// succeeds the model stays idle until it is dropped.
1458    pub(crate) fn hold_idle_activity(&self) -> Result<IdleActivity, ModelManagementError> {
1459        Ok(IdleActivity {
1460            _file: self.store.lock_idle_activity(&self.model_id)?,
1461        })
1462    }
1463
1464    pub(crate) fn remove(
1465        self,
1466        removal_generation: u64,
1467    ) -> Result<RemoveFromCarResult, ModelManagementError> {
1468        self.store
1469            .remove_with_mutation(&self.model_id, removal_generation)
1470    }
1471
1472    fn resume_existing(
1473        self,
1474        journal: RemovalJournal,
1475    ) -> Result<RemoveFromCarResult, ModelManagementError> {
1476        if journal.model_id != self.model_id {
1477            return Err(ModelManagementError::IdentityMismatch {
1478                path: self.store.removal_journal_path(&self.model_id),
1479                expected: self.model_id,
1480                actual: journal.model_id,
1481            });
1482        }
1483        let _activity = self.store.lock_idle_activity(&self.model_id)?;
1484        write_private_json(
1485            &self.store.tombstone_path(&self.model_id),
1486            &RemovalTombstone {
1487                model_id: self.model_id.clone(),
1488                removal_generation: journal.removal_generation,
1489                artifact_kind: Some(journal.receipt.artifact_kind),
1490                preserved_shared_cache_references: journal.receipt.shared_cache_references.clone(),
1491            },
1492        )?;
1493        self.store.resume_removal_journal(journal)
1494    }
1495}
1496
1497pub struct ModelLease {
1498    _file: OwnedFileLock,
1499}
1500
1501#[derive(Debug, Clone, PartialEq, Eq)]
1502pub struct RemoveFromCarResult {
1503    pub model_id: String,
1504    pub artifact_kind: ManagedArtifactKind,
1505    pub preserved_shared_cache_references: Vec<PathBuf>,
1506}
1507
1508#[derive(Debug, Serialize, Deserialize)]
1509#[serde(deny_unknown_fields)]
1510struct IdentityRecord {
1511    model_id: String,
1512}
1513
1514#[derive(Debug, Serialize, Deserialize)]
1515#[serde(deny_unknown_fields)]
1516struct RemovalTombstone {
1517    model_id: String,
1518    removal_generation: u64,
1519    #[serde(default)]
1520    artifact_kind: Option<ManagedArtifactKind>,
1521    #[serde(default)]
1522    preserved_shared_cache_references: Vec<PathBuf>,
1523}
1524
1525#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
1526#[serde(deny_unknown_fields)]
1527struct ArtifactIdentity {
1528    kind: ManagedArtifactKind,
1529    #[serde(default)]
1530    device: u64,
1531    #[serde(default)]
1532    inode: u64,
1533    #[serde(default)]
1534    file_len: u64,
1535}
1536
1537#[derive(Debug, Clone, Serialize, Deserialize)]
1538#[serde(deny_unknown_fields)]
1539struct RemovalJournal {
1540    model_id: String,
1541    removal_generation: u64,
1542    receipt: InstallReceipt,
1543    quarantine_path: PathBuf,
1544    artifact_identity: ArtifactIdentity,
1545}
1546
1547#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
1548#[serde(deny_unknown_fields)]
1549struct InstallJournal {
1550    model_id: String,
1551    receipt: InstallReceipt,
1552    #[serde(default)]
1553    staging_path: Option<PathBuf>,
1554    #[serde(default)]
1555    staging_identity: Option<ArtifactIdentity>,
1556}
1557
1558/// The receipt file name [`read_receipt`] reads for `model_id`, for tests that
1559/// stand a receipt up without a store. Its one caller is a Unix-only test.
1560#[cfg(all(test, unix))]
1561pub(crate) fn receipt_file_name_for_test(model_id: &str) -> String {
1562    hashed_filename(model_id)
1563}
1564
1565fn hashed_filename(model_id: &str) -> String {
1566    let digest = Sha256::digest(model_id.as_bytes());
1567    format!("{}.json", hex::encode(digest))
1568}
1569
1570fn is_hashed_state_filename(name: &std::ffi::OsStr) -> bool {
1571    let Some(name) = name.to_str() else {
1572        return false;
1573    };
1574    let Some(hex) = name.strip_suffix(".json") else {
1575        return false;
1576    };
1577    hex.len() == 64
1578        && hex
1579            .bytes()
1580            .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase())
1581}
1582
1583fn validate_model_id(model_id: &str) -> Result<(), ModelManagementError> {
1584    if !model_id.is_empty() && model_id.len() <= MAX_MODEL_ID_BYTES {
1585        return Ok(());
1586    }
1587    Err(ModelManagementError::InvalidState {
1588        path: PathBuf::new(),
1589        message: "model id must contain 1..=4096 UTF-8 bytes".into(),
1590    })
1591}
1592
1593fn validate_managed_leaf(
1594    managed_leaf: &str,
1595    models_dir: &Path,
1596) -> Result<(), ModelManagementError> {
1597    if Path::new(managed_leaf).components().count() == 1
1598        && matches!(
1599            Path::new(managed_leaf).components().next(),
1600            Some(std::path::Component::Normal(_))
1601        )
1602    {
1603        return Ok(());
1604    }
1605    Err(ModelManagementError::UnsafeManagedPath {
1606        model_id: managed_leaf.to_string(),
1607        path: models_dir.join(managed_leaf),
1608        reason: "managed model name must be one ordinary path component".into(),
1609    })
1610}
1611
1612fn validate_direct_child(
1613    root: &Path,
1614    candidate: &Path,
1615    model_id: &str,
1616) -> Result<(), ModelManagementError> {
1617    let canonical_root = validate_models_root(root, model_id)?;
1618    let parent = candidate
1619        .parent()
1620        .ok_or_else(|| ModelManagementError::UnsafeManagedPath {
1621            model_id: model_id.to_string(),
1622            path: candidate.to_path_buf(),
1623            reason: "managed path has no parent".into(),
1624        })?;
1625    if canonical_directory(parent)? != canonical_root {
1626        return Err(ModelManagementError::UnsafeManagedPath {
1627            model_id: model_id.to_string(),
1628            path: candidate.to_path_buf(),
1629            reason: "managed path is not a direct child of the models root".into(),
1630        });
1631    }
1632    Ok(())
1633}
1634
1635fn validate_models_root(root: &Path, model_id: &str) -> Result<PathBuf, ModelManagementError> {
1636    let metadata = std::fs::symlink_metadata(root).map_err(|source| ModelManagementError::Io {
1637        path: root.to_path_buf(),
1638        source,
1639    })?;
1640    if metadata.file_type().is_symlink() {
1641        return Err(ModelManagementError::UnsafeManagedPath {
1642            model_id: model_id.to_string(),
1643            path: root.to_path_buf(),
1644            reason: "managed models root must not be a symlink".into(),
1645        });
1646    }
1647    #[cfg(windows)]
1648    {
1649        use std::os::windows::fs::MetadataExt;
1650        const FILE_ATTRIBUTE_REPARSE_POINT: u32 = 0x400;
1651        if metadata.file_attributes() & FILE_ATTRIBUTE_REPARSE_POINT != 0 {
1652            return Err(ModelManagementError::UnsafeManagedPath {
1653                model_id: model_id.to_string(),
1654                path: root.to_path_buf(),
1655                reason: "managed models root must not be a reparse point".into(),
1656            });
1657        }
1658    }
1659    if !metadata.is_dir() {
1660        return Err(ModelManagementError::UnsafeManagedPath {
1661            model_id: model_id.to_string(),
1662            path: root.to_path_buf(),
1663            reason: "managed models root is not a directory".into(),
1664        });
1665    }
1666    canonical_directory(root)
1667}
1668
1669fn ensure_identity(path: &Path, expected: &str, actual: &str) -> Result<(), ModelManagementError> {
1670    if expected == actual {
1671        return Ok(());
1672    }
1673    Err(ModelManagementError::IdentityMismatch {
1674        path: path.to_path_buf(),
1675        expected: expected.to_string(),
1676        actual: actual.to_string(),
1677    })
1678}
1679
1680fn validate_lock_identity(
1681    file: &mut File,
1682    path: &Path,
1683    model_id: &str,
1684) -> Result<(), ModelManagementError> {
1685    file.rewind().map_err(|source| ModelManagementError::Io {
1686        path: path.to_path_buf(),
1687        source,
1688    })?;
1689    let mut bytes = Vec::new();
1690    file.take(16 * 1024)
1691        .read_to_end(&mut bytes)
1692        .map_err(|source| ModelManagementError::Io {
1693            path: path.to_path_buf(),
1694            source,
1695        })?;
1696    let record: IdentityRecord =
1697        serde_json::from_slice(&bytes).map_err(|error| ModelManagementError::InvalidState {
1698            path: path.to_path_buf(),
1699            message: error.to_string(),
1700        })?;
1701    ensure_identity(path, model_id, &record.model_id)
1702}
1703
1704fn canonical_directory(path: &Path) -> Result<PathBuf, ModelManagementError> {
1705    path.plain_canonicalize()
1706        .map_err(|source| ModelManagementError::Io {
1707            path: path.to_path_buf(),
1708            source,
1709        })
1710}
1711
1712fn artifact_identity(
1713    path: &Path,
1714    model_id: &str,
1715) -> Result<ArtifactIdentity, ModelManagementError> {
1716    let metadata = std::fs::symlink_metadata(path).map_err(|source| ModelManagementError::Io {
1717        path: path.to_path_buf(),
1718        source,
1719    })?;
1720    reject_windows_reparse(&metadata, path, model_id)?;
1721    let kind = if metadata.file_type().is_symlink() {
1722        ManagedArtifactKind::Symlink
1723    } else if metadata.is_dir() {
1724        ManagedArtifactKind::Directory
1725    } else if metadata.is_file() {
1726        ManagedArtifactKind::File
1727    } else {
1728        return Err(ModelManagementError::UnsafeManagedPath {
1729            model_id: model_id.to_string(),
1730            path: path.to_path_buf(),
1731            reason: "unsupported artifact type".into(),
1732        });
1733    };
1734    #[cfg(unix)]
1735    {
1736        use std::os::unix::fs::MetadataExt;
1737        Ok(ArtifactIdentity {
1738            kind,
1739            device: metadata.dev(),
1740            inode: metadata.ino(),
1741            file_len: metadata.len(),
1742        })
1743    }
1744    #[cfg(not(unix))]
1745    {
1746        Ok(ArtifactIdentity {
1747            kind,
1748            device: 0,
1749            inode: 0,
1750            file_len: metadata.len(),
1751        })
1752    }
1753}
1754
1755fn ensure_artifact_identity(
1756    path: &Path,
1757    model_id: &str,
1758    expected: &ArtifactIdentity,
1759) -> Result<(), ModelManagementError> {
1760    let actual = artifact_identity(path, model_id)?;
1761    if &actual == expected {
1762        return Ok(());
1763    }
1764    Err(ModelManagementError::UnsafeManagedPath {
1765        model_id: model_id.to_string(),
1766        path: path.to_path_buf(),
1767        reason: "managed artifact identity changed during removal".into(),
1768    })
1769}
1770
1771impl ArtifactIdentity {
1772    /// Removal-phase match. A directory's length changes as its entries are
1773    /// deleted, so a resumed directory removal compares kind, device, and inode
1774    /// only; files and symlinks keep the exact comparison. Install-intent
1775    /// recovery keeps using exact equality.
1776    fn matches_for_removal(&self, actual: &ArtifactIdentity) -> bool {
1777        self.kind == actual.kind
1778            && self.device == actual.device
1779            && self.inode == actual.inode
1780            && (self.kind == ManagedArtifactKind::Directory || self.file_len == actual.file_len)
1781    }
1782}
1783
1784fn ensure_removal_identity(
1785    path: &Path,
1786    model_id: &str,
1787    expected: &ArtifactIdentity,
1788) -> Result<(), ModelManagementError> {
1789    let actual = artifact_identity(path, model_id)?;
1790    if expected.matches_for_removal(&actual) {
1791        return Ok(());
1792    }
1793    Err(ModelManagementError::UnsafeManagedPath {
1794        model_id: model_id.to_string(),
1795        path: path.to_path_buf(),
1796        reason: "managed artifact identity changed during removal".into(),
1797    })
1798}
1799
1800fn artifact_kind(path: &Path, model_id: &str) -> Result<ManagedArtifactKind, ModelManagementError> {
1801    let metadata = std::fs::symlink_metadata(path).map_err(|source| ModelManagementError::Io {
1802        path: path.to_path_buf(),
1803        source,
1804    })?;
1805    reject_windows_reparse(&metadata, path, model_id)?;
1806    if metadata.file_type().is_symlink() {
1807        Ok(ManagedArtifactKind::Symlink)
1808    } else if metadata.is_dir() {
1809        Ok(ManagedArtifactKind::Directory)
1810    } else if metadata.is_file() {
1811        Ok(ManagedArtifactKind::File)
1812    } else {
1813        Err(ModelManagementError::UnsafeManagedPath {
1814            model_id: model_id.to_string(),
1815            path: path.to_path_buf(),
1816            reason: "unsupported artifact type".into(),
1817        })
1818    }
1819}
1820
1821fn collect_shared_references(
1822    path: &Path,
1823    model_id: &str,
1824) -> Result<Vec<PathBuf>, ModelManagementError> {
1825    let metadata = std::fs::symlink_metadata(path).map_err(|source| ModelManagementError::Io {
1826        path: path.to_path_buf(),
1827        source,
1828    })?;
1829    if metadata.file_type().is_symlink() {
1830        return path
1831            .plain_canonicalize()
1832            .map(|target| vec![target])
1833            .map_err(|source| ModelManagementError::Io {
1834                path: path.to_path_buf(),
1835                source,
1836            });
1837    }
1838    if !metadata.is_dir() {
1839        return Ok(Vec::new());
1840    }
1841    let mut references = Vec::new();
1842    for entry in std::fs::read_dir(path).map_err(|source| ModelManagementError::Io {
1843        path: path.to_path_buf(),
1844        source,
1845    })? {
1846        let entry = entry.map_err(|source| ModelManagementError::Io {
1847            path: path.to_path_buf(),
1848            source,
1849        })?;
1850        let entry_path = entry.path();
1851        let entry_metadata =
1852            std::fs::symlink_metadata(&entry_path).map_err(|source| ModelManagementError::Io {
1853                path: entry_path.clone(),
1854                source,
1855            })?;
1856        if entry_metadata.file_type().is_symlink() {
1857            let target =
1858                entry_path
1859                    .plain_canonicalize()
1860                    .map_err(|source| ModelManagementError::Io {
1861                        path: entry_path.clone(),
1862                        source,
1863                    })?;
1864            if std::fs::metadata(&target)
1865                .map_err(|source| ModelManagementError::Io {
1866                    path: target.clone(),
1867                    source,
1868                })?
1869                .is_dir()
1870            {
1871                return Err(ModelManagementError::UnsafeManagedPath {
1872                    model_id: model_id.to_string(),
1873                    path: entry_path,
1874                    reason: "managed directory contains a directory symlink".into(),
1875                });
1876            }
1877            references.push(target);
1878        } else if entry_metadata.is_dir() {
1879            references.extend(collect_shared_references(&entry_path, model_id)?);
1880        }
1881    }
1882    references.sort();
1883    references.dedup();
1884    Ok(references)
1885}
1886
1887#[cfg(unix)]
1888fn create_symlink(target: &Path, link: &Path) -> Result<(), ModelManagementError> {
1889    std::os::unix::fs::symlink(target, link).map_err(|source| ModelManagementError::Io {
1890        path: link.to_path_buf(),
1891        source,
1892    })
1893}
1894
1895#[cfg(windows)]
1896fn create_symlink(target: &Path, link: &Path) -> Result<(), ModelManagementError> {
1897    let result = if target.is_dir() {
1898        std::os::windows::fs::symlink_dir(target, link)
1899    } else {
1900        std::os::windows::fs::symlink_file(target, link)
1901    };
1902    result.map_err(|source| ModelManagementError::Io {
1903        path: link.to_path_buf(),
1904        source,
1905    })
1906}
1907
1908fn create_private_dir(path: &Path) -> Result<(), ModelManagementError> {
1909    std::fs::create_dir_all(path).map_err(|source| ModelManagementError::Io {
1910        path: path.to_path_buf(),
1911        source,
1912    })?;
1913    harden_private_directory(path)?;
1914    Ok(())
1915}
1916
1917fn harden_private_directory(path: &Path) -> Result<(), ModelManagementError> {
1918    let metadata = std::fs::symlink_metadata(path).map_err(|source| ModelManagementError::Io {
1919        path: path.to_path_buf(),
1920        source,
1921    })?;
1922    if metadata.file_type().is_symlink() {
1923        return Err(ModelManagementError::UnsafeManagedPath {
1924            model_id: "model-management-state".into(),
1925            path: path.to_path_buf(),
1926            reason: "private model-management directory must not be a symlink".into(),
1927        });
1928    }
1929    reject_windows_reparse(&metadata, path, "model-management-state")?;
1930    #[cfg(unix)]
1931    {
1932        use std::os::unix::fs::PermissionsExt;
1933        std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o700)).map_err(
1934            |source| ModelManagementError::Io {
1935                path: path.to_path_buf(),
1936                source,
1937            },
1938        )?;
1939    }
1940    Ok(())
1941}
1942
1943fn reject_windows_reparse(
1944    metadata: &std::fs::Metadata,
1945    path: &Path,
1946    model_id: &str,
1947) -> Result<(), ModelManagementError> {
1948    #[cfg(windows)]
1949    {
1950        use std::os::windows::fs::MetadataExt;
1951        const FILE_ATTRIBUTE_REPARSE_POINT: u32 = 0x400;
1952        if metadata.file_attributes() & FILE_ATTRIBUTE_REPARSE_POINT != 0 {
1953            return Err(ModelManagementError::UnsafeManagedPath {
1954                model_id: model_id.to_string(),
1955                path: path.to_path_buf(),
1956                reason: "model-management path is a Windows reparse point".into(),
1957            });
1958        }
1959    }
1960    #[cfg(not(windows))]
1961    let _ = (metadata, path, model_id);
1962    Ok(())
1963}
1964
1965fn open_private_new(path: &Path) -> Result<File, ModelManagementError> {
1966    let mut options = OpenOptions::new();
1967    options.read(true).write(true).create_new(true);
1968    #[cfg(unix)]
1969    {
1970        use std::os::unix::fs::OpenOptionsExt;
1971        options.mode(0o600);
1972    }
1973    options
1974        .open(path)
1975        .map_err(|source| ModelManagementError::Io {
1976            path: path.to_path_buf(),
1977            source,
1978        })
1979}
1980
1981fn open_existing_identity_lock(path: &Path) -> Result<File, ModelManagementError> {
1982    let metadata = std::fs::symlink_metadata(path).map_err(|source| ModelManagementError::Io {
1983        path: path.to_path_buf(),
1984        source,
1985    })?;
1986    if metadata.file_type().is_symlink() {
1987        return Err(ModelManagementError::InvalidState {
1988            path: path.to_path_buf(),
1989            message: "model-management lock sentinel must not be a symlink".into(),
1990        });
1991    }
1992    reject_windows_reparse(&metadata, path, "model-management-lock")?;
1993    #[cfg(unix)]
1994    {
1995        use std::os::unix::fs::MetadataExt;
1996        if metadata.nlink() != 1 {
1997            return Err(ModelManagementError::InvalidState {
1998                path: path.to_path_buf(),
1999                message: "model-management lock sentinel must not be hard-linked".into(),
2000            });
2001        }
2002    }
2003    let mut options = OpenOptions::new();
2004    options.read(true).write(true);
2005    #[cfg(unix)]
2006    {
2007        use std::os::unix::fs::OpenOptionsExt;
2008        options.custom_flags(libc::O_NOFOLLOW);
2009    }
2010    #[cfg(windows)]
2011    {
2012        use std::os::windows::fs::OpenOptionsExt;
2013        const FILE_FLAG_OPEN_REPARSE_POINT: u32 = 0x0020_0000;
2014        options.custom_flags(FILE_FLAG_OPEN_REPARSE_POINT);
2015    }
2016    let file = options
2017        .open(path)
2018        .map_err(|source| ModelManagementError::Io {
2019            path: path.to_path_buf(),
2020            source,
2021        })?;
2022    let opened_metadata = file.metadata().map_err(|source| ModelManagementError::Io {
2023        path: path.to_path_buf(),
2024        source,
2025    })?;
2026    if !opened_metadata.is_file() {
2027        return Err(ModelManagementError::InvalidState {
2028            path: path.to_path_buf(),
2029            message: "model-management lock sentinel is not a regular file".into(),
2030        });
2031    }
2032    reject_windows_reparse(&opened_metadata, path, "model-management-lock")?;
2033    #[cfg(unix)]
2034    {
2035        use std::os::unix::fs::MetadataExt;
2036        if opened_metadata.nlink() != 1 {
2037            return Err(ModelManagementError::InvalidState {
2038                path: path.to_path_buf(),
2039                message: "opened model-management lock sentinel is hard-linked".into(),
2040            });
2041        }
2042    }
2043    Ok(file)
2044}
2045
2046/// Read a model's install receipt under `state_root` without constructing a
2047/// [`ModelManagementStore`] — whose constructor resumes interrupted removals,
2048/// which deletes. For read-only paths such as `car doctor`.
2049pub fn read_receipt(
2050    state_root: &Path,
2051    model_id: &str,
2052) -> Result<Option<InstallReceipt>, ModelManagementError> {
2053    validate_model_id(model_id)?;
2054    let state_root = crate::resource_policy::normalized_state_root_key(state_root);
2055    read_receipt_at(
2056        &receipts_dir(&state_root).join(hashed_filename(model_id)),
2057        model_id,
2058    )
2059}
2060
2061/// Where install receipts live under an already-normalized state root — the
2062/// one spelling the store and [`read_receipt`] both use.
2063fn receipts_dir(state_root: &Path) -> PathBuf {
2064    state_root.join("model-management").join("receipts")
2065}
2066
2067fn read_receipt_at(
2068    path: &Path,
2069    model_id: &str,
2070) -> Result<Option<InstallReceipt>, ModelManagementError> {
2071    let bytes = match read_private_state(path) {
2072        Ok(bytes) => bytes,
2073        Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
2074        Err(source) => {
2075            return Err(ModelManagementError::Io {
2076                path: path.to_path_buf(),
2077                source,
2078            })
2079        }
2080    };
2081    let receipt: InstallReceipt =
2082        serde_json::from_slice(&bytes).map_err(|error| ModelManagementError::InvalidState {
2083            path: path.to_path_buf(),
2084            message: error.to_string(),
2085        })?;
2086    ensure_identity(path, model_id, &receipt.model_id)?;
2087    Ok(Some(receipt))
2088}
2089
2090fn read_private_state(path: &Path) -> std::io::Result<Vec<u8>> {
2091    let mut options = OpenOptions::new();
2092    options.read(true);
2093    #[cfg(unix)]
2094    {
2095        use std::os::unix::fs::OpenOptionsExt;
2096        options.custom_flags(libc::O_NOFOLLOW);
2097    }
2098    let file = options.open(path)?;
2099    let metadata = file.metadata()?;
2100    if !metadata.is_file() || metadata.len() > MAX_STATE_BYTES {
2101        return Err(std::io::Error::new(
2102            std::io::ErrorKind::InvalidData,
2103            "model-management state must be a bounded regular file",
2104        ));
2105    }
2106    #[cfg(unix)]
2107    {
2108        use std::os::unix::fs::MetadataExt;
2109        if metadata.nlink() != 1 {
2110            return Err(std::io::Error::new(
2111                std::io::ErrorKind::InvalidData,
2112                "model-management state must not be hard-linked",
2113            ));
2114        }
2115    }
2116    let mut bytes = Vec::with_capacity(metadata.len() as usize);
2117    file.take(MAX_STATE_BYTES + 1).read_to_end(&mut bytes)?;
2118    if bytes.len() as u64 > MAX_STATE_BYTES {
2119        return Err(std::io::Error::new(
2120            std::io::ErrorKind::InvalidData,
2121            "model-management state exceeds the size limit",
2122        ));
2123    }
2124    Ok(bytes)
2125}
2126
2127fn write_private_json<T: Serialize>(path: &Path, value: &T) -> Result<(), ModelManagementError> {
2128    let _guard = state_mutation_lock()
2129        .lock()
2130        .unwrap_or_else(std::sync::PoisonError::into_inner);
2131    let parent = path
2132        .parent()
2133        .expect("model-management record path has a parent");
2134    create_private_dir(parent)?;
2135    let tmp = (0..16)
2136        .map(|_| {
2137            parent.join(format!(
2138                ".{}.{}-{:032x}.tmp",
2139                std::process::id(),
2140                path.file_name()
2141                    .and_then(|name| name.to_str())
2142                    .unwrap_or("state"),
2143                rand::random::<u128>()
2144            ))
2145        })
2146        .find(|candidate| {
2147            std::fs::symlink_metadata(candidate)
2148                .is_err_and(|error| error.kind() == std::io::ErrorKind::NotFound)
2149        })
2150        .ok_or_else(|| ModelManagementError::InvalidState {
2151            path: parent.to_path_buf(),
2152            message: "could not allocate collision-resistant state staging file".into(),
2153        })?;
2154    let bytes =
2155        serde_json::to_vec_pretty(value).map_err(|error| ModelManagementError::InvalidState {
2156            path: path.to_path_buf(),
2157            message: error.to_string(),
2158        })?;
2159    let result = (|| {
2160        let mut file = open_private_new(&tmp)?;
2161        file.write_all(&bytes)
2162            .and_then(|_| file.sync_all())
2163            .map_err(|source| ModelManagementError::Io {
2164                path: tmp.clone(),
2165                source,
2166            })?;
2167        atomic_replace(&tmp, path).map_err(|source| ModelManagementError::Io {
2168            path: path.to_path_buf(),
2169            source,
2170        })?;
2171        car_secrets::harden_owner_only(path);
2172        sync_directory(parent).map_err(|source| ModelManagementError::Io {
2173            path: parent.to_path_buf(),
2174            source,
2175        })
2176    })();
2177    if result.is_err() {
2178        let _ = std::fs::remove_file(&tmp);
2179    }
2180    result
2181}
2182
2183fn state_mutation_lock() -> &'static Mutex<()> {
2184    static LOCK: OnceLock<Mutex<()>> = OnceLock::new();
2185    LOCK.get_or_init(|| Mutex::new(()))
2186}
2187
2188#[cfg(not(windows))]
2189fn atomic_replace(source: &Path, destination: &Path) -> std::io::Result<()> {
2190    std::fs::rename(source, destination)
2191}
2192
2193#[cfg(target_os = "macos")]
2194fn atomic_rename_noreplace(source: &Path, destination: &Path) -> std::io::Result<()> {
2195    use std::ffi::CString;
2196    use std::os::unix::ffi::OsStrExt;
2197
2198    let source = CString::new(source.as_os_str().as_bytes())
2199        .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidInput, error))?;
2200    let destination = CString::new(destination.as_os_str().as_bytes())
2201        .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidInput, error))?;
2202    // SAFETY: both C strings are NUL-terminated and live for the call.
2203    let result =
2204        unsafe { libc::renamex_np(source.as_ptr(), destination.as_ptr(), libc::RENAME_EXCL) };
2205    if result == 0 {
2206        Ok(())
2207    } else {
2208        Err(std::io::Error::last_os_error())
2209    }
2210}
2211
2212#[cfg(target_os = "linux")]
2213fn atomic_rename_noreplace(source: &Path, destination: &Path) -> std::io::Result<()> {
2214    use std::ffi::CString;
2215    use std::os::unix::ffi::OsStrExt;
2216
2217    let source = CString::new(source.as_os_str().as_bytes())
2218        .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidInput, error))?;
2219    let destination = CString::new(destination.as_os_str().as_bytes())
2220        .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidInput, error))?;
2221    // SAFETY: both C strings are NUL-terminated and live for the call.
2222    let result = unsafe {
2223        libc::renameat2(
2224            libc::AT_FDCWD,
2225            source.as_ptr(),
2226            libc::AT_FDCWD,
2227            destination.as_ptr(),
2228            libc::RENAME_NOREPLACE,
2229        )
2230    };
2231    if result == 0 {
2232        Ok(())
2233    } else {
2234        Err(std::io::Error::last_os_error())
2235    }
2236}
2237
2238#[cfg(windows)]
2239fn atomic_rename_noreplace(source: &Path, destination: &Path) -> std::io::Result<()> {
2240    use std::os::windows::ffi::OsStrExt;
2241
2242    const MOVEFILE_WRITE_THROUGH: u32 = 0x8;
2243    #[link(name = "kernel32")]
2244    unsafe extern "system" {
2245        fn MoveFileExW(existing: *const u16, new: *const u16, flags: u32) -> i32;
2246    }
2247    let source = source
2248        .as_os_str()
2249        .encode_wide()
2250        .chain(Some(0))
2251        .collect::<Vec<_>>();
2252    let destination = destination
2253        .as_os_str()
2254        .encode_wide()
2255        .chain(Some(0))
2256        .collect::<Vec<_>>();
2257    // No REPLACE_EXISTING flag: a pre-existing destination fails closed.
2258    let moved = unsafe {
2259        MoveFileExW(
2260            source.as_ptr(),
2261            destination.as_ptr(),
2262            MOVEFILE_WRITE_THROUGH,
2263        )
2264    };
2265    if moved == 0 {
2266        Err(std::io::Error::last_os_error())
2267    } else {
2268        Ok(())
2269    }
2270}
2271
2272#[cfg(not(any(target_os = "macos", target_os = "linux", windows)))]
2273fn atomic_rename_noreplace(_source: &Path, _destination: &Path) -> std::io::Result<()> {
2274    Err(std::io::Error::new(
2275        std::io::ErrorKind::Unsupported,
2276        "atomic no-replace publication is unavailable on this platform",
2277    ))
2278}
2279
2280#[cfg(windows)]
2281fn atomic_replace(source: &Path, destination: &Path) -> std::io::Result<()> {
2282    use std::os::windows::ffi::OsStrExt;
2283
2284    const MOVEFILE_REPLACE_EXISTING: u32 = 0x1;
2285    const MOVEFILE_WRITE_THROUGH: u32 = 0x8;
2286    #[link(name = "kernel32")]
2287    unsafe extern "system" {
2288        fn MoveFileExW(existing: *const u16, new: *const u16, flags: u32) -> i32;
2289    }
2290    let source = source
2291        .as_os_str()
2292        .encode_wide()
2293        .chain(Some(0))
2294        .collect::<Vec<_>>();
2295    let destination = destination
2296        .as_os_str()
2297        .encode_wide()
2298        .chain(Some(0))
2299        .collect::<Vec<_>>();
2300    // SAFETY: both pointers remain valid NUL-terminated UTF-16 buffers for
2301    // the duration of the call.
2302    let replaced = unsafe {
2303        MoveFileExW(
2304            source.as_ptr(),
2305            destination.as_ptr(),
2306            MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH,
2307        )
2308    };
2309    if replaced == 0 {
2310        Err(std::io::Error::last_os_error())
2311    } else {
2312        Ok(())
2313    }
2314}
2315
2316#[cfg(unix)]
2317fn sync_directory(path: &Path) -> std::io::Result<()> {
2318    File::open(path)?.sync_all()
2319}
2320
2321/// Object-bound deletion of a quarantined managed directory (macOS/Linux).
2322///
2323/// Everything after the first `open` is relative to descriptors CAR itself
2324/// opened: children are opened before they are trusted, symlinks are unlinked
2325/// and never followed, a change of device (and on Linux any mount crossing)
2326/// refuses the walk, entries are captured first and each is rechecked against
2327/// its captured identity immediately before its unlink, and a directory is
2328/// removed only when it is empty and still the object that was captured. POSIX
2329/// has no unlink-by-descriptor, so the one residual is the recheck → `unlinkat`
2330/// window on a captured name, the same window `std::fs::remove_dir_all` and
2331/// `rm -rf` accept; a name that appears after capture is never opened or
2332/// unlinked and makes its parent's `rmdir` fail closed instead.
2333#[cfg(any(target_os = "macos", target_os = "linux"))]
2334mod object_bound {
2335    use std::ffi::{CStr, CString};
2336    use std::fs::File;
2337    use std::os::fd::{AsRawFd, FromRawFd};
2338    use std::os::unix::ffi::OsStrExt;
2339    use std::os::unix::fs::MetadataExt;
2340    use std::path::Path;
2341
2342    pub(super) const MAX_REMOVAL_DEPTH: usize = 32;
2343
2344    const DIRECTORY_FLAGS: libc::c_int =
2345        libc::O_RDONLY | libc::O_DIRECTORY | libc::O_CLOEXEC | libc::O_NOFOLLOW;
2346
2347    #[derive(Clone, Copy, Debug, PartialEq, Eq)]
2348    pub(super) struct Identity {
2349        pub(super) device: u64,
2350        pub(super) inode: u64,
2351    }
2352
2353    #[derive(Clone, Copy, Debug, PartialEq, Eq)]
2354    pub(super) enum EntryKind {
2355        Directory,
2356        RegularFile,
2357        Symlink,
2358        Other,
2359    }
2360
2361    pub(super) fn c_name(path: &Path) -> std::io::Result<CString> {
2362        CString::new(path.as_os_str().as_bytes())
2363            .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidInput, error))
2364    }
2365
2366    /// Open a directory by path once, refusing to follow a symlink at the leaf.
2367    pub(super) fn open_directory(path: &Path) -> std::io::Result<File> {
2368        let name = c_name(path)?;
2369        // SAFETY: `name` is NUL-terminated and lives for the call.
2370        let raw = unsafe { libc::open(name.as_ptr(), DIRECTORY_FLAGS) };
2371        if raw < 0 {
2372            return Err(std::io::Error::last_os_error());
2373        }
2374        // SAFETY: `raw` is a newly-owned descriptor returned by `open`.
2375        Ok(unsafe { File::from_raw_fd(raw) })
2376    }
2377
2378    /// Open `name` relative to `parent` as a directory. Fails with `ENOTDIR`
2379    /// for a regular file and `ELOOP` for a symlink, which the walker treats
2380    /// as "not a directory".
2381    #[cfg(target_os = "macos")]
2382    pub(super) fn open_child_directory(parent: &File, name: &CStr) -> std::io::Result<File> {
2383        // SAFETY: the parent descriptor is open and `name` is NUL-terminated.
2384        let raw = unsafe { libc::openat(parent.as_raw_fd(), name.as_ptr(), DIRECTORY_FLAGS) };
2385        if raw < 0 {
2386            return Err(std::io::Error::last_os_error());
2387        }
2388        // SAFETY: `raw` is a newly-owned descriptor returned by `openat`.
2389        Ok(unsafe { File::from_raw_fd(raw) })
2390    }
2391
2392    /// Linux: `openat2` with `RESOLVE_BENEATH | RESOLVE_NO_SYMLINKS |
2393    /// RESOLVE_NO_XDEV` so the open can neither escape the parent, follow a
2394    /// symlink, nor cross a mount (bind mounts included, which share `st_dev`).
2395    #[cfg(target_os = "linux")]
2396    pub(super) fn open_child_directory(parent: &File, name: &CStr) -> std::io::Result<File> {
2397        // SAFETY: `open_how` is plain data; zero is a valid initial state.
2398        let mut how: libc::open_how = unsafe { std::mem::zeroed() };
2399        how.flags = DIRECTORY_FLAGS as u64;
2400        how.resolve = libc::RESOLVE_BENEATH | libc::RESOLVE_NO_SYMLINKS | libc::RESOLVE_NO_XDEV;
2401        // SAFETY: the parent descriptor is open, `name` is NUL-terminated, and
2402        // `how` is a valid `open_how` of the size passed.
2403        let raw = unsafe {
2404            libc::syscall(
2405                libc::SYS_openat2,
2406                parent.as_raw_fd(),
2407                name.as_ptr(),
2408                &how as *const libc::open_how,
2409                std::mem::size_of::<libc::open_how>(),
2410            )
2411        };
2412        if raw < 0 {
2413            return Err(std::io::Error::last_os_error());
2414        }
2415        // SAFETY: `raw` is a newly-owned descriptor returned by `openat2`.
2416        Ok(unsafe { File::from_raw_fd(raw as libc::c_int) })
2417    }
2418
2419    /// A child that reports a different device than the root is on another
2420    /// filesystem; the walk never descends across one. Linux additionally
2421    /// refuses bind mounts through `RESOLVE_NO_XDEV`, which share `st_dev`.
2422    pub(super) fn same_filesystem(root: &Identity, child: &Identity) -> bool {
2423        root.device == child.device
2424    }
2425
2426    pub(super) fn is_not_a_directory(error: &std::io::Error) -> bool {
2427        matches!(
2428            error.raw_os_error(),
2429            Some(libc::ENOTDIR) | Some(libc::ELOOP)
2430        )
2431    }
2432
2433    pub(super) fn identity_of(file: &File) -> std::io::Result<Identity> {
2434        let metadata = file.metadata()?;
2435        Ok(Identity {
2436            device: metadata.dev(),
2437            inode: metadata.ino(),
2438        })
2439    }
2440
2441    /// `fstatat(AT_SYMLINK_NOFOLLOW)` of `name` relative to `parent`.
2442    // `st_dev` / `st_ino` widths differ per target (macOS `dev_t` is `i32`,
2443    // Linux `dev_t` is `u64`), so the casts are needed on some targets and
2444    // redundant on others.
2445    #[allow(clippy::unnecessary_cast)]
2446    pub(super) fn stat_entry(parent: &File, name: &CStr) -> std::io::Result<(EntryKind, Identity)> {
2447        // SAFETY: `stat` is plain data; zero is a valid initial state.
2448        let mut stat: libc::stat = unsafe { std::mem::zeroed() };
2449        // SAFETY: the parent descriptor is open, `name` is NUL-terminated, and
2450        // `stat` is a valid out-pointer.
2451        let result = unsafe {
2452            libc::fstatat(
2453                parent.as_raw_fd(),
2454                name.as_ptr(),
2455                &mut stat,
2456                libc::AT_SYMLINK_NOFOLLOW,
2457            )
2458        };
2459        if result != 0 {
2460            return Err(std::io::Error::last_os_error());
2461        }
2462        let kind = match stat.st_mode & libc::S_IFMT {
2463            libc::S_IFDIR => EntryKind::Directory,
2464            libc::S_IFREG => EntryKind::RegularFile,
2465            libc::S_IFLNK => EntryKind::Symlink,
2466            _ => EntryKind::Other,
2467        };
2468        Ok((
2469            kind,
2470            Identity {
2471                device: stat.st_dev as u64,
2472                inode: stat.st_ino as u64,
2473            },
2474        ))
2475    }
2476
2477    /// Enumerate `directory` through a duplicate descriptor so the retained
2478    /// one keeps its position; names are sorted for deterministic order.
2479    pub(super) fn list_names(directory: &File) -> std::io::Result<Vec<CString>> {
2480        let dot = c".";
2481        // SAFETY: the directory descriptor is open and `dot` is NUL-terminated.
2482        let raw = unsafe { libc::openat(directory.as_raw_fd(), dot.as_ptr(), DIRECTORY_FLAGS) };
2483        if raw < 0 {
2484            return Err(std::io::Error::last_os_error());
2485        }
2486        // SAFETY: `raw` is a newly-owned descriptor; `fdopendir` takes ownership.
2487        let stream = unsafe { libc::fdopendir(raw) };
2488        if stream.is_null() {
2489            let error = std::io::Error::last_os_error();
2490            // SAFETY: `fdopendir` failed, so the descriptor is still ours to close.
2491            unsafe {
2492                libc::close(raw);
2493            }
2494            return Err(error);
2495        }
2496        let mut names = Vec::new();
2497        let enumeration = loop {
2498            set_errno(0);
2499            // SAFETY: `stream` is a valid open directory stream.
2500            let entry = unsafe { libc::readdir(stream) };
2501            if entry.is_null() {
2502                let errno = errno();
2503                break if errno == 0 {
2504                    Ok(())
2505                } else {
2506                    Err(std::io::Error::from_raw_os_error(errno))
2507                };
2508            }
2509            // SAFETY: `entry` points at a valid dirent whose name is NUL-terminated.
2510            let bytes = unsafe { CStr::from_ptr((*entry).d_name.as_ptr()) }.to_bytes();
2511            if bytes != b"." && bytes != b".." {
2512                match CString::new(bytes) {
2513                    Ok(name) => names.push(name),
2514                    Err(error) => {
2515                        break Err(std::io::Error::new(std::io::ErrorKind::InvalidData, error))
2516                    }
2517                }
2518            }
2519        };
2520        // SAFETY: `stream` is a valid open directory stream owned by us.
2521        let closed = unsafe { libc::closedir(stream) };
2522        enumeration?;
2523        if closed < 0 {
2524            return Err(std::io::Error::last_os_error());
2525        }
2526        names.sort();
2527        Ok(names)
2528    }
2529
2530    /// `unlinkat(parent, name, 0)`; a missing entry counts as already removed.
2531    pub(super) fn unlink_entry(parent: &File, name: &CStr) -> std::io::Result<()> {
2532        // SAFETY: the parent descriptor is open and `name` is NUL-terminated.
2533        let result = unsafe { libc::unlinkat(parent.as_raw_fd(), name.as_ptr(), 0) };
2534        if result == 0 {
2535            return Ok(());
2536        }
2537        let error = std::io::Error::last_os_error();
2538        if error.kind() == std::io::ErrorKind::NotFound {
2539            return Ok(());
2540        }
2541        Err(error)
2542    }
2543
2544    /// `unlinkat(parent, name, AT_REMOVEDIR)`: only ever removes an empty
2545    /// directory. A missing entry counts as already removed.
2546    pub(super) fn remove_empty_directory(parent: &File, name: &CStr) -> std::io::Result<()> {
2547        // SAFETY: the parent descriptor is open and `name` is NUL-terminated.
2548        let result =
2549            unsafe { libc::unlinkat(parent.as_raw_fd(), name.as_ptr(), libc::AT_REMOVEDIR) };
2550        if result == 0 {
2551            return Ok(());
2552        }
2553        let error = std::io::Error::last_os_error();
2554        if error.kind() == std::io::ErrorKind::NotFound {
2555            return Ok(());
2556        }
2557        Err(error)
2558    }
2559
2560    pub(super) fn is_not_empty(error: &std::io::Error) -> bool {
2561        matches!(
2562            error.raw_os_error(),
2563            Some(libc::ENOTEMPTY) | Some(libc::EEXIST)
2564        )
2565    }
2566
2567    #[cfg(target_os = "macos")]
2568    fn errno() -> i32 {
2569        // SAFETY: `__error` returns the thread-local errno pointer.
2570        unsafe { *libc::__error() }
2571    }
2572
2573    #[cfg(target_os = "macos")]
2574    fn set_errno(value: i32) {
2575        // SAFETY: `__error` returns the thread-local errno pointer.
2576        unsafe {
2577            *libc::__error() = value;
2578        }
2579    }
2580
2581    #[cfg(target_os = "linux")]
2582    fn errno() -> i32 {
2583        // SAFETY: `__errno_location` returns the thread-local errno pointer.
2584        unsafe { *libc::__errno_location() }
2585    }
2586
2587    #[cfg(target_os = "linux")]
2588    fn set_errno(value: i32) {
2589        // SAFETY: `__errno_location` returns the thread-local errno pointer.
2590        unsafe {
2591            *libc::__errno_location() = value;
2592        }
2593    }
2594}
2595
2596/// A hub directory being retired: journaled before it is detached into its
2597/// quarantine, removed once the quarantine is gone, so a crash between the
2598/// two leaves a record the next daemon start finishes from.
2599#[derive(Debug, Clone, Serialize, Deserialize)]
2600struct DetachedTreeJournal {
2601    source: PathBuf,
2602    quarantine: PathBuf,
2603    identity: ArtifactIdentity,
2604}
2605
2606/// A failed [`ModelManagementStore::detach_and_remove_tree`]: whether the
2607/// directory had already been renamed into its quarantine when it failed.
2608#[derive(Debug)]
2609pub(crate) struct DetachError {
2610    pub(crate) detached: bool,
2611    pub(crate) error: ModelManagementError,
2612}
2613
2614/// A retirement's intent to clear the tombstone its model removal writes:
2615/// retiring leaves a row fetchable, not disabled. Journaled before the
2616/// removal so a crash between removal and clear is finished at next start.
2617#[derive(Debug, Serialize, Deserialize)]
2618struct RetiringIntent {
2619    model_id: String,
2620}
2621
2622/// Quarantine names a retirement creates — the only names resume will delete.
2623pub(crate) const RETIRE_QUARANTINE_PREFIX: &str = ".car-retire-";
2624
2625impl ModelManagementStore {
2626    fn detached_tree_journals(&self) -> PathBuf {
2627        self.management_state_dir.join("hub-removals")
2628    }
2629
2630    fn retiring_intent_path(&self, model_id: &str) -> PathBuf {
2631        self.management_state_dir
2632            .join("retiring")
2633            .join(hashed_filename(model_id))
2634    }
2635
2636    /// Journal that `model_id`'s removal is a retirement, before it starts.
2637    pub(crate) fn note_retiring(&self, model_id: &str) -> Result<(), ModelManagementError> {
2638        validate_model_id(model_id)?;
2639        write_private_json(
2640            &self.retiring_intent_path(model_id),
2641            &RetiringIntent {
2642                model_id: model_id.to_string(),
2643            },
2644        )
2645    }
2646
2647    /// The retirement finished: its tombstone is cleared.
2648    pub(crate) fn retired(&self, model_id: &str) {
2649        let _ = std::fs::remove_file(self.retiring_intent_path(model_id));
2650    }
2651
2652    /// Finish retirements whose model removal completed but whose tombstone
2653    /// was never cleared. An intent whose removal has not completed — the
2654    /// receipt is still there — is kept while a removal journal can still
2655    /// finish it, and dropped once nothing can.
2656    pub fn resume_retiring(&self) -> Vec<ModelManagementError> {
2657        let mut errors = Vec::new();
2658        let Ok(read) = std::fs::read_dir(self.management_state_dir.join("retiring")) else {
2659            return errors;
2660        };
2661        for entry in read.filter_map(Result::ok) {
2662            let path = entry.path();
2663            let Some(intent) = read_private_state(&path)
2664                .ok()
2665                .and_then(|bytes| serde_json::from_slice::<RetiringIntent>(&bytes).ok())
2666                .filter(|intent| {
2667                    path.file_name() == Some(hashed_filename(&intent.model_id).as_ref())
2668                })
2669            else {
2670                tracing::warn!(path = %path.display(), "unreadable retirement intent left in place");
2671                continue;
2672            };
2673            let id = &intent.model_id;
2674            let receipt_present = self.receipt_path(id).exists();
2675            if receipt_present {
2676                if !self.removal_journal_path(id).exists() {
2677                    // The removal never ran; the row is still installed.
2678                    self.retired(id);
2679                }
2680                continue;
2681            }
2682            match self.clear_tombstone(id) {
2683                Ok(()) => self.retired(id),
2684                Err(error) => errors.push(error),
2685            }
2686        }
2687        errors
2688    }
2689
2690    /// Detach `source` — a hub repo directory CAR is retiring — into
2691    /// `quarantine` (a sibling, so the rename stays on one filesystem) and
2692    /// delete it through the same descriptor-bound walk a model removal uses:
2693    /// children opened by descriptor before they are trusted, symlinks
2694    /// unlinked and never followed, no filesystem crossing.
2695    ///
2696    /// The directory's identity is captured first, `verify` (the caller's
2697    /// containment check) runs next, and the identity is re-read immediately
2698    /// before the journal and rename, so what is renamed is what was checked.
2699    /// A failure after the rename says so ([`DetachError::detached`]): the
2700    /// directory is gone from its path and the next start finishes it.
2701    /// Refused outright where directory removal is unsupported, before
2702    /// anything is written or renamed.
2703    pub(crate) fn detach_and_remove_tree(
2704        &self,
2705        source: &Path,
2706        quarantine: &Path,
2707        label: &str,
2708        verify: impl FnOnce() -> Result<(), String>,
2709    ) -> Result<(), DetachError> {
2710        let before = |error| DetachError {
2711            detached: false,
2712            error,
2713        };
2714        if !directory_removal_supported() {
2715            return Err(before(ModelManagementError::UnsafeManagedPath {
2716                model_id: label.to_string(),
2717                path: source.to_path_buf(),
2718                reason: "directory removal is not supported on this platform".into(),
2719            }));
2720        }
2721        let identity = artifact_identity(source, label).map_err(before)?;
2722        if identity.kind != ManagedArtifactKind::Directory {
2723            return Err(before(ModelManagementError::UnsafeManagedPath {
2724                model_id: label.to_string(),
2725                path: source.to_path_buf(),
2726                reason: "a retired repo must be a directory".into(),
2727            }));
2728        }
2729        if quarantine.parent() != source.parent() {
2730            return Err(before(ModelManagementError::UnsafeManagedPath {
2731                model_id: label.to_string(),
2732                path: quarantine.to_path_buf(),
2733                reason: "the quarantine must sit beside the directory it detaches".into(),
2734            }));
2735        }
2736        verify().map_err(|reason| {
2737            before(ModelManagementError::UnsafeManagedPath {
2738                model_id: label.to_string(),
2739                path: source.to_path_buf(),
2740                reason,
2741            })
2742        })?;
2743        let again = artifact_identity(source, label).map_err(before)?;
2744        if again.device != identity.device || again.inode != identity.inode {
2745            return Err(before(ModelManagementError::UnsafeManagedPath {
2746                model_id: label.to_string(),
2747                path: source.to_path_buf(),
2748                reason: "the repo directory changed while it was being checked".into(),
2749            }));
2750        }
2751        let journal = DetachedTreeJournal {
2752            source: source.to_path_buf(),
2753            quarantine: quarantine.to_path_buf(),
2754            identity,
2755        };
2756        let journal_path = self
2757            .detached_tree_journals()
2758            .join(hashed_filename(&quarantine.to_string_lossy()));
2759        if let Some(parent) = journal_path.parent() {
2760            std::fs::create_dir_all(parent)
2761                .map_err(|source| ModelManagementError::Io {
2762                    path: parent.to_path_buf(),
2763                    source,
2764                })
2765                .map_err(before)?;
2766        }
2767        write_private_json(&journal_path, &journal).map_err(before)?;
2768        if let Err(io) = atomic_rename_noreplace(source, quarantine) {
2769            let _ = std::fs::remove_file(&journal_path);
2770            return Err(before(ModelManagementError::Io {
2771                path: source.to_path_buf(),
2772                source: io,
2773            }));
2774        }
2775        self.finish_detached_tree(&journal, &journal_path, label)
2776            .map_err(|error| DetachError {
2777                detached: true,
2778                error,
2779            })
2780    }
2781
2782    #[cfg(any(target_os = "macos", target_os = "linux"))]
2783    fn finish_detached_tree(
2784        &self,
2785        journal: &DetachedTreeJournal,
2786        journal_path: &Path,
2787        label: &str,
2788    ) -> Result<(), ModelManagementError> {
2789        self.remove_quarantined_directory(&journal.quarantine, &journal.identity, label)?;
2790        let _ = std::fs::remove_file(journal_path);
2791        Ok(())
2792    }
2793
2794    #[cfg(not(any(target_os = "macos", target_os = "linux")))]
2795    fn finish_detached_tree(
2796        &self,
2797        journal: &DetachedTreeJournal,
2798        _journal_path: &Path,
2799        label: &str,
2800    ) -> Result<(), ModelManagementError> {
2801        Err(ModelManagementError::UnsafeManagedPath {
2802            model_id: label.to_string(),
2803            path: journal.quarantine.clone(),
2804            reason: "directory removal is not supported on this platform".into(),
2805        })
2806    }
2807
2808    /// Finish retirements a crash interrupted: a journaled quarantine still
2809    /// on disk is deleted (identity-checked); a journal whose rename never
2810    /// happened is dropped. Daemon start only.
2811    pub fn resume_detached_trees(&self) -> Vec<ModelManagementError> {
2812        let mut errors = Vec::new();
2813        let Ok(read) = std::fs::read_dir(self.detached_tree_journals()) else {
2814            return errors;
2815        };
2816        for entry in read.filter_map(Result::ok) {
2817            let path = entry.path();
2818            let journal: DetachedTreeJournal = match read_private_state(&path)
2819                .ok()
2820                .and_then(|bytes| serde_json::from_slice(&bytes).ok())
2821            {
2822                Some(journal) => journal,
2823                None => {
2824                    tracing::warn!(path = %path.display(), "unreadable retirement journal left in place");
2825                    continue;
2826                }
2827            };
2828            // Only ever a name a retirement creates; the identity check in the
2829            // walk is what binds it to the directory that was journaled. (The
2830            // parent comparison is a consistency check on the journal, not a
2831            // guard: both paths come from it.)
2832            let named = journal
2833                .quarantine
2834                .file_name()
2835                .and_then(|name| name.to_str())
2836                .is_some_and(|name| name.starts_with(RETIRE_QUARANTINE_PREFIX));
2837            if !named || journal.quarantine.parent() != journal.source.parent() {
2838                tracing::warn!(path = %path.display(), "retirement journal names a foreign quarantine; left in place");
2839                continue;
2840            }
2841            match std::fs::symlink_metadata(&journal.quarantine) {
2842                // The rename never happened, or the walk already finished.
2843                Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
2844                    let _ = std::fs::remove_file(&path);
2845                    continue;
2846                }
2847                Err(source) => {
2848                    errors.push(ModelManagementError::Io {
2849                        path: journal.quarantine.clone(),
2850                        source,
2851                    });
2852                    continue;
2853                }
2854                Ok(_) => {}
2855            }
2856            if let Err(error) = self.finish_detached_tree(&journal, &path, "retired repo") {
2857                errors.push(error);
2858            }
2859        }
2860        errors
2861    }
2862}
2863
2864#[cfg(any(target_os = "macos", target_os = "linux"))]
2865impl ModelManagementStore {
2866    /// Delete the quarantined directory `quarantine` whose pre-rename identity
2867    /// is `expected`. See `object_bound` for the guarantees.
2868    fn remove_quarantined_directory(
2869        &self,
2870        quarantine: &Path,
2871        expected: &ArtifactIdentity,
2872        model_id: &str,
2873    ) -> Result<(), ModelManagementError> {
2874        use object_bound::{EntryKind, Identity, MAX_REMOVAL_DEPTH};
2875        use std::ffi::CString;
2876        use std::fs::File;
2877
2878        struct CapturedDirectory {
2879            file: File,
2880            name: CString,
2881            parent: usize,
2882            depth: usize,
2883            identity: Identity,
2884        }
2885        struct CapturedEntry {
2886            parent: usize,
2887            name: CString,
2888            identity: Identity,
2889        }
2890
2891        let unsafe_path = |reason: &str| ModelManagementError::UnsafeManagedPath {
2892            model_id: model_id.to_string(),
2893            path: quarantine.to_path_buf(),
2894            reason: reason.into(),
2895        };
2896        let io_error = |source: std::io::Error| ModelManagementError::Io {
2897            path: quarantine.to_path_buf(),
2898            source,
2899        };
2900
2901        #[cfg(all(test, any(target_os = "macos", target_os = "linux")))]
2902        if let Some(hook) = &self.removal_hook {
2903            (hook.0)(RemovalPhase::BeforeRootOpen, quarantine);
2904        }
2905
2906        // Bind the root: open it once, then trust only the descriptor. The
2907        // caller observed the quarantine moments ago, so its absence here means
2908        // something moved it; that is never a completed removal.
2909        let root = match object_bound::open_directory(quarantine) {
2910            Ok(root) => root,
2911            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
2912                return Err(unsafe_path(QUARANTINE_VANISHED_REASON));
2913            }
2914            Err(source) => return Err(io_error(source)),
2915        };
2916        let root_identity = object_bound::identity_of(&root).map_err(io_error)?;
2917        if root_identity.device != expected.device || root_identity.inode != expected.inode {
2918            return Err(unsafe_path(
2919                "quarantined directory identity changed before deletion",
2920            ));
2921        }
2922
2923        // Capture phase: open every child by descriptor before trusting it.
2924        let mut directories = vec![CapturedDirectory {
2925            file: root,
2926            name: CString::default(),
2927            parent: 0,
2928            depth: 0,
2929            identity: root_identity,
2930        }];
2931        let mut entries: Vec<CapturedEntry> = Vec::new();
2932        let mut pending = vec![0usize];
2933        while let Some(index) = pending.pop() {
2934            let depth = directories[index].depth;
2935            if depth > MAX_REMOVAL_DEPTH {
2936                return Err(unsafe_path(
2937                    "managed directory nesting exceeds the removal depth cap",
2938                ));
2939            }
2940            let names = object_bound::list_names(&directories[index].file).map_err(io_error)?;
2941            for name in names {
2942                match object_bound::open_child_directory(&directories[index].file, &name) {
2943                    Ok(child) => {
2944                        let identity = object_bound::identity_of(&child).map_err(io_error)?;
2945                        if !object_bound::same_filesystem(&root_identity, &identity) {
2946                            return Err(unsafe_path(
2947                                "managed directory crosses a filesystem boundary",
2948                            ));
2949                        }
2950                        directories.push(CapturedDirectory {
2951                            file: child,
2952                            name,
2953                            parent: index,
2954                            depth: depth + 1,
2955                            identity,
2956                        });
2957                        pending.push(directories.len() - 1);
2958                    }
2959                    Err(error) if object_bound::is_not_a_directory(&error) => {
2960                        let (kind, identity) =
2961                            object_bound::stat_entry(&directories[index].file, &name)
2962                                .map_err(io_error)?;
2963                        match kind {
2964                            EntryKind::RegularFile | EntryKind::Symlink => {
2965                                entries.push(CapturedEntry {
2966                                    parent: index,
2967                                    name,
2968                                    identity,
2969                                });
2970                            }
2971                            EntryKind::Directory | EntryKind::Other => {
2972                                return Err(unsafe_path(
2973                                    "managed directory contains an entry CAR cannot classify",
2974                                ));
2975                            }
2976                        }
2977                    }
2978                    Err(error) if error.raw_os_error() == Some(libc::EXDEV) => {
2979                        return Err(unsafe_path(
2980                            "managed directory crosses a filesystem boundary",
2981                        ));
2982                    }
2983                    Err(error)
2984                        if matches!(
2985                            error.raw_os_error(),
2986                            Some(libc::ENOSYS) | Some(libc::EINVAL) | Some(libc::E2BIG)
2987                        ) =>
2988                    {
2989                        return Err(unsafe_path(
2990                            "descriptor-bound directory open is unavailable on this kernel",
2991                        ));
2992                    }
2993                    Err(source) => return Err(io_error(source)),
2994                }
2995            }
2996        }
2997
2998        #[cfg(all(test, any(target_os = "macos", target_os = "linux")))]
2999        if let Some(hook) = &self.removal_hook {
3000            (hook.0)(RemovalPhase::AfterCapture, quarantine);
3001        }
3002
3003        // Delete phase: only captured entries, each rechecked against its
3004        // captured identity immediately before the unlink.
3005        for entry in &entries {
3006            let parent = &directories[entry.parent].file;
3007            match object_bound::stat_entry(parent, &entry.name) {
3008                Ok((_, identity)) if identity == entry.identity => {
3009                    object_bound::unlink_entry(parent, &entry.name).map_err(io_error)?;
3010                }
3011                Ok(_) => {
3012                    return Err(unsafe_path(
3013                        "managed directory entry identity changed during deletion",
3014                    ));
3015                }
3016                Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
3017                Err(source) => return Err(io_error(source)),
3018            }
3019        }
3020
3021        // Directories deepest first: every child index is greater than its
3022        // parent's, so popping from the end empties children before parents.
3023        while directories.len() > 1 {
3024            let CapturedDirectory {
3025                file,
3026                name,
3027                parent,
3028                identity,
3029                ..
3030            } = directories.pop().expect("non-empty");
3031            drop(file);
3032            let parent = &directories[parent].file;
3033            match object_bound::stat_entry(parent, &name) {
3034                Ok((EntryKind::Directory, actual)) if actual == identity => {
3035                    object_bound::remove_empty_directory(parent, &name).map_err(|error| {
3036                        if object_bound::is_not_empty(&error) {
3037                            unsafe_path("managed directory contains entries that were not captured")
3038                        } else {
3039                            io_error(error)
3040                        }
3041                    })?;
3042                }
3043                Ok(_) => {
3044                    return Err(unsafe_path(
3045                        "managed directory entry identity changed during deletion",
3046                    ));
3047                }
3048                Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
3049                Err(source) => return Err(io_error(source)),
3050            }
3051        }
3052
3053        // The root itself: relative to the directory the quarantine sits in —
3054        // the validated models root for a model removal, the hub for a retired
3055        // repo — and only if it is still the captured object and now empty.
3056        // (Looking it up under the models root alone found nothing for a hub
3057        // quarantine and reported success, leaving the directory behind.)
3058        let CapturedDirectory { file: root, .. } = directories.pop().expect("root");
3059        drop(root);
3060        let container = quarantine
3061            .parent()
3062            .ok_or_else(|| unsafe_path("quarantine path has no parent"))?;
3063        let models =
3064            object_bound::open_directory(container).map_err(|source| ModelManagementError::Io {
3065                path: container.to_path_buf(),
3066                source,
3067            })?;
3068        let leaf = quarantine
3069            .file_name()
3070            .map(Path::new)
3071            .ok_or_else(|| unsafe_path("quarantine path has no file name"))?;
3072        let leaf = object_bound::c_name(leaf).map_err(io_error)?;
3073        match object_bound::stat_entry(&models, &leaf) {
3074            Ok((EntryKind::Directory, actual)) if actual == root_identity => {
3075                object_bound::remove_empty_directory(&models, &leaf).map_err(|error| {
3076                    if object_bound::is_not_empty(&error) {
3077                        unsafe_path("managed directory contains entries that were not captured")
3078                    } else {
3079                        io_error(error)
3080                    }
3081                })
3082            }
3083            Ok(_) => Err(unsafe_path(
3084                "quarantined directory identity changed during deletion",
3085            )),
3086            // The walk held this root open moments ago; its absence now means
3087            // something moved it, never that the removal completed.
3088            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
3089                Err(unsafe_path(QUARANTINE_VANISHED_REASON))
3090            }
3091            Err(source) => Err(io_error(source)),
3092        }
3093    }
3094}
3095
3096/// What [`unlink_checked`] may remove.
3097#[derive(Debug, Clone, Copy, PartialEq, Eq)]
3098pub(crate) enum UnlinkKind {
3099    /// A regular file only.
3100    File,
3101    /// A symlink only.
3102    Link,
3103    /// Either.
3104    FileOrLink,
3105}
3106
3107/// Unlink `path`'s last component relative to a descriptor for its parent
3108/// directory — the parent opened once, refusing a symlink at its own leaf —
3109/// and only if the entry is still of `kind`. A path-based `remove_file` would
3110/// resolve the whole path again after the caller checked it.
3111#[cfg(any(target_os = "macos", target_os = "linux"))]
3112pub(crate) fn unlink_checked(path: &Path, kind: UnlinkKind) -> std::io::Result<()> {
3113    use object_bound::EntryKind;
3114    let parent = path.parent().ok_or_else(|| {
3115        std::io::Error::new(std::io::ErrorKind::InvalidInput, "path has no parent")
3116    })?;
3117    let name = path.file_name().ok_or_else(|| {
3118        std::io::Error::new(std::io::ErrorKind::InvalidInput, "path has no file name")
3119    })?;
3120    let dir = object_bound::open_directory(parent)?;
3121    let name = object_bound::c_name(Path::new(name))?;
3122    let (actual, _) = object_bound::stat_entry(&dir, &name)?;
3123    let allowed = matches!(
3124        (kind, actual),
3125        (
3126            UnlinkKind::File | UnlinkKind::FileOrLink,
3127            EntryKind::RegularFile
3128        ) | (
3129            UnlinkKind::Link | UnlinkKind::FileOrLink,
3130            EntryKind::Symlink
3131        )
3132    );
3133    if !allowed {
3134        return Err(std::io::Error::new(
3135            std::io::ErrorKind::InvalidInput,
3136            format!("{} is no longer a {kind:?} entry", path.display()),
3137        ));
3138    }
3139    object_bound::unlink_entry(&dir, &name)
3140}
3141
3142/// [`unlink_checked`] where descriptor-relative removal is unavailable: the
3143/// kind is re-checked, then the path is removed.
3144#[cfg(not(any(target_os = "macos", target_os = "linux")))]
3145pub(crate) fn unlink_checked(path: &Path, kind: UnlinkKind) -> std::io::Result<()> {
3146    let meta = std::fs::symlink_metadata(path)?;
3147    let allowed = match kind {
3148        UnlinkKind::File => meta.is_file(),
3149        UnlinkKind::Link => meta.file_type().is_symlink(),
3150        UnlinkKind::FileOrLink => meta.is_file() || meta.file_type().is_symlink(),
3151    };
3152    if !allowed {
3153        return Err(std::io::Error::new(
3154            std::io::ErrorKind::InvalidInput,
3155            format!("{} is no longer a {kind:?} entry", path.display()),
3156        ));
3157    }
3158    // Windows removes a directory symlink or junction with `RemoveDirectory`,
3159    // which takes the link and never what it points at; `DeleteFile` refuses
3160    // one outright.
3161    #[cfg(windows)]
3162    {
3163        use std::os::windows::fs::FileTypeExt;
3164        if meta.file_type().is_symlink_dir() {
3165            return std::fs::remove_dir(path);
3166        }
3167    }
3168    std::fs::remove_file(path)
3169}
3170
3171// Keep both cfg halves of `sync_directory` here, ABOVE `mod tests`. The
3172// non-unix half used to sit at the end of the file, where it was invisible on
3173// Unix (it compiles out) but tripped `clippy::items_after_test_module` on the
3174// Windows leg of `-D warnings`.
3175#[cfg(not(unix))]
3176fn sync_directory(_path: &Path) -> std::io::Result<()> {
3177    Ok(())
3178}
3179
3180#[cfg(test)]
3181mod tests {
3182    /// A retirement a crash interrupted after detaching its repo is finished
3183    /// at the next start; one whose rename never happened is just dropped.
3184    #[cfg(any(target_os = "macos", target_os = "linux"))]
3185    #[test]
3186    fn interrupted_hub_retirements_resume() {
3187        let root = tempfile::tempdir().unwrap();
3188        let store =
3189            ModelManagementStore::new(root.path().join("state"), root.path().join("models"));
3190        let hub = root.path().join("hub");
3191        let repo = hub.join("models--a--b");
3192        std::fs::create_dir_all(repo.join("blobs")).unwrap();
3193        std::fs::write(repo.join("blobs/x"), b"bytes").unwrap();
3194        let quarantine = hub.join(".car-retire-1");
3195        let journal = DetachedTreeJournal {
3196            source: repo.clone(),
3197            quarantine: quarantine.clone(),
3198            identity: artifact_identity(&repo, "t").unwrap(),
3199        };
3200        let journals = store.detached_tree_journals();
3201        std::fs::create_dir_all(&journals).unwrap();
3202        write_private_json(&journals.join("j1"), &journal).unwrap();
3203        std::fs::rename(&repo, &quarantine).unwrap();
3204        // A second journal whose rename never happened.
3205        let never = DetachedTreeJournal {
3206            source: hub.join("models--c--d"),
3207            quarantine: hub.join(".car-retire-2"),
3208            identity: journal.identity.clone(),
3209        };
3210        write_private_json(&journals.join("j2"), &never).unwrap();
3211
3212        assert!(store.resume_detached_trees().is_empty());
3213        assert!(!quarantine.exists());
3214        assert!(std::fs::read_dir(&journals).unwrap().next().is_none());
3215
3216        // A journal aimed at something that is not a retirement quarantine
3217        // is left alone, and so is its target.
3218        let foreign = hub.join("models--e--f");
3219        std::fs::create_dir_all(&foreign).unwrap();
3220        std::fs::write(foreign.join("keep"), b"k").unwrap();
3221        let aimed = DetachedTreeJournal {
3222            source: hub.join("models--g--h"),
3223            quarantine: foreign.clone(),
3224            identity: artifact_identity(&foreign, "t").unwrap(),
3225        };
3226        write_private_json(&journals.join("j3"), &aimed).unwrap();
3227        // A real quarantine whose contents are no longer what was journaled.
3228        let swapped = hub.join(".car-retire-3");
3229        std::fs::create_dir_all(&swapped).unwrap();
3230        std::fs::write(swapped.join("other"), b"o").unwrap();
3231        // The identity of a directory that still exists: the first journal's
3232        // was deleted above, and Linux reuses a freed inode, so `swapped` can
3233        // be created with that very identity and match it.
3234        let stale = DetachedTreeJournal {
3235            source: hub.join("models--i--j"),
3236            quarantine: swapped.clone(),
3237            identity: aimed.identity.clone(),
3238        };
3239        write_private_json(&journals.join("j4"), &stale).unwrap();
3240
3241        let errors = store.resume_detached_trees();
3242        assert_eq!(errors.len(), 1, "{errors:?}");
3243        assert!(foreign.join("keep").exists());
3244        assert!(swapped.join("other").exists());
3245        assert!(journals.join("j3").exists() && journals.join("j4").exists());
3246    }
3247
3248    /// Tracking starts when the daemon starts it, not at the first use — so
3249    /// a machine that never loads a model still has a start to measure from —
3250    /// and a later start never moves it.
3251    #[test]
3252    fn usage_tracking_starts_with_the_store_and_stays_put() {
3253        let root = tempfile::tempdir().unwrap();
3254        let store =
3255            ModelManagementStore::new(root.path().join("state"), root.path().join("models"));
3256        assert_eq!(
3257            store.usage_tracking_since(),
3258            None,
3259            "construction starts nothing"
3260        );
3261        store.start_usage_tracking();
3262        let since = store
3263            .usage_tracking_since()
3264            .expect("marker written when started");
3265        let marker = store.usage_root.join(USAGE_TRACKING_MARKER);
3266        write_private_json(&marker, &UsageTracking { since: 7 }).unwrap();
3267        let again =
3268            ModelManagementStore::new(root.path().join("state"), root.path().join("models"));
3269        again.start_usage_tracking();
3270        assert_eq!(again.usage_tracking_since(), Some(7));
3271        assert!(since > 7);
3272        assert_eq!(store.last_used("org/model"), None, "never used is not used");
3273    }
3274
3275    /// A lease stamps the model's use once per interval, not once per request.
3276    #[test]
3277    fn a_lease_stamps_use_at_most_once_per_interval() {
3278        let root = tempfile::tempdir().unwrap();
3279        let store =
3280            ModelManagementStore::new(root.path().join("state"), root.path().join("models"));
3281        drop(store.acquire_lease("org/model").unwrap());
3282        let stamped = store.last_used("org/model").expect("first lease stamps");
3283        assert!(stamped > 0);
3284        std::fs::remove_file(store.usage_root.join(hashed_filename("org/model"))).unwrap();
3285        drop(store.acquire_lease("org/model").unwrap());
3286        assert_eq!(
3287            store.last_used("org/model"),
3288            None,
3289            "throttled: no second write"
3290        );
3291        // Another model is its own stamp.
3292        drop(store.acquire_lease("org/other").unwrap());
3293        assert!(store.last_used("org/other").is_some());
3294        // A stamp naming a different model under this file is not this model's.
3295        write_private_json(
3296            &store.usage_root.join(hashed_filename("org/model")),
3297            &UsageStamp {
3298                model_id: "org/forged".into(),
3299                last_used: 1,
3300            },
3301        )
3302        .unwrap();
3303        assert_eq!(store.last_used("org/model"), None);
3304    }
3305
3306    /// A beat continues the run across a restart within the gap and across
3307    /// sleep within one process, and a long silence starts a new run.
3308    #[test]
3309    fn coverage_runs_continue_across_restarts_and_break_on_long_gaps() {
3310        let root = tempfile::tempdir().unwrap();
3311        let store =
3312            ModelManagementStore::new(root.path().join("state"), root.path().join("models"));
3313        assert_eq!(store.usage_covered_since(), None);
3314        store.beat();
3315        let first = store.usage_covered_since().unwrap();
3316        store.beat();
3317        assert_eq!(store.usage_covered_since(), Some(first), "same run");
3318        // A restart: a new process (fresh store) beating soon after.
3319        let restarted =
3320            ModelManagementStore::new(root.path().join("state"), root.path().join("models"));
3321        restarted.beat();
3322        assert_eq!(restarted.usage_covered_since(), Some(first));
3323        // The last beat was long ago and not ours: a new run.
3324        write_private_json(
3325            &restarted.usage_root.join(USAGE_COVERAGE),
3326            &UsageCoverage {
3327                run_start: 5,
3328                last_beat: 10,
3329            },
3330        )
3331        .unwrap();
3332        let later =
3333            ModelManagementStore::new(root.path().join("state"), root.path().join("models"));
3334        later.beat();
3335        assert!(later.usage_covered_since().unwrap() > 10);
3336    }
3337
3338    #[test]
3339    fn an_interrupted_retirement_clears_its_tombstone_at_resume() {
3340        let root = tempfile::tempdir().unwrap();
3341        let store =
3342            ModelManagementStore::new(root.path().join("state"), root.path().join("models"));
3343        let id = "mlx/some:4bit";
3344        store.note_retiring(id).unwrap();
3345        write_private_json(
3346            &store.tombstone_path(id),
3347            &RemovalTombstone {
3348                model_id: id.into(),
3349                removal_generation: 1,
3350                artifact_kind: None,
3351                preserved_shared_cache_references: Vec::new(),
3352            },
3353        )
3354        .unwrap();
3355        assert!(!store.car_enabled(id).unwrap());
3356        assert!(store.tombstone_path(id).exists());
3357        assert!(store.resume_retiring().is_empty());
3358        assert!(!store.tombstone_path(id).exists());
3359        assert!(!store.retiring_intent_path(id).exists());
3360    }
3361
3362    use super::*;
3363
3364    // Invoked only by the parent process fixture below, never by a model backend.
3365    #[test]
3366    #[ignore = "re-executed by identity_initialization_serializes_processes_before_json_is_written"]
3367    fn identity_initialization_child() {
3368        use std::io::{BufRead, Write};
3369        let Some(root) = std::env::var_os("CAR_IDENTITY_TEST_ROOT") else {
3370            return;
3371        };
3372        let root = PathBuf::from(root);
3373        let mut store = ModelManagementStore::new(root.join("state"), root.join("models"));
3374        if std::env::var("CAR_IDENTITY_TEST_ROLE").as_deref() == Ok("creator") {
3375            store.identity_init_hook = Some(|path| {
3376                let phase = std::env::var("CAR_IDENTITY_TEST_PHASE").unwrap();
3377                if path.parent().unwrap().file_name().unwrap() == phase.as_str() {
3378                    println!("IDENTITY_CREATED");
3379                    std::io::stdout().flush().unwrap();
3380                    let mut release = String::new();
3381                    std::io::stdin().lock().read_line(&mut release).unwrap();
3382                    assert_eq!(release.trim(), "release");
3383                }
3384            });
3385        } else {
3386            println!("READER_STARTED");
3387            std::io::stdout().flush().unwrap();
3388        }
3389        let _lease = store.acquire_lease("fixture/model").unwrap();
3390    }
3391
3392    #[test]
3393    fn identity_initialization_serializes_processes_before_json_is_written() {
3394        use std::io::{BufRead, BufReader, Write};
3395        use std::process::{Command, Stdio};
3396        fn wait_marker(reader: &mut impl BufRead, marker: &str) {
3397            let mut line = String::new();
3398            loop {
3399                line.clear();
3400                assert_ne!(
3401                    reader.read_line(&mut line).unwrap(),
3402                    0,
3403                    "child exited before {marker}"
3404                );
3405                if line.contains(marker) {
3406                    return;
3407                }
3408            }
3409        }
3410        for phase in ["mutation", "activity"] {
3411            let root = tempfile::tempdir().unwrap();
3412            let spawn = |role: &str| {
3413                Command::new(std::env::current_exe().unwrap())
3414                    .args([
3415                        "--exact",
3416                        "model_management::tests::identity_initialization_child",
3417                        "--nocapture",
3418                        "--include-ignored",
3419                    ])
3420                    .env("CAR_IDENTITY_TEST_ROOT", root.path())
3421                    .env("CAR_IDENTITY_TEST_ROLE", role)
3422                    .env("CAR_IDENTITY_TEST_PHASE", phase)
3423                    .stdin(Stdio::piped())
3424                    .stdout(Stdio::piped())
3425                    .stderr(Stdio::inherit())
3426                    .spawn()
3427                    .unwrap()
3428            };
3429            let mut creator = spawn("creator");
3430            let mut creator_out = BufReader::new(creator.stdout.take().unwrap());
3431            wait_marker(&mut creator_out, "IDENTITY_CREATED");
3432            let store =
3433                ModelManagementStore::new(root.path().join("state"), root.path().join("models"));
3434            let path = if phase == "mutation" {
3435                store.mutation_lock_path("fixture/model")
3436            } else {
3437                store.lease_path("fixture/model")
3438            };
3439            assert!(
3440                std::fs::read(&path).unwrap().is_empty(),
3441                "probe must hit the original partial-publication window"
3442            );
3443            let gate =
3444                open_existing_identity_lock(&path.parent().unwrap().join(".initialization.lock"))
3445                    .unwrap();
3446            assert!(
3447                gate.try_lock().is_err(),
3448                "creator must hold the initialization gate before publishing an empty identity"
3449            );
3450            let mut contender = spawn("reader");
3451            let mut contender_out = BufReader::new(contender.stdout.take().unwrap());
3452            wait_marker(&mut contender_out, "READER_STARTED");
3453            assert!(contender.try_wait().unwrap().is_none());
3454            writeln!(creator.stdin.take().unwrap(), "release").unwrap();
3455            assert!(creator.wait().unwrap().success());
3456            assert!(contender.wait().unwrap().success());
3457            let before = std::fs::metadata(&path).unwrap();
3458            drop(store.acquire_lease("fixture/model").unwrap());
3459            #[cfg(unix)]
3460            {
3461                use std::os::unix::fs::MetadataExt;
3462                assert_eq!(before.ino(), std::fs::metadata(&path).unwrap().ino());
3463            }
3464            #[cfg(not(unix))]
3465            let _ = before;
3466        }
3467    }
3468
3469    #[test]
3470    fn identity_initialization_preserves_persistent_corruption_refusal() {
3471        for phase in ["mutation", "activity"] {
3472            for corrupt in [b"".as_slice(), b"not json".as_slice()] {
3473                let root = tempfile::tempdir().unwrap();
3474                let store = ModelManagementStore::new(
3475                    root.path().join("state"),
3476                    root.path().join("models"),
3477                );
3478                drop(store.acquire_lease("fixture/model").unwrap());
3479                let path = if phase == "mutation" {
3480                    store.mutation_lock_path("fixture/model")
3481                } else {
3482                    store.lease_path("fixture/model")
3483                };
3484                std::fs::write(&path, corrupt).unwrap();
3485                assert!(matches!(
3486                    store.acquire_lease("fixture/model"),
3487                    Err(ModelManagementError::InvalidState { .. })
3488                ));
3489                assert_eq!(std::fs::read(&path).unwrap(), corrupt);
3490            }
3491        }
3492    }
3493
3494    #[cfg(unix)]
3495    #[test]
3496    fn identity_initialization_rejects_linked_gates_and_identity_files() {
3497        for phase in ["mutation", "activity"] {
3498            for gate in [false, true] {
3499                for symlink in [false, true] {
3500                    let root = tempfile::tempdir().unwrap();
3501                    let store = ModelManagementStore::new(
3502                        root.path().join("state"),
3503                        root.path().join("models"),
3504                    );
3505                    let identity = if phase == "mutation" {
3506                        store.mutation_lock_path("fixture/model")
3507                    } else {
3508                        store.lease_path("fixture/model")
3509                    };
3510                    create_private_dir(identity.parent().unwrap()).unwrap();
3511                    let path = if gate {
3512                        identity.parent().unwrap().join(".initialization.lock")
3513                    } else {
3514                        identity
3515                    };
3516                    let outside = root.path().join("outside");
3517                    std::fs::write(&outside, b"untouched").unwrap();
3518                    if symlink {
3519                        std::os::unix::fs::symlink(&outside, &path).unwrap();
3520                    } else {
3521                        std::fs::hard_link(&outside, &path).unwrap();
3522                    }
3523                    assert!(matches!(
3524                        store.acquire_lease("fixture/model"),
3525                        Err(ModelManagementError::InvalidState { .. })
3526                    ));
3527                    assert_eq!(std::fs::read(&outside).unwrap(), b"untouched");
3528                }
3529            }
3530        }
3531    }
3532
3533    fn directory_receipt(model_id: &str, managed_path: PathBuf) -> InstallReceipt {
3534        InstallReceipt {
3535            model_id: model_id.into(),
3536            managed_path,
3537            artifact_kind: ManagedArtifactKind::Directory,
3538            source_model_id: model_id.into(),
3539            source_revision: None,
3540            creation_generation: 1,
3541            shared_cache_references: vec![],
3542            adopted: false,
3543        }
3544    }
3545
3546    #[test]
3547    fn raw_receipt_identity_is_private_and_collision_fails_closed() {
3548        let root = tempfile::tempdir().unwrap();
3549        let models = root.path().join("models");
3550        std::fs::create_dir_all(&models).unwrap();
3551        let store = ModelManagementStore::new(root.path().join("state"), models.clone());
3552        let receipt = directory_receipt("org/model", models.join("Model"));
3553        store.write_receipt(&receipt).unwrap();
3554        assert_eq!(store.load_receipt("org/model").unwrap(), Some(receipt));
3555
3556        #[cfg(unix)]
3557        {
3558            use std::os::unix::fs::PermissionsExt;
3559            assert_eq!(
3560                std::fs::metadata(store.receipt_path("org/model"))
3561                    .unwrap()
3562                    .permissions()
3563                    .mode()
3564                    & 0o077,
3565                0
3566            );
3567        }
3568
3569        let collision = directory_receipt("different/model", models.join("Other"));
3570        std::fs::write(
3571            store.receipt_path("org/model"),
3572            serde_json::to_vec(&collision).unwrap(),
3573        )
3574        .unwrap();
3575        assert!(matches!(
3576            store.load_receipt("org/model"),
3577            Err(ModelManagementError::IdentityMismatch { .. })
3578        ));
3579    }
3580
3581    #[test]
3582    fn journaled_removal_rejects_leaf_identity_swap() {
3583        let root = tempfile::tempdir().unwrap();
3584        let models = root.path().join("models");
3585        std::fs::create_dir_all(&models).unwrap();
3586        let store = ModelManagementStore::new(root.path().join("state"), models.clone());
3587        let managed = models.join("Managed");
3588        std::fs::create_dir_all(&managed).unwrap();
3589        std::fs::write(managed.join("weights"), b"original").unwrap();
3590        let receipt = directory_receipt("org/model", managed.clone());
3591        store.write_receipt(&receipt).unwrap();
3592        let journal = RemovalJournal {
3593            model_id: "org/model".into(),
3594            removal_generation: 2,
3595            artifact_identity: artifact_identity(&managed, "org/model").unwrap(),
3596            receipt,
3597            quarantine_path: store.allocate_quarantine("org/model").unwrap(),
3598        };
3599        write_private_json(&store.removal_journal_path("org/model"), &journal).unwrap();
3600        let displaced = models.join("displaced-original");
3601        std::fs::rename(&managed, &displaced).unwrap();
3602        std::fs::create_dir_all(&managed).unwrap();
3603        std::fs::write(managed.join("sentinel"), b"replacement").unwrap();
3604
3605        assert!(matches!(
3606            store.begin_mutation("org/model").unwrap().remove(2),
3607            Err(ModelManagementError::UnsafeManagedPath { .. })
3608        ));
3609        assert_eq!(
3610            std::fs::read(managed.join("sentinel")).unwrap(),
3611            b"replacement"
3612        );
3613    }
3614
3615    #[test]
3616    fn journaled_removal_rejects_models_parent_swap() {
3617        let root = tempfile::tempdir().unwrap();
3618        let models = root.path().join("models");
3619        std::fs::create_dir_all(&models).unwrap();
3620        let store = ModelManagementStore::new(root.path().join("state"), models.clone());
3621        let managed = models.join("Managed");
3622        std::fs::create_dir_all(&managed).unwrap();
3623        std::fs::write(managed.join("weights"), b"original").unwrap();
3624        let receipt = directory_receipt("org/model", managed.clone());
3625        store.write_receipt(&receipt).unwrap();
3626        let journal = RemovalJournal {
3627            model_id: "org/model".into(),
3628            removal_generation: 2,
3629            artifact_identity: artifact_identity(&managed, "org/model").unwrap(),
3630            receipt,
3631            quarantine_path: store.allocate_quarantine("org/model").unwrap(),
3632        };
3633        write_private_json(&store.removal_journal_path("org/model"), &journal).unwrap();
3634        std::fs::rename(&models, root.path().join("models-original")).unwrap();
3635        std::fs::create_dir_all(&managed).unwrap();
3636        std::fs::write(managed.join("sentinel"), b"replacement-parent").unwrap();
3637
3638        assert!(matches!(
3639            store.begin_mutation("org/model").unwrap().remove(2),
3640            Err(ModelManagementError::UnsafeManagedPath { .. })
3641        ));
3642        assert_eq!(
3643            std::fs::read(managed.join("sentinel")).unwrap(),
3644            b"replacement-parent"
3645        );
3646    }
3647
3648    #[cfg(not(any(target_os = "macos", target_os = "linux")))]
3649    #[test]
3650    fn directory_receipts_fail_closed_without_recursive_cleanup() {
3651        let root = tempfile::tempdir().unwrap();
3652        let models = root.path().join("models");
3653        std::fs::create_dir_all(&models).unwrap();
3654        let store = ModelManagementStore::new(root.path().join("state"), models.clone());
3655        let managed = models.join("Managed");
3656        std::fs::create_dir_all(&managed).unwrap();
3657        std::fs::write(managed.join("shared-sentinel"), b"preserve").unwrap();
3658        store
3659            .write_receipt(&directory_receipt("org/model", managed.clone()))
3660            .unwrap();
3661
3662        assert!(!store.can_remove("org/model").unwrap());
3663        let error = store
3664            .begin_mutation("org/model")
3665            .unwrap()
3666            .remove(2)
3667            .unwrap_err();
3668        assert!(matches!(
3669            error,
3670            ModelManagementError::UnsafeManagedPath { .. }
3671        ));
3672        assert_eq!(
3673            std::fs::read(managed.join("shared-sentinel")).unwrap(),
3674            b"preserve"
3675        );
3676    }
3677
3678    #[test]
3679    fn failed_directory_staging_is_left_for_object_bound_cleanup() {
3680        let root = tempfile::tempdir().unwrap();
3681        let models = root.path().join("models");
3682        std::fs::create_dir_all(&models).unwrap();
3683        let store = ModelManagementStore::new(root.path().join("state"), models);
3684        let staging = store.create_install_staging("org/model").unwrap();
3685        std::fs::write(staging.join("shared-sentinel"), b"preserve").unwrap();
3686
3687        store.discard_install_staging(&staging);
3688
3689        assert_eq!(
3690            std::fs::read(staging.join("shared-sentinel")).unwrap(),
3691            b"preserve"
3692        );
3693    }
3694
3695    #[cfg(unix)]
3696    #[test]
3697    fn completed_symlink_removal_retry_is_idempotent() {
3698        use std::os::unix::fs::symlink;
3699
3700        let root = tempfile::tempdir().unwrap();
3701        let models = root.path().join("models");
3702        let shared = root.path().join("shared");
3703        std::fs::create_dir_all(&models).unwrap();
3704        std::fs::create_dir_all(&shared).unwrap();
3705        std::fs::write(shared.join("sentinel"), b"preserve").unwrap();
3706        let managed = models.join("Managed");
3707        symlink(&shared, &managed).unwrap();
3708        let store = ModelManagementStore::new(root.path().join("state"), models);
3709        let receipt = InstallReceipt {
3710            model_id: "org/model".into(),
3711            managed_path: managed,
3712            artifact_kind: ManagedArtifactKind::Symlink,
3713            source_model_id: "org/model".into(),
3714            source_revision: None,
3715            creation_generation: 1,
3716            shared_cache_references: vec![shared.plain_canonicalize().unwrap()],
3717            adopted: false,
3718        };
3719        store.write_receipt(&receipt).unwrap();
3720
3721        let first = store
3722            .begin_mutation("org/model")
3723            .unwrap()
3724            .remove(2)
3725            .unwrap();
3726        let retry = store
3727            .begin_mutation("org/model")
3728            .unwrap()
3729            .remove(3)
3730            .unwrap();
3731        assert_eq!(retry, first);
3732        assert!(shared.join("sentinel").exists());
3733    }
3734
3735    #[test]
3736    fn pulled_directory_publish_before_receipt_resumes_from_install_intent() {
3737        let root = tempfile::tempdir().unwrap();
3738        let models = root.path().join("models");
3739        std::fs::create_dir_all(&models).unwrap();
3740        let store = ModelManagementStore::new(root.path().join("state"), models.clone());
3741        let staging = store.create_install_staging("org/model").unwrap();
3742        std::fs::write(staging.join("weights"), b"owned").unwrap();
3743        let receipt = store
3744            .install_receipt_for_publication(
3745                "org/model",
3746                "org/model",
3747                1,
3748                false,
3749                models.join("Managed"),
3750                ManagedArtifactKind::Directory,
3751                &staging,
3752            )
3753            .unwrap();
3754        store.begin_install_intent(receipt, Some(&staging)).unwrap();
3755        store
3756            .publish_install_staging("org/model", &staging, "Managed")
3757            .unwrap();
3758        assert!(store.load_receipt("org/model").unwrap().is_none());
3759
3760        let resumed = store.resume_install_intent("org/model").unwrap().unwrap();
3761        assert_eq!(resumed.managed_path, models.join("Managed"));
3762        assert!(store.load_receipt("org/model").unwrap().is_some());
3763        #[cfg(any(target_os = "macos", target_os = "linux"))]
3764        assert!(store.can_remove("org/model").unwrap());
3765        #[cfg(not(any(target_os = "macos", target_os = "linux")))]
3766        assert!(!store.can_remove("org/model").unwrap());
3767    }
3768
3769    #[cfg(unix)]
3770    #[test]
3771    fn adopted_projection_publish_before_receipt_resumes_from_install_intent() {
3772        let root = tempfile::tempdir().unwrap();
3773        let models = root.path().join("models");
3774        let source = root.path().join("hand-installed");
3775        std::fs::create_dir_all(&models).unwrap();
3776        std::fs::create_dir_all(&source).unwrap();
3777        std::fs::write(source.join("sentinel"), b"preserve").unwrap();
3778        let store = ModelManagementStore::new(root.path().join("state"), models);
3779        let managed = store.adopted_projection_path("org/model").unwrap();
3780        let receipt = store
3781            .install_receipt_for_publication(
3782                "org/model",
3783                "org/model",
3784                1,
3785                true,
3786                managed.clone(),
3787                ManagedArtifactKind::Symlink,
3788                &source,
3789            )
3790            .unwrap();
3791        store.begin_install_intent(receipt, None).unwrap();
3792        store
3793            .materialize_adopted_projection("org/model", &source)
3794            .unwrap();
3795        assert!(store.load_receipt("org/model").unwrap().is_none());
3796
3797        let resumed = store.resume_install_intent("org/model").unwrap().unwrap();
3798        assert_eq!(resumed.managed_path, managed);
3799        assert!(store.can_remove("org/model").unwrap());
3800        assert!(source.join("sentinel").exists());
3801    }
3802
3803    #[cfg(unix)]
3804    #[test]
3805    fn install_intent_never_claims_a_preexisting_unreceipted_leaf() {
3806        use std::os::unix::fs::symlink;
3807
3808        let root = tempfile::tempdir().unwrap();
3809        let models = root.path().join("models");
3810        let source = root.path().join("shared");
3811        std::fs::create_dir_all(&models).unwrap();
3812        std::fs::create_dir_all(&source).unwrap();
3813        let managed = models.join("Managed");
3814        symlink(&source, &managed).unwrap();
3815        let store = ModelManagementStore::new(root.path().join("state"), models);
3816        let receipt = store
3817            .install_receipt_for_publication(
3818                "org/model",
3819                "org/model",
3820                1,
3821                false,
3822                managed.clone(),
3823                ManagedArtifactKind::Symlink,
3824                &source,
3825            )
3826            .unwrap();
3827
3828        let error = store.begin_install_intent(receipt, None).unwrap_err();
3829        assert!(matches!(
3830            error,
3831            ModelManagementError::UnsafeManagedPath { .. }
3832        ));
3833        assert!(managed.exists());
3834        assert!(store.load_receipt("org/model").unwrap().is_none());
3835    }
3836
3837    #[test]
3838    fn forged_install_journal_paths_fail_before_filesystem_mutation() {
3839        let root = tempfile::tempdir().unwrap();
3840        let models = root.path().join("models");
3841        std::fs::create_dir_all(&models).unwrap();
3842        let outside = root.path().join("outside");
3843        std::fs::create_dir_all(&outside).unwrap();
3844        std::fs::write(outside.join("sentinel"), b"preserve").unwrap();
3845        let store = ModelManagementStore::new(root.path().join("state"), models.clone());
3846
3847        let forged_managed = InstallJournal {
3848            model_id: "org/model".into(),
3849            receipt: directory_receipt("org/model", outside.clone()),
3850            staging_path: None,
3851            staging_identity: None,
3852        };
3853        write_private_json(&store.install_journal_path("org/model"), &forged_managed).unwrap();
3854        assert!(matches!(
3855            store.resume_install_intent("org/model"),
3856            Err(ModelManagementError::UnsafeManagedPath { .. })
3857        ));
3858        assert!(outside.join("sentinel").exists());
3859
3860        let forged_staging = InstallJournal {
3861            model_id: "org/other".into(),
3862            receipt: directory_receipt("org/other", models.join("Other")),
3863            staging_path: Some(outside.clone()),
3864            staging_identity: Some(artifact_identity(&outside, "org/other").unwrap()),
3865        };
3866        write_private_json(&store.install_journal_path("org/other"), &forged_staging).unwrap();
3867        assert!(matches!(
3868            store.resume_install_intent("org/other"),
3869            Err(ModelManagementError::UnsafeManagedPath { .. })
3870        ));
3871        assert!(outside.join("sentinel").exists());
3872    }
3873
3874    #[cfg(unix)]
3875    #[test]
3876    fn startup_ignores_noncanonical_removal_journal_without_touching_valid_model() {
3877        use std::os::unix::fs::symlink;
3878
3879        let root = tempfile::tempdir().unwrap();
3880        let state = root.path().join("state");
3881        let models = root.path().join("models");
3882        let shared = root.path().join("shared");
3883        let outside = root.path().join("outside");
3884        std::fs::create_dir_all(&models).unwrap();
3885        std::fs::create_dir_all(&shared).unwrap();
3886        std::fs::create_dir_all(&outside).unwrap();
3887        std::fs::write(shared.join("sentinel"), b"shared").unwrap();
3888        std::fs::write(outside.join("sentinel"), b"outside").unwrap();
3889        let managed = models.join("Managed");
3890        symlink(&shared, &managed).unwrap();
3891        let store = ModelManagementStore::new(state.clone(), models.clone());
3892        let legitimate = InstallReceipt {
3893            model_id: "org/model".into(),
3894            managed_path: managed.clone(),
3895            artifact_kind: ManagedArtifactKind::Symlink,
3896            source_model_id: "org/model".into(),
3897            source_revision: None,
3898            creation_generation: 1,
3899            shared_cache_references: vec![shared.plain_canonicalize().unwrap()],
3900            adopted: false,
3901        };
3902        store.write_receipt(&legitimate).unwrap();
3903        create_private_dir(&store.removals_root).unwrap();
3904        let forged = RemovalJournal {
3905            model_id: "org/model".into(),
3906            removal_generation: 99,
3907            receipt: InstallReceipt {
3908                managed_path: outside.join("missing-leaf"),
3909                ..legitimate
3910            },
3911            quarantine_path: models.join(".car-remove-forged"),
3912            artifact_identity: ArtifactIdentity {
3913                kind: ManagedArtifactKind::Symlink,
3914                device: 1,
3915                inode: 1,
3916                file_len: 0,
3917            },
3918        };
3919        write_private_json(&store.removals_root.join("junk.json"), &forged).unwrap();
3920        drop(store);
3921
3922        let recovered = ModelManagementStore::new(state, models);
3923        assert!(std::fs::symlink_metadata(&managed).is_ok());
3924        assert!(recovered.car_enabled("org/model").unwrap());
3925        assert_eq!(std::fs::read(shared.join("sentinel")).unwrap(), b"shared");
3926        assert_eq!(std::fs::read(outside.join("sentinel")).unwrap(), b"outside");
3927    }
3928
3929    // try_clone retains the same open-file description as an inherited fork
3930    // descriptor, without forking a multithreaded test process.
3931    #[cfg(any(target_os = "macos", target_os = "linux"))]
3932    #[test]
3933    fn mutation_guard_releases_lock_even_with_an_inherited_descriptor() {
3934        let (_root, _models, store) = supported_platform_store();
3935        let guard = store.begin_mutation("org/model").unwrap();
3936        let inherited = guard._file.try_clone().unwrap();
3937        assert!(matches!(
3938            store.begin_mutation("org/model"),
3939            Err(ModelManagementError::MutationInProgress { .. })
3940        ));
3941        drop(guard);
3942        let replacement = store.begin_mutation("org/model").unwrap();
3943        // Releasing the old duplicate must not unlock the replacement owner.
3944        drop(inherited);
3945        assert!(matches!(
3946            store.begin_mutation("org/model"),
3947            Err(ModelManagementError::MutationInProgress { .. })
3948        ));
3949        drop(replacement);
3950        assert!(store.begin_mutation("org/model").is_ok());
3951    }
3952
3953    #[cfg(any(target_os = "macos", target_os = "linux"))]
3954    #[test]
3955    fn lease_and_idle_guards_release_despite_inherited_descriptors() {
3956        let (_root, _models, store) = supported_platform_store();
3957        let lease = store.acquire_lease("org/model").unwrap();
3958        let inherited = lease._file.try_clone().unwrap();
3959        assert!(store.model_in_use("org/model").unwrap());
3960        drop(lease);
3961        assert!(!store.model_in_use("org/model").unwrap());
3962        let idle = store.lock_idle_activity("org/model").unwrap();
3963        let inherited_idle = idle.try_clone().unwrap();
3964        assert!(store.model_in_use("org/model").unwrap());
3965        drop(idle);
3966        assert!(!store.model_in_use("org/model").unwrap());
3967        drop((inherited, inherited_idle));
3968    }
3969
3970    #[cfg(any(target_os = "macos", target_os = "linux"))]
3971    #[test]
3972    fn temporary_owner_releases_initialization_gate_despite_inherited_descriptor() {
3973        let (_root, _models, store) = supported_platform_store();
3974        let identity = store
3975            .open_identity_lock(&store.mutation_lock_path("org/model"), "org/model")
3976            .unwrap();
3977        let path = store
3978            .mutation_lock_path("org/model")
3979            .parent()
3980            .unwrap()
3981            .join(".initialization.lock");
3982        let gate = open_existing_identity_lock(&path).unwrap();
3983        gate.lock().unwrap();
3984        let gate = OwnedFileLock::new(gate);
3985        let inherited = gate.try_clone().unwrap();
3986        let contender = open_existing_identity_lock(&path).unwrap();
3987        assert!(contender.try_lock().is_err());
3988        drop(gate);
3989        contender.try_lock().unwrap();
3990        contender.unlock().unwrap();
3991        drop((identity, inherited));
3992    }
3993
3994    #[cfg(any(target_os = "macos", target_os = "linux"))]
3995    fn supported_platform_store() -> (tempfile::TempDir, PathBuf, ModelManagementStore) {
3996        let root = tempfile::tempdir().unwrap();
3997        let models = root.path().join("models");
3998        std::fs::create_dir_all(&models).unwrap();
3999        let store = ModelManagementStore::new(root.path().join("state"), models.clone());
4000        (root, models, store)
4001    }
4002
4003    #[cfg(any(target_os = "macos", target_os = "linux"))]
4004    fn no_quarantine_left(models: &Path) -> bool {
4005        std::fs::read_dir(models).unwrap().all(|entry| {
4006            !entry
4007                .unwrap()
4008                .file_name()
4009                .to_string_lossy()
4010                .starts_with(".car-remove-")
4011        })
4012    }
4013
4014    #[cfg(any(target_os = "macos", target_os = "linux"))]
4015    const MAX_REMOVAL_DEPTH_FOR_TESTS: usize = super::object_bound::MAX_REMOVAL_DEPTH;
4016
4017    /// Unwraps a fail-closed refusal to its reason. Every race test funnels
4018    /// through here so no single site can quietly weaken into accepting a
4019    /// different error variant.
4020    #[cfg(any(target_os = "macos", target_os = "linux"))]
4021    fn unsafe_managed_reason(error: &ModelManagementError) -> &str {
4022        match error {
4023            ModelManagementError::UnsafeManagedPath { reason, .. } => reason,
4024            other => panic!("expected UnsafeManagedPath, got {other:?}"),
4025        }
4026    }
4027
4028    #[cfg(any(target_os = "macos", target_os = "linux"))]
4029    #[test]
4030    fn directory_receipt_is_removable_and_leaves_a_tombstone() {
4031        let (_root, models, store) = supported_platform_store();
4032        let managed = models.join("Managed");
4033        std::fs::create_dir_all(&managed).unwrap();
4034        std::fs::write(managed.join("weights"), b"owned").unwrap();
4035        store
4036            .write_receipt(&directory_receipt("org/model", managed.clone()))
4037            .unwrap();
4038
4039        assert!(store.can_remove("org/model").unwrap());
4040        let result = store
4041            .begin_mutation("org/model")
4042            .unwrap()
4043            .remove(2)
4044            .unwrap();
4045
4046        assert_eq!(result.artifact_kind, ManagedArtifactKind::Directory);
4047        assert!(std::fs::symlink_metadata(&managed).is_err());
4048        assert!(store.load_receipt("org/model").unwrap().is_none());
4049        let tombstone = store.load_tombstone("org/model").unwrap().unwrap();
4050        assert_eq!(
4051            tombstone.artifact_kind,
4052            Some(ManagedArtifactKind::Directory)
4053        );
4054        assert_eq!(tombstone.removal_generation, 2);
4055        assert!(no_quarantine_left(&models));
4056        assert!(store.load_removal_journal("org/model").unwrap().is_none());
4057    }
4058
4059    #[cfg(any(target_os = "macos", target_os = "linux"))]
4060    #[test]
4061    fn directory_removal_preserves_shared_cache_symlink_targets() {
4062        use std::os::unix::fs::symlink;
4063
4064        let (root, models, store) = supported_platform_store();
4065        let shared = root.path().join("shared-cache");
4066        std::fs::create_dir_all(&shared).unwrap();
4067        std::fs::write(shared.join("blob"), b"shared-bytes").unwrap();
4068        let managed = models.join("Managed");
4069        let nested = managed.join("a").join("b").join("c");
4070        std::fs::create_dir_all(&nested).unwrap();
4071        std::fs::write(managed.join("config.json"), b"{}").unwrap();
4072        std::fs::write(nested.join("weights"), b"owned").unwrap();
4073        symlink(shared.join("blob"), managed.join("blob.safetensors")).unwrap();
4074        let blob = shared.join("blob").plain_canonicalize().unwrap();
4075        let mut receipt = directory_receipt("org/model", managed.clone());
4076        receipt.shared_cache_references = vec![blob.clone()];
4077        store.write_receipt(&receipt).unwrap();
4078
4079        let result = store
4080            .begin_mutation("org/model")
4081            .unwrap()
4082            .remove(2)
4083            .unwrap();
4084
4085        assert!(std::fs::symlink_metadata(&managed).is_err());
4086        assert_eq!(std::fs::read(&blob).unwrap(), b"shared-bytes");
4087        assert_eq!(std::fs::read_dir(&shared).unwrap().count(), 1);
4088        assert_eq!(result.preserved_shared_cache_references, vec![blob]);
4089        assert!(no_quarantine_left(&models));
4090    }
4091
4092    /// Reading receipts for the revoked-copies report constructs no store, so
4093    /// a read-only path (`car doctor`, the portfolio) never finishes an
4094    /// interrupted removal. A store constructed afterwards does — the control
4095    /// that the quarantine below is one a store would resume.
4096    #[cfg(any(target_os = "macos", target_os = "linux"))]
4097    #[test]
4098    fn the_revoked_copies_report_never_resumes_a_removal() {
4099        let (root, models, store) = supported_platform_store();
4100        let state = root.path().join("state");
4101        let managed = models.join("Managed");
4102        std::fs::create_dir_all(&managed).unwrap();
4103        std::fs::write(managed.join("weights"), b"owned").unwrap();
4104        let receipt = directory_receipt("org/model", managed.clone());
4105        store.write_receipt(&receipt).unwrap();
4106        let quarantine = store.allocate_quarantine("org/model").unwrap();
4107        let journal = RemovalJournal {
4108            model_id: "org/model".into(),
4109            removal_generation: 2,
4110            artifact_identity: artifact_identity(&managed, "org/model").unwrap(),
4111            receipt: receipt.clone(),
4112            quarantine_path: quarantine.clone(),
4113        };
4114        write_private_json(&store.removal_journal_path("org/model"), &journal).unwrap();
4115        std::fs::rename(&managed, &quarantine).unwrap();
4116        drop(store);
4117        crate::catalog::save_revoked_names(
4118            &crate::catalog::revoked_names_path(&state),
4119            &std::collections::BTreeMap::from([("Managed".into(), "org/model".into())]),
4120        )
4121        .unwrap();
4122
4123        assert_eq!(read_receipt(&state, "org/model").unwrap(), Some(receipt));
4124        let _ = crate::retire::revoked_copies(&state, &models);
4125        assert!(quarantine.join("weights").exists(), "nothing resumed");
4126
4127        let _store = ModelManagementStore::new(state, models.clone());
4128        assert!(
4129            std::fs::symlink_metadata(&quarantine).is_err(),
4130            "control: a store resumes it"
4131        );
4132    }
4133
4134    #[cfg(any(target_os = "macos", target_os = "linux"))]
4135    #[test]
4136    fn directory_removal_resumes_after_a_partial_quarantine_delete() {
4137        let (_root, models, store) = supported_platform_store();
4138        let managed = models.join("Managed");
4139        std::fs::create_dir_all(managed.join("sub")).unwrap();
4140        std::fs::write(managed.join("weights"), b"owned").unwrap();
4141        std::fs::write(managed.join("sub").join("more"), b"owned").unwrap();
4142        let receipt = directory_receipt("org/model", managed.clone());
4143        store.write_receipt(&receipt).unwrap();
4144        let quarantine = store.allocate_quarantine("org/model").unwrap();
4145        let journal = RemovalJournal {
4146            model_id: "org/model".into(),
4147            removal_generation: 2,
4148            artifact_identity: artifact_identity(&managed, "org/model").unwrap(),
4149            receipt,
4150            quarantine_path: quarantine.clone(),
4151        };
4152        write_private_json(&store.removal_journal_path("org/model"), &journal).unwrap();
4153        // A crash after the rename and after some entries were already deleted.
4154        std::fs::rename(&managed, &quarantine).unwrap();
4155        std::fs::remove_file(quarantine.join("weights")).unwrap();
4156
4157        let resumed = store
4158            .begin_mutation("org/model")
4159            .unwrap()
4160            .remove(2)
4161            .unwrap();
4162        assert_eq!(resumed.artifact_kind, ManagedArtifactKind::Directory);
4163        assert!(std::fs::symlink_metadata(&quarantine).is_err());
4164        assert!(no_quarantine_left(&models));
4165
4166        // A third request replays the durable tombstone result. The generation
4167        // it reports must stay the one that actually performed the removal (2),
4168        // not the generation this caller asked for (3), and the preserved
4169        // reference list must replay empty as the receipt recorded it.
4170        let replayed = store
4171            .begin_mutation("org/model")
4172            .unwrap()
4173            .remove(3)
4174            .unwrap();
4175        assert_eq!(replayed.artifact_kind, ManagedArtifactKind::Directory);
4176        let tombstone = store.load_tombstone("org/model").unwrap().unwrap();
4177        assert_eq!(tombstone.removal_generation, 2);
4178        assert!(replayed.preserved_shared_cache_references.is_empty());
4179    }
4180
4181    #[cfg(any(target_os = "macos", target_os = "linux"))]
4182    #[test]
4183    fn directory_removal_refuses_trees_deeper_than_the_cap() {
4184        let (_root, models, store) = supported_platform_store();
4185        let managed = models.join("Managed");
4186        let mut deep = managed.clone();
4187        for level in 0..(MAX_REMOVAL_DEPTH_FOR_TESTS + 1) {
4188            deep = deep.join(format!("level{level}"));
4189        }
4190        std::fs::create_dir_all(&deep).unwrap();
4191        std::fs::write(deep.join("sentinel"), b"deep").unwrap();
4192        store
4193            .write_receipt(&directory_receipt("org/model", managed.clone()))
4194            .unwrap();
4195
4196        let error = store
4197            .begin_mutation("org/model")
4198            .unwrap()
4199            .remove(2)
4200            .unwrap_err();
4201
4202        let reason = unsafe_managed_reason(&error);
4203        assert!(reason.contains("depth"), "unexpected reason: {reason}");
4204        // The leaf was already detached into its quarantine; nothing in it was deleted.
4205        let journal = store.load_removal_journal("org/model").unwrap().unwrap();
4206        let quarantined_leaf = deep
4207            .strip_prefix(&managed)
4208            .map(|rest| journal.quarantine_path.join(rest))
4209            .unwrap();
4210        assert_eq!(
4211            std::fs::read(quarantined_leaf.join("sentinel")).unwrap(),
4212            b"deep"
4213        );
4214    }
4215
4216    // Deliberately ungated: `matches_for_removal` is pure struct logic that
4217    // compiles on every target, so narrowing this to macOS/Linux like its
4218    // siblings would silently drop Windows coverage.
4219    #[test]
4220    fn removal_identity_ignores_length_for_directories_only() {
4221        let directory = ArtifactIdentity {
4222            kind: ManagedArtifactKind::Directory,
4223            device: 7,
4224            inode: 42,
4225            file_len: 96,
4226        };
4227        let shrunk = ArtifactIdentity {
4228            file_len: 32,
4229            ..directory.clone()
4230        };
4231        assert!(directory.matches_for_removal(&shrunk));
4232        assert!(!directory.matches_for_removal(&ArtifactIdentity {
4233            inode: 43,
4234            ..directory.clone()
4235        }));
4236        assert!(!directory.matches_for_removal(&ArtifactIdentity {
4237            device: 8,
4238            ..directory.clone()
4239        }));
4240
4241        let file = ArtifactIdentity {
4242            kind: ManagedArtifactKind::File,
4243            device: 7,
4244            inode: 42,
4245            file_len: 96,
4246        };
4247        assert!(file.matches_for_removal(&file));
4248        assert!(!file.matches_for_removal(&ArtifactIdentity {
4249            file_len: 95,
4250            ..file.clone()
4251        }));
4252        assert!(!file.matches_for_removal(&ArtifactIdentity {
4253            kind: ManagedArtifactKind::Symlink,
4254            ..file.clone()
4255        }));
4256    }
4257
4258    #[cfg(any(target_os = "macos", target_os = "linux"))]
4259    #[test]
4260    fn same_filesystem_predicate_compares_devices_only() {
4261        use super::object_bound::{same_filesystem, Identity};
4262        let root = Identity {
4263            device: 5,
4264            inode: 1,
4265        };
4266        assert!(same_filesystem(
4267            &root,
4268            &Identity {
4269                device: 5,
4270                inode: 999
4271            }
4272        ));
4273        assert!(!same_filesystem(
4274            &root,
4275            &Identity {
4276                device: 6,
4277                inode: 1
4278            }
4279        ));
4280    }
4281
4282    #[cfg(any(target_os = "macos", target_os = "linux"))]
4283    #[test]
4284    fn directory_removal_refuses_entries_added_after_capture() {
4285        let (_root, models, store) = supported_platform_store();
4286        let managed = models.join("Managed");
4287        std::fs::create_dir_all(managed.join("sub")).unwrap();
4288        std::fs::write(managed.join("weights"), b"owned").unwrap();
4289        std::fs::write(managed.join("sub").join("more"), b"owned").unwrap();
4290        let store = store.with_removal_hook(|phase, quarantine| {
4291            if phase == RemovalPhase::AfterCapture {
4292                std::fs::write(quarantine.join("sub").join("late"), b"not captured").unwrap();
4293            }
4294        });
4295        store
4296            .write_receipt(&directory_receipt("org/model", managed.clone()))
4297            .unwrap();
4298
4299        let error = store
4300            .begin_mutation("org/model")
4301            .unwrap()
4302            .remove(2)
4303            .unwrap_err();
4304
4305        let reason = unsafe_managed_reason(&error);
4306        assert!(
4307            reason.contains("not captured"),
4308            "unexpected reason: {reason}"
4309        );
4310        let journal = store.load_removal_journal("org/model").unwrap().unwrap();
4311        let late = journal.quarantine_path.join("sub").join("late");
4312        assert_eq!(std::fs::read(&late).unwrap(), b"not captured");
4313
4314        // Once the foreign entry is gone the journaled removal completes.
4315        std::fs::remove_file(&late).unwrap();
4316        let store = store.with_removal_hook(|_, _| {});
4317        let resumed = store
4318            .begin_mutation("org/model")
4319            .unwrap()
4320            .remove(2)
4321            .unwrap();
4322        assert_eq!(resumed.artifact_kind, ManagedArtifactKind::Directory);
4323        assert!(no_quarantine_left(&models));
4324    }
4325
4326    #[cfg(any(target_os = "macos", target_os = "linux"))]
4327    #[test]
4328    fn directory_removal_refuses_root_swap_after_rename() {
4329        let (_root, models, store) = supported_platform_store();
4330        let managed = models.join("Managed");
4331        std::fs::create_dir_all(&managed).unwrap();
4332        std::fs::write(managed.join("weights"), b"owned").unwrap();
4333        let displaced = models.join("displaced");
4334        let store = store.with_removal_hook(move |phase, quarantine| {
4335            if phase == RemovalPhase::AfterRename {
4336                std::fs::rename(quarantine, &displaced).unwrap();
4337                std::fs::create_dir_all(quarantine).unwrap();
4338                std::fs::write(quarantine.join("sentinel"), b"replacement").unwrap();
4339            }
4340        });
4341        store
4342            .write_receipt(&directory_receipt("org/model", managed.clone()))
4343            .unwrap();
4344
4345        let error = store
4346            .begin_mutation("org/model")
4347            .unwrap()
4348            .remove(2)
4349            .unwrap_err();
4350
4351        let reason = unsafe_managed_reason(&error);
4352        assert!(
4353            reason.contains("identity changed"),
4354            "unexpected reason: {reason}"
4355        );
4356        let journal = store.load_removal_journal("org/model").unwrap().unwrap();
4357        assert_eq!(
4358            std::fs::read(journal.quarantine_path.join("sentinel")).unwrap(),
4359            b"replacement"
4360        );
4361        assert_eq!(
4362            std::fs::read(models.join("displaced").join("weights")).unwrap(),
4363            b"owned"
4364        );
4365    }
4366
4367    #[cfg(any(target_os = "macos", target_os = "linux"))]
4368    #[test]
4369    fn directory_symlink_planted_after_rename_is_unlinked_not_followed() {
4370        use std::os::unix::fs::symlink;
4371
4372        let (root, models, store) = supported_platform_store();
4373        let outside = root.path().join("outside");
4374        std::fs::create_dir_all(&outside).unwrap();
4375        std::fs::write(outside.join("sentinel"), b"outside").unwrap();
4376        let managed = models.join("Managed");
4377        std::fs::create_dir_all(&managed).unwrap();
4378        std::fs::write(managed.join("weights"), b"owned").unwrap();
4379        let target = outside.clone();
4380        let store = store.with_removal_hook(move |phase, quarantine| {
4381            if phase == RemovalPhase::AfterRename {
4382                symlink(&target, quarantine.join("escape")).unwrap();
4383            }
4384        });
4385        store
4386            .write_receipt(&directory_receipt("org/model", managed.clone()))
4387            .unwrap();
4388
4389        let result = store.begin_mutation("org/model").unwrap().remove(2);
4390
4391        assert_eq!(
4392            std::fs::read(outside.join("sentinel")).map_err(|e| e.to_string()),
4393            Ok(b"outside".to_vec()),
4394            "outside sentinel was deleted; removal result: {result:?}"
4395        );
4396        assert!(result.is_ok(), "{result:?}");
4397        assert!(std::fs::symlink_metadata(&managed).is_err());
4398        assert!(no_quarantine_left(&models));
4399    }
4400
4401    #[cfg(any(target_os = "macos", target_os = "linux"))]
4402    #[test]
4403    fn directory_removal_refuses_root_swap_between_path_check_and_descriptor_open() {
4404        let (_root, models, store) = supported_platform_store();
4405        let managed = models.join("Managed");
4406        std::fs::create_dir_all(&managed).unwrap();
4407        std::fs::write(managed.join("weights"), b"owned").unwrap();
4408        let displaced = models.join("displaced");
4409        let store = store.with_removal_hook(move |phase, quarantine| {
4410            if phase == RemovalPhase::BeforeRootOpen {
4411                std::fs::rename(quarantine, &displaced).unwrap();
4412                std::fs::create_dir_all(quarantine).unwrap();
4413                std::fs::write(quarantine.join("sentinel"), b"replacement").unwrap();
4414            }
4415        });
4416        store
4417            .write_receipt(&directory_receipt("org/model", managed.clone()))
4418            .unwrap();
4419
4420        let error = store
4421            .begin_mutation("org/model")
4422            .unwrap()
4423            .remove(2)
4424            .unwrap_err();
4425
4426        let reason = unsafe_managed_reason(&error);
4427        assert_eq!(
4428            reason,
4429            "quarantined directory identity changed before deletion"
4430        );
4431        let journal = store.load_removal_journal("org/model").unwrap().unwrap();
4432        assert_eq!(
4433            std::fs::read(journal.quarantine_path.join("sentinel")).unwrap(),
4434            b"replacement"
4435        );
4436        assert_eq!(
4437            std::fs::read(models.join("displaced").join("weights")).unwrap(),
4438            b"owned"
4439        );
4440    }
4441
4442    #[cfg(any(target_os = "macos", target_os = "linux"))]
4443    #[test]
4444    fn directory_removal_refuses_when_the_quarantine_vanishes_after_our_own_rename() {
4445        let (_root, models, store) = supported_platform_store();
4446        let managed = models.join("Managed");
4447        std::fs::create_dir_all(&managed).unwrap();
4448        std::fs::write(managed.join("weights"), b"owned").unwrap();
4449        let managed_for_hook = managed.clone();
4450        let store = store.with_removal_hook(move |phase, quarantine| {
4451            if phase == RemovalPhase::AfterRename {
4452                // A racer moves the quarantine straight back to its managed name.
4453                std::fs::rename(quarantine, &managed_for_hook).unwrap();
4454            }
4455        });
4456        store
4457            .write_receipt(&directory_receipt("org/model", managed.clone()))
4458            .unwrap();
4459
4460        let error = store
4461            .begin_mutation("org/model")
4462            .unwrap()
4463            .remove(2)
4464            .unwrap_err();
4465
4466        let reason = unsafe_managed_reason(&error);
4467        assert!(reason.contains("vanished"), "unexpected reason: {reason}");
4468        assert_eq!(std::fs::read(managed.join("weights")).unwrap(), b"owned");
4469        assert!(store.load_receipt("org/model").unwrap().is_some());
4470        assert!(store.load_removal_journal("org/model").unwrap().is_some());
4471    }
4472
4473    #[cfg(any(target_os = "macos", target_os = "linux"))]
4474    #[test]
4475    fn first_attempt_vanish_refusal_names_the_retry_that_completes_the_removal() {
4476        let (_root, models, store) = supported_platform_store();
4477        let managed = models.join("Managed");
4478        std::fs::create_dir_all(&managed).unwrap();
4479        std::fs::write(managed.join("weights"), b"owned").unwrap();
4480        let managed_for_hook = managed.clone();
4481        let sabotaged = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
4482        let store = store.with_removal_hook(move |phase, quarantine| {
4483            // Sabotage the first attempt only: a racer moves the quarantine back
4484            // to its managed name right after CAR's own rename. The retry below
4485            // then runs against an untouched store.
4486            if phase == RemovalPhase::AfterRename
4487                && !sabotaged.swap(true, std::sync::atomic::Ordering::SeqCst)
4488            {
4489                std::fs::rename(quarantine, &managed_for_hook).unwrap();
4490            }
4491        });
4492        store
4493            .write_receipt(&directory_receipt("org/model", managed.clone()))
4494            .unwrap();
4495
4496        let error = store
4497            .begin_mutation("org/model")
4498            .unwrap()
4499            .remove(2)
4500            .unwrap_err();
4501
4502        // Asserted verbatim: the reason is the operator's only signal that
4503        // retrying is the remedy rather than corruption, so dropping the
4504        // journal-retention half of the sentence must fail this test.
4505        assert_eq!(
4506            unsafe_managed_reason(&error),
4507            "quarantine vanished before deletion; the removal journal is retained, so retrying the removal resumes and completes it"
4508        );
4509        assert!(store.load_removal_journal("org/model").unwrap().is_some());
4510
4511        // And the retry the reason promises does finish the removal.
4512        let retried = store
4513            .begin_mutation("org/model")
4514            .unwrap()
4515            .remove(2)
4516            .unwrap();
4517
4518        assert_eq!(retried.artifact_kind, ManagedArtifactKind::Directory);
4519        assert!(std::fs::symlink_metadata(&managed).is_err());
4520        assert!(no_quarantine_left(&models));
4521        assert!(store.load_removal_journal("org/model").unwrap().is_none());
4522        assert!(store.load_receipt("org/model").unwrap().is_none());
4523    }
4524
4525    #[cfg(any(target_os = "macos", target_os = "linux"))]
4526    #[test]
4527    fn directory_removal_refuses_a_captured_file_replaced_before_unlink() {
4528        let (root, models, store) = supported_platform_store();
4529        let outside = root.path().join("outside");
4530        std::fs::create_dir_all(&outside).unwrap();
4531        let managed = models.join("Managed");
4532        std::fs::create_dir_all(&managed).unwrap();
4533        std::fs::write(managed.join("weights"), b"owned").unwrap();
4534        let parked = outside.join("parked");
4535        let store = store.with_removal_hook(move |phase, quarantine| {
4536            if phase == RemovalPhase::AfterCapture {
4537                std::fs::rename(quarantine.join("weights"), &parked).unwrap();
4538                std::fs::write(quarantine.join("weights"), b"victim").unwrap();
4539            }
4540        });
4541        store
4542            .write_receipt(&directory_receipt("org/model", managed.clone()))
4543            .unwrap();
4544
4545        let error = store
4546            .begin_mutation("org/model")
4547            .unwrap()
4548            .remove(2)
4549            .unwrap_err();
4550
4551        let reason = unsafe_managed_reason(&error);
4552        assert!(
4553            reason.contains("identity changed"),
4554            "unexpected reason: {reason}"
4555        );
4556        let journal = store.load_removal_journal("org/model").unwrap().unwrap();
4557        assert_eq!(
4558            std::fs::read(journal.quarantine_path.join("weights")).unwrap(),
4559            b"victim"
4560        );
4561        assert_eq!(std::fs::read(outside.join("parked")).unwrap(), b"owned");
4562    }
4563
4564    #[cfg(any(target_os = "macos", target_os = "linux"))]
4565    #[test]
4566    fn directory_removal_refuses_a_captured_directory_replaced_before_rmdir() {
4567        let (root, models, store) = supported_platform_store();
4568        let outside = root.path().join("outside");
4569        std::fs::create_dir_all(&outside).unwrap();
4570        let managed = models.join("Managed");
4571        std::fs::create_dir_all(managed.join("sub")).unwrap();
4572        std::fs::write(managed.join("sub").join("more"), b"owned").unwrap();
4573        let parked = outside.join("parked-sub");
4574        let parked_for_hook = parked.clone();
4575        let store = store.with_removal_hook(move |phase, quarantine| {
4576            if phase == RemovalPhase::AfterCapture {
4577                std::fs::rename(quarantine.join("sub"), &parked_for_hook).unwrap();
4578                std::fs::create_dir_all(quarantine.join("sub")).unwrap();
4579                std::fs::write(quarantine.join("sub").join("victim"), b"victim").unwrap();
4580            }
4581        });
4582        store
4583            .write_receipt(&directory_receipt("org/model", managed.clone()))
4584            .unwrap();
4585
4586        let error = store
4587            .begin_mutation("org/model")
4588            .unwrap()
4589            .remove(2)
4590            .unwrap_err();
4591
4592        let reason = unsafe_managed_reason(&error);
4593        assert!(
4594            reason.contains("identity changed"),
4595            "unexpected reason: {reason}"
4596        );
4597        let journal = store.load_removal_journal("org/model").unwrap().unwrap();
4598        assert_eq!(
4599            std::fs::read(journal.quarantine_path.join("sub").join("victim")).unwrap(),
4600            b"victim"
4601        );
4602        // The captured subtree moved elsewhere is still CAR's object: its
4603        // captured entry was unlinked there, the directory itself was not.
4604        assert!(std::fs::symlink_metadata(&parked).is_ok());
4605    }
4606
4607    #[cfg(any(target_os = "macos", target_os = "linux"))]
4608    #[test]
4609    fn directory_removal_accepts_a_tree_at_the_depth_cap() {
4610        let (_root, models, store) = supported_platform_store();
4611        let managed = models.join("Managed");
4612        let mut deep = managed.clone();
4613        for level in 0..MAX_REMOVAL_DEPTH_FOR_TESTS {
4614            deep = deep.join(format!("level{level}"));
4615        }
4616        std::fs::create_dir_all(&deep).unwrap();
4617        std::fs::write(deep.join("sentinel"), b"deep").unwrap();
4618        store
4619            .write_receipt(&directory_receipt("org/model", managed.clone()))
4620            .unwrap();
4621
4622        let result = store
4623            .begin_mutation("org/model")
4624            .unwrap()
4625            .remove(2)
4626            .unwrap();
4627
4628        assert_eq!(result.artifact_kind, ManagedArtifactKind::Directory);
4629        assert!(std::fs::symlink_metadata(&managed).is_err());
4630        assert!(no_quarantine_left(&models));
4631    }
4632
4633    #[cfg(any(target_os = "macos", target_os = "linux"))]
4634    #[test]
4635    fn directory_removal_refuses_when_the_quarantine_vanishes_before_the_descriptor_open() {
4636        let (_root, models, store) = supported_platform_store();
4637        let managed = models.join("Managed");
4638        std::fs::create_dir_all(&managed).unwrap();
4639        std::fs::write(managed.join("weights"), b"owned").unwrap();
4640        let managed_for_hook = managed.clone();
4641        let store = store.with_removal_hook(move |phase, quarantine| {
4642            if phase == RemovalPhase::BeforeRootOpen {
4643                // The path-based checks passed; a racer now moves the quarantine
4644                // back to its managed name before the binding descriptor opens.
4645                std::fs::rename(quarantine, &managed_for_hook).unwrap();
4646            }
4647        });
4648        store
4649            .write_receipt(&directory_receipt("org/model", managed.clone()))
4650            .unwrap();
4651
4652        let error = store
4653            .begin_mutation("org/model")
4654            .unwrap()
4655            .remove(2)
4656            .unwrap_err();
4657
4658        // Spelled out rather than compared against QUARANTINE_VANISHED_REASON:
4659        // this window is user-reachable on a first attempt too, and the receipt
4660        // and journal assertions below are exactly the retention the sentence
4661        // promises, so the promise is pinned here as well as at the guard.
4662        assert_eq!(
4663            unsafe_managed_reason(&error),
4664            "quarantine vanished before deletion; the removal journal is retained, so retrying the removal resumes and completes it"
4665        );
4666        assert_eq!(std::fs::read(managed.join("weights")).unwrap(), b"owned");
4667        assert!(store.load_receipt("org/model").unwrap().is_some());
4668        assert!(store.load_removal_journal("org/model").unwrap().is_some());
4669    }
4670
4671    /// Pins — does not endorse — what the retry does when the racer parked the
4672    /// quarantine somewhere other than the managed name. The resume finds
4673    /// neither the artifact nor the quarantine, so it cannot re-detach or
4674    /// delete anything; it unlinks the receipt and journal and reports success
4675    /// while the bytes survive at the parked path. Recorded so a deliberate
4676    /// change to that behaviour has to change this test.
4677    #[cfg(any(target_os = "macos", target_os = "linux"))]
4678    #[test]
4679    fn retry_after_the_quarantine_was_parked_elsewhere_settles_without_deleting_it() {
4680        let (root, models, store) = supported_platform_store();
4681        let outside = root.path().join("outside");
4682        std::fs::create_dir_all(&outside).unwrap();
4683        let managed = models.join("Managed");
4684        std::fs::create_dir_all(&managed).unwrap();
4685        std::fs::write(managed.join("weights"), b"owned").unwrap();
4686        let parked = outside.join("parked");
4687        let parked_for_hook = parked.clone();
4688        let store = store.with_removal_hook(move |phase, quarantine| {
4689            if phase == RemovalPhase::AfterRename {
4690                // Not back to the managed name: out of the models root entirely.
4691                std::fs::rename(quarantine, &parked_for_hook).unwrap();
4692            }
4693        });
4694        store
4695            .write_receipt(&directory_receipt("org/model", managed.clone()))
4696            .unwrap();
4697
4698        let error = store
4699            .begin_mutation("org/model")
4700            .unwrap()
4701            .remove(2)
4702            .unwrap_err();
4703
4704        assert_eq!(
4705            unsafe_managed_reason(&error),
4706            "quarantine vanished before deletion; the removal journal is retained, so retrying the removal resumes and completes it"
4707        );
4708        assert!(store.load_removal_journal("org/model").unwrap().is_some());
4709
4710        let retried = store
4711            .begin_mutation("org/model")
4712            .unwrap()
4713            .remove(2)
4714            .unwrap();
4715
4716        assert_eq!(retried.artifact_kind, ManagedArtifactKind::Directory);
4717        assert!(store.load_removal_journal("org/model").unwrap().is_none());
4718        assert!(store.load_receipt("org/model").unwrap().is_none());
4719        assert!(no_quarantine_left(&models));
4720        // The parked copy is untouched: CAR never followed it out of the root.
4721        assert_eq!(std::fs::read(parked.join("weights")).unwrap(), b"owned");
4722    }
4723}